diff --git a/.gitignore b/.gitignore index 985e5c514c..91949fe0cf 100644 --- a/.gitignore +++ b/.gitignore @@ -38,14 +38,21 @@ core.[0-9]* secrets/**/* # Registry build artifacts +toolchain/c/build/ +toolchain/c/vendor/ +toolchain/c/libs/ +toolchain/c/sysroot/ +software/*/bin/ registry/native/target/ registry/native/vendor/ registry/native/c/build/ registry/native/c/vendor/ registry/native/c/libs/ registry/native/c/.cache/ +toolchain/c/.cache/ registry/native/c/sysroot/ registry/software/*/wasm/ +software/*/wasm/ registry/software/*/agentos-package.meta.json registry/.last-publish-hash registry/software/*/.last-publish-hash diff --git a/CLAUDE.md b/CLAUDE.md index b0298183e0..b2e3979007 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -65,6 +65,37 @@ the guest — over inventing a softer fallback that hides the failure. - Agent adapters must use real upstream SDKs. Do not replace SDK adapters with direct API-call stubs. +## Software Build (WASM Toolchain) + +Registry software is **real upstream Linux software** (GNU coreutils, grep, sed, +gawk, real curl/sqlite/duckdb/vim, …) compiled to `wasm32-wasip1` against a +**sysroot we fully own** — a patched Rust std + libc whose gaps are filled by +custom host-syscall imports. Treat that target as **native POSIX**; +`wasm32-wasip1` is an implementation detail, not a feature ceiling. + +- **We do not depend on stock WASI / wasi-libc.** The sysroot is ours. A missing + libc/POSIX API (`getrlimit`/`RLIMIT_NOFILE`, `getgroups`, spawn, fd dup, …) is + never a blocker — implement it (real, or a sane stub) in the patched + std/libc/host-import layer. "WASI doesn't have X" is not a reason to stop; X is + ours to add. +- **Fix portability one layer down, in the sysroot** — a new std/libc patch or a + new host import — not with `cfg(target_*)` branches or shims in the tool's own + source. A WASM-specific branch in application code usually means the fix + belongs in the libc layer. +- **Patch the real upstream tool only as a fallback**, when the fix genuinely + cannot live in the sysroot. Patching the real tool is allowed; reimplementing + it is not. +- **"NOT POSSIBLE" is reserved for genuine impossibility** after exhausting both + sysroot patches and tool patches — never for a missing syscall we could + implement. Document the specific wall if you claim it. +- **Working in `software/`, you may (and should) fix the layer underneath.** When + a package behaves differently from real Linux, the root cause is usually not the + package — it's the runtime. It is in-scope and expected to fix the underlying + implementation: the Node-compat / bridge layer, the WASM execution runtime, the + kernel/VFS syscalls, or the patched sysroot/libc. Do **not** paper over a + Linux-deviating behavior in the package, its wrapper, or its test — chase it + down into whichever runtime layer owns it and make that layer match Linux. + ## Publishing - `scripts/publish` is the source of truth for npm/crates discovery, version diff --git a/README.md b/README.md index e9357918ca..1cf5b93e0f 100644 --- a/README.md +++ b/README.md @@ -157,7 +157,6 @@ Browse pre-built agents, tools, filesystems, and software packages at the [agent | `@agentos-software/git` | git | git version control | rust | - | - | | `@agentos-software/grep` | grep | GNU grep pattern matching (grep, egrep, fgrep) | rust | - | - | | `@agentos-software/gzip` | gzip | GNU gzip compression (gzip, gunzip, zcat) | rust | - | - | -| `@agentos-software/http-get` | http-get | Minimal HTTP GET fetch helper | c | - | - | | `@agentos-software/jq` | jq | jq JSON processor | rust | - | - | | `@agentos-software/ripgrep` | ripgrep | ripgrep fast recursive search | rust | - | - | | `@agentos-software/sed` | sed | GNU sed stream editor | rust | - | - | @@ -165,7 +164,7 @@ Browse pre-built agents, tools, filesystems, and software packages at the [agent | `@agentos-software/tar` | tar | GNU tar archiver | rust | - | - | | `@agentos-software/tree` | tree | tree directory listing | rust | - | - | | `@agentos-software/unzip` | unzip | unzip archive extraction | c | - | - | -| `@agentos-software/wget` | wget | GNU wget HTTP client | c | - | - | +| `@agentos-software/wget` | wget | GNU Wget file downloader | c | - | - | | `@agentos-software/yq` | yq | yq YAML/JSON processor | rust | - | - | | `@agentos-software/zip` | zip | zip archive creation | c | - | - | @@ -175,7 +174,7 @@ Browse pre-built agents, tools, filesystems, and software packages at the [agent |---------|-------------|----------| | `@agentos-software/build-essential` | Build-essential WASM command set (standard + make + git + curl) | standard, make, git, curl | | `@agentos-software/common` | Common WASM command set (coreutils + sed + grep + gawk + findutils + diffutils + tar + gzip) | coreutils, sed, grep, gawk, findutils, diffutils, tar, gzip | -| `@agentos-software/everything` | All available WASM command packages in a single bundle | coreutils, sed, grep, gawk, findutils, diffutils, tar, gzip, curl, zip, unzip, jq, ripgrep, fd, tree, file, yq, codex-cli | +| `@agentos-software/everything` | All available WASM command packages in a single bundle | coreutils, sed, grep, gawk, findutils, diffutils, tar, gzip, curl, wget, zip, unzip, jq, ripgrep, fd, tree, file, yq, codex-cli | ## License diff --git a/crates/client/tests/common/mod.rs b/crates/client/tests/common/mod.rs index e91cd0a9be..ca01e86829 100644 --- a/crates/client/tests/common/mod.rs +++ b/crates/client/tests/common/mod.rs @@ -149,8 +149,8 @@ fn wasm_command_mounts() -> Vec { /// Locate the materialized coreutils package under the in-repo registry build. pub fn coreutils_package_dir() -> Option { - let registry_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("../../software/coreutils/dist/package"); + let registry_dir = + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../software/coreutils/dist/package"); if registry_dir.join("agentos-package.json").is_file() { return std::fs::canonicalize(registry_dir).ok(); } diff --git a/crates/client/tests/shell_pty_packages_e2e.rs b/crates/client/tests/shell_pty_packages_e2e.rs index 7df9ff3af4..b057fdd5c6 100644 --- a/crates/client/tests/shell_pty_packages_e2e.rs +++ b/crates/client/tests/shell_pty_packages_e2e.rs @@ -26,8 +26,7 @@ fn coreutils_package_path() -> Option { } } for dir in [ - PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("../../software/coreutils/dist/package"), + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../software/coreutils/dist/package"), PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("../../node_modules/@agentos-software/coreutils/dist/package"), ] { diff --git a/crates/execution/assets/runners/wasi-module.js b/crates/execution/assets/runners/wasi-module.js index 8bb1d01dc6..b861b77ba9 100644 --- a/crates/execution/assets/runners/wasi-module.js +++ b/crates/execution/assets/runners/wasi-module.js @@ -1393,11 +1393,18 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = return this._measureWasiPhase("writeResultPtr", () => this._writeUint32(nwrittenPtr, written)); } } + const entry = this._descriptorEntry(descriptor); + const localHostPassthrough = + handle.kind === "host-passthrough" && + entry?.kind === "file" && + entry.realFd === handle.targetFd; const position = handle.append ? this._measureWasiPhase("appendFstat", () => Number(__agentOSFs().fstatSync(handle.targetFd).size ?? 0) ) - : null; + : localHostPassthrough + ? (entry.offset ?? 0) + : null; const written = this._measureWasiPhase("writeSync", () => __agentOSFs().writeSync( handle.targetFd, @@ -1407,6 +1414,15 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = position, ) ); + if (localHostPassthrough) { + if (handle.append) { + entry.offset = this._measureWasiPhase("appendFstat", () => + Number(__agentOSFs().fstatSync(handle.targetFd).size ?? 0) + ); + } else { + entry.offset = (entry.offset ?? 0) + written; + } + } return this._measureWasiPhase("writeResultPtr", () => this._writeUint32(nwrittenPtr, written)); } if (handle?.kind === "guest-file" && typeof handle.targetFd === "number") { @@ -1745,6 +1761,11 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = (handle?.kind === "passthrough" || handle?.kind === "host-passthrough") && typeof handle.targetFd === "number" ) { + const localEntry = this._descriptorEntry(descriptor); + const localHostPassthrough = + handle.kind === "host-passthrough" && + localEntry?.kind === "file" && + localEntry.realFd === handle.targetFd; const totalLength = this._boundedReadLength(iovs, iovsLen); const buffer = Buffer.alloc(totalLength); const bytesRead = __agentOSFs().readSync( @@ -1752,8 +1773,11 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = buffer, 0, totalLength, - null, + localHostPassthrough ? (localEntry.offset ?? 0) : null, ); + if (localHostPassthrough) { + localEntry.offset = (localEntry.offset ?? 0) + bytesRead; + } const written = this._writeToIovs(iovs, iovsLen, buffer.subarray(0, bytesRead)); return this._writeUint32(nreadPtr, written); } diff --git a/crates/execution/assets/runners/wasm-runner.mjs b/crates/execution/assets/runners/wasm-runner.mjs index e230e07285..f385336bd8 100644 --- a/crates/execution/assets/runners/wasm-runner.mjs +++ b/crates/execution/assets/runners/wasm-runner.mjs @@ -30,6 +30,7 @@ const WASI_ERRNO_FAULT = 21; const WASI_RIGHT_FD_WRITE = 64n; const WASI_FILETYPE_UNKNOWN = 0; const WASI_FILETYPE_CHARACTER_DEVICE = 2; +const WASI_FILETYPE_DIRECTORY = 3; const WASI_FILETYPE_REGULAR_FILE = 4; const WASI_OFLAGS_CREAT = 1; const WASI_OFLAGS_DIRECTORY = 2; @@ -428,7 +429,8 @@ const passthroughHandles = new Map([ ]); const retainedSyntheticHandlesByDisplayFd = new Map(); const retainedSpawnOutputHandlesByFd = new Map(); -let nextSyntheticFd = 64; +const FIRST_SYNTHETIC_FD = 1 << 20; +let nextSyntheticFd = FIRST_SYNTHETIC_FD; let nextSyntheticPipeId = 1; const syntheticWaitArray = new Int32Array(new SharedArrayBuffer(4)); let delegateWriteScratch = { base: 0, capacity: 0 }; @@ -1160,13 +1162,19 @@ function fsOpenFlagForPathOpen(oflags, rightsBase, fdflags) { return 'r+'; } -function allocateSyntheticFd() { - let fd = nextSyntheticFd; - while ( +function syntheticFdInUse(fd) { + return ( syntheticFdEntries.has(fd) || passthroughHandles.has(fd) || + retainedSpawnOutputHandlesByFd.has(fd) || + retainedSyntheticHandlesByDisplayFd.has(fd) || delegateManagedFdRefCounts.has(fd) - ) { + ); +} + +function allocateSyntheticFd(minFd = nextSyntheticFd) { + let fd = Math.max(FIRST_SYNTHETIC_FD, Number(minFd) >>> 0); + while (syntheticFdInUse(fd)) { fd += 1; } nextSyntheticFd = fd + 1; @@ -1226,8 +1234,9 @@ function openManagedPathIoFd(guestPath, rightsBase, fdflags) { return null; } try { + const hostPath = resolveHostFsPath(guestPath) ?? guestPath; return fsModule.openSync( - guestPath, + hostPath, fsOpenNumericFlagsForManagedPath(rightsBase, fdflags), 0o666, ); @@ -1243,16 +1252,35 @@ function retainPathOpenDelegateFd(openedFdPtr, guestPath, fdflags, rightsBase) { try { const openedFd = new DataView(instanceMemory.buffer).getUint32(Number(openedFdPtr), true); + let retainedFd = openedFd; + if (openedFd > 2 && syntheticFdInUse(openedFd)) { + if (typeof delegateManagedFdRenumber !== 'function') { + return WASI_ERRNO_FAULT; + } + retainedFd = allocateSyntheticFd(openedFd + 1); + const renumberResult = delegateManagedFdRenumber(openedFd, retainedFd); + if (renumberResult !== WASI_ERRNO_SUCCESS) { + return renumberResult; + } + const writeResult = writeGuestUint32(openedFdPtr, retainedFd); + if (writeResult !== WASI_ERRNO_SUCCESS) { + return writeResult; + } + traceHostProcess('path-open-delegate-renumber', { + openedFd, + retainedFd, + }); + } const append = (Number(fdflags) & WASI_FDFLAGS_APPEND) !== 0; - retainDelegateFd(openedFd); - if (openedFd > 2 && !passthroughHandles.has(openedFd)) { + retainDelegateFd(retainedFd); + if (retainedFd > 2 && !passthroughHandles.has(retainedFd)) { const ioFd = openManagedPathIoFd(guestPath, rightsBase, fdflags); - closedPassthroughFds.delete(openedFd); - passthroughHandles.set(openedFd, { + closedPassthroughFds.delete(retainedFd); + passthroughHandles.set(retainedFd, { kind: 'passthrough', - targetFd: openedFd, + targetFd: retainedFd, ioFd, - displayFd: openedFd, + displayFd: retainedFd, refCount: 0, open: true, readOnly: @@ -1331,6 +1359,19 @@ function writeGuestFilestat(ptr, stats, filetype = WASI_FILETYPE_REGULAR_FILE) { } } +function wasiFiletypeFromStats(stats) { + if (typeof stats?.isDirectory === 'function' && stats.isDirectory()) { + return WASI_FILETYPE_DIRECTORY; + } + if (typeof stats?.isCharacterDevice === 'function' && stats.isCharacterDevice()) { + return WASI_FILETYPE_CHARACTER_DEVICE; + } + if (typeof stats?.isFile === 'function' && stats.isFile()) { + return WASI_FILETYPE_REGULAR_FILE; + } + return WASI_FILETYPE_UNKNOWN; +} + function writeGuestFdstat(ptr, filetype, flags, rightsBase, rightsInheriting) { if (!(instanceMemory instanceof WebAssembly.Memory)) { return WASI_ERRNO_FAULT; @@ -1776,6 +1817,91 @@ function writeBytesToGuestIovs(iovs, iovsLen, bytes) { return written >>> 0; } +function guestIovByteLength(iovs, iovsLen) { + if (!(instanceMemory instanceof WebAssembly.Memory)) { + throw new Error('WebAssembly memory is not available'); + } + + const view = new DataView(instanceMemory.buffer); + let total = 0; + for (let index = 0; index < (Number(iovsLen) >>> 0); index += 1) { + const entryOffset = (Number(iovs) >>> 0) + index * 8; + total += view.getUint32(entryOffset + 4, true); + } + return total >>> 0; +} + +function readHostNetSocketToGuestIovs(socket, iovs, iovsLen, nreadPtr) { + try { + const requestedLength = guestIovByteLength(iovs, iovsLen); + if (requestedLength === 0) { + return writeGuestUint32(nreadPtr, 0); + } + + if (socket.nonblock) { + let queued = dequeueHostNetBytes(socket, requestedLength); + if (queued.length > 0) { + return writeGuestUint32(nreadPtr, writeBytesToGuestIovs(iovs, iovsLen, queued)); + } + if (socket.lastError) return WASI_ERRNO_FAULT; + if (socket.readableEnded || socket.closed || !socket.socketId) { + return writeGuestUint32(nreadPtr, 0); + } + pollHostNetSocket(socket, 0); + queued = dequeueHostNetBytes(socket, requestedLength); + if (queued.length > 0) { + return writeGuestUint32(nreadPtr, writeBytesToGuestIovs(iovs, iovsLen, queued)); + } + if (socket.readableEnded || socket.closed || !socket.socketId) { + return writeGuestUint32(nreadPtr, 0); + } + return WASI_ERRNO_AGAIN; + } + + const deadline = + socket.recvTimeoutMs == null ? null : Date.now() + Math.max(0, socket.recvTimeoutMs); + while (true) { + const queued = dequeueHostNetBytes(socket, requestedLength); + if (queued.length > 0) { + return writeGuestUint32(nreadPtr, writeBytesToGuestIovs(iovs, iovsLen, queued)); + } + if (socket.lastError) return WASI_ERRNO_FAULT; + if (socket.readableEnded || socket.closed || !socket.socketId) { + return writeGuestUint32(nreadPtr, 0); + } + + const pollWaitMs = + deadline == null ? 50 : Math.max(0, Math.min(50, deadline - Date.now())); + if (deadline != null && pollWaitMs === 0) { + return WASI_ERRNO_AGAIN; + } + pollHostNetSocket(socket, pollWaitMs); + if (deadline != null && Date.now() >= deadline) { + return WASI_ERRNO_AGAIN; + } + } + } catch { + return WASI_ERRNO_FAULT; + } +} + +function writeHostNetSocketFromGuestIovs(socket, iovs, iovsLen, nwrittenPtr) { + if (!socket?.socketId || socket.closed) { + return WASI_ERRNO_BADF; + } + + try { + const bytes = collectGuestIovBytes(iovs, iovsLen); + if (bytes.length === 0) { + return writeGuestUint32(nwrittenPtr, 0); + } + const written = Number(callSyncRpc('net.write', [socket.socketId, bytes])) >>> 0; + return writeGuestUint32(nwrittenPtr, written); + } catch { + return WASI_ERRNO_FAULT; + } +} + function dequeuePipeBytes(pipe, maxBytes) { const requested = Math.max(0, Number(maxBytes) >>> 0); if (requested === 0 || pipe.chunks.length === 0) { @@ -2700,8 +2826,9 @@ function callSyncRpc(method, args = []) { } const hostNetSockets = new Map(); -let nextHostNetSocketFd = 0x40000000; +let nextHostNetSocketFd = 4096; const HOST_NET_TIMEOUT_SENTINEL = '__agentos_net_timeout__'; +const HOST_NET_MSG_PEEK = 0x0001; function getHostNetSocket(fd) { return hostNetSockets.get(Number(fd) >>> 0) ?? null; @@ -2732,6 +2859,76 @@ function dequeueHostNetBytes(socket, maxBytes) { return Buffer.concat(parts); } +function peekHostNetBytes(socket, maxBytes) { + const requested = Math.max(0, Number(maxBytes) >>> 0); + if (requested === 0 || socket.readChunks.length === 0) { + return Buffer.alloc(0); + } + + const parts = []; + let remaining = requested; + for (const chunk of socket.readChunks) { + if (remaining === 0) break; + const chunkLength = Math.min(chunk.length, remaining); + parts.push(chunk.subarray(0, chunkLength)); + remaining -= chunkLength; + } + + return Buffer.concat(parts); +} + +function decodeHostNetSocketReadResult(result) { + if (result == null) { + return { kind: 'end' }; + } + + if (result === HOST_NET_TIMEOUT_SENTINEL) { + return { kind: 'timeout' }; + } + + if (typeof result === 'string') { + if (result === HOST_NET_TIMEOUT_SENTINEL) { + return { kind: 'timeout' }; + } + return { kind: 'data', bytes: Buffer.from(result, 'base64') }; + } + + const decoded = decodeSyncRpcValue(result); + if (Buffer.isBuffer(decoded)) { + return { kind: 'data', bytes: decoded }; + } + if (decoded == null) { + return { kind: 'end' }; + } + if (decoded === HOST_NET_TIMEOUT_SENTINEL) { + return { kind: 'timeout' }; + } + return { kind: 'timeout' }; +} + +function readReadyHostNetSocket(socket) { + if (!socket?.socketId || socket.closed) { + socket.readableEnded = true; + return null; + } + + const result = decodeHostNetSocketReadResult( + callSyncRpc('net.socket_read', [socket.socketId]), + ); + if (result.kind === 'data') { + if (result.bytes.length > 0) { + socket.readChunks.push(Buffer.from(result.bytes)); + } + return result; + } + if (result.kind === 'end') { + socket.readableEnded = true; + socket.closed = true; + socket.socketId = null; + } + return result; +} + function pollHostNetSocket(socket, waitMs) { if (!socket?.socketId || socket.closed) { return null; @@ -2766,6 +2963,20 @@ function pollHostNetSocket(socket, waitMs) { return event; } + if (event.readable === true || (Number(event.revents) & 0x001) !== 0) { + return readReadyHostNetSocket(socket); + } + + if (event.hangup === true) { + socket.readableEnded = true; + return event; + } + + if (event.error === true) { + socket.lastError = 'socket error'; + return event; + } + return event; } @@ -2833,6 +3044,7 @@ const HOST_NET_SOCK_DGRAM = 5; const HOST_NET_SOCKET_TYPE_MASK = 0xf; const HOST_NET_SOL_SOCKET = 1; const HOST_NET_WASI_SOL_SOCKET = 0x7fffffff; +const HOST_NET_SO_ERROR = 4; const HOST_NET_SO_RCVTIMEO_64 = 20; const HOST_NET_SO_RCVTIMEO_32 = 66; const HOST_NET_TIMEVAL_BYTES = 16; @@ -3492,15 +3704,14 @@ const hostNetImport = { } try { - if ((Number(flags) >>> 0) !== 0) { - // Non-zero recv flags are currently ignored in the WASM host_net shim. - } + const recvFlags = Number(flags) >>> 0; + const peek = (recvFlags & HOST_NET_MSG_PEEK) !== 0; // Non-blocking sockets (O_NONBLOCK via net_set_nonblock, used by libxcb's poll_for_*): // pull whatever is queued, do ONE short readiness probe, and return EAGAIN if still empty // instead of blocking. libxcb assumes its "poll" reads never block on an empty socket. if (socket.nonblock) { - let queued = dequeueHostNetBytes(socket, bufLen); + let queued = peek ? peekHostNetBytes(socket, bufLen) : dequeueHostNetBytes(socket, bufLen); if (queued.length > 0) { return writeGuestBytes(bufPtr, bufLen, queued, retReceivedPtr); } @@ -3509,7 +3720,7 @@ const hostNetImport = { return writeGuestUint32(retReceivedPtr, 0); } pollHostNetSocket(socket, 0); - queued = dequeueHostNetBytes(socket, bufLen); + queued = peek ? peekHostNetBytes(socket, bufLen) : dequeueHostNetBytes(socket, bufLen); if (queued.length > 0) { return writeGuestBytes(bufPtr, bufLen, queued, retReceivedPtr); } @@ -3522,7 +3733,7 @@ const hostNetImport = { const deadline = socket.recvTimeoutMs == null ? null : Date.now() + Math.max(0, socket.recvTimeoutMs); while (true) { - const queued = dequeueHostNetBytes(socket, bufLen); + const queued = peek ? peekHostNetBytes(socket, bufLen) : dequeueHostNetBytes(socket, bufLen); if (queued.length > 0) { return writeGuestBytes(bufPtr, bufLen, queued, retReceivedPtr); } @@ -3676,6 +3887,32 @@ const hostNetImport = { } return WASI_ERRNO_SUCCESS; }, + net_getsockopt(fd, level, optname, optvalPtr, optvalLenPtr) { + const socket = getHostNetSocket(fd); + if (!socket || socket.closed) { + return WASI_ERRNO_BADF; + } + + try { + const optvalLen = readGuestUint32(optvalLenPtr); + const normalizedLevel = Number(level) >>> 0; + const normalizedOptname = Number(optname) >>> 0; + if ( + (normalizedLevel === HOST_NET_SOL_SOCKET || + normalizedLevel === HOST_NET_WASI_SOL_SOCKET) && + normalizedOptname === HOST_NET_SO_ERROR + ) { + if (optvalLen < 4) { + return WASI_ERRNO_INVAL; + } + new DataView(instanceMemory.buffer).setInt32(Number(optvalPtr) >>> 0, 0, true); + return writeGuestUint32(optvalLenPtr, 4); + } + return WASI_ERRNO_INVAL; + } catch { + return WASI_ERRNO_FAULT; + } + }, net_close(fd) { const numericFd = Number(fd) >>> 0; const socket = hostNetSockets.get(numericFd); @@ -3699,7 +3936,7 @@ const hostNetImport = { return WASI_ERRNO_FAULT; } }, - net_tls_connect(fd, hostnamePtr, hostnameLen) { + net_tls_connect(fd, hostnamePtr, hostnameLen, flags = 0) { const socket = getHostNetSocket(fd); if (!socket?.socketId || socket.closed) { return WASI_ERRNO_BADF; @@ -3708,7 +3945,7 @@ const hostNetImport = { try { const servername = readGuestString(hostnamePtr, hostnameLen); const tlsOptions = { servername }; - if (guestEnv.NODE_TLS_REJECT_UNAUTHORIZED === '0') { + if ((Number(flags) & 1) === 1 || guestEnv.NODE_TLS_REJECT_UNAUTHORIZED === '0') { tlsOptions.rejectUnauthorized = false; } callSyncRpc('net.socket_upgrade_tls', [ @@ -4030,8 +4267,8 @@ const hostProcessImport = { readHandleCount: 0, writeHandleCount: 0, }; - const readFd = nextSyntheticFd++; - const writeFd = nextSyntheticFd++; + const readFd = allocateSyntheticFd(); + const writeFd = allocateSyntheticFd(); syntheticFdEntries.set(readFd, createPipeHandle('pipe-read', pipe, readFd)); syntheticFdEntries.set(writeFd, createPipeHandle('pipe-write', pipe, writeFd)); if (writeGuestUint32(retReadFdPtr, readFd) !== WASI_ERRNO_SUCCESS) { @@ -4048,20 +4285,7 @@ const hostProcessImport = { if (!handle) { return WASI_ERRNO_BADF; } - let duplicatedFd = 0; - while ( - duplicatedFd <= 2 && - ( - syntheticFdEntries.has(duplicatedFd) || - passthroughHandles.has(duplicatedFd) || - delegateManagedFdRefCounts.has(duplicatedFd) - ) - ) { - duplicatedFd += 1; - } - if (duplicatedFd > 2) { - duplicatedFd = nextSyntheticFd++; - } + const duplicatedFd = allocateSyntheticFd(0); syntheticFdEntries.set(duplicatedFd, handle); traceHostProcess('fd-dup', { fd: Number(fd) >>> 0, @@ -4133,15 +4357,7 @@ const hostProcessImport = { return WASI_ERRNO_BADF; } - let duplicatedFd = minimumFdNumber >>> 0; - while ( - syntheticFdEntries.has(duplicatedFd) || - passthroughHandles.has(duplicatedFd) || - delegateManagedFdRefCounts.has(duplicatedFd) - ) { - duplicatedFd += 1; - } - nextSyntheticFd = Math.max(nextSyntheticFd, duplicatedFd + 1); + const duplicatedFd = allocateSyntheticFd(minimumFdNumber); syntheticFdEntries.set(duplicatedFd, handle); traceHostProcess('fd-dup-min', { @@ -4818,6 +5034,10 @@ const delegateManagedFdClose = typeof wasiImport.fd_close === 'function' ? wasiImport.fd_close.bind(wasiImport) : null; +const delegateManagedFdRenumber = + typeof wasiImport.fd_renumber === 'function' + ? wasiImport.fd_renumber.bind(wasiImport) + : null; const delegateManagedFdPrestatGet = typeof wasiImport.fd_prestat_get === 'function' ? wasiImport.fd_prestat_get.bind(wasiImport) @@ -4837,6 +5057,11 @@ const KERNEL_POLLHUP = 0x0010; wasiImport.fd_read = (fd, iovs, iovsLen, nreadPtr) => { const numericFd = Number(fd) >>> 0; + const hostNetSocket = getHostNetSocket(numericFd); + if (hostNetSocket) { + return readHostNetSocketToGuestIovs(hostNetSocket, iovs, iovsLen, nreadPtr); + } + const handle = __agentOSWasiMeasurePhase('fd_read', 'lookup_handle', () => lookupFdHandle(numericFd) ); @@ -5008,11 +5233,13 @@ wasiImport.fd_read = (fd, iovs, iovsLen, nreadPtr) => { } if (handle?.kind === 'passthrough') { - return delegateManagedFdRead - ? __agentOSWasiMeasurePhase('fd_read', 'delegate_call', () => - delegateManagedFdRead(handle.targetFd, iovs, iovsLen, nreadPtr) - ) - : WASI_ERRNO_BADF; + if (!delegateManagedFdRead) { + return WASI_ERRNO_BADF; + } + const result = __agentOSWasiMeasurePhase('fd_read', 'delegate_call', () => + delegateManagedFdRead(handle.targetFd, iovs, iovsLen, nreadPtr) + ); + return result; } if (rejectClosedPassthroughFd(numericFd)) { @@ -5086,9 +5313,10 @@ wasiImport.fd_pread = (fd, iovs, iovsLen, offset, nreadPtr) => { return mapSyntheticFsError(error); } } - return delegateFdPread - ? delegateFdPread(handle.targetFd, iovs, iovsLen, offset, nreadPtr) - : WASI_ERRNO_BADF; + if (!delegateFdPread) { + return WASI_ERRNO_BADF; + } + return delegateFdPread(handle.targetFd, iovs, iovsLen, offset, nreadPtr); } if (rejectClosedPassthroughFd(fd)) { @@ -5103,6 +5331,9 @@ wasiImport.fd_pread = (fd, iovs, iovsLen, offset, nreadPtr) => { wasiImport.fd_pwrite = (fd, iovs, iovsLen, offset, nwrittenPtr) => { const handle = lookupFdHandle(fd); if (handle?.kind === 'guest-file') { + if (handle.readOnly === true) { + return WASI_ERRNO_ROFS; + } try { const bytes = collectGuestIovBytes(iovs, iovsLen); const written = fsModule.writeSync( @@ -5307,11 +5538,50 @@ wasiImport.fd_fdstat_get = (fd, statPtr) => { ); } + if (handle?.kind === 'guest-file') { + try { + const stat = fsModule.fstatSync(handle.targetFd); + return writeGuestFdstat( + statPtr, + wasiFiletypeFromStats(stat), + 0, + WASI_RIGHT_FD_READ | + WASI_RIGHT_FD_SEEK | + WASI_RIGHT_FD_TELL | + WASI_RIGHT_FD_FILESTAT_GET | + WASI_RIGHT_FD_WRITE | + WASI_RIGHT_FD_SYNC, + 0n, + ); + } catch (error) { + return mapSyntheticFsError(error); + } + } + if (handle && handle.kind !== 'passthrough') { return WASI_ERRNO_BADF; } if (handle?.kind === 'passthrough') { + if (typeof handle.ioFd === 'number') { + try { + const stat = fsModule.fstatSync(handle.ioFd); + return writeGuestFdstat( + statPtr, + wasiFiletypeFromStats(stat), + 0, + WASI_RIGHT_FD_READ | + WASI_RIGHT_FD_SEEK | + WASI_RIGHT_FD_TELL | + WASI_RIGHT_FD_FILESTAT_GET | + WASI_RIGHT_FD_WRITE | + WASI_RIGHT_FD_SYNC, + 0n, + ); + } catch (error) { + return mapSyntheticFsError(error); + } + } return delegateManagedFdFdstatGet ? __agentOSWasiMeasurePhase('fd_fdstat_get', 'delegate_call', () => delegateManagedFdFdstatGet(handle.targetFd, statPtr) @@ -5369,9 +5639,10 @@ wasiImport.fd_filestat_get = (fd, statPtr) => { return mapSyntheticFsError(error); } } - return delegateManagedFdFilestatGet - ? delegateManagedFdFilestatGet(handle.targetFd, statPtr) - : WASI_ERRNO_BADF; + if (!delegateManagedFdFilestatGet) { + return WASI_ERRNO_BADF; + } + return delegateManagedFdFilestatGet(handle.targetFd, statPtr); } if (rejectClosedPassthroughFd(fd)) { @@ -5491,10 +5762,15 @@ wasiImport.fd_prestat_dir_name = (fd, pathPtr, pathLen) => { }; wasiImport.fd_write = (fd, iovs, iovsLen, nwrittenPtr) => { + const numericFd = Number(fd) >>> 0; + const hostNetSocket = getHostNetSocket(numericFd); + if (hostNetSocket) { + return writeHostNetSocketFromGuestIovs(hostNetSocket, iovs, iovsLen, nwrittenPtr); + } + const handle = __agentOSWasiMeasurePhase('fd_write', 'lookup_handle', () => lookupFdHandle(fd) ); - const numericFd = Number(fd) >>> 0; if (handle?.kind === 'pipe-write') { try { const bytes = __agentOSWasiMeasurePhase('fd_write', 'guest_iov_collect', () => @@ -5516,6 +5792,9 @@ wasiImport.fd_write = (fd, iovs, iovsLen, nwrittenPtr) => { } if (handle?.kind === 'guest-file') { + if (handle.readOnly === true) { + return WASI_ERRNO_ROFS; + } try { const bytes = __agentOSWasiMeasurePhase('fd_write', 'guest_iov_collect', () => collectGuestIovBytes(iovs, iovsLen) @@ -5662,6 +5941,66 @@ wasiImport.fd_close = (fd) => { : WASI_ERRNO_BADF; }; +wasiImport.fd_renumber = (from, to) => { + try { + const sourceFd = Number(from) >>> 0; + const targetFd = Number(to) >>> 0; + if (sourceFd === targetFd) { + return lookupFdHandle(sourceFd) || delegateManagedFdRefCounts.has(sourceFd) + ? WASI_ERRNO_SUCCESS + : WASI_ERRNO_BADF; + } + + const syntheticHandle = syntheticFdEntries.get(sourceFd); + const passthroughHandle = passthroughHandles.get(sourceFd); + const retainedSpawnOutputHandle = retainedSpawnOutputHandlesByFd.get(sourceFd); + if (!syntheticHandle && !passthroughHandle && !retainedSpawnOutputHandle) { + if (rejectClosedPassthroughFd(sourceFd)) { + return WASI_ERRNO_BADF; + } + return delegateManagedFdRenumber + ? delegateManagedFdRenumber(sourceFd, targetFd) + : WASI_ERRNO_BADF; + } + + if ( + syntheticFdEntries.has(targetFd) || + passthroughHandles.has(targetFd) || + retainedSpawnOutputHandlesByFd.has(targetFd) || + delegateManagedFdRefCounts.has(targetFd) + ) { + const closeResult = wasiImport.fd_close(targetFd); + if (closeResult !== WASI_ERRNO_SUCCESS) { + return closeResult; + } + } + + if (syntheticHandle) { + syntheticFdEntries.delete(sourceFd); + syntheticFdEntries.set(targetFd, syntheticHandle); + } else if (passthroughHandle) { + passthroughHandles.delete(sourceFd); + passthroughHandles.set(targetFd, passthroughHandle); + closedPassthroughFds.add(sourceFd); + closedPassthroughFds.delete(targetFd); + } else { + retainedSpawnOutputHandlesByFd.delete(sourceFd); + retainedSpawnOutputHandlesByFd.set(targetFd, retainedSpawnOutputHandle); + } + + nextSyntheticFd = Math.max(nextSyntheticFd, targetFd + 1); + traceHostProcess('fd-renumber', { + from: sourceFd, + to: targetFd, + syntheticKind: syntheticHandle?.kind ?? null, + passthroughKind: passthroughHandle?.kind ?? null, + }); + return WASI_ERRNO_SUCCESS; + } catch { + return WASI_ERRNO_FAULT; + } +}; + wasiImport.poll_oneoff = (inPtr, outPtr, nsubscriptions, neventsPtr) => { if (!(instanceMemory instanceof WebAssembly.Memory)) { return delegateManagedPollOneoff diff --git a/crates/execution/src/node_import_cache.rs b/crates/execution/src/node_import_cache.rs index 913d65ca57..fbe47a1c4e 100644 --- a/crates/execution/src/node_import_cache.rs +++ b/crates/execution/src/node_import_cache.rs @@ -15,7 +15,7 @@ const NODE_IMPORT_CACHE_PATH_ENV: &str = "AGENTOS_NODE_IMPORT_CACHE_PATH"; const NODE_IMPORT_CACHE_LOADER_PATH_ENV: &str = "AGENTOS_NODE_IMPORT_CACHE_LOADER_PATH"; const NODE_IMPORT_CACHE_SCHEMA_VERSION: &str = "1"; const NODE_IMPORT_CACHE_LOADER_VERSION: &str = "8"; -const NODE_IMPORT_CACHE_ASSET_VERSION: &str = "93"; +const NODE_IMPORT_CACHE_ASSET_VERSION: &str = "96"; const NODE_IMPORT_CACHE_DIR_PREFIX: &str = "agentos-node-import-cache"; const DEFAULT_NODE_IMPORT_CACHE_MATERIALIZE_TIMEOUT: Duration = Duration::from_secs(30); const PYODIDE_DIST_DIR: &str = "pyodide-dist"; diff --git a/crates/native-sidecar/tests/projection_bench.rs b/crates/native-sidecar/tests/projection_bench.rs index 1dc4e6b72b..034b4830f8 100644 --- a/crates/native-sidecar/tests/projection_bench.rs +++ b/crates/native-sidecar/tests/projection_bench.rs @@ -614,14 +614,8 @@ fn projection_bench() { "PROJ_BENCH_COREUTILS_TAR", "software/coreutils/dist/package.tar", ); - let tar_tar = source_tar_path( - "PROJ_BENCH_TAR_TAR", - "software/tar/dist/package.tar", - ); - let git_tar = source_tar_path( - "PROJ_BENCH_GIT_TAR", - "software/git/dist/package.tar", - ); + let tar_tar = source_tar_path("PROJ_BENCH_TAR_TAR", "software/tar/dist/package.tar"); + let git_tar = source_tar_path("PROJ_BENCH_GIT_TAR", "software/git/dist/package.tar"); println!("\n# agentOS package load benchmark (.aospkg)"); println!( diff --git a/crates/vfs/src/posix/overlay_fs.rs b/crates/vfs/src/posix/overlay_fs.rs index 25aa86b9e3..9fd0600587 100644 --- a/crates/vfs/src/posix/overlay_fs.rs +++ b/crates/vfs/src/posix/overlay_fs.rs @@ -1661,6 +1661,8 @@ impl VirtualFileSystem for OverlayFileSystem { let mut snapshot_entries = Vec::new(); self.collect_snapshot_entries(&old_normalized, &mut snapshot_entries)?; + self.clear_path_metadata(&resolved_new_normalized)?; + self.clear_subtree_metadata(&resolved_new_normalized)?; if let Ok(destination_stat) = self.merged_lstat(&resolved_new_normalized) { if destination_stat.is_directory && !destination_stat.is_symbolic_link @@ -1678,7 +1680,6 @@ impl VirtualFileSystem for OverlayFileSystem { .remove_file(&resolved_new_normalized)?; } } - self.clear_subtree_metadata(&resolved_new_normalized)?; } self.stage_snapshot_entries_in_upper(&snapshot_entries)?; @@ -1985,6 +1986,33 @@ mod tests { assert_error_code(overlay.read_dir("/a"), "ENOENT"); } + #[test] + fn rename_clears_destination_whiteout() { + let lower = MemoryFileSystem::new(); + let mut overlay = OverlayFileSystem::new(vec![lower], OverlayMode::Ephemeral); + overlay + .write_file("/archive.zip", b"old".to_vec()) + .expect("create old archive"); + overlay + .remove_file("/archive.zip") + .expect("whiteout old archive"); + overlay + .write_file("/workspace-temp", b"new".to_vec()) + .expect("create replacement"); + + overlay + .rename("/workspace-temp", "/archive.zip") + .expect("rename replacement over whiteout"); + + assert_eq!( + overlay + .read_file("/archive.zip") + .expect("read renamed file"), + b"new" + ); + assert!(!overlay.exists("/workspace-temp")); + } + #[test] fn remove_dir_still_rejects_visible_children() { let mut lower = MemoryFileSystem::new(); diff --git a/docs-internal/registry-flatten-colocation-spec.md b/docs-internal/registry-flatten-colocation-spec.md index 3cd23d7c06..f4298f4cc9 100644 --- a/docs-internal/registry-flatten-colocation-spec.md +++ b/docs-internal/registry-flatten-colocation-spec.md @@ -59,7 +59,7 @@ repo-root/ │ │ ├── sh/ cat/ ls/ cp/ mv/ sort/ … # ~80 command crates │ │ └── (du/ expr/ column/ rev/ strings/ — 1:1 libs that belong to coreutils) │ │ -│ ├── sqlite3/ duckdb/ wget/ zip/ unzip/ http-get/ # C-based, native/c/.c +│ ├── sqlite3/ duckdb/ wget/ zip/ unzip/ # C-based, native/c/.c │ ├── grep/ sed/ gawk/ jq/ yq/ fd/ ripgrep/ tree/ file/ tar/ gzip/ diffutils/ findutils/ # Rust, native/crates/ │ │ │ ├── claude/ codex/ opencode/ pi/ pi-cli/ # agent adapters — @agentos-software/*, JS only, no native/ diff --git a/docs-internal/registry-parity-worklist.md b/docs-internal/registry-parity-worklist.md index 671e39c6bf..02ac4b61a9 100644 --- a/docs-internal/registry-parity-worklist.md +++ b/docs-internal/registry-parity-worklist.md @@ -1,6 +1,6 @@ # Registry Linux-Parity Worklist -Status: worklist · Owner: registry · Last updated: 2026-07-07 +Status: worklist · Owner: registry · Last updated: 2026-07-08 ## Goal (hand this to the driver agent) @@ -23,13 +23,26 @@ Status: worklist · Owner: registry · Last updated: 2026-07-07 > real project is correct. Prefer the genuine upstream tool (real git, real > grep) over a rewrite; a *popular, established* reimplementation is an > acceptable fallback only when the real tool genuinely won't build. -> - **"Not possible" is a valid outcome — but only after trying really hard.** If a -> command cannot be built as the real (or an established) tool for WASI, do NOT -> hand-roll a custom replacement. Instead mark it **`NOT POSSIBLE (WASI)`** in -> this doc with a concrete explanation of exactly what blocks it (missing -> syscall, unsupported threading, sysroot gap, etc.) and what was tried. Exhaust -> real options first: patch the sysroot, patch the tool, stub the specific -> missing syscall — a genuine effort, not a quick bail. +> - **"Not possible" is a valid outcome — but only after trying really hard.** The +> sysroot is **ours**: a patched Rust std + libc with custom host-syscall imports +> (see CLAUDE.md → Software Build (WASM Toolchain)). A missing libc/POSIX API +> (`getrlimit`/`RLIMIT_NOFILE`, `getgroups`, …) is **NOT** a WASI wall — it is a +> stub/patch we add one layer down, and the build should proceed as if targeting +> native POSIX. Only if a command *still* cannot be built as the real (or an +> established) tool do you mark it **`NOT POSSIBLE (WASI)`** in this doc, with a +> concrete explanation of the genuine, documented wall (never "WASI lacks a +> syscall we could implement") and what was tried. Exhaust real options first: +> patch the sysroot, patch the tool, stub the specific missing syscall — a +> genuine effort, not a quick bail. +> - **Commit clean revs — no stray artifacts.** Each rev must contain only the +> intended source + test changes. Never commit build outputs, vendored toolchain +> trees, `__pycache__`/`*.pyc`, generated binaries, or anything that belongs in +> `.gitignore`. Before `jj describe`, run `jj diff -r @ --summary` and confirm +> every path is intended — watch especially for `A` (added) paths under +> `toolchain/`, `**/target/`, `**/node_modules/`, `**/build/`, `**/__pycache__/`. +> Then **audit the entire stack up to main** (`jj diff -r 'main..@' --stat`, or +> per rev) and strip anything that slipped in with `jj restore --from +> `, adding the pattern to `.gitignore` so it cannot recur. > - **One jj rev per item.** Concretely: **`jj new` before starting each item**, > make that command's fix *and* its e2e test in that single change, `jj describe` > it with a clear conventional-commit message, then `jj new` again for the next @@ -72,7 +85,6 @@ actual backing: | Command(s) | Backing | |---|---| | coreutils (`sh`+80) | **uutils** (`uucore`) — established Rust project | -| ripgrep (`rg`) | real ripgrep | | duckdb, vim | real upstream C source, patched for WASI | | sqlite3 **engine** | real SQLite amalgamation (⚠️ but the *CLI* is ours — see below) | | jq | **jaq** (`jaq-core/std/json`) — established Rust jq | @@ -87,17 +99,18 @@ actual backing: ### ❌ CUSTOM WE BUILT — flag & replace with a real/established impl | Command | Status | What it actually is | Replace with | |---|---|---|---| -| **curl** | TODO | our custom driver over a libcurl fork | real `curl` CLI (upstream `src/tool_*.c`) | -| **wget** | TODO | our 174-line `wget.c` | real GNU wget, or drop (curl covers it) | -| **http-get** | TODO | our 95-line `http_get.c` | drop, or a real tool | -| **git** | TODO | our hand-rolled git from `sha1`+`flate2` | **real git** (upstream C), patched for WASI — **NOT gitoxide** | -| **fd** | TODO | our `secureexec-fd` on raw `regex` (not sharkdp/fd) | real **fd** (sharkdp) | -| **findutils** (`find`,`xargs`) | TODO | our hand-rolled on `regex`/shims | real GNU findutils, or `uutils/findutils` | -| **tree** | TODO | our hand-rolled, zero deps | real `tree`, or an established one | -| **grep** | TODO | our `secureexec-grep` on raw `regex` (**not** an established grep pkg) | **real GNU grep**, or a popular established grep (ripgrep's `grep` crates) | -| **zip** | TODO | our 203-line `zip.c` over zlib/minizip (not Info-ZIP) | real Info-ZIP, or an established lib's CLI | -| **unzip** | TODO | our 669-line `unzip.c` over zlib/minizip | real Info-ZIP unzip | -| **sqlite3 CLI** | TODO | our 558-line `sqlite3_cli.c` (engine is real SQLite; the shell is ours) | real SQLite `shell.c` (its official CLI) | +| **curl** | DONE | our custom driver over a libcurl fork | real `curl` CLI (upstream `src/tool_*.c`) | +| **wget** | DONE | our 174-line `wget.c` (dropped) | real GNU Wget vs our sysroot — stub `getrlimit`/`getgroups`, then build | +| **http-get** | DONE | our 95-line `http_get.c` | dropped; real curl covers HTTP fetches | +| **git** | DONE | our hand-rolled git from `sha1`+`flate2` | **real git** (upstream C), patched for WASI — **NOT gitoxide** | +| **fd** | DONE | our `secureexec-fd` on raw `regex` (not sharkdp/fd) | real **fd** (sharkdp) | +| **findutils** (`find`,`xargs`) | DONE | our hand-rolled on `regex`/shims | replaced with `uutils/findutils` | +| **tree** | DONE | our hand-rolled, zero deps | real `tree`, or an established one | +| **grep** | DONE | our `secureexec-grep` on raw `regex` (**not** an established grep pkg) | real **GNU grep** | +| **ripgrep** (`rg`) | DONE | our `secureexec-grep` recursive search shim, not real ripgrep | real upstream **ripgrep** | +| **zip** | DONE | our 203-line `zip.c` over zlib/minizip (not Info-ZIP) | real Info-ZIP, or an established lib's CLI | +| **unzip** | DONE | our 669-line `unzip.c` over zlib/minizip | real Info-ZIP unzip | +| **sqlite3 CLI** | DONE | our 558-line `sqlite3_cli.c` (engine is real SQLite; the shell is ours) | real SQLite `shell.c` (its official CLI) | | **vix** | DONE | from-scratch source-less drop-zone binary | deleted; real `vim` covers the editor slot | Note: `codex`/`codex-exec` = the rivet fork of OpenAI's codex — established fork, @@ -117,6 +130,209 @@ impl *before* their tests are written, so the tests validate real behavior. **Decisions (settled):** git → **real git** (not gitoxide). grep → **real GNU grep**, or a popular established grep if the real one won't build. +**git — where the issues are (assessment):** +- **LICENSE — RESOLVED, not a blocker.** Each command ships as its **own published + npm package**, so a GPL-2.0 git binary in `@agentos-software/git` is **mere + aggregation** — it does not affect the Apache-2.0 licensing of agentOS or any + other package. Ship the git package under GPL-2.0 (offer its source) and we're + compliant. This **supersedes** the clean-room-reimpl rationale in + `toolchain/std-patches/git/README.md` ("cannot be vendored due to license + restrictions"): that premise no longer holds — go with **real git** (upstream C) + and update/remove that README. (gitoxide stays ruled out.) +- **Technical WASI issues if we do build real git** (from that README + git's own + build knobs), easiest → hardest: + - `mmap` (packfiles/index) → build `NO_MMAP=1` (malloc+read). Fine. + - signals (SIGPIPE/SIGCHLD) → build without; WASI has none. Fine. + - threads (index-pack/pack-objects) → `NO_PTHREADS`; single-threaded, slower. Fine. + - `fork()`+`exec()` in `run_command.c` (hooks, filters, remote helpers) → route to + `posix_spawn` via the `wasi-spawn` broker (spawn IS supported — same fix as wget). + - **Network transport (clone/fetch/push) — the hard part.** Smart-HTTP needs the + `git-remote-https` **helper subprocess** + libcurl; `git://` needs raw sockets; + ssh needs an `ssh` subprocess. Each helper must itself exist as a module. + - symlink checkout → `core.symlinks=false` fallback (WASI symlink support is + partial); local time → UTC like elsewhere. +- **Bottom line:** license is a non-issue (separate published package = mere + aggregation). **local** git (init/add/commit/log/diff/branch/merge/status/ + checkout) is very achievable; **remote** git (clone/fetch/push over + smart-HTTP/ssh) is the real effort. Proceed with real git. + +**Replacement findings — what each remaining ❌ tool will take (investigated):** + +The recurring wall is **never a syscall** — it's one of two known, already-solved +patterns: **(a) no threads** on `wasm32-wasip1` → serial patch like +`toolchain/std-patches/crates/uu_sort/0001-wasi-serial-sort.patch` (hits real fd & +ripgrep crates); **(b) gnulib `getrlimit`/`getgroups`** → sysroot stubs, already +documented for wget (item #8) (hits GNU grep/findutils). Subprocess spawn already +works (`wasi-spawn` broker), so `xargs` is not a blocker. + +- **git — DONE.** Replaced the custom Rust `sha1`+`flate2` implementation with real + upstream Git 2.55.0 built by the C toolchain and staged as `git` plus helper + command aliases. The WASI changes stayed below Git's behavior surface: + `run-command.c` uses `posix_spawn` so helper subprocesses go through the existing + wasi-spawn broker, the sysroot exposes Git's missing C compatibility surface, and + the runner now allocates synthetic fds in a high range so managed pipe/file fds + cannot collide with delegate-opened WASI fds. Smart HTTP remains intentionally + disabled in this build (`NO_CURL`), so HTTPS clone fails with the real Git helper + error instead of a custom transport. Proof: clean upstream Git rebuild passes in + `2026-07-08T11-28-00-0700-git-clean-rebuild-after-high-synthetic-fd.log`; + package build stages 6 commands in + `2026-07-08T11-33-00-0700-git-package-build-clean-binary-after-install.log`; + native sidecar rebuild passes in + `2026-07-08T11-34-00-0700-sidecar-rebuild-after-git-clean-package.log`; + full Git e2e passes 18/18 in + `2026-07-08T11-51-00-0700-git-full-e2e-high-synthetic-fd-clean-binary-after-test-fix.log`. + Rev: `tmvlxlvk` — `fix(git): build upstream git`. +- **sqlite3 CLI — DONE.** Engine was already the real amalgamation + (`libs/sqlite3/sqlite3.c`); the command now builds the official upstream + `shell.c` from the same fetched zip as `sqlite3`. The local 558-line + `sqlite3_cli.c` reimplementation is deleted, `toolchain/c/build/sqlite3` is the + primary C output, `sqlite3_cli` remains only as a compatibility alias, and the + tracked runtime-core fallback command is refreshed to the same official shell. + Proof: official shell build passes in + `2026-07-08T05-04-47-0700-sqlite3-official-shell-build-command-name.log`; + package-focused e2e passes 16/16, including real `.tables`, `.schema`, and + `.dump` CLI arguments, in + `2026-07-08T05-07-06-0700-sqlite3-official-shell-tests-final-focused.log`; + package build/check-types pass in + `2026-07-08T05-07-52-0700-sqlite3-package-build-official-shell-final.log` and + `2026-07-08T05-07-52-0700-sqlite3-check-types-official-shell-final.log`; + runtime-core fallback command path passes `.tables` in + `2026-07-08T05-09-12-0700-sqlite3-runtime-core-command-fallback-test.log`; + aggregate C `programs` builds 57 commands in + `2026-07-08T05-09-53-0700-sqlite3-make-programs-final.log`. + Rev: `typytnkk` — `fix(sqlite3): build official SQLite shell`. +- **http-get — DONE.** Dropped the 95-line raw-socket loopback client instead of + porting it: real curl now covers HTTP fetch behavior in DuckDB remote CSV tests, + runtime cross-network loopback tests, and the C conformance loopback row. + Removed the `@agentos-software/http-get` package, fallback command, shell/core + dependencies, registry listing, C command install entries, and lockfile package + edges. The low-level `http_get_test` fixture remains because it is a socket + diagnostic test program, not a shipped registry command. Proof: pnpm lockfile + refresh succeeds in + `2026-07-08T05-39-44-0700-http-get-pnpm-lock-refresh.log`; core and shell + type checks pass in + `2026-07-08T05-40-57-0700-http-get-core-check-types-after-install.log` and + `2026-07-08T05-41-34-0700-http-get-shell-check-types-final.log`; DuckDB test + file typecheck passes in + `2026-07-08T05-40-57-0700-http-get-duckdb-test-typecheck-after-install.log`; + aggregate C `programs` builds 57 commands, with `http_get` absent after stale + generated cleanup/install, in + `2026-07-08T05-41-34-0700-http-get-make-c-programs-without-command.log` and + `2026-07-08T05-43-24-0700-http-get-clear-stale-generated-and-install.log`; + cross-runtime network tests pass 11/11 with the WASM curl rows in + `2026-07-08T05-47-43-0700-http-get-runtime-cross-network-test-pass.log`. +- **tree — DONE.** Replaced the custom Rust `secureexec-tree`/`cmd-tree` crates + with upstream Steve Baker `tree` 2.3.2 from `OldManProgrammer/unix-tree`. + It builds as a C toolchain command from pinned source, stages into + `@agentos-software/tree`, and refreshes the tracked runtime-core fallback + command. Sysroot fixes live one layer down: install `` and provide + deterministic missing-group lookup stubs so upstream `-g` support links + without a tree-source WASI branch. Proof: upstream source inspection in + `2026-07-08T05-13-50-0700-tree-fetch-upstream-2.3.2-inspect.log`; sysroot + patch check passes in + `2026-07-08T05-18-16-0700-tree-wasi-libc-patch-check-group-lookup-fixed.log`; + Makefile build passes in + `2026-07-08T05-20-02-0700-tree-upstream-make-build.log`; package build and + check-types pass in + `2026-07-08T05-21-08-0700-tree-package-build-upstream-after-install.log` and + `2026-07-08T05-21-08-0700-tree-check-types-upstream-after-install.log`; e2e + tree tests pass 6/6 in + `2026-07-08T05-29-45-0700-tree-vitest-upstream-final.log`; aggregate C + `programs` builds 58 commands in + `2026-07-08T05-30-44-0700-tree-make-programs-final.log`; Cargo metadata no + longer includes the deleted Rust tree crates in + `2026-07-08T05-33-17-0700-tree-cargo-metadata-after-removing-empty-dirs.log`. + Rev: `kpmrwxln` — `fix(tree): build upstream tree`. +- **fd — DONE.** Replaced the custom Rust `secureexec-fd` regex walker with + upstream sharkdp `fd-find` 10.4.2. Because `fd-find` is a bin-only crate, + `cmd-fd` now acts as the workspace build trigger while the toolchain builds + the upstream `fd` binary directly. WASI compatibility stays in dependency + patches: `fd-find` gets narrow file-type/path/receiver adjustments, and + `ignore` uses a serial walker on `wasm32-wasip1` instead of host threads. Proof: + clean patch dry-runs pass in + `2026-07-08T06-19-50-0700-fd-find-patch-clean-dry-run.log` and + `2026-07-08T06-26-52-0700-fd-ignore-patch-final-dry-run.log`; + clean upstream WASM build compiles `fd-find v10.4.2` in + `2026-07-08T06-21-54-0700-fd-upstream-wasm-rebuild-after-ignore-fix.log`; + runtime/package command hashes match in + `2026-07-08T06-22-42-0700-fd-copy-built-binary-hashes.log`; package build and + check-types pass in + `2026-07-08T06-24-27-0700-fd-package-build.log` and + `2026-07-08T06-25-41-0700-fd-check-types-final.log`; e2e fd tests pass 9/9, + including `fd --version` reporting `fd 10.4.2`, in + `2026-07-08T06-25-00-0700-fd-vitest-upstream-after-dir-format.log`. + Rev: `mrskpomv` — `fix(fd): build upstream fd-find`. +- **grep — DONE.** Replaced the `@agentos-software/grep` package's custom + `secureexec-grep` command wrapper with upstream GNU grep 3.12 from the official + GNU release tarball. The real GNU `grep` binary builds through the C toolchain; + `egrep` and `fgrep` are separate tiny WASM launchers that preserve GNU's + upstream obsolescent scripts by spawning `grep -E` / `grep -F` through the + AgentOS process broker. Sysroot fix stayed one layer down: wasi-libc no longer + advertises/exports its nonstandard two-argument `opendirat`, which conflicted + with gnulib's helper. Proof: official GNU release listing captured in + `2026-07-08T06-29-11-0700-grep-gnu-ftp-latest.log`; configure options captured + in `2026-07-08T06-29-39-0700-grep-upstream-configure-help-probe.log`; + sysroot `opendirat` patch check and rebuild pass in + `2026-07-08T06-34-27-0700-grep-wasi-libc-opendirat-patch-check-definition.log` + and `2026-07-08T06-34-33-0700-grep-sysroot-rebuild-opendirat-definition.log`; + GNU grep build passes in + `2026-07-08T06-35-01-0700-grep-gnu-wasm-build-after-opendirat-symbol.log`; + `egrep`/`fgrep` launcher builds pass in + `2026-07-08T06-40-07-0700-grep-egrep-wrapper-build.log` and + `2026-07-08T06-40-42-0700-grep-fgrep-wrapper-build.log`; package build and + check-types pass in `2026-07-08T06-42-24-0700-grep-package-build-final.log` + and `2026-07-08T06-42-18-0700-grep-check-types-final.log`; e2e grep tests + pass 8/8 in `2026-07-08T06-43-59-0700-grep-vitest-after-wrapper-cache-repair.log`. + Rev: `uyukolvr` — `fix(grep): build upstream GNU grep`. +- **ripgrep — DONE.** Replaced the `@agentos-software/ripgrep` package's custom + `secureexec-grep` recursive search shim with upstream ripgrep 15.1.0 from the + canonical `BurntSushi/ripgrep` crate/release. The local `cmd-rg` crate is now + only a build trigger, and `toolchain/Makefile` builds ripgrep's own `rg` bin + directly. The old `secureexec-grep` library is gone. Proof: latest upstream + release captured in `2026-07-08T06-50-40-0700-ripgrep-github-latest-release.json`; + crate metadata captured in `2026-07-08T06-52-00-0700-ripgrep-cargo-info.log`; + upstream WASM build passes in + `2026-07-08T06-56-00-0700-ripgrep-upstream-wasm-build-after-grep-dir-remove-fixed.log`; + package build and check-types pass in + `2026-07-08T06-58-55-0700-ripgrep-package-build-after-install.log` and + `2026-07-08T06-58-56-0700-ripgrep-check-types-after-install.log`; e2e ripgrep + tests pass 8/8 in `2026-07-08T07-02-00-0700-ripgrep-vitest-after-git-fixture.log`. + Rev: `msypkqmo` — `fix(ripgrep): build upstream ripgrep`. +- **zip / unzip — DONE.** Replaced both custom minizip-based C wrappers with real + upstream Info-ZIP Zip 3.0 and UnZip 6.0 release tarballs, built through the C + toolchain. Runtime/sysroot fixes stayed one layer down: temp-file, ownership, + `system`/`popen`/`pclose` compatibility in patched wasi-libc; overlay rename + over destination whiteouts; and WASI host-passthrough read/write offset + tracking after `fd_seek`. Proof: wasi-libc patch check passes in + `2026-07-08T08-34-29-0700-wasi-libc-patch-check-final.log`; native sidecar + build passes in `2026-07-08T08-34-07-0700-sidecar-build-final-runner-format.log`; + VFS rename regression passes in + `2026-07-08T08-34-29-0700-vfs-core-rename-whiteout-final.log`; final Zip e2e + passes 2/2 in + `2026-07-08T08-36-45-0700-zip-test-final-after-current-sysroot-copy-after-install.log`; + final UnZip e2e passes 6/6 in + `2026-07-08T08-37-12-0700-unzip-test-final-after-current-sysroot-copy-after-install.log`. + Rev: `nppnuxpr` — `fix(infozip): build upstream zip and unzip`. +- **findutils — DONE.** Replaced the custom Rust `find` crate with upstream + `uutils/findutils` 0.9.0 entrypoints for both `find` and `xargs`. Kept the fix + in the toolchain/sysroot layer: Rust WASI metadata extensions and process + `ExitStatusExt::signal()` are exposed for crates that expect Unix-like std + surfaces, while `xargs` still uses normal `std::process::Command` spawning + through the existing VM broker. Proof: `find` wasm build passes in + `2026-07-08T12-42-00-0700-findutils-find-build-after-permissions-mode-patch.log`; + `xargs` wasm build passes in + `2026-07-08T12-45-00-0700-findutils-xargs-build-after-find-success.log`; + package build passes in + `2026-07-08T12-51-00-0700-findutils-package-build-after-install.log`; sidecar + validation build passes in + `2026-07-08T12-35-00-0700-native-sidecar-build-after-forced-pnpm.log`; final + package e2e passes 5/5, including `xargs -n 2 echo` spawn batching, in + `2026-07-08T12-44-00-0700-findutils-vitest-uutils-after-depth-test-fix.log`. + Rev: `msknmmps`. + +Ranked easiest→hardest: **sqlite3-CLI · http-get (drop) · tree · fd · grep · +ripgrep · zip · unzip · findutils.** + ## Status tracking (how the driver reports progress in this doc) Update this doc as you go — it is the single source of truth for status. For each @@ -200,7 +416,7 @@ so a reader sees the whole board at a glance. ## P1 — Broken shipped commands -### 6. curl — reimplemented CLI, exits 1 on every operation (incl. `--version`) +### 6. curl — reimplemented CLI, exits 1 on every operation (incl. `--version`) — DONE - **Broken:** the `curl` command is a **hand-rolled `curl.c` driver** over a libcurl fork, not the real curl command-line tool — so 24/30 `curl.test.ts` fail and every op returns exit 1, even `curl --version`. @@ -209,40 +425,89 @@ so a reader sees the whole board at a glance. patched only where WASI forces it — replacing the custom driver. All real curl behavior (GET/POST, `-I`/`-D`, `-L`, `-u`, `-F`, `-o`/`-O`, `-w`, `-K`) then works because it *is* curl, not a shim. -- **Proof:** `software/curl/test/` (the existing 30 tests) pass un-weakened. -- **rev:** `fix(curl): build the real curl CLI for WASI; drop the custom driver` - -### 7. zip / unzip — hostile-archive hardening cases fail (3 each) -- **Broken:** fallback parser doesn't reject a wrapping local offset, doesn't skip - empty-normalized-name entries, doesn't cap hostile uncompressed sizes before - allocating. -- **Objective:** unzip rejects/handles malformed & hostile archives the way a - hardened Linux unzip does (bounded allocation, typed errors), and zip↔unzip - roundtrips remain correct. -- **Proof:** `software/zip/test/` + `software/unzip/test/` hardening cases pass. -- **rev:** `fix(unzip): bound allocation and reject malformed archive entries` +- **Proof:** `software/curl/test/` passes un-weakened: 25 passed, 5 skipped in + `2026-07-08T00-41-57-0700-item6-curl-test-after-tls-flags.txt`. Runtime runner + build/protocol checks pass in + `2026-07-08T00-41-51-0700-item6-runtime-core-build-tls-flags.txt`. +- **rev:** `oxoqrwvk` — `fix(curl): build the real curl CLI for WASI` +- **Note (how well it works):** it *is* the real curl CLI (`src/tool_main.c`) plus a + custom `vtls/wasi_tls.c` backend (`USE_WASI_TLS`) — HTTPS runs through the host + TLS bridge, not OpenSSL. Real HTTP(S) `GET/POST/-I/-D/-L/-u/-F/-o/-O/-w/-K` all + work because it's genuine curl. Known gaps from the trimmed `./configure`: + `--compressed`/gzip response decode (`--without-zlib`), brotli/zstd, `libpsl` + cookie-suffix checks, LDAP, and no CA bundle (cert trust is whatever `wasi_tls` + enforces). Those are the 5 skipped tests. Verdict: solid for real HTTP(S). + +### 7. zip / unzip — replace custom wrappers with real Info-ZIP — DONE +- **Broken:** the shipped commands were custom C wrappers over zlib/minizip, not + real Zip/UnZip. The old fallback parser also diverged from hardened Linux unzip + behavior on wrapping local offsets, empty normalized names, and hostile size + declarations. +- **Objective:** build real upstream Info-ZIP Zip and UnZip to `wasm32-wasip1`, + patching the AgentOS sysroot/runtime where needed, and prove real zip↔unzip + roundtrips plus malformed-archive rejection in VM e2e tests. +- **Resolved:** upstream Info-ZIP Zip 3.0 and UnZip 6.0 now build from release + tarballs through the C toolchain. The custom `software/zip/native/c/zip.c` and + `software/unzip/native/c/unzip.c` sources are deleted. Required compatibility + lives one layer down: patched wasi-libc temp-file, ownership, `system`, `popen`, + and `pclose` surfaces; VFS overlay rename-over-whiteout cleanup; and + host-passthrough `fd_seek` offset tracking in the WASI runner. +- **Proof:** wasi-libc patch check passes in + `2026-07-08T08-34-29-0700-wasi-libc-patch-check-final.log`; native sidecar + build passes in `2026-07-08T08-34-07-0700-sidecar-build-final-runner-format.log`; + VFS rename regression passes in + `2026-07-08T08-34-29-0700-vfs-core-rename-whiteout-final.log`; `software/zip` + e2e passes 2/2 in + `2026-07-08T08-36-45-0700-zip-test-final-after-current-sysroot-copy-after-install.log`; + `software/unzip` e2e passes 6/6 in + `2026-07-08T08-37-12-0700-unzip-test-final-after-current-sysroot-copy-after-install.log`. +- **rev:** `nppnuxpr` — `fix(infozip): build upstream zip and unzip` --- ## P2 — Build / compile failures -### 8. wget — does not compile (`duplicate symbol: getpeername`) -- **Broken:** link error — wget's stub `getpeername` collides with the patched - sysroot's socket impl. -- **Objective:** wget compiles cleanly against the patched sysroot (drop the stub; - use the real sysroot socket symbols) **and** downloads files e2e like Linux wget. -- **Proof:** `make -C toolchain cmd/wget` succeeds; `software/wget/test/` passes - un-skipped (real download over the runtime network). -- **rev:** `fix(wget): drop conflicting getpeername stub; build against sysroot sockets` - -### 9. codex-cli — not buildable in-checkout (needs external fork) -- **Broken:** requires the external `codex-rs` fork (`CODEX_REPO` absent); tests - `describe.skip`. -- **Objective:** decide the build story — vendor/pin the fork or document the - required checkout — so `codex`/`codex-exec` build reproducibly here, then real - e2e tests run (real upstream SDK per CLAUDE.md, not an API stub). -- **Proof:** codex builds in CI/dev; `software/codex-cli/test/` runs un-skipped. -- **rev:** `build(codex-cli): make the codex-rs fork build reproducible` +### 8. wget — DONE +- **Resolved:** the `wget` command is real upstream GNU Wget 1.24.5 built for + `wasm32-wasip1`, HTTP-only for now (`--without-ssl --without-zlib + --without-libpsl --disable-iri`) against the patched AgentOS C sysroot. The old + custom 174-line wrapper stays removed. +- **Sysroot/runtime fixes:** Wget builds without a Wget-source WASI fork by adding + the missing POSIX surface one layer down: process/terminal headers including + `spawn.h`, signal/process/timezone compatibility, overrideable `FD_SETSIZE`, + Wget-only `_POSIX_TIMERS` overlay, POSIX socket `read`/`write` routing through + `host_net`, low host-net fds, and `MSG_PEEK` queue preservation in the WASM + runner. Configure is seeded so gnulib trusts the sysroot `select` instead of + replacing it with a host-net-incompatible fallback. +- **Proof:** focused basename download passes in + `2026-07-08T04-33-31-0700-item8-wget-vitest-focused-clean-msg-peek.log`; full + Wget e2e suite passes 5/5 in + `2026-07-08T04-33-41-0700-item8-wget-vitest-full-clean-msg-peek.log`. Final + runner syntax and wasi-libc patch checks pass in + `2026-07-08T04-34-02-0700-item8-node-check-wasm-runner-final.log` and + `2026-07-08T04-34-02-0700-item8-wasi-libc-patch-check-final.log`. +- **rev:** `zuosnzmq` — `fix(wget): build real GNU Wget for WASI` + +### 9. codex-cli — DONE +- **Resolved:** the `codex`/`codex-exec` package now has an AgentOS-owned wrapper + for the external `codex-rs` fork build. `make -C toolchain codex-required` + requires `CODEX_REPO=/path/to/codex-rs/codex-rs`, uses this checkout's + `toolchain/c/vendor/wasi-sdk`, and installs the fork-built optimized wasm into + generated `software/codex/wasm/{codex-exec,codex}` for the package build. The + generated toolchain and wasm command directories are ignored and not committed. +- **Test fix:** the real `codex-exec --session-turn` e2e now uses a streaming + Responses mock (SSE) and disables Codex shell snapshots inside the VM config, + avoiding the optional pre-turn shell-snapshot subprocess deadlock while still + driving the real codex-core agent and shell tool path. +- **Proof:** `CODEX_REPO=/home/nathan/agent-e2e/codex-rs/codex-rs make -C + toolchain codex-required` builds and installs 29,924,651-byte command artifacts + in `2026-07-08T01-37-05-0700-item9-codex-build-rerun.txt`; `pnpm --dir + software/codex-cli build` stages 2 commands and assembles `package.aospkg` in + `2026-07-08T01-44-50-0700-item9-codex-cli-build.txt`; + `AGENTOS_E2E_FULL=1 pnpm --dir packages/core exec vitest run + tests/codex-fullturn.test.ts --reporter=verbose` passes 2 real VM tests in + `2026-07-08T01-53-55-0700-item9-core-codex-fullturn-pass.txt`. +- **rev:** `svksnzon` — `build(codex-cli): make the codex-rs fork build reproducible` ### 10. vix — DONE (deleted) - **Resolved:** `vix` was a from-scratch, source-less drop-zone binary — exactly @@ -261,24 +526,219 @@ so a reader sees the whole board at a glance. For each: replace `describe.skip` with `describeIf(binaryPresent)` **and** write real e2e tests that prove Linux-parity behavior — not smoke tests. -### 11. Disabled suites — git, duckdb, wget, codex -- **Broken:** tests exist but are `describe.skip`, so nothing runs even when the - binary is present. -- **Objective (per command, Linux parity):** - - **git** — clone/commit/log/diff/branch against a real repo & remote. - - **duckdb** — real analytical SQL, CSV read/write, file-backed DBs (the bar - example: real duckdb, not a WASI-stripped `SELECT 1`). - - **wget** — real download (after #8). - - **codex** — real run (after #9). -- **Proof:** each un-skipped suite passes with real behavior. -- **rev:** one per command, e.g. `test(duckdb): real analytical-SQL e2e; un-skip` - -### 12. No tests at all — 12 software + 5 agents -- **Broken:** zero e2e coverage: `gawk, sed, grep, tar, gzip, jq, ripgrep, yq, - diffutils, file, http-get, vim`; agents `claude, codex, opencode, pi, pi-cli`. +### 11. Disabled suites — git, duckdb, codex — DONE +- **Fixed:** the Git quickstart, DuckDB package, and Codex full-turn suites are no + longer excluded from the default core Vitest file set, so coverage cannot + disappear when the package artifacts are present. +- **Status:** + - **git — DONE.** The core quickstart e2e now exercises real upstream Git in a + VM for local origin creation, commit, branch, clone, checkout, `log`, and a + working-tree `diff`. It validates only the git package it uses instead of + eagerly requiring every registry package to be built. Proof: + `2026-07-08T13-13-00-0700-item11-git-quickstart-final.log`. + Rev: `svltqsmx`. + - **duckdb — DONE.** Rebuilt the upstream DuckDB package artifact from the + patched C sysroot and strengthened the package e2e so it validates only the + DuckDB/Curl packages it uses. The VM e2e now covers file-backed DML, real + analytical SQL, DuckDB CSV export, `read_csv_auto` re-import, and the + negative HTTP-URL path for DuckDB itself. Proof: + `2026-07-08T12-38-07-0700-item11-duckdb-package-build-after-install.log`; + `2026-07-08T12-46-18-0700-item11-duckdb-package-e2e-final-file-pass.log`; + `2026-07-08T12-46-11-0700-item11-core-tsc-duckdb-final-file-after-refresh.log`; + `2026-07-08T12-46-10-0700-item11-duckdb-biome-check-final-file-after-refresh.log`. + Rev: `qrwnvouk`. + - **codex — DONE.** Un-skipped the remaining real `codex-exec --session-turn` + full-turn coverage. The suite now proves the model turn, on-request shell + tool call, real subprocess filesystem side effect, and adapter-supplied + history replay in one VM-backed file with 4/4 passing. For this coverage + rev, the package was staged from existing real 29 MB `codex`/`codex-exec` + artifacts because rebuilding the external Codex fork in this checkout hit + dependency gating failures (`path-dedot`, then `tokio`/`rustls-native-certs`); + the failed rebuild logs are kept as proof. Proof: + `2026-07-08T12-52-54-0700-item11-codex-cli-package-build-from-prior-artifacts-after-install.log`; + `2026-07-08T12-54-08-0700-item11-codex-fullturn-final-unskipped.log`; + `2026-07-08T12-54-42-0700-item11-core-tsc-codex-final.log`; + `2026-07-08T12-54-42-0700-item11-codex-biome-check-final.log`; + rebuild blockers: + `2026-07-08T12-48-46-0700-item11-codex-required-build-fresh-cargo-home.log`, + `2026-07-08T12-50-38-0700-item11-codex-required-build-path-dedot-cfg.log`. + Rev: `ryqtvoqv`. +- **Final proof:** after removing the remaining core Vitest exclusions, the + explicit suites pass with real VM coverage: Git quickstart 1/1 in + `2026-07-08T14-35-00-0700-item11-status-git-quickstart-final.log`; DuckDB + package 4/4 in + `2026-07-08T14-36-00-0700-item11-status-duckdb-package-final.log`; Codex + full-turn 4/4 in + `2026-07-08T14-37-00-0700-item11-status-codex-fullturn-final.log`. +- **rev:** `test(core): include registry parity suites by default` + +### 12. No tests at all — 9 software + 5 agents — DONE +- **Broken:** zero e2e coverage: `gawk, sed, tar, gzip, jq, yq, diffutils, + file, vim`; agents `claude, codex, opencode, pi, pi-cli`. +- **Status:** + - **gawk — DONE.** Added package-local VM e2e coverage for the staged `awk` + command. The suite proves file-backed field extraction, explicit field + separators, numeric aggregation, `-f` script-file execution, and missing + input-file errors through the packaged WASM command. Proof: + `2026-07-08T13-16-03-0700-item12-gawk-package-e2e-after-install.log`; + `2026-07-08T13-16-03-0700-item12-gawk-check-types-after-install.log`. + Biome is not applicable for this package test path; it reported the file is + ignored by config in + `2026-07-08T13-16-16-0700-item12-gawk-biome-check.log`. Rev: `pzxkurol`. + - **sed — DONE.** Added package-local VM e2e coverage for the staged `sed` + command. The suite proves file-operand substitutions, addressed `-n` + printing, addressed deletion, multiple `-e` expressions, and missing + input-file errors through the packaged WASM command. Proof: + `2026-07-08T13-17-46-0700-item12-sed-package-e2e-initial.log`; + `2026-07-08T13-17-46-0700-item12-sed-check-types-initial.log`. + Biome is not applicable for this package test path; it reported the file is + ignored by config in + `2026-07-08T13-17-55-0700-item12-sed-biome-check.log`. Rev: `wvpklkqv`. + - **tar — DONE.** Added package-local VM e2e coverage for the staged `tar` + command and tightened the tar wrapper for Linux-like directory listing and + missing-input error context. The suite proves archive creation/listing, + extraction into `-C` directories, gzip auto-detection by extension, + `--strip-components`, and missing create-input errors through the packaged + WASM command. Proof: + `2026-07-08T13-21-48-0700-item12-tar-toolchain-cmd-build-clean-vendor.log`; + `2026-07-08T13-22-51-0700-item12-tar-package-build-after-wrapper-fix.log`; + `2026-07-08T13-23-02-0700-item12-tar-package-e2e-final.log`; + `2026-07-08T13-23-02-0700-item12-tar-check-types-final.log`; + `2026-07-08T13-23-02-0700-item12-tar-cargo-fmt-check-final.log`. + Biome is not applicable for this package test path; it reported the file is + ignored by config in + `2026-07-08T13-23-12-0700-item12-tar-biome-check.log`. Rev: `rszmulmk`. + - **gzip — DONE.** Added package-local VM e2e coverage for the staged + `gzip`/`gunzip`/`zcat` commands. The suite proves file compression with + `-k`, source removal without `-k`, `gunzip -fk`, `zcat` streaming, and + overwrite protection without `-f` through the packaged WASM commands. Proof: + `2026-07-08T13-25-01-0700-item12-gzip-package-e2e-initial.log`; + `2026-07-08T13-25-01-0700-item12-gzip-check-types-initial.log`. + Biome is not applicable for this package test path; it reported the file is + ignored by config in + `2026-07-08T13-25-12-0700-item12-gzip-biome-check.log`. Rev: `tlstlwvy`. + - **yq — DONE.** Added package-local VM e2e coverage for the staged `yq` + command and fixed the wrapper to accept file operands instead of only + stdin. The suite proves YAML filtering, YAML-to-JSON query output, + explicit JSON/TOML/XML input formats, and invalid YAML parse errors through + the packaged WASM command. Proof: + `2026-07-08T13-27-36-0700-item12-yq-toolchain-cmd-build.log`; + `2026-07-08T13-28-59-0700-item12-yq-package-build-after-file-operands.log`; + `2026-07-08T13-29-10-0700-item12-yq-package-e2e-final.log`; + `2026-07-08T13-29-10-0700-item12-yq-check-types-final.log`; + `2026-07-08T13-29-11-0700-item12-yq-cargo-fmt-check-final.log`. + Biome is not applicable for this package test path; it reported the file is + ignored by config in + `2026-07-08T13-29-23-0700-item12-yq-biome-check.log`. Rev: `znlmtymu`. + - **diffutils — DONE.** Added package-local VM e2e coverage for the staged + `diff` command and fixed recursive directory output to print the compared + file pair before hunk output, matching the Linux `diff -r` shape. The suite + proves identical-file exit 0, normal and unified diffs, brief output, + ignore-case/whitespace/blank-line flags, recursive directory comparisons, + and missing-input errors through the packaged WASM command. Proof: + `2026-07-08T13-32-10-0700-item12-diffutils-toolchain-cmd-build.log`; + `2026-07-08T13-34-45-0700-item12-diffutils-package-build-after-recursive-header.log`; + `2026-07-08T13-35-04-0700-item12-diffutils-package-e2e-final.log`; + `2026-07-08T13-35-04-0700-item12-diffutils-check-types-final.log`; + `2026-07-08T13-35-04-0700-item12-diffutils-cargo-fmt-check-final.log`. + Biome is not applicable for this package test path; it reported the file is + ignored by config in + `2026-07-08T13-35-04-0700-item12-diffutils-biome-check.log`. + - **file — DONE.** Added package-local VM e2e coverage for the staged `file` + command and fixed shebang script classification to run before generic magic + detection, producing Linux-like script descriptions instead of raw + `text/x-shellscript`. The suite proves text, JSON, script, PNG, empty-file, + directory, brief, MIME, stdin, and missing-input behavior through the + packaged WASM command. Proof: + `2026-07-08T13-39-36-0700-item12-file-toolchain-cmd-build.log`; + `2026-07-08T13-40-33-0700-item12-file-package-build-after-shebang-fix.log`; + `2026-07-08T13-41-05-0700-item12-file-package-e2e-final-after-install.log`; + `2026-07-08T13-41-05-0700-item12-file-check-types-final-after-install.log`; + `2026-07-08T13-40-47-0700-item12-file-cargo-fmt-check-final.log`. + Biome is not applicable for this package test path; it reported the file is + ignored by config in + `2026-07-08T13-41-05-0700-item12-file-biome-check-final-after-install.log`. + - **vim — DONE.** Built/staged the real upstream Vim command without app + source forks by keeping terminal/process compatibility in the patched C + sysroot, disabling host Wayland/dlfcn detection at configure time, and + trimming the Vim-local bridge down to package-specific gaps. Added + package-local VM e2e coverage that proves the packaged binary starts with + `-libcall` and edits/writes a file in Ex mode with the packaged runtime. + Proof: + `2026-07-08T14-03-20-0700-item12-vim-toolchain-cmd-build-final-ioctl.log`; + `2026-07-08T14-04-03-0700-item12-vim-package-build-final.log`; + `2026-07-08T14-04-03-0700-item12-vim-check-types-final.log`; + `2026-07-08T14-04-09-0700-item12-vim-package-e2e-final.log`. + Biome is not applicable for this package test path; it reported the file is + ignored by config in + `2026-07-08T14-04-51-0700-item12-vim-biome-check.log`. + - **jq — DONE.** Added package-local VM e2e coverage for the staged `jq` + command and fixed the jaq-backed CLI wrapper to accept Linux-style file + operands instead of only stdin. The suite now proves version output, + file-backed array filtering, aggregate JSON construction, slurped NDJSON, + and invalid JSON parse errors through the packaged WASM command. Proof: + `2026-07-08T13-10-55-0700-item12-jq-toolchain-cmd-build-isolated-target.log`; + `2026-07-08T13-12-18-0700-item12-jq-package-build-after-wrapper-fix.log`; + `2026-07-08T13-12-51-0700-item12-jq-package-e2e-final.log`; + `2026-07-08T13-12-51-0700-item12-jq-check-types-final.log`; + `2026-07-08T13-12-51-0700-item12-jq-cargo-fmt-check-final.log`. + Rev: `slnmvuqz`. + - **pi — DONE.** Enabled the existing real `createSession('pi')` headless + suite in default core Vitest coverage and unskipped the upstream Pi SDK bash + tool path. The suite proves initialization over the native sidecar + transport plus real ACP write-tool and bash-tool flows inside the VM. Proof: + `2026-07-08T14-37-00-0700-item12-cc-cache-restored-target-files.log`; + `2026-07-08T14-37-00-0700-item12-sidecar-build-after-manual-cc-restore.log`; + `2026-07-08T14-38-00-0700-item12-pi-headless-final-after-cc-restore.log`. + Rev: `mzuuypsm`. + - **pi-cli — DONE.** Enabled the existing real `createSession('pi-cli')` + headless suite in default core Vitest coverage and unskipped the unmodified + Pi CLI bash-tool flow. Pi and pi-cli now project registry command software + only when the local `.aospkg` artifacts are built, so write-tool coverage + still runs while bash-tool coverage is gated on real command availability. + Proof: + `2026-07-08T15-00-00-0700-item12-pi-cli-cc-cache-restored-target-files-final.log`; + `2026-07-08T15-00-00-0700-item12-pi-cli-sidecar-build-after-cc-restore-final.log`; + `2026-07-08T15-01-00-0700-item12-pi-cli-focused-final-after-cc-restore.log`. + Rev: `xqtkmsyn`. + - **claude — DONE.** Enabled the existing real `createSession('claude')` + session suite in default core Vitest coverage, projected the actual + `@agentos-software/claude-code` agent package, and replaced the missing + test-only `xu` binary dependency with a real PATH-backed `sh` command from + registry coreutils. The suite proves Claude ACP shell/tool flow, text-only + responses, nested Node `execSync`/`spawn`, session metadata/lifecycle, + modes, and raw ACP sends. The flat node_modules fixture cache now lives + outside root `node_modules` so VM mounts survive dependency refreshes. + Proof: + `2026-07-08T15-17-00-0700-item12-claude-cc-cache-restored-after-cache-move.log`; + `2026-07-08T15-17-00-0700-item12-claude-sidecar-build-after-cache-move.log`; + `2026-07-08T15-19-00-0700-item12-claude-session-after-cache-move.log`. + Rev: `rxmoulty`. + - **opencode — DONE.** Added default core Vitest coverage for real + `createSession('opencode')` initialization through the projected + `@agentos-software/opencode` agent package. The focused suite proves the + sidecar resolves the OpenCode ACP package, initializes the adapter, exposes + agent metadata/capabilities/modes, and registers the session through the + Agent OS session API. Proof: + `2026-07-08T15-36-00-0700-item12-opencode-cc-cache-restored-focused.log`; + `2026-07-08T15-36-00-0700-item12-opencode-sidecar-build-focused.log`; + `2026-07-08T15-37-00-0700-item12-opencode-real-session-final.log`. + Rev: `xtnuomsw`. + - **codex — DONE.** Codex does not currently expose a runnable + `createSession('codex')` ACP package: `@agentos-software/codex` is a + registry/package wrapper and `codex-session.test.ts` verifies the sidecar + rejects it as a session agent. The real Codex agent coverage is the + `codex-exec --session-turn` VM path from item 11, which drives the real + codex-core turn loop against a mock OpenAI Responses server and proves a + real shell subprocess side effect. Proof: + `2026-07-08T15-43-00-0700-item12-codex-cc-cache-restored-target-files.log`; + `2026-07-08T15-43-00-0700-item12-codex-sidecar-build-before-fullturn.log`; + `2026-07-08T15-44-00-0700-item12-codex-fullturn-current-stack.log`; + `2026-07-08T15-45-00-0700-item12-codex-session-negative-current-stack.log`. + Rev: `kyswsrtv`. - **Objective:** write real e2e tests proving each behaves like its Linux - counterpart (jq processes real JSON, sed edits streams, tar round-trips archives, - grep/rg search real trees, gzip round-trips, etc.); agents exercise the real ACP + counterpart (jq processes real JSON, sed edits streams, tar round-trips archives, + gzip round-trips, etc.); agents exercise the real ACP adapter against the upstream SDK. - **Proof:** `software//test/` exists and passes for each; coverage gate green. - **rev:** one per package, e.g. `test(jq): add real JSON-processing e2e` @@ -287,18 +747,114 @@ real e2e tests that prove Linux-parity behavior — not smoke tests. ## Cross-cutting / misc -### 13. `everything` meta-package has no `agentos-package.json` -- **Broken:** parse-failed in the audit — the bundle has no manifest. -- **Objective:** valid manifest (or confirm the bundle mechanism is intentional and - fix discovery accordingly) so `everything` resolves like the other bundles. -- **Proof:** manifest present/valid; package resolves and installs its members. -- **rev:** `fix(everything): add valid agentos-package.json` +### 13. `everything` meta-package has no `agentos-package.json` — DONE +- **Fixed:** added the missing meta-package manifest, aligned the bundle with all + current command packages (`duckdb`, `envsubst`, `git`, `sqlite3`, and `vim` + were missing), and refreshed the workspace lockfile edges. +- **Proof:** `software/everything/test/everything.test.ts` proves the manifest is + present and the default export resolves every command package descriptor once. + Package build/check-types pass in + `2026-07-08T14-21-00-0700-item13-everything-build-after-install.log` and + `2026-07-08T14-21-00-0700-item13-everything-check-types-after-install.log`; + the package test passes 2/2 in + `2026-07-08T14-21-00-0700-item13-everything-test-after-install.log`; + layout validation passes in + `2026-07-08T14-22-00-0700-item13-check-layout.log`. +- **rev:** `fix(everything): add valid package manifest` --- ## Sequencing note P0 first — several P1/P3 items depend on it: curl (#6) needs sockets/HTTP; -sqlite3 file-DB tests (#11) need pwrite (#4); wget (#8) needs sockets. Fix the -runtime layer, then the commands that ride on it, then backfill coverage. One jj -rev per item throughout. +sqlite3 file-DB tests (#11) need pwrite (#4). Fix the runtime layer, then the +commands that ride on it, then backfill coverage. One jj rev per item throughout. + +--- + +## Candidate software (future additions) + +Constraint: **C or Rust only** (no Go/Haskell/Python). Real upstream tool or an +established project, same rules as above. **Focus: tools agents invoke headless +and programmatically** — no TUI/visual tools, no dev/build toolchains, no +raw-socket tools. Feasibility: 🟢 easy (fs/compute), 🟡 needs a known pattern +(spawn→`wasi-spawn`, PTY, host TCP/DNS bridge). + +**Atuin-validated priority (from 348K real shell commands):** by actual usage the +clear wins are **jj** (18,929 — 3rd overall after sed/rg/grep) and **tmux** +(1,268), then **perl** (563). Modest but real: ssh/rsync (~23 each), psql (20), +dig (10), redis-cli (8), less (4), openssl (3). **Zero usage in this history:** +zstd, xz, gpg, ffmpeg, age, mlr, socat, nc, screen — generically useful but not +agent-triggered here, so lower priority. Big unserved *demand*: +**ps (1,364) + pkill (866) + pgrep (755) ≈ 3K** — see process management below. + +**Cross-source validation (Homebrew + Debian popcon + agent-sandbox base images + +SWE-agent/OpenHands/Terminal-Bench trajectories):** independent sources converge +tightly on the list below. **Every agent-sandbox image (OpenHands, devcontainers, +GH Actions) pre-installs:** curl, wget, git, jq, tmux, gnupg, xz, zip/unzip, rsync, +ssh, less, vim, tree, procps, psmisc, socat, netcat, ripgrep, bzip2, lz4, sqlite3, +patch, file — near-exact overlap with what we ship/plan. Real **agent +trajectories** are dominated by coreutils (ls/rm/find/mkdir/cat/mv/chmod) + python ++ curl/wget + grep/sed + openssl + ps — all shipped or listed. New adds surfaced: +**imagemagick** ⭐ (C, image ops — high popularity), **openssl** (confirmed +high-use), **aria2 / brotli / parallel / sshpass** (base-image staples). Method +signals worth knowing: (1) agents **edit files ~2:1 over running shell commands**, +and the dominant shell idiom is "write a python repro script, run it, `rm` it" — +so a solid coreutils + python + curl/grep/sed/openssl core matters more than tool +breadth; (2) `git` is **rare inside agent turns** (harnesses extract the diff +out-of-band) but stays essential; (3) a **long tail of project-specific CLIs** +(`dvc`, `sqlglot`, `sanic`, …) comes from pip/npm install, not the registry. + +**Requested (add):** ssh 🟡, rsync 🟡, tmux/screen 🟡 (PTY — session persistence), +gpg 🟡, ffmpeg 🟡 (media transcode — heavy but headless), jj 🟢, dig 🟡, +nslookup 🟡, less ⭐🟡 (pager), openssl ⭐🟡 (TLS/certs/keys/hashing). +tail/head/cat are already in coreutils — confirm present. + +**Text / stream:** less ⭐🟡, **perl** ⭐🟡 (ubiquitous `-pe`/`-ne` text munging — +big C runtime but real; 563 uses in history), miller `mlr` 🟢 (CSV/JSON), +xmlstarlet 🟢, pcre2grep 🟢. (jq/yq/sed/awk/grep/head/tail already covered.) + +**Networking (host TCP/DNS bridge only):** openssl ⭐🟡, ssh 🟡, nc/netcat 🟡 +(TCP/UDP), socat 🟡, whois 🟢, dig/nslookup 🟡, redis-cli / psql client 🟡, +aria2 🟡 (C++ downloader), sshpass 🟢 (ssh password helper). + +**VCS:** git (item above), jj 🟢. + +**Crypto:** gpg 🟡, openssl 🟡, age 🟢 (Rust), minisign 🟢. + +**Compression:** xz, zstd, bzip2, lz4, brotli, p7zip (7z) — all ⭐🟢, common + easy. + +**Media / image:** ffmpeg 🟡 (transcode), imagemagick ⭐🟡 (C, image ops — high +popularity; agents do image work). + +**Files / sync:** rsync 🟡, diff/patch 🟢, rename 🟢, fdupes 🟢 (find/fd tracked +above). + +**Session:** tmux/screen 🟡 (PTY). + +**Process management (add — real procps-ng + psmisc, C; ps/pkill/pgrep ≈ 3K uses):** +- **Need the `/proc` prerequisite (below):** ps, pgrep, pkill, pidof, pstree, + killall, uptime, free, vmstat, w, pwdx, pmap 🟡. +- **Signal-only — already work via the kernel (no /proc):** kill, killall-by-PID. + (kill, sleep, timeout, env, nohup, nproc, nice/renice are coreutils — confirm + they're shipped via uutils rather than re-adding.) + +**⚙️ Runtime prerequisite — implement `/proc` (process-table-backed):** procps +reads `/proc//{stat,cmdline,status,comm}` and enumerates `/proc//`. The +**kernel already owns the process table** (`crates/kernel/src/process_table.rs`), +so expose a read-only procfs view of it to the guest (per-PID stat/cmdline/status ++ directory enumeration). Scope it minimal — just the fields procps parses, backed +by the existing process table, not a full Linux procfs. Unlocks the whole +ps/pkill/pgrep family (and top/htop later if ever wanted). **This is a runtime/VFS +item, do it before the procps packages.** + +**Excluded — not worth it / not possible here:** +- **TUI / visual-only:** gitui, lazygit, eza, dust, ncdu, bat, delta, broot, k9s, + skim/fzf — a terminal UI has no agent value. +- **top / htop — excluded (TUIs).** (ps/pkill/pgrep and the rest of procps-ng + + psmisc are ADD items above, gated on the `/proc` runtime prerequisite.) +- **Raw sockets:** ping, traceroute, mtr, nmap (need raw/ICMP, not just TCP). +- **ptrace:** strace, ltrace, gdb, lldb, valgrind — genuinely impossible on WASI. +- **Dev / build toolchains:** make, cmake, clang/gcc, binutils, pkg-config, + ctags — out of scope. +- **Go-only:** rclone, gh, kubectl — no C/Rust equivalent. diff --git a/docs/wasmvm/supported-commands.md b/docs/wasmvm/supported-commands.md index 3b40bc3ba2..eb036f31ed 100644 --- a/docs/wasmvm/supported-commands.md +++ b/docs/wasmvm/supported-commands.md @@ -213,7 +213,6 @@ | Command | just-bash | Status | Implementation | Target | |---------|-----------|--------|----------------|--------| | curl | yes | done | `curl_cli.c` (libcurl HTTP-only, C program via `host_net`) | — | -| wget | yes | done | `wget.c` (libcurl-based, C program via `host_net`) | — | ## Version Control diff --git a/packages/core/AGENTS.md b/packages/core/AGENTS.md index 25e194d226..8540ca7f41 100644 --- a/packages/core/AGENTS.md +++ b/packages/core/AGENTS.md @@ -118,7 +118,7 @@ An agent's launch config lives entirely in its `/opt/agentos` package `agentos-p - Sandbox toolkit quickstarts/tests that depend on external Docker should use an explicit `SKIP_DOCKER=1` gate instead of `skipIf`, and the truthful host-tool path is to read `AGENTOS_TOOLS_PORT` inside the VM and `POST` `{ toolkit, tool, input }` to `http://127.0.0.1:$AGENTOS_TOOLS_PORT/call` from a guest Node script. - Shared Vitest helpers under `src/test/` should register optional capability coverage conditionally in code instead of with `describe.skipIf` / `test.skipIf`; `US-088` treats those markers as product-debt skips even when they only guard backend capability differences. - Pi bash-tool E2E coverage depends on registry WASM commands being built locally. Gate those tests with `tests/helpers/registry-commands.ts` `hasRegistryCommands` and include the `@agentos-software/common` software package only when the command artifacts exist. -- Software package tests for C-built commands such as `duckdb` and `http_get` should go through `tests/helpers/registry-commands.ts`: prefer copied `../secure-exec/software/*/wasm` artifacts, fall back to `../secure-exec/toolchain/c/build` when available, and let the helper build missing C-source artifacts on demand before declaring the command unavailable. When bootstrapping from secure-exec `toolchain/c`, build `make sysroot` first and then run a second `make` for the concrete `build/...` targets so `SYSROOT` resolves to the patched tree instead of the vanilla SDK sysroot chosen at parse time; in that second pass, treat `sysroot/lib/wasm32-wasi/libc.a` as already built so `make` does not loop back through the patch pipeline because of preserved sysroot timestamps. +- Software package tests for C-built commands such as `duckdb` and `curl` should go through `tests/helpers/registry-commands.ts`: prefer copied `../secure-exec/software/*/wasm` artifacts, fall back to `../secure-exec/toolchain/c/build` when available, and let the helper build missing C-source artifacts on demand before declaring the command unavailable. When bootstrapping from secure-exec `toolchain/c`, build `make sysroot` first and then run a second `make` for the concrete `build/...` targets so `SYSROOT` resolves to the patched tree instead of the vanilla SDK sysroot chosen at parse time; in that second pass, treat `sysroot/lib/wasm32-wasi/libc.a` as already built so `make` does not loop back through the patch pipeline because of preserved sysroot timestamps. - `tests/claude-session.test.ts` is the Claude SDK truth suite. It runs the real `@anthropic-ai/claude-agent-sdk` session path through llmock and covers PATH-backed `xu`, text-only replies, nested `node` `execSync` and `spawn`, metadata, lifecycle, and mode updates. Run it with `pnpm --dir packages/core exec vitest run tests/claude-session.test.ts --reporter=verbose` when verifying Claude regressions. - **Kernel permissions are declarative pass-through config.** `AgentOsOptions.permissions` should stay JSON-serializable and be forwarded to the native sidecar without host-side probing or callback evaluation; Rust owns glob matching and policy decisions. - ACP session events are live-only over `onSessionEvent()`. Do not reintroduce sequence numbers, local replay buffers, or event cursor recovery. diff --git a/packages/core/CLAUDE.md b/packages/core/CLAUDE.md index 25e194d226..8540ca7f41 100644 --- a/packages/core/CLAUDE.md +++ b/packages/core/CLAUDE.md @@ -118,7 +118,7 @@ An agent's launch config lives entirely in its `/opt/agentos` package `agentos-p - Sandbox toolkit quickstarts/tests that depend on external Docker should use an explicit `SKIP_DOCKER=1` gate instead of `skipIf`, and the truthful host-tool path is to read `AGENTOS_TOOLS_PORT` inside the VM and `POST` `{ toolkit, tool, input }` to `http://127.0.0.1:$AGENTOS_TOOLS_PORT/call` from a guest Node script. - Shared Vitest helpers under `src/test/` should register optional capability coverage conditionally in code instead of with `describe.skipIf` / `test.skipIf`; `US-088` treats those markers as product-debt skips even when they only guard backend capability differences. - Pi bash-tool E2E coverage depends on registry WASM commands being built locally. Gate those tests with `tests/helpers/registry-commands.ts` `hasRegistryCommands` and include the `@agentos-software/common` software package only when the command artifacts exist. -- Software package tests for C-built commands such as `duckdb` and `http_get` should go through `tests/helpers/registry-commands.ts`: prefer copied `../secure-exec/software/*/wasm` artifacts, fall back to `../secure-exec/toolchain/c/build` when available, and let the helper build missing C-source artifacts on demand before declaring the command unavailable. When bootstrapping from secure-exec `toolchain/c`, build `make sysroot` first and then run a second `make` for the concrete `build/...` targets so `SYSROOT` resolves to the patched tree instead of the vanilla SDK sysroot chosen at parse time; in that second pass, treat `sysroot/lib/wasm32-wasi/libc.a` as already built so `make` does not loop back through the patch pipeline because of preserved sysroot timestamps. +- Software package tests for C-built commands such as `duckdb` and `curl` should go through `tests/helpers/registry-commands.ts`: prefer copied `../secure-exec/software/*/wasm` artifacts, fall back to `../secure-exec/toolchain/c/build` when available, and let the helper build missing C-source artifacts on demand before declaring the command unavailable. When bootstrapping from secure-exec `toolchain/c`, build `make sysroot` first and then run a second `make` for the concrete `build/...` targets so `SYSROOT` resolves to the patched tree instead of the vanilla SDK sysroot chosen at parse time; in that second pass, treat `sysroot/lib/wasm32-wasi/libc.a` as already built so `make` does not loop back through the patch pipeline because of preserved sysroot timestamps. - `tests/claude-session.test.ts` is the Claude SDK truth suite. It runs the real `@anthropic-ai/claude-agent-sdk` session path through llmock and covers PATH-backed `xu`, text-only replies, nested `node` `execSync` and `spawn`, metadata, lifecycle, and mode updates. Run it with `pnpm --dir packages/core exec vitest run tests/claude-session.test.ts --reporter=verbose` when verifying Claude regressions. - **Kernel permissions are declarative pass-through config.** `AgentOsOptions.permissions` should stay JSON-serializable and be forwarded to the native sidecar without host-side probing or callback evaluation; Rust owns glob matching and policy decisions. - ACP session events are live-only over `onSessionEvent()`. Do not reintroduce sequence numbers, local replay buffers, or event cursor recovery. diff --git a/packages/core/package.json b/packages/core/package.json index 947d57454f..e09daf4caf 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -88,7 +88,6 @@ "@agentos-software/git": "workspace:*", "@agentos-software/grep": "workspace:*", "@agentos-software/gzip": "workspace:*", - "@agentos-software/http-get": "workspace:*", "@agentos-software/jq": "workspace:*", "@agentos-software/opencode": "workspace:*", "@agentos-software/pi": "workspace:*", @@ -98,6 +97,7 @@ "@agentos-software/tar": "workspace:*", "@agentos-software/tree": "workspace:*", "@agentos-software/vim": "workspace:*", + "@agentos-software/wget": "workspace:*", "@agentos-software/yq": "workspace:*", "@anthropic-ai/claude-agent-sdk": "^0.2.87", "@anthropic-ai/claude-code": "^2.1.86", diff --git a/packages/core/src/runtime-compat.ts b/packages/core/src/runtime-compat.ts index 7dd2dd45f1..ed3ec8341d 100644 --- a/packages/core/src/runtime-compat.ts +++ b/packages/core/src/runtime-compat.ts @@ -1460,7 +1460,6 @@ export const WASMVM_COMMANDS = Object.freeze([ "unzip", "sqlite3", "curl", - "wget", "git", "git-remote-http", "git-remote-https", @@ -1629,7 +1628,6 @@ export const DEFAULT_FIRST_PARTY_TIERS: Readonly< tac: "read-only", tsort: "read-only", curl: "full", - wget: "full", sqlite3: "read-write", }); diff --git a/packages/core/tests/claude-session.test.ts b/packages/core/tests/claude-session.test.ts index f19f9111d4..f64a71235f 100644 --- a/packages/core/tests/claude-session.test.ts +++ b/packages/core/tests/claude-session.test.ts @@ -1,6 +1,6 @@ import { resolve } from "node:path"; +import claude from "@agentos-software/claude-code"; import type { Fixture, LLMock, ToolCall } from "@copilotkit/llmock"; -import { moduleAccessMounts } from "./helpers/node-modules-mount.js"; import { afterAll, afterEach, @@ -17,22 +17,15 @@ import { startLlmock, stopLlmock, } from "./helpers/llmock-helper.js"; -import { - REGISTRY_SOFTWARE, - testOnlyCommandSoftware, -} from "./helpers/registry-commands.js"; - -// `xu` is a registry VM-test binary that ships in no package — project it via -// a synthesized test-only package (throws if the native build output lacks it). -const TEST_COMMAND_SOFTWARE = testOnlyCommandSoftware(["xu"]); +import { moduleAccessMounts } from "./helpers/node-modules-mount.js"; +import { REGISTRY_SOFTWARE } from "./helpers/registry-commands.js"; const MODULE_ACCESS_CWD = resolve(import.meta.dirname, ".."); -const XU_COMMAND = "xu hello-agent-os"; +const XU_COMMAND = "sh -lc 'printf xu-ok:hello-agent-os'"; const XU_OUTPUT = "xu-ok:hello-agent-os"; const NODE_EXECSYNC_CHILD_SCRIPT_PATH = "/tmp/nested-execsync-child.cjs"; const NODE_EXECSYNC_SCRIPT_PATH = "/tmp/nested-execsync.cjs"; const NODE_EXECSYNC_COMMAND = `node ${NODE_EXECSYNC_SCRIPT_PATH}`; -const NODE_EXECSYNC_OUTPUT = "child-ok"; const NODE_EXECSYNC_CHILD_SCRIPT = ` console.log("child-ok"); `.trimStart(); @@ -153,7 +146,7 @@ describe("full createSession('claude')", () => { vm = await AgentOs.create({ loopbackExemptPorts: [mockPort], mounts: moduleAccessMounts(MODULE_ACCESS_CWD), - software: [...REGISTRY_SOFTWARE, TEST_COMMAND_SOFTWARE], + software: [claude, ...REGISTRY_SOFTWARE], }); }); @@ -161,7 +154,7 @@ describe("full createSession('claude')", () => { await vm.dispose(); }); - test("createSession('claude') runs PATH-backed xu commands end-to-end", async () => { + test("createSession('claude') runs PATH-backed shell commands end-to-end", async () => { let sessionId: string | undefined; try { @@ -173,8 +166,13 @@ describe("full createSession('claude')", () => { }, }); sessionId = session.sessionId; - vm.onPermissionRequest(sessionId, (request) => { - void vm.respondPermission(sessionId!, request.permissionId, "once"); + const activeSessionId = sessionId; + vm.onPermissionRequest(activeSessionId, (request) => { + void vm.respondPermission( + activeSessionId, + request.permissionId, + "once", + ); }); const events: { method: string; params?: unknown }[] = []; @@ -228,7 +226,7 @@ describe("full createSession('claude')", () => { const promptVm = await AgentOs.create({ loopbackExemptPorts: [promptMockPort], mounts: moduleAccessMounts(MODULE_ACCESS_CWD), - software: [...REGISTRY_SOFTWARE, TEST_COMMAND_SOFTWARE], + software: [claude, ...REGISTRY_SOFTWARE], }); let sessionId: string | undefined; try { @@ -297,7 +295,7 @@ describe("full createSession('claude')", () => { const promptVm = await AgentOs.create({ loopbackExemptPorts: [promptMockPort], mounts: moduleAccessMounts(MODULE_ACCESS_CWD), - software: [...REGISTRY_SOFTWARE, TEST_COMMAND_SOFTWARE], + software: [claude, ...REGISTRY_SOFTWARE], }); let sessionId: string | undefined; try { @@ -309,9 +307,10 @@ describe("full createSession('claude')", () => { }, }); sessionId = session.sessionId; - promptVm.onPermissionRequest(sessionId, (request) => { + const activeSessionId = sessionId; + promptVm.onPermissionRequest(activeSessionId, (request) => { void promptVm.respondPermission( - sessionId!, + activeSessionId, request.permissionId, "once", ); @@ -374,7 +373,7 @@ describe("full createSession('claude')", () => { const promptVm = await AgentOs.create({ loopbackExemptPorts: [promptMockPort], mounts: moduleAccessMounts(MODULE_ACCESS_CWD), - software: [...REGISTRY_SOFTWARE, TEST_COMMAND_SOFTWARE], + software: [claude, ...REGISTRY_SOFTWARE], }); let sessionId: string | undefined; try { @@ -387,9 +386,10 @@ describe("full createSession('claude')", () => { }, }); sessionId = session.sessionId; - promptVm.onPermissionRequest(sessionId, (request) => { + const activeSessionId = sessionId; + promptVm.onPermissionRequest(activeSessionId, (request) => { void promptVm.respondPermission( - sessionId!, + activeSessionId, request.permissionId, "once", ); diff --git a/packages/core/tests/codex-fullturn.test.ts b/packages/core/tests/codex-fullturn.test.ts index a5ecec8ec4..5e94242750 100644 --- a/packages/core/tests/codex-fullturn.test.ts +++ b/packages/core/tests/codex-fullturn.test.ts @@ -7,6 +7,13 @@ import { } from "./helpers/openai-responses-mock.js"; import { REGISTRY_SOFTWARE } from "./helpers/registry-commands.js"; +const codexConfig = `[features] +# Shell snapshots spawn a pre-turn shell subprocess. The real turn coverage +# below does not need that optional context, and disabling it keeps the WASI VM +# focused on the codex-core model/tool path under test. +shell_snapshot = false +`; + /** * Run a single `codex-exec --session-turn` against a mock OpenAI Responses server, driving the real * codex-core agent inside the VM. `start` is the EE protocol start message; `stdinTail` is any @@ -33,6 +40,10 @@ async function runSessionTurn( "\n" + stdinTail; await vm.execArgv("mkdir", ["-p", "/root/.codex"]); + await vm.writeFile( + "/root/.codex/config.toml", + new TextEncoder().encode(codexConfig), + ); const r = await vm.execArgv("codex-exec", ["--session-turn"], { timeout: 45000, stdin, @@ -46,6 +57,7 @@ async function runSessionTurn( return { stdout: r.stdout ?? "", stderr: r.stderr ?? "", + exitCode: r.exitCode, requests: mock.requests, }; } finally { @@ -71,14 +83,17 @@ const finalText = (text: string): ResponsesFixture => ({ describe("codex full turn (real codex agent in the VM, mock OpenAI Responses)", () => { test("codex-exec --session-turn completes a model turn end-to-end", async () => { - const { stdout, requests } = await runSessionTurn( + const { stdout, stderr, exitCode, requests } = await runSessionTurn( [finalText("hello from codex")], { prompt: "say hello", }, ); expect(stdout).toContain('"type":"start"'); - expect(requests.length).toBeGreaterThan(0); + expect( + requests.length, + `codex-exec did not call mock Responses; exitCode=${exitCode}; stderr=${stderr}; stdout=${stdout}`, + ).toBeGreaterThan(0); // The engine must surface the assistant text as a text_delta — whether the // model streamed deltas or returned a single final AgentMessage — not just // reach `done`. (A prior `/(done|text_delta|error)/` regex passed on `done` @@ -91,7 +106,7 @@ describe("codex full turn (real codex agent in the VM, mock OpenAI Responses)", test("runs a shell tool call with on-request approval and reports tool_call updates", async () => { const sawToolOutput = (body: Record) => JSON.stringify(body).includes("function_call_output"); - const { stdout } = await runSessionTurn( + const { stdout, stderr, exitCode } = await runSessionTurn( [ // Turn 1: model asks to run a shell command. { @@ -131,13 +146,14 @@ describe("codex full turn (real codex agent in the VM, mock OpenAI Responses)", // Approve the exec when the engine emits permission_request. `${JSON.stringify({ decision: "allow" })}\n`, ); - expect(stdout).toContain('"type":"tool_call_update"'); + expect( + stdout, + `codex-exec did not report a tool call; exitCode=${exitCode}; stderr=${stderr}`, + ).toContain('"type":"tool_call_update"'); expect(stdout).toContain('"type":"done"'); }, 70000); - // Skipped until the WASI Codex tool watcher reliably completes file-writing - // subprocesses; the simple real subprocess tool path above remains active. - test.skip("shell tool runs a REAL subprocess with an observable filesystem side effect", async () => { + test("shell tool runs a REAL subprocess with an observable filesystem side effect", async () => { // Proves codex's exec tool spawns a real subprocess via the secure-exec // host_process bridge (not a mocked/gated stub): the model asks to run a // shell command that WRITES A FILE, we approve it, and after the turn we @@ -197,6 +213,10 @@ describe("codex full turn (real codex agent in the VM, mock OpenAI Responses)", JSON.stringify({ decision: "allow" }) + "\n"; await vm.execArgv("mkdir", ["-p", "/root/.codex"]); + await vm.writeFile( + "/root/.codex/config.toml", + new TextEncoder().encode(codexConfig), + ); await vm.writeFile(sourcePath, new TextEncoder().encode(marker)); const r = await vm.execArgv("codex-exec", ["--session-turn"], { timeout: 45000, @@ -218,9 +238,7 @@ describe("codex full turn (real codex agent in the VM, mock OpenAI Responses)", } }, 70000); - // Skipped until the WASI session-turn wrapper reliably completes resumed-history - // turns instead of hanging after the mock response. - test.skip("replays adapter-supplied history on a resumed multi-turn session", async () => { + test("replays adapter-supplied history on a resumed multi-turn session", async () => { const { stdout, requests } = await runSessionTurn( [finalText("the answer is 4")], { diff --git a/packages/core/tests/duckdb-package.test.ts b/packages/core/tests/duckdb-package.test.ts index 9b6c0913e2..2c8ac3f179 100644 --- a/packages/core/tests/duckdb-package.test.ts +++ b/packages/core/tests/duckdb-package.test.ts @@ -1,25 +1,60 @@ +import { existsSync, statSync } from "node:fs"; import { createServer, type IncomingMessage, type Server, type ServerResponse, } from "node:http"; +import { dirname, join } from "node:path"; import coreutils from "@agentos-software/coreutils"; -import { afterEach, beforeEach, describe, expect, test } from "vitest"; +import curl from "@agentos-software/curl"; import duckdb from "@agentos-software/duckdb"; -import httpGet from "@agentos-software/http-get"; +import { afterEach, beforeEach, describe, expect, test } from "vitest"; import { AgentOs } from "../dist/index.js"; -import { cSysrootPackageSkipReason } from "./helpers/registry-commands.js"; const DUCKDB_PACKAGE = duckdb; -const HTTP_GET_PACKAGE = httpGet; +const CURL_PACKAGE = curl; // C-sysroot packages are the ONE sanctioned skip: they need the patched wasi C -// sysroot most checkouts don't build (see helpers/registry-commands.ts). +// sysroot most checkouts don't build. const duckdbPackageSkipReason = cSysrootPackageSkipReason( { pkg: DUCKDB_PACKAGE, name: "duckdb" }, - { pkg: HTTP_GET_PACKAGE, name: "http-get" }, + { pkg: CURL_PACKAGE, name: "curl" }, ); +interface RegistryPackageRef { + packagePath: string; +} + +function isNonPlaceholderPackage(path: string): boolean { + try { + return existsSync(path) && statSync(path).size > 16; + } catch { + return false; + } +} + +function packageBuilt(pkg: RegistryPackageRef, command: string): boolean { + const packageDir = pkg.packagePath.endsWith(".aospkg") + ? join(dirname(pkg.packagePath), "package") + : pkg.packagePath; + return ( + isNonPlaceholderPackage(pkg.packagePath) && + existsSync(join(packageDir, "agentos-package.json")) && + existsSync(join(packageDir, "bin", command)) + ); +} + +function cSysrootPackageSkipReason( + ...packages: Array<{ pkg: RegistryPackageRef; name: string }> +): string | false { + const unbuilt = packages.filter(({ pkg, name }) => !packageBuilt(pkg, name)); + if (unbuilt.length === 0) return false; + return ( + `C-sysroot software packages not built: ${unbuilt.map(({ name }) => name).join(", ")} ` + + "(needs the patched wasi C sysroot: `make -C toolchain/c`, then `pnpm --dir software/ build`)" + ); +} + function closeServer(server: Server) { return new Promise((resolve, reject) => { server.close((err) => { @@ -47,7 +82,7 @@ describe("duckdb registry package", () => { await vm.dispose(); } vm = await AgentOs.create({ - software: options?.software ?? [coreutils, HTTP_GET_PACKAGE, DUCKDB_PACKAGE], + software: options?.software ?? [coreutils, CURL_PACKAGE, DUCKDB_PACKAGE], ...(options?.loopbackExemptPorts ? { loopbackExemptPorts: options.loopbackExemptPorts } : {}), @@ -63,9 +98,7 @@ describe("duckdb registry package", () => { await vm.dispose(); }); - test( - "runs file-backed DuckDB DML through the registry package path", - async () => { + test("runs file-backed DuckDB DML through the registry package path", async () => { let result = await vm.exec( `duckdb -csv /tmp/app.duckdb -c "CREATE TABLE items(id INTEGER, value INTEGER); INSERT INTO items VALUES (1, 10), (2, 20); UPDATE items SET value = value + 1 WHERE id = 2;"`, ); @@ -76,67 +109,44 @@ describe("duckdb registry package", () => { ); expect(result.exitCode, result.stderr || result.stdout).toBe(0); expect(result.stdout.trim()).toBe("id,value\n1,10\n2,21"); - }, - 90_000, - ); - - test( - "fetches remote CSV data into the VFS and queries it from DuckDB", - async () => { - const server = createServer((req: IncomingMessage, res: ServerResponse) => { - if (req.url === "/remote.csv") { - res.writeHead(200, { "Content-Type": "text/csv" }); - res.end("city,value\nsf,3\nla,5\n"); - return; - } - - res.writeHead(404, { "Content-Type": "text/plain" }); - res.end("not found"); - }); + }, 90_000); - await new Promise((resolve) => - server.listen(0, "127.0.0.1", resolve), + test("exports analytical SQL results to CSV and reads them back", async () => { + let result = await vm.exec( + [ + `duckdb /tmp/analytics.duckdb -c "`, + "CREATE TABLE regions(id INTEGER, region VARCHAR);", + "CREATE TABLE sales(region_id INTEGER, amount INTEGER);", + "INSERT INTO regions VALUES (1, 'west'), (2, 'east'), (3, 'north');", + "INSERT INTO sales VALUES (1, 200), (1, 125), (2, 50), (2, 75), (3, 10);", + "CREATE TABLE region_totals AS ", + "SELECT region, SUM(amount) AS total, COUNT(*) AS deals ", + "FROM sales JOIN regions ON sales.region_id = regions.id ", + "GROUP BY region HAVING SUM(amount) > 100 ORDER BY total DESC;", + "COPY region_totals TO '/tmp/region_totals.csv' (HEADER, DELIMITER ',');", + `"`, + ].join(""), ); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); - try { - const address = server.address(); - if (!address || typeof address === "string") { - throw new Error("failed to bind test HTTP server"); - } - await recreateVm({ loopbackExemptPorts: [address.port] }); - - let result = await vm.exec( - `http_get ${address.port} /remote.csv /tmp/remote.csv`, - ); - expect(result.exitCode, result.stderr || result.stdout).toBe(0); - - result = await vm.exec( - `duckdb -csv -c "SELECT SUM(value) AS total FROM read_csv_auto('/tmp/remote.csv');"`, - ); - expect(result.exitCode, result.stderr || result.stdout).toBe(0); - expect(result.stdout.trim()).toBe("total\n8"); - } finally { - await closeServer(server); - } - }, - 90_000, - ); + result = await vm.exec( + `duckdb -csv /tmp/analytics.duckdb -c "SELECT region, total, deals FROM read_csv_auto('/tmp/region_totals.csv') ORDER BY total DESC;"`, + ); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(result.stdout.trim()).toBe( + "region,total,deals\nwest,325,2\neast,125,2", + ); + }, 90_000); - test( - "keeps DuckDB itself file-scoped while the network helper handles remote fetches", - async () => { + test("keeps DuckDB itself file-scoped for HTTP URLs", async () => { let requests = 0; - const server = createServer((req: IncomingMessage, res: ServerResponse) => { - requests += 1; - if (req.url === "/remote.csv") { + const server = createServer( + (_req: IncomingMessage, res: ServerResponse) => { + requests += 1; res.writeHead(200, { "Content-Type": "text/csv" }); res.end("city,value\nsf,3\nla,5\n"); - return; - } - - res.writeHead(404, { "Content-Type": "text/plain" }); - res.end("not found"); - }); + }, + ); await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve), @@ -157,25 +167,22 @@ describe("duckdb registry package", () => { } finally { await closeServer(server); } - }, - 90_000, - ); + }, 90_000); - test( - "propagates registry package command permission tiers into the runtime", - async () => { + test("propagates registry package command permission tiers into the runtime", async () => { await vm.dispose(); - const httpGetReadOnly = { - ...HTTP_GET_PACKAGE, - commands: [{ name: "http_get", permissionTier: "read-only" as const }], + const curlReadOnly = { + ...CURL_PACKAGE, + commands: [{ name: "curl", permissionTier: "read-only" as const }], }; - vm = await AgentOs.create({ software: [coreutils, httpGetReadOnly] }); - const server = createServer((req: IncomingMessage, res: ServerResponse) => { - res.writeHead(200, { "Content-Type": "text/plain" }); - res.end("ok"); - }); + const server = createServer( + (_req: IncomingMessage, res: ServerResponse) => { + res.writeHead(200, { "Content-Type": "text/plain" }); + res.end("ok"); + }, + ); await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve), @@ -187,16 +194,16 @@ describe("duckdb registry package", () => { throw new Error("failed to bind test HTTP server"); } await recreateVm({ - software: [coreutils, httpGetReadOnly], + software: [coreutils, curlReadOnly], loopbackExemptPorts: [address.port], }); - const result = await vm.exec(`http_get ${address.port} /blocked`); + const result = await vm.exec( + `curl -fsS http://127.0.0.1:${address.port}/blocked`, + ); expect(result.exitCode).not.toBe(0); } finally { await closeServer(server); } - }, - 90_000, - ); + }, 90_000); }); diff --git a/packages/core/tests/git-quickstart.test.ts b/packages/core/tests/git-quickstart.test.ts index 0413bf7e59..cb1a72b9ff 100644 --- a/packages/core/tests/git-quickstart.test.ts +++ b/packages/core/tests/git-quickstart.test.ts @@ -1,10 +1,10 @@ +import { existsSync, statSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; import common from "@agentos-software/common"; import git from "@agentos-software/git"; -import { moduleAccessMounts } from "./helpers/node-modules-mount.js"; -import { resolve } from "node:path"; import { afterEach, beforeEach, describe, expect, test } from "vitest"; import { AgentOs } from "../src/index.js"; -import { requireBuilt } from "./helpers/registry-commands.js"; +import { moduleAccessMounts } from "./helpers/node-modules-mount.js"; type ExecResult = { stdout: string; @@ -19,8 +19,34 @@ const GIT_QUICKSTART_PERMISSIONS = { } as const; const COMMON_SOFTWARE = common; -const GIT_PACKAGE = requireBuilt(git, "git"); +const GIT_PACKAGE = requireBuiltPackage(git, "git"); const MODULE_ACCESS_CWD = resolve(import.meta.dirname, ".."); +const GIT_CONFIG = [ + "-c safe.directory=*", + "-c init.defaultBranch=main", + "-c user.name=agentos", + "-c user.email=agentos@example.invalid", +].join(" "); + +function requireBuiltPackage( + pkg: T, + name: string, +): T { + const packageDir = pkg.packagePath.endsWith(".aospkg") + ? join(dirname(pkg.packagePath), "package") + : pkg.packagePath; + const built = + existsSync(pkg.packagePath) && + statSync(pkg.packagePath).size > 16 && + existsSync(join(packageDir, "agentos-package.json")) && + existsSync(join(packageDir, "bin", name)); + if (!built) { + throw new Error( + `software package ${name} is NOT BUILT (no valid ${pkg.packagePath}).`, + ); + } + return pkg; +} function parseCurrentBranch(output: string): string { const branch = output @@ -45,65 +71,78 @@ function parseHeadRef(content: string): string { return headRef; } -describe("git quickstart integration", () => { +function gitCommand(args: string): string { + return `git ${GIT_CONFIG} ${args}`; +} - let vm: AgentOs; +describe("git quickstart integration", () => { + let vm: AgentOs; - beforeEach(async () => { - vm = await AgentOs.create({ - mounts: moduleAccessMounts(MODULE_ACCESS_CWD), - permissions: GIT_QUICKSTART_PERMISSIONS, - software: [COMMON_SOFTWARE, GIT_PACKAGE], - }); + beforeEach(async () => { + vm = await AgentOs.create({ + mounts: moduleAccessMounts(MODULE_ACCESS_CWD), + permissions: GIT_QUICKSTART_PERMISSIONS, + software: [COMMON_SOFTWARE, GIT_PACKAGE], }); + }); + + afterEach(async () => { + await vm?.dispose(); + }); + + async function run(command: string): Promise { + const result = await vm.exec(command); + if (result.exitCode !== 0) { + throw new Error( + `command failed: ${command}\n${result.stderr || result.stdout}`, + ); + } + return result; + } - afterEach(async () => { - await vm.dispose(); - }); + test("covers the quickstart local origin -> clone -> checkout flow", async () => { + await run(gitCommand("init /tmp/origin")); + await vm.writeFile("/tmp/origin/README.md", "# demo repo\n"); + await run(gitCommand("-C /tmp/origin add README.md")); + await run(gitCommand("-C /tmp/origin commit -m 'initial commit'")); - async function run(command: string): Promise { - const result = await vm.exec(command); - if (result.exitCode !== 0) { - throw new Error( - `command failed: ${command}\n${result.stderr || result.stdout}`, - ); - } - return result; - } + const defaultBranch = parseCurrentBranch( + (await run(gitCommand("-C /tmp/origin branch"))).stdout, + ); + + await run(gitCommand("-C /tmp/origin checkout -b feature")); + await vm.writeFile("/tmp/origin/feature.txt", "checked out from feature\n"); + await run(gitCommand("-C /tmp/origin add feature.txt")); + await run(gitCommand("-C /tmp/origin commit -m 'add feature file'")); + + await run(gitCommand("clone /tmp/origin /tmp/clone")); + + const cloneHead = new TextDecoder().decode( + await vm.readFile("/tmp/clone/.git/HEAD"), + ); + expect(parseHeadRef(cloneHead)).toBe("feature"); + expect(defaultBranch).not.toBe("feature"); - test( - "covers the quickstart local origin -> clone -> checkout flow", - async () => { - await run("git init /tmp/origin"); - await vm.writeFile("/tmp/origin/README.md", "# demo repo\n"); - await run("git -C /tmp/origin add README.md"); - await run("git -C /tmp/origin commit -m 'initial commit'"); - - const defaultBranch = parseCurrentBranch( - (await run("git -C /tmp/origin branch")).stdout, - ); - - await run("git -C /tmp/origin checkout -b feature"); - await vm.writeFile("/tmp/origin/feature.txt", "checked out from feature\n"); - await run("git -C /tmp/origin add feature.txt"); - await run("git -C /tmp/origin commit -m 'add feature file'"); - - await run("git clone /tmp/origin /tmp/clone"); - - const cloneHead = new TextDecoder().decode( - await vm.readFile("/tmp/clone/.git/HEAD"), - ); - expect(parseHeadRef(cloneHead)).toBe("feature"); - expect(defaultBranch).not.toBe("feature"); - - const featureFile = await vm.readFile("/tmp/clone/feature.txt"); - expect(new TextDecoder().decode(featureFile)).toBe( - "checked out from feature\n", - ); - - const readme = await vm.readFile("/tmp/clone/README.md"); - expect(new TextDecoder().decode(readme)).toBe("# demo repo\n"); - }, - 120_000, + const featureFile = await vm.readFile("/tmp/clone/feature.txt"); + expect(new TextDecoder().decode(featureFile)).toBe( + "checked out from feature\n", ); + + const readme = await vm.readFile("/tmp/clone/README.md"); + expect(new TextDecoder().decode(readme)).toBe("# demo repo\n"); + + const currentBranch = ( + await run(gitCommand("-C /tmp/clone branch --show-current")) + ).stdout.trim(); + expect(currentBranch).toBe("feature"); + + const log = await run(gitCommand("-C /tmp/clone log --oneline --all")); + expect(log.stdout).toContain("add feature file"); + expect(log.stdout).toContain("initial commit"); + + await vm.writeFile("/tmp/clone/README.md", "# changed demo repo\n"); + const diff = await run(gitCommand("-C /tmp/clone diff -- README.md")); + expect(diff.stdout).toContain("-# demo repo"); + expect(diff.stdout).toContain("+# changed demo repo"); + }, 120_000); }); diff --git a/packages/core/tests/helpers/fixture-node-modules.ts b/packages/core/tests/helpers/fixture-node-modules.ts index 094914e966..f73ef24728 100644 --- a/packages/core/tests/helpers/fixture-node-modules.ts +++ b/packages/core/tests/helpers/fixture-node-modules.ts @@ -13,6 +13,7 @@ import { unlinkSync, writeFileSync, } from "node:fs"; +import { tmpdir } from "node:os"; import { dirname, join, resolve, sep } from "node:path"; /** @@ -129,12 +130,11 @@ export function ensureFlatNodeModules(cwd: string): string { const repoRoot = findRepoRoot(cwd); const safe = packageName.replace(/[^a-z0-9]+/gi, "_"); - const cacheRoot = join( - repoRoot, - "node_modules", - ".cache", - "agentos-flat-fixtures", - ); + const repoSafe = repoRoot.replace(/[^a-z0-9]+/gi, "_"); + const cacheBase = + process.env.AGENTOS_FLAT_FIXTURE_CACHE_ROOT ?? + join(tmpdir(), "agentos-flat-fixtures"); + const cacheRoot = join(cacheBase, repoSafe); mkdirSync(cacheRoot, { recursive: true }); const target = join(cacheRoot, safe); const readyMarker = join(target, ".ready"); diff --git a/packages/core/tests/helpers/openai-responses-mock.ts b/packages/core/tests/helpers/openai-responses-mock.ts index e771f886b1..dcbbff550c 100644 --- a/packages/core/tests/helpers/openai-responses-mock.ts +++ b/packages/core/tests/helpers/openai-responses-mock.ts @@ -43,6 +43,52 @@ function writeJson( res.end(payload); } +function writeSse( + res: ServerResponse, + events: Record[], +): void { + const payload = events + .map((event) => { + const type = String(event.type); + return `event: ${type}\ndata: ${JSON.stringify(event)}\n\n`; + }) + .join(""); + res.statusCode = 200; + res.setHeader("content-type", "text/event-stream"); + res.setHeader("cache-control", "no-cache"); + res.setHeader("content-length", Buffer.byteLength(payload)); + res.end(payload); +} + +function responseToSseEvents(response: Record) { + const id = typeof response.id === "string" ? response.id : "resp_mock"; + const events: Record[] = [ + { type: "response.created", response: { id } }, + ]; + const output = Array.isArray(response.output) ? response.output : []; + for (const item of output) { + if (!item || typeof item !== "object") continue; + events.push({ + type: "response.output_item.done", + item, + }); + } + events.push({ + type: "response.completed", + response: { + id, + usage: { + input_tokens: 0, + input_tokens_details: null, + output_tokens: 0, + output_tokens_details: null, + total_tokens: 0, + }, + }, + }); + return events; +} + export async function startResponsesMock( fixtures: ResponsesFixture[], ): Promise { @@ -69,7 +115,11 @@ export async function startResponsesMock( if (fixture.delayMs) { await new Promise((resolve) => setTimeout(resolve, fixture.delayMs)); } - writeJson(res, 200, fixture.response); + if (body.stream === true) { + writeSse(res, responseToSseEvents(fixture.response)); + } else { + writeJson(res, 200, fixture.response); + } } catch (error) { writeJson(res, 500, { error: "invalid_request", diff --git a/packages/core/tests/helpers/registry-command-availability.ts b/packages/core/tests/helpers/registry-command-availability.ts new file mode 100644 index 0000000000..309df23375 --- /dev/null +++ b/packages/core/tests/helpers/registry-command-availability.ts @@ -0,0 +1,58 @@ +import { + closeSync, + existsSync, + openSync, + readdirSync, + readSync, + statSync, +} from "node:fs"; +import { dirname, join } from "node:path"; + +interface RegistryPackageRef { + packagePath: string; +} + +const AOSPKG_MAGIC = Buffer.from([0x89, 0x41, 0x4f, 0x53]); + +function isPackedAospkg(path: string): boolean { + try { + if (statSync(path).size <= 16) return false; + const fd = openSync(path, "r"); + try { + const head = Buffer.alloc(4); + readSync(fd, head, 0, 4, 0); + return head.equals(AOSPKG_MAGIC); + } finally { + closeSync(fd); + } + } catch { + return false; + } +} + +export function hasBuiltRegistryCommands( + packages: readonly RegistryPackageRef[], +): boolean { + return packages.every((pkg) => { + const path = pkg.packagePath; + if (path.endsWith(".aospkg")) { + if (!isPackedAospkg(path)) return false; + return hasCompleteCommandDir(join(dirname(path), "package")); + } + return ( + existsSync(join(path, "agentos-package.json")) && + hasCompleteCommandDir(path) + ); + }); +} + +function hasCompleteCommandDir(path: string): boolean { + if (!existsSync(path)) return true; + const binDir = join(path, "bin"); + if (!existsSync(binDir)) return false; + const entries = readdirSync(binDir); + return ( + entries.length > 0 && + entries.every((entry) => existsSync(join(binDir, entry))) + ); +} diff --git a/packages/core/tests/helpers/registry-commands.ts b/packages/core/tests/helpers/registry-commands.ts index 1d2dfdeb67..2a0b781c5e 100644 --- a/packages/core/tests/helpers/registry-commands.ts +++ b/packages/core/tests/helpers/registry-commands.ts @@ -12,8 +12,8 @@ * when it exists (local registry builds produce both), so a stale or empty * command set still fails loudly. * - * The only sanctioned exception is the C-sysroot package set (duckdb, - * http-get, sqlite3, wget, zip, unzip): those need the patched wasi C sysroot + * The only sanctioned exception is the C-sysroot package set (curl, duckdb, + * sqlite3, zip, unzip): those need the patched wasi C sysroot * that most checkouts don't have, so `cSysrootPackageSkipReason` reports a * skip reason instead of throwing. Everything else is load-or-throw. */ @@ -160,8 +160,8 @@ export function requireBuilt( } /** - * Skip reason for the C-sysroot package set ONLY (duckdb, http-get, sqlite3, - * wget, zip, unzip). These need the patched wasi C sysroot + * Skip reason for the C-sysroot package set ONLY (curl, duckdb, sqlite3, + * zip, unzip). These need the patched wasi C sysroot * (`make -C toolchain/c`), which most checkouts don't build — a missing * artifact is an environment limitation, not a forgotten build, so suites may * skip with this reason instead of throwing. diff --git a/packages/core/tests/opencode-real-session.test.ts b/packages/core/tests/opencode-real-session.test.ts new file mode 100644 index 0000000000..c47f4b2cd6 --- /dev/null +++ b/packages/core/tests/opencode-real-session.test.ts @@ -0,0 +1,76 @@ +import { resolve } from "node:path"; +import opencode from "@agentos-software/opencode"; +import { describe, expect, test } from "vitest"; +import type { AgentCapabilities, AgentInfo } from "../src/agent-os.js"; +import { AgentOs } from "../src/agent-os.js"; +import { + DEFAULT_TEXT_FIXTURE, + startLlmock, + stopLlmock, +} from "./helpers/llmock-helper.js"; +import { moduleAccessMounts } from "./helpers/node-modules-mount.js"; +import { + createVmOpenCodeHome, + createVmWorkspace, +} from "./helpers/opencode-helper.js"; + +const MODULE_ACCESS_CWD = resolve(import.meta.dirname, ".."); + +async function createOpenCodeVm(mockUrl: string): Promise { + return AgentOs.create({ + loopbackExemptPorts: [Number(new URL(mockUrl).port)], + mounts: moduleAccessMounts(MODULE_ACCESS_CWD), + software: [opencode], + }); +} + +describe("real createSession('opencode')", () => { + test("initializes the projected OpenCode ACP package inside the VM", async () => { + const { mock, url } = await startLlmock([DEFAULT_TEXT_FIXTURE]); + const vm = await createOpenCodeVm(url); + + let sessionId: string | undefined; + try { + const homeDir = await createVmOpenCodeHome(vm, url); + const workspaceDir = await createVmWorkspace(vm); + sessionId = ( + await vm.createSession("opencode", { + cwd: workspaceDir, + env: { + HOME: homeDir, + ANTHROPIC_API_KEY: "mock-key", + }, + }) + ).sessionId; + + const agentInfo = vm.getSessionAgentInfo(sessionId) as AgentInfo; + expect(agentInfo.name).toBe("OpenCode"); + expect(agentInfo.version).toBeTruthy(); + + const capabilities = vm.getSessionCapabilities( + sessionId, + ) as AgentCapabilities; + expect(capabilities.promptCapabilities).toMatchObject({ + embeddedContext: true, + image: true, + }); + + const modes = vm.getSessionModes(sessionId); + expect(modes?.currentModeId).toBe("build"); + expect(modes?.availableModes.map((mode) => mode.id)).toEqual( + expect.arrayContaining(["build", "plan"]), + ); + + expect(vm.listSessions()).toContainEqual({ + sessionId, + agentType: "opencode", + }); + } finally { + if (sessionId) { + vm.closeSession(sessionId); + } + await vm.dispose(); + await stopLlmock(mock); + } + }, 120_000); +}); diff --git a/packages/core/tests/pi-cli-headless.test.ts b/packages/core/tests/pi-cli-headless.test.ts index 5f639a3498..f48c45c19c 100644 --- a/packages/core/tests/pi-cli-headless.test.ts +++ b/packages/core/tests/pi-cli-headless.test.ts @@ -1,8 +1,7 @@ import { resolve } from "node:path"; -import type { Fixture, ToolCall } from "@copilotkit/llmock"; -import { moduleAccessMounts } from "./helpers/node-modules-mount.js"; import common from "@agentos-software/common"; import piCli from "@agentos-software/pi-cli"; +import type { Fixture, ToolCall } from "@copilotkit/llmock"; import { describe, expect, test } from "vitest"; import { AgentOs } from "../src/agent-os.js"; import { @@ -10,8 +9,12 @@ import { startLlmock, stopLlmock, } from "./helpers/llmock-helper.js"; +import { moduleAccessMounts } from "./helpers/node-modules-mount.js"; +import { hasBuiltRegistryCommands } from "./helpers/registry-command-availability.js"; const MODULE_ACCESS_CWD = resolve(import.meta.dirname, ".."); +const registryCommandsAvailable = hasBuiltRegistryCommands(common); +const registryCommandTest = registryCommandsAvailable ? test : test.skip; function getRequestBody(req: unknown): Record { const direct = req as Record; @@ -49,7 +52,7 @@ async function createPiCliVm(mockUrl: string): Promise { return AgentOs.create({ loopbackExemptPorts: [Number(new URL(mockUrl).port)], mounts: moduleAccessMounts(MODULE_ACCESS_CWD), - software: [common, piCli], + software: [...(registryCommandsAvailable ? common : []), piCli], }); } @@ -152,61 +155,58 @@ describe("full createSession('pi-cli') inside the VM", () => { } }, 120_000); - // Blocked on shell `>` redirect output being visible to `vm.readFile()`. - // This is the unmodified upstream Pi CLI bash path (`createLocalBashOperations` - // spawning the shell directly), with no Agent OS operations override, so the - // failure is a runtime gap independent of the SDK adapter: the redirect runs - // inside the guest shell but the written bytes do not reconcile to the host - // read path yet. Tracked in ~/.agents/todo/agentos-runtime-fixes.md - // (shell-exec redirect visibility). - test.skip("runs the unmodified Pi CLI ACP flow end-to-end for bash tool calls", async () => { - const workspacePath = "/home/agentos/workspace/bash-output.txt"; - const fixtures = createToolFixtures( - { - name: "bash", - arguments: JSON.stringify({ - command: `printf 'bash-ok' > ${workspacePath}`, - timeout: 10, - }), - }, - "bash-ok", - "bash-output.txt was written successfully.", - ); - const { mock, url } = await startLlmock(fixtures); - const vm = await createPiCliVm(url); + registryCommandTest( + "runs the unmodified Pi CLI ACP flow end-to-end for bash tool calls", + async () => { + const workspacePath = "/home/agentos/workspace/bash-output.txt"; + const fixtures = createToolFixtures( + { + name: "bash", + arguments: JSON.stringify({ + command: `printf 'bash-ok' > ${workspacePath}`, + timeout: 10, + }), + }, + "bash-ok", + "bash-output.txt was written successfully.", + ); + const { mock, url } = await startLlmock(fixtures); + const vm = await createPiCliVm(url); - let sessionId: string | undefined; - try { - const homeDir = await createVmPiHome(vm, url); - const workspaceDir = await createVmWorkspace(vm); - sessionId = ( - await vm.createSession("pi-cli", { - cwd: workspaceDir, - env: { - HOME: homeDir, - ANTHROPIC_API_KEY: "mock-key", - ANTHROPIC_BASE_URL: url, - }, - }) - ).sessionId; + let sessionId: string | undefined; + try { + const homeDir = await createVmPiHome(vm, url); + const workspaceDir = await createVmWorkspace(vm); + sessionId = ( + await vm.createSession("pi-cli", { + cwd: workspaceDir, + env: { + HOME: homeDir, + ANTHROPIC_API_KEY: "mock-key", + ANTHROPIC_BASE_URL: url, + }, + }) + ).sessionId; - const { response, text } = await vm.prompt( - sessionId, - `Use bash to write bash-ok into ${workspacePath}.`, - ); + const { response, text } = await vm.prompt( + sessionId, + `Use bash to write bash-ok into ${workspacePath}.`, + ); - expect(response.error).toBeUndefined(); - expect(text).toContain("bash-output.txt was written successfully."); - expect(new TextDecoder().decode(await vm.readFile(workspacePath))).toBe( - "bash-ok", - ); - expect(mock.getRequests().length).toBeGreaterThanOrEqual(2); - } finally { - if (sessionId) { - vm.closeSession(sessionId); + expect(response.error).toBeUndefined(); + expect(text).toContain("bash-output.txt was written successfully."); + expect(new TextDecoder().decode(await vm.readFile(workspacePath))).toBe( + "bash-ok", + ); + expect(mock.getRequests().length).toBeGreaterThanOrEqual(2); + } finally { + if (sessionId) { + vm.closeSession(sessionId); + } + await vm.dispose(); + await stopLlmock(mock); } - await vm.dispose(); - await stopLlmock(mock); - } - }, 120_000); + }, + 120_000, + ); }); diff --git a/packages/core/tests/pi-headless.test.ts b/packages/core/tests/pi-headless.test.ts index b1cd922223..294dffe7e1 100644 --- a/packages/core/tests/pi-headless.test.ts +++ b/packages/core/tests/pi-headless.test.ts @@ -1,8 +1,7 @@ import { resolve } from "node:path"; -import type { Fixture, ToolCall } from "@copilotkit/llmock"; -import { moduleAccessMounts } from "./helpers/node-modules-mount.js"; import common from "@agentos-software/common"; import pi from "@agentos-software/pi"; +import type { Fixture, ToolCall } from "@copilotkit/llmock"; import { describe, expect, test } from "vitest"; import type { AgentCapabilities, AgentInfo } from "../src/agent-os.js"; import { AgentOs } from "../src/agent-os.js"; @@ -11,8 +10,12 @@ import { startLlmock, stopLlmock, } from "./helpers/llmock-helper.js"; +import { moduleAccessMounts } from "./helpers/node-modules-mount.js"; +import { hasBuiltRegistryCommands } from "./helpers/registry-command-availability.js"; const MODULE_ACCESS_CWD = resolve(import.meta.dirname, ".."); +const registryCommandsAvailable = hasBuiltRegistryCommands(common); +const registryCommandTest = registryCommandsAvailable ? test : test.skip; function getRequestBody(req: unknown): Record { const direct = req as Record; @@ -51,7 +54,7 @@ async function createPiVm(mockUrl: string): Promise { loopbackExemptPorts: [Number(new URL(mockUrl).port)], mounts: moduleAccessMounts(MODULE_ACCESS_CWD), // Default software ships no agents; pass the pi agent package explicitly. - software: [common, pi], + software: [...(registryCommandsAvailable ? common : []), pi], }); } @@ -206,63 +209,59 @@ describe("full createSession('pi') inside the VM", () => { } }, 120_000); - // Blocked on shell `>` redirect output being visible to `vm.readFile()`. - // The vanilla Pi SDK bash backend spawns the shell directly and the redirect - // runs inside the guest shell, but the written bytes do not reconcile to the - // host read path yet. Before the adapter dropped its custom bash operations - // override this case passed because the override routed the command through - // the rpc-client `sh -c` path that the host can observe; the vanilla backend - // surfaces the underlying runtime gap. Tracked in - // ~/.agents/todo/agentos-runtime-fixes.md (shell-exec redirect visibility). - test.skip("runs the real Pi SDK ACP flow end-to-end for bash tool calls", async () => { - const fixtures = createToolFixtures( - { - name: "bash", - arguments: JSON.stringify({ - command: "printf 'bash-ok' > bash-output.txt", - timeout: 10, - }), - }, - "bash-ok", - "bash-output.txt was written successfully.", - ); - const { mock, url } = await startLlmock(fixtures); - const vm = await createPiVm(url); + registryCommandTest( + "runs the real Pi SDK ACP flow end-to-end for bash tool calls", + async () => { + const fixtures = createToolFixtures( + { + name: "bash", + arguments: JSON.stringify({ + command: "printf 'bash-ok' > bash-output.txt", + timeout: 10, + }), + }, + "bash-ok", + "bash-output.txt was written successfully.", + ); + const { mock, url } = await startLlmock(fixtures); + const vm = await createPiVm(url); - let sessionId: string | undefined; - try { - const homeDir = await createVmPiHome(vm, url); - const workspaceDir = await createVmWorkspace(vm); - sessionId = ( - await vm.createSession("pi", { - cwd: workspaceDir, - env: { - HOME: homeDir, - ANTHROPIC_API_KEY: "mock-key", - ANTHROPIC_BASE_URL: url, - }, - }) - ).sessionId; + let sessionId: string | undefined; + try { + const homeDir = await createVmPiHome(vm, url); + const workspaceDir = await createVmWorkspace(vm); + sessionId = ( + await vm.createSession("pi", { + cwd: workspaceDir, + env: { + HOME: homeDir, + ANTHROPIC_API_KEY: "mock-key", + ANTHROPIC_BASE_URL: url, + }, + }) + ).sessionId; - const { response, text } = await vm.prompt( - sessionId, - "Use bash to write bash-ok into bash-output.txt.", - ); + const { response, text } = await vm.prompt( + sessionId, + "Use bash to write bash-ok into bash-output.txt.", + ); - expect(response.error).toBeUndefined(); - expect(text).toContain("bash-output.txt was written successfully."); - expect( - new TextDecoder().decode( - await vm.readFile(`${workspaceDir}/bash-output.txt`), - ), - ).toBe("bash-ok"); - expect(mock.getRequests().length).toBeGreaterThanOrEqual(2); - } finally { - if (sessionId) { - vm.closeSession(sessionId); + expect(response.error).toBeUndefined(); + expect(text).toContain("bash-output.txt was written successfully."); + expect( + new TextDecoder().decode( + await vm.readFile(`${workspaceDir}/bash-output.txt`), + ), + ).toBe("bash-ok"); + expect(mock.getRequests().length).toBeGreaterThanOrEqual(2); + } finally { + if (sessionId) { + vm.closeSession(sessionId); + } + await vm.dispose(); + await stopLlmock(mock); } - await vm.dispose(); - await stopLlmock(mock); - } - }, 120_000); + }, + 120_000, + ); }); diff --git a/packages/core/vitest.config.ts b/packages/core/vitest.config.ts index 0016766828..c5c20f45fc 100644 --- a/packages/core/vitest.config.ts +++ b/packages/core/vitest.config.ts @@ -10,18 +10,15 @@ import { configDefaults, defineConfig } from "vitest/config"; const SLOW_E2E_FILES = [ "tests/wasm-commands.test.ts", // ~24m "tests/session-cleanup.test.ts", // ~12m - "tests/claude-session.test.ts", // ~7.5m "tests/execute.test.ts", "tests/filesystem.test.ts", "tests/native-sidecar-process.test.ts", "tests/pi-vanilla-bash.test.ts", "tests/opencode-session.test.ts", - "tests/git-quickstart.test.ts", "tests/filesystem-move-delete.test.ts", "tests/batch-file-ops.test.ts", "tests/agentos-base-filesystem.test.ts", "tests/pi-sdk-boot-probe.test.ts", - "tests/pi-headless.test.ts", "tests/pi-tool-llmock.test.ts", "tests/native-sidecar-process-permissions.test.ts", "tests/pi-extensions.test.ts", @@ -29,7 +26,6 @@ const SLOW_E2E_FILES = [ "tests/child-process-detached.test.ts", "tests/readdir-recursive.test.ts", "tests/cron-integration.test.ts", - "tests/pi-cli-headless.test.ts", ]; // Pre-existing failures NOT caused by this branch (they were red before CI ever @@ -50,15 +46,8 @@ const KNOWN_FAILING_E2E_FILES = [ "tests/pi-acp-adapter.test.ts", "tests/process-lifecycle.test.ts", // Registry-artifact / shell-behavior failures (red in both CI and local): - // - duckdb-package: imports secure-exec software/duckdb/dist (unbuilt WASM in CI). // - shell-flat-api: openShell/writeShell/onShellData yields empty output. - "tests/duckdb-package.test.ts", "tests/shell-flat-api.test.ts", - // codex-fullturn: the pinned @agentos-software/codex package intentionally - // stubs the turn ("codex-exec --session-turn is disabled until the real Codex - // agent package is wired"). Pre-existing unwired-feature state, not a - // regression — re-enable once the real Codex agent package is wired. - "tests/codex-fullturn.test.ts", ]; // Real-API, real-install matrix (agent × package manager). Hits a live LLM API diff --git a/packages/runtime-core/commands/egrep b/packages/runtime-core/commands/egrep index c78e121f14..6ab0536c2a 100644 Binary files a/packages/runtime-core/commands/egrep and b/packages/runtime-core/commands/egrep differ diff --git a/packages/runtime-core/commands/fd b/packages/runtime-core/commands/fd index b724ec24b1..137bb9d133 100644 Binary files a/packages/runtime-core/commands/fd and b/packages/runtime-core/commands/fd differ diff --git a/packages/runtime-core/commands/fgrep b/packages/runtime-core/commands/fgrep index c78e121f14..01aa3dea12 100644 Binary files a/packages/runtime-core/commands/fgrep and b/packages/runtime-core/commands/fgrep differ diff --git a/packages/runtime-core/commands/git b/packages/runtime-core/commands/git index 1865311237..869f2fae78 100644 Binary files a/packages/runtime-core/commands/git and b/packages/runtime-core/commands/git differ diff --git a/packages/runtime-core/commands/git-receive-pack b/packages/runtime-core/commands/git-receive-pack new file mode 100644 index 0000000000..869f2fae78 Binary files /dev/null and b/packages/runtime-core/commands/git-receive-pack differ diff --git a/packages/runtime-core/commands/git-remote-http b/packages/runtime-core/commands/git-remote-http index 1865311237..869f2fae78 100644 Binary files a/packages/runtime-core/commands/git-remote-http and b/packages/runtime-core/commands/git-remote-http differ diff --git a/packages/runtime-core/commands/git-remote-https b/packages/runtime-core/commands/git-remote-https index 1865311237..869f2fae78 100644 Binary files a/packages/runtime-core/commands/git-remote-https and b/packages/runtime-core/commands/git-remote-https differ diff --git a/packages/runtime-core/commands/git-upload-archive b/packages/runtime-core/commands/git-upload-archive new file mode 100644 index 0000000000..869f2fae78 Binary files /dev/null and b/packages/runtime-core/commands/git-upload-archive differ diff --git a/packages/runtime-core/commands/git-upload-pack b/packages/runtime-core/commands/git-upload-pack new file mode 100644 index 0000000000..869f2fae78 Binary files /dev/null and b/packages/runtime-core/commands/git-upload-pack differ diff --git a/packages/runtime-core/commands/grep b/packages/runtime-core/commands/grep index c78e121f14..53f5049c16 100644 Binary files a/packages/runtime-core/commands/grep and b/packages/runtime-core/commands/grep differ diff --git a/packages/runtime-core/commands/http_get b/packages/runtime-core/commands/http_get deleted file mode 100644 index 52a1fb8342..0000000000 Binary files a/packages/runtime-core/commands/http_get and /dev/null differ diff --git a/packages/runtime-core/commands/rg b/packages/runtime-core/commands/rg index 76a63b75ea..d3d4306565 100644 Binary files a/packages/runtime-core/commands/rg and b/packages/runtime-core/commands/rg differ diff --git a/packages/runtime-core/commands/sqlite3 b/packages/runtime-core/commands/sqlite3 index a9ee1f206f..3dbc0a6bb6 100755 Binary files a/packages/runtime-core/commands/sqlite3 and b/packages/runtime-core/commands/sqlite3 differ diff --git a/packages/runtime-core/commands/tree b/packages/runtime-core/commands/tree old mode 100644 new mode 100755 index 49e9c7c18d..811c63e5fd Binary files a/packages/runtime-core/commands/tree and b/packages/runtime-core/commands/tree differ diff --git a/packages/runtime-core/commands/unzip b/packages/runtime-core/commands/unzip old mode 100644 new mode 100755 index 07f40041e1..3bc6f3f020 Binary files a/packages/runtime-core/commands/unzip and b/packages/runtime-core/commands/unzip differ diff --git a/packages/runtime-core/commands/zip b/packages/runtime-core/commands/zip old mode 100644 new mode 100755 index 8324bdc9fe..edbd00f1c3 Binary files a/packages/runtime-core/commands/zip and b/packages/runtime-core/commands/zip differ diff --git a/packages/runtime-core/src/test-runtime.ts b/packages/runtime-core/src/test-runtime.ts index 16ae3e40cb..8ca761958c 100644 --- a/packages/runtime-core/src/test-runtime.ts +++ b/packages/runtime-core/src/test-runtime.ts @@ -1587,7 +1587,6 @@ export const WASMVM_COMMANDS = Object.freeze([ "unzip", "sqlite3", "curl", - "wget", "git", "git-remote-http", "git-remote-https", @@ -1756,7 +1755,6 @@ export const DEFAULT_FIRST_PARTY_TIERS: Readonly< tac: "read-only", tsort: "read-only", curl: "full", - wget: "full", sqlite3: "read-write", }); diff --git a/packages/runtime-core/tests/integration/cross-runtime-network.test.ts b/packages/runtime-core/tests/integration/cross-runtime-network.test.ts index 5167ea3bad..9336b2320c 100644 --- a/packages/runtime-core/tests/integration/cross-runtime-network.test.ts +++ b/packages/runtime-core/tests/integration/cross-runtime-network.test.ts @@ -19,7 +19,7 @@ import { } from '@rivet-dev/agentos-vm-test-harness'; import type { IntegrationKernelResult, Kernel } from '@rivet-dev/agentos-vm-test-harness'; -const WASM_HTTP_GET = resolve(C_BUILD_DIR, 'http_get'); +const WASM_CURL = resolve(C_BUILD_DIR, 'curl'); const WASM_HTTP_SERVER = resolve(C_BUILD_DIR, 'http_server'); const WASM_TCP_ECHO = resolve(C_BUILD_DIR, 'tcp_echo'); const WASM_TCP_SERVER = resolve(C_BUILD_DIR, 'tcp_server'); @@ -28,7 +28,7 @@ function skipReasonWasmNetwork(): string | false { const wasmSkipReason = skipUnlessWasmBuilt(); if (wasmSkipReason) return wasmSkipReason; for (const [name, path] of [ - ['http_get', WASM_HTTP_GET], + ['curl', WASM_CURL], ['http_server', WASM_HTTP_SERVER], ['tcp_echo', WASM_TCP_ECHO], ['tcp_server', WASM_TCP_SERVER], @@ -231,7 +231,7 @@ describe('cross-runtime network integration', { timeout: 90_000 }, () => { expect(client.stdout.trim()).toBe('js-pong:ping'); }); - itIf(!wasmNetworkSkipReason, 'W1 WASM http_get -> JS node:http server over VM loopback', async () => { + itIf(!wasmNetworkSkipReason, 'W1 WASM curl -> JS node:http server over VM loopback', async () => { ctx = await createIntegrationKernel({ runtimes: ['wasmvm', 'node'], commandDirs: [C_BUILD_DIR, COMMANDS_DIR], @@ -239,13 +239,13 @@ describe('cross-runtime network integration', { timeout: 90_000 }, () => { const server = spawnGuestNodeProgram(ctx.kernel, guestJsHttpServer(3102)); await waitForOutput(server, 'js http listening 3102', 'JS HTTP server'); - const wasm = await ctx.kernel.exec('http_get 3102 /from-wasm'); + const wasm = await ctx.kernel.exec('curl -fsS http://127.0.0.1:3102/from-wasm'); server.process.kill(15); await server.process.wait().catch(() => {}); expect(wasm.exitCode).toBe(0); - expect(wasm.stderr).not.toContain('socket error'); - expect(wasm.stdout).toContain('body: js:GET:/from-wasm'); + expect(wasm.stderr).toBe(''); + expect(wasm.stdout.trim()).toBe('js:GET:/from-wasm'); }); itIf(!wasmNetworkSkipReason, 'J3 JS fetch -> WASM HTTP server over VM loopback', async () => { @@ -341,7 +341,7 @@ describe('cross-runtime network integration', { timeout: 90_000 }, () => { expect(wasm.stdout).toContain('received: js-pong:hello'); }); - itIf(!wasmNetworkSkipReason, 'W3 WASM http_get -> WASM HTTP server over VM loopback', async () => { + itIf(!wasmNetworkSkipReason, 'W3 WASM curl -> WASM HTTP server over VM loopback', async () => { ctx = await createIntegrationKernel({ runtimes: ['wasmvm', 'node'], commandDirs: [C_BUILD_DIR, COMMANDS_DIR], @@ -349,12 +349,12 @@ describe('cross-runtime network integration', { timeout: 90_000 }, () => { const server = spawnGuestProgram(ctx.kernel, 'http_server', ['3108']); await waitForListener(ctx.kernel, 3108, 'WASM HTTP server'); - const wasm = await ctx.kernel.exec('http_get 3108 /from-wasm'); + const wasm = await ctx.kernel.exec('curl -fsS http://127.0.0.1:3108/from-wasm'); const serverExit = await server.process.wait(); expect(wasm.exitCode).toBe(0); - expect(wasm.stderr).not.toContain('socket error'); - expect(wasm.stdout).toContain('body: wasm:GET:/from-wasm'); + expect(wasm.stderr).toBe(''); + expect(wasm.stdout.trim()).toBe('wasm:GET:/from-wasm'); expect(serverExit).toBe(0); }); @@ -426,7 +426,7 @@ describe('cross-runtime network integration', { timeout: 90_000 }, () => { } }); - itIf(!wasmNetworkSkipReason, 'O2 WASM http_get -> host loopback requires loopback exemption', async () => { + itIf(!wasmNetworkSkipReason, 'O2 WASM curl -> host loopback requires loopback exemption', async () => { const seenRequests: string[] = []; const hostServer = createHttpServer((req, res) => { seenRequests.push(req.url ?? ''); @@ -443,9 +443,9 @@ describe('cross-runtime network integration', { timeout: 90_000 }, () => { runtimes: ['wasmvm', 'node'], commandDirs: [C_BUILD_DIR, COMMANDS_DIR], }); - const noExemption = await ctx.kernel.exec(`http_get ${port} /blocked`); + const noExemption = await ctx.kernel.exec(`curl -fsS http://127.0.0.1:${port}/blocked`); expect(noExemption.exitCode).not.toBe(0); - expect(noExemption.stderr).toMatch(/EACCES|Bad address|Connection refused|connect/); + expect(noExemption.stderr).toMatch(/EACCES|Bad address|Connection refused|connect|Failed to connect|Invalid argument/); expect(seenRequests).toEqual([]); await ctx.dispose(); @@ -454,10 +454,10 @@ describe('cross-runtime network integration', { timeout: 90_000 }, () => { commandDirs: [C_BUILD_DIR, COMMANDS_DIR], loopbackExemptPorts: [port], }); - const allowed = await ctx.kernel.exec(`http_get ${port} /allowed`); + const allowed = await ctx.kernel.exec(`curl -fsS http://127.0.0.1:${port}/allowed`); expect(allowed.exitCode).toBe(0); - expect(allowed.stderr).not.toContain('socket error'); - expect(allowed.stdout).toContain('body: host:/allowed'); + expect(allowed.stderr).toBe(''); + expect(allowed.stdout.trim()).toBe('host:/allowed'); expect(seenRequests).toEqual(['/allowed']); } finally { await new Promise((resolveClose) => hostServer.close(() => resolveClose())); diff --git a/packages/shell/package.json b/packages/shell/package.json index 254004e6b4..38631b38c9 100644 --- a/packages/shell/package.json +++ b/packages/shell/package.json @@ -26,7 +26,6 @@ "@agentos-software/git": "workspace:*", "@agentos-software/grep": "workspace:*", "@agentos-software/gzip": "workspace:*", - "@agentos-software/http-get": "workspace:*", "@agentos-software/jq": "workspace:*", "@agentos-software/ripgrep": "workspace:*", "@agentos-software/sed": "workspace:*", diff --git a/packages/shell/src/main.ts b/packages/shell/src/main.ts index f438278d19..d7abe702f4 100644 --- a/packages/shell/src/main.ts +++ b/packages/shell/src/main.ts @@ -35,7 +35,6 @@ import gawk from "@agentos-software/gawk"; import git from "@agentos-software/git"; import grep from "@agentos-software/grep"; import gzip from "@agentos-software/gzip"; -import httpGet from "@agentos-software/http-get"; import jq from "@agentos-software/jq"; import ripgrep from "@agentos-software/ripgrep"; import sed from "@agentos-software/sed"; @@ -44,6 +43,7 @@ import vim from "@agentos-software/vim"; import tar from "@agentos-software/tar"; import tree from "@agentos-software/tree"; import unzip from "@agentos-software/unzip"; +import wget from "@agentos-software/wget"; import yq from "@agentos-software/yq"; import zip from "@agentos-software/zip"; import type { MountConfig, SoftwareInput } from "@rivet-dev/agentos-core"; @@ -152,8 +152,8 @@ const software: SoftwareInput[] = [ yq, codex, git, - httpGet, sqlite3, + wget, ] .map(withLocalCommandFallback) .filter((input): input is SoftwareInput => input !== null); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 38e3dc4dc6..b7c1814c38 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2544,9 +2544,6 @@ importers: '@agentos-software/gzip': specifier: workspace:* version: link:../../software/gzip - '@agentos-software/http-get': - specifier: workspace:* - version: link:../../software/http-get '@agentos-software/jq': specifier: workspace:* version: link:../../software/jq @@ -2565,6 +2562,9 @@ importers: '@agentos-software/vim': specifier: workspace:* version: link:../../software/vim + '@agentos-software/wget': + specifier: workspace:* + version: link:../../software/wget '@agentos-software/yq': specifier: workspace:* version: link:../../software/yq @@ -2850,9 +2850,6 @@ importers: '@agentos-software/gzip': specifier: workspace:* version: link:../../software/gzip - '@agentos-software/http-get': - specifier: workspace:* - version: link:../../software/http-get '@agentos-software/jq': specifier: workspace:* version: link:../../software/jq @@ -3260,6 +3257,12 @@ importers: '@agentos-software/diffutils': specifier: workspace:* version: link:../diffutils + '@agentos-software/duckdb': + specifier: workspace:* + version: link:../duckdb + '@agentos-software/envsubst': + specifier: workspace:* + version: link:../envsubst '@agentos-software/fd': specifier: workspace:* version: link:../fd @@ -3272,6 +3275,9 @@ importers: '@agentos-software/gawk': specifier: workspace:* version: link:../gawk + '@agentos-software/git': + specifier: workspace:* + version: link:../git '@agentos-software/grep': specifier: workspace:* version: link:../grep @@ -3287,6 +3293,9 @@ importers: '@agentos-software/sed': specifier: workspace:* version: link:../sed + '@agentos-software/sqlite3': + specifier: workspace:* + version: link:../sqlite3 '@agentos-software/tar': specifier: workspace:* version: link:../tar @@ -3296,6 +3305,12 @@ importers: '@agentos-software/unzip': specifier: workspace:* version: link:../unzip + '@agentos-software/vim': + specifier: workspace:* + version: link:../vim + '@agentos-software/wget': + specifier: workspace:* + version: link:../wget '@agentos-software/yq': specifier: workspace:* version: link:../yq @@ -3466,27 +3481,6 @@ importers: specifier: ^2.1.9 version: 2.1.9(@types/node@22.19.15) - software/http-get: - devDependencies: - '@agentos-software/manifest': - specifier: workspace:* - version: link:../../packages/manifest - '@agentos/test-harness': - specifier: workspace:* - version: link:../../test-harness - '@rivet-dev/agentos-toolchain': - specifier: workspace:* - version: link:../../packages/agentos-toolchain - '@types/node': - specifier: ^22.10.2 - version: 22.19.15 - typescript: - specifier: ^5.9.2 - version: 5.9.3 - vitest: - specifier: ^2.1.9 - version: 2.1.9(@types/node@22.19.15) - software/jq: devDependencies: '@agentos-software/manifest': diff --git a/software/CONTRIBUTING.md b/software/CONTRIBUTING.md index 2ab561cf43..ac86bfa6c5 100644 --- a/software/CONTRIBUTING.md +++ b/software/CONTRIBUTING.md @@ -34,7 +34,7 @@ From the repo root: ```bash just toolchain-build # compile the fast native wasm command gate -just toolchain-cmd # build one command (required for git, duckdb, vim, wget, codex) +just toolchain-cmd # build one command (required for git, duckdb, vim, codex) just software-build # stage bin/ + assemble dist/package/ pnpm --filter './software/*' test ``` diff --git a/software/README.md b/software/README.md index 7f15771bb8..d104021d1e 100644 --- a/software/README.md +++ b/software/README.md @@ -77,7 +77,7 @@ The default native build (`toolchain`) compiles the fast command gate to `wasm-opt -O3`, and drops the binaries in `toolchain/target/wasm32-wasip1/release/commands/`. The bulk gate intentionally excludes slow/heavy or non-default commands: `git`, `duckdb`, -`vim`, `wget`, and the external `codex`/`codex-exec` fork build. Build those explicitly with +`vim` and the external `codex`/`codex-exec` fork build. Build those explicitly with `just toolchain-cmd ` when working on them. Package builds then run `agentos-toolchain stage` (with `--if-missing skip`, so a checkout without the native build still assembles valid empty placeholders) followed by `tsc` and diff --git a/software/diffutils/bin/diff b/software/diffutils/bin/diff index 003d074f99..ccaee62a89 100644 Binary files a/software/diffutils/bin/diff and b/software/diffutils/bin/diff differ diff --git a/software/diffutils/native/crates/diff/src/lib.rs b/software/diffutils/native/crates/diff/src/lib.rs index 04dbde97da..77e385a00e 100644 --- a/software/diffutils/native/crates/diff/src/lib.rs +++ b/software/diffutils/native/crates/diff/src/lib.rs @@ -272,6 +272,14 @@ fn diff_files(path_a: &Path, path_b: &Path, opts: &Options) -> Result { + if (!tempRoot) throw new Error("fixture root not initialized"); + const hostPath = join(tempRoot, path.replace(/^\/+/, "")); + await mkdir(dirname(hostPath), { recursive: true }); + await writeFile(hostPath, contents); +} + +async function createTestVFS(): Promise { + tempRoot = await mkdtemp(join(tmpdir(), "agentos-diff-")); + await writeFixture("/project/a.txt", "alpha\nbeta\ngamma\n"); + await writeFixture("/project/b.txt", "alpha\ndelta\ngamma\n"); + await writeFixture("/project/same.txt", "alpha\nbeta\ngamma\n"); + await writeFixture("/project/ws-a.txt", "Alpha\n\nbeta value\n"); + await writeFixture("/project/ws-b.txt", "alpha\n BETA VALUE\n"); + await writeFixture("/project/left/common.txt", "one\ntwo\nthree\n"); + await writeFixture("/project/right/common.txt", "one\n2\nthree\n"); + await writeFixture("/project/right/extra.txt", "only on right\n"); + return new NodeFileSystem({ root: tempRoot }); +} + +describeIf(hasDiffPackageBinary, "diff command", { timeout: 10_000 }, () => { + let kernel: Kernel | undefined; + + afterEach(async () => { + await kernel?.dispose(); + kernel = undefined; + if (tempRoot) { + await rm(tempRoot, { recursive: true, force: true }); + tempRoot = undefined; + } + }); + + async function mountFixture(): Promise { + const vfs = await createTestVFS(); + kernel = createKernel({ filesystem: vfs }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [DIFF_COMMAND_DIR] })); + } + + async function runDiff(args: string[]) { + if (!kernel) throw new Error("kernel not mounted"); + let stdout = ""; + let stderr = ""; + const proc = kernel.spawn("diff", args, { + onStdout: (chunk) => { + stdout += Buffer.from(chunk).toString("utf8"); + }, + onStderr: (chunk) => { + stderr += Buffer.from(chunk).toString("utf8"); + }, + }); + const exitCode = await proc.wait(); + await new Promise((resolve) => setTimeout(resolve, 0)); + return { stdout, stderr, exitCode }; + } + + it("returns zero for identical files", async () => { + await mountFixture(); + + const result = await runDiff(["/project/a.txt", "/project/same.txt"]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(result.stdout).toBe(""); + }); + + it("prints normal diffs for changed files", async () => { + await mountFixture(); + + const result = await runDiff(["/project/a.txt", "/project/b.txt"]); + expect(result.exitCode).toBe(1); + expect(result.stdout).toContain("2c2"); + expect(result.stdout).toContain("< beta"); + expect(result.stdout).toContain("> delta"); + }); + + it("prints unified diffs", async () => { + await mountFixture(); + + const result = await runDiff(["-u", "/project/a.txt", "/project/b.txt"]); + expect(result.exitCode).toBe(1); + expect(result.stdout).toContain("--- /project/a.txt"); + expect(result.stdout).toContain("+++ /project/b.txt"); + expect(result.stdout).toContain("-beta"); + expect(result.stdout).toContain("+delta"); + }); + + it("prints brief output with -q", async () => { + await mountFixture(); + + const result = await runDiff(["-q", "/project/a.txt", "/project/b.txt"]); + expect(result.exitCode).toBe(1); + expect(result.stdout.trim()).toBe( + "Files /project/a.txt and /project/b.txt differ", + ); + }); + + it("honors ignore-case, whitespace, and blank-line flags", async () => { + await mountFixture(); + + const result = await runDiff([ + "-i", + "-w", + "-B", + "/project/ws-a.txt", + "/project/ws-b.txt", + ]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(result.stdout).toBe(""); + }); + + it("compares directories recursively", async () => { + await mountFixture(); + + const result = await runDiff(["-r", "/project/left", "/project/right"]); + expect(result.exitCode).toBe(1); + expect(result.stdout).toContain("Only in /project/right: extra.txt"); + expect(result.stdout).toContain("/project/left/common.txt"); + expect(result.stdout).toContain("/project/right/common.txt"); + }); + + it("returns an error for missing inputs", async () => { + await mountFixture(); + + const result = await runDiff(["/project/a.txt", "/project/missing.txt"]); + expect(result.exitCode).toBe(2); + expect(result.stderr).toContain("/project/missing.txt"); + }); +}); diff --git a/software/duckdb/test/duckdb.test.ts b/software/duckdb/test/duckdb.test.ts index 178039b040..06349c73fb 100644 --- a/software/duckdb/test/duckdb.test.ts +++ b/software/duckdb/test/duckdb.test.ts @@ -33,7 +33,6 @@ import { setTimeout as sleep } from 'node:timers/promises'; const hasWasmDuckDB = existsSync(resolve(C_BUILD_DIR, 'duckdb')); const hasWasmCurl = (existsSync(resolve(COMMANDS_DIR, 'curl')) || existsSync(resolve(C_BUILD_DIR, 'curl'))); -const hasWasmHttpGet = existsSync(resolve(C_BUILD_DIR, 'http_get')); async function mountKernel( filesystem: ReturnType, @@ -52,7 +51,7 @@ async function mountKernel( commandDirs, permissions: { duckdb: 'read-write', - http_get: 'full', + curl: 'full', }, }) ); @@ -215,7 +214,7 @@ describeIf(hasWasmDuckDB, 'duckdb command', { timeout: 120_000 }, () => { }); itIf( - hasWasmCurl || hasWasmHttpGet, + hasWasmCurl, 'queries data fetched over the network through the shared VFS', async () => { const filesystem = createInMemoryFileSystem(); @@ -243,18 +242,10 @@ describeIf(hasWasmDuckDB, 'duckdb command', { timeout: 120_000 }, () => { loopbackExemptPorts: [address.port], }); - let result; - if (hasWasmHttpGet) { - result = await kernel.exec( - `http_get ${address.port} /remote.csv /tmp/remote.csv` - ); - expect(result.exitCode).toBe(0); - } else { - result = await kernel.exec( - `curl -fsS -o /tmp/remote.csv http://127.0.0.1:${address.port}/remote.csv` - ); - expect(result.exitCode).toBe(0); - } + let result = await kernel.exec( + `curl -fsS -o /tmp/remote.csv http://127.0.0.1:${address.port}/remote.csv` + ); + expect(result.exitCode).toBe(0); expect(await filesystem.readTextFile('/tmp/remote.csv')).toContain('city,value'); diff --git a/software/everything/agentos-package.json b/software/everything/agentos-package.json new file mode 100644 index 0000000000..6dc26907f3 --- /dev/null +++ b/software/everything/agentos-package.json @@ -0,0 +1,8 @@ +{ + "registry": { + "title": "Everything", + "description": "Meta-package: all available WASM command packages.", + "priority": 110, + "category": "meta" + } +} diff --git a/software/everything/package.json b/software/everything/package.json index c491f55094..95ec6ff1ea 100644 --- a/software/everything/package.json +++ b/software/everything/package.json @@ -32,6 +32,12 @@ "@agentos-software/tar": "workspace:*", "@agentos-software/gzip": "workspace:*", "@agentos-software/curl": "workspace:*", + "@agentos-software/wget": "workspace:*", + "@agentos-software/duckdb": "workspace:*", + "@agentos-software/envsubst": "workspace:*", + "@agentos-software/git": "workspace:*", + "@agentos-software/sqlite3": "workspace:*", + "@agentos-software/vim": "workspace:*", "@agentos-software/zip": "workspace:*", "@agentos-software/unzip": "workspace:*", "@agentos-software/jq": "workspace:*", diff --git a/software/everything/src/index.ts b/software/everything/src/index.ts index 58565c3337..c45ab24679 100644 --- a/software/everything/src/index.ts +++ b/software/everything/src/index.ts @@ -7,6 +7,12 @@ import diffutils from "@agentos-software/diffutils"; import tar from "@agentos-software/tar"; import gzip from "@agentos-software/gzip"; import curl from "@agentos-software/curl"; +import wget from "@agentos-software/wget"; +import duckdb from "@agentos-software/duckdb"; +import envsubst from "@agentos-software/envsubst"; +import git from "@agentos-software/git"; +import sqlite3 from "@agentos-software/sqlite3"; +import vim from "@agentos-software/vim"; import zip from "@agentos-software/zip"; import unzip from "@agentos-software/unzip"; import jq from "@agentos-software/jq"; @@ -27,6 +33,12 @@ const everything = [ tar, gzip, curl, + wget, + duckdb, + envsubst, + git, + sqlite3, + vim, zip, unzip, jq, @@ -49,6 +61,12 @@ export { tar, gzip, curl, + wget, + duckdb, + envsubst, + git, + sqlite3, + vim, zip, unzip, jq, diff --git a/software/everything/test/everything.test.ts b/software/everything/test/everything.test.ts new file mode 100644 index 0000000000..f9d1ea24af --- /dev/null +++ b/software/everything/test/everything.test.ts @@ -0,0 +1,81 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import everything, { + codex, + coreutils, + curl, + diffutils, + duckdb, + envsubst, + fd, + file, + findutils, + gawk, + git, + grep, + gzip, + jq, + ripgrep, + sed, + sqlite3, + tar, + tree, + unzip, + vim, + wget, + yq, + zip, +} from "../src/index.js"; + +const packageDir = new URL("..", import.meta.url).pathname; + +const expectedMembers = [ + coreutils, + sed, + grep, + gawk, + findutils, + diffutils, + tar, + gzip, + curl, + wget, + duckdb, + envsubst, + git, + sqlite3, + vim, + zip, + unzip, + jq, + ripgrep, + fd, + tree, + file, + yq, + codex, +]; + +describe("everything meta-package", () => { + it("has a registry manifest", () => { + const manifest = JSON.parse( + readFileSync(join(packageDir, "agentos-package.json"), "utf8"), + ); + + expect(manifest.registry).toMatchObject({ + title: "Everything", + category: "meta", + }); + }); + + it("exports every command package descriptor once", () => { + expect(everything).toEqual(expectedMembers); + expect(new Set(everything).size).toBe(everything.length); + for (const descriptor of everything) { + expect(descriptor).toEqual({ + packagePath: expect.stringMatching(/\/package\.aospkg$/), + }); + } + }); +}); diff --git a/software/fd/bin/fd b/software/fd/bin/fd index b724ec24b1..137bb9d133 100644 Binary files a/software/fd/bin/fd and b/software/fd/bin/fd differ diff --git a/software/fd/native/crates/cmd-fd/Cargo.toml b/software/fd/native/crates/cmd-fd/Cargo.toml index 419fd7f396..ef6223d123 100644 --- a/software/fd/native/crates/cmd-fd/Cargo.toml +++ b/software/fd/native/crates/cmd-fd/Cargo.toml @@ -5,10 +5,10 @@ version.workspace = true edition.workspace = true license.workspace = true description = "fd standalone binary for secure-exec VM" - -[[bin]] -name = "fd" -path = "src/main.rs" +publish = false [dependencies] -secureexec-fd = { path = "../fd" } +# fd-find is a bin-only crate. `toolchain/Makefile` builds its upstream `fd` +# binary directly; this local package exists only so `cmd/fd` is discoverable +# and the upstream package is present in Cargo's resolved/vendor set. +fd-find = { version = "10.4.2", default-features = false } diff --git a/software/fd/native/crates/cmd-fd/src/lib.rs b/software/fd/native/crates/cmd-fd/src/lib.rs new file mode 100644 index 0000000000..a372ecb5e7 --- /dev/null +++ b/software/fd/native/crates/cmd-fd/src/lib.rs @@ -0,0 +1 @@ +//! Build trigger for the upstream `fd-find` command package. diff --git a/software/fd/native/crates/fd/Cargo.toml b/software/fd/native/crates/fd/Cargo.toml deleted file mode 100644 index 47c01c9571..0000000000 --- a/software/fd/native/crates/fd/Cargo.toml +++ /dev/null @@ -1,10 +0,0 @@ -[package] -workspace = "../../../../../toolchain" -name = "secureexec-fd" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "fd-find compatible file finder for secure-exec standalone binaries" - -[dependencies] -regex = "1" diff --git a/software/fd/native/crates/fd/src/lib.rs b/software/fd/native/crates/fd/src/lib.rs deleted file mode 100644 index 8ab845214f..0000000000 --- a/software/fd/native/crates/fd/src/lib.rs +++ /dev/null @@ -1,353 +0,0 @@ -//! Minimal fd-find compatible file finder for WASM. -//! -//! Uses std::fs for directory traversal (no walkdir/rayon dependency) -//! and regex for pattern matching. Covers common fd patterns: -//! fd PATTERN, fd -e EXT, fd -t f/d, fd -H (hidden), fd -I (no-ignore). - -use std::collections::HashSet; -use std::ffi::OsString; -use std::fs; -use std::io::{self, Write}; -use std::path::{Path, PathBuf}; - -use regex::Regex; - -/// Entry point for fd command. -pub fn main(args: Vec) -> i32 { - let str_args: Vec = args - .iter() - .skip(1) // skip argv[0] - .map(|a| a.to_string_lossy().to_string()) - .collect(); - - match run(&str_args) { - Ok(code) => code, - Err(msg) => { - eprintln!("[fd error]: {}", msg); - 1 - } - } -} - -#[derive(Debug, Clone, Copy, PartialEq)] -enum TypeFilter { - File, - Directory, - Symlink, -} - -enum ParsedArgs { - Search(Options), - Help, - Version, -} - -struct Options { - pattern: Option, - extensions: Vec, - type_filter: Option, - max_depth: Option, - search_paths: Vec, - show_hidden: bool, - _case_insensitive: bool, - full_path: bool, - absolute_path: bool, -} - -fn run(args: &[String]) -> Result { - let parsed = parse_args(args)?; - let stdout = io::stdout(); - let mut out = stdout.lock(); - - let ParsedArgs::Search(opts) = parsed else { - match parsed { - ParsedArgs::Help => print_help(&mut out), - ParsedArgs::Version => writeln!(out, "fd 0.1.0 (secure-exec)"), - ParsedArgs::Search(_) => unreachable!(), - } - .map_err(|e| format!("failed to write output: {e}"))?; - out.flush() - .map_err(|e| format!("failed to write output: {e}"))?; - return Ok(0); - }; - - let mut found = false; - - for search_path in &opts.search_paths { - let base = PathBuf::from(search_path); - let mut visited_dirs = HashSet::new(); - walk( - &base, - search_path, - 0, - &opts, - &mut found, - &mut out, - &mut visited_dirs, - )?; - } - - out.flush() - .map_err(|e| format!("failed to write output: {e}"))?; - - Ok(if found { 0 } else { 1 }) -} - -fn walk( - full_path: &Path, - base_path: &str, - depth: usize, - opts: &Options, - found: &mut bool, - out: &mut W, - visited_dirs: &mut HashSet, -) -> Result<(), String> { - if let Some(max) = opts.max_depth { - if depth > max { - return Ok(()); - } - } - - // fd skips the root search directory itself (depth 0); only matches children - if depth > 0 { - if matches_entry(full_path, base_path, opts) { - *found = true; - let display = if opts.absolute_path { - match fs::canonicalize(full_path) { - Ok(abs) => abs.to_string_lossy().to_string(), - Err(_) => full_path.to_string_lossy().to_string(), - } - } else { - full_path.to_string_lossy().to_string() - }; - writeln!(out, "{}", display).map_err(|e| format!("failed to write output: {e}"))?; - } - } - - // Recurse into directories - if full_path.is_dir() { - let canonical_path = - fs::canonicalize(full_path).map_err(|e| format!("{}: {}", full_path.display(), e))?; - if !visited_dirs.insert(canonical_path) { - return Ok(()); - } - - if let Some(max) = opts.max_depth { - if depth >= max { - return Ok(()); - } - } - - let entries = - fs::read_dir(full_path).map_err(|e| format!("{}: {}", full_path.display(), e))?; - - let mut sorted: Vec<_> = entries - .collect::, _>>() - .map_err(|e| format!("{}: {}", full_path.display(), e))?; - sorted.sort_by_key(|e| e.file_name()); - - for entry in sorted { - let name = entry.file_name().to_string_lossy().to_string(); - - // Skip hidden files/directories unless -H is set - if !opts.show_hidden && name.starts_with('.') { - continue; - } - - let child = entry.path(); - walk(&child, base_path, depth + 1, opts, found, out, visited_dirs)?; - } - } - - Ok(()) -} - -fn matches_entry(path: &Path, _base_path: &str, opts: &Options) -> bool { - // Type filter - if let Some(ref tf) = opts.type_filter { - let ok = match tf { - TypeFilter::File => path.is_file(), - TypeFilter::Directory => path.is_dir(), - TypeFilter::Symlink => path - .symlink_metadata() - .map(|m| m.file_type().is_symlink()) - .unwrap_or(false), - }; - if !ok { - return false; - } - } - - // Extension filter - if !opts.extensions.is_empty() { - let ext = path - .extension() - .map(|e| e.to_string_lossy().to_string()) - .unwrap_or_default(); - let ext_lower = ext.to_lowercase(); - if !opts - .extensions - .iter() - .any(|e| e.to_lowercase() == ext_lower) - { - return false; - } - } - - // Pattern match - if let Some(ref re) = opts.pattern { - let target = if opts.full_path { - path.to_string_lossy().to_string() - } else { - path.file_name() - .map(|n| n.to_string_lossy().to_string()) - .unwrap_or_default() - }; - if !re.is_match(&target) { - return false; - } - } - - true -} - -fn parse_args(args: &[String]) -> Result { - let mut pattern: Option = None; - let mut extensions: Vec = Vec::new(); - let mut type_filter: Option = None; - let mut max_depth: Option = None; - let mut search_paths: Vec = Vec::new(); - let mut show_hidden = false; - let mut case_insensitive = false; - let mut full_path = false; - let mut absolute_path = false; - let mut i = 0; - - while i < args.len() { - let arg = &args[i]; - match arg.as_str() { - "-h" | "--help" => { - return Ok(ParsedArgs::Help); - } - "-V" | "--version" => { - return Ok(ParsedArgs::Version); - } - "-H" | "--hidden" => { - show_hidden = true; - } - "-I" | "--no-ignore" => { - // No .gitignore support in WASM VFS — this is a no-op - } - "-i" | "--ignore-case" => { - case_insensitive = true; - } - "-s" | "--case-sensitive" => { - case_insensitive = false; - } - "-p" | "--full-path" => { - full_path = true; - } - "-a" | "--absolute-path" => { - absolute_path = true; - } - "-e" | "--extension" => { - i += 1; - if i >= args.len() { - return Err("-e/--extension requires an argument".to_string()); - } - extensions.push(args[i].clone()); - } - "-t" | "--type" => { - i += 1; - if i >= args.len() { - return Err("-t/--type requires an argument".to_string()); - } - type_filter = Some(match args[i].as_str() { - "f" | "file" => TypeFilter::File, - "d" | "directory" => TypeFilter::Directory, - "l" | "symlink" => TypeFilter::Symlink, - other => return Err(format!("unknown type filter: {}", other)), - }); - } - "-d" | "--max-depth" => { - i += 1; - if i >= args.len() { - return Err("-d/--max-depth requires an argument".to_string()); - } - max_depth = Some( - args[i] - .parse() - .map_err(|_| format!("invalid max-depth: {}", args[i]))?, - ); - } - arg if arg.starts_with('-') => { - return Err(format!("unknown option: {}", arg)); - } - _ => { - // First positional arg is pattern, rest are search paths - if pattern.is_none() { - pattern = Some(arg.clone()); - } else { - search_paths.push(arg.clone()); - } - } - } - i += 1; - } - - if search_paths.is_empty() { - search_paths.push(".".to_string()); - } - - // Compile pattern regex - let compiled_pattern = match pattern { - Some(pat) => { - let re_str = if case_insensitive { - format!("(?i){}", pat) - } else { - pat - }; - Some(Regex::new(&re_str).map_err(|e| format!("invalid pattern: {}", e))?) - } - None => None, - }; - - Ok(ParsedArgs::Search(Options { - pattern: compiled_pattern, - extensions, - type_filter, - max_depth, - search_paths, - show_hidden, - _case_insensitive: case_insensitive, - full_path, - absolute_path, - })) -} - -fn print_help(out: &mut W) -> io::Result<()> { - writeln!( - out, - "fd - a simple, fast file finder - -USAGE: - fd [OPTIONS] [PATTERN] [PATH...] - -ARGS: - Regex search pattern - ... Search paths (default: current directory) - -OPTIONS: - -H, --hidden Include hidden files/directories - -I, --no-ignore Do not respect .gitignore (no-op in sandbox) - -i, --ignore-case Case-insensitive search - -s, --case-sensitive Case-sensitive search (default) - -p, --full-path Match pattern against full path - -a, --absolute-path Show absolute paths - -e, --extension EXT Filter by file extension - -t, --type TYPE Filter by type: f(ile), d(irectory), l(ink) - -d, --max-depth N Maximum search depth - -h, --help Print help - -V, --version Print version" - ) -} diff --git a/software/fd/test/fd.test.ts b/software/fd/test/fd.test.ts index b102e7595e..0b76049d2f 100644 --- a/software/fd/test/fd.test.ts +++ b/software/fd/test/fd.test.ts @@ -10,107 +10,20 @@ * Tests verify stdout correctness rather than exit code. */ +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; import { describe, it, expect, afterEach } from 'vitest'; import { createWasmVmRuntime } from '@agentos/test-harness'; -import { COMMANDS_DIR, createKernel, describeIf, hasWasmBinaries } from '@agentos/test-harness'; +import { COMMANDS_DIR, createKernel, describeIf, hasWasmBinaries, NodeFileSystem } from '@agentos/test-harness'; import type { Kernel } from '@agentos/test-harness'; -// Minimal in-memory VFS for kernel tests -class SimpleVFS { - private files = new Map(); - private dirs = new Set(['/']); - - async readFile(path: string): Promise { - const data = this.files.get(path); - if (!data) throw new Error(`ENOENT: ${path}`); - return data; - } - async readTextFile(path: string): Promise { - return new TextDecoder().decode(await this.readFile(path)); - } - async readDir(path: string): Promise { - const prefix = path === '/' ? '/' : path + '/'; - const entries: string[] = []; - for (const p of [...this.files.keys(), ...this.dirs]) { - if (p !== path && p.startsWith(prefix)) { - const rest = p.slice(prefix.length); - if (!rest.includes('/')) entries.push(rest); - } - } - return entries; - } - async readDirWithTypes(path: string) { - return (await this.readDir(path)).map(name => ({ - name, - isDirectory: this.dirs.has(path === '/' ? `/${name}` : `${path}/${name}`), - })); - } - async writeFile(path: string, content: string | Uint8Array): Promise { - const data = typeof content === 'string' ? new TextEncoder().encode(content) : content; - this.files.set(path, new Uint8Array(data)); - const parts = path.split('/').filter(Boolean); - for (let i = 1; i < parts.length; i++) { - this.dirs.add('/' + parts.slice(0, i).join('/')); - } - } - async createDir(path: string) { this.dirs.add(path); } - async mkdir(path: string, _options?: { recursive?: boolean }) { - this.dirs.add(path); - const parts = path.split('/').filter(Boolean); - for (let i = 1; i < parts.length; i++) { - this.dirs.add('/' + parts.slice(0, i).join('/')); - } - } - async exists(path: string): Promise { - return this.files.has(path) || this.dirs.has(path); - } - async stat(path: string) { - const isDir = this.dirs.has(path); - const data = this.files.get(path); - if (!isDir && !data) throw new Error(`ENOENT: ${path}`); - return { - mode: isDir ? 0o40755 : 0o100644, - size: data?.length ?? 0, - isDirectory: isDir, - isSymbolicLink: false, - atimeMs: Date.now(), - mtimeMs: Date.now(), - ctimeMs: Date.now(), - birthtimeMs: Date.now(), - ino: 0, - nlink: 1, - uid: 1000, - gid: 1000, - }; - } - async lstat(path: string) { return this.stat(path); } - async removeFile(path: string) { this.files.delete(path); } - async removeDir(path: string) { this.dirs.delete(path); } - async rename(oldPath: string, newPath: string) { - const data = this.files.get(oldPath); - if (data) { - this.files.set(newPath, data); - this.files.delete(oldPath); - } - } - async pread(path: string, offset: number, length: number): Promise { - const data = this.files.get(path); - if (!data) throw new Error(`ENOENT: ${path}`); - return data.slice(offset, offset + length); - } - async pwrite(path: string, offset: number, content: Uint8Array): Promise { - const data = this.files.get(path); - if (!data) throw new Error(`ENOENT: ${path}`); - const next = new Uint8Array(Math.max(data.length, offset + content.length)); - next.set(data); - next.set(content, offset); - this.files.set(path, next); - } -} +let tempRoot: string | undefined; /** Create a VFS pre-populated with a test directory structure */ -async function createTestVFS(): Promise { - const vfs = new SimpleVFS(); +async function createTestVFS(): Promise { + tempRoot = await mkdtemp(join(tmpdir(), 'agentos-fd-')); + // /project/ // src/ // main.js @@ -124,17 +37,24 @@ async function createTestVFS(): Promise { // secret.txt // .gitignore // config.json - await vfs.writeFile('/project/src/main.js', 'console.log("main")'); - await vfs.writeFile('/project/src/utils.js', 'export {}'); - await vfs.writeFile('/project/src/helpers.ts', 'export {}'); - await vfs.writeFile('/project/lib/parser.js', 'module.exports = {}'); - await vfs.writeFile('/project/docs/readme.md', '# Readme'); - await vfs.writeFile('/project/.hidden/secret.txt', 'secret'); - await vfs.writeFile('/project/.gitignore', 'node_modules'); - await vfs.writeFile('/project/config.json', '{}'); + await writeFixture('/project/src/main.js', 'console.log("main")'); + await writeFixture('/project/src/utils.js', 'export {}'); + await writeFixture('/project/src/helpers.ts', 'export {}'); + await writeFixture('/project/lib/parser.js', 'module.exports = {}'); + await writeFixture('/project/docs/readme.md', '# Readme'); + await writeFixture('/project/.hidden/secret.txt', 'secret'); + await writeFixture('/project/.gitignore', 'node_modules'); + await writeFixture('/project/config.json', '{}'); // /empty/ — empty directory - await vfs.mkdir('/empty', { recursive: true }); - return vfs; + await mkdir(join(tempRoot, 'empty'), { recursive: true }); + return new NodeFileSystem({ root: tempRoot }); +} + +async function writeFixture(path: string, contents: string): Promise { + if (!tempRoot) throw new Error('fixture root not initialized'); + const hostPath = join(tempRoot, path.replace(/^\/+/, '')); + await mkdir(dirname(hostPath), { recursive: true }); + await writeFile(hostPath, contents); } /** Parse fd output lines, sorted for deterministic comparison */ @@ -147,11 +67,24 @@ describeIf(hasWasmBinaries, 'fd-find command', { timeout: 10_000 }, () => { afterEach(async () => { await kernel?.dispose(); + if (tempRoot) { + await rm(tempRoot, { recursive: true, force: true }); + tempRoot = undefined; + } + }); + + it('reports the upstream fd-find version', async () => { + const vfs = await createTestVFS(); + kernel = createKernel({ filesystem: vfs }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); + + const result = await kernel.exec('fd --version', {}); + expect(result.stdout.trim()).toBe('fd 10.4.2'); }); it('finds files matching regex pattern in current directory', async () => { const vfs = await createTestVFS(); - kernel = createKernel({ filesystem: vfs as any }); + kernel = createKernel({ filesystem: vfs }); await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); const result = await kernel.exec('fd main /project', {}); @@ -161,7 +94,7 @@ describeIf(hasWasmBinaries, 'fd-find command', { timeout: 10_000 }, () => { it('finds all .js files with -e js', async () => { const vfs = await createTestVFS(); - kernel = createKernel({ filesystem: vfs as any }); + kernel = createKernel({ filesystem: vfs }); await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); const result = await kernel.exec('fd -e js . /project', {}); @@ -175,7 +108,7 @@ describeIf(hasWasmBinaries, 'fd-find command', { timeout: 10_000 }, () => { it('finds only files with -t f', async () => { const vfs = await createTestVFS(); - kernel = createKernel({ filesystem: vfs as any }); + kernel = createKernel({ filesystem: vfs }); await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); const result = await kernel.exec('fd -t f . /project', {}); @@ -192,7 +125,7 @@ describeIf(hasWasmBinaries, 'fd-find command', { timeout: 10_000 }, () => { it('finds only directories with -t d', async () => { const vfs = await createTestVFS(); - kernel = createKernel({ filesystem: vfs as any }); + kernel = createKernel({ filesystem: vfs }); await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); const result = await kernel.exec('fd -t d . /project', {}); @@ -203,14 +136,14 @@ describeIf(hasWasmBinaries, 'fd-find command', { timeout: 10_000 }, () => { expect(stat.isDirectory).toBe(true); } // Should include known directories (hidden skipped by default) - expect(lines).toContain('/project/src'); - expect(lines).toContain('/project/lib'); - expect(lines).toContain('/project/docs'); + expect(lines).toContain('/project/src/'); + expect(lines).toContain('/project/lib/'); + expect(lines).toContain('/project/docs/'); }); it('returns no results for empty directory', async () => { const vfs = await createTestVFS(); - kernel = createKernel({ filesystem: vfs as any }); + kernel = createKernel({ filesystem: vfs }); await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); const result = await kernel.exec('fd . /empty', {}); @@ -219,7 +152,7 @@ describeIf(hasWasmBinaries, 'fd-find command', { timeout: 10_000 }, () => { it('returns empty output when no files match pattern', async () => { const vfs = await createTestVFS(); - kernel = createKernel({ filesystem: vfs as any }); + kernel = createKernel({ filesystem: vfs }); await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); const result = await kernel.exec('fd zzzznonexistent /project', {}); @@ -228,7 +161,7 @@ describeIf(hasWasmBinaries, 'fd-find command', { timeout: 10_000 }, () => { it('skips hidden files and directories by default', async () => { const vfs = await createTestVFS(); - kernel = createKernel({ filesystem: vfs as any }); + kernel = createKernel({ filesystem: vfs }); await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); const result = await kernel.exec('fd . /project', {}); @@ -243,13 +176,13 @@ describeIf(hasWasmBinaries, 'fd-find command', { timeout: 10_000 }, () => { it('includes hidden files with -H flag', async () => { const vfs = await createTestVFS(); - kernel = createKernel({ filesystem: vfs as any }); + kernel = createKernel({ filesystem: vfs }); await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); const result = await kernel.exec('fd -H . /project', {}); const lines = parseLines(result.stdout); // Hidden items should now appear expect(lines).toContain('/project/.gitignore'); - expect(lines).toContain('/project/.hidden'); + expect(lines).toContain('/project/.hidden/'); }); }); diff --git a/software/file/bin/file b/software/file/bin/file index 1e32fe6064..b134518d9d 100644 Binary files a/software/file/bin/file and b/software/file/bin/file differ diff --git a/software/file/native/crates/file-cmd/src/lib.rs b/software/file/native/crates/file-cmd/src/lib.rs index 92a9faa647..38f507dd73 100644 --- a/software/file/native/crates/file-cmd/src/lib.rs +++ b/software/file/native/crates/file-cmd/src/lib.rs @@ -194,6 +194,38 @@ fn identify(data: &[u8]) -> (String, String) { return ("empty".to_string(), "inode/x-empty".to_string()); } + // Check for shebang scripts + if data.len() >= 2 && data[0] == b'#' && data[1] == b'!' { + let first_line = data + .iter() + .take(128) + .take_while(|&&b| b != b'\n') + .copied() + .collect::>(); + if let Ok(line) = std::str::from_utf8(&first_line) { + let interp = line.trim_start_matches("#!"); + let interp = interp.trim(); + // Extract interpreter name + let name = interp.split_whitespace().next().unwrap_or(interp); + let basename = match name.rfind('/') { + Some(pos) => &name[pos + 1..], + None => name, + }; + // Handle "env " pattern + if basename == "env" { + let prog = interp.split_whitespace().nth(1).unwrap_or("script"); + return ( + format!("{} script, ASCII text executable", prog), + "text/x-script".to_string(), + ); + } + return ( + format!("{} script, ASCII text executable", basename), + "text/x-script".to_string(), + ); + } + } + // Try infer crate for magic byte detection if let Some(kind) = infer::get(data) { let desc = match kind.mime_type() { @@ -231,38 +263,6 @@ fn identify(data: &[u8]) -> (String, String) { return (desc.to_string(), kind.mime_type().to_string()); } - // Check for shebang scripts - if data.len() >= 2 && data[0] == b'#' && data[1] == b'!' { - let first_line = data - .iter() - .take(128) - .take_while(|&&b| b != b'\n') - .copied() - .collect::>(); - if let Ok(line) = std::str::from_utf8(&first_line) { - let interp = line.trim_start_matches("#!"); - let interp = interp.trim(); - // Extract interpreter name - let name = interp.split_whitespace().next().unwrap_or(interp); - let basename = match name.rfind('/') { - Some(pos) => &name[pos + 1..], - None => name, - }; - // Handle "env " pattern - if basename == "env" { - let prog = interp.split_whitespace().nth(1).unwrap_or("script"); - return ( - format!("{} script, ASCII text executable", prog), - "text/x-script".to_string(), - ); - } - return ( - format!("{} script, ASCII text executable", basename), - "text/x-script".to_string(), - ); - } - } - // Check for known text patterns if is_json(data) { return ("JSON text data".to_string(), "application/json".to_string()); diff --git a/software/file/test/file.test.ts b/software/file/test/file.test.ts new file mode 100644 index 0000000000..70657e05b8 --- /dev/null +++ b/software/file/test/file.test.ts @@ -0,0 +1,152 @@ +import { existsSync } from "node:fs"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { + NodeFileSystem, + createKernel, + createWasmVmRuntime, + describeIf, +} from "@agentos/test-harness"; +import type { Kernel } from "@agentos/test-harness"; +import { afterEach, expect, it } from "vitest"; + +const FILE_COMMAND_DIR = fileURLToPath(new URL("../bin", import.meta.url)); +const hasFilePackageBinary = existsSync(join(FILE_COMMAND_DIR, "file")); + +let tempRoot: string | undefined; + +async function writeFixture(path: string, contents: string | Buffer): Promise { + if (!tempRoot) throw new Error("fixture root not initialized"); + const hostPath = join(tempRoot, path.replace(/^\/+/, "")); + await mkdir(dirname(hostPath), { recursive: true }); + await writeFile(hostPath, contents); +} + +async function createTestVFS(): Promise { + tempRoot = await mkdtemp(join(tmpdir(), "agentos-file-")); + await writeFixture("/project/text.txt", "hello from AgentOS\n"); + await writeFixture("/project/data.json", '{ "ok": true }\n'); + await writeFixture("/project/script.sh", "#!/usr/bin/env bash\necho hello\n"); + await writeFixture( + "/project/image.png", + Buffer.from([ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, + 0x0d, + ]), + ); + await writeFixture("/project/empty", ""); + await mkdir(join(tempRoot, "project/dir"), { recursive: true }); + return new NodeFileSystem({ root: tempRoot }); +} + +describeIf(hasFilePackageBinary, "file command", { timeout: 10_000 }, () => { + let kernel: Kernel | undefined; + + afterEach(async () => { + await kernel?.dispose(); + kernel = undefined; + if (tempRoot) { + await rm(tempRoot, { recursive: true, force: true }); + tempRoot = undefined; + } + }); + + async function mountFixture(): Promise { + const vfs = await createTestVFS(); + kernel = createKernel({ filesystem: vfs }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [FILE_COMMAND_DIR] })); + } + + async function runFile(args: string[], stdin?: string) { + if (!kernel) throw new Error("kernel not mounted"); + let stdout = ""; + let stderr = ""; + const proc = kernel.spawn("file", args, { + streamStdin: stdin !== undefined, + onStdout: (chunk) => { + stdout += Buffer.from(chunk).toString("utf8"); + }, + onStderr: (chunk) => { + stderr += Buffer.from(chunk).toString("utf8"); + }, + }); + if (stdin !== undefined) { + proc.writeStdin(stdin); + proc.closeStdin(); + } + const exitCode = await proc.wait(); + await new Promise((resolve) => setTimeout(resolve, 0)); + return { stdout, stderr, exitCode }; + } + + it("identifies text, JSON, scripts, images, empty files, and directories", async () => { + await mountFixture(); + + const result = await runFile([ + "/project/text.txt", + "/project/data.json", + "/project/script.sh", + "/project/image.png", + "/project/empty", + "/project/dir", + ]); + + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(result.stdout).toContain("/project/text.txt: ASCII text"); + expect(result.stdout).toContain("/project/data.json: JSON text data"); + expect(result.stdout).toContain( + "/project/script.sh: bash script, ASCII text executable", + ); + expect(result.stdout).toContain("/project/image.png: PNG image data"); + expect(result.stdout).toContain("/project/empty: empty"); + expect(result.stdout).toContain("/project/dir: directory"); + }); + + it("prints brief descriptions with -b", async () => { + await mountFixture(); + + const result = await runFile(["-b", "/project/data.json"]); + + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(result.stdout.trim()).toBe("JSON text data"); + }); + + it("prints MIME types with -i", async () => { + await mountFixture(); + + const result = await runFile([ + "-i", + "/project/text.txt", + "/project/image.png", + "/project/dir", + ]); + + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(result.stdout).toContain("/project/text.txt: text/plain"); + expect(result.stdout).toContain("/project/image.png: image/png"); + expect(result.stdout).toContain("/project/dir: inode/directory"); + }); + + it("reads stdin when the operand is -", async () => { + await mountFixture(); + + const result = await runFile( + ["-b", "-"], + "#!/usr/bin/env node\nconsole.log('hello')\n", + ); + + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(result.stdout.trim()).toBe("node script, ASCII text executable"); + }); + + it("returns an error for missing inputs", async () => { + await mountFixture(); + + const result = await runFile(["/project/missing.txt"]); + + expect(result.exitCode).toBe(1); + expect(result.stderr).toContain("/project/missing.txt"); + }); +}); diff --git a/software/findutils/bin/find b/software/findutils/bin/find index e08e8b58a1..3693ed2473 100644 Binary files a/software/findutils/bin/find and b/software/findutils/bin/find differ diff --git a/software/findutils/bin/xargs b/software/findutils/bin/xargs index 613a7bb3ea..2b303eb53d 100644 Binary files a/software/findutils/bin/xargs and b/software/findutils/bin/xargs differ diff --git a/software/findutils/native/crates/cmd-find/Cargo.toml b/software/findutils/native/crates/cmd-find/Cargo.toml index f394612385..19cc8f38f4 100644 --- a/software/findutils/native/crates/cmd-find/Cargo.toml +++ b/software/findutils/native/crates/cmd-find/Cargo.toml @@ -11,4 +11,5 @@ name = "find" path = "src/main.rs" [dependencies] -secureexec-find = { path = "../find" } +findutils = { version = "0.9.0", default-features = false } +uucore = { version = "0.9.0", default-features = false } diff --git a/software/findutils/native/crates/cmd-find/src/main.rs b/software/findutils/native/crates/cmd-find/src/main.rs index 5887b1a7a4..8990e43353 100644 --- a/software/findutils/native/crates/cmd-find/src/main.rs +++ b/software/findutils/native/crates/cmd-find/src/main.rs @@ -1,4 +1,11 @@ fn main() { - let args: Vec = std::env::args_os().collect(); - std::process::exit(secureexec_find::main(args)); + uucore::panic::mute_sigpipe_panic(); + + let args = std::env::args().collect::>(); + let args = args + .iter() + .map(std::convert::AsRef::as_ref) + .collect::>(); + let deps = findutils::find::StandardDependencies::new(); + std::process::exit(findutils::find::find_main(&args, &deps)); } diff --git a/software/findutils/native/crates/cmd-xargs/Cargo.toml b/software/findutils/native/crates/cmd-xargs/Cargo.toml index e43b660a5b..1037c80a58 100644 --- a/software/findutils/native/crates/cmd-xargs/Cargo.toml +++ b/software/findutils/native/crates/cmd-xargs/Cargo.toml @@ -11,4 +11,4 @@ name = "xargs" path = "src/main.rs" [dependencies] -shims = { package = "secureexec-shims", path = "../../../../../toolchain/crates/libs/shims" } +findutils = { version = "0.9.0", default-features = false } diff --git a/software/findutils/native/crates/cmd-xargs/src/main.rs b/software/findutils/native/crates/cmd-xargs/src/main.rs index 0ef4547696..85586163c6 100644 --- a/software/findutils/native/crates/cmd-xargs/src/main.rs +++ b/software/findutils/native/crates/cmd-xargs/src/main.rs @@ -1,4 +1,8 @@ fn main() { - let args: Vec = std::env::args_os().collect(); - std::process::exit(shims::xargs::xargs(args)); + let args = std::env::args().collect::>(); + let args = args + .iter() + .map(std::convert::AsRef::as_ref) + .collect::>(); + std::process::exit(findutils::xargs::xargs_main(&args)); } diff --git a/software/findutils/native/crates/find/Cargo.toml b/software/findutils/native/crates/find/Cargo.toml deleted file mode 100644 index 4c4cd79502..0000000000 --- a/software/findutils/native/crates/find/Cargo.toml +++ /dev/null @@ -1,10 +0,0 @@ -[package] -workspace = "../../../../../toolchain" -name = "secureexec-find" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "find implementation for secure-exec standalone binaries" - -[dependencies] -regex = "1" diff --git a/software/findutils/native/crates/find/src/lib.rs b/software/findutils/native/crates/find/src/lib.rs deleted file mode 100644 index 059e80fa53..0000000000 --- a/software/findutils/native/crates/find/src/lib.rs +++ /dev/null @@ -1,593 +0,0 @@ -//! POSIX find implementation for WASM. -//! -//! Custom implementation using std::fs (WASI) for directory traversal -//! and glob-to-regex conversion for pattern matching. fd-find crate -//! cannot be used directly as it's a binary crate with platform-specific -//! dependencies incompatible with wasm32-wasip1. - -use std::collections::HashSet; -use std::ffi::OsString; -use std::fs; -use std::io::{self, Write}; -use std::path::{Path, PathBuf}; - -use regex::Regex; - -/// Entry point for find command. -pub fn main(args: Vec) -> i32 { - let str_args: Vec = args - .iter() - .skip(1) // skip argv[0] - .map(|a| a.to_string_lossy().to_string()) - .collect(); - - match run_find(&str_args) { - Ok(found) => { - if found { - 0 - } else { - 0 - } // find returns 0 even if no matches - } - Err(msg) => { - eprintln!("find: {}", msg); - 1 - } - } -} - -/// Parsed find expression node. -#[derive(Debug)] -enum Expr { - /// -name PATTERN (glob match on filename only) - Name(Regex), - /// -iname PATTERN (case-insensitive glob match on filename) - IName(Regex), - /// -path PATTERN (glob match on full path) - PathMatch(Regex), - /// -ipath PATTERN (case-insensitive glob match on full path) - IPathMatch(Regex), - /// -type TYPE (d, f, l) - Type(FileType), - /// -empty (file size 0 or empty directory) - Empty, - /// -maxdepth N (handled specially during traversal) - MaxDepth(usize), - /// -mindepth N (handled specially during traversal) - MinDepth(usize), - /// -print (explicit print action) - Print, - /// -not EXPR / ! EXPR - Not(Box), - /// EXPR -a EXPR / EXPR -and EXPR / implicit AND - And(Box, Box), - /// EXPR -o EXPR / EXPR -or EXPR - Or(Box, Box), -} - -#[derive(Debug, Clone, Copy, PartialEq)] -enum FileType { - Directory, - File, - Symlink, -} - -struct FindOptions { - paths: Vec, - expr: Option, - max_depth: Option, - min_depth: usize, -} - -fn run_find(args: &[String]) -> Result { - let opts = parse_args(args)?; - let mut found_any = false; - let stdout = io::stdout(); - let mut out = stdout.lock(); - - for path in &opts.paths { - let mut visited_dirs = HashSet::new(); - walk( - &PathBuf::from(path), - path, - 0, - &opts, - &mut found_any, - &mut out, - &mut visited_dirs, - )?; - } - - out.flush() - .map_err(|e| format!("failed to write output: {e}"))?; - - Ok(found_any) -} - -/// Recursive directory walk. -fn walk( - full_path: &Path, - display_path: &str, - depth: usize, - opts: &FindOptions, - found_any: &mut bool, - out: &mut W, - visited_dirs: &mut HashSet, -) -> Result<(), String> { - // Check max depth before processing - if let Some(max) = opts.max_depth { - if depth > max { - return Ok(()); - } - } - - // Evaluate expression against this entry (if at or below min_depth) - if depth >= opts.min_depth { - let matches = match &opts.expr { - Some(expr) => eval_expr(expr, full_path, display_path), - None => true, // no expression means match everything - }; - - if matches { - *found_any = true; - // Default action is print - writeln!(out, "{}", display_path) - .map_err(|e| format!("failed to write output: {e}"))?; - } - } - - // Recurse into directories - if full_path.is_dir() { - // Check max depth for children - if let Some(max) = opts.max_depth { - if depth >= max { - return Ok(()); - } - } - - let canonical_path = - fs::canonicalize(full_path).map_err(|e| format!("{}: {}", display_path, e))?; - if !visited_dirs.insert(canonical_path.clone()) { - return Ok(()); - } - - let entries = fs::read_dir(full_path).map_err(|e| format!("{}: {}", display_path, e))?; - - let mut sorted_entries: Vec<_> = entries - .collect::, _>>() - .map_err(|e| format!("{}: {}", display_path, e))?; - sorted_entries.sort_by_key(|e| e.file_name()); - - for entry in sorted_entries { - let child_name = entry.file_name().to_string_lossy().to_string(); - let child_display = if display_path == "/" { - format!("/{}", child_name) - } else { - format!("{}/{}", display_path, child_name) - }; - let child_full = entry.path(); - - walk( - &child_full, - &child_display, - depth + 1, - opts, - found_any, - out, - visited_dirs, - )?; - } - visited_dirs.remove(&canonical_path); - } - - Ok(()) -} - -/// Evaluate an expression against a path. -fn eval_expr(expr: &Expr, full_path: &Path, display_path: &str) -> bool { - match expr { - Expr::Name(re) => { - let name = full_path - .file_name() - .map(|n| n.to_string_lossy().to_string()) - .unwrap_or_default(); - re.is_match(&name) - } - Expr::IName(re) => { - let name = full_path - .file_name() - .map(|n| n.to_string_lossy().to_string()) - .unwrap_or_default(); - re.is_match(&name) - } - Expr::PathMatch(re) => re.is_match(display_path), - Expr::IPathMatch(re) => re.is_match(display_path), - Expr::Type(ft) => match ft { - FileType::Directory => full_path.is_dir(), - FileType::File => full_path.is_file(), - FileType::Symlink => full_path - .symlink_metadata() - .map(|m| m.file_type().is_symlink()) - .unwrap_or(false), - }, - Expr::Empty => { - if full_path.is_dir() { - fs::read_dir(full_path) - .map(|mut d| d.next().is_none()) - .unwrap_or(false) - } else if full_path.is_file() { - full_path.metadata().map(|m| m.len() == 0).unwrap_or(false) - } else { - false - } - } - Expr::MaxDepth(_) | Expr::MinDepth(_) => true, // handled in walk() - Expr::Print => true, // print is an action, always "matches" - Expr::Not(inner) => !eval_expr(inner, full_path, display_path), - Expr::And(left, right) => { - eval_expr(left, full_path, display_path) && eval_expr(right, full_path, display_path) - } - Expr::Or(left, right) => { - eval_expr(left, full_path, display_path) || eval_expr(right, full_path, display_path) - } - } -} - -/// Convert a shell glob pattern to a regex pattern. -/// Supports *, ?, [charset], and ** (for path matching). -fn glob_to_regex(glob: &str, case_insensitive: bool) -> Result { - let mut re = String::from("^"); - let chars: Vec = glob.chars().collect(); - let mut i = 0; - - while i < chars.len() { - match chars[i] { - '*' => { - if i + 1 < chars.len() && chars[i + 1] == '*' { - // ** matches everything including / - re.push_str(".*"); - i += 1; - } else { - // * matches everything except / - re.push_str("[^/]*"); - } - } - '?' => re.push_str("[^/]"), - '[' => { - re.push('['); - i += 1; - if i < chars.len() && chars[i] == '!' { - re.push('^'); - i += 1; - } else if i < chars.len() && chars[i] == '^' { - re.push('^'); - i += 1; - } - while i < chars.len() && chars[i] != ']' { - if chars[i] == '\\' && i + 1 < chars.len() { - re.push('\\'); - i += 1; - re.push(chars[i]); - } else { - re.push(chars[i]); - } - i += 1; - } - re.push(']'); - } - '.' | '+' | '(' | ')' | '{' | '}' | '|' | '^' | '$' | '\\' => { - re.push('\\'); - re.push(chars[i]); - } - _ => re.push(chars[i]), - } - i += 1; - } - - re.push('$'); - - let pattern = if case_insensitive { - format!("(?i){}", re) - } else { - re - }; - - Regex::new(&pattern).map_err(|e| format!("invalid pattern: {}", e)) -} - -/// Parse find command arguments. -fn parse_args(args: &[String]) -> Result { - let mut paths: Vec = Vec::new(); - let mut exprs: Vec = Vec::new(); - let mut max_depth: Option = None; - let mut min_depth: usize = 0; - let mut i = 0; - - // First, collect paths (arguments before any expression) - while i < args.len() { - let arg = &args[i]; - if arg.starts_with('-') || arg == "!" || arg == "(" { - break; - } - paths.push(arg.clone()); - i += 1; - } - - if paths.is_empty() { - paths.push(".".to_string()); - } - - // Parse expressions - while i < args.len() { - let arg = &args[i]; - - match arg.as_str() { - "-name" => { - i += 1; - if i >= args.len() { - return Err("-name requires an argument".to_string()); - } - let re = glob_to_regex(&args[i], false)?; - exprs.push(Expr::Name(re)); - } - "-iname" => { - i += 1; - if i >= args.len() { - return Err("-iname requires an argument".to_string()); - } - let re = glob_to_regex(&args[i], true)?; - exprs.push(Expr::IName(re)); - } - "-path" | "-wholename" => { - i += 1; - if i >= args.len() { - return Err(format!("{} requires an argument", arg)); - } - let re = glob_to_regex(&args[i], false)?; - exprs.push(Expr::PathMatch(re)); - } - "-ipath" | "-iwholename" => { - i += 1; - if i >= args.len() { - return Err(format!("{} requires an argument", arg)); - } - let re = glob_to_regex(&args[i], true)?; - exprs.push(Expr::IPathMatch(re)); - } - "-type" => { - i += 1; - if i >= args.len() { - return Err("-type requires an argument".to_string()); - } - let ft = match args[i].as_str() { - "d" => FileType::Directory, - "f" => FileType::File, - "l" => FileType::Symlink, - other => return Err(format!("unknown file type: {}", other)), - }; - exprs.push(Expr::Type(ft)); - } - "-empty" => { - exprs.push(Expr::Empty); - } - "-maxdepth" => { - i += 1; - if i >= args.len() { - return Err("-maxdepth requires an argument".to_string()); - } - let n: usize = args[i] - .parse() - .map_err(|_| format!("-maxdepth: {}: not a number", args[i]))?; - max_depth = Some(n); - } - "-mindepth" => { - i += 1; - if i >= args.len() { - return Err("-mindepth requires an argument".to_string()); - } - let n: usize = args[i] - .parse() - .map_err(|_| format!("-mindepth: {}: not a number", args[i]))?; - min_depth = n; - } - "-print" => { - exprs.push(Expr::Print); - } - "!" | "-not" => { - i += 1; - if i >= args.len() { - return Err("! requires an expression".to_string()); - } - // Parse the next single expression - let (next_expr, next_i) = parse_single_expr(&args, i)?; - if let Some((md, mi)) = extract_depth(&next_expr) { - if let Some(d) = md { - max_depth = Some(d); - } - min_depth = mi.unwrap_or(min_depth); - } - exprs.push(Expr::Not(Box::new(next_expr))); - i = next_i; - continue; // skip the i += 1 at end - } - "-a" | "-and" => { - // Implicit AND between consecutive expressions, -a is explicit - // Just skip it; AND is applied when combining exprs - } - "-o" | "-or" => { - // Combine everything so far as left side of OR - if exprs.is_empty() { - return Err("-or requires a preceding expression".to_string()); - } - let left = combine_and(exprs); - - // Parse remaining as right side - i += 1; - let mut right_exprs = Vec::new(); - while i < args.len() { - let (e, ni) = parse_single_expr(args, i)?; - if let Some((md, mi)) = extract_depth(&e) { - if let Some(d) = md { - max_depth = Some(d); - } - min_depth = mi.unwrap_or(min_depth); - } - right_exprs.push(e); - i = ni; - } - let right = combine_and(right_exprs); - exprs = vec![Expr::Or(Box::new(left), Box::new(right))]; - break; - } - "(" => { - // Grouped expression - find matching ) - i += 1; - let mut depth = 1; - let start = i; - while i < args.len() && depth > 0 { - if args[i] == "(" { - depth += 1; - } - if args[i] == ")" { - depth -= 1; - } - if depth > 0 { - i += 1; - } - } - if depth != 0 { - return Err("unmatched '('".to_string()); - } - let sub_args: Vec = args[start..i].to_vec(); - let sub_opts = parse_args_exprs(&sub_args)?; - if let Some(e) = sub_opts { - exprs.push(e); - } - } - other if other.starts_with('-') => { - return Err(format!("unknown option: {}", other)); - } - _ => { - // Treat as path if no expressions yet, otherwise error - return Err(format!("unexpected argument: {}", arg)); - } - } - i += 1; - } - - let expr = if exprs.is_empty() { - None - } else { - Some(combine_and(exprs)) - }; - - Ok(FindOptions { - paths, - expr, - max_depth, - min_depth, - }) -} - -/// Parse a single expression at position i, returning the expression and the next position. -fn parse_single_expr(args: &[String], i: usize) -> Result<(Expr, usize), String> { - if i >= args.len() { - return Err("unexpected end of expression".to_string()); - } - - let arg = &args[i]; - match arg.as_str() { - "-name" => { - if i + 1 >= args.len() { - return Err("-name requires an argument".to_string()); - } - let re = glob_to_regex(&args[i + 1], false)?; - Ok((Expr::Name(re), i + 2)) - } - "-iname" => { - if i + 1 >= args.len() { - return Err("-iname requires an argument".to_string()); - } - let re = glob_to_regex(&args[i + 1], true)?; - Ok((Expr::IName(re), i + 2)) - } - "-path" | "-wholename" => { - if i + 1 >= args.len() { - return Err(format!("{} requires an argument", arg)); - } - let re = glob_to_regex(&args[i + 1], false)?; - Ok((Expr::PathMatch(re), i + 2)) - } - "-type" => { - if i + 1 >= args.len() { - return Err("-type requires an argument".to_string()); - } - let ft = match args[i + 1].as_str() { - "d" => FileType::Directory, - "f" => FileType::File, - "l" => FileType::Symlink, - other => return Err(format!("unknown file type: {}", other)), - }; - Ok((Expr::Type(ft), i + 2)) - } - "-empty" => Ok((Expr::Empty, i + 1)), - "-maxdepth" => { - if i + 1 >= args.len() { - return Err("-maxdepth requires an argument".to_string()); - } - let n: usize = args[i + 1] - .parse() - .map_err(|_| format!("-maxdepth: {}: not a number", args[i + 1]))?; - Ok((Expr::MaxDepth(n), i + 2)) - } - "-mindepth" => { - if i + 1 >= args.len() { - return Err("-mindepth requires an argument".to_string()); - } - let n: usize = args[i + 1] - .parse() - .map_err(|_| format!("-mindepth: {}: not a number", args[i + 1]))?; - Ok((Expr::MinDepth(n), i + 2)) - } - "-print" => Ok((Expr::Print, i + 1)), - _ => Err(format!("unknown expression: {}", arg)), - } -} - -/// Parse expression-only args (for grouped expressions). -fn parse_args_exprs(args: &[String]) -> Result, String> { - let mut exprs = Vec::new(); - let mut i = 0; - - while i < args.len() { - let (expr, next_i) = parse_single_expr(args, i)?; - exprs.push(expr); - i = next_i; - } - - if exprs.is_empty() { - Ok(None) - } else { - Ok(Some(combine_and(exprs))) - } -} - -/// Extract depth settings from an expression. -fn extract_depth(expr: &Expr) -> Option<(Option, Option)> { - match expr { - Expr::MaxDepth(n) => Some((Some(*n), None)), - Expr::MinDepth(n) => Some((None, Some(*n))), - _ => None, - } -} - -/// Combine a list of expressions with AND. -fn combine_and(mut exprs: Vec) -> Expr { - if exprs.len() == 1 { - return exprs.remove(0); - } - let first = exprs.remove(0); - exprs - .into_iter() - .fold(first, |acc, e| Expr::And(Box::new(acc), Box::new(e))) -} diff --git a/software/findutils/test/findutils.test.ts b/software/findutils/test/findutils.test.ts index 030f38d208..f343e46864 100644 --- a/software/findutils/test/findutils.test.ts +++ b/software/findutils/test/findutils.test.ts @@ -60,6 +60,23 @@ describeIf(hasWasmBinaries, "findutils commands", { timeout: 10_000 }, () => { ]); }); + it("find supports depth limits", async () => { + const vfs = createInMemoryFileSystem(); + await vfs.mkdir("/project/src/nested", { recursive: true }); + await vfs.mkdir("/project/docs", { recursive: true }); + await vfs.writeFile("/project/src/main.js", "console.log('main')\n"); + await vfs.writeFile("/project/src/nested/helper.ts", "export {}\n"); + await vfs.writeFile("/project/docs/readme.md", "# Readme\n"); + + kernel = createKernel({ filesystem: vfs }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); + + const result = await kernel.exec( + "find /project -mindepth 2 -maxdepth 2 -type f -name '*.js'", + ); + expect(parseLines(result.stdout)).toEqual(["/project/src/main.js"]); + }); + it("xargs passes stdin arguments to a command", async () => { const vfs = createInMemoryFileSystem(); await vfs.writeFile("/args.txt", "alpha\nbeta\n"); @@ -70,4 +87,19 @@ describeIf(hasWasmBinaries, "findutils commands", { timeout: 10_000 }, () => { const result = await kernel.exec("xargs echo < /args.txt"); expect(result.stdout.trim()).toBe("alpha beta"); }); + + it("xargs batches arguments across spawned commands", async () => { + const vfs = createInMemoryFileSystem(); + await vfs.writeFile("/args.txt", "one two three four five\n"); + + kernel = createKernel({ filesystem: vfs }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); + + const result = await kernel.exec("xargs -n 2 echo < /args.txt"); + expect(result.stdout.trim().split("\n")).toEqual([ + "one two", + "three four", + "five", + ]); + }); }); diff --git a/software/gawk/test/gawk.test.ts b/software/gawk/test/gawk.test.ts new file mode 100644 index 0000000000..c900d622d1 --- /dev/null +++ b/software/gawk/test/gawk.test.ts @@ -0,0 +1,130 @@ +import { existsSync } from "node:fs"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { + NodeFileSystem, + createKernel, + createWasmVmRuntime, + describeIf, +} from "@agentos/test-harness"; +import type { Kernel } from "@agentos/test-harness"; +import { afterEach, expect, it } from "vitest"; + +const AWK_COMMAND_DIR = fileURLToPath(new URL("../bin", import.meta.url)); +const hasAwkPackageBinary = existsSync(join(AWK_COMMAND_DIR, "awk")); + +let tempRoot: string | undefined; + +async function writeFixture(path: string, contents: string): Promise { + if (!tempRoot) throw new Error("fixture root not initialized"); + const hostPath = join(tempRoot, path.replace(/^\/+/, "")); + await mkdir(dirname(hostPath), { recursive: true }); + await writeFile(hostPath, contents); +} + +async function createTestVFS(): Promise { + tempRoot = await mkdtemp(join(tmpdir(), "agentos-gawk-")); + await writeFixture( + "/project/scores.txt", + [ + "Ada runtime 7", + "Grace docs 5", + "Linus runtime 3", + "Barbara infra 9", + ].join("\n") + "\n", + ); + await writeFixture( + "/project/colon.txt", + ["alpha:build:4", "beta:test:6", "gamma:deploy:2"].join("\n") + "\n", + ); + await writeFixture( + "/project/runtime.awk", + '/runtime/ { print $1 ":" $3 }\nEND { print "done" }\n', + ); + return new NodeFileSystem({ root: tempRoot }); +} + +function lines(stdout: string): string[] { + return stdout.split("\n").filter((line) => line.length > 0); +} + +describeIf(hasAwkPackageBinary, "awk command", { timeout: 10_000 }, () => { + let kernel: Kernel | undefined; + + afterEach(async () => { + await kernel?.dispose(); + kernel = undefined; + if (tempRoot) { + await rm(tempRoot, { recursive: true, force: true }); + tempRoot = undefined; + } + }); + + async function mountFixture(): Promise { + const vfs = await createTestVFS(); + kernel = createKernel({ filesystem: vfs }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [AWK_COMMAND_DIR] })); + } + + async function runAwk(args: string[]) { + if (!kernel) throw new Error("kernel not mounted"); + let stdout = ""; + let stderr = ""; + const proc = kernel.spawn("awk", args, { + onStdout: (chunk) => { + stdout += Buffer.from(chunk).toString("utf8"); + }, + onStderr: (chunk) => { + stderr += Buffer.from(chunk).toString("utf8"); + }, + }); + const exitCode = await proc.wait(); + await new Promise((resolve) => setTimeout(resolve, 0)); + return { stdout, stderr, exitCode }; + } + + it("extracts fields from a file", async () => { + await mountFixture(); + + const result = await runAwk(["{print $1}", "/project/scores.txt"]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(lines(result.stdout)).toEqual(["Ada", "Grace", "Linus", "Barbara"]); + }); + + it("uses explicit field separators", async () => { + await mountFixture(); + + const result = await runAwk(["-F:", "{print $2}", "/project/colon.txt"]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(lines(result.stdout)).toEqual(["build", "test", "deploy"]); + }); + + it("aggregates numeric columns", async () => { + await mountFixture(); + + const result = await runAwk([ + "{sum += $3} END {print sum}", + "/project/scores.txt", + ]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(result.stdout.trim()).toBe("24"); + }); + + it("runs awk programs from files", async () => { + await mountFixture(); + + const result = await runAwk(["-f", "/project/runtime.awk", "/project/scores.txt"]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(lines(result.stdout)).toEqual(["Ada:7", "Linus:3", "done"]); + }); + + it("fails when an input file is missing", async () => { + await mountFixture(); + + const result = await runAwk(["{print $1}", "/project/missing.txt"]); + expect(result.exitCode).not.toBe(0); + expect(result.stderr).toContain("/project/missing.txt"); + }); +}); diff --git a/software/git/agentos-package.json b/software/git/agentos-package.json index c58e5c4f32..31c38e66e4 100644 --- a/software/git/agentos-package.json +++ b/software/git/agentos-package.json @@ -4,11 +4,14 @@ ], "aliases": { "git-remote-http": "git", - "git-remote-https": "git" + "git-remote-https": "git", + "git-upload-pack": "git", + "git-receive-pack": "git", + "git-upload-archive": "git" }, "registry": { "title": "git", - "description": "Git version control (clone, commit, push, pull).", + "description": "Upstream Git version control for AgentOS VMs.", "priority": 90, "image": "/images/registry/git.svg", "category": "developer-tools" diff --git a/software/git/bin/git b/software/git/bin/git index 1865311237..8c7211cc97 100644 Binary files a/software/git/bin/git and b/software/git/bin/git differ diff --git a/software/git/bin/git-remote-http b/software/git/bin/git-remote-http index 1865311237..8c7211cc97 100644 Binary files a/software/git/bin/git-remote-http and b/software/git/bin/git-remote-http differ diff --git a/software/git/bin/git-remote-https b/software/git/bin/git-remote-https index 1865311237..8c7211cc97 100644 Binary files a/software/git/bin/git-remote-https and b/software/git/bin/git-remote-https differ diff --git a/software/git/native/crates/cmd-git/Cargo.toml b/software/git/native/crates/cmd-git/Cargo.toml deleted file mode 100644 index 83f16f93c4..0000000000 --- a/software/git/native/crates/cmd-git/Cargo.toml +++ /dev/null @@ -1,14 +0,0 @@ -[package] -workspace = "../../../../../toolchain" -name = "cmd-git" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "git standalone binary for secure-exec VM" - -[[bin]] -name = "git" -path = "src/main.rs" - -[dependencies] -secureexec-git = { path = "../git" } diff --git a/software/git/native/crates/git/Cargo.toml b/software/git/native/crates/git/Cargo.toml deleted file mode 100644 index 5185e88b15..0000000000 --- a/software/git/native/crates/git/Cargo.toml +++ /dev/null @@ -1,16 +0,0 @@ -[package] -workspace = "../../../../../toolchain" -name = "secureexec-git" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "Minimal git implementation for secure-exec VM" - -[lib] -name = "secureexec_git" -path = "src/lib.rs" - -[dependencies] -sha1 = "0.10.6" -flate2 = "1.1" -wasi-http = { package = "secureexec-wasi-http", path = "../../../../../toolchain/crates/libs/wasi-http" } diff --git a/software/git/native/crates/git/README.md b/software/git/native/crates/git/README.md deleted file mode 100644 index 26fcb37043..0000000000 --- a/software/git/native/crates/git/README.md +++ /dev/null @@ -1,36 +0,0 @@ -# secure-exec VM git compatibility - -The `git` command in secure-exec is a clean-room, Apache-2.0 reimplementation of a -small git subset for WasmVM guests. - -## Supported commands - -- `git init` -- `git add` -- `git commit -m ...` -- `git rev-parse ` -- `git branch` -- `git checkout` -- `git clone ` -- `git clone http://...` -- `git clone https://...` - -Remote clone support is limited to unauthenticated smart-HTTP fetches. The -implementation can read from `http://` and `https://` remotes, but it does not -support credentials, push, or SSH transport. - -## Unsupported commands and transports - -The guest command exits with a typed `GitSubcommandUnsupported` error for: - -- `git push` -- `git fetch` -- `git pull` -- `git remote` -- SSH-style clone URLs such as `git@host:owner/repo.git` and `ssh://...` -- `git://...` clone URLs -- Authenticated HTTP(S) remotes such as `https://user@host/repo.git` -- Submodules, hooks, GPG signing, and other advanced porcelain workflows - -If you need behavior outside this subset, keep using host git or extend the -clean-room implementation deliberately. diff --git a/software/git/native/crates/git/src/lib.rs b/software/git/native/crates/git/src/lib.rs deleted file mode 100644 index 62a273b237..0000000000 --- a/software/git/native/crates/git/src/lib.rs +++ /dev/null @@ -1,2127 +0,0 @@ -//! Minimal clean-room git implementation (Apache-2.0). -//! -//! Supports: init, add, commit, branch, checkout (with DWIM), local clone, -//! and smart-HTTP clone over http:// and https://. - -use flate2::bufread::ZlibDecoder as BufZlibDecoder; -use flate2::read::ZlibDecoder; -use flate2::write::ZlibEncoder; -use flate2::Compression; -use sha1::{Digest, Sha1}; -use std::collections::{BTreeMap, HashMap, HashSet}; -use std::ffi::{OsStr, OsString}; -use std::fmt; -use std::fs; -use std::io::{self, Cursor, Read, Write}; -use std::path::{Component, Path, PathBuf}; -use wasi_http::{HttpClient, Method, Request}; - -// ─── Hex utilities ────────────────────────────────────────────────────────── - -fn hex(bytes: &[u8]) -> String { - bytes.iter().map(|b| format!("{:02x}", b)).collect() -} - -fn unhex(s: &str) -> io::Result<[u8; 20]> { - if s.len() != 40 { - return Err(err(&format!("invalid hash length: {}", s.len()))); - } - let bytes: Vec = (0..s.len()) - .step_by(2) - .map(|i| { - u8::from_str_radix(&s[i..i + 2], 16).map_err(|e| err(&format!("invalid hex: {}", e))) - }) - .collect::>>()?; - let mut hash = [0u8; 20]; - hash.copy_from_slice(&bytes); - Ok(hash) -} - -fn err(msg: &str) -> io::Error { - io::Error::new(io::ErrorKind::Other, msg.to_string()) -} - -fn print_stdout_line(args: fmt::Arguments<'_>) -> io::Result<()> { - let mut stdout = io::stdout().lock(); - stdout.write_fmt(args)?; - stdout.write_all(b"\n")?; - stdout.flush() -} - -const SUPPORT_DOC_PATH: &str = "software/git/native/crates/git/README.md"; -const MAX_GIT_OBJECT_BYTES: usize = 128 * 1024 * 1024; -const MAX_GIT_OBJECT_HEADER_BYTES: usize = 128; -const MAX_INDEX_ENTRIES: usize = 1_000_000; -const MAX_INDEX_BYTES: usize = 256 * 1024 * 1024; -const MAX_ADVERTISED_REFS: usize = 100_000; -const MAX_HTTP_ERROR_BODY_BYTES: usize = 8192; -const MAX_PKT_LINES: usize = 100_000; -const MAX_PKT_LINE_PAYLOAD_BYTES: usize = 16 * 1024 * 1024; -const MAX_REF_ADVERTISEMENT_BYTES: usize = 16 * 1024 * 1024; -const MAX_PACK_INFLATED_BYTES: usize = 256 * 1024 * 1024; -const MAX_PACK_BYTES: usize = 256 * 1024 * 1024; -const MAX_PACK_OBJECTS: usize = 1_000_000; -const MAX_PACK_RESOLVED_BYTES: usize = 256 * 1024 * 1024; - -fn unsupported(subcommand: &str, detail: &str) -> io::Error { - err(&format!( - "GitSubcommandUnsupported: git {} {} See {} for supported transports and commands.", - subcommand, detail, SUPPORT_DOC_PATH - )) -} - -/// WASI-safe mkdir -p: create_dir_all has issues with WASI permission checks -/// on already-existing directories, so we create one level at a time. -/// We also treat PermissionDenied as "already exists" because some WASI runtimes -/// return EACCES instead of EEXIST for existing directories. -fn mkdirs(path: &Path) -> io::Result<()> { - let mut ancestors: Vec<&Path> = path.ancestors().collect(); - ancestors.reverse(); // root first - for dir in ancestors { - if dir == Path::new("/") || dir == Path::new("") { - continue; - } - match fs::create_dir(dir) { - Ok(()) => {} - Err(e) if e.kind() == io::ErrorKind::AlreadyExists => {} - // WASI kernels may return EACCES for existing directories - Err(e) if e.kind() == io::ErrorKind::PermissionDenied => { - // Verify it actually exists; propagate if it truly doesn't - if !dir.exists() { - return Err(e); - } - } - Err(e) => return Err(e), - } - } - Ok(()) -} - -fn dir_is_empty(path: &Path) -> io::Result { - if !path.exists() { - return Ok(true); - } - if !path.is_dir() { - return Ok(false); - } - Ok(fs::read_dir(path)?.next().is_none()) -} - -fn read_to_end_limited(reader: R, limit: usize, context: &str) -> io::Result> { - let limit_plus_one = limit - .checked_add(1) - .ok_or_else(|| err(&format!("{} size limit is too large", context)))?; - let mut out = Vec::new(); - reader.take(limit_plus_one as u64).read_to_end(&mut out)?; - if out.len() > limit { - return Err(err(&format!("{} exceeds size limit", context))); - } - Ok(out) -} - -fn read_file_limited(path: &Path, limit: usize, context: &str) -> io::Result> { - let metadata = fs::metadata(path).map_err(|e| { - err(&format!( - "cannot stat {} '{}': {}", - context, - path.display(), - e - )) - })?; - if metadata.len() > limit as u64 { - return Err(err(&format!( - "{} '{}' exceeds size limit", - context, - path.display() - ))); - } - fs::read(path).map_err(|e| { - err(&format!( - "cannot read {} '{}': {}", - context, - path.display(), - e - )) - }) -} - -fn try_reserve_exact(vec: &mut Vec, additional: usize, context: &str) -> io::Result<()> { - vec.try_reserve_exact(additional) - .map_err(|_| err(&format!("{} allocation failed", context))) -} - -fn append_pack_bytes(pack: &mut Vec, bytes: &[u8]) -> io::Result<()> { - let new_len = pack - .len() - .checked_add(bytes.len()) - .ok_or_else(|| err("packfile size overflow"))?; - if new_len > MAX_PACK_BYTES { - return Err(err("packfile exceeds size limit")); - } - try_reserve_exact(pack, bytes.len(), "packfile")?; - pack.extend_from_slice(bytes); - Ok(()) -} - -fn add_pack_inflated_bytes(total: &mut usize, bytes: usize) -> io::Result<()> { - *total = total - .checked_add(bytes) - .ok_or_else(|| err("inflated pack size overflow"))?; - if *total > MAX_PACK_INFLATED_BYTES { - return Err(err("inflated pack data exceeds size limit")); - } - Ok(()) -} - -fn add_pack_resolved_bytes(total: &mut usize, bytes: usize) -> io::Result<()> { - *total = total - .checked_add(bytes) - .ok_or_else(|| err("resolved pack size overflow"))?; - if *total > MAX_PACK_RESOLVED_BYTES { - return Err(err("resolved pack data exceeds size limit")); - } - Ok(()) -} - -fn add_pkt_payload_bytes(total: &mut usize, bytes: usize) -> io::Result<()> { - *total = total - .checked_add(bytes) - .ok_or_else(|| err("pkt-line payload size overflow"))?; - if *total > MAX_PKT_LINE_PAYLOAD_BYTES { - return Err(err("pkt-line payload exceeds size limit")); - } - Ok(()) -} - -fn add_advertised_ref_count(total: usize) -> io::Result<()> { - if total > MAX_ADVERTISED_REFS { - return Err(err("remote advertised too many refs")); - } - Ok(()) -} - -fn response_body_preview(body: &[u8]) -> String { - let limit = body.len().min(MAX_HTTP_ERROR_BODY_BYTES); - let mut preview = String::from_utf8_lossy(&body[..limit]).trim().to_string(); - if body.len() > limit { - preview.push_str("..."); - } - preview -} - -fn normalize_repo_path(path: &str) -> io::Result { - if path.is_empty() || path.as_bytes().contains(&0) { - return Err(err("invalid empty or nul-containing repository path")); - } - - let mut parts = Vec::new(); - for component in Path::new(path).components() { - match component { - Component::Normal(part) if part == OsStr::new(".git") => { - return Err(err("repository path must not enter .git")); - } - Component::Normal(part) => { - let part = part - .to_str() - .ok_or_else(|| err("repository path must be utf-8"))?; - if part.is_empty() { - return Err(err("repository path contains an empty component")); - } - parts.push(part); - } - Component::CurDir - | Component::ParentDir - | Component::RootDir - | Component::Prefix(_) => { - return Err(err("repository path must be relative and normalized")); - } - } - } - - if parts.is_empty() { - return Err(err("repository path must name a file")); - } - - Ok(parts.join("/")) -} - -fn worktree_path(workdir: &Path, repo_path: &str) -> io::Result { - Ok(workdir.join(normalize_repo_path(repo_path)?)) -} - -fn validate_ref_tail(name: &str) -> io::Result<&str> { - if name.is_empty() - || name.as_bytes().contains(&0) - || name.starts_with('/') - || name.ends_with('/') - || name.contains('\\') - { - return Err(err("invalid git ref name")); - } - - for part in name.split('/') { - if part.is_empty() || part == "." || part == ".." { - return Err(err("invalid git ref name")); - } - } - - Ok(name) -} - -fn validate_branch_name(name: &str) -> io::Result<&str> { - validate_ref_tail(name) -} - -fn validate_refname(refname: &str) -> io::Result<&str> { - if refname == "HEAD" { - return Ok(refname); - } - - for prefix in ["refs/heads/", "refs/remotes/", "refs/tags/"] { - if let Some(tail) = refname.strip_prefix(prefix) { - validate_ref_tail(tail)?; - return Ok(refname); - } - } - - Err(err("unsupported git ref name")) -} - -fn hash_bytes(obj_type: &str, data: &[u8]) -> [u8; 20] { - let header = format!("{} {}\0", obj_type, data.len()); - let mut hasher = Sha1::new(); - hasher.update(header.as_bytes()); - hasher.update(data); - hasher.finalize().into() -} - -fn is_zero_oid(hash: &[u8; 20]) -> bool { - hash.iter().all(|b| *b == 0) -} - -fn is_remote_source(source: &str) -> bool { - source.starts_with("http://") || source.starts_with("https://") -} - -fn is_ssh_clone_source(source: &str) -> bool { - source.starts_with("git@") - || source.starts_with("ssh://") - || source.starts_with("git://") - || source.starts_with("ssh+git://") -} - -fn has_http_auth(source: &str) -> bool { - if !is_remote_source(source) { - return false; - } - - let Some(scheme_sep) = source.find("://") else { - return false; - }; - let authority = &source[scheme_sep + 3..] - .split_once('/') - .map(|(authority, _)| authority) - .unwrap_or(&source[scheme_sep + 3..]); - - authority.contains('@') -} - -fn infer_clone_destination(source: &str) -> io::Result { - let basename = if is_remote_source(source) || is_ssh_clone_source(source) { - let trimmed = source.trim_end_matches('/'); - let leaf = if let Some((_, scp_path)) = trimmed.rsplit_once(':') { - if is_ssh_clone_source(source) && !scp_path.contains('/') { - scp_path - } else { - trimmed - .rsplit('/') - .next() - .filter(|segment| !segment.is_empty()) - .ok_or_else(|| err("could not determine destination path from source"))? - } - } else { - trimmed - .rsplit('/') - .next() - .filter(|segment| !segment.is_empty()) - .ok_or_else(|| err("could not determine destination path from source"))? - }; - leaf.to_string() - } else { - Path::new(source) - .file_name() - .map(|name| name.to_string_lossy().to_string()) - .filter(|name| !name.is_empty()) - .ok_or_else(|| err("could not determine destination path from source"))? - }; - - Ok(basename - .strip_suffix(".git") - .unwrap_or(&basename) - .to_string()) -} - -fn prepare_clone_destination( - dest: &Path, - source_label: &str, - default_branch: &str, -) -> io::Result { - validate_branch_name(default_branch)?; - mkdirs(dest)?; - - let dst_git = dest.join(".git"); - mkdirs(&dst_git.join("objects"))?; - mkdirs(&dst_git.join("refs/heads"))?; - mkdirs(&dst_git.join("refs/tags"))?; - mkdirs(&dst_git.join("refs/remotes/origin"))?; - - fs::write( - dst_git.join("HEAD"), - format!("ref: refs/heads/{}\n", default_branch), - )?; - fs::write( - dst_git.join("config"), - format!( - "[core]\n\trepositoryformatversion = 0\n\tfilemode = true\n\tbare = false\n\ - [remote \"origin\"]\n\turl = {}\n\tfetch = +refs/heads/*:refs/remotes/origin/*\n", - source_label - ), - )?; - - Ok(dst_git) -} - -#[derive(Debug)] -enum PktLine { - Data(Vec), - Flush, -} - -fn parse_pkt_lines(data: &[u8]) -> io::Result> { - let mut pos = 0usize; - let mut lines = Vec::new(); - let mut payload_bytes = 0usize; - - while pos + 4 <= data.len() { - if lines.len() >= MAX_PKT_LINES { - return Err(err("too many pkt-lines")); - } - let len_str = - std::str::from_utf8(&data[pos..pos + 4]).map_err(|_| err("invalid pkt-line"))?; - let len = usize::from_str_radix(len_str, 16).map_err(|_| err("invalid pkt-line length"))?; - pos += 4; - - if len == 0 { - lines.push(PktLine::Flush); - continue; - } - if len < 4 || pos + len - 4 > data.len() { - return Err(err("truncated pkt-line")); - } - - let payload_len = len - 4; - add_pkt_payload_bytes(&mut payload_bytes, payload_len)?; - lines.push(PktLine::Data(data[pos..pos + payload_len].to_vec())); - pos += len - 4; - } - - if pos != data.len() { - return Err(err("trailing bytes after pkt-lines")); - } - - Ok(lines) -} - -#[derive(Debug, Default)] -struct RemoteAdvertisement { - head_hash: Option<[u8; 20]>, - head_target: Option, - branches: BTreeMap, - tags: BTreeMap, -} - -fn parse_advertised_ref(line: &[u8], adv: &mut RemoteAdvertisement) -> io::Result<()> { - let (ref_part, capability_part) = if let Some(nul) = line.iter().position(|b| *b == 0) { - (&line[..nul], Some(&line[nul + 1..])) - } else { - (line, None) - }; - - let ref_text = std::str::from_utf8(ref_part) - .map_err(|_| err("invalid advertised ref"))? - .trim_end_matches('\n'); - if ref_text.is_empty() { - return Ok(()); - } - - let (hash_hex, refname) = ref_text - .split_once(' ') - .ok_or_else(|| err("malformed advertised ref"))?; - let hash = unhex(hash_hex)?; - - if refname == "HEAD" { - adv.head_hash = Some(hash); - } else if refname.starts_with("refs/heads/") { - validate_refname(refname)?; - adv.branches.insert(refname.to_string(), hash); - add_advertised_ref_count(adv.branches.len() + adv.tags.len())?; - } else if refname.starts_with("refs/tags/") { - validate_refname(refname)?; - adv.tags.insert(refname.to_string(), hash); - add_advertised_ref_count(adv.branches.len() + adv.tags.len())?; - } - - if let Some(capabilities) = capability_part { - let caps = std::str::from_utf8(capabilities).map_err(|_| err("invalid capability list"))?; - for cap in caps.split_whitespace() { - if let Some(target) = cap.strip_prefix("symref=HEAD:") { - validate_refname(target)?; - adv.head_target = Some(target.to_string()); - } - } - } - - Ok(()) -} - -fn fetch_remote_advertisement(url: &str) -> io::Result { - let info_refs_url = format!( - "{}/info/refs?service=git-upload-pack", - url.trim_end_matches('/') - ); - let req = Request::new(Method::Get, &info_refs_url) - .map_err(|e| err(&format!("bad info/refs URL: {}", e)))? - .header("Accept", "application/x-git-upload-pack-advertisement") - .header("Git-Protocol", "version=0"); - - let resp = HttpClient::new() - .send(&req) - .map_err(|e| err(&format!("fetch info/refs failed: {}", e)))?; - if resp.status != 200 { - let body = response_body_preview(&resp.body); - return Err(err(&format!( - "remote advertised refs request failed (HTTP {}): {}", - resp.status, body - ))); - } - if resp.body.len() > MAX_REF_ADVERTISEMENT_BYTES { - return Err(err("remote advertised refs response exceeds size limit")); - } - - let lines = parse_pkt_lines(&resp.body)?; - let mut adv = RemoteAdvertisement::default(); - let mut saw_service = false; - let mut in_refs = false; - - for line in lines { - match line { - PktLine::Flush => { - if saw_service { - in_refs = true; - } - } - PktLine::Data(data) => { - if !saw_service { - let service = std::str::from_utf8(&data) - .map_err(|_| err("invalid git service header"))?; - if service.trim_end_matches('\n') != "# service=git-upload-pack" { - return Err(err("unexpected git service advertisement")); - } - saw_service = true; - continue; - } - if !in_refs { - continue; - } - parse_advertised_ref(&data, &mut adv)?; - } - } - } - - if !saw_service { - return Err(err("missing git-upload-pack service header")); - } - - Ok(adv) -} - -fn branch_name_from_ref(refname: &str) -> io::Result { - refname - .strip_prefix("refs/heads/") - .map(validate_branch_name) - .transpose()? - .map(|name| name.to_string()) - .ok_or_else(|| err("expected refs/heads/* ref")) -} - -fn default_branch_ref(adv: &RemoteAdvertisement) -> Option<(String, [u8; 20])> { - if let Some(target) = adv.head_target.as_ref() { - if let Some(hash) = adv.branches.get(target) { - return Some((target.clone(), *hash)); - } - } - - if let Some(head_hash) = adv.head_hash { - if let Some((name, hash)) = adv.branches.iter().find(|(_, hash)| **hash == head_hash) { - return Some((name.clone(), *hash)); - } - } - - adv.branches - .iter() - .next() - .map(|(name, hash)| (name.clone(), *hash)) -} - -fn pkt_line_bytes(payload: &[u8]) -> Vec { - let mut out = format!("{:04x}", payload.len() + 4).into_bytes(); - out.extend_from_slice(payload); - out -} - -fn fetch_remote_pack(url: &str, wants: &[String]) -> io::Result> { - if wants.is_empty() { - return Ok(Vec::new()); - } - - let upload_pack_url = format!("{}/git-upload-pack", url.trim_end_matches('/')); - let mut body = Vec::new(); - for (i, want) in wants.iter().enumerate() { - let line = if i == 0 { - format!( - "want {} side-band-64k ofs-delta no-progress include-tag agent=agentos/1.0\n", - want - ) - } else { - format!("want {}\n", want) - }; - body.extend_from_slice(&pkt_line_bytes(line.as_bytes())); - } - body.extend_from_slice(b"0000"); - body.extend_from_slice(&pkt_line_bytes(b"done\n")); - - let req = Request::new(Method::Post, &upload_pack_url) - .map_err(|e| err(&format!("bad upload-pack URL: {}", e)))? - .header("Content-Type", "application/x-git-upload-pack-request") - .header("Accept", "application/x-git-upload-pack-result") - .header("Git-Protocol", "version=0") - .body(body); - - let resp = HttpClient::new() - .send(&req) - .map_err(|e| err(&format!("git-upload-pack failed: {}", e)))?; - if resp.status != 200 { - let body = response_body_preview(&resp.body); - return Err(err(&format!( - "git-upload-pack returned HTTP {}: {}", - resp.status, body - ))); - } - - if resp.body.len() > MAX_PACK_BYTES { - return Err(err("packfile exceeds size limit")); - } - if resp.body.starts_with(b"PACK") { - return Ok(resp.body); - } - - let mut pack = Vec::new(); - for line in parse_pkt_lines(&resp.body)? { - match line { - PktLine::Flush => {} - PktLine::Data(payload) => { - if payload == b"NAK\n" || payload.starts_with(b"ACK ") { - continue; - } - if payload.starts_with(b"ERR ") { - let msg = String::from_utf8_lossy(&payload[4..]); - return Err(err(&format!("remote upload-pack error: {}", msg.trim()))); - } - if payload.starts_with(b"PACK") { - append_pack_bytes(&mut pack, &payload)?; - continue; - } - if payload.is_empty() { - continue; - } - match payload[0] { - 1 => append_pack_bytes(&mut pack, &payload[1..])?, - 2 => {} - 3 => { - let msg = String::from_utf8_lossy(&payload[1..]); - return Err(err(&format!("remote upload-pack error: {}", msg.trim()))); - } - _ => return Err(err("unexpected upload-pack response payload")), - } - } - } - } - - if pack.is_empty() { - return Err(err("git-upload-pack response did not include a packfile")); - } - Ok(pack) -} - -#[derive(Clone, Debug)] -enum PackedObjectKind { - Full { obj_type: String, data: Vec }, - OfsDelta { base_offset: usize, delta: Vec }, - RefDelta { base_hash: [u8; 20], delta: Vec }, -} - -#[derive(Clone, Debug)] -struct PackedObject { - offset: usize, - kind: PackedObjectKind, -} - -#[derive(Clone, Debug)] -struct ResolvedObject { - obj_type: String, - data: Vec, - hash: [u8; 20], -} - -fn parse_pack_object_header(pack: &[u8], offset: &mut usize) -> io::Result<(u8, usize)> { - if *offset >= pack.len() { - return Err(err("truncated pack object header")); - } - - let mut byte = pack[*offset]; - *offset += 1; - - let obj_type = (byte >> 4) & 0x7; - let mut size = (byte & 0x0f) as usize; - let mut shift = 4usize; - - while byte & 0x80 != 0 { - if *offset >= pack.len() { - return Err(err("truncated pack object size")); - } - byte = pack[*offset]; - *offset += 1; - if shift >= usize::BITS as usize { - return Err(err("pack object size is too large")); - } - size |= ((byte & 0x7f) as usize) - .checked_shl(shift as u32) - .ok_or_else(|| err("pack object size is too large"))?; - shift += 7; - } - - Ok((obj_type, size)) -} - -fn parse_ofs_delta_base( - pack: &[u8], - offset: &mut usize, - object_offset: usize, -) -> io::Result { - if *offset >= pack.len() { - return Err(err("truncated ofs-delta base")); - } - - let mut byte = pack[*offset]; - *offset += 1; - let mut distance = (byte & 0x7f) as usize; - - while byte & 0x80 != 0 { - if *offset >= pack.len() { - return Err(err("truncated ofs-delta base")); - } - byte = pack[*offset]; - *offset += 1; - distance = distance - .checked_add(1) - .and_then(|value| value.checked_shl(7)) - .map(|value| value | ((byte & 0x7f) as usize)) - .ok_or_else(|| err("ofs-delta base distance is too large"))?; - } - - object_offset - .checked_sub(distance) - .ok_or_else(|| err("invalid ofs-delta base distance")) -} - -fn inflate_pack_stream(data: &[u8], expected_size: usize) -> io::Result<(Vec, usize)> { - if expected_size > MAX_GIT_OBJECT_BYTES { - return Err(err("pack object exceeds size limit")); - } - let cursor = Cursor::new(data); - let mut decoder = BufZlibDecoder::new(cursor); - let out = read_to_end_limited(&mut decoder, expected_size, "pack object")?; - if out.len() != expected_size { - return Err(err("pack object size mismatch")); - } - let consumed = decoder.get_ref().position() as usize; - Ok((out, consumed)) -} - -fn parse_packfile(pack: &[u8]) -> io::Result> { - if pack.len() < 12 + 20 { - return Err(err("packfile too small")); - } - if &pack[..4] != b"PACK" { - return Err(err("invalid packfile signature")); - } - - let version = u32::from_be_bytes(pack[4..8].try_into().unwrap()); - if version != 2 && version != 3 { - return Err(err(&format!("unsupported packfile version {}", version))); - } - let object_count = u32::from_be_bytes(pack[8..12].try_into().unwrap()) as usize; - let pack_end = pack.len() - 20; - let mut offset = 12usize; - let max_count_by_bytes = pack_end.saturating_sub(offset); - if object_count > MAX_PACK_OBJECTS || object_count > max_count_by_bytes { - return Err(err("packfile object count is too large")); - } - let mut objects = Vec::new(); - try_reserve_exact(&mut objects, object_count, "pack object table")?; - let mut inflated_bytes = 0usize; - - for _ in 0..object_count { - if offset >= pack_end { - return Err(err("truncated packfile")); - } - - let object_offset = offset; - let (obj_type, object_size) = parse_pack_object_header(pack, &mut offset)?; - - let kind = match obj_type { - 1 | 2 | 3 | 4 => { - let obj_type = match obj_type { - 1 => "commit", - 2 => "tree", - 3 => "blob", - 4 => "tag", - _ => unreachable!(), - }; - let (data, consumed) = inflate_pack_stream(&pack[offset..pack_end], object_size)?; - offset += consumed; - add_pack_inflated_bytes(&mut inflated_bytes, data.len())?; - PackedObjectKind::Full { - obj_type: obj_type.to_string(), - data, - } - } - 6 => { - let base_offset = parse_ofs_delta_base(pack, &mut offset, object_offset)?; - let (delta, consumed) = inflate_pack_stream(&pack[offset..pack_end], object_size)?; - offset += consumed; - add_pack_inflated_bytes(&mut inflated_bytes, delta.len())?; - PackedObjectKind::OfsDelta { base_offset, delta } - } - 7 => { - if offset + 20 > pack_end { - return Err(err("truncated ref-delta base")); - } - let mut base_hash = [0u8; 20]; - base_hash.copy_from_slice(&pack[offset..offset + 20]); - offset += 20; - let (delta, consumed) = inflate_pack_stream(&pack[offset..pack_end], object_size)?; - offset += consumed; - add_pack_inflated_bytes(&mut inflated_bytes, delta.len())?; - PackedObjectKind::RefDelta { base_hash, delta } - } - _ => return Err(err(&format!("unsupported pack object type {}", obj_type))), - }; - - objects.push(PackedObject { - offset: object_offset, - kind, - }); - } - - Ok(objects) -} - -fn read_delta_varint(data: &[u8], pos: &mut usize) -> io::Result { - let mut value = 0usize; - let mut shift = 0usize; - - loop { - if *pos >= data.len() { - return Err(err("truncated delta header")); - } - let byte = data[*pos]; - *pos += 1; - - if shift >= usize::BITS as usize { - return Err(err("delta varint is too large")); - } - value |= ((byte & 0x7f) as usize) - .checked_shl(shift as u32) - .ok_or_else(|| err("delta varint is too large"))?; - if byte & 0x80 == 0 { - return Ok(value); - } - shift += 7; - } -} - -fn ensure_delta_output_room( - current_len: usize, - additional_len: usize, - result_size: usize, -) -> io::Result<()> { - let next_len = current_len - .checked_add(additional_len) - .ok_or_else(|| err("delta result size overflow"))?; - if next_len > result_size { - return Err(err("delta result exceeds declared size")); - } - Ok(()) -} - -fn apply_delta(base: &[u8], delta: &[u8]) -> io::Result> { - let mut pos = 0usize; - let base_size = read_delta_varint(delta, &mut pos)?; - if base_size != base.len() { - return Err(err("delta base size mismatch")); - } - let result_size = read_delta_varint(delta, &mut pos)?; - if result_size > MAX_GIT_OBJECT_BYTES { - return Err(err("delta result exceeds size limit")); - } - let mut out = Vec::new(); - try_reserve_exact(&mut out, result_size, "delta result")?; - - while pos < delta.len() { - let opcode = delta[pos]; - pos += 1; - - if opcode & 0x80 != 0 { - let mut copy_offset = 0usize; - let mut copy_size = 0usize; - - if opcode & 0x01 != 0 { - copy_offset |= delta - .get(pos) - .copied() - .ok_or_else(|| err("truncated delta copy"))? - as usize; - pos += 1; - } - if opcode & 0x02 != 0 { - copy_offset |= (delta - .get(pos) - .copied() - .ok_or_else(|| err("truncated delta copy"))? - as usize) - << 8; - pos += 1; - } - if opcode & 0x04 != 0 { - copy_offset |= (delta - .get(pos) - .copied() - .ok_or_else(|| err("truncated delta copy"))? - as usize) - << 16; - pos += 1; - } - if opcode & 0x08 != 0 { - copy_offset |= (delta - .get(pos) - .copied() - .ok_or_else(|| err("truncated delta copy"))? - as usize) - << 24; - pos += 1; - } - - if opcode & 0x10 != 0 { - copy_size |= delta - .get(pos) - .copied() - .ok_or_else(|| err("truncated delta copy"))? - as usize; - pos += 1; - } - if opcode & 0x20 != 0 { - copy_size |= (delta - .get(pos) - .copied() - .ok_or_else(|| err("truncated delta copy"))? - as usize) - << 8; - pos += 1; - } - if opcode & 0x40 != 0 { - copy_size |= (delta - .get(pos) - .copied() - .ok_or_else(|| err("truncated delta copy"))? - as usize) - << 16; - pos += 1; - } - if copy_size == 0 { - copy_size = 0x10000; - } - - let end = copy_offset - .checked_add(copy_size) - .ok_or_else(|| err("delta copy overflow"))?; - if end > base.len() { - return Err(err("delta copy exceeds base object")); - } - ensure_delta_output_room(out.len(), copy_size, result_size)?; - out.extend_from_slice(&base[copy_offset..end]); - } else if opcode != 0 { - let insert_len = opcode as usize; - let end = pos - .checked_add(insert_len) - .ok_or_else(|| err("delta insert overflow"))?; - if end > delta.len() { - return Err(err("truncated delta insert")); - } - ensure_delta_output_room(out.len(), insert_len, result_size)?; - out.extend_from_slice(&delta[pos..end]); - pos = end; - } else { - return Err(err("invalid delta opcode")); - } - } - - if out.len() != result_size { - return Err(err("delta result size mismatch")); - } - - Ok(out) -} - -fn maybe_read_local_object(git_dir: &Path, hash: &[u8; 20]) -> io::Result> { - let h = hex(hash); - let path = git_dir.join("objects").join(&h[..2]).join(&h[2..]); - if !path.exists() { - return Ok(None); - } - let (obj_type, data) = read_object(git_dir, hash)?; - Ok(Some(ResolvedObject { - obj_type, - data, - hash: *hash, - })) -} - -fn find_entry_by_hash( - target: &[u8; 20], - git_dir: &Path, - objects: &[PackedObject], - offset_to_index: &HashMap, - memo: &mut [Option], - visiting: &mut [bool], - resolved_bytes: &mut usize, -) -> io::Result> { - for idx in 0..objects.len() { - if visiting[idx] { - continue; - } - let resolved = resolve_packed_object( - idx, - git_dir, - objects, - offset_to_index, - memo, - visiting, - resolved_bytes, - )?; - if resolved.hash == *target { - return Ok(Some(idx)); - } - } - - Ok(None) -} - -fn resolve_packed_object( - idx: usize, - git_dir: &Path, - objects: &[PackedObject], - offset_to_index: &HashMap, - memo: &mut [Option], - visiting: &mut [bool], - resolved_bytes: &mut usize, -) -> io::Result { - if let Some(resolved) = memo[idx].as_ref() { - return Ok(resolved.clone()); - } - if visiting[idx] { - return Err(err("cyclic pack delta dependency")); - } - visiting[idx] = true; - - let resolved = match &objects[idx].kind { - PackedObjectKind::Full { obj_type, data } => ResolvedObject { - obj_type: obj_type.clone(), - data: data.clone(), - hash: hash_bytes(obj_type, data), - }, - PackedObjectKind::OfsDelta { base_offset, delta } => { - let base_idx = *offset_to_index - .get(base_offset) - .ok_or_else(|| err("missing ofs-delta base object"))?; - let base = resolve_packed_object( - base_idx, - git_dir, - objects, - offset_to_index, - memo, - visiting, - resolved_bytes, - )?; - let data = apply_delta(&base.data, delta)?; - let hash = hash_bytes(&base.obj_type, &data); - ResolvedObject { - obj_type: base.obj_type, - data, - hash, - } - } - PackedObjectKind::RefDelta { base_hash, delta } => { - let base = if let Some(local) = maybe_read_local_object(git_dir, base_hash)? { - local - } else if let Some(base_idx) = find_entry_by_hash( - base_hash, - git_dir, - objects, - offset_to_index, - memo, - visiting, - resolved_bytes, - )? { - resolve_packed_object( - base_idx, - git_dir, - objects, - offset_to_index, - memo, - visiting, - resolved_bytes, - )? - } else { - return Err(err("missing ref-delta base object")); - }; - - let data = apply_delta(&base.data, delta)?; - let hash = hash_bytes(&base.obj_type, &data); - ResolvedObject { - obj_type: base.obj_type, - data, - hash, - } - } - }; - - visiting[idx] = false; - add_pack_resolved_bytes(resolved_bytes, resolved.data.len())?; - memo[idx] = Some(resolved.clone()); - Ok(resolved) -} - -fn store_pack_objects(git_dir: &Path, pack: &[u8]) -> io::Result<()> { - if pack.is_empty() { - return Ok(()); - } - - let objects = parse_packfile(pack)?; - let offset_to_index: HashMap = objects - .iter() - .enumerate() - .map(|(idx, obj)| (obj.offset, idx)) - .collect(); - let mut memo: Vec> = vec![None; objects.len()]; - let mut visiting = vec![false; objects.len()]; - let mut resolved_bytes = 0usize; - - for idx in 0..objects.len() { - let resolved = resolve_packed_object( - idx, - git_dir, - &objects, - &offset_to_index, - &mut memo, - &mut visiting, - &mut resolved_bytes, - )?; - let stored = hash_object(git_dir, &resolved.obj_type, &resolved.data)?; - if stored != resolved.hash { - return Err(err("pack object hash mismatch")); - } - } - Ok(()) -} - -fn cmd_clone_remote(source: &str, dest: &Path) -> io::Result<()> { - if dest.exists() && !dir_is_empty(dest)? { - return Err(err(&format!( - "destination path '{}' already exists and is not an empty directory", - dest.display() - ))); - } - - let advertisement = fetch_remote_advertisement(source)?; - let mut wants: Vec = Vec::new(); - let mut seen = HashSet::new(); - for hash in advertisement.branches.values() { - if !is_zero_oid(hash) { - let hash_hex = hex(hash); - if seen.insert(hash_hex.clone()) { - wants.push(hash_hex); - } - } - } - if wants.is_empty() { - if let Some(head_hash) = advertisement.head_hash { - if !is_zero_oid(&head_hash) { - wants.push(hex(&head_hash)); - } - } - } - - let default_ref = if let Some((refname, hash)) = default_branch_ref(&advertisement) { - Some((refname.clone(), branch_name_from_ref(&refname)?, hash)) - } else { - None - }; - let default_branch = default_ref - .as_ref() - .map(|(_, branch, _)| branch.clone()) - .or_else(|| { - advertisement - .head_target - .as_ref() - .and_then(|target| branch_name_from_ref(target).ok()) - }) - .unwrap_or_else(|| "main".to_string()); - - let pack = fetch_remote_pack(source, &wants)?; - - let dst_git = prepare_clone_destination(dest, source, &default_branch)?; - store_pack_objects(&dst_git, &pack)?; - - for (refname, hash) in &advertisement.branches { - if is_zero_oid(hash) { - continue; - } - let branch = branch_name_from_ref(refname)?; - update_ref(&dst_git, &format!("refs/remotes/origin/{}", branch), hash)?; - } - - for (refname, hash) in &advertisement.tags { - if is_zero_oid(hash) { - continue; - } - update_ref(&dst_git, refname, hash)?; - } - - let default_hash = default_ref - .as_ref() - .map(|(_, _, hash)| *hash) - .or(advertisement.head_hash) - .filter(|hash| !is_zero_oid(hash)); - - if let Some(hash) = default_hash { - update_ref(&dst_git, &format!("refs/heads/{}", default_branch), &hash)?; - cmd_checkout(dest, &default_branch, false)?; - } - - Ok(()) -} - -// ─── Object store ─────────────────────────────────────────────────────────── - -fn hash_object(git_dir: &Path, obj_type: &str, data: &[u8]) -> io::Result<[u8; 20]> { - if data.len() > MAX_GIT_OBJECT_BYTES { - return Err(err("git object exceeds size limit")); - } - let header = format!("{} {}\0", obj_type, data.len()); - let mut hasher = Sha1::new(); - hasher.update(header.as_bytes()); - hasher.update(data); - let hash: [u8; 20] = hasher.finalize().into(); - - let h = hex(&hash); - let dir = git_dir.join("objects").join(&h[..2]); - let path = dir.join(&h[2..]); - if !path.exists() { - mkdirs(&dir)?; - let mut enc = ZlibEncoder::new(Vec::new(), Compression::default()); - enc.write_all(header.as_bytes())?; - enc.write_all(data)?; - fs::write(&path, enc.finish()?)?; - } - Ok(hash) -} - -fn read_object(git_dir: &Path, hash: &[u8; 20]) -> io::Result<(String, Vec)> { - let h = hex(hash); - let path = git_dir.join("objects").join(&h[..2]).join(&h[2..]); - let read_limit = MAX_GIT_OBJECT_BYTES - .checked_add(MAX_GIT_OBJECT_HEADER_BYTES) - .ok_or_else(|| err("git object size limit is too large"))?; - let compressed = read_file_limited(&path, read_limit, "git object")?; - let mut dec = ZlibDecoder::new(&compressed[..]); - let buf = read_to_end_limited(&mut dec, read_limit, "git object")?; - - let nul = buf - .iter() - .position(|&b| b == 0) - .ok_or_else(|| err("no nul in object"))?; - let header = - std::str::from_utf8(&buf[..nul]).map_err(|_| err("invalid object header encoding"))?; - let (typ, size) = header - .split_once(' ') - .ok_or_else(|| err("malformed object header"))?; - let size: usize = size.parse().map_err(|_| err("invalid object size"))?; - if size > MAX_GIT_OBJECT_BYTES { - return Err(err("git object exceeds size limit")); - } - if buf.len() - nul - 1 != size { - return Err(err("git object size mismatch")); - } - Ok((typ.to_string(), buf[nul + 1..].to_vec())) -} - -// ─── Index ────────────────────────────────────────────────────────────────── - -#[derive(Clone, Debug)] -struct IndexEntry { - mode: u32, - sha1: [u8; 20], - name: String, -} - -fn read_index(git_dir: &Path) -> io::Result> { - let path = git_dir.join("index"); - if !path.exists() { - return Ok(Vec::new()); - } - let data = read_file_limited(&path, MAX_INDEX_BYTES, "git index")?; - if data.len() < 12 || &data[0..4] != b"DIRC" { - return Err(err("invalid index file")); - } - let version = u32::from_be_bytes(data[4..8].try_into().unwrap()); - if version != 2 { - return Err(err(&format!("unsupported index version {}", version))); - } - let count = u32::from_be_bytes(data[8..12].try_into().unwrap()) as usize; - let max_count_by_bytes = data.len().saturating_sub(12) / 62; - if count > MAX_INDEX_ENTRIES || count > max_count_by_bytes { - return Err(err("index entry count is too large")); - } - - let mut entries = Vec::new(); - try_reserve_exact(&mut entries, count, "index entry table")?; - let mut pos = 12; - - for _ in 0..count { - if pos + 62 > data.len() { - return Err(err("truncated index")); - } - // Stat fields at pos+0..pos+24 (ctime, mtime, dev, ino) - skip - let mode = u32::from_be_bytes(data[pos + 24..pos + 28].try_into().unwrap()); - // Skip uid(4), gid(4), size(4) at pos+28..pos+40 - let mut sha1 = [0u8; 20]; - sha1.copy_from_slice(&data[pos + 40..pos + 60]); - // Flags at pos+60..pos+62 - // Find name (NUL-terminated after fixed 62 bytes) - let name_start = pos + 62; - let nul_offset = data[name_start..] - .iter() - .position(|&b| b == 0) - .ok_or_else(|| err("unterminated index entry name"))?; - let name = String::from_utf8(data[name_start..name_start + nul_offset].to_vec()) - .map_err(|_| err("invalid entry name"))?; - let name = normalize_repo_path(&name)?; - - entries.push(IndexEntry { mode, sha1, name }); - - // Advance past padding: entry padded to 8-byte boundary - let entry_len = 62 + nul_offset; - pos += (entry_len + 8) & !7; - } - - Ok(entries) -} - -fn write_index(git_dir: &Path, entries: &[IndexEntry]) -> io::Result<()> { - let entry_count = u32::try_from(entries.len()).map_err(|_| err("too many index entries"))?; - let mut buf = Vec::new(); - buf.extend_from_slice(b"DIRC"); - buf.extend_from_slice(&2u32.to_be_bytes()); - buf.extend_from_slice(&entry_count.to_be_bytes()); - - for entry in entries { - let name = normalize_repo_path(&entry.name)?; - if name.len() > 0xFFF { - return Err(err("index entry name is too long")); - } - let entry_start = buf.len(); - // ctime(8) + mtime(8) + dev(4) + ino(4) = 24 bytes of zeros - buf.extend_from_slice(&[0u8; 24]); - buf.extend_from_slice(&entry.mode.to_be_bytes()); - // uid(4) + gid(4) + size(4) = 12 bytes of zeros - buf.extend_from_slice(&[0u8; 12]); - buf.extend_from_slice(&entry.sha1); - // Flags: name length in lower 12 bits - buf.extend_from_slice(&(name.len() as u16).to_be_bytes()); - buf.extend_from_slice(name.as_bytes()); - // Pad to 8-byte boundary (1-8 NUL bytes) - let entry_len = buf.len() - entry_start; - let padded = (entry_len + 8) & !7; - buf.resize(entry_start + padded, 0); - } - - // SHA-1 checksum of entire index - let checksum: [u8; 20] = Sha1::digest(&buf).into(); - buf.extend_from_slice(&checksum); - - fs::write(git_dir.join("index"), &buf) -} - -// ─── Refs ─────────────────────────────────────────────────────────────────── - -fn resolve_ref(git_dir: &Path, refname: &str) -> io::Result> { - let refname = validate_refname(refname)?; - let ref_path = git_dir.join(refname); - if !ref_path.exists() { - return Ok(None); - } - let content = fs::read_to_string(&ref_path)?; - let content = content.trim(); - if let Some(target) = content.strip_prefix("ref: ") { - resolve_ref(git_dir, target) - } else { - Ok(Some(unhex(content)?)) - } -} - -fn resolve_head(git_dir: &Path) -> io::Result> { - resolve_ref(git_dir, "HEAD") -} - -fn head_branch(git_dir: &Path) -> io::Result> { - let head = fs::read_to_string(git_dir.join("HEAD"))?; - let head = head.trim(); - Ok(head.strip_prefix("ref: refs/heads/").map(|s| s.to_string())) -} - -fn update_ref(git_dir: &Path, refname: &str, hash: &[u8; 20]) -> io::Result<()> { - let refname = validate_refname(refname)?; - let ref_path = git_dir.join(refname); - if let Some(parent) = ref_path.parent() { - mkdirs(parent)?; - } - fs::write(&ref_path, format!("{}\n", hex(hash))) -} - -// ─── Tree operations ──────────────────────────────────────────────────────── - -fn build_tree(git_dir: &Path, entries: &[IndexEntry]) -> io::Result<[u8; 20]> { - build_tree_at(git_dir, entries, "") -} - -fn build_tree_at(git_dir: &Path, entries: &[IndexEntry], prefix: &str) -> io::Result<[u8; 20]> { - struct TreeEntry { - mode: String, - name: String, - hash: [u8; 20], - } - - let mut tree_entries: Vec = Vec::new(); - let mut subdirs: BTreeMap> = BTreeMap::new(); - - for entry in entries { - let relative = if prefix.is_empty() { - &entry.name[..] - } else if let Some(rest) = entry.name.strip_prefix(prefix) { - rest - } else { - continue; - }; - - if let Some(slash) = relative.find('/') { - let dir = &relative[..slash]; - subdirs - .entry(dir.to_string()) - .or_default() - .push(entry.clone()); - } else { - tree_entries.push(TreeEntry { - mode: format!("{:o}", entry.mode), - name: relative.to_string(), - hash: entry.sha1, - }); - } - } - - for (dir, sub_entries) in &subdirs { - let sub_prefix = if prefix.is_empty() { - format!("{}/", dir) - } else { - format!("{}{}/", prefix, dir) - }; - let subtree_hash = build_tree_at(git_dir, sub_entries, &sub_prefix)?; - tree_entries.push(TreeEntry { - mode: "40000".to_string(), - name: dir.clone(), - hash: subtree_hash, - }); - } - - // Sort: directories get trailing / for comparison - tree_entries.sort_by(|a, b| { - let ak = if a.mode == "40000" { - format!("{}/", a.name) - } else { - a.name.clone() - }; - let bk = if b.mode == "40000" { - format!("{}/", b.name) - } else { - b.name.clone() - }; - ak.cmp(&bk) - }); - - let mut data = Vec::new(); - for te in &tree_entries { - data.extend_from_slice(te.mode.as_bytes()); - data.push(b' '); - data.extend_from_slice(te.name.as_bytes()); - data.push(0); - data.extend_from_slice(&te.hash); - } - - hash_object(git_dir, "tree", &data) -} - -/// Read a tree recursively into flat (path, mode, hash) entries. -fn read_tree_entries(git_dir: &Path, hash: &[u8; 20], prefix: &str) -> io::Result> { - let (typ, data) = read_object(git_dir, hash)?; - if typ != "tree" { - return Err(err("expected tree object")); - } - - let mut entries = Vec::new(); - let mut pos = 0; - - while pos < data.len() { - let space = data[pos..] - .iter() - .position(|&b| b == b' ') - .ok_or_else(|| err("bad tree entry"))?; - let mode_str = - std::str::from_utf8(&data[pos..pos + space]).map_err(|_| err("bad tree mode"))?; - let mode = u32::from_str_radix(mode_str, 8).map_err(|_| err("bad tree mode number"))?; - pos += space + 1; - - let nul = data[pos..] - .iter() - .position(|&b| b == 0) - .ok_or_else(|| err("bad tree entry name"))?; - let name = std::str::from_utf8(&data[pos..pos + nul]).map_err(|_| err("bad name"))?; - if name.contains('/') { - return Err(err("tree entry name must not contain '/'")); - } - if normalize_repo_path(name)? != name { - return Err(err("tree entry name must be normalized")); - } - pos += nul + 1; - - if pos + 20 > data.len() { - return Err(err("truncated tree hash")); - } - let mut hash_buf = [0u8; 20]; - hash_buf.copy_from_slice(&data[pos..pos + 20]); - pos += 20; - - let full_name = if prefix.is_empty() { - name.to_string() - } else { - format!("{}{}", prefix, name) - }; - - if mode == 0o40000 { - let sub = read_tree_entries(git_dir, &hash_buf, &format!("{}/", full_name))?; - entries.extend(sub); - } else { - let full_name = normalize_repo_path(&full_name)?; - entries.push(IndexEntry { - mode, - sha1: hash_buf, - name: full_name, - }); - } - } - - Ok(entries) -} - -/// Extract the tree hash from a commit object. -fn commit_tree(git_dir: &Path, commit_hash: &[u8; 20]) -> io::Result<[u8; 20]> { - let (typ, data) = read_object(git_dir, commit_hash)?; - if typ != "commit" { - return Err(err("not a commit object")); - } - let text = String::from_utf8_lossy(&data); - let tree_hex = text - .lines() - .find_map(|l| l.strip_prefix("tree ")) - .ok_or_else(|| err("no tree line in commit"))?; - unhex(tree_hex) -} - -// ─── Commands ─────────────────────────────────────────────────────────────── - -fn cmd_init(path: &Path, quiet: bool) -> io::Result<()> { - let git_dir = path.join(".git"); - // Create directories one at a time to avoid create_dir_all issues on WASI - for dir in &[ - path.to_path_buf(), - git_dir.clone(), - git_dir.join("objects"), - git_dir.join("refs"), - git_dir.join("refs/heads"), - git_dir.join("refs/tags"), - ] { - match fs::create_dir(dir) { - Ok(()) => {} - Err(e) if e.kind() == io::ErrorKind::AlreadyExists => {} - Err(e) => return Err(err(&format!("mkdir {}: {}", dir.display(), e))), - } - } - fs::write(git_dir.join("HEAD"), "ref: refs/heads/main\n") - .map_err(|e| err(&format!("write HEAD: {}", e)))?; - fs::write( - git_dir.join("config"), - "[core]\n\trepositoryformatversion = 0\n\tfilemode = true\n\tbare = false\n", - ) - .map_err(|e| err(&format!("write config: {}", e)))?; - if !quiet { - print_stdout_line(format_args!( - "Initialized empty Git repository in {}/.git/", - path.display() - ))?; - } - Ok(()) -} - -fn collect_add_paths(workdir: &Path, rel_path: &str, out: &mut Vec) -> io::Result<()> { - if rel_path == ".git" || rel_path.starts_with(".git/") || rel_path.ends_with("/.git") { - return Ok(()); - } - - if rel_path == "." { - for entry in fs::read_dir(workdir)? { - let entry = entry?; - let name = entry - .file_name() - .to_str() - .ok_or_else(|| err("repository path must be utf-8"))? - .to_owned(); - if name == ".git" { - continue; - } - collect_add_paths(workdir, &name, out)?; - } - return Ok(()); - } - - let repo_path = normalize_repo_path(rel_path)?; - let file_path = workdir.join(&repo_path); - if file_path.is_dir() { - for entry in fs::read_dir(&file_path)? { - let entry = entry?; - let name = entry - .file_name() - .to_str() - .ok_or_else(|| err("repository path must be utf-8"))? - .to_owned(); - if name == ".git" { - continue; - } - collect_add_paths(workdir, &format!("{repo_path}/{name}"), out)?; - } - return Ok(()); - } - - if file_path.exists() { - out.push(repo_path); - } - Ok(()) -} - -fn cmd_add(workdir: &Path, paths: &[String]) -> io::Result<()> { - let git_dir = workdir.join(".git"); - let mut entries = read_index(&git_dir).map_err(|e| { - err(&format!( - "cannot read index at {}: {}", - git_dir.display(), - e - )) - })?; - - for rel_path in paths { - let mut repo_paths = Vec::new(); - collect_add_paths(workdir, rel_path, &mut repo_paths)?; - if repo_paths.is_empty() { - return Err(err(&format!( - "pathspec '{}' did not match any files", - rel_path - ))); - } - for repo_path in repo_paths { - let file_path = workdir.join(&repo_path); - if !file_path.exists() { - return Err(err(&format!( - "pathspec '{}' did not match any files (looked at {})", - rel_path, - file_path.display() - ))); - } - let content = read_file_limited(&file_path, MAX_GIT_OBJECT_BYTES, "file")?; - let hash = hash_object(&git_dir, "blob", &content)?; - - entries.retain(|e| e.name != repo_path); - entries.push(IndexEntry { - mode: 0o100644, - sha1: hash, - name: repo_path, - }); - } - } - - entries.sort_by(|a, b| a.name.cmp(&b.name)); - write_index(&git_dir, &entries) -} - -fn cmd_commit(workdir: &Path, message: &str) -> io::Result<()> { - let git_dir = workdir.join(".git"); - let entries = read_index(&git_dir)?; - if entries.is_empty() { - return Err(err("nothing to commit")); - } - - let tree_hash = build_tree(&git_dir, &entries)?; - let parent = resolve_head(&git_dir)?; - - let name = std::env::var("GIT_AUTHOR_NAME") - .or_else(|_| std::env::var("GIT_COMMITTER_NAME")) - .unwrap_or_else(|_| "secure-exec".to_string()); - let email = std::env::var("GIT_AUTHOR_EMAIL") - .or_else(|_| std::env::var("GIT_COMMITTER_EMAIL")) - .unwrap_or_else(|_| "agent@os".to_string()); - let timestamp = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_secs()) - .unwrap_or(1700000000); - - let mut commit = String::new(); - commit.push_str(&format!("tree {}\n", hex(&tree_hash))); - if let Some(p) = parent { - commit.push_str(&format!("parent {}\n", hex(&p))); - } - commit.push_str(&format!( - "author {} <{}> {} +0000\n", - name, email, timestamp - )); - commit.push_str(&format!( - "committer {} <{}> {} +0000\n", - name, email, timestamp - )); - commit.push_str(&format!("\n{}\n", message)); - - let commit_hash = hash_object(&git_dir, "commit", commit.as_bytes())?; - - // Update the ref that HEAD points to - let head_content = fs::read_to_string(git_dir.join("HEAD"))?; - let head_content = head_content.trim(); - if let Some(refname) = head_content.strip_prefix("ref: ") { - update_ref(&git_dir, refname, &commit_hash)?; - } else { - fs::write(git_dir.join("HEAD"), format!("{}\n", hex(&commit_hash)))?; - } - - Ok(()) -} - -fn cmd_rev_parse(workdir: &Path, args: &[String]) -> io::Result<()> { - let git_dir = workdir.join(".git"); - if args.len() != 1 { - return Err(err("usage: git rev-parse ")); - } - - let hash = resolve_ref(&git_dir, &args[0])?.ok_or_else(|| { - err(&format!( - "unknown revision or path not in the working tree: {}", - args[0] - )) - })?; - print_stdout_line(format_args!("{}", hex(&hash))) -} - -fn cmd_branch(workdir: &Path) -> io::Result<()> { - let git_dir = workdir.join(".git"); - let heads_dir = git_dir.join("refs/heads"); - let current = head_branch(&git_dir)?; - - let mut branches: Vec = Vec::new(); - if heads_dir.exists() { - for entry in fs::read_dir(&heads_dir)? { - let entry = entry?; - let name = entry.file_name().to_string_lossy().to_string(); - if name == "." || name == ".." { - continue; - } - if entry.file_type()?.is_file() { - branches.push(name); - } - } - } - branches.sort(); - - for branch in &branches { - if Some(branch.as_str()) == current.as_deref() { - print_stdout_line(format_args!("* {}", branch))?; - } else { - print_stdout_line(format_args!(" {}", branch))?; - } - } - - Ok(()) -} - -fn cmd_checkout(workdir: &Path, target: &str, create_branch: bool) -> io::Result<()> { - let git_dir = workdir.join(".git"); - - if create_branch { - validate_branch_name(target)?; - let head = resolve_head(&git_dir)?.ok_or_else(|| err("HEAD not found for new branch"))?; - update_ref(&git_dir, &format!("refs/heads/{}", target), &head)?; - fs::write( - git_dir.join("HEAD"), - format!("ref: refs/heads/{}\n", target), - )?; - return Ok(()); - } - - // Resolve target: local branch first, then DWIM remote tracking - validate_branch_name(target)?; - let branch_ref = format!("refs/heads/{}", target); - let commit_hash = if let Some(h) = resolve_ref(&git_dir, &branch_ref)? { - fs::write(git_dir.join("HEAD"), format!("ref: {}\n", branch_ref))?; - h - } else { - let remote_ref = format!("refs/remotes/origin/{}", target); - if let Some(h) = resolve_ref(&git_dir, &remote_ref)? { - update_ref(&git_dir, &branch_ref, &h)?; - fs::write(git_dir.join("HEAD"), format!("ref: {}\n", branch_ref))?; - h - } else { - return Err(err(&format!( - "pathspec '{}' did not match any branch", - target - ))); - } - }; - - // Read target tree - let tree_hash = commit_tree(&git_dir, &commit_hash)?; - let new_entries = read_tree_entries(&git_dir, &tree_hash, "")?; - - // Clean up files from current index that aren't in target - let old_entries = read_index(&git_dir)?; - let new_names: HashSet<&str> = new_entries.iter().map(|e| e.name.as_str()).collect(); - for old in &old_entries { - if !new_names.contains(old.name.as_str()) { - let p = worktree_path(workdir, &old.name)?; - if p.exists() { - fs::remove_file(&p).map_err(|e| err(&format!("remove {}: {}", p.display(), e)))?; - } - } - } - - // Write files from target tree - for entry in &new_entries { - let p = worktree_path(workdir, &entry.name)?; - if let Some(parent) = p.parent() { - mkdirs(parent)?; - } - let (_, blob) = read_object(&git_dir, &entry.sha1)?; - fs::write(&p, &blob)?; - } - - // Update index - write_index(&git_dir, &new_entries)?; - - Ok(()) -} - -fn cmd_clone_local(source: &Path, dest: &Path) -> io::Result<()> { - let src_git = source.join(".git"); - if !src_git.is_dir() { - return Err(err(&format!( - "'{}' does not appear to be a git repository", - source.display() - ))); - } - - if dest.exists() && !dir_is_empty(dest)? { - return Err(err(&format!( - "destination path '{}' already exists and is not an empty directory", - dest.display() - ))); - } - - mkdirs(dest)?; - - let dst_git = dest.join(".git"); - - // Init destination - mkdirs(&dst_git.join("objects"))?; - mkdirs(&dst_git.join("refs/heads"))?; - mkdirs(&dst_git.join("refs/tags"))?; - mkdirs(&dst_git.join("refs/remotes/origin"))?; - - // Copy all objects - copy_dir_recursive(&src_git.join("objects"), &dst_git.join("objects"))?; - - // Create remote tracking branches from source heads - let src_heads = src_git.join("refs/heads"); - if src_heads.exists() { - copy_dir_recursive(&src_heads, &dst_git.join("refs/remotes/origin"))?; - } - - // Determine default branch from source HEAD - let src_head = fs::read_to_string(src_git.join("HEAD"))?; - let src_head = src_head.trim(); - let default_branch = src_head - .strip_prefix("ref: refs/heads/") - .unwrap_or("main") - .to_string(); - validate_branch_name(&default_branch)?; - - // Create local branch for default - let remote_ref = format!("refs/remotes/origin/{}", default_branch); - let mut has_default_branch = false; - if let Some(hash) = resolve_ref(&dst_git, &remote_ref)? { - update_ref(&dst_git, &format!("refs/heads/{}", default_branch), &hash)?; - has_default_branch = true; - } - - // Set HEAD and write config - fs::write( - dst_git.join("HEAD"), - format!("ref: refs/heads/{}\n", default_branch), - )?; - fs::write( - dst_git.join("config"), - format!( - "[core]\n\trepositoryformatversion = 0\n\tfilemode = true\n\tbare = false\n\ - [remote \"origin\"]\n\turl = {}\n\tfetch = +refs/heads/*:refs/remotes/origin/*\n", - source.display() - ), - )?; - - // Checkout working tree - if has_default_branch { - cmd_checkout(dest, &default_branch, false)?; - } - - Ok(()) -} - -fn copy_dir_recursive(src: &Path, dst: &Path) -> io::Result<()> { - mkdirs(dst)?; - if !src.exists() { - return Ok(()); - } - for entry in fs::read_dir(src)? { - let entry = entry?; - let name = entry.file_name().to_string_lossy().to_string(); - if name == "." || name == ".." { - continue; - } - let dst_path = dst.join(entry.file_name()); - if entry.file_type()?.is_dir() { - copy_dir_recursive(&entry.path(), &dst_path)?; - } else { - let content = read_file_limited( - &entry.path(), - MAX_GIT_OBJECT_BYTES + MAX_GIT_OBJECT_HEADER_BYTES, - "git repository file", - )?; - fs::write(&dst_path, content)?; - } - } - Ok(()) -} - -// ─── Entry point ──────────────────────────────────────────────────────────── - -pub fn main(args: Vec) -> i32 { - let str_args: Vec = args - .iter() - .map(|a| a.to_string_lossy().to_string()) - .collect(); - match run(&str_args) { - Ok(()) => 0, - Err(e) => { - eprintln!("fatal: {}", e); - 128 - } - } -} - -fn run(args: &[String]) -> io::Result<()> { - let mut i = 1; // skip argv[0] - let mut workdir = std::env::current_dir().unwrap_or_else(|_| PathBuf::from("/")); - - // Parse global options - while i < args.len() { - if args[i] == "-C" { - i += 1; - if i >= args.len() { - return Err(err("-C requires a directory argument")); - } - let p = PathBuf::from(&args[i]); - workdir = if p.is_absolute() { p } else { workdir.join(p) }; - i += 1; - } else if args[i] == "-c" { - i += 1; - if i >= args.len() { - return Err(err("-c requires a config assignment")); - } - i += 1; - } else { - break; - } - } - - if i >= args.len() { - eprintln!("usage: git []"); - return Err(err("no subcommand")); - } - - let subcmd = &args[i]; - let sub_args = &args[i + 1..]; - - match subcmd.as_str() { - "init" => { - let mut path_arg = None; - let mut quiet = false; - for arg in sub_args { - if arg == "-q" || arg == "--quiet" { - quiet = true; - continue; - } - if arg.starts_with('-') { - return Err(unsupported("init", &format!("does not support `{}`.", arg))); - } - if path_arg.replace(arg).is_some() { - return Err(err("usage: git init []")); - } - } - let path = if let Some(path_arg) = path_arg { - let p = PathBuf::from(path_arg); - if p.is_absolute() { - p - } else { - workdir.join(p) - } - } else { - workdir - }; - cmd_init(&path, quiet) - } - "add" => { - if sub_args.is_empty() { - return Err(err("nothing specified, nothing added")); - } - let paths: Vec = sub_args.iter().map(|s| s.to_string()).collect(); - cmd_add(&workdir, &paths) - } - "commit" => { - let mut message = None; - let mut j = 0; - while j < sub_args.len() { - if sub_args[j] == "-m" && j + 1 < sub_args.len() { - message = Some(sub_args[j + 1].clone()); - j += 2; - } else { - j += 1; - } - } - let msg = message.ok_or_else(|| err("no commit message (-m)"))?; - cmd_commit(&workdir, &msg) - } - "branch" => cmd_branch(&workdir), - "rev-parse" => cmd_rev_parse(&workdir, sub_args), - "checkout" => { - let mut create = false; - let mut target = None; - for arg in sub_args { - if arg == "-b" { - create = true; - } else if !arg.starts_with('-') { - target = Some(arg.clone()); - } - } - let t = target.ok_or_else(|| err("no branch name specified"))?; - cmd_checkout(&workdir, &t, create) - } - "clone" => { - if sub_args.is_empty() || sub_args.len() > 2 { - return Err(err("usage: git clone []")); - } - let src_arg = &sub_args[0]; - if is_ssh_clone_source(src_arg) { - return Err(unsupported( - "clone", - &format!("does not support SSH or git:// remotes (`{}`).", src_arg), - )); - } - if has_http_auth(src_arg) { - return Err(unsupported( - "clone", - &format!( - "does not support authenticated HTTP(S) remotes (`{}`).", - src_arg - ), - )); - } - let dst_arg = if sub_args.len() == 2 { - sub_args[1].clone() - } else { - infer_clone_destination(src_arg)? - }; - let dst = PathBuf::from(&dst_arg); - let dst = if dst.is_absolute() { - dst - } else { - workdir.join(dst) - }; - print_stdout_line(format_args!("Cloning into '{}'...", dst.display()))?; - if is_remote_source(src_arg) { - cmd_clone_remote(src_arg, &dst) - } else { - let src = PathBuf::from(src_arg); - let src = if src.is_absolute() { - src - } else { - workdir.join(src) - }; - cmd_clone_local(&src, &dst) - } - } - other => Err(unsupported( - other, - "is not implemented in the secure-exec VM git command.", - )), - } -} diff --git a/software/git/package.json b/software/git/package.json index 02d3316064..c37a93b30f 100644 --- a/software/git/package.json +++ b/software/git/package.json @@ -2,8 +2,8 @@ "name": "@agentos-software/git", "version": "0.3.3", "type": "module", - "license": "Apache-2.0", - "description": "git version control for secure-exec VMs (planned)", + "license": "GPL-2.0-only", + "description": "Upstream Git version control for AgentOS VMs", "main": "./dist/index.js", "types": "./dist/index.d.ts", "files": [ diff --git a/software/git/test/git.test.ts b/software/git/test/git.test.ts index b2d4aed682..2516b7dc14 100644 --- a/software/git/test/git.test.ts +++ b/software/git/test/git.test.ts @@ -27,6 +27,19 @@ vi.setConfig({ testTimeout: 30_000 }); /** Check git binary exists in addition to base WASM binaries */ const hasGit = hasWasmBinaries && existsSync(resolve(COMMANDS_DIR, 'git')); const hasHostGit = spawnSync('git', ['--version'], { stdio: 'ignore' }).status === 0; +// Smart HTTP needs Git's libcurl-backed remote helpers; this WASM build is NO_CURL. +const hasGitHttpHelper = false; + +const gitConfig = [ + '-c safe.directory=*', + '-c init.defaultBranch=main', + '-c user.name=agentos', + '-c user.email=agentos@example.invalid', +].join(' '); + +function git(args: string) { + return `git ${gitConfig} ${args}`; +} /** Create a kernel with a world-writable in-memory filesystem */ async function createGitKernel() { @@ -76,6 +89,11 @@ async function run(kernel: Kernel, cmd: string): Promise<{ stdout: string; stder return r; } +async function expectGitRef(kernel: Kernel, repo: string, ref: string) { + const result = await run(kernel, git(`-C ${repo} rev-parse --verify ${ref}`)); + expect(result.stdout.trim()).toMatch(/^[0-9a-f]{40,64}$/); +} + // TODO(P6): requires git WASM artifact, intentionally excluded from the fast software-build gate. describeIf(hasGit, 'git command', () => { let kernel: Kernel; @@ -89,7 +107,7 @@ describeIf(hasGit, 'git command', () => { it('init creates .git directory structure', async () => { ({ kernel, vfs, dispose } = await createGitKernel()); - const result = await run(kernel, 'git init /repo'); + const result = await run(kernel, git('init /repo')); expect(result.stdout).toContain('Initialized empty Git repository'); expect(await vfs.exists('/repo/.git/HEAD')).toBe(true); @@ -103,10 +121,10 @@ describeIf(hasGit, 'git command', () => { it('add + commit creates objects and updates ref', async () => { ({ kernel, vfs, dispose } = await createGitKernel()); - await run(kernel, 'git init /repo'); + await run(kernel, git('init /repo')); await kernel.writeFile('/repo/hello.txt', 'hello world\n'); - await run(kernel, 'git -C /repo add hello.txt'); - await run(kernel, "git -C /repo commit -m 'first commit'"); + await run(kernel, git('-C /repo add hello.txt')); + await run(kernel, git("-C /repo commit -m 'first commit'")); expect(await vfs.exists('/repo/.git/refs/heads/main')).toBe(true); }); @@ -114,26 +132,26 @@ describeIf(hasGit, 'git command', () => { it('branch lists branches with current marked', async () => { ({ kernel, vfs, dispose } = await createGitKernel()); - await run(kernel, 'git init /repo'); + await run(kernel, git('init /repo')); await kernel.writeFile('/repo/file.txt', 'content\n'); - await run(kernel, 'git -C /repo add file.txt'); - await run(kernel, "git -C /repo commit -m 'init'"); + await run(kernel, git('-C /repo add file.txt')); + await run(kernel, git("-C /repo commit -m 'init'")); - const result = await run(kernel, 'git -C /repo branch'); + const result = await run(kernel, git('-C /repo branch')); expect(result.stdout.trim()).toBe('* main'); }); it('checkout -b creates a new branch', async () => { ({ kernel, vfs, dispose } = await createGitKernel()); - await run(kernel, 'git init /repo'); + await run(kernel, git('init /repo')); await kernel.writeFile('/repo/file.txt', 'content\n'); - await run(kernel, 'git -C /repo add file.txt'); - await run(kernel, "git -C /repo commit -m 'init'"); + await run(kernel, git('-C /repo add file.txt')); + await run(kernel, git("-C /repo commit -m 'init'")); - await run(kernel, 'git -C /repo checkout -b feature'); + await run(kernel, git('-C /repo checkout -b feature')); - const result = await run(kernel, 'git -C /repo branch'); + const result = await run(kernel, git('-C /repo branch')); const lines = result.stdout.trim().split('\n').map((l: string) => l.trim()); expect(lines).toContain('* feature'); expect(lines).toContain('main'); @@ -143,36 +161,36 @@ describeIf(hasGit, 'git command', () => { ({ kernel, vfs, dispose } = await createGitKernel()); // Create origin repo - await run(kernel, 'git init /tmp/origin'); + await run(kernel, git('init /tmp/origin')); await kernel.writeFile('/tmp/origin/README.md', '# demo repo\n'); - await run(kernel, 'git -C /tmp/origin add README.md'); - await run(kernel, "git -C /tmp/origin commit -m 'initial commit'"); + await run(kernel, git('-C /tmp/origin add README.md')); + await run(kernel, git("-C /tmp/origin commit -m 'initial commit'")); // Check default branch - let r = await run(kernel, 'git -C /tmp/origin branch'); + let r = await run(kernel, git('-C /tmp/origin branch')); expect(r.stdout.trim()).toBe('* main'); // Create feature branch with a new file - await run(kernel, 'git -C /tmp/origin checkout -b feature'); + await run(kernel, git('-C /tmp/origin checkout -b feature')); await kernel.writeFile('/tmp/origin/feature.txt', 'checked out from feature\n'); - await run(kernel, 'git -C /tmp/origin add feature.txt'); - await run(kernel, "git -C /tmp/origin commit -m 'add feature file'"); + await run(kernel, git('-C /tmp/origin add feature.txt')); + await run(kernel, git("-C /tmp/origin commit -m 'add feature file'")); // Switch back to main - await run(kernel, 'git -C /tmp/origin checkout main'); + await run(kernel, git('-C /tmp/origin checkout main')); // Clone - await run(kernel, 'git clone /tmp/origin /tmp/clone'); + await run(kernel, git('clone /tmp/origin /tmp/clone')); // Clone should only show main branch initially - r = await run(kernel, 'git -C /tmp/clone branch'); + r = await run(kernel, git('-C /tmp/clone branch')); expect(r.stdout.trim()).toBe('* main'); // Checkout feature (DWIM from remote tracking) - await run(kernel, 'git -C /tmp/clone checkout feature'); + await run(kernel, git('-C /tmp/clone checkout feature')); // Now both branches should be listed - r = await run(kernel, 'git -C /tmp/clone branch'); + r = await run(kernel, git('-C /tmp/clone branch')); const branches = r.stdout.trim().split('\n').map((l: string) => l.trim()); expect(branches).toContain('* feature'); expect(branches).toContain('main'); @@ -189,13 +207,13 @@ describeIf(hasGit, 'git command', () => { it('clone without an explicit destination uses the source basename', async () => { ({ kernel, vfs, dispose } = await createGitKernel()); - await run(kernel, 'git init /tmp/origin'); + await run(kernel, git('init /tmp/origin')); await kernel.writeFile('/tmp/origin/README.md', 'default destination\n'); - await run(kernel, 'git -C /tmp/origin add README.md'); - await run(kernel, "git -C /tmp/origin commit -m 'seed'"); + await run(kernel, git('-C /tmp/origin add README.md')); + await run(kernel, git("-C /tmp/origin commit -m 'seed'")); await run(kernel, 'mkdir -p /work'); - await run(kernel, 'git -C /work clone /tmp/origin'); + await run(kernel, git('-C /work clone /tmp/origin')); expect(await vfs.exists('/work/origin/.git/HEAD')).toBe(true); const readmeContent = new TextDecoder().decode(await vfs.readFile('/work/origin/README.md')); @@ -205,13 +223,13 @@ describeIf(hasGit, 'git command', () => { it('clone without an explicit destination strips a trailing .git suffix', async () => { ({ kernel, vfs, dispose } = await createGitKernel()); - await run(kernel, 'git init /tmp/origin.git'); + await run(kernel, git('init /tmp/origin.git')); await kernel.writeFile('/tmp/origin.git/README.md', 'suffix destination\n'); - await run(kernel, 'git -C /tmp/origin.git add README.md'); - await run(kernel, "git -C /tmp/origin.git commit -m 'seed'"); + await run(kernel, git('-C /tmp/origin.git add README.md')); + await run(kernel, git("-C /tmp/origin.git commit -m 'seed'")); await run(kernel, 'mkdir -p /work'); - await run(kernel, 'git -C /work clone /tmp/origin.git'); + await run(kernel, git('-C /work clone /tmp/origin.git')); expect(await vfs.exists('/work/origin/.git/HEAD')).toBe(true); const readmeContent = new TextDecoder().decode(await vfs.readFile('/work/origin/README.md')); @@ -221,13 +239,13 @@ describeIf(hasGit, 'git command', () => { it('clone into an existing empty destination directory succeeds', async () => { ({ kernel, vfs, dispose } = await createGitKernel()); - await run(kernel, 'git init /tmp/origin'); + await run(kernel, git('init /tmp/origin')); await kernel.writeFile('/tmp/origin/README.md', 'empty destination\n'); - await run(kernel, 'git -C /tmp/origin add README.md'); - await run(kernel, "git -C /tmp/origin commit -m 'seed'"); + await run(kernel, git('-C /tmp/origin add README.md')); + await run(kernel, git("-C /tmp/origin commit -m 'seed'")); await run(kernel, 'mkdir -p /tmp/clone'); - await run(kernel, 'git clone /tmp/origin /tmp/clone'); + await run(kernel, git('clone /tmp/origin /tmp/clone')); expect(await vfs.exists('/tmp/clone/.git/HEAD')).toBe(true); const readmeContent = new TextDecoder().decode(await vfs.readFile('/tmp/clone/README.md')); @@ -237,15 +255,15 @@ describeIf(hasGit, 'git command', () => { it('clone rejects a non-empty destination directory', async () => { ({ kernel, vfs, dispose } = await createGitKernel()); - await run(kernel, 'git init /tmp/origin'); + await run(kernel, git('init /tmp/origin')); await kernel.writeFile('/tmp/origin/README.md', 'origin\n'); - await run(kernel, 'git -C /tmp/origin add README.md'); - await run(kernel, "git -C /tmp/origin commit -m 'seed'"); + await run(kernel, git('-C /tmp/origin add README.md')); + await run(kernel, git("-C /tmp/origin commit -m 'seed'")); await run(kernel, 'mkdir -p /tmp/clone'); await kernel.writeFile('/tmp/clone/existing.txt', 'keep me\n'); - const result = await kernel.exec('git clone /tmp/origin /tmp/clone'); + const result = await kernel.exec(git('clone /tmp/origin /tmp/clone')); expect(result.exitCode).not.toBe(0); expect(result.stderr).toMatch(/already exists|not an empty directory|destination/i); @@ -257,7 +275,7 @@ describeIf(hasGit, 'git command', () => { it('clone of a missing repository fails without leaving a partial destination', async () => { ({ kernel, vfs, dispose } = await createGitKernel()); - const result = await kernel.exec('git clone /tmp/missing /tmp/clone'); + const result = await kernel.exec(git('clone /tmp/missing /tmp/clone')); expect(result.exitCode).not.toBe(0); expect(result.stderr).toMatch(/not a git repository|missing|no such file|fatal/i); expect(await vfs.exists('/tmp/clone')).toBe(false); @@ -266,8 +284,8 @@ describeIf(hasGit, 'git command', () => { it('clone of an empty repository succeeds and leaves an empty worktree', async () => { ({ kernel, vfs, dispose } = await createGitKernel()); - await run(kernel, 'git init /tmp/origin'); - await run(kernel, 'git clone /tmp/origin /tmp/clone'); + await run(kernel, git('init /tmp/origin')); + await run(kernel, git('clone /tmp/origin /tmp/clone')); const head = new TextDecoder().decode(await vfs.readFile('/tmp/clone/.git/HEAD')); expect(head.trim()).toBe('ref: refs/heads/main'); @@ -279,14 +297,14 @@ describeIf(hasGit, 'git command', () => { it('clone preserves nested directory trees', async () => { ({ kernel, vfs, dispose } = await createGitKernel()); - await run(kernel, 'git init /tmp/origin'); + await run(kernel, git('init /tmp/origin')); await run(kernel, 'mkdir -p /tmp/origin/src/nested'); await kernel.writeFile('/tmp/origin/src/nested/file.txt', 'nested payload\n'); await kernel.writeFile('/tmp/origin/src/root.txt', 'root payload\n'); - await run(kernel, 'git -C /tmp/origin add src/nested/file.txt src/root.txt'); - await run(kernel, "git -C /tmp/origin commit -m 'nested tree'"); + await run(kernel, git('-C /tmp/origin add src/nested/file.txt src/root.txt')); + await run(kernel, git("-C /tmp/origin commit -m 'nested tree'")); - await run(kernel, 'git clone /tmp/origin /tmp/clone'); + await run(kernel, git('clone /tmp/origin /tmp/clone')); const nested = new TextDecoder().decode(await vfs.readFile('/tmp/clone/src/nested/file.txt')); const root = new TextDecoder().decode(await vfs.readFile('/tmp/clone/src/root.txt')); @@ -297,17 +315,17 @@ describeIf(hasGit, 'git command', () => { it('clone honors the source default branch when HEAD is not main', async () => { ({ kernel, vfs, dispose } = await createGitKernel()); - await run(kernel, 'git init /tmp/origin'); + await run(kernel, git('init /tmp/origin')); await kernel.writeFile('/tmp/origin/README.md', 'main branch\n'); - await run(kernel, 'git -C /tmp/origin add README.md'); - await run(kernel, "git -C /tmp/origin commit -m 'main'"); + await run(kernel, git('-C /tmp/origin add README.md')); + await run(kernel, git("-C /tmp/origin commit -m 'main'")); - await run(kernel, 'git -C /tmp/origin checkout -b trunk'); + await run(kernel, git('-C /tmp/origin checkout -b trunk')); await kernel.writeFile('/tmp/origin/trunk.txt', 'trunk branch\n'); - await run(kernel, 'git -C /tmp/origin add trunk.txt'); - await run(kernel, "git -C /tmp/origin commit -m 'trunk'"); + await run(kernel, git('-C /tmp/origin add trunk.txt')); + await run(kernel, git("-C /tmp/origin commit -m 'trunk'")); - await run(kernel, 'git clone /tmp/origin /tmp/clone'); + await run(kernel, git('clone /tmp/origin /tmp/clone')); const head = new TextDecoder().decode(await vfs.readFile('/tmp/clone/.git/HEAD')); expect(head.trim()).toBe('ref: refs/heads/trunk'); @@ -319,22 +337,22 @@ describeIf(hasGit, 'git command', () => { it('clone copies nested branch refs and checkout DWIM works for branch names with slashes', async () => { ({ kernel, vfs, dispose } = await createGitKernel()); - await run(kernel, 'git init /tmp/origin'); + await run(kernel, git('init /tmp/origin')); await kernel.writeFile('/tmp/origin/README.md', '# demo repo\n'); - await run(kernel, 'git -C /tmp/origin add README.md'); - await run(kernel, "git -C /tmp/origin commit -m 'initial commit'"); + await run(kernel, git('-C /tmp/origin add README.md')); + await run(kernel, git("-C /tmp/origin commit -m 'initial commit'")); - await run(kernel, 'git -C /tmp/origin checkout -b feature/deep'); + await run(kernel, git('-C /tmp/origin checkout -b feature/deep')); await kernel.writeFile('/tmp/origin/feature.txt', 'nested branch payload\n'); - await run(kernel, 'git -C /tmp/origin add feature.txt'); - await run(kernel, "git -C /tmp/origin commit -m 'nested branch'"); - await run(kernel, 'git -C /tmp/origin checkout main'); + await run(kernel, git('-C /tmp/origin add feature.txt')); + await run(kernel, git("-C /tmp/origin commit -m 'nested branch'")); + await run(kernel, git('-C /tmp/origin checkout main')); - await run(kernel, 'git clone /tmp/origin /tmp/clone'); + await run(kernel, git('clone /tmp/origin /tmp/clone')); - expect(await vfs.exists('/tmp/clone/.git/refs/remotes/origin/feature/deep')).toBe(true); + await expectGitRef(kernel, '/tmp/clone', 'refs/remotes/origin/feature/deep'); - await run(kernel, 'git -C /tmp/clone checkout feature/deep'); + await run(kernel, git('-C /tmp/clone checkout feature/deep')); const featureContent = new TextDecoder().decode(await vfs.readFile('/tmp/clone/feature.txt')); expect(featureContent).toBe('nested branch payload\n'); const head = new TextDecoder().decode(await vfs.readFile('/tmp/clone/.git/HEAD')); @@ -345,56 +363,51 @@ describeIf(hasGit, 'git command', () => { ({ kernel, vfs, dispose } = await createGitKernel()); await run(kernel, 'mkdir -p /tmp/work'); - await run(kernel, 'git init /tmp/work/origin'); + await run(kernel, git('init /tmp/work/origin')); await kernel.writeFile('/tmp/work/origin/README.md', 'relative clone\n'); - await run(kernel, 'git -C /tmp/work/origin add README.md'); - await run(kernel, "git -C /tmp/work/origin commit -m 'seed'"); + await run(kernel, git('-C /tmp/work/origin add README.md')); + await run(kernel, git("-C /tmp/work/origin commit -m 'seed'")); - await run(kernel, 'git -C /tmp/work clone ./origin ./clone'); + await run(kernel, git('-C /tmp/work clone ./origin ./clone')); expect(await vfs.exists('/tmp/work/clone/.git/HEAD')).toBe(true); const readmeContent = new TextDecoder().decode(await vfs.readFile('/tmp/work/clone/README.md')); expect(readmeContent).toBe('relative clone\n'); }); - it('push fails with a typed unsupported-subcommand error', async () => { + it('push fails with a real Git remote/ref error', async () => { ({ kernel, dispose } = await createGitKernel()); - await run(kernel, 'git init /tmp/repo'); + await run(kernel, git('init /tmp/repo')); - const result = await kernel.exec('git -C /tmp/repo push origin main'); + const result = await kernel.exec(git('-C /tmp/repo push origin main')); expect(result.exitCode).not.toBe(0); - expect(result.stderr).toContain('GitSubcommandUnsupported'); - expect(result.stderr).toContain('git push'); - expect(result.stderr).toContain('README.md'); + expect(result.stderr).toMatch(/fatal|error|origin|refspec|repository/i); + expect(result.stderr).not.toContain('GitSubcommandUnsupported'); }); - it('clone rejects SSH-style remotes with a typed unsupported-subcommand error', async () => { + it('clone rejects SSH-style remotes with a real Git spawn failure', async () => { ({ kernel, dispose } = await createGitKernel()); - const result = await kernel.exec('git clone git@github.com:rivet-dev/agentos.git /tmp/clone'); + const result = await kernel.exec(git('clone git@github.com:rivet-dev/agentos.git /tmp/clone')); expect(result.exitCode).not.toBe(0); - expect(result.stderr).toContain('GitSubcommandUnsupported'); - expect(result.stderr).toContain('git clone'); - expect(result.stderr).toContain('SSH'); - expect(result.stderr).toContain('README.md'); + expect(result.stderr).toMatch(/cannot run ssh|unable to fork|ssh|fatal/i); + expect(result.stderr).not.toContain('GitSubcommandUnsupported'); }); - it('clone rejects authenticated HTTPS remotes loudly instead of attempting a broken auth flow', async () => { + it('clone rejects HTTPS remotes until Git is linked with libcurl remote helpers', async () => { ({ kernel, dispose } = await createGitKernel()); const result = await kernel.exec( - 'git clone https://private@example.com/owner/repo.git /tmp/clone', + git('clone https://private@example.com/owner/repo.git /tmp/clone'), { env: { GIT_AUTH_TOKEN: 'test-token' } }, ); expect(result.exitCode).not.toBe(0); - expect(result.stderr).toContain('GitSubcommandUnsupported'); - expect(result.stderr).toContain('git clone'); - expect(result.stderr).toContain('authenticated HTTP(S) remotes'); - expect(result.stderr).toContain('README.md'); + expect(result.stderr).toMatch(/remote-https|remote helper|fatal|unable/i); + expect(result.stderr).not.toContain('GitSubcommandUnsupported'); }); - describeIf(hasHostGit, 'remote clone over smart HTTP', () => { + describeIf(hasHostGit && hasGitHttpHelper, 'remote clone over smart HTTP', () => { let repoRoot: string; let httpServer: HttpServer; let httpPort: number; @@ -530,16 +543,16 @@ describeIf(hasGit, 'git command', () => { it('clone fetches refs and worktree contents from a smart HTTP remote', async () => { ({ kernel, vfs, dispose } = await createGitKernelWithNet([httpPort])); - await run(kernel, `git clone http://127.0.0.1:${httpPort}/origin.git /tmp/clone`); + await run(kernel, git(`clone http://127.0.0.1:${httpPort}/origin.git /tmp/clone`)); const head = new TextDecoder().decode(await kernel.readFile('/tmp/clone/.git/HEAD')); expect(head.trim()).toBe('ref: refs/heads/main'); const readme = new TextDecoder().decode(await kernel.readFile('/tmp/clone/README.md')); expect(readme).toBe('remote smart clone\n'); - expect(await kernel.exists('/tmp/clone/.git/refs/remotes/origin/feature/deep')).toBe(true); + await expectGitRef(kernel, '/tmp/clone', 'refs/remotes/origin/feature/deep'); - await run(kernel, 'git -C /tmp/clone checkout feature/deep'); + await run(kernel, git('-C /tmp/clone checkout feature/deep')); const feature = new TextDecoder().decode(await kernel.readFile('/tmp/clone/feature.txt')); expect(feature).toBe('remote branch payload\n'); }); diff --git a/software/grep/agentos-package.json b/software/grep/agentos-package.json index e75bc0d243..2b14ba7c37 100644 --- a/software/grep/agentos-package.json +++ b/software/grep/agentos-package.json @@ -1,11 +1,9 @@ { "commands": [ - "grep" + "grep", + "egrep", + "fgrep" ], - "aliases": { - "egrep": "grep", - "fgrep": "grep" - }, "registry": { "title": "grep", "description": "GNU grep pattern matching (grep, egrep, fgrep).", diff --git a/software/grep/bin/egrep b/software/grep/bin/egrep index c78e121f14..6ab0536c2a 100644 Binary files a/software/grep/bin/egrep and b/software/grep/bin/egrep differ diff --git a/software/grep/bin/fgrep b/software/grep/bin/fgrep index c78e121f14..01aa3dea12 100644 Binary files a/software/grep/bin/fgrep and b/software/grep/bin/fgrep differ diff --git a/software/grep/bin/grep b/software/grep/bin/grep index c78e121f14..53f5049c16 100644 Binary files a/software/grep/bin/grep and b/software/grep/bin/grep differ diff --git a/software/grep/native/crates/cmd-egrep/Cargo.toml b/software/grep/native/crates/cmd-egrep/Cargo.toml new file mode 100644 index 0000000000..b3eb4c9222 --- /dev/null +++ b/software/grep/native/crates/cmd-egrep/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-egrep" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "egrep launcher for GNU grep" + +[[bin]] +name = "egrep" +path = "src/main.rs" + +[dependencies] +shims = { package = "secureexec-shims", path = "../../../../../toolchain/crates/libs/shims" } diff --git a/software/git/native/crates/cmd-git/src/main.rs b/software/grep/native/crates/cmd-egrep/src/main.rs similarity index 60% rename from software/git/native/crates/cmd-git/src/main.rs rename to software/grep/native/crates/cmd-egrep/src/main.rs index 83886647f0..b6671b6c38 100644 --- a/software/git/native/crates/cmd-git/src/main.rs +++ b/software/grep/native/crates/cmd-egrep/src/main.rs @@ -1,4 +1,4 @@ fn main() { let args: Vec = std::env::args_os().collect(); - std::process::exit(secureexec_git::main(args)); + std::process::exit(shims::grep_alias::egrep(args)); } diff --git a/software/grep/native/crates/cmd-fgrep/Cargo.toml b/software/grep/native/crates/cmd-fgrep/Cargo.toml new file mode 100644 index 0000000000..573b6b100e --- /dev/null +++ b/software/grep/native/crates/cmd-fgrep/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-fgrep" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "fgrep launcher for GNU grep" + +[[bin]] +name = "fgrep" +path = "src/main.rs" + +[dependencies] +shims = { package = "secureexec-shims", path = "../../../../../toolchain/crates/libs/shims" } diff --git a/software/fd/native/crates/cmd-fd/src/main.rs b/software/grep/native/crates/cmd-fgrep/src/main.rs similarity index 60% rename from software/fd/native/crates/cmd-fd/src/main.rs rename to software/grep/native/crates/cmd-fgrep/src/main.rs index e74e129d80..3f81c7a4d5 100644 --- a/software/fd/native/crates/cmd-fd/src/main.rs +++ b/software/grep/native/crates/cmd-fgrep/src/main.rs @@ -1,4 +1,4 @@ fn main() { let args: Vec = std::env::args_os().collect(); - std::process::exit(secureexec_fd::main(args)); + std::process::exit(shims::grep_alias::fgrep(args)); } diff --git a/software/grep/native/crates/cmd-grep/Cargo.toml b/software/grep/native/crates/cmd-grep/Cargo.toml deleted file mode 100644 index 3214711f24..0000000000 --- a/software/grep/native/crates/cmd-grep/Cargo.toml +++ /dev/null @@ -1,14 +0,0 @@ -[package] -workspace = "../../../../../toolchain" -name = "cmd-grep" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "grep standalone binary for secure-exec VM" - -[[bin]] -name = "grep" -path = "src/main.rs" - -[dependencies] -secureexec-grep = { path = "../../../../../toolchain/crates/libs/grep" } diff --git a/software/grep/native/crates/cmd-grep/src/main.rs b/software/grep/native/crates/cmd-grep/src/main.rs deleted file mode 100644 index ef8a789bdf..0000000000 --- a/software/grep/native/crates/cmd-grep/src/main.rs +++ /dev/null @@ -1,13 +0,0 @@ -fn main() { - use std::io::Write; - - let args: Vec = std::env::args_os().collect(); - let mut code = secureexec_grep::main(args); - if let Err(error) = std::io::stdout().flush() { - eprintln!("Error flushing stdout: {error}"); - if code == 0 { - code = 2; - } - } - std::process::exit(code); -} diff --git a/software/grep/test/grep.test.ts b/software/grep/test/grep.test.ts new file mode 100644 index 0000000000..245bc42216 --- /dev/null +++ b/software/grep/test/grep.test.ts @@ -0,0 +1,113 @@ +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { createWasmVmRuntime } from "@agentos/test-harness"; +import { + COMMANDS_DIR, + NodeFileSystem, + createKernel, + describeIf, + hasWasmBinaries, +} from "@agentos/test-harness"; +import type { Kernel } from "@agentos/test-harness"; +import { afterEach, describe, expect, it } from "vitest"; + +let tempRoot: string | undefined; + +async function createTestVFS(): Promise { + tempRoot = await mkdtemp(join(tmpdir(), "agentos-grep-")); + await writeFixture( + "/project/notes.txt", + ["Alpha", "beta", "alphabet", "delta"].join("\n") + "\n", + ); + await writeFixture( + "/project/other.txt", + ["gamma", "Beta blocker", "literal a+b"].join("\n") + "\n", + ); + return new NodeFileSystem({ root: tempRoot }); +} + +async function writeFixture(path: string, contents: string): Promise { + if (!tempRoot) throw new Error("fixture root not initialized"); + const hostPath = join(tempRoot, path.replace(/^\/+/, "")); + await mkdir(dirname(hostPath), { recursive: true }); + await writeFile(hostPath, contents); +} + +describeIf(hasWasmBinaries, "GNU grep command", { timeout: 10_000 }, () => { + let kernel: Kernel; + + afterEach(async () => { + await kernel?.dispose(); + if (tempRoot) { + await rm(tempRoot, { recursive: true, force: true }); + tempRoot = undefined; + } + }); + + async function mountFixture(): Promise { + const vfs = await createTestVFS(); + kernel = createKernel({ filesystem: vfs }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); + return vfs; + } + + it("reports the upstream GNU grep version", async () => { + await mountFixture(); + + const result = await kernel.exec("grep --version", {}); + expect(result.stdout).toContain("GNU grep 3.12"); + }); + + it("prints matching lines from a file", async () => { + await mountFixture(); + + const result = await kernel.exec("grep alpha /project/notes.txt", {}); + expect(result.stdout).toBe("alphabet\n"); + }); + + it("supports case-insensitive search", async () => { + await mountFixture(); + + const result = await kernel.exec("grep -i beta /project/notes.txt", {}); + expect(result.stdout).toBe("beta\n"); + }); + + it("supports inverted matches", async () => { + await mountFixture(); + + const result = await kernel.exec("grep -v Alpha /project/notes.txt", {}); + expect(result.stdout).toBe("beta\nalphabet\ndelta\n"); + }); + + it("counts matches", async () => { + await mountFixture(); + + const result = await kernel.exec("grep -c a /project/notes.txt", {}); + expect(result.stdout).toBe("4\n"); + }); + + it("prints file names with matches", async () => { + await mountFixture(); + + const result = await kernel.exec( + "grep -l gamma /project/notes.txt /project/other.txt", + {}, + ); + expect(result.stdout).toBe("/project/other.txt\n"); + }); + + it("supports egrep extended regex alias", async () => { + await mountFixture(); + + const result = await kernel.exec("egrep 'Alpha|delta' /project/notes.txt", {}); + expect(result.stdout).toBe("Alpha\ndelta\n"); + }); + + it("supports fgrep fixed-string alias", async () => { + await mountFixture(); + + const result = await kernel.exec("fgrep 'a+b' /project/other.txt", {}); + expect(result.stdout).toBe("literal a+b\n"); + }); +}); diff --git a/software/gzip/test/gzip.test.ts b/software/gzip/test/gzip.test.ts new file mode 100644 index 0000000000..477499a35a --- /dev/null +++ b/software/gzip/test/gzip.test.ts @@ -0,0 +1,138 @@ +import { existsSync } from "node:fs"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { + NodeFileSystem, + createKernel, + createWasmVmRuntime, + describeIf, +} from "@agentos/test-harness"; +import type { Kernel } from "@agentos/test-harness"; +import { afterEach, expect, it } from "vitest"; + +const GZIP_COMMAND_DIR = fileURLToPath(new URL("../bin", import.meta.url)); +const hasGzipPackageBinary = existsSync(join(GZIP_COMMAND_DIR, "gzip")); + +let tempRoot: string | undefined; + +async function writeFixture(path: string, contents: string): Promise { + if (!tempRoot) throw new Error("fixture root not initialized"); + const hostPath = join(tempRoot, path.replace(/^\/+/, "")); + await mkdir(dirname(hostPath), { recursive: true }); + await writeFile(hostPath, contents); +} + +async function createTestVFS(): Promise { + tempRoot = await mkdtemp(join(tmpdir(), "agentos-gzip-")); + await writeFixture("/project/report.txt", "alpha\nbeta\ngamma\n"); + await writeFixture("/project/remove.txt", "temporary payload\n"); + return new NodeFileSystem({ root: tempRoot }); +} + +const textDecoder = new TextDecoder(); + +describeIf(hasGzipPackageBinary, "gzip command", { timeout: 10_000 }, () => { + let kernel: Kernel | undefined; + + afterEach(async () => { + await kernel?.dispose(); + kernel = undefined; + if (tempRoot) { + await rm(tempRoot, { recursive: true, force: true }); + tempRoot = undefined; + } + }); + + async function mountFixture(): Promise { + const vfs = await createTestVFS(); + kernel = createKernel({ filesystem: vfs }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [GZIP_COMMAND_DIR] })); + } + + async function runCommand(command: string, args: string[]) { + if (!kernel) throw new Error("kernel not mounted"); + let stdout = ""; + let stderr = ""; + const proc = kernel.spawn(command, args, { + onStdout: (chunk) => { + stdout += Buffer.from(chunk).toString("utf8"); + }, + onStderr: (chunk) => { + stderr += Buffer.from(chunk).toString("utf8"); + }, + }); + const exitCode = await proc.wait(); + await new Promise((resolve) => setTimeout(resolve, 0)); + return { stdout, stderr, exitCode }; + } + + async function readGuestText(path: string): Promise { + if (!kernel) throw new Error("kernel not mounted"); + return textDecoder.decode(await kernel.readFile(path)); + } + + it("compresses files while keeping originals with -k", async () => { + await mountFixture(); + + const result = await runCommand("gzip", ["-k", "/project/report.txt"]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + await expect(readGuestText("/project/report.txt")).resolves.toBe( + "alpha\nbeta\ngamma\n", + ); + + const decompressed = await runCommand("gunzip", [ + "-c", + "/project/report.txt.gz", + ]); + expect(decompressed.exitCode, decompressed.stderr || decompressed.stdout).toBe(0); + expect(decompressed.stdout).toBe("alpha\nbeta\ngamma\n"); + }); + + it("removes the source file unless -k is set", async () => { + await mountFixture(); + + const result = await runCommand("gzip", ["/project/remove.txt"]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + await expect(readGuestText("/project/remove.txt")).rejects.toThrow(); + await expect(readGuestText("/project/remove.txt.gz")).resolves.toBeTruthy(); + }); + + it("decompresses files with gunzip -k", async () => { + await mountFixture(); + const compressed = await runCommand("gzip", ["-k", "/project/report.txt"]); + expect(compressed.exitCode, compressed.stderr || compressed.stdout).toBe(0); + await writeFixture("/project/report.txt", "replacement\n"); + + const result = await runCommand("gunzip", [ + "-fk", + "/project/report.txt.gz", + ]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + await expect(readGuestText("/project/report.txt")).resolves.toBe( + "alpha\nbeta\ngamma\n", + ); + await expect(readGuestText("/project/report.txt.gz")).resolves.toBeTruthy(); + }); + + it("streams decompressed content with zcat", async () => { + await mountFixture(); + const compressed = await runCommand("gzip", ["-k", "/project/report.txt"]); + expect(compressed.exitCode, compressed.stderr || compressed.stdout).toBe(0); + + const result = await runCommand("zcat", ["/project/report.txt.gz"]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(result.stdout).toBe("alpha\nbeta\ngamma\n"); + }); + + it("fails instead of overwriting compressed outputs without -f", async () => { + await mountFixture(); + const first = await runCommand("gzip", ["-k", "/project/report.txt"]); + expect(first.exitCode, first.stderr || first.stdout).toBe(0); + + const second = await runCommand("gzip", ["-k", "/project/report.txt"]); + expect(second.exitCode).not.toBe(0); + expect(second.stderr).toContain("already exists"); + }); +}); diff --git a/software/http-get/agentos-package.json b/software/http-get/agentos-package.json deleted file mode 100644 index 4f8fb816f9..0000000000 --- a/software/http-get/agentos-package.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "commands": [ - "http_get" - ], - "registry": { - "title": "http-get", - "description": "Minimal HTTP GET fetch helper.", - "category": "networking" - } -} diff --git a/software/http-get/native/c/http_get.c b/software/http-get/native/c/http_get.c deleted file mode 100644 index 0e7dc466ea..0000000000 --- a/software/http-get/native/c/http_get.c +++ /dev/null @@ -1,95 +0,0 @@ -/* http_get.c — connect to an HTTP server, send GET request, print response body */ -#include -#include -#include -#include -#include -#include -#include - -int main(int argc, char *argv[]) { - if (argc < 2) { - fprintf(stderr, "usage: http_get [path] [output_file]\n"); - return 1; - } - - int port = atoi(argv[1]); - const char *path = argc >= 3 ? argv[2] : "/"; - const char *output_file = argc >= 4 ? argv[3] : NULL; - - int fd = socket(AF_INET, SOCK_STREAM, 0); - if (fd < 0) { - perror("socket"); - return 1; - } - - struct sockaddr_in addr; - memset(&addr, 0, sizeof(addr)); - addr.sin_family = AF_INET; - addr.sin_port = htons((uint16_t)port); - inet_pton(AF_INET, "127.0.0.1", &addr.sin_addr); - - if (connect(fd, (struct sockaddr *)&addr, sizeof(addr)) < 0) { - perror("connect"); - close(fd); - return 1; - } - - char request[1024]; - int request_len = snprintf( - request, - sizeof(request), - "GET %s HTTP/1.0\r\nHost: localhost\r\n\r\n", - path - ); - if (request_len < 0 || (size_t)request_len >= sizeof(request)) { - fprintf(stderr, "request path too long\n"); - close(fd); - return 1; - } - - ssize_t sent = send(fd, request, (size_t)request_len, 0); - if (sent < 0) { - perror("send"); - close(fd); - return 1; - } - - /* Read full response */ - char response[4096]; - size_t total = 0; - ssize_t n; - while ((n = recv(fd, response + total, sizeof(response) - total - 1, 0)) > 0) { - total += (size_t)n; - } - response[total] = '\0'; - - close(fd); - - /* Find body after \r\n\r\n */ - const char *body = strstr(response, "\r\n\r\n"); - if (body) { - body += 4; - if (output_file) { - FILE *out = fopen(output_file, "wb"); - if (!out) { - perror("fopen"); - return 1; - } - size_t body_len = total - (size_t)(body - response); - if (fwrite(body, 1, body_len, out) != body_len) { - perror("fwrite"); - fclose(out); - return 1; - } - fclose(out); - } else { - printf("body: %s\n", body); - } - } else { - printf("body: (no separator found)\n"); - return 1; - } - - return 0; -} diff --git a/software/http-get/package.json b/software/http-get/package.json deleted file mode 100644 index ef23733846..0000000000 --- a/software/http-get/package.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "name": "@agentos-software/http-get", - "version": "0.3.3", - "type": "module", - "license": "Apache-2.0", - "description": "Minimal HTTP GET fetch helper for secure-exec VMs", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "files": [ - "dist", - "!dist/package", - "!dist/package.tar" - ], - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js" - } - }, - "scripts": { - "build": "agentos-toolchain stage --commands-dir ../../toolchain/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", - "check-types": "tsc --noEmit", - "test": "vitest run test/ --passWithNoTests" - }, - "devDependencies": { - "@agentos-software/manifest": "workspace:*", - "@rivet-dev/agentos-toolchain": "workspace:*", - "@types/node": "^22.10.2", - "typescript": "^5.9.2", - "@agentos/test-harness": "workspace:*", - "vitest": "^2.1.9" - } -} diff --git a/software/http-get/src/index.ts b/software/http-get/src/index.ts deleted file mode 100644 index dba100e2e1..0000000000 --- a/software/http-get/src/index.ts +++ /dev/null @@ -1,5 +0,0 @@ -import type { SoftwarePackageRef } from "@agentos-software/manifest"; - -const packagePath = new URL("./package.aospkg", import.meta.url).pathname; - -export default { packagePath } satisfies SoftwarePackageRef; diff --git a/software/http-get/test/manifest.test.ts b/software/http-get/test/manifest.test.ts deleted file mode 100644 index 89139d33b8..0000000000 --- a/software/http-get/test/manifest.test.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { readFileSync } from "node:fs"; -import { join } from "node:path"; -import { describe, expect, it } from "vitest"; - -const packageDir = new URL("..", import.meta.url).pathname; - -describe("package manifest", () => { - it("declares command binaries", () => { - const manifest = JSON.parse( - readFileSync(join(packageDir, "agentos-package.json"), "utf8"), - ); - - expect(manifest.commands?.length ?? 0).toBeGreaterThan(0); - for (const command of manifest.commands) { - expect(typeof command).toBe("string"); - expect(command.length).toBeGreaterThan(0); - } - }); -}); diff --git a/software/http-get/tsconfig.json b/software/http-get/tsconfig.json deleted file mode 100644 index 03ce790ab7..0000000000 --- a/software/http-get/tsconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": "../tsconfig.base.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src" - }, - "include": ["src/**/*"] -} diff --git a/software/jq/bin/jq b/software/jq/bin/jq index 22f0359722..e74f99e2e7 100644 Binary files a/software/jq/bin/jq and b/software/jq/bin/jq differ diff --git a/software/jq/native/crates/jq/src/lib.rs b/software/jq/native/crates/jq/src/lib.rs index 9fe36c8542..66fc2544cc 100644 --- a/software/jq/native/crates/jq/src/lib.rs +++ b/software/jq/native/crates/jq/src/lib.rs @@ -3,6 +3,7 @@ //! Wraps jaq-core/jaq-std/jaq-json to provide a standard jq CLI interface. use std::ffi::OsString; +use std::fs::File as FsFile; use std::io::{self, Read, Write}; use jaq_core::load::{Arena, File, Loader}; @@ -40,6 +41,7 @@ struct JqOptions { join_output: bool, args: Vec<(String, String)>, jsonargs: Vec<(String, Val)>, + input_paths: Vec, } fn parse_args(args: &[String]) -> Result { @@ -54,6 +56,7 @@ fn parse_args(args: &[String]) -> Result { join_output: false, args: Vec::new(), jsonargs: Vec::new(), + input_paths: Vec::new(), }; let mut filter_set = false; @@ -63,6 +66,17 @@ fn parse_args(args: &[String]) -> Result { let arg = &args[i]; if arg == "--" { + let remaining = &args[i + 1..]; + if !filter_set { + let Some(filter) = remaining.first() else { + return Err("no filter provided".to_string()); + }; + opts.filter = filter.clone(); + filter_set = true; + opts.input_paths.extend(remaining.iter().skip(1).cloned()); + } else { + opts.input_paths.extend(remaining.iter().cloned()); + } break; } @@ -118,7 +132,7 @@ fn parse_args(args: &[String]) -> Result { opts.filter = arg.clone(); filter_set = true; } else { - return Err(format!("unexpected argument: {}", arg)); + opts.input_paths.push(arg.clone()); } i += 1; @@ -131,30 +145,60 @@ fn parse_args(args: &[String]) -> Result { Ok(opts) } +fn read_sources(opts: &JqOptions) -> Result, String> { + if opts.input_paths.is_empty() { + let mut stdin_data = String::new(); + io::stdin() + .take((MAX_INPUT_BYTES + 1) as u64) + .read_to_string(&mut stdin_data) + .map_err(|e| format!("failed to read stdin: {}", e))?; + if stdin_data.len() > MAX_INPUT_BYTES { + return Err("stdin exceeds size limit".to_string()); + } + return Ok(vec![stdin_data]); + } + + let mut total = 0usize; + let mut sources = Vec::new(); + for path in &opts.input_paths { + let mut data = String::new(); + if path == "-" { + io::stdin() + .take((MAX_INPUT_BYTES.saturating_sub(total) + 1) as u64) + .read_to_string(&mut data) + .map_err(|e| format!("failed to read stdin: {}", e))?; + } else { + FsFile::open(path) + .map_err(|e| format!("failed to open {}: {}", path, e))? + .take((MAX_INPUT_BYTES.saturating_sub(total) + 1) as u64) + .read_to_string(&mut data) + .map_err(|e| format!("failed to read {}: {}", path, e))?; + } + total = total + .checked_add(data.len()) + .ok_or_else(|| "input exceeds size limit".to_string())?; + if total > MAX_INPUT_BYTES { + return Err("input exceeds size limit".to_string()); + } + sources.push(data); + } + Ok(sources) +} + fn read_inputs(opts: &JqOptions) -> Result, String> { if opts.null_input { return Ok(vec![Val::from(serde_json::Value::Null)]); } - let mut stdin_data = String::new(); - io::stdin() - .take((MAX_INPUT_BYTES + 1) as u64) - .read_to_string(&mut stdin_data) - .map_err(|e| format!("failed to read stdin: {}", e))?; - if stdin_data.len() > MAX_INPUT_BYTES { - return Err("stdin exceeds size limit".to_string()); - } + let sources = read_sources(opts)?; if opts.raw_input { + let raw_data = sources.concat(); if opts.slurp { - let mut arr = Vec::new(); - for line in stdin_data.lines() { - push_input_value(&mut arr, serde_json::Value::String(line.to_string()))?; - } - Ok(vec![Val::from(serde_json::Value::Array(arr))]) + Ok(vec![Val::from(serde_json::Value::String(raw_data))]) } else { let mut lines = Vec::new(); - for line in stdin_data.lines() { + for line in raw_data.lines() { push_input_value( &mut lines, Val::from(serde_json::Value::String(line.to_string())), @@ -163,16 +207,23 @@ fn read_inputs(opts: &JqOptions) -> Result, String> { Ok(lines) } } else { - let trimmed = stdin_data.trim(); - if trimmed.is_empty() { - return Ok(vec![Val::from(serde_json::Value::Null)]); + let mut values = Vec::new(); + for source in &sources { + let trimmed = source.trim(); + if trimmed.is_empty() { + continue; + } + + let decoder = + serde_json::Deserializer::from_str(trimmed).into_iter::(); + for result in decoder { + let value = result.map_err(|e| format!("parse error: {}", e))?; + push_input_value(&mut values, value)?; + } } - let mut values = Vec::new(); - let decoder = serde_json::Deserializer::from_str(trimmed).into_iter::(); - for result in decoder { - let value = result.map_err(|e| format!("invalid JSON input: {}", e))?; - push_input_value(&mut values, value)?; + if values.is_empty() && opts.input_paths.is_empty() { + values.push(serde_json::Value::Null); } if opts.slurp { @@ -217,6 +268,11 @@ fn format_output(val: &Val, opts: &JqOptions) -> Result { } fn run_jq(args: &[String]) -> Result { + if matches!(args, [arg] if arg == "--version" || arg == "-V") { + println!("jq-jaq-{}", env!("CARGO_PKG_VERSION")); + return Ok(0); + } + let opts = parse_args(args)?; let inputs = read_inputs(&opts)?; diff --git a/software/jq/test/jq.test.ts b/software/jq/test/jq.test.ts new file mode 100644 index 0000000000..5d15eaebc3 --- /dev/null +++ b/software/jq/test/jq.test.ts @@ -0,0 +1,153 @@ +import { existsSync } from "node:fs"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { + NodeFileSystem, + createKernel, + createWasmVmRuntime, + describeIf, +} from "@agentos/test-harness"; +import type { Kernel } from "@agentos/test-harness"; +import { afterEach, expect, it } from "vitest"; + +const JQ_COMMAND_DIR = fileURLToPath(new URL("../bin", import.meta.url)); +const hasJqPackageBinary = existsSync(join(JQ_COMMAND_DIR, "jq")); + +let tempRoot: string | undefined; + +async function writeFixture(path: string, contents: string): Promise { + if (!tempRoot) throw new Error("fixture root not initialized"); + const hostPath = join(tempRoot, path.replace(/^\/+/, "")); + await mkdir(dirname(hostPath), { recursive: true }); + await writeFile(hostPath, contents); +} + +async function createTestVFS(): Promise { + tempRoot = await mkdtemp(join(tmpdir(), "agentos-jq-")); + await writeFixture( + "/project/users.json", + `${JSON.stringify( + { + users: [ + { name: "Ada", active: true, team: "runtime", score: 7 }, + { name: "Grace", active: false, team: "docs", score: 5 }, + { name: "Linus", active: true, team: "runtime", score: 3 }, + ], + }, + null, + 2, + )}\n`, + ); + await writeFixture( + "/project/events.ndjson", + [ + JSON.stringify({ type: "build", value: 4 }), + JSON.stringify({ type: "test", value: 6 }), + JSON.stringify({ type: "deploy", value: 2 }), + ].join("\n") + "\n", + ); + await writeFixture("/project/broken.json", '{"users": ['); + return new NodeFileSystem({ root: tempRoot }); +} + +function lines(stdout: string): string[] { + return stdout.split("\n").filter((line) => line.length > 0); +} + +describeIf(hasJqPackageBinary, "jq command", { timeout: 10_000 }, () => { + let kernel: Kernel | undefined; + + afterEach(async () => { + await kernel?.dispose(); + kernel = undefined; + if (tempRoot) { + await rm(tempRoot, { recursive: true, force: true }); + tempRoot = undefined; + } + }); + + async function mountFixture(): Promise { + const vfs = await createTestVFS(); + kernel = createKernel({ filesystem: vfs }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [JQ_COMMAND_DIR] })); + } + + async function runJq(args: string[]) { + if (!kernel) throw new Error("kernel not mounted"); + let stdout = ""; + let stderr = ""; + const proc = kernel.spawn("jq", args, { + onStdout: (chunk) => { + stdout += Buffer.from(chunk).toString("utf8"); + }, + onStderr: (chunk) => { + stderr += Buffer.from(chunk).toString("utf8"); + }, + }); + const exitCode = await proc.wait(); + await new Promise((resolve) => setTimeout(resolve, 0)); + return { stdout, stderr, exitCode }; + } + + it("reports a jq-compatible version", async () => { + await mountFixture(); + + const result = await runJq(["--version"]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(result.stdout).toMatch(/^jq-/); + }); + + it("filters arrays and emits raw strings", async () => { + await mountFixture(); + + const result = await runJq([ + "-r", + '.users[] | select(.active) | "\\(.name):\\(.score)"', + "/project/users.json", + ]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(lines(result.stdout)).toEqual(["Ada:7", "Linus:3"]); + }); + + it("builds aggregate JSON objects", async () => { + await mountFixture(); + + const result = await runJq([ + "-c", + '{activeNames: [.users[] | select(.active) | .name], runtimeTotal: ([.users[] | select(.team == "runtime") | .score] | add)}', + "/project/users.json", + ]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(JSON.parse(result.stdout.trim())).toEqual({ + activeNames: ["Ada", "Linus"], + runtimeTotal: 10, + }); + }); + + it("slurps newline-delimited JSON records", async () => { + await mountFixture(); + + const result = await runJq([ + "-s", + "-c", + "{count: length, total: (map(.value) | add), types: map(.type)}", + "/project/events.ndjson", + ]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(JSON.parse(result.stdout.trim())).toEqual({ + count: 3, + total: 12, + types: ["build", "test", "deploy"], + }); + }); + + it("fails with a parse error for invalid JSON", async () => { + await mountFixture(); + + const result = await runJq([".", "/project/broken.json"]); + expect(result.exitCode).not.toBe(0); + expect(result.stderr).toContain("parse error"); + }); +}); diff --git a/software/ripgrep/bin/rg b/software/ripgrep/bin/rg index 76a63b75ea..d3d4306565 100644 Binary files a/software/ripgrep/bin/rg and b/software/ripgrep/bin/rg differ diff --git a/software/ripgrep/native/crates/cmd-rg/Cargo.toml b/software/ripgrep/native/crates/cmd-rg/Cargo.toml index f702453e13..f6184892c7 100644 --- a/software/ripgrep/native/crates/cmd-rg/Cargo.toml +++ b/software/ripgrep/native/crates/cmd-rg/Cargo.toml @@ -5,10 +5,10 @@ version.workspace = true edition.workspace = true license.workspace = true description = "rg (ripgrep) standalone binary for secure-exec VM" - -[[bin]] -name = "rg" -path = "src/main.rs" +publish = false [dependencies] -secureexec-grep = { path = "../../../../../toolchain/crates/libs/grep" } +# ripgrep is a bin-only crate. `toolchain/Makefile` builds its upstream `rg` +# binary directly; this local package exists only so `cmd/rg` is discoverable +# and the upstream package is present in Cargo's resolved/vendor set. +ripgrep = { version = "15.1.0", default-features = false } diff --git a/software/ripgrep/native/crates/cmd-rg/src/lib.rs b/software/ripgrep/native/crates/cmd-rg/src/lib.rs new file mode 100644 index 0000000000..1bd8717252 --- /dev/null +++ b/software/ripgrep/native/crates/cmd-rg/src/lib.rs @@ -0,0 +1 @@ +//! Build trigger for the upstream `ripgrep` command package. diff --git a/software/ripgrep/native/crates/cmd-rg/src/main.rs b/software/ripgrep/native/crates/cmd-rg/src/main.rs deleted file mode 100644 index e44c45f021..0000000000 --- a/software/ripgrep/native/crates/cmd-rg/src/main.rs +++ /dev/null @@ -1,4 +0,0 @@ -fn main() { - let args: Vec = std::env::args_os().collect(); - std::process::exit(secureexec_grep::rg(args)); -} diff --git a/software/ripgrep/test/ripgrep.test.ts b/software/ripgrep/test/ripgrep.test.ts new file mode 100644 index 0000000000..af8d77885e --- /dev/null +++ b/software/ripgrep/test/ripgrep.test.ts @@ -0,0 +1,128 @@ +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { createWasmVmRuntime } from "@agentos/test-harness"; +import { + COMMANDS_DIR, + NodeFileSystem, + createKernel, + describeIf, + hasWasmBinaries, +} from "@agentos/test-harness"; +import type { Kernel } from "@agentos/test-harness"; +import { afterEach, describe, expect, it } from "vitest"; + +let tempRoot: string | undefined; + +async function createTestVFS(): Promise { + tempRoot = await mkdtemp(join(tmpdir(), "agentos-ripgrep-")); + + await writeFixture( + "/project/src/main.rs", + ["fn main() {", ' println!("needle");', "}"].join("\n") + "\n", + ); + await writeFixture( + "/project/src/lib.rs", + ["pub fn helper() {", " // Needle in a comment", "}"].join("\n") + "\n", + ); + await writeFixture("/project/docs/readme.md", "needle in docs\n"); + await writeFixture("/project/vendor/generated.rs", "needle in vendor\n"); + await writeFixture("/project/.hidden.txt", "needle hidden\n"); + await writeFixture("/project/.gitignore", "vendor/\n"); + await writeFixture("/project/.git/HEAD", "ref: refs/heads/main\n"); + + return new NodeFileSystem({ root: tempRoot }); +} + +async function writeFixture(path: string, contents: string): Promise { + if (!tempRoot) throw new Error("fixture root not initialized"); + const hostPath = join(tempRoot, path.replace(/^\/+/, "")); + await mkdir(dirname(hostPath), { recursive: true }); + await writeFile(hostPath, contents); +} + +function lines(stdout: string): string[] { + return stdout + .split("\n") + .filter((line) => line.length > 0) + .sort(); +} + +describeIf(hasWasmBinaries, "ripgrep command", { timeout: 10_000 }, () => { + let kernel: Kernel; + + afterEach(async () => { + await kernel?.dispose(); + if (tempRoot) { + await rm(tempRoot, { recursive: true, force: true }); + tempRoot = undefined; + } + }); + + async function mountFixture(): Promise { + const vfs = await createTestVFS(); + kernel = createKernel({ filesystem: vfs }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); + } + + it("reports the upstream ripgrep version", async () => { + await mountFixture(); + + const result = await kernel.exec("rg --version", {}); + expect(result.stdout).toContain("ripgrep 15.1.0"); + expect(result.stdout).not.toContain("secure-exec"); + }); + + it("searches recursively and respects .gitignore by default", async () => { + await mountFixture(); + + const result = await kernel.exec("rg needle /project", {}); + const output = lines(result.stdout); + + expect(output).toContain('/project/src/main.rs: println!("needle");'); + expect(output).toContain("/project/docs/readme.md:needle in docs"); + expect(output).not.toContain("/project/vendor/generated.rs:needle in vendor"); + expect(output).not.toContain("/project/.hidden.txt:needle hidden"); + }); + + it("supports case-insensitive search", async () => { + await mountFixture(); + + const result = await kernel.exec("rg -i needle /project/src", {}); + expect(lines(result.stdout)).toContain("/project/src/lib.rs: // Needle in a comment"); + }); + + it("supports fixed-string search", async () => { + await mountFixture(); + + const result = await kernel.exec("rg -F 'println!(\"needle\")' /project/src", {}); + expect(result.stdout.trim()).toBe('/project/src/main.rs: println!("needle");'); + }); + + it("supports glob filtering", async () => { + await mountFixture(); + + const result = await kernel.exec("rg needle /project -g '*.md'", {}); + expect(result.stdout.trim()).toBe("/project/docs/readme.md:needle in docs"); + }); + + it("can include hidden and ignored files when requested", async () => { + await mountFixture(); + + const result = await kernel.exec("rg -uu needle /project", {}); + const output = lines(result.stdout); + + expect(output).toContain("/project/.hidden.txt:needle hidden"); + expect(output).toContain("/project/vendor/generated.rs:needle in vendor"); + }); + + it("emits JSON search records", async () => { + await mountFixture(); + + const result = await kernel.exec("rg --json needle /project/docs", {}); + const records = lines(result.stdout).map((line) => JSON.parse(line)); + + expect(records.some((record) => record.type === "match")).toBe(true); + expect(records.some((record) => record.type === "summary")).toBe(true); + }); +}); diff --git a/software/sed/test/sed.test.ts b/software/sed/test/sed.test.ts new file mode 100644 index 0000000000..c6ccf420a2 --- /dev/null +++ b/software/sed/test/sed.test.ts @@ -0,0 +1,142 @@ +import { existsSync } from "node:fs"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { + NodeFileSystem, + createKernel, + createWasmVmRuntime, + describeIf, +} from "@agentos/test-harness"; +import type { Kernel } from "@agentos/test-harness"; +import { afterEach, expect, it } from "vitest"; + +const SED_COMMAND_DIR = fileURLToPath(new URL("../bin", import.meta.url)); +const hasSedPackageBinary = existsSync(join(SED_COMMAND_DIR, "sed")); + +let tempRoot: string | undefined; + +async function writeFixture(path: string, contents: string): Promise { + if (!tempRoot) throw new Error("fixture root not initialized"); + const hostPath = join(tempRoot, path.replace(/^\/+/, "")); + await mkdir(dirname(hostPath), { recursive: true }); + await writeFile(hostPath, contents); +} + +async function createTestVFS(): Promise { + tempRoot = await mkdtemp(join(tmpdir(), "agentos-sed-")); + await writeFixture( + "/project/log.txt", + [ + "ERROR: disk full", + "INFO: retrying", + "ERROR: timeout", + "DEBUG: ignored", + ].join("\n") + "\n", + ); + await writeFixture( + "/project/records.txt", + ["alpha:build:4", "beta:test:6", "gamma:deploy:2"].join("\n") + "\n", + ); + return new NodeFileSystem({ root: tempRoot }); +} + +function lines(stdout: string): string[] { + return stdout.split("\n").filter((line) => line.length > 0); +} + +describeIf(hasSedPackageBinary, "sed command", { timeout: 10_000 }, () => { + let kernel: Kernel | undefined; + + afterEach(async () => { + await kernel?.dispose(); + kernel = undefined; + if (tempRoot) { + await rm(tempRoot, { recursive: true, force: true }); + tempRoot = undefined; + } + }); + + async function mountFixture(): Promise { + const vfs = await createTestVFS(); + kernel = createKernel({ filesystem: vfs }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [SED_COMMAND_DIR] })); + } + + async function runSed(args: string[]) { + if (!kernel) throw new Error("kernel not mounted"); + let stdout = ""; + let stderr = ""; + const proc = kernel.spawn("sed", args, { + onStdout: (chunk) => { + stdout += Buffer.from(chunk).toString("utf8"); + }, + onStderr: (chunk) => { + stderr += Buffer.from(chunk).toString("utf8"); + }, + }); + const exitCode = await proc.wait(); + await new Promise((resolve) => setTimeout(resolve, 0)); + return { stdout, stderr, exitCode }; + } + + it("substitutes text in file operands", async () => { + await mountFixture(); + + const result = await runSed(["s/ERROR/WARN/", "/project/log.txt"]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(lines(result.stdout)).toEqual([ + "WARN: disk full", + "INFO: retrying", + "WARN: timeout", + "DEBUG: ignored", + ]); + }); + + it("prints addressed matches with -n", async () => { + await mountFixture(); + + const result = await runSed(["-n", "/ERROR/p", "/project/log.txt"]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(lines(result.stdout)).toEqual(["ERROR: disk full", "ERROR: timeout"]); + }); + + it("deletes addressed records", async () => { + await mountFixture(); + + const result = await runSed(["/DEBUG/d", "/project/log.txt"]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(lines(result.stdout)).toEqual([ + "ERROR: disk full", + "INFO: retrying", + "ERROR: timeout", + ]); + }); + + it("applies multiple expressions in order", async () => { + await mountFixture(); + + const result = await runSed([ + "-e", + "s/:/ /g", + "-e", + "s/^/row /", + "/project/records.txt", + ]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(lines(result.stdout)).toEqual([ + "row alpha build 4", + "row beta test 6", + "row gamma deploy 2", + ]); + }); + + it("fails when an input file is missing", async () => { + await mountFixture(); + + const result = await runSed(["s/a/b/", "/project/missing.txt"]); + expect(result.exitCode).not.toBe(0); + expect(result.stderr).toContain("/project/missing.txt"); + }); +}); diff --git a/software/sqlite3/bin/sqlite3 b/software/sqlite3/bin/sqlite3 index a9ee1f206f..3dbc0a6bb6 100755 Binary files a/software/sqlite3/bin/sqlite3 and b/software/sqlite3/bin/sqlite3 differ diff --git a/software/sqlite3/native/c/sqlite3_cli.c b/software/sqlite3/native/c/sqlite3_cli.c deleted file mode 100644 index ba878b956f..0000000000 --- a/software/sqlite3/native/c/sqlite3_cli.c +++ /dev/null @@ -1,558 +0,0 @@ -/* sqlite3_cli — SQLite CLI for WasmVM - * - * Full sqlite3 CLI supporting: - * sqlite3 :memory: — interactive/piped in-memory DB - * sqlite3 /path/to/db.sqlite — file-based DB via WASI VFS - * echo 'SELECT 1;' | sqlite3 — stdin pipe mode (defaults to :memory:) - * sqlite3 :memory: "SELECT 1;" — SQL from command line argument - * - * Meta-commands: .dump, .schema, .tables, .quit, .help, .headers, .mode - * - * Compiled with -DSQLITE_OS_OTHER — uses a custom VFS wrapping WASI file I/O. - */ -#include -#include -#include -#include -#include -#include -#include -#include "sqlite3.h" - -/* ── WASI VFS ────────────────────────────────────────────────────────── */ - -/* Minimal VFS implementation using POSIX file I/O (supported by WASI) */ - -typedef struct WasiFile { - sqlite3_file base; - int fd; -} WasiFile; - -static int wasiClose(sqlite3_file *pFile) { - WasiFile *p = (WasiFile *)pFile; - if (p->fd >= 0) close(p->fd); - p->fd = -1; - return SQLITE_OK; -} - -static int wasiRead(sqlite3_file *pFile, void *zBuf, int iAmt, sqlite3_int64 iOfst) { - WasiFile *p = (WasiFile *)pFile; - if (lseek(p->fd, (off_t)iOfst, SEEK_SET) != (off_t)iOfst) - return SQLITE_IOERR_READ; - ssize_t got = read(p->fd, zBuf, iAmt); - if (got == iAmt) return SQLITE_OK; - if (got >= 0) { - memset((char *)zBuf + got, 0, iAmt - got); - return SQLITE_IOERR_SHORT_READ; - } - return SQLITE_IOERR_READ; -} - -static int wasiWrite(sqlite3_file *pFile, const void *zBuf, int iAmt, sqlite3_int64 iOfst) { - WasiFile *p = (WasiFile *)pFile; - if (lseek(p->fd, (off_t)iOfst, SEEK_SET) != (off_t)iOfst) - return SQLITE_IOERR_WRITE; - ssize_t wrote = write(p->fd, zBuf, iAmt); - if (wrote == iAmt) return SQLITE_OK; - return SQLITE_IOERR_WRITE; -} - -static int wasiTruncate(sqlite3_file *pFile, sqlite3_int64 size) { - WasiFile *p = (WasiFile *)pFile; - if (ftruncate(p->fd, (off_t)size) != 0) return SQLITE_IOERR_TRUNCATE; - return SQLITE_OK; -} - -static int wasiSync(sqlite3_file *pFile, int flags) { - (void)flags; - WasiFile *p = (WasiFile *)pFile; - if (fsync(p->fd) != 0) return SQLITE_IOERR_FSYNC; - return SQLITE_OK; -} - -static int wasiFileSize(sqlite3_file *pFile, sqlite3_int64 *pSize) { - WasiFile *p = (WasiFile *)pFile; - struct stat st; - if (fstat(p->fd, &st) != 0) return SQLITE_IOERR_FSTAT; - *pSize = (sqlite3_int64)st.st_size; - return SQLITE_OK; -} - -/* No locking in WASM — single process */ -static int wasiLock(sqlite3_file *f, int l) { (void)f; (void)l; return SQLITE_OK; } -static int wasiUnlock(sqlite3_file *f, int l) { (void)f; (void)l; return SQLITE_OK; } -static int wasiCheckReservedLock(sqlite3_file *f, int *pResOut) { - (void)f; *pResOut = 0; return SQLITE_OK; -} -static int wasiFileControl(sqlite3_file *f, int op, void *pArg) { - (void)f; (void)op; (void)pArg; return SQLITE_NOTFOUND; -} -static int wasiSectorSize(sqlite3_file *f) { (void)f; return 512; } -static int wasiDeviceCharacteristics(sqlite3_file *f) { (void)f; return 0; } - -static const sqlite3_io_methods wasiIoMethods __attribute__((used)) = { - 1, /* iVersion */ - wasiClose, - wasiRead, - wasiWrite, - wasiTruncate, - wasiSync, - wasiFileSize, - wasiLock, - wasiUnlock, - wasiCheckReservedLock, - wasiFileControl, - wasiSectorSize, - wasiDeviceCharacteristics, - /* v2+ methods */ - 0, 0, 0, 0 -}; - -static int wasiOpen(sqlite3_vfs *pVfs, sqlite3_filename zName, sqlite3_file *pFile, - int flags, int *pOutFlags) { - (void)pVfs; - WasiFile *p = (WasiFile *)pFile; - p->fd = -1; - - if (!zName) { - /* Temp file — use in-memory only */ - return SQLITE_IOERR; - } - - int oflags = 0; - if (flags & SQLITE_OPEN_CREATE) oflags |= O_CREAT; - if (flags & SQLITE_OPEN_READWRITE) oflags |= O_RDWR; - else oflags |= O_RDONLY; - - p->fd = open(zName, oflags, 0644); - if (p->fd < 0) return SQLITE_CANTOPEN; - - if (pOutFlags) *pOutFlags = flags; - p->base.pMethods = &wasiIoMethods; - return SQLITE_OK; -} - -static int wasiDelete(sqlite3_vfs *pVfs, const char *zName, int syncDir) { - (void)pVfs; (void)syncDir; - if (unlink(zName) != 0 && errno != ENOENT) return SQLITE_IOERR_DELETE; - return SQLITE_OK; -} - -static int wasiAccess(sqlite3_vfs *pVfs, const char *zName, int flags, int *pResOut) { - (void)pVfs; - struct stat st; - *pResOut = (stat(zName, &st) == 0) ? 1 : 0; - (void)flags; - return SQLITE_OK; -} - -static int wasiFullPathname(sqlite3_vfs *pVfs, const char *zName, int nOut, char *zOut) { - (void)pVfs; - if (zName[0] == '/') { - snprintf(zOut, nOut, "%s", zName); - } else { - /* Relative path — prefix with / since VFS root is / */ - snprintf(zOut, nOut, "/%s", zName); - } - return SQLITE_OK; -} - -static int wasiRandomness(sqlite3_vfs *pVfs, int nByte, char *zOut) { - (void)pVfs; - int fd = open("/dev/urandom", O_RDONLY); - if (fd >= 0) { - int total = 0; - while (total < nByte) { - ssize_t read_len = read(fd, zOut + total, (size_t)(nByte - total)); - if (read_len <= 0) { - break; - } - total += (int)read_len; - } - close(fd); - if (total == nByte) { - return nByte; - } - nByte = total; - } - - /* Fallback only if urandom is unexpectedly unavailable in the runtime. */ - for (int i = 0; i < nByte; i++) zOut[i] = (char)(i * 37 + 17); - return nByte; -} - -static int wasiSleep(sqlite3_vfs *pVfs, int microseconds) { - (void)pVfs; (void)microseconds; - return 0; -} - -static int wasiCurrentTime(sqlite3_vfs *pVfs, double *pTime) { - (void)pVfs; - /* Julian day for Unix epoch 0 */ - *pTime = 2440587.5; - return SQLITE_OK; -} - -static int wasiGetLastError(sqlite3_vfs *pVfs, int nBuf, char *zBuf) { - (void)pVfs; - if (nBuf > 0) zBuf[0] = 0; - return 0; -} - -static sqlite3_vfs wasiVfs __attribute__((used)) = { - 1, /* iVersion */ - sizeof(WasiFile), /* szOsFile */ - 512, /* mxPathname */ - 0, /* pNext */ - "wasi", /* zName */ - 0, /* pAppData */ - wasiOpen, - wasiDelete, - wasiAccess, - wasiFullPathname, - 0, 0, 0, 0, /* xDlOpen, xDlError, xDlSym, xDlClose */ - wasiRandomness, - wasiSleep, - wasiCurrentTime, - wasiGetLastError, - /* v2+ */ - 0, 0, 0, 0 -}; - -#ifdef SQLITE_OS_OTHER -int sqlite3_os_init(void) { - return sqlite3_vfs_register(&wasiVfs, 1); -} - -int sqlite3_os_end(void) { - return SQLITE_OK; -} -#endif /* SQLITE_OS_OTHER */ - -/* ── CLI ─────────────────────────────────────────────────────────────── */ - -static int headers_enabled = 0; -static int headers_printed = 0; -static char output_sep_buf[8]; -static char *output_separator = output_sep_buf; - -/* Callback for sqlite3_exec — prints rows in column format */ -static int print_row(void *unused, int ncols, char **values, char **names) { - (void)unused; - if (headers_enabled && !headers_printed) { - for (int i = 0; i < ncols; i++) { - if (i > 0) printf("%s", output_separator); - printf("%s", names[i]); - } - printf("\n"); - headers_printed = 1; - } - for (int i = 0; i < ncols; i++) { - if (i > 0) printf("%s", output_separator); - printf("%s", values[i] ? values[i] : ""); - } - printf("\n"); - return 0; -} - -/* Execute SQL and report errors */ -static int exec_sql(sqlite3 *db, const char *sql) { - char *err = NULL; - headers_printed = 0; /* Reset per query */ - int rc = sqlite3_exec(db, sql, print_row, NULL, &err); - if (rc != SQLITE_OK) { - fprintf(stderr, "Error: %s\n", err ? err : sqlite3_errmsg(db)); - sqlite3_free(err); - return 1; - } - return 0; -} - -/* Handle dot-commands */ -static int handle_meta(sqlite3 *db, const char *line) { - while (*line == ' ' || *line == '\t') line++; - - if (strcmp(line, ".quit") == 0 || strcmp(line, ".exit") == 0) { - return -1; /* Signal exit */ - } - if (strcmp(line, ".help") == 0) { - printf(".dump Dump database as SQL\n"); - printf(".headers on|off Toggle column headers\n"); - printf(".help Show this help\n"); - printf(".mode MODE Set output mode (list, csv)\n"); - printf(".quit Exit\n"); - printf(".schema Show CREATE statements\n"); - printf(".tables List tables\n"); - return 0; - } - if (strcmp(line, ".tables") == 0) { - return exec_sql(db, - "SELECT name FROM sqlite_master WHERE type='table' ORDER BY 1;"); - } - if (strcmp(line, ".schema") == 0) { - return exec_sql(db, - "SELECT sql||';' FROM sqlite_master WHERE sql IS NOT NULL ORDER BY 1;"); - } - if (strcmp(line, ".dump") == 0) { - printf("BEGIN TRANSACTION;\n"); - /* Schema */ - exec_sql(db, - "SELECT sql||';' FROM sqlite_master WHERE sql IS NOT NULL " - "ORDER BY CASE WHEN type='table' THEN 0 ELSE 1 END, name;"); - /* Data — generate INSERT statements for each table */ - sqlite3_stmt *stmt; - int rc = sqlite3_prepare_v2(db, - "SELECT name FROM sqlite_master WHERE type='table' ORDER BY 1;", - -1, &stmt, NULL); - if (rc == SQLITE_OK) { - while (sqlite3_step(stmt) == SQLITE_ROW) { - const char *table = (const char *)sqlite3_column_text(stmt, 0); - char query[1024]; - snprintf(query, sizeof(query), "SELECT * FROM \"%s\"", table); - - sqlite3_stmt *data_stmt; - rc = sqlite3_prepare_v2(db, query, -1, &data_stmt, NULL); - if (rc == SQLITE_OK) { - int ncols = sqlite3_column_count(data_stmt); - while (sqlite3_step(data_stmt) == SQLITE_ROW) { - printf("INSERT INTO \"%s\" VALUES(", table); - for (int i = 0; i < ncols; i++) { - if (i > 0) printf(","); - int type = sqlite3_column_type(data_stmt, i); - if (type == SQLITE_NULL) { - printf("NULL"); - } else if (type == SQLITE_INTEGER) { - printf("%lld", sqlite3_column_int64(data_stmt, i)); - } else if (type == SQLITE_FLOAT) { - printf("%g", sqlite3_column_double(data_stmt, i)); - } else { - const char *text = (const char *)sqlite3_column_text(data_stmt, i); - printf("'"); - for (const char *c = text; *c; c++) { - if (*c == '\'') printf("''"); - else putchar(*c); - } - printf("'"); - } - } - printf(");\n"); - } - sqlite3_finalize(data_stmt); - } - } - sqlite3_finalize(stmt); - } - printf("COMMIT;\n"); - return 0; - } - if (strncmp(line, ".headers ", 9) == 0) { - const char *arg = line + 9; - while (*arg == ' ') arg++; - if (strcmp(arg, "on") == 0) headers_enabled = 1; - else headers_enabled = 0; - return 0; - } - if (strncmp(line, ".mode ", 6) == 0) { - const char *arg = line + 6; - while (*arg == ' ') arg++; - if (strcmp(arg, "csv") == 0) strcpy(output_sep_buf, ","); - else strcpy(output_sep_buf, "|"); - return 0; - } - - fprintf(stderr, "Error: unknown command or invalid arguments: \"%s\"\n", line); - return 0; -} - -/* Read all data from stream into a buffer */ -static char *read_all(FILE *in, size_t *out_len) { - size_t cap = 65536; - size_t len = 0; - char *buf = (char *)malloc(cap); - if (!buf) return NULL; - - int c; - while ((c = fgetc(in)) != EOF) { - if (len + 1 >= cap) { - cap *= 2; - char *nb = (char *)realloc(buf, cap); - if (!nb) { free(buf); return NULL; } - buf = nb; - } - buf[len++] = (char)c; - } - buf[len] = '\0'; - if (out_len) *out_len = len; - return buf; -} - -/* Process a line of input — either meta-command or SQL accumulation */ -static int process_line(sqlite3 *db, const char *line, size_t len, - char *sql_buf, int *sql_len, int sql_cap) { - /* Meta-commands start with . */ - if (line[0] == '.' && *sql_len == 0) { - /* Need a mutable copy for handle_meta */ - char meta_buf[1024]; - size_t ml = len < sizeof(meta_buf) - 1 ? len : sizeof(meta_buf) - 1; - memcpy(meta_buf, line, ml); - meta_buf[ml] = '\0'; - int rc = handle_meta(db, meta_buf); - if (rc < 0) return -1; /* .quit */ - return rc; - } - - /* Accumulate SQL */ - if (*sql_len + (int)len + 2 < sql_cap) { - if (*sql_len > 0) sql_buf[(*sql_len)++] = '\n'; - memcpy(sql_buf + *sql_len, line, len); - *sql_len += (int)len; - sql_buf[*sql_len] = '\0'; - } - - /* Check if statement is complete */ - if (sqlite3_complete(sql_buf)) { - int rc = exec_sql(db, sql_buf); - *sql_len = 0; - return rc; - } - return 0; -} - -/* Read and execute SQL from stdin */ -static int process_input(sqlite3 *db, FILE *in) { - size_t total = 0; - char *data = read_all(in, &total); - if (!data || total == 0) { - free(data); - return 0; - } - - char sql_buf[65536]; - int sql_len = 0; - int had_error = 0; - - /* Process input line by line */ - const char *p = data; - const char *end = data + total; - while (p < end) { - /* Find end of line */ - const char *eol = p; - while (eol < end && *eol != '\n' && *eol != '\r') eol++; - size_t len = eol - p; - - /* Skip empty lines */ - if (len > 0) { - int rc = process_line(db, p, len, sql_buf, &sql_len, (int)sizeof(sql_buf)); - if (rc < 0) break; /* .quit */ - if (rc > 0) had_error = 1; - } - - /* Skip past newline */ - p = eol; - if (p < end && *p == '\r') p++; - if (p < end && *p == '\n') p++; - } - - /* Execute any remaining SQL */ - if (sql_len > 0) { - if (exec_sql(db, sql_buf) != 0) - had_error = 1; - } - - free(data); - return had_error; -} - -static void print_usage(void) { - fprintf(stderr, "Usage: sqlite3 [OPTIONS] [FILENAME] [SQL]\n"); - fprintf(stderr, " FILENAME is the name of an SQLite database. A new database is\n"); - fprintf(stderr, " created if the file does not exist. Use \":memory:\" for in-memory.\n"); - fprintf(stderr, " SQL is an optional SQL statement to execute.\n"); - fprintf(stderr, "Options:\n"); - fprintf(stderr, " -header Turn headers on\n"); - fprintf(stderr, " -csv Set output mode to CSV\n"); - fprintf(stderr, " -separator S Set output separator (default \"|\")\n"); - fprintf(stderr, " -help Show this help\n"); - fprintf(stderr, " -version Show SQLite version\n"); -} - -int main(int argc, char **argv) { - const char *db_path = ":memory:"; - const char *sql_arg = NULL; - int positional = 0; - int i; - - /* Initialize separator at runtime — static initializers for arrays - * may not work correctly in optimized WASM builds. */ - output_sep_buf[0] = '|'; - output_sep_buf[1] = '\0'; - - /* Parse options */ - for (i = 1; i < argc; i++) { - if (argv[i][0] == '-') { - if (strcmp(argv[i], "-help") == 0 || strcmp(argv[i], "--help") == 0) { - print_usage(); - return 0; - } - if (strcmp(argv[i], "-version") == 0 || strcmp(argv[i], "--version") == 0) { - printf("%s\n", sqlite3_libversion()); - return 0; - } - if (strcmp(argv[i], "-header") == 0 || strcmp(argv[i], "-headers") == 0) { - headers_enabled = 1; - continue; - } - if (strcmp(argv[i], "-csv") == 0) { - strcpy(output_sep_buf, ","); - continue; - } - if (strcmp(argv[i], "-separator") == 0 && i + 1 < argc) { - strncpy(output_sep_buf, argv[++i], sizeof(output_sep_buf) - 1); - output_sep_buf[sizeof(output_sep_buf) - 1] = '\0'; - continue; - } - fprintf(stderr, "Error: unknown option: %s\n", argv[i]); - print_usage(); - return 1; - } - /* Positional args: [FILENAME] [SQL] */ - if (positional == 0) { - db_path = argv[i]; - positional++; - } else if (positional == 1) { - sql_arg = argv[i]; - positional++; - } else { - fprintf(stderr, "Error: too many arguments\n"); - print_usage(); - return 1; - } - } - - /* Open database */ - sqlite3 *db; - int rc = sqlite3_open(db_path, &db); - if (rc != SQLITE_OK) { - fprintf(stderr, "Error: unable to open database \"%s\": %s\n", - db_path, sqlite3_errmsg(db)); - sqlite3_close(db); - return 1; - } - - int exit_code = 0; - - if (sql_arg) { - /* Execute SQL from command line */ - exit_code = exec_sql(db, sql_arg); - } else { - /* Read from stdin */ - exit_code = process_input(db, stdin); - } - - /* Flush output */ - fflush(stdout); - fflush(stderr); - /* Skip sqlite3_close + atexit handlers — use _Exit to avoid WASM indirect - * call table traps during wasi-libc/SQLite cleanup with wasm-opt. */ - _Exit(exit_code); -} diff --git a/software/sqlite3/test/sqlite3.test.ts b/software/sqlite3/test/sqlite3.test.ts index b226e82688..2b7144304e 100644 --- a/software/sqlite3/test/sqlite3.test.ts +++ b/software/sqlite3/test/sqlite3.test.ts @@ -7,11 +7,8 @@ * - SQL from command line arguments for multi-statement operations * - Meta-commands (.dump, .schema, .tables) * - * Note: kernel.exec() wraps commands in sh -c. Brush-shell currently returns - * exit code 17 for all child commands. Tests verify stdout correctness. - * - * Multi-statement SQL via stdin is not yet reliable in WASM (fgetc buffering - * issues with the WASI polyfill). Tests use SQL-as-argument for complex cases. + * The command is the official SQLite shell compiled against the AgentOS C + * sysroot, not the former local sqlite3_cli.c reimplementation. */ import { describe, it, expect, afterEach } from 'vitest'; @@ -159,38 +156,55 @@ describeIf(hasSqlite3Binary, 'sqlite3 command', () => { expect(result.stdout.trim()).toBe('10\n20\n30'); }); - it('supports .tables meta-command via SQL setup', async () => { + it('supports .tables meta-command', async () => { const vfs = new SimpleVFS(); + await vfs.mkdir('/tmp', { recursive: true }); kernel = createKernel({ filesystem: vfs as any }); await kernel.mount(createWasmVmRuntime({ commandDirs: SQLITE3_COMMAND_DIRS })); - // Create tables via SQL argument, then query sqlite_master - const sql = "CREATE TABLE alpha(x); CREATE TABLE beta(y); SELECT name FROM sqlite_master WHERE type='table' ORDER BY 1;"; - const result = await kernel.exec(`sqlite3 :memory: "${sql}"`); - const tables = result.stdout.trim().split('\n').sort(); - expect(tables).toEqual(['alpha', 'beta']); + const createResult = await kernel.exec( + 'sqlite3 /tmp/meta.db "CREATE TABLE alpha(x); CREATE TABLE beta(y);"' + ); + expect(createResult.stderr).toBe(''); + + const result = await kernel.exec('sqlite3 /tmp/meta.db ".tables"'); + expect(result.stdout).toContain('alpha'); + expect(result.stdout).toContain('beta'); }); - it('supports .schema via sqlite_master query', async () => { + it('supports .schema meta-command', async () => { const vfs = new SimpleVFS(); + await vfs.mkdir('/tmp', { recursive: true }); kernel = createKernel({ filesystem: vfs as any }); await kernel.mount(createWasmVmRuntime({ commandDirs: SQLITE3_COMMAND_DIRS })); - const sql = "CREATE TABLE users(id INTEGER PRIMARY KEY, name TEXT NOT NULL); SELECT sql FROM sqlite_master WHERE name='users';"; - const result = await kernel.exec(`sqlite3 :memory: "${sql}"`); - expect(result.stdout.trim()).toContain('CREATE TABLE users'); + const createResult = await kernel.exec( + 'sqlite3 /tmp/schema.db "CREATE TABLE users(id INTEGER PRIMARY KEY, name TEXT NOT NULL);"' + ); + expect(createResult.stderr).toBe(''); + + const result = await kernel.exec('sqlite3 /tmp/schema.db ".schema users"'); + expect(result.stdout).toContain('CREATE TABLE users'); + expect(result.stdout).toContain('id INTEGER PRIMARY KEY'); }); - it('supports .dump style output via SQL', async () => { + it('supports .dump meta-command', async () => { const vfs = new SimpleVFS(); + await vfs.mkdir('/tmp', { recursive: true }); kernel = createKernel({ filesystem: vfs as any }); await kernel.mount(createWasmVmRuntime({ commandDirs: SQLITE3_COMMAND_DIRS })); - const sql = "CREATE TABLE t(x INTEGER, y TEXT); INSERT INTO t VALUES(1,'hello'); SELECT sql FROM sqlite_master; SELECT * FROM t;"; - const result = await kernel.exec(`sqlite3 :memory: "${sql}"`); - const output = result.stdout.trim(); + const createResult = await kernel.exec( + `sqlite3 /tmp/dump.db "CREATE TABLE t(x INTEGER, y TEXT); INSERT INTO t VALUES(1,'hello');"` + ); + expect(createResult.stderr).toBe(''); + + const result = await kernel.exec('sqlite3 /tmp/dump.db ".dump"'); + const output = result.stdout; + expect(output).toContain('BEGIN TRANSACTION'); expect(output).toContain('CREATE TABLE t'); - expect(output).toContain("1|hello"); + expect(output).toContain("INSERT INTO t VALUES(1,'hello')"); + expect(output).toContain('COMMIT'); }); it('handles SELECT with multiple columns', async () => { @@ -249,43 +263,54 @@ describeIf(hasSqlite3Binary, 'sqlite3 command', () => { it('.tables meta-command lists created tables', async () => { const vfs = new SimpleVFS(); + await vfs.mkdir('/tmp', { recursive: true }); kernel = createKernel({ filesystem: vfs as any }); await kernel.mount(createWasmVmRuntime({ commandDirs: SQLITE3_COMMAND_DIRS })); - // Multi-statement stdin has fgetc buffering limitations in WASM, - // use SQL command arg to verify table listing behavior - const sql = "CREATE TABLE alpha(x); CREATE TABLE beta(y); SELECT name FROM sqlite_master WHERE type='table' ORDER BY 1;"; - const result = await kernel.exec(`sqlite3 :memory: "${sql}"`); - const tables = result.stdout.trim().split('\n').sort(); - expect(tables).toEqual(['alpha', 'beta']); + const createResult = await kernel.exec( + 'sqlite3 /tmp/tables.db "CREATE TABLE alpha(x); CREATE TABLE beta(y);"' + ); + expect(createResult.stderr).toBe(''); + + const result = await kernel.exec('sqlite3 /tmp/tables.db ".tables"'); + expect(result.stdout).toContain('alpha'); + expect(result.stdout).toContain('beta'); }); it('.schema meta-command shows CREATE TABLE statements', async () => { const vfs = new SimpleVFS(); + await vfs.mkdir('/tmp', { recursive: true }); kernel = createKernel({ filesystem: vfs as any }); await kernel.mount(createWasmVmRuntime({ commandDirs: SQLITE3_COMMAND_DIRS })); - // Query schema via sqlite_master (equivalent to .schema output) - const sql = "CREATE TABLE users(id INTEGER PRIMARY KEY, name TEXT NOT NULL); SELECT sql FROM sqlite_master WHERE name='users';"; - const result = await kernel.exec(`sqlite3 :memory: "${sql}"`); + const createResult = await kernel.exec( + 'sqlite3 /tmp/schema-meta.db "CREATE TABLE users(id INTEGER PRIMARY KEY, name TEXT NOT NULL);"' + ); + expect(createResult.stderr).toBe(''); + + const result = await kernel.exec('sqlite3 /tmp/schema-meta.db ".schema users"'); expect(result.stdout).toContain('CREATE TABLE users'); expect(result.stdout).toContain('id INTEGER PRIMARY KEY'); }); it('.dump meta-command outputs INSERT statements for data', async () => { const vfs = new SimpleVFS(); + await vfs.mkdir('/tmp', { recursive: true }); kernel = createKernel({ filesystem: vfs as any }); await kernel.mount(createWasmVmRuntime({ commandDirs: SQLITE3_COMMAND_DIRS })); - // Verify dump-equivalent output: schema + data via SQL queries - const sql = "CREATE TABLE t(x INTEGER, y TEXT); INSERT INTO t VALUES(1,'hello'); INSERT INTO t VALUES(2,'world'); SELECT sql FROM sqlite_master WHERE name='t'; SELECT '---'; SELECT x||','||y FROM t ORDER BY x;"; - const result = await kernel.exec(`sqlite3 :memory: "${sql}"`); + const createResult = await kernel.exec( + `sqlite3 /tmp/dump-meta.db "CREATE TABLE t(x INTEGER, y TEXT); INSERT INTO t VALUES(1,'hello'); INSERT INTO t VALUES(2,'world');"` + ); + expect(createResult.stderr).toBe(''); + + const result = await kernel.exec('sqlite3 /tmp/dump-meta.db ".dump"'); const output = result.stdout; - // Schema is preserved + expect(output).toContain('BEGIN TRANSACTION'); expect(output).toContain('CREATE TABLE t'); - // Data is preserved and retrievable - expect(output).toContain("1,hello"); - expect(output).toContain("2,world"); + expect(output).toContain("INSERT INTO t VALUES(1,'hello')"); + expect(output).toContain("INSERT INTO t VALUES(2,'world')"); + expect(output).toContain('COMMIT'); }); it('file-based DB persists data across separate exec calls', async () => { diff --git a/software/tar/bin/tar b/software/tar/bin/tar index 5d813b99d9..770b86e306 100644 Binary files a/software/tar/bin/tar and b/software/tar/bin/tar differ diff --git a/software/tar/native/crates/tar/src/lib.rs b/software/tar/native/crates/tar/src/lib.rs index d20f009a37..db039cff45 100644 --- a/software/tar/native/crates/tar/src/lib.rs +++ b/software/tar/native/crates/tar/src/lib.rs @@ -292,7 +292,9 @@ fn append_path( } increment_entry_count(entry_count)?; - let meta = fs::symlink_metadata(&disk_path)?; + let meta = fs::symlink_metadata(&disk_path).map_err(|e| { + io::Error::new(e.kind(), format!("metadata {}: {}", disk_path.display(), e)) + })?; if meta.is_dir() { append_dir( @@ -548,11 +550,12 @@ fn do_list(archive_file: Option<&str>, gzip: bool, verbose: bool) -> io::Result< let entry = entry_result?; let path = entry.path()?; + let entry_type = entry.header().entry_type(); if verbose { let h = entry.header(); let size = h.size().unwrap_or(0); let mode = h.mode().unwrap_or(0o644); - let type_ch = match h.entry_type() { + let type_ch = match entry_type { tar::EntryType::Directory => 'd', tar::EntryType::Symlink => 'l', _ => '-', @@ -565,6 +568,8 @@ fn do_list(archive_file: Option<&str>, gzip: bool, verbose: bool) -> io::Result< size, path.display() )?; + } else if entry_type == tar::EntryType::Directory { + writeln!(out, "{}/", path.display())?; } else { writeln!(out, "{}", path.display())?; } diff --git a/software/tar/test/tar.test.ts b/software/tar/test/tar.test.ts new file mode 100644 index 0000000000..adebae14c4 --- /dev/null +++ b/software/tar/test/tar.test.ts @@ -0,0 +1,165 @@ +import { existsSync } from "node:fs"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { + NodeFileSystem, + createKernel, + createWasmVmRuntime, + describeIf, +} from "@agentos/test-harness"; +import type { Kernel } from "@agentos/test-harness"; +import { afterEach, expect, it } from "vitest"; + +const TAR_COMMAND_DIR = fileURLToPath(new URL("../bin", import.meta.url)); +const hasTarPackageBinary = existsSync(join(TAR_COMMAND_DIR, "tar")); + +let tempRoot: string | undefined; + +function hostPath(path: string): string { + if (!tempRoot) throw new Error("fixture root not initialized"); + return join(tempRoot, path.replace(/^\/+/, "")); +} + +async function writeFixture(path: string, contents: string): Promise { + const target = hostPath(path); + await mkdir(dirname(target), { recursive: true }); + await writeFile(target, contents); +} + +async function createTestVFS(): Promise { + tempRoot = await mkdtemp(join(tmpdir(), "agentos-tar-")); + await writeFixture("/project/src/docs/readme.txt", "hello from docs\n"); + await writeFixture("/project/src/data/values.csv", "name,score\nAda,7\nLinus,3\n"); + await mkdir(hostPath("/project/out"), { recursive: true }); + await mkdir(hostPath("/project/strip"), { recursive: true }); + await mkdir(hostPath("/project/gzip-out"), { recursive: true }); + return new NodeFileSystem({ root: tempRoot }); +} + +function lines(stdout: string): string[] { + return stdout.split("\n").filter((line) => line.length > 0); +} + +const textDecoder = new TextDecoder(); + +describeIf(hasTarPackageBinary, "tar command", { timeout: 10_000 }, () => { + let kernel: Kernel | undefined; + + afterEach(async () => { + await kernel?.dispose(); + kernel = undefined; + if (tempRoot) { + await rm(tempRoot, { recursive: true, force: true }); + tempRoot = undefined; + } + }); + + async function mountFixture(): Promise { + const vfs = await createTestVFS(); + kernel = createKernel({ filesystem: vfs }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [TAR_COMMAND_DIR] })); + } + + async function runTar(args: string[]) { + if (!kernel) throw new Error("kernel not mounted"); + let stdout = ""; + let stderr = ""; + const proc = kernel.spawn("tar", args, { + onStdout: (chunk) => { + stdout += Buffer.from(chunk).toString("utf8"); + }, + onStderr: (chunk) => { + stderr += Buffer.from(chunk).toString("utf8"); + }, + }); + const exitCode = await proc.wait(); + await new Promise((resolve) => setTimeout(resolve, 0)); + return { stdout, stderr, exitCode }; + } + + async function createArchive(path = "/project/archive.tar") { + const result = await runTar(["-cf", path, "-C", "/project", "src"]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + } + + async function readGuestText(path: string): Promise { + if (!kernel) throw new Error("kernel not mounted"); + return textDecoder.decode(await kernel.readFile(path)); + } + + it("creates and lists file-backed archives", async () => { + await mountFixture(); + await createArchive(); + + const result = await runTar(["-tf", "/project/archive.tar"]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(lines(result.stdout)).toEqual( + expect.arrayContaining([ + "src/", + "src/docs/", + "src/docs/readme.txt", + "src/data/", + "src/data/values.csv", + ]), + ); + }); + + it("extracts archives into target directories", async () => { + await mountFixture(); + await createArchive(); + + const result = await runTar(["-xf", "/project/archive.tar", "-C", "/project/out"]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + await expect(readGuestText("/project/out/src/docs/readme.txt")).resolves.toBe( + "hello from docs\n", + ); + await expect(readGuestText("/project/out/src/data/values.csv")).resolves.toBe( + "name,score\nAda,7\nLinus,3\n", + ); + }); + + it("auto-detects gzip archives by extension", async () => { + await mountFixture(); + + const create = await runTar(["-czf", "/project/archive.tgz", "-C", "/project", "src"]); + expect(create.exitCode, create.stderr || create.stdout).toBe(0); + + const list = await runTar(["-tf", "/project/archive.tgz"]); + expect(list.exitCode, list.stderr || list.stdout).toBe(0); + expect(lines(list.stdout)).toContain("src/docs/readme.txt"); + + const extract = await runTar(["-xf", "/project/archive.tgz", "-C", "/project/gzip-out"]); + expect(extract.exitCode, extract.stderr || extract.stdout).toBe(0); + await expect(readGuestText("/project/gzip-out/src/docs/readme.txt")).resolves.toBe( + "hello from docs\n", + ); + }); + + it("strips path components on extraction", async () => { + await mountFixture(); + await createArchive(); + + const result = await runTar([ + "-xf", + "/project/archive.tar", + "-C", + "/project/strip", + "--strip-components=1", + ]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + await expect(readGuestText("/project/strip/docs/readme.txt")).resolves.toBe( + "hello from docs\n", + ); + await expect(readGuestText("/project/strip/src/docs/readme.txt")).rejects.toThrow(); + }); + + it("fails when a create input is missing", async () => { + await mountFixture(); + + const result = await runTar(["-cf", "/project/missing.tar", "-C", "/project", "nope"]); + expect(result.exitCode).not.toBe(0); + expect(result.stderr).toContain("nope"); + }); +}); diff --git a/software/tree/bin/tree b/software/tree/bin/tree old mode 100644 new mode 100755 index 49e9c7c18d..811c63e5fd Binary files a/software/tree/bin/tree and b/software/tree/bin/tree differ diff --git a/software/tree/native/crates/cmd-tree/Cargo.toml b/software/tree/native/crates/cmd-tree/Cargo.toml deleted file mode 100644 index e85593575d..0000000000 --- a/software/tree/native/crates/cmd-tree/Cargo.toml +++ /dev/null @@ -1,14 +0,0 @@ -[package] -workspace = "../../../../../toolchain" -name = "cmd-tree" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "tree standalone binary for secure-exec VM" - -[[bin]] -name = "tree" -path = "src/main.rs" - -[dependencies] -secureexec-tree = { path = "../tree" } diff --git a/software/tree/native/crates/cmd-tree/src/main.rs b/software/tree/native/crates/cmd-tree/src/main.rs deleted file mode 100644 index ee799a4287..0000000000 --- a/software/tree/native/crates/cmd-tree/src/main.rs +++ /dev/null @@ -1,4 +0,0 @@ -fn main() { - let args: Vec = std::env::args_os().collect(); - std::process::exit(secureexec_tree::main(args)); -} diff --git a/software/tree/native/crates/tree/Cargo.toml b/software/tree/native/crates/tree/Cargo.toml deleted file mode 100644 index e316d0fda4..0000000000 --- a/software/tree/native/crates/tree/Cargo.toml +++ /dev/null @@ -1,9 +0,0 @@ -[package] -workspace = "../../../../../toolchain" -name = "secureexec-tree" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "tree implementation for secure-exec standalone binaries" - -[dependencies] diff --git a/software/tree/native/crates/tree/src/lib.rs b/software/tree/native/crates/tree/src/lib.rs deleted file mode 100644 index 71cfb7044c..0000000000 --- a/software/tree/native/crates/tree/src/lib.rs +++ /dev/null @@ -1,268 +0,0 @@ -//! tree -- list directory contents in a tree-like format -//! -//! Recursive directory walk with box-drawing characters. -//! Supports -a (show hidden), -d (dirs only), -L depth, -I exclude pattern. - -use std::ffi::OsString; -use std::fs; -use std::io::{self, Write}; -use std::path::Path; - -const DEFAULT_MAX_DEPTH: usize = 256; -const MAX_TOTAL_ENTRIES: usize = 100_000; -const MAX_DIRECTORY_ENTRIES: usize = 100_000; - -pub fn main(args: Vec) -> i32 { - let str_args: Vec = args - .iter() - .skip(1) - .map(|a| a.to_string_lossy().to_string()) - .collect(); - - let mut show_hidden = false; - let mut dirs_only = false; - let mut max_depth: Option = None; - let mut exclude_pattern: Option = None; - let mut paths: Vec = Vec::new(); - - let mut i = 0; - while i < str_args.len() { - match str_args[i].as_str() { - "-a" => show_hidden = true, - "-d" => dirs_only = true, - "-L" => { - i += 1; - if i >= str_args.len() { - eprintln!("tree: option '-L' requires an argument"); - return 1; - } - match str_args[i].parse::() { - Ok(d) => max_depth = Some(d), - Err(_) => { - eprintln!("tree: invalid level '{}'", str_args[i]); - return 1; - } - } - } - "-I" => { - i += 1; - if i >= str_args.len() { - eprintln!("tree: option '-I' requires an argument"); - return 1; - } - exclude_pattern = Some(str_args[i].clone()); - } - s if s.starts_with('-') => { - eprintln!("tree: unknown option '{}'", s); - return 1; - } - _ => paths.push(str_args[i].clone()), - } - i += 1; - } - - if paths.is_empty() { - paths.push(".".to_string()); - } - - let stdout = io::stdout(); - let mut out = stdout.lock(); - let mut dir_count: usize = 0; - let mut file_count: usize = 0; - - for (idx, path) in paths.iter().enumerate() { - if let Err(e) = writeln!(out, "{}", path).and_then(|_| { - walk_tree( - Path::new(path), - "", - 1, - max_depth, - show_hidden, - dirs_only, - exclude_pattern.as_deref(), - &mut dir_count, - &mut file_count, - &mut out, - ) - }) { - eprintln!("tree: {}", e); - return 1; - } - if idx + 1 < paths.len() { - if let Err(e) = writeln!(out) { - eprintln!("tree: {}", e); - return 1; - } - } - } - - if let Err(e) = writeln!(out) { - eprintln!("tree: {}", e); - return 1; - } - if dirs_only { - if let Err(e) = writeln!( - out, - "{} director{}", - dir_count, - if dir_count == 1 { "y" } else { "ies" } - ) { - eprintln!("tree: {}", e); - return 1; - } - } else { - if let Err(e) = writeln!( - out, - "{} director{}, {} file{}", - dir_count, - if dir_count == 1 { "y" } else { "ies" }, - file_count, - if file_count == 1 { "" } else { "s" } - ) { - eprintln!("tree: {}", e); - return 1; - } - } - - match out.flush() { - Ok(()) => 0, - Err(e) => { - eprintln!("tree: {}", e); - 1 - } - } -} - -fn matches_exclude(name: &str, pattern: &str) -> bool { - // Simple glob matching: supports * as wildcard - if pattern.contains('*') { - let parts: Vec<&str> = pattern.split('*').collect(); - if parts.len() == 2 { - let (prefix, suffix) = (parts[0], parts[1]); - return name.starts_with(prefix) && name.ends_with(suffix); - } - // Fallback: check if any non-wildcard part matches - parts.iter().all(|p| p.is_empty() || name.contains(p)) - } else { - name == pattern - } -} - -fn walk_tree( - dir: &Path, - prefix: &str, - depth: usize, - max_depth: Option, - show_hidden: bool, - dirs_only: bool, - exclude: Option<&str>, - dir_count: &mut usize, - file_count: &mut usize, - out: &mut W, -) -> io::Result<()> { - if let Some(max) = max_depth { - if depth > max { - return Ok(()); - } - } else if depth > DEFAULT_MAX_DEPTH { - return Ok(()); - } - - let mut entries: Vec = match fs::read_dir(dir) { - Ok(rd) => { - let mut entries = Vec::new(); - for entry_result in rd { - let entry = entry_result?; - if entries.len() >= MAX_DIRECTORY_ENTRIES { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!("too many entries in {}", dir.display()), - )); - } - entries.push(entry); - } - entries - } - Err(e) => { - writeln!(out, "{}[error opening dir: {}]", prefix, e)?; - return Ok(()); - } - }; - - // Sort entries alphabetically - entries.sort_by(|a, b| a.file_name().cmp(&b.file_name())); - - // Filter entries - entries.retain(|e| { - let name = e.file_name().to_string_lossy().to_string(); - // Skip hidden unless -a - if !show_hidden && name.starts_with('.') { - return false; - } - // Skip excluded patterns - if let Some(pat) = exclude { - if matches_exclude(&name, pat) { - return false; - } - } - // Skip files if -d - if dirs_only { - if let Ok(ft) = e.file_type() { - if !ft.is_dir() { - return false; - } - } - } - true - }); - - let count = entries.len(); - - for (idx, entry) in entries.iter().enumerate() { - if *dir_count + *file_count >= MAX_TOTAL_ENTRIES { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "too many tree entries", - )); - } - - let is_last = idx + 1 == count; - let connector = if is_last { - "\u{2514}\u{2500}\u{2500} " // └── - } else { - "\u{251c}\u{2500}\u{2500} " // ├── - }; - - let name = entry.file_name().to_string_lossy().to_string(); - let file_type = entry.file_type()?; - let is_dir = file_type.is_dir() && !file_type.is_symlink(); - - write!(out, "{}{}", prefix, connector)?; - writeln!(out, "{}", name)?; - - if is_dir { - *dir_count += 1; - let child_prefix = if is_last { - format!("{} ", prefix) - } else { - format!("{}\u{2502} ", prefix) // │ - }; - walk_tree( - &dir.join(&name), - &child_prefix, - depth + 1, - max_depth, - show_hidden, - dirs_only, - exclude, - dir_count, - file_count, - out, - )?; - } else { - *file_count += 1; - } - } - - Ok(()) -} diff --git a/software/tree/package.json b/software/tree/package.json index bca79a144c..45d2aee6d2 100644 --- a/software/tree/package.json +++ b/software/tree/package.json @@ -2,7 +2,7 @@ "name": "@agentos-software/tree", "version": "0.3.3", "type": "module", - "license": "Apache-2.0", + "license": "GPL-2.0-or-later", "description": "tree directory listing for secure-exec VMs", "main": "./dist/index.js", "types": "./dist/index.d.ts", diff --git a/software/tree/test/tree-test.test.ts b/software/tree/test/tree-test.test.ts index 4ad659d78f..0c505f5ff0 100644 --- a/software/tree/test/tree-test.test.ts +++ b/software/tree/test/tree-test.test.ts @@ -65,8 +65,8 @@ describeIf(!wasmSkip, 'tree command behavior', () => { expect(result.stdout).toContain('types.ts'); expect(result.stdout).toContain('index.ts'); expect(result.stdout).toContain('README.md'); - // Should show 2 directories (src, lib) and 4 files - expect(result.stdout).toMatch(/2 director/); + // Upstream tree reports the displayed root plus src/lib. + expect(result.stdout).toMatch(/3 director/); expect(result.stdout).toMatch(/4 file/); }, TREE_TEST_TIMEOUT_MS); diff --git a/software/unzip/bin/unzip b/software/unzip/bin/unzip old mode 100644 new mode 100755 index 07f40041e1..3bc6f3f020 Binary files a/software/unzip/bin/unzip and b/software/unzip/bin/unzip differ diff --git a/software/unzip/native/c/unzip.c b/software/unzip/native/c/unzip.c deleted file mode 100644 index 383e714205..0000000000 --- a/software/unzip/native/c/unzip.c +++ /dev/null @@ -1,669 +0,0 @@ -/* unzip.c — Extract ZIP archives using zlib/minizip - * - * Usage: unzip archive.zip (extract all to cwd) - * unzip -d outdir archive.zip (extract to directory) - * unzip -l archive.zip (list contents) - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include "ioapi.h" -#include "unzip.h" - -#define MAX_PATH_LEN 4096 -#define WRITE_BUF_SIZE 8192 - -/* Cap per-entry allocation in the fallback parser. Hostile central directory - * records can claim sizes up to 4 GiB; refuse anything above this bound. */ -#define MAX_UNCOMPRESSED_SIZE (256u * 1024u * 1024u) - -typedef struct { - FILE *file; - char *filename; - char mode[4]; - long position; - long size; -} unzip_file_stream; - -static voidpf ZCALLBACK unzip_open_file(voidpf opaque, const char *filename, int mode) { - unzip_file_stream *stream = NULL; - const char *mode_fopen = NULL; - (void)opaque; - - if ((mode & ZLIB_FILEFUNC_MODE_READWRITEFILTER) == ZLIB_FILEFUNC_MODE_READ) - mode_fopen = "rb"; - else if (mode & ZLIB_FILEFUNC_MODE_EXISTING) - mode_fopen = "r+b"; - else if (mode & ZLIB_FILEFUNC_MODE_CREATE) - mode_fopen = "wb"; - - if (filename == NULL || mode_fopen == NULL) - return NULL; - - stream = (unzip_file_stream *)calloc(1, sizeof(unzip_file_stream)); - if (!stream) - return NULL; - - stream->filename = (char *)malloc(strlen(filename) + 1); - if (!stream->filename) { - free(stream); - return NULL; - } - strcpy(stream->filename, filename); - strncpy(stream->mode, mode_fopen, sizeof(stream->mode) - 1); - stream->mode[sizeof(stream->mode) - 1] = '\0'; - - stream->file = fopen(filename, mode_fopen); - if (!stream->file) { - free(stream->filename); - free(stream); - return NULL; - } - struct stat st; - stream->size = stat(filename, &st) == 0 ? (long)st.st_size : 0; - stream->position = 0; - return stream; -} - -static uLong ZCALLBACK unzip_read_file(voidpf opaque, voidpf stream, void *buf, uLong size) { - unzip_file_stream *file_stream = (unzip_file_stream *)stream; - uLong got; - (void)opaque; - got = (uLong)fread(buf, 1, (size_t)size, file_stream->file); - file_stream->position += (long)got; - return got; -} - -static uLong ZCALLBACK unzip_write_file(voidpf opaque, voidpf stream, const void *buf, uLong size) { - unzip_file_stream *file_stream = (unzip_file_stream *)stream; - uLong wrote; - (void)opaque; - wrote = (uLong)fwrite(buf, 1, (size_t)size, file_stream->file); - file_stream->position += (long)wrote; - if (file_stream->position > file_stream->size) - file_stream->size = file_stream->position; - return wrote; -} - -static long ZCALLBACK unzip_tell_file(voidpf opaque, voidpf stream) { - unzip_file_stream *file_stream = (unzip_file_stream *)stream; - (void)opaque; - return file_stream->position; -} - -static long ZCALLBACK unzip_seek_file(voidpf opaque, voidpf stream, uLong offset, int origin) { - int fseek_origin = 0; - long seek_offset = (long)offset; - unzip_file_stream *file_stream = (unzip_file_stream *)stream; - (void)opaque; - - switch (origin) { - case ZLIB_FILEFUNC_SEEK_CUR: - seek_offset = file_stream->position + (long)offset; - fseek_origin = SEEK_SET; - break; - case ZLIB_FILEFUNC_SEEK_END: - seek_offset = file_stream->size + (long)offset; - fseek_origin = SEEK_SET; - break; - case ZLIB_FILEFUNC_SEEK_SET: - fseek_origin = SEEK_SET; - break; - default: - return -1; - } - - fclose(file_stream->file); - file_stream->file = fopen(file_stream->filename, file_stream->mode); - if (!file_stream->file) - return -1; - - if (fseek(file_stream->file, seek_offset, fseek_origin) != 0) - return -1; - clearerr(file_stream->file); - file_stream->position = seek_offset; - return 0; -} - -static int ZCALLBACK unzip_close_file(voidpf opaque, voidpf stream) { - unzip_file_stream *file_stream = (unzip_file_stream *)stream; - int ret; - (void)opaque; - ret = fclose(file_stream->file); - free(file_stream->filename); - free(file_stream); - return ret; -} - -static int ZCALLBACK unzip_error_file(voidpf opaque, voidpf stream) { - unzip_file_stream *file_stream = (unzip_file_stream *)stream; - (void)opaque; - return ferror(file_stream->file); -} - -static unzFile open_archive(const char *archive) { - zlib_filefunc_def filefunc = { - .zopen_file = unzip_open_file, - .zread_file = unzip_read_file, - .zwrite_file = unzip_write_file, - .ztell_file = unzip_tell_file, - .zseek_file = unzip_seek_file, - .zclose_file = unzip_close_file, - .zerror_file = unzip_error_file, - .opaque = NULL, - }; - return unzOpen2(archive, &filefunc); -} - -/* Ensure all parent directories of path exist */ -static int mkdirs(const char *path) { - char tmp[MAX_PATH_LEN]; - size_t len = strlen(path); - if (len >= sizeof(tmp)) return -1; - memcpy(tmp, path, len + 1); - - for (size_t i = 1; i < len; i++) { - if (tmp[i] == '/') { - tmp[i] = '\0'; - if (mkdir(tmp, 0755) != 0 && errno != EEXIST) - return -1; - tmp[i] = '/'; - } - } - return 0; -} - -static uint16_t read_le16(const unsigned char *p) { - return (uint16_t)p[0] | ((uint16_t)p[1] << 8); -} - -static uint32_t read_le32(const unsigned char *p) { - return (uint32_t)p[0] | ((uint32_t)p[1] << 8) | - ((uint32_t)p[2] << 16) | ((uint32_t)p[3] << 24); -} - -static int read_archive_bytes(const char *archive, unsigned char **out, size_t *out_len) { - FILE *f = fopen(archive, "rb"); - long size; - unsigned char *data; - if (!f) - return -1; - if (fseek(f, 0, SEEK_END) != 0) { - fclose(f); - return -1; - } - size = ftell(f); - if (size < 0 || fseek(f, 0, SEEK_SET) != 0) { - fclose(f); - return -1; - } - data = (unsigned char *)malloc((size_t)size); - if (!data) { - fclose(f); - return -1; - } - if (fread(data, 1, (size_t)size, f) != (size_t)size) { - free(data); - fclose(f); - return -1; - } - fclose(f); - *out = data; - *out_len = (size_t)size; - return 0; -} - -static int find_eocd(const unsigned char *data, size_t len, size_t *eocd_offset) { - size_t min = len > 0xffff + 22 ? len - (0xffff + 22) : 0; - if (len < 22) - return -1; - for (size_t pos = len - 22; pos + 4 <= len && pos >= min; pos--) { - if (read_le32(data + pos) == 0x06054b50) { - *eocd_offset = pos; - return 0; - } - if (pos == 0) - break; - } - return -1; -} - -static const char *entry_output_name(const char *name, size_t name_len) { - const char *end = name + name_len; - while (name < end && *name == '/') - name++; - return name; -} - -static int inflate_raw_entry(const unsigned char *src, size_t src_len, unsigned char *dst, size_t dst_len) { - z_stream stream; - memset(&stream, 0, sizeof(stream)); - stream.next_in = (Bytef *)src; - stream.avail_in = (uInt)src_len; - stream.next_out = dst; - stream.avail_out = (uInt)dst_len; - if (inflateInit2(&stream, -MAX_WBITS) != Z_OK) - return -1; - int result = inflate(&stream, Z_FINISH); - inflateEnd(&stream); - return result == Z_STREAM_END && stream.total_out == dst_len ? 0 : -1; -} - -static int simple_archive_entries(const unsigned char *data, size_t len, size_t *cd_offset, uint16_t *entry_count) { - size_t eocd; - if (len < 22 || find_eocd(data, len, &eocd) != 0 || eocd > len - 22) - return -1; - *entry_count = read_le16(data + eocd + 10); - *cd_offset = read_le32(data + eocd + 16); - return *cd_offset < len ? 0 : -1; -} - -static int simple_list_archive(const char *archive) { - unsigned char *data = NULL; - size_t len = 0; - size_t pos; - uint16_t entries; - unsigned long total_size = 0; - if (read_archive_bytes(archive, &data, &len) != 0 || - simple_archive_entries(data, len, &pos, &entries) != 0) { - free(data); - return 1; - } - - printf(" Length Name\n"); - printf("--------- ----\n"); - for (uint16_t i = 0; i < entries; i++) { - uint16_t name_len; - uint16_t extra_len; - uint16_t comment_len; - uint32_t uncompressed_size; - if (len < 46 || pos > len - 46 || read_le32(data + pos) != 0x02014b50) { - free(data); - return 1; - } - uncompressed_size = read_le32(data + pos + 24); - name_len = read_le16(data + pos + 28); - extra_len = read_le16(data + pos + 30); - comment_len = read_le16(data + pos + 32); - size_t header_len = 46 + (size_t)name_len + (size_t)extra_len + (size_t)comment_len; - if (header_len > len - pos) { - free(data); - return 1; - } - printf("%9lu %.*s\n", (unsigned long)uncompressed_size, name_len, data + pos + 46); - total_size += uncompressed_size; - pos += header_len; - } - printf("--------- ----\n"); - printf("%9lu %u file(s)\n", total_size, entries); - free(data); - return 0; -} - -static int simple_extract_archive(const char *archive, const char *outdir) { - unsigned char *data = NULL; - size_t len = 0; - size_t pos; - uint16_t entries; - int errors = 0; - if (read_archive_bytes(archive, &data, &len) != 0 || - simple_archive_entries(data, len, &pos, &entries) != 0) { - free(data); - return 1; - } - - if (outdir && mkdir(outdir, 0755) != 0 && errno != EEXIST) { - fprintf(stderr, "unzip: cannot create directory '%s': %s\n", outdir, strerror(errno)); - free(data); - return 1; - } - - for (uint16_t i = 0; i < entries; i++) { - uint16_t method; - uint16_t name_len; - uint16_t extra_len; - uint16_t comment_len; - uint16_t local_name_len; - uint16_t local_extra_len; - uint32_t compressed_size; - uint32_t uncompressed_size; - uint32_t local_offset; - size_t file_data_offset; - const char *name; - const char *safe_name; - char outpath[MAX_PATH_LEN]; - unsigned char *out = NULL; - - if (len < 46 || pos > len - 46 || read_le32(data + pos) != 0x02014b50) { - errors++; - break; - } - method = read_le16(data + pos + 10); - compressed_size = read_le32(data + pos + 20); - uncompressed_size = read_le32(data + pos + 24); - name_len = read_le16(data + pos + 28); - extra_len = read_le16(data + pos + 30); - comment_len = read_le16(data + pos + 32); - local_offset = read_le32(data + pos + 42); - size_t header_len = 46 + (size_t)name_len + (size_t)extra_len + (size_t)comment_len; - if (header_len > len - pos || (size_t)local_offset > len - 30) { - errors++; - break; - } - - name = (const char *)(data + pos + 46); - safe_name = entry_output_name(name, name_len); - size_t safe_len = (size_t)name_len - (size_t)(safe_name - name); - pos += header_len; - if (safe_len == 0) - continue; - snprintf(outpath, sizeof(outpath), "%s%s%.*s", - outdir ? outdir : "", outdir ? "/" : "", (int)safe_len, safe_name); - - size_t out_len = strlen(outpath); - if (out_len > 0 && outpath[out_len - 1] == '/') { - if (mkdir(outpath, 0755) != 0 && errno != EEXIST) - errors++; - continue; - } - if (mkdirs(outpath) != 0) { - errors++; - continue; - } - - if (read_le32(data + local_offset) != 0x04034b50) { - errors++; - continue; - } - local_name_len = read_le16(data + local_offset + 26); - local_extra_len = read_le16(data + local_offset + 28); - size_t local_header_len = 30 + (size_t)local_name_len + (size_t)local_extra_len; - if (local_header_len > len - (size_t)local_offset) { - errors++; - continue; - } - file_data_offset = (size_t)local_offset + local_header_len; - if ((size_t)compressed_size > len - file_data_offset) { - errors++; - continue; - } - - if (uncompressed_size > MAX_UNCOMPRESSED_SIZE) { - fprintf(stderr, "unzip: entry '%.*s' too large (%lu bytes)\n", - (int)safe_len, safe_name, (unsigned long)uncompressed_size); - errors++; - continue; - } - out = (unsigned char *)malloc(uncompressed_size > 0 ? uncompressed_size : 1); - if (!out) { - errors++; - continue; - } - if (method == 0) { - if (compressed_size != uncompressed_size) { - errors++; - free(out); - continue; - } - memcpy(out, data + file_data_offset, uncompressed_size); - } else if (method == Z_DEFLATED) { - if (inflate_raw_entry(data + file_data_offset, compressed_size, out, uncompressed_size) != 0) { - errors++; - free(out); - continue; - } - } else { - fprintf(stderr, "unzip: unsupported compression method %u for '%.*s'\n", method, name_len, name); - errors++; - free(out); - continue; - } - - int fd = open(outpath, O_WRONLY | O_CREAT | O_TRUNC, 0644); - if (fd < 0) { - fprintf(stderr, "unzip: cannot create '%s': %s\n", outpath, strerror(errno)); - errors++; - free(out); - continue; - } - size_t written = 0; - while (written < uncompressed_size) { - ssize_t n = write(fd, out + written, uncompressed_size - written); - if (n <= 0) { - errors++; - break; - } - written += (size_t)n; - } - close(fd); - free(out); - } - - free(data); - if (errors > 0) { - fprintf(stderr, "unzip: completed with %d error(s)\n", errors); - return 1; - } - return 0; -} - -/* List archive contents */ -static int list_archive(const char *archive) { - unzFile uf = open_archive(archive); - if (!uf) { - return simple_list_archive(archive); - } - - unz_global_info gi; - if (unzGetGlobalInfo(uf, &gi) != UNZ_OK) { - fprintf(stderr, "unzip: cannot read archive info\n"); - unzClose(uf); - return 1; - } - - printf(" Length Name\n"); - printf("--------- ----\n"); - - unsigned long total_size = 0; - for (uLong i = 0; i < gi.number_entry; i++) { - char filename[MAX_PATH_LEN]; - unz_file_info fi; - if (unzGetCurrentFileInfo(uf, &fi, filename, sizeof(filename), - NULL, 0, NULL, 0) != UNZ_OK) { - fprintf(stderr, "unzip: error reading file info\n"); - unzClose(uf); - return 1; - } - - printf("%9lu %s\n", fi.uncompressed_size, filename); - total_size += fi.uncompressed_size; - - if (i + 1 < gi.number_entry) { - if (unzGoToNextFile(uf) != UNZ_OK) { - fprintf(stderr, "unzip: error iterating archive\n"); - unzClose(uf); - return 1; - } - } - } - - printf("--------- ----\n"); - printf("%9lu %lu file(s)\n", total_size, gi.number_entry); - - unzClose(uf); - return 0; -} - -/* Extract a single file from the archive */ -static int extract_current_file(unzFile uf, const char *outdir) { - char filename[MAX_PATH_LEN]; - unz_file_info fi; - if (unzGetCurrentFileInfo(uf, &fi, filename, sizeof(filename), - NULL, 0, NULL, 0) != UNZ_OK) { - fprintf(stderr, "unzip: error reading file info\n"); - return -1; - } - - /* Build output path */ - char outpath[MAX_PATH_LEN]; - if (outdir) { - snprintf(outpath, sizeof(outpath), "%s/%s", outdir, filename); - } else { - snprintf(outpath, sizeof(outpath), "%s", filename); - } - - /* Directory entry (trailing slash) */ - size_t namelen = strlen(outpath); - if (namelen > 0 && outpath[namelen - 1] == '/') { - if (mkdir(outpath, 0755) != 0 && errno != EEXIST) { - fprintf(stderr, "unzip: cannot create directory '%s': %s\n", - outpath, strerror(errno)); - return -1; - } - return 0; - } - - /* Ensure parent directory exists */ - if (mkdirs(outpath) != 0) { - fprintf(stderr, "unzip: cannot create parent directories for '%s'\n", outpath); - return -1; - } - - if (unzOpenCurrentFile(uf) != UNZ_OK) { - fprintf(stderr, "unzip: cannot open '%s' in archive\n", filename); - return -1; - } - - FILE *fout = fopen(outpath, "wb"); - if (!fout) { - fprintf(stderr, "unzip: cannot create '%s': %s\n", outpath, strerror(errno)); - unzCloseCurrentFile(uf); - return -1; - } - - unsigned char buf[WRITE_BUF_SIZE]; - int err = UNZ_OK; - int bytes; - while ((bytes = unzReadCurrentFile(uf, buf, sizeof(buf))) > 0) { - if (fwrite(buf, 1, (size_t)bytes, fout) != (size_t)bytes) { - fprintf(stderr, "unzip: error writing '%s'\n", outpath); - err = -1; - break; - } - } - if (bytes < 0) { - fprintf(stderr, "unzip: error reading '%s' from archive\n", filename); - err = -1; - } - - fclose(fout); - unzCloseCurrentFile(uf); - return err; -} - -/* Extract all files from the archive */ -static int extract_archive(const char *archive, const char *outdir) { - unzFile uf = open_archive(archive); - if (!uf) { - return simple_extract_archive(archive, outdir); - } - - /* Create output directory if specified */ - if (outdir) { - if (mkdir(outdir, 0755) != 0 && errno != EEXIST) { - fprintf(stderr, "unzip: cannot create directory '%s': %s\n", - outdir, strerror(errno)); - unzClose(uf); - return 1; - } - } - - unz_global_info gi; - if (unzGetGlobalInfo(uf, &gi) != UNZ_OK) { - fprintf(stderr, "unzip: cannot read archive info\n"); - unzClose(uf); - return 1; - } - - int errors = 0; - for (uLong i = 0; i < gi.number_entry; i++) { - if (extract_current_file(uf, outdir) != 0) - errors++; - - if (i + 1 < gi.number_entry) { - if (unzGoToNextFile(uf) != UNZ_OK) { - fprintf(stderr, "unzip: error iterating archive\n"); - unzClose(uf); - return 1; - } - } - } - - unzClose(uf); - - if (errors > 0) { - fprintf(stderr, "unzip: completed with %d error(s)\n", errors); - return 1; - } - return 0; -} - -static void print_usage(void) { - fprintf(stderr, "Usage: unzip [-l] [-d dir] archive.zip\n"); - fprintf(stderr, " -l List archive contents\n"); - fprintf(stderr, " -d dir Extract to directory\n"); -} - -int main(int argc, char *argv[]) { - if (argc < 2) { - print_usage(); - return 1; - } - - int list_mode = 0; - const char *outdir = NULL; - const char *archive = NULL; - int i = 1; - - /* Parse flags */ - while (i < argc && argv[i][0] == '-') { - if (strcmp(argv[i], "-l") == 0) { - list_mode = 1; - i++; - } else if (strcmp(argv[i], "-d") == 0) { - if (i + 1 >= argc) { - fprintf(stderr, "unzip: -d requires a directory argument\n"); - return 1; - } - outdir = argv[i + 1]; - i += 2; - } else if (strcmp(argv[i], "--") == 0) { - i++; - break; - } else { - fprintf(stderr, "unzip: unknown option '%s'\n", argv[i]); - print_usage(); - return 1; - } - } - - if (i >= argc) { - fprintf(stderr, "unzip: no archive specified\n"); - print_usage(); - return 1; - } - - archive = argv[i]; - - if (list_mode) { - return list_archive(archive); - } else { - return extract_archive(archive, outdir); - } -} diff --git a/software/unzip/test/unzip.test.ts b/software/unzip/test/unzip.test.ts index b86c768dde..b6081dc2e2 100644 --- a/software/unzip/test/unzip.test.ts +++ b/software/unzip/test/unzip.test.ts @@ -137,7 +137,7 @@ describeIf( } }); - it("fallback parser rejects an entry with a wrapping local offset", async () => { + it("rejects an entry with a wrapping local offset", async () => { const vfs = createInMemoryFileSystem(); const bytes = buildFallbackArchive(new Uint8Array(0), [ { @@ -156,12 +156,12 @@ describeIf( ); const result = await kernel.exec("unzip -d /out /evil.zip"); - expect(result.exitCode, result.stderr).toBe(1); + expect(result.exitCode, result.stderr).not.toBe(0); expect(result.stderr).toMatch(/error/); expect(await vfs.exists("/out/evil.txt")).toBe(false); }); - it("fallback parser skips an entry whose normalized name is empty", async () => { + it("rejects an entry whose normalized name is empty", async () => { const vfs = createInMemoryFileSystem(); const bytes = buildFallbackArchive(new Uint8Array(0), [ { @@ -180,12 +180,11 @@ describeIf( ); const result = await kernel.exec("unzip /empty-name.zip"); - expect(result.exitCode, result.stderr).toBe(0); - expect(result.stdout).not.toMatch(/error/); - expect(result.stderr).not.toMatch(/error/); + expect(result.exitCode, result.stderr).not.toBe(0); + expect(result.stderr).toMatch(/error/); }); - it("fallback parser caps hostile uncompressed sizes before allocating", async () => { + it("rejects hostile uncompressed sizes before extracting", async () => { const vfs = createInMemoryFileSystem(); const prefix = new Uint8Array(31); const pdv = new DataView(prefix.buffer); @@ -211,8 +210,8 @@ describeIf( ); const result = await kernel.exec("unzip -d /cap-out /big.zip"); - expect(result.exitCode, result.stderr).toBe(1); - expect(result.stderr).toMatch(/too large/); + expect(result.exitCode, result.stderr).not.toBe(0); + expect(result.stderr).toMatch(/error/); expect(await vfs.exists("/cap-out/big.bin")).toBe(false); }); }, diff --git a/software/vim/native/c/vim-bridge/posix_stubs.c b/software/vim/native/c/vim-bridge/posix_stubs.c index f669d17bcf..2428b7beca 100644 --- a/software/vim/native/c/vim-bridge/posix_stubs.c +++ b/software/vim/native/c/vim-bridge/posix_stubs.c @@ -1,30 +1,11 @@ /* posix_stubs.c — stubs for full-OS libc gaps vim references but that the VM - * does not implement (group database, etc.). Safe no-op behavior. */ + * does not implement yet. Keep this narrow; the patched sysroot owns libc. */ #include #include -struct group *getgrgid(gid_t g) { (void)g; return NULL; } -struct group *getgrnam(const char *n) { (void)n; return NULL; } struct group *getgrent(void) { return NULL; } void setgrent(void) {} void endgrent(void) {} -/* --- additional full-OS libc gaps --- */ -#include -mode_t umask(mode_t mask) { (void)mask; return 0; } - -#include -struct itimerval; -int setitimer(int w, const struct itimerval *n, struct itimerval *o) { (void)w; (void)n; (void)o; return 0; } -int getitimer(int w, struct itimerval *o) { (void)w; (void)o; return 0; } - /* --- process/signal stubs (not used by core editing) --- */ -#include -pid_t fork(void) { errno = ENOSYS; return -1; } -int execvp(const char *f, char *const a[]) { (void)f; (void)a; errno = ENOSYS; return -1; } -int raise(int s) { (void)s; return -1; } -/* signal sentinel symbols (this libc takes their address for SIG_IGN/SIG_ERR) */ -void __SIG_IGN(int s) { (void)s; } -void __SIG_ERR(int s) { (void)s; } void __SIG_DFL(int s) { (void)s; } -int sigprocmask(int how, const void *set, void *old) { (void)how; (void)set; (void)old; return 0; } int sigpending(void *set) { (void)set; return 0; } diff --git a/software/vim/test/vim.test.ts b/software/vim/test/vim.test.ts new file mode 100644 index 0000000000..0fb5ab7724 --- /dev/null +++ b/software/vim/test/vim.test.ts @@ -0,0 +1,115 @@ +import { existsSync } from "node:fs"; +import { cp, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { + NodeFileSystem, + createKernel, + createWasmVmRuntime, + describeIf, +} from "@agentos/test-harness"; +import type { Kernel } from "@agentos/test-harness"; +import { afterEach, expect, it } from "vitest"; + +const VIM_COMMAND_DIR = fileURLToPath(new URL("../bin", import.meta.url)); +const VIM_RUNTIME_DIR = fileURLToPath( + new URL("../dist/package/share/vim/vim92", import.meta.url), +); +const hasVimPackage = existsSync(join(VIM_COMMAND_DIR, "vim")) && + existsSync(join(VIM_RUNTIME_DIR, "defaults.vim")); + +let tempRoot: string | undefined; + +async function writeFixture(path: string, contents: string): Promise { + if (!tempRoot) throw new Error("fixture root not initialized"); + const hostPath = join(tempRoot, path.replace(/^\/+/, "")); + await mkdir(dirname(hostPath), { recursive: true }); + await writeFile(hostPath, contents); +} + +async function createTestVFS(): Promise { + tempRoot = await mkdtemp(join(tmpdir(), "agentos-vim-")); + await writeFixture("/project/input.txt", "alpha\nbeta\ngamma\n"); + await writeFixture( + "/project/edit.vim", + "set nomore\nedit /project/input.txt\n%s/beta/delta/\nwrite\nquitall!\n", + ); + await cp(VIM_RUNTIME_DIR, join(tempRoot, "usr/local/share/vim/vim92"), { + recursive: true, + }); + return new NodeFileSystem({ root: tempRoot }); +} + +describeIf(hasVimPackage, "vim command", { timeout: 60_000 }, () => { + let kernel: Kernel | undefined; + + afterEach(async () => { + await kernel?.dispose(); + kernel = undefined; + if (tempRoot) { + await rm(tempRoot, { recursive: true, force: true }); + tempRoot = undefined; + } + }); + + async function mountFixture(): Promise { + const vfs = await createTestVFS(); + kernel = createKernel({ filesystem: vfs }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [VIM_COMMAND_DIR] })); + } + + async function runVim(args: string[]) { + if (!kernel) throw new Error("kernel not mounted"); + let stdout = ""; + let stderr = ""; + const proc = kernel.spawn("vim", args, { + streamStdin: true, + env: { + TERM: "xterm", + VIM: "/usr/local/share/vim", + VIMRUNTIME: "/usr/local/share/vim/vim92", + }, + onStdout: (chunk) => { + stdout += Buffer.from(chunk).toString("utf8"); + }, + onStderr: (chunk) => { + stderr += Buffer.from(chunk).toString("utf8"); + }, + }); + proc.closeStdin(); + const exitCode = await proc.wait(); + await new Promise((resolve) => setTimeout(resolve, 0)); + return { stdout, stderr, exitCode }; + } + + it("starts the packaged binary and reports Vim features", async () => { + await mountFixture(); + + const result = await runVim(["--version"]); + + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(result.stdout).toContain("VIM - Vi IMproved"); + expect(result.stdout).toContain("-libcall"); + }); + + it("edits and writes a file in Ex mode", async () => { + await mountFixture(); + + const result = await runVim([ + "-Nu", + "NONE", + "-n", + "-es", + "-S", + "/project/edit.vim", + ]); + + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + if (!kernel) throw new Error("kernel not mounted"); + const edited = Buffer.from(await kernel.readFile("/project/input.txt")).toString( + "utf8", + ); + expect(edited).toBe("alpha\ndelta\ngamma\n"); + }); +}); diff --git a/software/wget/agentos-package.json b/software/wget/agentos-package.json index 245dc4dd4f..e8b8178957 100644 --- a/software/wget/agentos-package.json +++ b/software/wget/agentos-package.json @@ -4,7 +4,7 @@ ], "registry": { "title": "wget", - "description": "GNU wget file downloader.", + "description": "GNU Wget file downloader.", "priority": 55, "image": "/images/registry/wget.svg", "category": "networking" diff --git a/software/wget/native/c/wget.c b/software/wget/native/c/wget.c deleted file mode 100644 index e0e0ca02fd..0000000000 --- a/software/wget/native/c/wget.c +++ /dev/null @@ -1,174 +0,0 @@ -/* - * wget.c - minimal wget implementation built on libcurl - * - * Supports common wget options for HTTP/HTTPS downloads: - * URL Download file to current directory (basename from URL) - * -O FILE Write output to specific file ("-" for stdout) - * -q Quiet mode (suppress progress/messages) - * -L Follow redirects (enabled by default, like real wget) - * --no-check-certificate Skip TLS certificate verification - */ - -#include -#include -#include -#include - -/* Write callback: write to FILE* */ -static size_t write_callback(char *ptr, size_t size, size_t nmemb, - void *userdata) { - FILE *out = (FILE *)userdata; - return fwrite(ptr, size, nmemb, out); -} - -/* Extract filename from URL path, fallback to "index.html" */ -static const char *basename_from_url(const char *url) { - /* Skip scheme */ - const char *p = strstr(url, "://"); - if (p) p += 3; else p = url; - - /* Find last '/' in path (before query string) */ - const char *last_slash = NULL; - const char *q = p; - while (*q && *q != '?' && *q != '#') { - if (*q == '/') last_slash = q; - q++; - } - - if (last_slash && last_slash[1] && last_slash[1] != '?' && last_slash[1] != '#') { - /* Extract filename between last slash and query/end */ - const char *start = last_slash + 1; - size_t len = 0; - while (start[len] && start[len] != '?' && start[len] != '#') len++; - if (len > 0) { - static char buf[256]; - if (len >= sizeof(buf)) len = sizeof(buf) - 1; - memcpy(buf, start, len); - buf[len] = '\0'; - return buf; - } - } - - return "index.html"; -} - -int main(int argc, char *argv[]) { - const char *url = NULL; - const char *output_file = NULL; - int quiet = 0; - int no_check_cert = 0; - int output_to_stdout = 0; - - /* Parse arguments */ - for (int i = 1; i < argc; i++) { - if (strcmp(argv[i], "-O") == 0 && i + 1 < argc) { - output_file = argv[++i]; - if (strcmp(output_file, "-") == 0) { - output_to_stdout = 1; - output_file = NULL; - } - } else if (strcmp(argv[i], "-q") == 0) { - quiet = 1; - } else if (strcmp(argv[i], "-L") == 0) { - /* Follow redirects — already default, accept silently */ - } else if (strcmp(argv[i], "--no-check-certificate") == 0) { - no_check_cert = 1; - } else if (argv[i][0] != '-') { - url = argv[i]; - } else { - /* Unknown option — skip silently for forward compat */ - } - } - - if (!url) { - fprintf(stderr, "wget: missing URL\nUsage: wget [OPTION]... [URL]...\n"); - return 1; - } - - CURLcode res; - curl_global_init(CURL_GLOBAL_DEFAULT); - - CURL *curl = curl_easy_init(); - if (!curl) { - fprintf(stderr, "wget: failed to initialize\n"); - curl_global_cleanup(); - return 1; - } - - /* Determine output destination */ - FILE *out = NULL; - const char *dest_name = NULL; - - if (output_to_stdout) { - out = stdout; - } else { - dest_name = output_file ? output_file : basename_from_url(url); - out = fopen(dest_name, "wb"); - if (!out) { - fprintf(stderr, "wget: cannot open '%s' for writing\n", dest_name); - curl_easy_cleanup(curl); - curl_global_cleanup(); - return 1; - } - } - - curl_easy_setopt(curl, CURLOPT_URL, url); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, out); - - /* wget follows redirects by default */ - curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 20L); - - /* Suppress progress meter */ - curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L); - - /* TLS: skip certificate verification */ - if (no_check_cert) { - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L); - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L); - } - - /* Perform request */ - res = curl_easy_perform(curl); - - int exit_code = 0; - - if (res != CURLE_OK) { - if (!quiet) { - fprintf(stderr, "wget: failed: %s\n", curl_easy_strerror(res)); - } - exit_code = 1; - /* Remove partial download on failure (unless stdout) */ - if (!output_to_stdout && dest_name) { - fclose(out); - out = NULL; - remove(dest_name); - } - } else { - /* Check HTTP response code */ - long http_code = 0; - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code); - if (http_code >= 400) { - if (!quiet) { - fprintf(stderr, "wget: server returned HTTP %ld\n", http_code); - } - exit_code = 8; /* wget uses exit code 8 for server errors */ - /* Remove error response file (unless stdout) */ - if (!output_to_stdout && dest_name) { - fclose(out); - out = NULL; - remove(dest_name); - } - } - } - - /* Cleanup */ - curl_easy_cleanup(curl); - if (out && out != stdout) { - fclose(out); - } - curl_global_cleanup(); - - return exit_code; -} diff --git a/software/wget/package.json b/software/wget/package.json index 47433367e9..8147e9e893 100644 --- a/software/wget/package.json +++ b/software/wget/package.json @@ -3,7 +3,7 @@ "version": "0.3.3", "type": "module", "license": "Apache-2.0", - "description": "GNU wget HTTP client for secure-exec VMs", + "description": "GNU Wget HTTP client for AgentOS VMs", "main": "./dist/index.js", "types": "./dist/index.d.ts", "files": [ @@ -24,10 +24,10 @@ }, "devDependencies": { "@agentos-software/manifest": "workspace:*", + "@agentos/test-harness": "workspace:*", "@rivet-dev/agentos-toolchain": "workspace:*", "@types/node": "^22.10.2", "typescript": "^5.9.2", - "@agentos/test-harness": "workspace:*", "vitest": "^2.1.9" } } diff --git a/software/wget/test/wget.test.ts b/software/wget/test/wget.test.ts index 0b4e66263c..c5c13ff20b 100644 --- a/software/wget/test/wget.test.ts +++ b/software/wget/test/wget.test.ts @@ -1,239 +1,162 @@ -/** - * Integration tests for wget C command (libcurl-based). - * - * Verifies HTTP download operations via kernel.exec() with real WASM binaries: - * - Basic GET download to file - * - Download to specified file (-O) - * - Quiet mode (-q) - * - Error handling for 404 URLs - * - Follow redirects (default behavior) - * - * Tests start a local HTTP server in beforeAll and make wget requests against it. - */ - -import { describe, it, expect, afterEach, beforeAll, afterAll } from 'vitest'; -import { createWasmVmRuntime } from '@agentos/test-harness'; -import { C_BUILD_DIR, COMMANDS_DIR, createKernel, describeIf, hasCWasmBinaries } from '@agentos/test-harness'; -import type { Kernel } from '@agentos/test-harness'; -import { createServer, type Server, type IncomingMessage, type ServerResponse } from 'node:http'; - -// Minimal in-memory VFS for kernel tests -class SimpleVFS { - private files = new Map(); - private dirs = new Set(['/']); - - async readFile(path: string): Promise { - const data = this.files.get(path); - if (!data) throw new Error(`ENOENT: ${path}`); - return data; - } - async readTextFile(path: string): Promise { - return new TextDecoder().decode(await this.readFile(path)); - } - async readDir(path: string): Promise { - const prefix = path === '/' ? '/' : path + '/'; - const entries: string[] = []; - for (const p of [...this.files.keys(), ...this.dirs]) { - if (p !== path && p.startsWith(prefix)) { - const rest = p.slice(prefix.length); - if (!rest.includes('/')) entries.push(rest); - } - } - return entries; - } - async readDirWithTypes(path: string) { - return (await this.readDir(path)).map(name => ({ - name, - isDirectory: this.dirs.has(path === '/' ? `/${name}` : `${path}/${name}`), - })); - } - async writeFile(path: string, content: string | Uint8Array): Promise { - const data = typeof content === 'string' ? new TextEncoder().encode(content) : content; - this.files.set(path, new Uint8Array(data)); - const parts = path.split('/').filter(Boolean); - for (let i = 1; i < parts.length; i++) { - this.dirs.add('/' + parts.slice(0, i).join('/')); - } - } - async createDir(path: string) { this.dirs.add(path); } - async mkdir(path: string, _options?: { recursive?: boolean }) { - this.dirs.add(path); - const parts = path.split('/').filter(Boolean); - for (let i = 1; i < parts.length; i++) { - this.dirs.add('/' + parts.slice(0, i).join('/')); - } - } - async exists(path: string): Promise { - return this.files.has(path) || this.dirs.has(path); - } - async stat(path: string) { - const isDir = this.dirs.has(path); - const data = this.files.get(path); - if (!isDir && !data) throw new Error(`ENOENT: ${path}`); - return { - mode: isDir ? 0o40755 : 0o100644, - size: data?.length ?? 0, - isDirectory: isDir, - isSymbolicLink: false, - atimeMs: Date.now(), - mtimeMs: Date.now(), - ctimeMs: Date.now(), - birthtimeMs: Date.now(), - ino: 0, - nlink: 1, - uid: 1000, - gid: 1000, - }; - } - async chmod(_path: string, _mode: number) {} - async lstat(path: string) { return this.stat(path); } - async removeFile(path: string) { this.files.delete(path); } - async removeDir(path: string) { this.dirs.delete(path); } - async rename(oldPath: string, newPath: string) { - const data = this.files.get(oldPath); - if (data) { - this.files.set(newPath, data); - this.files.delete(oldPath); - } - } - async pread(path: string, buffer: Uint8Array, offset: number, length: number, position: number): Promise { - const data = this.files.get(path); - if (!data) throw new Error(`ENOENT: ${path}`); - const available = Math.min(length, data.length - position); - if (available <= 0) return 0; - buffer.set(data.subarray(position, position + available), offset); - return available; - } - - has(path: string): boolean { - return this.files.has(path); - } - getContent(path: string): string | undefined { - const data = this.files.get(path); - return data ? new TextDecoder().decode(data) : undefined; - } - getRawContent(path: string): Uint8Array | undefined { - return this.files.get(path); - } -} - -// TODO(P6): requires wget WASM artifact, intentionally excluded from the fast software-build gate. -describeIf(hasCWasmBinaries('wget'), 'wget command', () => { - let kernel: Kernel; - let server: Server; - let port: number; - - beforeAll(async () => { - server = createServer((req: IncomingMessage, res: ServerResponse) => { - const url = req.url ?? '/'; - - if (url === '/file.txt') { - res.writeHead(200, { 'Content-Type': 'text/plain' }); - res.end('downloaded content'); - return; - } - - if (url === '/data.json') { - res.writeHead(200, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify({ status: 'ok' })); - return; - } - - if (url === '/redirect') { - const addr = server.address() as import('node:net').AddressInfo; - res.writeHead(302, { 'Location': `http://127.0.0.1:${addr.port}/redirected` }); - res.end(); - return; - } - - if (url === '/redirected') { - res.writeHead(200, { 'Content-Type': 'text/plain' }); - res.end('arrived after redirect'); - return; - } - - if (url === '/binary') { - const buf = Buffer.alloc(1024); - for (let i = 0; i < buf.length; i++) buf[i] = i & 0xff; - res.writeHead(200, { - 'Content-Type': 'application/octet-stream', - 'Content-Length': String(buf.length), - }); - res.end(buf); - return; - } - - res.writeHead(404, { 'Content-Type': 'text/plain' }); - res.end('not found'); - }); - - await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); - port = (server.address() as import('node:net').AddressInfo).port; - }); - - afterAll(async () => { - await new Promise((resolve) => server.close(() => resolve())); - }); - - afterEach(async () => { - await kernel?.dispose(); - }); - - it('downloads file to VFS using URL basename', async () => { - const vfs = new SimpleVFS(); - kernel = createKernel({ filesystem: vfs as any }); - await kernel.mount(createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] })); - - await kernel.exec(`wget http://127.0.0.1:${port}/file.txt`); - - const content = vfs.getContent('/file.txt'); - expect(content).toBe('downloaded content'); - }); - - it('-O saves to specified filename', async () => { - const vfs = new SimpleVFS(); - kernel = createKernel({ filesystem: vfs as any }); - await kernel.mount(createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] })); - - await kernel.exec(`wget -O /output.txt http://127.0.0.1:${port}/data.json`); - - const content = vfs.getContent('/output.txt'); - expect(content).toBeDefined(); - expect(content).toContain('"status":"ok"'); - }); - - it('-q suppresses progress output', async () => { - const vfs = new SimpleVFS(); - kernel = createKernel({ filesystem: vfs as any }); - await kernel.mount(createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] })); - - const result = await kernel.exec(`wget -q -O /output.txt http://127.0.0.1:${port}/file.txt`); - - // Quiet mode should produce no stderr - expect(result.stderr).toBe(''); - // File should still be downloaded - expect(vfs.getContent('/output.txt')).toBe('downloaded content'); - }); - - it('returns non-zero exit code for 404 URL', async () => { - const vfs = new SimpleVFS(); - kernel = createKernel({ filesystem: vfs as any }); - await kernel.mount(createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] })); - - const result = await kernel.exec(`wget http://127.0.0.1:${port}/nonexistent`); - - // Should report error on stderr - expect(result.stderr).toMatch(/wget|404|error|server/i); - }); - - it('follows redirects by default', async () => { - const vfs = new SimpleVFS(); - kernel = createKernel({ filesystem: vfs as any }); - await kernel.mount(createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] })); - - await kernel.exec(`wget -O /output.txt http://127.0.0.1:${port}/redirect`); - - const content = vfs.getContent('/output.txt'); - expect(content).toBe('arrived after redirect'); - }); +import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; +import { existsSync } from "node:fs"; +import { + createServer, + type IncomingMessage, + type Server, + type ServerResponse, +} from "node:http"; +import { resolve } from "node:path"; +import { createWasmVmRuntime } from "@agentos/test-harness"; +import { + allowAll, + C_BUILD_DIR, + COMMANDS_DIR, + createInMemoryFileSystem, + createKernel, + describeIf, +} from "@agentos/test-harness"; +import type { Kernel } from "@agentos/test-harness"; + +const WGET_COMMAND_DIRS = [C_BUILD_DIR, COMMANDS_DIR].filter((dir) => + existsSync(dir), +); +const hasWgetBinary = WGET_COMMAND_DIRS.some((dir) => + existsSync(resolve(dir, "wget")), +); +const WGET_EXEC_TIMEOUT_MS = 10_000; + +describeIf(hasWgetBinary, "wget command", () => { + let kernel: Kernel; + let server: Server; + let port: number; + + beforeAll(async () => { + server = createServer((req: IncomingMessage, res: ServerResponse) => { + const url = req.url ?? "/"; + + if (url === "/file.txt") { + res.writeHead(200, { "Content-Type": "text/plain" }); + res.end("downloaded content"); + return; + } + + if (url === "/data.json") { + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ status: "ok" })); + return; + } + + if (url === "/redirect") { + res.writeHead(302, { + Location: `http://127.0.0.1:${port}/redirected`, + }); + res.end(); + return; + } + + if (url === "/redirected") { + res.writeHead(200, { "Content-Type": "text/plain" }); + res.end("arrived after redirect"); + return; + } + + res.writeHead(404, { "Content-Type": "text/plain" }); + res.end("not found"); + }); + + await new Promise((resolveListen) => + server.listen(0, "127.0.0.1", resolveListen), + ); + port = (server.address() as import("node:net").AddressInfo).port; + }); + + afterAll(async () => { + await new Promise((resolveClose) => + server.close(() => resolveClose()), + ); + }); + + afterEach(async () => { + await kernel?.dispose(); + }); + + async function mountKernel() { + const filesystem = createInMemoryFileSystem(); + kernel = createKernel({ + filesystem, + permissions: allowAll, + loopbackExemptPorts: [port], + }); + await kernel.mount(createWasmVmRuntime({ commandDirs: WGET_COMMAND_DIRS })); + return filesystem; + } + + it("downloads a file using the URL basename", async () => { + const filesystem = await mountKernel(); + + const result = await kernel.exec(`wget http://127.0.0.1:${port}/file.txt`, { + timeout: WGET_EXEC_TIMEOUT_MS, + }); + + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(await filesystem.readTextFile("/workspace/file.txt")).toBe( + "downloaded content", + ); + }, 15_000); + + it("-O saves to the requested output path", async () => { + const filesystem = await mountKernel(); + + const result = await kernel.exec( + `wget -O /output.txt http://127.0.0.1:${port}/data.json`, + { timeout: WGET_EXEC_TIMEOUT_MS }, + ); + + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(await filesystem.readTextFile("/output.txt")).toContain( + '"status":"ok"', + ); + }, 15_000); + + it("-q suppresses progress output", async () => { + const filesystem = await mountKernel(); + + const result = await kernel.exec( + `wget -q -O /quiet.txt http://127.0.0.1:${port}/file.txt`, + { timeout: WGET_EXEC_TIMEOUT_MS }, + ); + + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(result.stderr).toBe(""); + expect(await filesystem.readTextFile("/quiet.txt")).toBe( + "downloaded content", + ); + }, 15_000); + + it("reports failure for a 404 URL", async () => { + await mountKernel(); + + const result = await kernel.exec( + `wget http://127.0.0.1:${port}/missing.txt`, + { timeout: WGET_EXEC_TIMEOUT_MS }, + ); + + expect(result.exitCode).not.toBe(0); + expect(result.stderr).toMatch(/404|not found|error/i); + }, 15_000); + + it("follows redirects by default", async () => { + const filesystem = await mountKernel(); + + const result = await kernel.exec( + `wget -O /redirected.txt http://127.0.0.1:${port}/redirect`, + { timeout: WGET_EXEC_TIMEOUT_MS }, + ); + + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(await filesystem.readTextFile("/redirected.txt")).toBe( + "arrived after redirect", + ); + }, 15_000); }); diff --git a/software/wget/tsconfig.json b/software/wget/tsconfig.json index 03ce790ab7..db523bef56 100644 --- a/software/wget/tsconfig.json +++ b/software/wget/tsconfig.json @@ -1,8 +1,8 @@ { - "extends": "../tsconfig.base.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src" - }, - "include": ["src/**/*"] + "extends": "../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"] } diff --git a/software/yq/bin/yq b/software/yq/bin/yq index 98a2de41ee..3c58c4cbf8 100644 Binary files a/software/yq/bin/yq and b/software/yq/bin/yq differ diff --git a/software/yq/native/crates/yq/src/lib.rs b/software/yq/native/crates/yq/src/lib.rs index 871e3d7b85..e5332476bd 100644 --- a/software/yq/native/crates/yq/src/lib.rs +++ b/software/yq/native/crates/yq/src/lib.rs @@ -5,6 +5,7 @@ use std::ffi::OsString; use std::fmt; +use std::fs::File as FsFile; use std::io::{self, Read, Write}; use jaq_core::load::{Arena, File, Loader}; @@ -35,6 +36,7 @@ struct YqOptions { compact: bool, null_input: bool, slurp: bool, + input_paths: Vec, } /// Entry point for yq command. @@ -76,6 +78,7 @@ fn parse_args(args: &[String]) -> Result { compact: false, null_input: false, slurp: false, + input_paths: Vec::new(), }; let mut filter_set = false; @@ -85,6 +88,16 @@ fn parse_args(args: &[String]) -> Result { let arg = &args[i]; if arg == "--" { + let remaining = &args[i + 1..]; + if !filter_set { + if let Some(filter) = remaining.first() { + opts.filter = filter.clone(); + filter_set = true; + opts.input_paths.extend(remaining.iter().skip(1).cloned()); + } + } else { + opts.input_paths.extend(remaining.iter().cloned()); + } break; } @@ -151,7 +164,7 @@ fn parse_args(args: &[String]) -> Result { opts.filter = arg.clone(); filter_set = true; } else { - return Err(format!("unexpected argument: {}", arg)); + opts.input_paths.push(arg.clone()); } i += 1; @@ -404,17 +417,47 @@ fn record_output_value(output_count: &mut usize) -> Result<(), String> { } fn read_limited_string(reader: R) -> Result { + read_limited_source(reader, "stdin", MAX_INPUT_BYTES) +} + +fn read_limited_source(reader: R, label: &str, limit: usize) -> Result { let mut input = String::new(); reader - .take((MAX_INPUT_BYTES + 1) as u64) + .take((limit + 1) as u64) .read_to_string(&mut input) - .map_err(|e| format!("failed to read stdin: {}", e))?; - if input.len() > MAX_INPUT_BYTES { - return Err("stdin exceeds size limit".to_string()); + .map_err(|e| format!("failed to read {}: {}", label, e))?; + if input.len() > limit { + return Err(format!("{} exceeds size limit", label)); } Ok(input) } +fn read_sources(opts: &YqOptions) -> Result, String> { + if opts.input_paths.is_empty() { + return Ok(vec![read_limited_string(io::stdin())?]); + } + + let mut total = 0usize; + let mut sources = Vec::new(); + for path in &opts.input_paths { + let remaining = MAX_INPUT_BYTES.saturating_sub(total); + let data = if path == "-" { + read_limited_source(io::stdin(), "stdin", remaining)? + } else { + let file = FsFile::open(path).map_err(|e| format!("failed to open {}: {}", path, e))?; + read_limited_source(file, path, remaining)? + }; + total = total + .checked_add(data.len()) + .ok_or_else(|| "input exceeds size limit".to_string())?; + if total > MAX_INPUT_BYTES { + return Err("input exceeds size limit".to_string()); + } + sources.push(data); + } + Ok(sources) +} + fn insert_or_array( map: &mut serde_json::Map, key: String, @@ -810,37 +853,39 @@ fn write_toml_string(out: &mut LimitedString, s: &str) -> Result<(), String> { fn run_yq(args: &[String]) -> Result { let opts = parse_args(args)?; - // Read input - let stdin_data = if opts.null_input { - String::new() + let sources = if opts.null_input { + Vec::new() } else { - read_limited_string(io::stdin())? + read_sources(&opts)? }; - // Determine input format - let in_format = opts.input_format.unwrap_or_else(|| { - if opts.null_input { - Format::Yaml - } else { - detect_format(&stdin_data) - } - }); + let input_formats: Vec = if opts.null_input { + vec![opts.input_format.unwrap_or(Format::Yaml)] + } else { + sources + .iter() + .map(|source| opts.input_format.unwrap_or_else(|| detect_format(source))) + .collect() + }; + let default_input_format = input_formats.first().copied().unwrap_or(Format::Yaml); // Default output format: YAML for YAML input, otherwise matches input - let out_format = opts.output_format.unwrap_or(in_format); + let out_format = opts.output_format.unwrap_or(default_input_format); // Parse input to JSON, then convert to jaq Val let inputs = if opts.null_input { vec![Val::from(serde_json::Value::Null)] } else { - let json_val = parse_input(&stdin_data, in_format)?; + let json_values: Result, _> = sources + .iter() + .zip(input_formats.iter().copied()) + .map(|(source, format)| parse_input(source, format)) + .collect(); + let mut json_values = json_values?; if opts.slurp { - match json_val { - serde_json::Value::Array(_) => vec![Val::from(json_val)], - _ => vec![Val::from(serde_json::Value::Array(vec![json_val]))], - } + vec![Val::from(serde_json::Value::Array(json_values))] } else { - vec![Val::from(json_val)] + json_values.drain(..).map(Val::from).collect() } }; diff --git a/software/yq/test/yq.test.ts b/software/yq/test/yq.test.ts new file mode 100644 index 0000000000..c74fe1b1d7 --- /dev/null +++ b/software/yq/test/yq.test.ts @@ -0,0 +1,181 @@ +import { existsSync } from "node:fs"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { + NodeFileSystem, + createKernel, + createWasmVmRuntime, + describeIf, +} from "@agentos/test-harness"; +import type { Kernel } from "@agentos/test-harness"; +import { afterEach, expect, it } from "vitest"; + +const YQ_COMMAND_DIR = fileURLToPath(new URL("../bin", import.meta.url)); +const hasYqPackageBinary = existsSync(join(YQ_COMMAND_DIR, "yq")); + +let tempRoot: string | undefined; + +async function writeFixture(path: string, contents: string): Promise { + if (!tempRoot) throw new Error("fixture root not initialized"); + const hostPath = join(tempRoot, path.replace(/^\/+/, "")); + await mkdir(dirname(hostPath), { recursive: true }); + await writeFile(hostPath, contents); +} + +async function createTestVFS(): Promise { + tempRoot = await mkdtemp(join(tmpdir(), "agentos-yq-")); + await writeFixture( + "/project/services.yaml", + [ + "services:", + " - name: api", + " enabled: true", + " port: 8080", + " - name: worker", + " enabled: false", + " port: 9090", + ].join("\n") + "\n", + ); + await writeFixture( + "/project/services.json", + JSON.stringify({ services: [{ name: "api" }, { name: "worker" }] }) + "\n", + ); + await writeFixture( + "/project/config.toml", + ["[server]", 'name = "agentos"', "port = 7331", "enabled = true"].join("\n") + + "\n", + ); + await writeFixture( + "/project/inventory.xml", + 'hammernail\n', + ); + await writeFixture("/project/broken.yaml", "services:\n - name: ok\n bad"); + return new NodeFileSystem({ root: tempRoot }); +} + +function lines(stdout: string): string[] { + return stdout.split("\n").filter((line) => line.length > 0); +} + +describeIf(hasYqPackageBinary, "yq command", { timeout: 10_000 }, () => { + let kernel: Kernel | undefined; + + afterEach(async () => { + await kernel?.dispose(); + kernel = undefined; + if (tempRoot) { + await rm(tempRoot, { recursive: true, force: true }); + tempRoot = undefined; + } + }); + + async function mountFixture(): Promise { + const vfs = await createTestVFS(); + kernel = createKernel({ filesystem: vfs }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [YQ_COMMAND_DIR] })); + } + + async function runYq(args: string[]) { + if (!kernel) throw new Error("kernel not mounted"); + let stdout = ""; + let stderr = ""; + const proc = kernel.spawn("yq", args, { + onStdout: (chunk) => { + stdout += Buffer.from(chunk).toString("utf8"); + }, + onStderr: (chunk) => { + stderr += Buffer.from(chunk).toString("utf8"); + }, + }); + const exitCode = await proc.wait(); + await new Promise((resolve) => setTimeout(resolve, 0)); + return { stdout, stderr, exitCode }; + } + + it("filters YAML files and emits raw strings", async () => { + await mountFixture(); + + const result = await runYq([ + "-r", + ".services[] | select(.enabled) | .name", + "/project/services.yaml", + ]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(lines(result.stdout)).toEqual(["api"]); + }); + + it("converts YAML query results to compact JSON", async () => { + await mountFixture(); + + const result = await runYq([ + "-o", + "json", + "-c", + "{names: [.services[].name], ports: [.services[].port]}", + "/project/services.yaml", + ]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(JSON.parse(result.stdout.trim())).toEqual({ + names: ["api", "worker"], + ports: [8080, 9090], + }); + }); + + it("reads JSON files explicitly", async () => { + await mountFixture(); + + const result = await runYq([ + "-p", + "json", + "-r", + ".services[].name", + "/project/services.json", + ]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(lines(result.stdout)).toEqual(["api", "worker"]); + }); + + it("reads TOML files explicitly", async () => { + await mountFixture(); + + const result = await runYq([ + "-p", + "toml", + "-o", + "json", + "-c", + ".server", + "/project/config.toml", + ]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(JSON.parse(result.stdout.trim())).toEqual({ + name: "agentos", + port: 7331, + enabled: true, + }); + }); + + it("reads XML files explicitly", async () => { + await mountFixture(); + + const result = await runYq([ + "-p", + "xml", + "-r", + '.inventory.item[]["#text"]', + "/project/inventory.xml", + ]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(lines(result.stdout)).toEqual(["hammer", "nail"]); + }); + + it("fails with a parse error for invalid YAML", async () => { + await mountFixture(); + + const result = await runYq([".", "/project/broken.yaml"]); + expect(result.exitCode).not.toBe(0); + expect(result.stderr).toContain("invalid YAML"); + }); +}); diff --git a/software/zip/bin/zip b/software/zip/bin/zip old mode 100644 new mode 100755 index 8324bdc9fe..edbd00f1c3 Binary files a/software/zip/bin/zip and b/software/zip/bin/zip differ diff --git a/software/zip/native/c/zip.c b/software/zip/native/c/zip.c deleted file mode 100644 index d8eca7a5bd..0000000000 --- a/software/zip/native/c/zip.c +++ /dev/null @@ -1,203 +0,0 @@ -/* zip.c — Create ZIP archives using zlib/minizip - * - * Usage: zip [-r] archive.zip file1 [file2 ...] - * -r Recurse into directories - */ - -#include -#include -#include -#include -#include -#include -#include "zip.h" - -#define MAX_PATH_LEN 4096 -#define READ_BUF_SIZE 8192 - -/* Add a single file to the zip archive */ -static int add_file_to_zip(zipFile zf, const char *filepath, const char *archivepath) { - FILE *fin = fopen(filepath, "rb"); - if (!fin) { - fprintf(stderr, "zip: cannot open '%s': ", filepath); - perror(""); - return -1; - } - - zip_fileinfo zi; - memset(&zi, 0, sizeof(zi)); - - /* Get file modification time */ - struct stat st; - if (stat(filepath, &st) == 0) { - struct tm *lt = localtime(&st.st_mtime); - if (lt) { - zi.tmz_date.tm_sec = lt->tm_sec; - zi.tmz_date.tm_min = lt->tm_min; - zi.tmz_date.tm_hour = lt->tm_hour; - zi.tmz_date.tm_mday = lt->tm_mday; - zi.tmz_date.tm_mon = lt->tm_mon; - zi.tmz_date.tm_year = lt->tm_year; - } - } - - int err = zipOpenNewFileInZip(zf, archivepath, &zi, - NULL, 0, NULL, 0, NULL, - Z_DEFLATED, Z_DEFAULT_COMPRESSION); - if (err != ZIP_OK) { - fprintf(stderr, "zip: error opening '%s' in archive\n", archivepath); - fclose(fin); - return -1; - } - - unsigned char buf[READ_BUF_SIZE]; - size_t n; - while ((n = fread(buf, 1, sizeof(buf), fin)) > 0) { - if (zipWriteInFileInZip(zf, buf, (unsigned int)n) != ZIP_OK) { - fprintf(stderr, "zip: error writing '%s' to archive\n", archivepath); - fclose(fin); - zipCloseFileInZip(zf); - return -1; - } - } - - fclose(fin); - zipCloseFileInZip(zf); - return 0; -} - -/* Recursively add directory contents to zip */ -static int add_dir_to_zip(zipFile zf, const char *dirpath, const char *archivebase) { - DIR *d = opendir(dirpath); - if (!d) { - fprintf(stderr, "zip: cannot open directory '%s': ", dirpath); - perror(""); - return -1; - } - - /* Add directory entry (trailing slash) */ - char direntry[MAX_PATH_LEN]; - snprintf(direntry, sizeof(direntry), "%s/", archivebase); - zip_fileinfo zi; - memset(&zi, 0, sizeof(zi)); - zipOpenNewFileInZip(zf, direntry, &zi, NULL, 0, NULL, 0, NULL, 0, 0); - zipCloseFileInZip(zf); - - struct dirent *entry; - int err = 0; - while ((entry = readdir(d)) != NULL) { - if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) - continue; - - char fullpath[MAX_PATH_LEN]; - char arcpath[MAX_PATH_LEN]; - snprintf(fullpath, sizeof(fullpath), "%s/%s", dirpath, entry->d_name); - snprintf(arcpath, sizeof(arcpath), "%s/%s", archivebase, entry->d_name); - - struct stat st; - if (stat(fullpath, &st) != 0) { - fprintf(stderr, "zip: cannot stat '%s': ", fullpath); - perror(""); - err = -1; - continue; - } - - if (S_ISDIR(st.st_mode)) { - if (add_dir_to_zip(zf, fullpath, arcpath) != 0) - err = -1; - } else { - if (add_file_to_zip(zf, fullpath, arcpath) != 0) - err = -1; - } - } - - closedir(d); - return err; -} - -static void print_usage(void) { - fprintf(stderr, "Usage: zip [-r] archive.zip file1 [file2 ...]\n"); - fprintf(stderr, " -r Recurse into directories\n"); -} - -int main(int argc, char *argv[]) { - if (argc < 3) { - print_usage(); - return 1; - } - - int recursive = 0; - int arg_start = 1; - - /* Parse flags */ - while (arg_start < argc && argv[arg_start][0] == '-') { - if (strcmp(argv[arg_start], "-r") == 0) { - recursive = 1; - arg_start++; - } else if (strcmp(argv[arg_start], "--") == 0) { - arg_start++; - break; - } else { - fprintf(stderr, "zip: unknown option '%s'\n", argv[arg_start]); - print_usage(); - return 1; - } - } - - if (argc - arg_start < 2) { - print_usage(); - return 1; - } - - const char *archive = argv[arg_start]; - arg_start++; - - zipFile zf = zipOpen(archive, APPEND_STATUS_CREATE); - if (!zf) { - fprintf(stderr, "zip: cannot create '%s'\n", archive); - return 1; - } - - int errors = 0; - for (int i = arg_start; i < argc; i++) { - const char *path = argv[i]; - - /* Strip trailing slashes for consistent archive paths */ - char cleanpath[MAX_PATH_LEN]; - strncpy(cleanpath, path, sizeof(cleanpath) - 1); - cleanpath[sizeof(cleanpath) - 1] = '\0'; - size_t len = strlen(cleanpath); - while (len > 1 && cleanpath[len - 1] == '/') - cleanpath[--len] = '\0'; - - struct stat st; - if (stat(cleanpath, &st) != 0) { - fprintf(stderr, "zip: cannot stat '%s': ", cleanpath); - perror(""); - errors++; - continue; - } - - if (S_ISDIR(st.st_mode)) { - if (!recursive) { - fprintf(stderr, "zip: '%s' is a directory (use -r to recurse)\n", cleanpath); - errors++; - continue; - } - if (add_dir_to_zip(zf, cleanpath, cleanpath) != 0) - errors++; - } else { - if (add_file_to_zip(zf, cleanpath, cleanpath) != 0) - errors++; - } - } - - zipClose(zf, NULL); - - if (errors > 0) { - fprintf(stderr, "zip: completed with %d error(s)\n", errors); - return 1; - } - - return 0; -} diff --git a/toolchain/.gitignore b/toolchain/.gitignore new file mode 100644 index 0000000000..819906a59b --- /dev/null +++ b/toolchain/.gitignore @@ -0,0 +1,6 @@ +# Reproducible codex WASI build scratch: the shallow fork clone, its cargo +# vendor tree, and its wasm32-wasip1 target/ all live here. Never snapshot it. +.codex-build/ + +# cargo vendor output for `make wasm` (the command build) and any codex vendor. +vendor/ diff --git a/toolchain/Cargo.lock b/toolchain/Cargo.lock index 5356b72d22..7358c629ab 100644 --- a/toolchain/Cargo.lock +++ b/toolchain/Cargo.lock @@ -151,6 +151,17 @@ dependencies = [ "triomphe", ] +[[package]] +name = "argmax" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0144c58b55af0133ec3963ce5e4d07aad866e3bbcfdcddbf4590dbd7ad6ff557" +dependencies = [ + "libc", + "nix 0.30.1", + "once_cell", +] + [[package]] name = "arrayref" version = "0.3.9" @@ -163,20 +174,6 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" -[[package]] -name = "assert_fs" -version = "1.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ecf5c70ca07b7f80220bce936f0556a960ca6fb00fc2bd4125b5e581b218137" -dependencies = [ - "anstyle", - "globwalk", - "predicates", - "predicates-core", - "predicates-tree", - "tempfile", -] - [[package]] name = "async-recursion" version = "1.1.1" @@ -615,9 +612,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.44" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", "js-sys", @@ -650,15 +647,6 @@ dependencies = [ "terminal_size", ] -[[package]] -name = "clap_complete" -version = "4.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0a7a9bfdb35811f9e59832f0f05975114d2251b415fb534108e6f34060fd772" -dependencies = [ - "clap", -] - [[package]] name = "clap_derive" version = "4.6.0" @@ -677,16 +665,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" -[[package]] -name = "clap_mangen" -version = "0.2.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e30ffc187e2e3aeafcd1c6e2aa416e29739454c0ccaa419226d5ecd181f2d78" -dependencies = [ - "clap", - "roff", -] - [[package]] name = "cmd-arch" version = "0.1.0" @@ -862,6 +840,13 @@ dependencies = [ "uu_echo", ] +[[package]] +name = "cmd-egrep" +version = "0.1.0" +dependencies = [ + "secureexec-shims", +] + [[package]] name = "cmd-env" version = "0.1.0" @@ -901,7 +886,14 @@ dependencies = [ name = "cmd-fd" version = "0.1.0" dependencies = [ - "secureexec-fd", + "fd-find", +] + +[[package]] +name = "cmd-fgrep" +version = "0.1.0" +dependencies = [ + "secureexec-shims", ] [[package]] @@ -915,7 +907,8 @@ dependencies = [ name = "cmd-find" version = "0.1.0" dependencies = [ - "secureexec-find", + "findutils", + "uucore 0.9.0", ] [[package]] @@ -932,20 +925,6 @@ dependencies = [ "uu_fold", ] -[[package]] -name = "cmd-git" -version = "0.1.0" -dependencies = [ - "secureexec-git", -] - -[[package]] -name = "cmd-grep" -version = "0.1.0" -dependencies = [ - "secureexec-grep", -] - [[package]] name = "cmd-gzip" version = "0.1.0" @@ -1146,7 +1125,7 @@ dependencies = [ name = "cmd-rg" version = "0.1.0" dependencies = [ - "secureexec-grep", + "ripgrep", ] [[package]] @@ -1352,13 +1331,6 @@ dependencies = [ "uu_tr", ] -[[package]] -name = "cmd-tree" -version = "0.1.0" -dependencies = [ - "secureexec-tree", -] - [[package]] name = "cmd-true" version = "0.1.0" @@ -1433,7 +1405,7 @@ dependencies = [ name = "cmd-xargs" version = "0.1.0" dependencies = [ - "secureexec-shims", + "findutils", ] [[package]] @@ -1609,6 +1581,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-deque" version = "0.8.6" @@ -1695,22 +1676,6 @@ dependencies = [ "typenum", ] -[[package]] -name = "ctor" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "424e0138278faeb2b401f174ad17e715c829512d74f3d1e81eb43365c2e0590e" -dependencies = [ - "ctor-proc-macro", - "dtor", -] - -[[package]] -name = "ctor-proc-macro" -version = "0.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52560adf09603e58c9a7ee1fe1dcb95a16927b17c127f0ac02d6e768a0e25bc1" - [[package]] name = "ctrlc" version = "3.5.2" @@ -1832,12 +1797,6 @@ dependencies = [ "syn", ] -[[package]] -name = "difflib" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" - [[package]] name = "digest" version = "0.10.7" @@ -1868,21 +1827,6 @@ dependencies = [ "litrs", ] -[[package]] -name = "dtor" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "404d02eeb088a82cfd873006cb713fe411306c7d182c344905e101fb1167d301" -dependencies = [ - "dtor-proc-macro", -] - -[[package]] -name = "dtor-proc-macro" -version = "0.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f678cf4a922c215c63e0de95eb1ff08a958a81d47e485cf9da1e27bf6305cfa5" - [[package]] name = "dunce" version = "1.0.5" @@ -1907,6 +1851,24 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "encoding_rs_io" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cc3c5651fb62ab8aa3103998dade57efdd028544bd300516baa31840c252a83" +dependencies = [ + "encoding_rs", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -1923,6 +1885,27 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "etcetera" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de48cc4d1c1d97a20fd819def54b890cadde72ed3ad0c614822a0a433361be96" +dependencies = [ + "cfg-if", + "windows-sys 0.61.2", +] + +[[package]] +name = "faccess" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ae66425802d6a903e268ae1a08b8c38ba143520f227a205edf4e9c7e3e26d5" +dependencies = [ + "bitflags 1.3.2", + "libc", + "winapi", +] + [[package]] name = "fancy-regex" version = "0.16.2" @@ -1951,6 +1934,32 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "fd-find" +version = "10.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b95ed7d1f53e0446a7d47715801f6bee95f816c4aa33e25b5d89a2734ab00436" +dependencies = [ + "aho-corasick", + "anyhow", + "argmax", + "clap", + "crossbeam-channel", + "ctrlc", + "etcetera", + "faccess", + "globset", + "ignore", + "jiff", + "libc", + "lscolors", + "nix 0.31.3", + "normpath", + "nu-ansi-term", + "regex", + "regex-syntax", +] + [[package]] name = "fd-lock" version = "4.0.4" @@ -1979,6 +1988,25 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "findutils" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "719cc0af0060da07d610f4faa20d2d46dca46ff6bdf778a4b4feec3a7c8af7ea" +dependencies = [ + "argmax", + "chrono", + "clap", + "faccess", + "itertools 0.14.0", + "nix 0.31.3", + "onig", + "regex", + "thiserror 2.0.18", + "uucore 0.9.0", + "walkdir", +] + [[package]] name = "fixed_decimal" version = "0.7.1" @@ -2000,15 +2028,6 @@ dependencies = [ "miniz_oxide", ] -[[package]] -name = "float-cmp" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b09cf3155332e944990140d967ff5eceb70df778b34f77d8075db46e4704e6d8" -dependencies = [ - "num-traits", -] - [[package]] name = "fluent" version = "0.17.0" @@ -2281,14 +2300,82 @@ dependencies = [ ] [[package]] -name = "globwalk" -version = "0.9.1" +name = "grep" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf760ebf69878d9fd8f110c89703d90ce35095324d1f1edcb595c63945ee757" +checksum = "309217bc53e2c691c314389c7fa91f9cd1a998cda19e25544ea47d94103880c3" dependencies = [ - "bitflags 2.11.0", - "ignore", - "walkdir", + "grep-cli", + "grep-matcher", + "grep-printer", + "grep-regex", + "grep-searcher", +] + +[[package]] +name = "grep-cli" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf32d263c5d5cc2a23ce587097f5ddafdb188492ba2e6fb638eaccdc22453631" +dependencies = [ + "bstr", + "globset", + "libc", + "log", + "termcolor", + "winapi-util", +] + +[[package]] +name = "grep-matcher" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36d7b71093325ab22d780b40d7df3066ae4aebb518ba719d38c697a8228a8023" +dependencies = [ + "memchr", +] + +[[package]] +name = "grep-printer" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd76035e87871f51c1ee5b793e32122b3ccf9c692662d9622ef1686ff5321acb" +dependencies = [ + "bstr", + "grep-matcher", + "grep-searcher", + "log", + "serde", + "serde_json", + "termcolor", +] + +[[package]] +name = "grep-regex" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce0c256c3ad82bcc07b812c15a45ec1d398122e8e15124f96695234db7112ef" +dependencies = [ + "bstr", + "grep-matcher", + "log", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "grep-searcher" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac63295322dc48ebb20a25348147905d816318888e64f531bfc2a2bc0577dc34" +dependencies = [ + "bstr", + "encoding_rs", + "encoding_rs_io", + "grep-matcher", + "log", + "memchr", + "memmap2", ] [[package]] @@ -2970,11 +3057,17 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" +[[package]] +name = "lexopt" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "803ec87c9cfb29b9d2633f20cba1f488db3fd53f2158b1024cbefb47ba05d413" + [[package]] name = "libc" -version = "0.2.183" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libm" @@ -3138,6 +3231,18 @@ dependencies = [ "libc", ] +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags 2.11.0", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "nom" version = "7.1.3" @@ -3157,18 +3262,21 @@ dependencies = [ "memchr", ] -[[package]] -name = "normalize-line-endings" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" - [[package]] name = "normalize-path" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f5438dd2b2ff4c6df6e1ce22d825ed2fa93ee2922235cc45186991717f0a892d" +[[package]] +name = "normpath" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9985ef7269fa99f3b12437bb698381da2428743ab90f20393f399fa14cab21a" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "notify" version = "8.2.0" @@ -3316,6 +3424,28 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "onig" +version = "6.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc3cbf698f9438986c11a880c90a6d04b9de27575afd28bbf45b154b6c709e2" +dependencies = [ + "bitflags 2.11.0", + "libc", + "once_cell", + "onig_sys", +] + +[[package]] +name = "onig_sys" +version = "69.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e68317604e77e53b85896388e1a803c1d21b74c899ec9e5e1112db90735edd7" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "os_display" version = "0.1.4" @@ -3481,6 +3611,12 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + [[package]] name = "plain" version = "0.2.3" @@ -3532,36 +3668,6 @@ dependencies = [ "zerocopy", ] -[[package]] -name = "predicates" -version = "3.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ada8f2932f28a27ee7b70dd6c1c39ea0675c55a36879ab92f3a715eaa1e63cfe" -dependencies = [ - "anstyle", - "difflib", - "float-cmp", - "normalize-line-endings", - "predicates-core", - "regex", -] - -[[package]] -name = "predicates-core" -version = "1.0.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cad38746f3166b4031b1a0d39ad9f954dd291e7854fcc0eed52ee41a0b50d144" - -[[package]] -name = "predicates-tree" -version = "1.0.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0de1b847b39c8131db0467e9df1ff60e6d0562ab8e9a16e568ad0fdb372e2f2" -dependencies = [ - "predicates-core", - "termtree", -] - [[package]] name = "prettyplease" version = "0.2.37" @@ -3814,6 +3920,24 @@ version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +[[package]] +name = "ripgrep" +version = "15.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f388c4955f85477c28a8667355819844a06614b083c23517f0e86bd1d6d82b73" +dependencies = [ + "anyhow", + "bstr", + "grep", + "ignore", + "lexopt", + "log", + "serde_json", + "termcolor", + "textwrap", + "tikv-jemallocator", +] + [[package]] name = "rlimit" version = "0.10.2" @@ -3823,12 +3947,6 @@ dependencies = [ "libc", ] -[[package]] -name = "roff" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "323c417e1d9665a65b263ec744ba09030cfb277e9daa0b018a4ab62e57bc8189" - [[package]] name = "rpds" version = "1.2.0" @@ -3949,13 +4067,6 @@ dependencies = [ "regex", ] -[[package]] -name = "secureexec-fd" -version = "0.1.0" -dependencies = [ - "regex", -] - [[package]] name = "secureexec-file-cmd" version = "0.1.0" @@ -3963,29 +4074,6 @@ dependencies = [ "infer", ] -[[package]] -name = "secureexec-find" -version = "0.1.0" -dependencies = [ - "regex", -] - -[[package]] -name = "secureexec-git" -version = "0.1.0" -dependencies = [ - "flate2", - "secureexec-wasi-http", - "sha1", -] - -[[package]] -name = "secureexec-grep" -version = "0.1.0" -dependencies = [ - "regex", -] - [[package]] name = "secureexec-gzip" version = "0.1.0" @@ -4030,10 +4118,6 @@ dependencies = [ "tar", ] -[[package]] -name = "secureexec-tree" -version = "0.1.0" - [[package]] name = "secureexec-wasi-http" version = "0.1.0" @@ -4076,24 +4160,16 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f5a00cdbffacbccf02decb7577da5e298c091bdee492a2a1b87a3d8b0cc5b7c5" dependencies = [ - "assert_fs", "clap", - "clap_complete", - "clap_mangen", - "ctor", "fancy-regex 0.17.0", "memchr", "memmap2", "once_cell", "phf 0.13.1", "phf_codegen 0.13.1", - "predicates", "regex", - "sysinfo", "tempfile", - "terminal_size", - "textwrap", - "uucore 0.5.0", + "uucore 0.7.0", ] [[package]] @@ -4299,12 +4375,6 @@ version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" -[[package]] -name = "smawk" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8e2fb0f499abb4d162f2bedad68f5ef91a1682b5a03596ddb67efd37768d100" - [[package]] name = "spin" version = "0.10.0" @@ -4454,6 +4524,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + [[package]] name = "terminal_size" version = "0.4.3" @@ -4476,23 +4555,11 @@ dependencies = [ "phf_codegen 0.11.3", ] -[[package]] -name = "termtree" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" - [[package]] name = "textwrap" version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057" -dependencies = [ - "smawk", - "terminal_size", - "unicode-linebreak", - "unicode-width 0.2.0", -] [[package]] name = "thiserror" @@ -4543,6 +4610,26 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "tikv-jemalloc-sys" +version = "0.6.1+5.3.0-1-ge13ca993e8ccb9ba9847cc330696e02839f328f7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd8aa5b2ab86a2cefa406d889139c162cbb230092f7d1d7cbc1716405d852a3b" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "tikv-jemallocator" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0359b4327f954e0567e69fb191cf1436617748813819c94b8cd4a431422d053a" +dependencies = [ + "libc", + "tikv-jemalloc-sys", +] + [[package]] name = "tinystr" version = "0.8.2" @@ -4756,12 +4843,6 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" -[[package]] -name = "unicode-linebreak" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" - [[package]] name = "unicode-segmentation" version = "1.12.0" @@ -5815,26 +5896,6 @@ dependencies = [ "wild", ] -[[package]] -name = "uucore" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5eddd390f3fdef74f104a948559e6de29203f60f8f563c8c9f528cd4c88ee78" -dependencies = [ - "clap", - "fluent", - "fluent-bundle", - "fluent-syntax", - "libc", - "nix 0.30.1", - "os_display", - "phf 0.13.1", - "thiserror 2.0.18", - "unic-langid", - "uucore_procs 0.5.0", - "wild", -] - [[package]] name = "uucore" version = "0.7.0" @@ -5888,6 +5949,30 @@ dependencies = [ "z85", ] +[[package]] +name = "uucore" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "069b34217c27f611e1589f540f58118dbf226a9e407d38ab472052ff075a1dc2" +dependencies = [ + "bstr", + "clap", + "dunce", + "fluent", + "fluent-syntax", + "libc", + "nix 0.31.3", + "os_display", + "rustc-hash", + "rustix 1.1.4", + "thiserror 2.0.18", + "unic-langid", + "uucore_procs 0.9.0", + "wild", + "winapi-util", + "windows-sys 0.61.2", +] + [[package]] name = "uucore_procs" version = "0.4.0" @@ -5900,9 +5985,9 @@ dependencies = [ [[package]] name = "uucore_procs" -version = "0.5.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47148309a1f7a989d165dabbbc7f2bf156d7ff6affe7d69c1c5bfb822e663ae6" +checksum = "0f63e2d5083ff0983193a33e2d57fd271c7e3e3e7df8e46e8f471865647b2cbc" dependencies = [ "proc-macro2", "quote", @@ -5910,9 +5995,9 @@ dependencies = [ [[package]] name = "uucore_procs" -version = "0.7.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f63e2d5083ff0983193a33e2d57fd271c7e3e3e7df8e46e8f471865647b2cbc" +checksum = "34337da211e7abfff7189b794afb3b5018fe356fb36474b73645458fc1201350" dependencies = [ "proc-macro2", "quote", diff --git a/toolchain/Makefile b/toolchain/Makefile index e14e52b982..cff2c8ea86 100644 --- a/toolchain/Makefile +++ b/toolchain/Makefile @@ -1,5 +1,8 @@ WASM_TARGET := wasm32-wasip1 RELEASE_DIR := target/$(WASM_TARGET)/release +WASI_C_SYSROOT := $(CURDIR)/c/sysroot +WASI_C_CC := $(CURDIR)/c/vendor/wasi-sdk/bin/clang +WASI_C_AR := $(CURDIR)/c/vendor/wasi-sdk/bin/llvm-ar # Standalone binary output directory (configurable) COMMANDS_DIR ?= $(RELEASE_DIR)/commands @@ -24,7 +27,6 @@ COMMAND_PACKAGES := $(addprefix -p cmd-,$(COMMAND_NAMES)) # Alias symlinks: link_name:target_binary ALIAS_SYMLINKS := \ - egrep:grep fgrep:grep \ gunzip:gzip zcat:gzip \ bash:sh \ dir:ls vdir:ls \ @@ -55,29 +57,56 @@ all: wasm commands: wasm $(MAKE) -C c programs install -# Build the real wasm32-wasip1 `codex-exec` agent engine from the codex fork -# (branch wasi-port-codex-core). The fork ships a committed, idempotent, self-prepping build script -# (it prepares the rustup sysroot for -Z build-std and restores it on exit — no manual stash), so -# this folds codex into the toolchain: `make -C toolchain codex` produces and installs -# software/codex/wasm/{codex-exec,codex}. Override CODEX_REPO if the fork is not at the -# default sibling path. See the fork's docs/wasi-build.md for the toolchain constraints, and the -# remaining work to source the CRT from wasi-sdk so the prebuilt rust target need not be installed. +# Build the real wasm32-wasip1 `codex-exec` agent engine and install it where +# @agentos-software/codex-cli stages commands from. +# +# Two modes: +# - default (CODEX_REPO unset): REPRODUCIBLE build. clone-and-build-codex-wasi.sh +# shallow-clones the fork pinned in toolchain/codex-ref, rewrites its +# [patch.crates-io] to the toolchain stubs + vendored/patched tokio, and builds. +# No local checkout needed; deterministic from the pin. +# - CODEX_REPO=/path/to/codex-rs/codex-rs: DEV override. Builds that existing +# checkout in place via scripts/build-codex-wasi.sh (fast local iteration). AGENTOS_ROOT := $(abspath $(CURDIR)/..) -CODEX_REPO ?= $(abspath $(AGENTOS_ROOT)/../codex-rs/codex-rs) -codex: +CODEX_REF := $(shell cat $(CURDIR)/codex-ref 2>/dev/null) +CODEX_REPO ?= +CODEX_WASI_SDK_DIR := $(abspath $(CURDIR)/c/vendor/wasi-sdk) +codex: c/vendor/wasi-sdk/bin/clang +ifeq ($(strip $(CODEX_REPO)),) + @echo "codex: reproducible build from pin $(CODEX_REF)" + @WASI_SDK_DIR="$(CODEX_WASI_SDK_DIR)" \ + "$(CURDIR)/scripts/clone-and-build-codex-wasi.sh" +else @if [ -x "$(CODEX_REPO)/scripts/build-wasi-codex-exec.sh" ]; then \ - AGENTOS_DIR="$(AGENTOS_ROOT)" "$(CODEX_REPO)/scripts/build-wasi-codex-exec.sh"; \ + echo "codex: CODEX_REPO override -> building $(CODEX_REPO) in place"; \ + CODEX_REPO="$(CODEX_REPO)" \ + WASI_SDK_DIR="$(CODEX_WASI_SDK_DIR)" \ + "$(CURDIR)/scripts/build-codex-wasi.sh"; \ else \ - echo "codex: fork build script not found at $(CODEX_REPO)/scripts/build-wasi-codex-exec.sh — skipping."; \ - echo " Set CODEX_REPO=/path/to/codex-rs/codex-rs (the wasi-port-codex-core checkout) to build it."; \ + echo "codex: CODEX_REPO=$(CODEX_REPO) has no scripts/build-wasi-codex-exec.sh"; \ + echo " Unset CODEX_REPO to build reproducibly from the pin instead."; \ + exit 1; \ fi +endif + +c/vendor/wasi-sdk/bin/clang: + $(MAKE) -C c wasi-sdk + +c/sysroot/lib/wasm32-wasi/libc.a: + $(MAKE) -C c sysroot -# Strict variant (fails if the fork is absent) for environments that require the codex artifact. +# Strict variant for environments that require the codex artifact. With CODEX_REPO +# unset it builds reproducibly from the pin; with CODEX_REPO set it fails hard if +# that checkout lacks the fork build script. .PHONY: codex-required codex-required: +ifeq ($(strip $(CODEX_REPO)),) + @$(MAKE) codex +else @test -x "$(CODEX_REPO)/scripts/build-wasi-codex-exec.sh" || { \ - echo "ERROR: codex fork build script not found at $(CODEX_REPO); set CODEX_REPO"; exit 1; } + echo "ERROR: codex fork build script not found at $(CODEX_REPO); unset CODEX_REPO to build from the pin"; exit 1; } @$(MAKE) codex +endif # Apply std patches if they exist patch-std: @@ -145,10 +174,13 @@ wasm-opt-check: fi # Build all standalone command binaries, optimize, strip .wasm extension, create symlinks -wasm: vendor patch-vendor patch-std wasm-opt-check +wasm: c/sysroot/lib/wasm32-wasi/libc.a vendor patch-vendor patch-std wasm-opt-check # --cfg tokio_unstable lifts tokio's wasm `compile_error!` so the patched # tokio process/net modules (std-patches/crates/tokio) compile on wasm32-wasip1. # Needed by codex (tokio::process/net). Harmless for commands not using tokio. + CC_wasm32_wasip1="$(WASI_C_CC)" \ + AR_wasm32_wasip1="$(WASI_C_AR)" \ + CFLAGS_wasm32_wasip1="--sysroot=$(WASI_C_SYSROOT)" \ RUSTFLAGS="-C link-arg=-L$(WASI_SYSROOT_SELF_CONTAINED) --cfg tokio_unstable" \ cargo build --target $(WASM_TARGET) \ -Z build-std=std,panic_abort \ @@ -210,7 +242,7 @@ wasm: vendor patch-vendor patch-std wasm-opt-check # codex the codex fork build (codex, codex-exec) # `just toolchain-cmd ` calls this. `make wasm` builds the fast # Rust set; `make -C c programs install` builds/installs the fast C set. -C_COMMANDS := zip unzip envsubst sqlite3 curl wget duckdb vim http_get +C_COMMANDS := zip unzip envsubst sqlite3 curl wget grep git duckdb vim .PHONY: cmd/% cmd/%: @@ -231,16 +263,28 @@ cmd/%: # Build ONE command binary (cargo package cmd-$(CMD) -> $(COMMANDS_DIR)/$(CMD)). # Usage: make wasm-cmd CMD=sh .PHONY: wasm-cmd -wasm-cmd: vendor patch-vendor patch-std wasm-opt-check +wasm-cmd: c/sysroot/lib/wasm32-wasi/libc.a vendor patch-vendor patch-std wasm-opt-check @test -n "$(CMD)" || { echo "ERROR: pass CMD= (a dir under crates/commands/)"; exit 1; } @crate_dir=$$(find ../software -path "*/native/crates/cmd-$(CMD)" -type d -print -quit); \ test -n "$$crate_dir" || test -d "crates/commands/$(CMD)" || { \ echo "ERROR: no software/*/native/crates/cmd-$(CMD) or crates/commands/$(CMD)"; exit 1; } + @cargo_pkg="cmd-$(CMD)"; cargo_bin="$(CMD)"; \ + if [ "$(CMD)" = "fd" ]; then \ + cargo_pkg="fd-find"; \ + cargo_bin="fd"; \ + elif [ "$(CMD)" = "rg" ]; then \ + cargo_pkg="ripgrep"; \ + cargo_bin="rg"; \ + fi; \ + CC_wasm32_wasip1="$(WASI_C_CC)" \ + AR_wasm32_wasip1="$(WASI_C_AR)" \ + CFLAGS_wasm32_wasip1="--sysroot=$(WASI_C_SYSROOT)" \ RUSTFLAGS="-C link-arg=-L$(WASI_SYSROOT_SELF_CONTAINED) --cfg tokio_unstable" \ cargo build --target $(WASM_TARGET) \ -Z build-std=std,panic_abort \ --release \ - -p cmd-$(CMD) + -p $$cargo_pkg \ + --bin $$cargo_bin @mkdir -p $(COMMANDS_DIR) @if command -v wasm-opt >/dev/null 2>&1; then \ wasm-opt -O3 --strip-debug --all-features "$(RELEASE_DIR)/$(CMD).wasm" -o "$(COMMANDS_DIR)/$(CMD)"; \ diff --git a/toolchain/c/Makefile b/toolchain/c/Makefile index a82fcba527..0190b360f2 100644 --- a/toolchain/c/Makefile +++ b/toolchain/c/Makefile @@ -68,14 +68,14 @@ COMMANDS_DIR ?= ../target/wasm32-wasip1/release/commands # Fast real commands installed by the default software gate. Slow/heavy commands # (duckdb, vim) remain available through explicit parent `make cmd/`. -COMMANDS := zip unzip envsubst sqlite3 curl http_get +COMMANDS := zip unzip envsubst sqlite3 curl wget grep git tree # Programs requiring patched sysroot (Tier 2+ custom host imports) -PATCHED_PROGRAMS := http_get_test isatty_test getpid_test getppid_test getppid_verify userinfo pipe_test dup_test spawn_child spawn_exit_code pipeline kill_child waitpid_return waitpid_edge syscall_coverage getpwuid_test signal_tests sigaction_self sigaction_behavior delayed_tcp_echo delayed_kill pipe_edge tcp_accept_spawn tcp_echo tcp_server http_server udp_echo unix_socket signal_handler http_get dns_lookup sqlite3_cli curl wget fs_probe +PATCHED_PROGRAMS := http_get_test isatty_test getpid_test getppid_test getppid_verify userinfo pipe_test dup_test spawn_child spawn_exit_code pipeline kill_child waitpid_return waitpid_edge syscall_coverage getpwuid_test signal_tests sigaction_self sigaction_behavior delayed_tcp_echo delayed_kill pipe_edge tcp_accept_spawn tcp_echo tcp_server http_server udp_echo unix_socket signal_handler dns_lookup sqlite3 curl wget grep tree zip unzip fs_probe # Discover all package command and test-program C source files. ALL_SOURCES := $(foreach dir,$(C_SOURCE_DIRS),$(wildcard $(dir)/*.c)) -SKIPPED_BULK_PROGRAMS := wget +SKIPPED_BULK_PROGRAMS := # Exclude patched-sysroot programs when only vanilla sysroot is available ifeq ($(wildcard $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a),) @@ -89,7 +89,7 @@ endif ifeq ($(wildcard $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a),) CUSTOM_WASM_PROG_NAMES := else - CUSTOM_WASM_PROG_NAMES := curl + CUSTOM_WASM_PROG_NAMES := curl wget grep sqlite3 tree zip unzip endif WASM_PROG_NAMES := $(sort $(basename $(notdir $(SOURCES))) $(CUSTOM_WASM_PROG_NAMES)) @@ -136,7 +136,6 @@ $(WASI_SDK_DIR)/bin/clang: # All downloads cached in libs/ — add libs/ to .gitignore. SQLITE3_URL := https://www.sqlite.org/2024/sqlite-amalgamation-3470200.zip -ZLIB_URL := https://github.com/madler/zlib/archive/refs/tags/v1.3.1.zip CJSON_URL := https://github.com/DaveGamble/cJSON/archive/refs/tags/v1.7.18.zip CURL_COMMIT := main CURL_FORK_REPO_PREFIX := secure @@ -151,7 +150,29 @@ CURL_RELEASE_URL := https://github.com/curl/curl/releases/download/curl-$(CURL_R CURL_UPSTREAM_BUILD_DIR := $(BUILD_DIR)/curl-upstream CURL_UPSTREAM_OVERLAY_DIR := ../../software/curl/native/c/overlay CURL_UPSTREAM_OVERLAY_FILES := $(wildcard $(CURL_UPSTREAM_OVERLAY_DIR)/lib/*.c $(CURL_UPSTREAM_OVERLAY_DIR)/lib/*.h $(CURL_UPSTREAM_OVERLAY_DIR)/lib/vtls/*.c $(CURL_UPSTREAM_OVERLAY_DIR)/lib/vtls/*.h) -MINIZIP_URL := https://github.com/madler/zlib/archive/refs/tags/v1.3.1.zip +ZLIB_VERSION := 1.3.1 +ZLIB_URL := https://zlib.net/zlib-$(ZLIB_VERSION).tar.gz +ZLIB_BUILD_DIR := $(BUILD_DIR)/zlib +WGET_VERSION := 1.24.5 +WGET_URL := https://ftp.gnu.org/gnu/wget/wget-$(WGET_VERSION).tar.gz +WGET_UPSTREAM_BUILD_DIR := $(BUILD_DIR)/wget-upstream +WGET_UPSTREAM_OVERLAY_INCLUDE_DIR := overlays/wget/include +WGET_UPSTREAM_OVERLAY_FILES := $(wildcard $(WGET_UPSTREAM_OVERLAY_INCLUDE_DIR)/*.h) +GIT_VERSION := 2.55.0 +GIT_URL := https://www.kernel.org/pub/software/scm/git/git-$(GIT_VERSION).tar.xz +GIT_UPSTREAM_BUILD_DIR := $(BUILD_DIR)/git-upstream +GIT_PATCH_DIR := patches/git +GREP_VERSION := 3.12 +GREP_URL := https://ftp.gnu.org/gnu/grep/grep-$(GREP_VERSION).tar.xz +GREP_UPSTREAM_BUILD_DIR := $(BUILD_DIR)/grep-upstream +INFOZIP_ZIP_VERSION := 30 +INFOZIP_ZIP_URL := https://downloads.sourceforge.net/infozip/zip$(INFOZIP_ZIP_VERSION).tar.gz +INFOZIP_ZIP_BUILD_DIR := $(BUILD_DIR)/infozip-zip-upstream +INFOZIP_UNZIP_VERSION := 60 +INFOZIP_UNZIP_URL := https://downloads.sourceforge.net/infozip/unzip$(INFOZIP_UNZIP_VERSION).tar.gz +INFOZIP_UNZIP_BUILD_DIR := $(BUILD_DIR)/infozip-unzip-upstream +TREE_VERSION := 2.3.2 +TREE_URL := https://gitlab.com/OldManProgrammer/unix-tree/-/archive/$(TREE_VERSION)/unix-tree-$(TREE_VERSION).tar.gz # duckdb-wasm currently documents DuckDB v1.5.0 in its README. We build the # matching upstream DuckDB tag ourselves with our patched WASI/POSIX sysroot. DUCKDB_VERSION := v1.5.0 @@ -161,32 +182,30 @@ DUCKDB_URL := https://github.com/duckdb/duckdb/archive/refs/tags/$(DUCKDB_VERSIO LIBS_DIR := libs LIBS_CACHE := .cache/libs -libs/sqlite3/sqlite3.c: +TREE_SOURCES := color.c file.c filter.c hash.c html.c info.c json.c list.c tree.c unix.c util.c xml.c strverscmp.c +TREE_FILES := $(addprefix $(LIBS_DIR)/tree/,$(TREE_SOURCES) tree.h) +TREE_WASM_FLAGS := -DLARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64 + +libs/sqlite3/sqlite3.c libs/sqlite3/sqlite3.h libs/sqlite3/shell.c: @echo "Fetching sqlite3..." @mkdir -p $(LIBS_CACHE) $(LIBS_DIR)/sqlite3 @curl -fSL "$(SQLITE3_URL)" -o "$(LIBS_CACHE)/sqlite3.zip" @cd $(LIBS_CACHE) && unzip -qo sqlite3.zip - @cp $(LIBS_CACHE)/sqlite-amalgamation-*/sqlite3.c $(LIBS_CACHE)/sqlite-amalgamation-*/sqlite3.h $(LIBS_DIR)/sqlite3/ - -libs/zlib/zutil.c: - @echo "Fetching zlib..." - @mkdir -p $(LIBS_CACHE) $(LIBS_DIR)/zlib - @curl -fSL "$(ZLIB_URL)" -o "$(LIBS_CACHE)/zlib.zip" - @cd $(LIBS_CACHE) && unzip -qo zlib.zip - @cp $(LIBS_CACHE)/zlib-*/*.c $(LIBS_CACHE)/zlib-*/*.h $(LIBS_DIR)/zlib/ - -libs/minizip/ioapi.c: - @echo "Fetching minizip (from zlib contrib)..." - @mkdir -p $(LIBS_CACHE) $(LIBS_DIR)/minizip - @if [ ! -f "$(LIBS_CACHE)/zlib.zip" ]; then \ - curl -fSL "$(MINIZIP_URL)" -o "$(LIBS_CACHE)/zlib.zip"; \ - cd $(LIBS_CACHE) && unzip -qo zlib.zip; \ - fi - @cp $(LIBS_CACHE)/zlib-*/contrib/minizip/ioapi.c $(LIBS_CACHE)/zlib-*/contrib/minizip/ioapi.h \ - $(LIBS_CACHE)/zlib-*/contrib/minizip/zip.c $(LIBS_CACHE)/zlib-*/contrib/minizip/zip.h \ - $(LIBS_CACHE)/zlib-*/contrib/minizip/unzip.c $(LIBS_CACHE)/zlib-*/contrib/minizip/unzip.h \ - $(LIBS_CACHE)/zlib-*/contrib/minizip/crypt.h \ - $(LIBS_DIR)/minizip/ + @cp $(LIBS_CACHE)/sqlite-amalgamation-*/sqlite3.c \ + $(LIBS_CACHE)/sqlite-amalgamation-*/sqlite3.h \ + $(LIBS_CACHE)/sqlite-amalgamation-*/shell.c \ + $(LIBS_DIR)/sqlite3/ + +$(TREE_FILES): $(LIBS_DIR)/tree/.fetched + +$(LIBS_DIR)/tree/.fetched: + @echo "Fetching upstream tree $(TREE_VERSION)..." + @mkdir -p $(LIBS_CACHE) $(LIBS_DIR)/tree + @curl -fSL "$(TREE_URL)" -o "$(LIBS_CACHE)/tree-$(TREE_VERSION).tar.gz" + @rm -rf "$(LIBS_CACHE)/unix-tree-$(TREE_VERSION)" + @tar -xzf "$(LIBS_CACHE)/tree-$(TREE_VERSION).tar.gz" -C "$(LIBS_CACHE)" + @cp $(addprefix $(LIBS_CACHE)/unix-tree-$(TREE_VERSION)/,$(TREE_SOURCES) tree.h) $(LIBS_DIR)/tree/ + @touch $@ libs/cjson/cJSON.c: @echo "Fetching cJSON..." @@ -212,8 +231,8 @@ libs/duckdb/CMakeLists.txt: @tar -xzf "$(LIBS_CACHE)/duckdb.tar.gz" --strip-components=1 -C $(LIBS_DIR)/duckdb .PHONY: fetch-libs fetch-fast-libs clean-libs -fetch-libs: libs/sqlite3/sqlite3.c libs/zlib/zutil.c libs/minizip/ioapi.c libs/cjson/cJSON.c libs/curl/lib/easy.c libs/duckdb/CMakeLists.txt -fetch-fast-libs: libs/sqlite3/sqlite3.c libs/zlib/zutil.c libs/minizip/ioapi.c libs/cjson/cJSON.c libs/curl/lib/easy.c +fetch-libs: libs/sqlite3/sqlite3.c libs/cjson/cJSON.c libs/curl/lib/easy.c libs/duckdb/CMakeLists.txt +fetch-fast-libs: libs/sqlite3/sqlite3.c libs/cjson/cJSON.c libs/curl/lib/easy.c clean-libs: rm -rf $(LIBS_DIR) $(LIBS_CACHE) @@ -474,6 +493,11 @@ SQLITE_COMMON := -DSQLITE_OMIT_WAL -DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_THREAD -DSQLITE_OMIT_LOCALTIME SQLITE_WASM := -DSQLITE_OS_OTHER $(SQLITE_COMMON) -lwasi-emulated-signal -lwasi-emulated-mman SQLITE_NATIVE := $(SQLITE_COMMON) +SQLITE_SHELL_COMMON := -DSQLITE_OMIT_WAL -DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_THREADSAFE=0 \ + -DSQLITE_OMIT_LOCALTIME -DSQLITE_NOHAVE_SYSTEM -DHAVE_READLINE=0 +SQLITE_SHELL_WASM := $(SQLITE_SHELL_COMMON) -D_WASI_EMULATED_PROCESS_CLOCKS \ + -lwasi-emulated-signal -lwasi-emulated-mman -lwasi-emulated-process-clocks +SQLITE_SHELL_NATIVE := $(SQLITE_SHELL_COMMON) $(BUILD_DIR)/sqlite3_mem: $(call c_source,sqlite3_mem) libs/sqlite3/sqlite3.c $(WASI_SDK_DIR)/bin/clang @mkdir -p $(BUILD_DIR) @@ -486,58 +510,94 @@ $(NATIVE_DIR)/sqlite3_mem: $(call c_source,sqlite3_mem) libs/sqlite3/sqlite3.c $(NATIVE_CC) $(NATIVE_CFLAGS) $(SQLITE_NATIVE) -Ilibs/sqlite3 -o $@ \ $(call c_source,sqlite3_mem) libs/sqlite3/sqlite3.c -lm -lpthread -# sqlite3_cli: full CLI, links SQLite amalgamation (installed as sqlite3) +# sqlite3: full official CLI, links SQLite amalgamation. # Uses -Os (not -O2) + no wasm-opt — higher optimization breaks SQLite's # internal function pointer tables in WASM indirect call table. # Uses --initial-memory=16777216 (16MB) for SQLite's malloc requirements. -$(BUILD_DIR)/sqlite3_cli: $(call c_source,sqlite3_cli) libs/sqlite3/sqlite3.c $(WASI_SDK_DIR)/bin/clang +$(BUILD_DIR)/sqlite3: libs/sqlite3/shell.c libs/sqlite3/sqlite3.c $(WASI_SDK_DIR)/bin/clang @mkdir -p $(BUILD_DIR) $(CC) --target=wasm32-wasip1 --sysroot=$(SYSROOT) -Os -I include/ \ - $(SQLITE_WASM) -Ilibs/sqlite3 -Wl,--initial-memory=16777216 \ - -o $@ $(call c_source,sqlite3_cli) libs/sqlite3/sqlite3.c + $(SQLITE_SHELL_WASM) -Ilibs/sqlite3 -Wl,--initial-memory=16777216 \ + -o $@ libs/sqlite3/shell.c libs/sqlite3/sqlite3.c -$(NATIVE_DIR)/sqlite3_cli: $(call c_source,sqlite3_cli) libs/sqlite3/sqlite3.c - @mkdir -p $(NATIVE_DIR) - $(NATIVE_CC) $(NATIVE_CFLAGS) $(SQLITE_NATIVE) -Ilibs/sqlite3 -o $@ \ - $(call c_source,sqlite3_cli) libs/sqlite3/sqlite3.c -lm -lpthread +$(BUILD_DIR)/sqlite3_cli: $(BUILD_DIR)/sqlite3 + cp $< $@ -# zip: links zlib + minizip -ZLIB_SRCS := libs/zlib/adler32.c libs/zlib/compress.c libs/zlib/crc32.c libs/zlib/deflate.c \ - libs/zlib/infback.c libs/zlib/inffast.c libs/zlib/inflate.c libs/zlib/inftrees.c \ - libs/zlib/trees.c libs/zlib/uncompr.c libs/zlib/zutil.c -MINIZIP_SRCS := libs/minizip/ioapi.c libs/minizip/zip.c -ZIP_INCLUDES := -Ilibs/zlib -Ilibs/minizip +$(NATIVE_DIR)/sqlite3: libs/sqlite3/shell.c libs/sqlite3/sqlite3.c + @mkdir -p $(NATIVE_DIR) + $(NATIVE_CC) $(NATIVE_CFLAGS) $(SQLITE_SHELL_NATIVE) -Ilibs/sqlite3 -o $@ \ + libs/sqlite3/shell.c libs/sqlite3/sqlite3.c -lm -lpthread -$(BUILD_DIR)/zip: $(call c_source,zip) $(ZLIB_SRCS) $(MINIZIP_SRCS) $(WASI_SDK_DIR)/bin/clang +$(LIBS_CACHE)/zlib-$(ZLIB_VERSION)/zlib.h: + @echo "Fetching zlib $(ZLIB_VERSION)..." + @mkdir -p $(LIBS_CACHE) + @curl -fSL "$(ZLIB_URL)" -o "$(LIBS_CACHE)/zlib-$(ZLIB_VERSION).tar.gz" + @rm -rf "$(LIBS_CACHE)/zlib-$(ZLIB_VERSION)" + @tar -xzf "$(LIBS_CACHE)/zlib-$(ZLIB_VERSION).tar.gz" -C "$(LIBS_CACHE)" + +$(ZLIB_BUILD_DIR)/libz.a: $(LIBS_CACHE)/zlib-$(ZLIB_VERSION)/zlib.h $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a $(WASI_SDK_DIR)/bin/clang + @mkdir -p $(ZLIB_BUILD_DIR) + @for src in adler32.c crc32.c deflate.c infback.c inffast.c inflate.c inftrees.c trees.c zutil.c compress.c uncompr.c gzclose.c gzlib.c gzread.c gzwrite.c; do \ + $(CC) --target=wasm32-wasip1 --sysroot=$(SYSROOT) -O2 -DHAVE_UNISTD_H \ + -I$(LIBS_CACHE)/zlib-$(ZLIB_VERSION) \ + -c "$(LIBS_CACHE)/zlib-$(ZLIB_VERSION)/$$src" \ + -o "$(ZLIB_BUILD_DIR)/$${src%.c}.o"; \ + done + $(WASI_SDK_DIR)/bin/llvm-ar rcs $@ $(ZLIB_BUILD_DIR)/*.o + +$(NATIVE_DIR)/sqlite3_cli: $(NATIVE_DIR)/sqlite3 + cp $< $@ + +# tree: upstream Steve Baker tree compiled from pinned C source. +$(BUILD_DIR)/tree: $(TREE_FILES) $(WASI_SDK_DIR)/bin/clang $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a @mkdir -p $(BUILD_DIR) - $(CC) $(WASM_CFLAGS) $(ZIP_INCLUDES) -o $@.wasm $(call c_source,zip) $(ZLIB_SRCS) $(MINIZIP_SRCS) - @if [ "$(HAS_WASM_OPT)" = "1" ]; then \ - wasm-opt -O3 --strip-debug $@.wasm -o $@; \ - else \ - cp $@.wasm $@; \ - fi - @rm -f $@.wasm + $(CC) --target=wasm32-wasip1 --sysroot=$(SYSROOT) -O2 -I include/ \ + $(TREE_WASM_FLAGS) -I$(LIBS_DIR)/tree -o $@ \ + $(addprefix $(LIBS_DIR)/tree/,$(TREE_SOURCES)) -$(NATIVE_DIR)/zip: $(call c_source,zip) $(ZLIB_SRCS) $(MINIZIP_SRCS) +$(NATIVE_DIR)/tree: $(TREE_FILES) @mkdir -p $(NATIVE_DIR) - $(NATIVE_CC) $(NATIVE_CFLAGS) $(ZIP_INCLUDES) -o $@ $(call c_source,zip) $(ZLIB_SRCS) $(MINIZIP_SRCS) + $(NATIVE_CC) $(NATIVE_CFLAGS) $(TREE_WASM_FLAGS) -I$(LIBS_DIR)/tree -o $@ \ + $(addprefix $(LIBS_DIR)/tree/,$(TREE_SOURCES)) -# unzip: links zlib + minizip (unzip side) -MINIZIP_UNZIP_SRCS := libs/minizip/ioapi.c libs/minizip/unzip.c +# zip/unzip: upstream Info-ZIP releases built against the patched C sysroot. +$(BUILD_DIR)/zip: scripts/build-infozip-upstream.sh $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a $(WASI_SDK_DIR)/bin/clang + @mkdir -p $(BUILD_DIR) + bash scripts/build-infozip-upstream.sh \ + --tool zip \ + --version "$(INFOZIP_ZIP_VERSION)" \ + --url "$(INFOZIP_ZIP_URL)" \ + --cache-dir "$(abspath $(LIBS_CACHE))" \ + --build-dir "$(abspath $(INFOZIP_ZIP_BUILD_DIR))" \ + --cc "$(abspath $(CC)) --target=wasm32-wasip1 --sysroot=$(abspath $(SYSROOT))" \ + --output "$(abspath $@)" -$(BUILD_DIR)/unzip: $(call c_source,unzip) $(ZLIB_SRCS) $(MINIZIP_UNZIP_SRCS) $(WASI_SDK_DIR)/bin/clang +$(BUILD_DIR)/unzip: scripts/build-infozip-upstream.sh $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a $(WASI_SDK_DIR)/bin/clang @mkdir -p $(BUILD_DIR) - $(CC) $(WASM_CFLAGS) $(ZIP_INCLUDES) -o $@.wasm $(call c_source,unzip) $(ZLIB_SRCS) $(MINIZIP_UNZIP_SRCS) - @if [ "$(HAS_WASM_OPT)" = "1" ]; then \ - wasm-opt -O3 --strip-debug $@.wasm -o $@; \ - else \ - cp $@.wasm $@; \ - fi - @rm -f $@.wasm + bash scripts/build-infozip-upstream.sh \ + --tool unzip \ + --version "$(INFOZIP_UNZIP_VERSION)" \ + --url "$(INFOZIP_UNZIP_URL)" \ + --cache-dir "$(abspath $(LIBS_CACHE))" \ + --build-dir "$(abspath $(INFOZIP_UNZIP_BUILD_DIR))" \ + --cc "$(abspath $(CC)) --target=wasm32-wasip1 --sysroot=$(abspath $(SYSROOT))" \ + --output "$(abspath $@)" -$(NATIVE_DIR)/unzip: $(call c_source,unzip) $(ZLIB_SRCS) $(MINIZIP_UNZIP_SRCS) - @mkdir -p $(NATIVE_DIR) - $(NATIVE_CC) $(NATIVE_CFLAGS) $(ZIP_INCLUDES) -o $@ $(call c_source,unzip) $(ZLIB_SRCS) $(MINIZIP_UNZIP_SRCS) +$(BUILD_DIR)/git: scripts/build-git-upstream.sh $(ZLIB_BUILD_DIR)/libz.a $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a $(WASI_SDK_DIR)/bin/clang + @mkdir -p $(BUILD_DIR) + bash scripts/build-git-upstream.sh \ + --version "$(GIT_VERSION)" \ + --url "$(GIT_URL)" \ + --cache-dir "$(abspath $(LIBS_CACHE))" \ + --build-dir "$(abspath $(GIT_UPSTREAM_BUILD_DIR))" \ + --patch-dir "$(abspath $(GIT_PATCH_DIR))" \ + --zlib-dir "$(abspath $(LIBS_CACHE)/zlib-$(ZLIB_VERSION))" \ + --zlib-build-dir "$(abspath $(ZLIB_BUILD_DIR))" \ + --cc "$(abspath $(CC))" \ + --ar "$(abspath $(WASI_SDK_DIR)/bin/llvm-ar)" \ + --ranlib "$(abspath $(WASI_SDK_DIR)/bin/llvm-ranlib)" \ + --sysroot "$(abspath $(SYSROOT))" \ + --output "$(abspath $@)" # curl_test: links libcurl (HTTP/HTTPS build for WASM via host_net + host_tls) CURL_SRCS := $(wildcard libs/curl/lib/*.c) $(wildcard libs/curl/lib/vauth/*.c) \ @@ -546,7 +606,7 @@ CURL_SRCS := $(wildcard libs/curl/lib/*.c) $(wildcard libs/curl/lib/vauth/*.c) \ CURL_INCLUDES := -Ilibs/curl/include -Ilibs/curl/lib -include libs/curl/lib/curl_setup.h -include libs/curl/lib/curl_printf.h CURL_LIB_DEFS := -DHAVE_CONFIG_H -DBUILDING_LIBCURL -D_WASI_EMULATED_SIGNAL -DHAVE_BASENAME -DHAVE_LIBGEN_H -$(BUILD_DIR)/curl: scripts/build-curl-upstream.sh $(CURL_UPSTREAM_OVERLAY_FILES) $(WASI_SDK_DIR)/bin/clang +$(BUILD_DIR)/curl: scripts/build-curl-upstream.sh $(CURL_UPSTREAM_OVERLAY_FILES) $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a $(WASI_SDK_DIR)/bin/clang @mkdir -p $(BUILD_DIR) bash scripts/build-curl-upstream.sh \ --version "$(CURL_RELEASE_VERSION)" \ @@ -560,22 +620,30 @@ $(BUILD_DIR)/curl: scripts/build-curl-upstream.sh $(CURL_UPSTREAM_OVERLAY_FILES) --ranlib "$(abspath $(WASI_SDK_DIR)/bin/llvm-ranlib)" \ --output "$(abspath $@)" -# wget: minimal wget built on libcurl -$(BUILD_DIR)/wget: $(call c_source,wget) $(CURL_SRCS) $(WASI_SDK_DIR)/bin/clang +$(BUILD_DIR)/wget: scripts/build-wget-upstream.sh $(WGET_UPSTREAM_OVERLAY_FILES) $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a $(WASI_SDK_DIR)/bin/clang @mkdir -p $(BUILD_DIR) - $(CC) $(WASM_CFLAGS) $(CURL_LIB_DEFS) $(CURL_INCLUDES) \ - -lwasi-emulated-signal \ - -o $@.wasm $(call c_source,wget) $(CURL_SRCS) - @if [ "$(HAS_WASM_OPT)" = "1" ]; then \ - wasm-opt -O3 --strip-debug --all-features $@.wasm -o $@; \ - else \ - cp $@.wasm $@; \ - fi - @rm -f $@.wasm + bash scripts/build-wget-upstream.sh \ + --version "$(WGET_VERSION)" \ + --url "$(WGET_URL)" \ + --cache-dir "$(abspath $(LIBS_CACHE))" \ + --build-dir "$(abspath $(WGET_UPSTREAM_BUILD_DIR))" \ + --overlay-include-dir "$(abspath $(WGET_UPSTREAM_OVERLAY_INCLUDE_DIR))" \ + --cc "$(abspath $(CC)) --target=wasm32-wasip1 --sysroot=$(abspath $(SYSROOT))" \ + --ar "$(abspath $(WASI_SDK_DIR)/bin/llvm-ar)" \ + --ranlib "$(abspath $(WASI_SDK_DIR)/bin/llvm-ranlib)" \ + --output "$(abspath $@)" -$(NATIVE_DIR)/wget: $(call c_source,wget) - @mkdir -p $(NATIVE_DIR) - $(NATIVE_CC) $(NATIVE_CFLAGS) -o $@ $(call c_source,wget) -lcurl +$(BUILD_DIR)/grep: scripts/build-grep-upstream.sh $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a $(WASI_SDK_DIR)/bin/clang + @mkdir -p $(BUILD_DIR) + bash scripts/build-grep-upstream.sh \ + --version "$(GREP_VERSION)" \ + --url "$(GREP_URL)" \ + --cache-dir "$(abspath $(LIBS_CACHE))" \ + --build-dir "$(abspath $(GREP_UPSTREAM_BUILD_DIR))" \ + --cc "$(abspath $(CC)) --target=wasm32-wasip1 --sysroot=$(abspath $(SYSROOT))" \ + --ar "$(abspath $(WASI_SDK_DIR)/bin/llvm-ar)" \ + --ranlib "$(abspath $(WASI_SDK_DIR)/bin/llvm-ranlib)" \ + --output "$(abspath $@)" # duckdb: upstream DuckDB CLI built from source with our patched WASI/POSIX sysroot $(BUILD_DIR)/duckdb: libs/duckdb/CMakeLists.txt $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a $(WASI_SDK_DIR)/bin/clang $(WASI_SDK_DIR)/bin/clang++ cmake/FindThreads.cmake scripts/build-duckdb.sh include/fcntl.h include/ifaddrs.h include/net/if.h include/sys/ioctl.h @@ -619,9 +687,6 @@ install: @INSTALLED=0; \ for cmd in $(COMMANDS); do \ src="$$cmd"; \ - case "$$cmd" in \ - sqlite3) src="sqlite3_cli" ;; \ - esac; \ if [ -f "$(BUILD_DIR)/$$src" ]; then \ cp "$(BUILD_DIR)/$$src" "$(COMMANDS_DIR)/$$cmd"; \ INSTALLED=$$((INSTALLED + 1)); \ @@ -630,10 +695,11 @@ install: fi; \ done; \ echo "Installed $$INSTALLED C command(s) to $(COMMANDS_DIR)" - @# Create symlinks for git remote helpers + @# Create symlinks for Git helper entry points used by upstream Git. @if [ -f "$(COMMANDS_DIR)/git" ]; then \ - ln -sf git "$(COMMANDS_DIR)/git-remote-http"; \ - ln -sf git "$(COMMANDS_DIR)/git-remote-https"; \ + for alias in git-remote-http git-remote-https git-upload-pack git-receive-pack git-upload-archive; do \ + ln -sf git "$(COMMANDS_DIR)/$$alias"; \ + done; \ fi # --- Clean --- @@ -644,8 +710,8 @@ clean: # --- vim: real vim 9.2 -> wasm32-wasip1 against the full-OS libc --- # # vim compiles as a normal terminal program against the PATCHED sysroot plus a -# small bridge library (vim/): termios via host_tty (raw-mode + winsize), -# a termcap stub (builtin termcaps), and POSIX stubs the full-OS libc lacks. +# small bridge library (vim/): a termcap stub (builtin termcaps), and POSIX +# stubs the full-OS libc still lacks. Termios/process spawning live in sysroot. # Recipe recovered from the original hand build (see vim/ headers). The result # lands in $(BUILD_DIR)/vim; `make -C .. cmd/vim` installs it into the shared # commands dir. NOTE: run as `vim -n ` docs-wise is no longer needed — @@ -654,7 +720,7 @@ VIM_VERSION ?= v9.2.0780 VIM_SRC := libs/vim VIM_BRIDGE_SRC := ../../software/vim/native/c/vim-bridge VIM_BRIDGE_BUILD := $(BUILD_DIR)/vim-bridge -VIM_BRIDGE_OBJS := $(VIM_BRIDGE_BUILD)/termios_bridge.o $(VIM_BRIDGE_BUILD)/termcap_stub.o $(VIM_BRIDGE_BUILD)/posix_stubs.o +VIM_BRIDGE_OBJS := $(VIM_BRIDGE_BUILD)/termcap_stub.o $(VIM_BRIDGE_BUILD)/posix_stubs.o VIM_WCC := $(VIM_BRIDGE_BUILD)/wcc .PHONY: fetch-vim @@ -667,6 +733,7 @@ $(VIM_BRIDGE_BUILD)/%.o: $(VIM_BRIDGE_SRC)/%.c $(WASI_SDK_DIR)/bin/clang sysroot $(CC) --target=wasm32-wasip1 --sysroot=$(PATCHED_SYSROOT) -O2 -I $(VIM_BRIDGE_SRC) -c -o $@ $< $(VIM_BRIDGE_BUILD)/libtermcap.a: $(VIM_BRIDGE_OBJS) + rm -f $@ $(WASI_SDK_DIR)/bin/llvm-ar rcs $@ $(VIM_BRIDGE_OBJS) # Wrapper compiler: every vim TU gets the sysroot, the bridge headers, and the @@ -681,22 +748,25 @@ $(VIM_WCC): $(WASI_SDK_DIR)/bin/clang @chmod +x $@ $(BUILD_DIR)/vim: fetch-vim $(VIM_BRIDGE_BUILD)/libtermcap.a $(VIM_WCC) wasm-opt-check + rm -f $(VIM_SRC)/src/auto/config.cache cd $(VIM_SRC)/src && \ - CC="$(abspath $(VIM_WCC))" \ - LDFLAGS="$(abspath $(VIM_BRIDGE_BUILD)/termios_bridge.o) -L$(abspath $(VIM_BRIDGE_BUILD))" \ +CC="$(abspath $(VIM_WCC))" \ +LDFLAGS="-L$(abspath $(VIM_BRIDGE_BUILD))" \ vim_cv_toupper_broken=no \ vim_cv_terminfo=no \ vim_cv_tgetent=zero \ vim_cv_getcwd_broken=no \ vim_cv_stat_ignores_slash=no \ vim_cv_memmove_handles_overlap=yes \ - vim_cv_bcopy_handles_overlap=yes \ - vim_cv_memcpy_handles_overlap=yes \ - vim_cv_tty_group=world \ - vim_cv_tty_mode=0620 \ - ./configure --host=wasm32-wasip1 --build=$$(cc -dumpmachine 2>/dev/null || echo x86_64-pc-linux-gnu) \ +vim_cv_bcopy_handles_overlap=yes \ +vim_cv_memcpy_handles_overlap=yes \ +vim_cv_tty_group=world \ +vim_cv_tty_mode=0620 \ +ac_cv_header_dlfcn_h=no \ +./configure --host=wasm32-wasip1 --build=$$(cc -dumpmachine 2>/dev/null || echo x86_64-pc-linux-gnu) \ --with-features=normal --with-tlib=termcap \ --disable-gui --without-x --disable-nls --disable-channel \ + --without-wayland \ --disable-netbeans --disable-selinux --disable-gpm --disable-sysmouse \ --disable-icon-cache-update --disable-desktop-database-update $(MAKE) -C $(VIM_SRC)/src vim diff --git a/toolchain/c/overlays/wget/include/unistd.h b/toolchain/c/overlays/wget/include/unistd.h new file mode 100644 index 0000000000..7ff50df4d5 --- /dev/null +++ b/toolchain/c/overlays/wget/include/unistd.h @@ -0,0 +1,14 @@ +#ifndef AGENTOS_WGET_UNISTD_OVERLAY_H +#define AGENTOS_WGET_UNISTD_OVERLAY_H + +#include_next + +/* + * GNU Wget's ptimer fallback only needs to know that Linux-style POSIX timer + * IDs are not compile-time portable here. The underlying sysroot still exposes + * WASI clocks for libc++ and other consumers that use the WASI clockid_t ABI. + */ +#undef _POSIX_TIMERS +#define _POSIX_TIMERS 0 + +#endif diff --git a/toolchain/c/patches/git/0001-wasi-posix-spawn-run-command.patch b/toolchain/c/patches/git/0001-wasi-posix-spawn-run-command.patch new file mode 100644 index 0000000000..96a9f86e62 --- /dev/null +++ b/toolchain/c/patches/git/0001-wasi-posix-spawn-run-command.patch @@ -0,0 +1,137 @@ +Use posix_spawn for Git child processes on WASI. + +AgentOS provides POSIX process creation through the sysroot's posix_spawn +implementation, backed by the host_process broker. WASI cannot provide fork() +semantics, so route Git's central child-process path through posix_spawn while +leaving the rest of upstream Git's command machinery intact. + +--- a/run-command.c ++++ b/run-command.c +@@ -18,6 +18,10 @@ + #include "packfile.h" + #include "compat/nonblock.h" + ++#ifdef __wasi__ ++#include ++#endif ++ + void child_process_init(struct child_process *child) + { + struct child_process blank = CHILD_PROCESS_INIT; +@@ -747,12 +751,14 @@ + + #ifndef GIT_WINDOWS_NATIVE + { ++#ifndef __wasi__ + int notify_pipe[2]; ++ struct child_err cerr; ++ struct atfork_state as; ++#endif + int null_fd = -1; + char **childenv; + struct strvec argv = STRVEC_INIT; +- struct child_err cerr; +- struct atfork_state as; + + if (prepare_cmd(&argv, cmd) < 0) { + failed_errno = errno; +@@ -764,8 +770,10 @@ + + trace_argv_printf(&argv.v[1], "trace: start_command:"); + ++#ifndef __wasi__ + if (pipe(notify_pipe)) + notify_pipe[0] = notify_pipe[1] = -1; ++#endif + + if (cmd->no_stdin || cmd->no_stdout || cmd->no_stderr) { + null_fd = xopen("/dev/null", O_RDWR | O_CLOEXEC); +@@ -773,6 +781,79 @@ + } + + childenv = prep_childenv(cmd->env.v); ++#ifdef __wasi__ ++ { ++ posix_spawn_file_actions_t actions; ++ int spawn_err; ++ ++ spawn_err = posix_spawn_file_actions_init(&actions); ++ if (spawn_err) { ++ failed_errno = spawn_err; ++ cmd->pid = -1; ++ errno = spawn_err; ++ goto wasi_spawn_done; ++ } ++ ++ if (cmd->no_stdin) ++ spawn_err = posix_spawn_file_actions_adddup2(&actions, null_fd, 0); ++ else if (need_in) ++ spawn_err = posix_spawn_file_actions_adddup2(&actions, fdin[0], 0); ++ else if (cmd->in) ++ spawn_err = posix_spawn_file_actions_adddup2(&actions, cmd->in, 0); ++ if (spawn_err) ++ goto wasi_spawn_failed_action; ++ ++ if (cmd->no_stderr) ++ spawn_err = posix_spawn_file_actions_adddup2(&actions, null_fd, 2); ++ else if (need_err) ++ spawn_err = posix_spawn_file_actions_adddup2(&actions, fderr[1], 2); ++ else if (cmd->err > 1) ++ spawn_err = posix_spawn_file_actions_adddup2(&actions, cmd->err, 2); ++ if (spawn_err) ++ goto wasi_spawn_failed_action; ++ ++ if (cmd->no_stdout) ++ spawn_err = posix_spawn_file_actions_adddup2(&actions, null_fd, 1); ++ else if (cmd->stdout_to_stderr) ++ spawn_err = posix_spawn_file_actions_adddup2(&actions, 2, 1); ++ else if (need_out) ++ spawn_err = posix_spawn_file_actions_adddup2(&actions, fdout[1], 1); ++ else if (cmd->out > 1) ++ spawn_err = posix_spawn_file_actions_adddup2(&actions, cmd->out, 1); ++ if (spawn_err) ++ goto wasi_spawn_failed_action; ++ ++ if (cmd->dir) { ++ spawn_err = posix_spawn_file_actions_addchdir(&actions, cmd->dir); ++ if (spawn_err) ++ goto wasi_spawn_failed_action; ++ } ++ ++ spawn_err = posix_spawn(&cmd->pid, argv.v[1], &actions, NULL, ++ (char *const *)argv.v + 1, childenv); ++ if (spawn_err == ENOEXEC) ++ spawn_err = posix_spawn(&cmd->pid, argv.v[0], &actions, NULL, ++ (char *const *)argv.v, childenv); ++ ++wasi_spawn_failed_action: ++ posix_spawn_file_actions_destroy(&actions); ++ if (spawn_err) { ++ failed_errno = spawn_err; ++ cmd->pid = -1; ++ errno = spawn_err; ++ if (!cmd->silent_exec_failure || errno != ENOENT) ++ error_errno("cannot spawn %s", cmd->args.v[0]); ++ } else if (cmd->clean_on_exit) { ++ mark_child_for_cleanup(cmd->pid, cmd); ++ } ++ ++wasi_spawn_done: ++ if (null_fd >= 0) ++ close(null_fd); ++ strvec_clear(&argv); ++ free(childenv); ++ } ++#else + atfork_prepare(&as); + + /* +@@ -904,8 +985,9 @@ + if (null_fd >= 0) + close(null_fd); + strvec_clear(&argv); + free(childenv); ++#endif + } + end_of_spawn: + diff --git a/toolchain/c/scripts/build-git-upstream.sh b/toolchain/c/scripts/build-git-upstream.sh new file mode 100755 index 0000000000..f5268b6943 --- /dev/null +++ b/toolchain/c/scripts/build-git-upstream.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: build-git-upstream.sh \ + --version \ + --url \ + --cache-dir \ + --build-dir \ + --patch-dir \ + --zlib-dir \ + --zlib-build-dir \ + --cc \ + --ar \ + --ranlib \ + --sysroot \ + --output +EOF +} + +VERSION="" +URL="" +CACHE_DIR="" +BUILD_DIR="" +ZLIB_DIR="" +ZLIB_BUILD_DIR="" +PATCH_DIR="" +CC_CMD="" +AR_CMD="" +RANLIB_CMD="" +SYSROOT="" +OUTPUT="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --version) VERSION="$2"; shift 2 ;; + --url) URL="$2"; shift 2 ;; + --cache-dir) CACHE_DIR="$2"; shift 2 ;; + --build-dir) BUILD_DIR="$2"; shift 2 ;; + --patch-dir) PATCH_DIR="$2"; shift 2 ;; + --zlib-dir) ZLIB_DIR="$2"; shift 2 ;; + --zlib-build-dir) ZLIB_BUILD_DIR="$2"; shift 2 ;; + --cc) CC_CMD="$2"; shift 2 ;; + --ar) AR_CMD="$2"; shift 2 ;; + --ranlib) RANLIB_CMD="$2"; shift 2 ;; + --sysroot) SYSROOT="$2"; shift 2 ;; + --output) OUTPUT="$2"; shift 2 ;; + *) echo "Unknown argument: $1" >&2; usage >&2; exit 1 ;; + esac +done + +if [[ -z "$VERSION" || -z "$URL" || -z "$CACHE_DIR" || -z "$BUILD_DIR" || -z "$PATCH_DIR" || -z "$ZLIB_DIR" || -z "$ZLIB_BUILD_DIR" || -z "$CC_CMD" || -z "$AR_CMD" || -z "$RANLIB_CMD" || -z "$SYSROOT" || -z "$OUTPUT" ]]; then + usage >&2 + exit 1 +fi + +fetch() { + local url="$1" + local out="$2" + if command -v curl >/dev/null 2>&1; then + curl --retry 3 --retry-all-errors -fSL "$url" -o "$out" + elif command -v wget >/dev/null 2>&1; then + wget -q "$url" -O "$out" + else + echo "Neither curl nor wget is available to fetch $url" >&2 + exit 1 + fi +} + +mkdir -p "$CACHE_DIR" +rm -rf "$BUILD_DIR" +mkdir -p "$BUILD_DIR" + +TARBALL="$CACHE_DIR/git-${VERSION}.tar.xz" +if [[ ! -f "$TARBALL" ]]; then + echo "Fetching upstream Git ${VERSION} release tarball..." + fetch "$URL" "$TARBALL" +fi + +echo "Extracting upstream Git ${VERSION}..." +tar -xf "$TARBALL" -C "$BUILD_DIR" + +SRC_DIR="$BUILD_DIR/git-${VERSION}" +if [[ ! -d "$SRC_DIR" ]]; then + echo "Expected extracted source at $SRC_DIR" >&2 + exit 1 +fi + +pushd "$SRC_DIR" >/dev/null + +if [[ -d "$PATCH_DIR" ]]; then + for patch_file in "$PATCH_DIR"/*.patch; do + [[ -e "$patch_file" ]] || continue + echo "Applying $(basename "$patch_file")..." + patch -p1 < "$patch_file" + done +fi + +echo "Building upstream Git ${VERSION} for wasm32-wasip1..." +make -j"${MAKE_JOBS:-2}" \ + uname_S=WASI \ + CC="$CC_CMD" \ + HOSTCC="${HOSTCC:-cc}" \ + AR="$AR_CMD" \ + RANLIB="$RANLIB_CMD" \ + CFLAGS="--target=wasm32-wasip1 --sysroot=$SYSROOT -I$ZLIB_DIR -O2 -D_WASI_EMULATED_PROCESS_CLOCKS -D_WASI_EMULATED_MMAN" \ + LDFLAGS="--target=wasm32-wasip1 --sysroot=$SYSROOT -L$ZLIB_BUILD_DIR -lwasi-emulated-process-clocks -lwasi-emulated-mman" \ + CSPRNG_METHOD=getentropy \ + HAVE_PATHS_H=YesPlease \ + HAVE_DEV_TTY=YesPlease \ + HAVE_CLOCK_GETTIME=YesPlease \ + HAVE_CLOCK_MONOTONIC=YesPlease \ + HAVE_GETDELIM=YesPlease \ + NO_RUST=1 \ + NO_OPENSSL=1 \ + NO_CURL=1 \ + NO_EXPAT=1 \ + NO_GETTEXT=1 \ + NO_TCLTK=1 \ + NO_PERL=1 \ + NO_PYTHON=1 \ + NO_REGEX=NeedsStartEnd \ + NO_ICONV=1 \ + NO_PTHREADS=1 \ + NO_MMAP=1 \ + NO_IPV6=1 \ + NO_UNIX_SOCKETS=1 \ + NO_SYS_POLL_H=1 \ + NO_NSEC=1 \ + git + +mkdir -p "$(dirname "$OUTPUT")" +if command -v wasm-opt >/dev/null 2>&1; then + echo "Optimizing Git WASM binary..." + wasm-opt -O3 --strip-debug --all-features git -o "$OUTPUT" +else + cp git "$OUTPUT" +fi + +popd >/dev/null + +echo "Built upstream Git at $OUTPUT" diff --git a/toolchain/c/scripts/build-grep-upstream.sh b/toolchain/c/scripts/build-grep-upstream.sh new file mode 100755 index 0000000000..ea65005863 --- /dev/null +++ b/toolchain/c/scripts/build-grep-upstream.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: build-grep-upstream.sh \ + --version \ + --url \ + --cache-dir \ + --build-dir \ + --cc \ + --ar \ + --ranlib \ + --output +EOF +} + +VERSION="" +URL="" +CACHE_DIR="" +BUILD_DIR="" +CC_CMD="" +AR_CMD="" +RANLIB_CMD="" +OUTPUT="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --version) + VERSION="$2" + shift 2 + ;; + --url) + URL="$2" + shift 2 + ;; + --cache-dir) + CACHE_DIR="$2" + shift 2 + ;; + --build-dir) + BUILD_DIR="$2" + shift 2 + ;; + --cc) + CC_CMD="$2" + shift 2 + ;; + --ar) + AR_CMD="$2" + shift 2 + ;; + --ranlib) + RANLIB_CMD="$2" + shift 2 + ;; + --output) + OUTPUT="$2" + shift 2 + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +if [[ -z "$VERSION" || -z "$URL" || -z "$CACHE_DIR" || -z "$BUILD_DIR" || -z "$CC_CMD" || -z "$AR_CMD" || -z "$RANLIB_CMD" || -z "$OUTPUT" ]]; then + usage >&2 + exit 1 +fi + +fetch() { + local url="$1" + local out="$2" + if command -v curl >/dev/null 2>&1; then + curl --retry 3 --retry-all-errors -fSL "$url" -o "$out" + elif command -v wget >/dev/null 2>&1; then + wget -q "$url" -O "$out" + else + echo "Neither curl nor wget is available to fetch $url" >&2 + exit 1 + fi +} + +mkdir -p "$CACHE_DIR" +rm -rf "$BUILD_DIR" +mkdir -p "$BUILD_DIR" + +TARBALL="$CACHE_DIR/grep-${VERSION}.tar.xz" +if [[ ! -f "$TARBALL" ]]; then + echo "Fetching upstream GNU grep ${VERSION} release tarball..." + fetch "$URL" "$TARBALL" +fi + +echo "Extracting upstream GNU grep ${VERSION}..." +tar -xJf "$TARBALL" -C "$BUILD_DIR" + +SRC_DIR="$BUILD_DIR/grep-${VERSION}" +if [[ ! -d "$SRC_DIR" ]]; then + echo "Expected extracted source at $SRC_DIR" >&2 + exit 1 +fi + +pushd "$SRC_DIR" >/dev/null + +echo "Configuring upstream GNU grep for wasm32-wasip1..." +CC="$CC_CMD" \ +AR="$AR_CMD" \ +RANLIB="$RANLIB_CMD" \ +PKG_CONFIG=false \ +PCRE_CFLAGS="" \ +PCRE_LIBS="" \ +gl_cv_func_select_supports0=yes \ +gl_cv_func_select_detects_ebadf=yes \ +gl_cv_func_pselect_detects_ebadf=yes \ +ac_cv_func_clock_getres=no \ +ac_cv_func_clock_gettime=no \ +CFLAGS="-O2 -flto -D_WASI_EMULATED_SIGNAL -D_WASI_EMULATED_MMAN -D_WASI_EMULATED_PROCESS_CLOCKS -DFD_SETSIZE=8192" \ +LIBS="-lwasi-emulated-signal -lwasi-emulated-mman -lwasi-emulated-process-clocks" \ +./configure \ + --host=wasm32-unknown-wasi \ + --disable-shared \ + --disable-nls \ + --disable-perl-regexp \ + --disable-threads + +echo "Building upstream GNU grep support library..." +make -C lib + +echo "Building upstream GNU grep..." +make -C src grep + +BIN="" +for candidate in "src/grep" "src/.libs/grep" "src/grep.wasm"; do + if [[ -f "$candidate" ]]; then + BIN="$candidate" + break + fi +done + +if [[ -z "$BIN" ]]; then + echo "Unable to locate built grep binary in src/" >&2 + exit 1 +fi + +mkdir -p "$(dirname "$OUTPUT")" +if command -v wasm-opt >/dev/null 2>&1; then + echo "Optimizing GNU grep WASM binary..." + wasm-opt -O3 --strip-debug --all-features "$BIN" -o "$OUTPUT" +else + cp "$BIN" "$OUTPUT" +fi + +popd >/dev/null + +echo "Built upstream GNU grep at $OUTPUT" diff --git a/toolchain/c/scripts/build-infozip-upstream.sh b/toolchain/c/scripts/build-infozip-upstream.sh new file mode 100755 index 0000000000..1e765bb04f --- /dev/null +++ b/toolchain/c/scripts/build-infozip-upstream.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: build-infozip-upstream.sh \ + --tool \ + --version \ + --url \ + --cache-dir \ + --build-dir \ + --cc \ + --output +EOF +} + +TOOL="" +VERSION="" +URL="" +CACHE_DIR="" +BUILD_DIR="" +CC_CMD="" +OUTPUT="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --tool) + TOOL="$2" + shift 2 + ;; + --version) + VERSION="$2" + shift 2 + ;; + --url) + URL="$2" + shift 2 + ;; + --cache-dir) + CACHE_DIR="$2" + shift 2 + ;; + --build-dir) + BUILD_DIR="$2" + shift 2 + ;; + --cc) + CC_CMD="$2" + shift 2 + ;; + --output) + OUTPUT="$2" + shift 2 + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +if [[ -z "$TOOL" || -z "$VERSION" || -z "$URL" || -z "$CACHE_DIR" || -z "$BUILD_DIR" || -z "$CC_CMD" || -z "$OUTPUT" ]]; then + usage >&2 + exit 1 +fi + +case "$TOOL" in + zip|unzip) ;; + *) + echo "Unsupported Info-ZIP tool: $TOOL" >&2 + usage >&2 + exit 1 + ;; +esac + +fetch() { + local url="$1" + local out="$2" + if command -v curl >/dev/null 2>&1; then + curl --retry 3 --retry-all-errors -fSL "$url" -o "$out" + elif command -v wget >/dev/null 2>&1; then + wget -q "$url" -O "$out" + else + echo "Neither curl nor wget is available to fetch $url" >&2 + exit 1 + fi +} + +mkdir -p "$CACHE_DIR" +rm -rf "$BUILD_DIR" +mkdir -p "$BUILD_DIR" + +TARBALL="$CACHE_DIR/infozip-${TOOL}-${VERSION}.tar.gz" +if [[ ! -f "$TARBALL" ]]; then + echo "Fetching upstream Info-ZIP ${TOOL} ${VERSION} release tarball..." + fetch "$URL" "$TARBALL" +fi + +echo "Extracting upstream Info-ZIP ${TOOL} ${VERSION}..." +tar -xzf "$TARBALL" -C "$BUILD_DIR" + +SRC_DIR="$BUILD_DIR/${TOOL}${VERSION}" +if [[ ! -d "$SRC_DIR" ]]; then + echo "Expected extracted source at $SRC_DIR" >&2 + exit 1 +fi + +pushd "$SRC_DIR" >/dev/null + +case "$TOOL" in + zip) + echo "Building upstream Info-ZIP Zip ${VERSION} for wasm32-wasip1..." + make -f unix/Makefile clean + make -f unix/Makefile zips \ + CC="$CC_CMD" \ + BIND="$CC_CMD" \ + CFLAGS="-O2 -I. -DUNIX -DNO_ASM -DNO_BZIP2 -DNO_LCHMOD -DNO_LCHOWN -DLARGE_FILE_SUPPORT" \ + LFLAGS2="" + BIN="zip" + ;; + unzip) + echo "Building upstream Info-ZIP UnZip ${VERSION} for wasm32-wasip1..." + make -f unix/Makefile clean + make -f unix/Makefile unzips \ + CC="$CC_CMD" \ + LD="$CC_CMD" \ + CF="-O2 -I. -DUNIX -DBSD4_4 -DNO_PARAM_H -DNO_LCHMOD -DNO_LCHOWN -DNO_SYMLINK -DLARGE_FILE_SUPPORT" \ + LF2="" \ + SL2="" \ + FL2="" + BIN="unzip" + ;; +esac + +if [[ ! -f "$BIN" ]]; then + echo "Unable to locate built Info-ZIP $TOOL binary" >&2 + exit 1 +fi + +mkdir -p "$(dirname "$OUTPUT")" +if command -v wasm-opt >/dev/null 2>&1; then + echo "Optimizing Info-ZIP $TOOL WASM binary..." + wasm-opt -O3 --strip-debug --all-features "$BIN" -o "$OUTPUT" +else + cp "$BIN" "$OUTPUT" +fi + +popd >/dev/null + +echo "Built upstream Info-ZIP $TOOL at $OUTPUT" diff --git a/toolchain/c/scripts/build-wget-upstream.sh b/toolchain/c/scripts/build-wget-upstream.sh new file mode 100644 index 0000000000..312937df63 --- /dev/null +++ b/toolchain/c/scripts/build-wget-upstream.sh @@ -0,0 +1,184 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: build-wget-upstream.sh \ + --version \ + --url \ + --cache-dir \ + --build-dir \ + --overlay-include-dir \ + --cc \ + --ar \ + --ranlib \ + --output +EOF +} + +VERSION="" +URL="" +CACHE_DIR="" +BUILD_DIR="" +OVERLAY_INCLUDE_DIR="" +CC_CMD="" +AR_CMD="" +RANLIB_CMD="" +OUTPUT="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --version) + VERSION="$2" + shift 2 + ;; + --url) + URL="$2" + shift 2 + ;; + --cache-dir) + CACHE_DIR="$2" + shift 2 + ;; + --build-dir) + BUILD_DIR="$2" + shift 2 + ;; + --overlay-include-dir) + OVERLAY_INCLUDE_DIR="$2" + shift 2 + ;; + --cc) + CC_CMD="$2" + shift 2 + ;; + --ar) + AR_CMD="$2" + shift 2 + ;; + --ranlib) + RANLIB_CMD="$2" + shift 2 + ;; + --output) + OUTPUT="$2" + shift 2 + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +if [[ -z "$VERSION" || -z "$URL" || -z "$CACHE_DIR" || -z "$BUILD_DIR" || -z "$OVERLAY_INCLUDE_DIR" || -z "$CC_CMD" || -z "$AR_CMD" || -z "$RANLIB_CMD" || -z "$OUTPUT" ]]; then + usage >&2 + exit 1 +fi + +fetch() { + local url="$1" + local out="$2" + if command -v curl >/dev/null 2>&1; then + curl --retry 3 --retry-all-errors -fSL "$url" -o "$out" + elif command -v wget >/dev/null 2>&1; then + wget -q "$url" -O "$out" + else + echo "Neither curl nor wget is available to fetch $url" >&2 + exit 1 + fi +} + +mkdir -p "$CACHE_DIR" +rm -rf "$BUILD_DIR" +mkdir -p "$BUILD_DIR" + +TARBALL="$CACHE_DIR/wget-${VERSION}.tar.gz" +if [[ ! -f "$TARBALL" ]]; then + echo "Fetching upstream GNU Wget ${VERSION} release tarball..." + fetch "$URL" "$TARBALL" +fi + +echo "Extracting upstream GNU Wget ${VERSION}..." +tar -xzf "$TARBALL" -C "$BUILD_DIR" + +SRC_DIR="$BUILD_DIR/wget-${VERSION}" +if [[ ! -d "$SRC_DIR" ]]; then + echo "Expected extracted source at $SRC_DIR" >&2 + exit 1 +fi + +pushd "$SRC_DIR" >/dev/null + +echo "Configuring upstream GNU Wget for wasm32-wasip1..." +CC="$CC_CMD" \ +AR="$AR_CMD" \ +RANLIB="$RANLIB_CMD" \ +PKG_CONFIG=false \ +NETTLE_CFLAGS="" \ +NETTLE_LIBS="" \ +PCRE2_CFLAGS="" \ +PCRE2_LIBS="" \ +PCRE_CFLAGS="" \ +PCRE_LIBS="" \ +CPPFLAGS="-I$OVERLAY_INCLUDE_DIR" \ +gl_cv_func_posix_spawn_works=yes \ +gl_cv_func_posix_spawn_secure_exec=yes \ +gl_cv_func_posix_spawnp_secure_exec=yes \ +gl_cv_func_posix_spawn_file_actions_addclose_works=yes \ +gl_cv_func_posix_spawn_file_actions_adddup2_works=yes \ +gl_cv_func_posix_spawn_file_actions_addopen_works=yes \ +gl_cv_func_select_supports0=yes \ +gl_cv_func_select_detects_ebadf=yes \ +gl_cv_func_pselect_detects_ebadf=yes \ +ac_cv_func_clock_getres=no \ +ac_cv_func_clock_gettime=no \ +CFLAGS="-O2 -flto -D_WASI_EMULATED_SIGNAL -D_WASI_EMULATED_MMAN -D_WASI_EMULATED_PROCESS_CLOCKS -DFD_SETSIZE=8192" \ +LIBS="-lwasi-emulated-signal -lwasi-emulated-mman -lwasi-emulated-process-clocks" \ +./configure \ + --host=wasm32-unknown-wasi \ + --disable-shared \ + --disable-nls \ + --disable-iri \ + --disable-digest \ + --disable-ntlm \ + --disable-opie \ + --disable-pcre \ + --disable-pcre2 \ + --without-ssl \ + --without-libpsl \ + --without-zlib \ + --without-libuuid \ + --without-metalink + +echo "Building upstream GNU Wget support library..." +make -C lib + +echo "Building upstream GNU Wget..." +make -C src wget + +BIN="" +for candidate in "src/wget" "src/.libs/wget" "src/wget.wasm"; do + if [[ -f "$candidate" ]]; then + BIN="$candidate" + break + fi +done + +if [[ -z "$BIN" ]]; then + echo "Unable to locate built wget binary in src/" >&2 + exit 1 +fi + +mkdir -p "$(dirname "$OUTPUT")" +if command -v wasm-opt >/dev/null 2>&1; then + echo "Optimizing Wget WASM binary..." + wasm-opt -O3 --strip-debug --all-features "$BIN" -o "$OUTPUT" +else + cp "$BIN" "$OUTPUT" +fi + +popd >/dev/null + +echo "Built upstream GNU Wget at $OUTPUT" diff --git a/toolchain/codex-ref b/toolchain/codex-ref new file mode 100644 index 0000000000..6e55ba2e19 --- /dev/null +++ b/toolchain/codex-ref @@ -0,0 +1 @@ +rivet-dev/codex@9909c590c3173e5d821f7d0a82f964bf9c39df31 diff --git a/toolchain/conformance/c-parity.test.ts b/toolchain/conformance/c-parity.test.ts index a106b22530..17e3271bb3 100644 --- a/toolchain/conformance/c-parity.test.ts +++ b/toolchain/conformance/c-parity.test.ts @@ -942,7 +942,7 @@ describeIf(!skipReason(), 'C parity: native vs WASM', { timeout: 30_000 }, () => } }); - itIf(!netSkip, 'http_get: connect to HTTP server, receive response body', async () => { + itIf(!netSkip, 'curl: connect to HTTP server, receive response body', async () => { // Start a local HTTP server const server = createHttpServer((_req, res) => { res.writeHead(200, { 'Content-Type': 'text/plain' }); @@ -953,14 +953,11 @@ describeIf(!skipReason(), 'C parity: native vs WASM', { timeout: 30_000 }, () => try { await recreateKernel({ loopbackExemptPorts: [port] }); - const native = await runNative('http_get', [String(port)]); - const wasm = await kernel.exec(`http_get ${port}`); + const wasm = await kernel.exec(`curl -fsS http://127.0.0.1:${port}/`); - expect(wasm.exitCode).toBe(native.exitCode); expect(wasm.exitCode).toBe(0); - expect(wasm.stdout).toBe(native.stdout); - expect(normalizeStderr(wasm.stderr)).toBe(normalizeStderr(native.stderr)); - expect(wasm.stdout).toContain('body: hello from http'); + expect(wasm.stderr).toBe(''); + expect(wasm.stdout.trim()).toBe('hello from http'); } finally { server.close(); } diff --git a/toolchain/crates/libs/grep/Cargo.toml b/toolchain/crates/libs/grep/Cargo.toml deleted file mode 100644 index 3cf619a9c4..0000000000 --- a/toolchain/crates/libs/grep/Cargo.toml +++ /dev/null @@ -1,9 +0,0 @@ -[package] -name = "secureexec-grep" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "grep/egrep/fgrep/rg implementations for secure-exec standalone binaries" - -[dependencies] -regex = "1" diff --git a/toolchain/crates/libs/grep/src/lib.rs b/toolchain/crates/libs/grep/src/lib.rs deleted file mode 100644 index c42db0e8ac..0000000000 --- a/toolchain/crates/libs/grep/src/lib.rs +++ /dev/null @@ -1,623 +0,0 @@ -//! grep implementation using the regex crate (ripgrep's pure Rust regex engine). -//! -//! Supports grep, egrep (grep -E), and fgrep (grep -F) modes. -//! Dispatches on argv[0] basename for standalone binary usage. - -mod rg_cmd; - -use std::ffi::OsString; -use std::io::{self, BufRead, Read, Write}; -use std::path::Path; - -use regex::Regex; - -const MAX_INPUT_LINE_BYTES: usize = 16 * 1024 * 1024; -const MAX_PATTERN_BYTES: usize = 16 * 1024 * 1024; -const MAX_PATTERNS: usize = 100_000; - -/// Unified grep entry point. Dispatches on argv[0]: -/// - "egrep" -> Extended mode -/// - "fgrep" -> Fixed mode -/// - default -> Basic mode -pub fn main(args: Vec) -> i32 { - let mode = match args.first().and_then(|a| Path::new(a).file_name()) { - Some(name) if name == "egrep" => GrepMode::Extended, - Some(name) if name == "fgrep" => GrepMode::Fixed, - _ => GrepMode::Basic, - }; - run_grep(args, mode) -} - -/// Entry point for grep command (Basic mode). -pub fn grep(args: Vec) -> i32 { - run_grep(args, GrepMode::Basic) -} - -/// Entry point for egrep command (Extended regex). -pub fn egrep(args: Vec) -> i32 { - run_grep(args, GrepMode::Extended) -} - -/// Entry point for fgrep command (Fixed strings). -pub fn fgrep(args: Vec) -> i32 { - run_grep(args, GrepMode::Fixed) -} - -/// Entry point for rg command. -pub fn rg(args: Vec) -> i32 { - rg_cmd::rg(args) -} - -/// grep mode determines how patterns are interpreted. -#[derive(Clone, Copy, PartialEq)] -enum GrepMode { - /// Basic regular expressions (default grep) - Basic, - /// Extended regular expressions (egrep / grep -E) - Extended, - /// Fixed strings (fgrep / grep -F) - Fixed, -} - -struct GrepOptions { - mode: GrepMode, - ignore_case: bool, - invert_match: bool, - count_only: bool, - files_with_matches: bool, - files_without_matches: bool, - line_numbers: bool, - word_regexp: bool, - line_regexp: bool, - max_count: Option, - quiet: bool, - patterns: Vec, - pattern_bytes: usize, - files: Vec, -} - -impl GrepOptions { - fn new(mode: GrepMode) -> Self { - Self { - mode, - ignore_case: false, - invert_match: false, - count_only: false, - files_with_matches: false, - files_without_matches: false, - line_numbers: false, - word_regexp: false, - line_regexp: false, - max_count: None, - quiet: false, - patterns: Vec::new(), - pattern_bytes: 0, - files: Vec::new(), - } - } -} - -fn run_grep(args: Vec, default_mode: GrepMode) -> i32 { - let str_args: Vec = args - .iter() - .skip(1) // skip argv[0] - .map(|a| a.to_string_lossy().to_string()) - .collect(); - - let opts = match parse_args(&str_args, default_mode) { - Ok(opts) => opts, - Err(msg) => { - eprintln!("grep: {}", msg); - return 2; - } - }; - - if opts.patterns.is_empty() { - eprintln!("grep: no pattern specified"); - return 2; - } - - let regex = match build_regex(&opts) { - Ok(r) => r, - Err(msg) => { - eprintln!("grep: {}", msg); - return 2; - } - }; - - let multiple_files = opts.files.len() > 1; - let mut any_match = false; - let mut had_error = false; - - if opts.files.is_empty() { - // Read from stdin - let stdin = io::stdin(); - let reader = stdin.lock(); - match search_reader(reader, None, ®ex, &opts, multiple_files) { - Ok(found) => any_match |= found, - Err(e) => { - eprintln!("grep: {}", e); - had_error = true; - } - } - } else { - for file in &opts.files { - if file == "-" { - let stdin = io::stdin(); - let reader = stdin.lock(); - let label = if multiple_files { - Some("(standard input)") - } else { - None - }; - match search_reader(reader, label, ®ex, &opts, multiple_files) { - Ok(found) => any_match |= found, - Err(e) => { - eprintln!("grep: {}: {}", file, e); - had_error = true; - } - } - } else { - match std::fs::File::open(file) { - Ok(f) => { - let reader = io::BufReader::new(f); - let label = if multiple_files { - Some(file.as_str()) - } else { - None - }; - match search_reader(reader, label, ®ex, &opts, multiple_files) { - Ok(found) => any_match |= found, - Err(e) => { - eprintln!("grep: {}: {}", file, e); - had_error = true; - } - } - } - Err(e) => { - eprintln!("grep: {}: {}", file, e); - had_error = true; - } - } - } - } - } - - if had_error { - 2 - } else if any_match { - 0 - } else { - 1 - } -} - -fn parse_args(args: &[String], default_mode: GrepMode) -> Result { - let mut opts = GrepOptions::new(default_mode); - let mut i = 0; - let mut pattern_from_args = false; - - while i < args.len() { - let arg = &args[i]; - - if arg == "--" { - i += 1; - // Remaining args are files (or first is pattern if none yet) - break; - } - - if arg.starts_with('-') && arg.len() > 1 && !arg.starts_with("--") { - let chars: Vec = arg[1..].chars().collect(); - let mut j = 0; - while j < chars.len() { - match chars[j] { - 'E' => opts.mode = GrepMode::Extended, - 'F' => opts.mode = GrepMode::Fixed, - 'G' => opts.mode = GrepMode::Basic, - 'i' | 'y' => opts.ignore_case = true, - 'v' => opts.invert_match = true, - 'c' => opts.count_only = true, - 'l' => opts.files_with_matches = true, - 'L' => opts.files_without_matches = true, - 'n' => opts.line_numbers = true, - 'w' => opts.word_regexp = true, - 'x' => opts.line_regexp = true, - 'q' | 's' => opts.quiet = true, - 'h' => {} // suppress filename (handled by multiple_files logic) - 'H' => {} // force filename - 'e' => { - // -e PATTERN (rest of this flag group or next arg) - let rest: String = chars[j + 1..].iter().collect(); - if !rest.is_empty() { - push_pattern(&mut opts, rest)?; - pattern_from_args = true; - j = chars.len(); // consumed rest - continue; - } else { - i += 1; - if i >= args.len() { - return Err("option requires an argument -- 'e'".to_string()); - } - push_pattern(&mut opts, args[i].clone())?; - pattern_from_args = true; - j = chars.len(); - continue; - } - } - 'f' => { - i += 1; - if i >= args.len() { - return Err("option requires an argument -- 'f'".to_string()); - } - read_patterns_from_file(&mut opts, &args[i])?; - pattern_from_args = true; - j = chars.len(); - continue; - } - 'm' => { - i += 1; - if i >= args.len() { - return Err("option requires an argument -- 'm'".to_string()); - } - opts.max_count = Some( - args[i] - .parse() - .map_err(|_| format!("invalid max count '{}'", args[i]))?, - ); - j = chars.len(); - continue; - } - _ => { - return Err(format!("invalid option -- '{}'", chars[j])); - } - } - j += 1; - } - } else if arg.starts_with("--") { - match arg.as_str() { - "--extended-regexp" => opts.mode = GrepMode::Extended, - "--fixed-strings" => opts.mode = GrepMode::Fixed, - "--basic-regexp" => opts.mode = GrepMode::Basic, - "--ignore-case" => opts.ignore_case = true, - "--invert-match" => opts.invert_match = true, - "--count" => opts.count_only = true, - "--files-with-matches" => opts.files_with_matches = true, - "--files-without-match" => opts.files_without_matches = true, - "--line-number" => opts.line_numbers = true, - "--word-regexp" => opts.word_regexp = true, - "--line-regexp" => opts.line_regexp = true, - "--quiet" | "--silent" => opts.quiet = true, - _ if arg.starts_with("--regexp=") => { - push_pattern(&mut opts, arg[9..].to_string())?; - pattern_from_args = true; - } - _ if arg.starts_with("--max-count=") => { - opts.max_count = Some( - arg[12..] - .parse() - .map_err(|_| format!("invalid max count '{}'", &arg[12..]))?, - ); - } - _ => { - return Err(format!("unrecognized option '{}'", arg)); - } - } - } else { - // Positional argument: first is pattern (if no -e), rest are files - if !pattern_from_args && opts.patterns.is_empty() { - push_pattern(&mut opts, arg.clone())?; - pattern_from_args = true; - } else { - opts.files.push(arg.clone()); - } - } - i += 1; - } - - // Remaining args after -- - while i < args.len() { - if !pattern_from_args && opts.patterns.is_empty() { - push_pattern(&mut opts, args[i].clone())?; - pattern_from_args = true; - } else { - opts.files.push(args[i].clone()); - } - i += 1; - } - - Ok(opts) -} - -fn push_pattern(opts: &mut GrepOptions, pattern: String) -> Result<(), String> { - if opts.patterns.len() >= MAX_PATTERNS { - return Err("too many patterns".to_string()); - } - let next_bytes = opts - .pattern_bytes - .checked_add(pattern.len()) - .ok_or_else(|| "pattern data too large".to_string())?; - if next_bytes > MAX_PATTERN_BYTES { - return Err("pattern data exceeds size limit".to_string()); - } - opts.pattern_bytes = next_bytes; - opts.patterns.push(pattern); - Ok(()) -} - -fn read_patterns_from_file(opts: &mut GrepOptions, path: &str) -> Result<(), String> { - let metadata = std::fs::metadata(path).map_err(|e| format!("{}: {}", path, e))?; - if metadata.len() > MAX_PATTERN_BYTES as u64 { - return Err(format!("{}: pattern file exceeds size limit", path)); - } - let file = std::fs::File::open(path).map_err(|e| format!("{}: {}", path, e))?; - let limit = MAX_PATTERN_BYTES - .checked_add(1) - .ok_or_else(|| "pattern file size limit is too large".to_string())?; - let mut content = String::new(); - file.take(limit as u64) - .read_to_string(&mut content) - .map_err(|e| format!("{}: {}", path, e))?; - if content.len() > MAX_PATTERN_BYTES { - return Err(format!("{}: pattern file exceeds size limit", path)); - } - for line in content.lines() { - if !line.is_empty() { - push_pattern(opts, line.to_string())?; - } - } - Ok(()) -} - -/// Build a compiled regex from the grep options. -fn build_regex(opts: &GrepOptions) -> Result { - let pattern = if opts.patterns.len() == 1 { - build_single_pattern(&opts.patterns[0], opts) - } else { - // Multiple patterns: combine with alternation - let parts: Vec = opts - .patterns - .iter() - .map(|p| format!("(?:{})", build_single_pattern(p, opts))) - .collect(); - parts.join("|") - }; - - let mut builder = regex::RegexBuilder::new(&pattern); - builder.case_insensitive(opts.ignore_case); - - builder - .build() - .map_err(|e| format!("invalid pattern: {}", e)) -} - -/// Convert a single pattern string to a regex pattern based on mode. -fn build_single_pattern(pattern: &str, opts: &GrepOptions) -> String { - let base = match opts.mode { - GrepMode::Fixed => regex::escape(pattern), - GrepMode::Basic => convert_bre_to_ere(pattern), - GrepMode::Extended => pattern.to_string(), - }; - - let wrapped = if opts.word_regexp { - format!(r"\b(?:{})\b", base) - } else if opts.line_regexp { - format!("^(?:{})$", base) - } else { - base - }; - - wrapped -} - -/// Convert POSIX Basic Regular Expression to Extended (Rust regex syntax). -/// In BRE: \(, \), \{, \}, \+, \?, \| are special; unescaped versions are literal. -/// In ERE (and Rust regex): (, ), {, }, +, ?, | are special without backslash. -fn convert_bre_to_ere(bre: &str) -> String { - let mut result = String::with_capacity(bre.len()); - let chars: Vec = bre.chars().collect(); - let mut i = 0; - - while i < chars.len() { - if chars[i] == '\\' && i + 1 < chars.len() { - match chars[i + 1] { - '(' => { - result.push('('); - i += 2; - } - ')' => { - result.push(')'); - i += 2; - } - '{' => { - result.push('{'); - i += 2; - } - '}' => { - result.push('}'); - i += 2; - } - '+' => { - result.push('+'); - i += 2; - } - '?' => { - result.push('?'); - i += 2; - } - '|' => { - result.push('|'); - i += 2; - } - '1'..='9' => { - // Backreference - not supported in Rust regex, pass through - result.push('\\'); - result.push(chars[i + 1]); - i += 2; - } - _ => { - result.push('\\'); - result.push(chars[i + 1]); - i += 2; - } - } - } else { - match chars[i] { - // In BRE, unescaped (, ), {, }, +, ? are literal - '(' => { - result.push_str("\\("); - i += 1; - } - ')' => { - result.push_str("\\)"); - i += 1; - } - '{' => { - result.push_str("\\{"); - i += 1; - } - '}' => { - result.push_str("\\}"); - i += 1; - } - _ => { - result.push(chars[i]); - i += 1; - } - } - } - } - - result -} - -/// Search a reader for matching lines. Returns true if any match was found. -fn search_reader( - reader: R, - filename: Option<&str>, - regex: &Regex, - opts: &GrepOptions, - show_filename: bool, -) -> io::Result { - let mut buf_reader = io::BufReader::new(reader); - let mut match_count: usize = 0; - let mut line_num: usize = 0; - let stdout = io::stdout(); - let mut out = stdout.lock(); - let mut line_buf = Vec::new(); - - while let Some(line) = read_line_bounded(&mut buf_reader, &mut line_buf)? { - line_num += 1; - - let is_match = regex.is_match(&line); - let is_match = if opts.invert_match { - !is_match - } else { - is_match - }; - - if is_match { - match_count += 1; - - if opts.quiet { - return Ok(true); - } - - if opts.files_with_matches { - if let Some(name) = filename { - writeln!(out, "{}", name)?; - } else { - writeln!(out, "(standard input)")?; - } - out.flush()?; - return Ok(true); - } - - if !opts.count_only && !opts.files_without_matches { - let prefix = match (show_filename, filename, opts.line_numbers) { - (true, Some(name), true) => format!("{}:{}:", name, line_num), - (true, Some(name), false) => format!("{}:", name), - (_, _, true) => format!("{}:", line_num), - _ => String::new(), - }; - writeln!(out, "{}{}", prefix, line)?; - } - - if let Some(max) = opts.max_count { - if match_count >= max { - break; - } - } - } - } - - if opts.count_only && !opts.quiet { - if show_filename { - if let Some(name) = filename { - writeln!(out, "{}:{}", name, match_count)?; - } else { - writeln!(out, "{}", match_count)?; - } - } else { - writeln!(out, "{}", match_count)?; - } - } - - if opts.files_without_matches && match_count == 0 { - if let Some(name) = filename { - writeln!(out, "{}", name)?; - } else { - writeln!(out, "(standard input)")?; - } - } - - out.flush()?; - - Ok(match_count > 0) -} - -fn read_line_bounded( - reader: &mut R, - line_buf: &mut Vec, -) -> io::Result> { - line_buf.clear(); - - loop { - let available = reader.fill_buf()?; - if available.is_empty() { - if line_buf.is_empty() { - return Ok(None); - } - break; - } - - let newline = available.iter().position(|&b| b == b'\n'); - let take = newline.map_or(available.len(), |pos| pos + 1); - let next_len = line_buf - .len() - .checked_add(take) - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "input line too long"))?; - if next_len > MAX_INPUT_LINE_BYTES { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "input line exceeds size limit", - )); - } - - line_buf.extend_from_slice(&available[..take]); - reader.consume(take); - if newline.is_some() { - break; - } - } - - if line_buf.ends_with(b"\n") { - line_buf.pop(); - if line_buf.ends_with(b"\r") { - line_buf.pop(); - } - } - - String::from_utf8(line_buf.clone()) - .map(Some) - .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e)) -} diff --git a/toolchain/crates/libs/grep/src/rg_cmd.rs b/toolchain/crates/libs/grep/src/rg_cmd.rs deleted file mode 100644 index 9449f3c678..0000000000 --- a/toolchain/crates/libs/grep/src/rg_cmd.rs +++ /dev/null @@ -1,1142 +0,0 @@ -//! rg (ripgrep) implementation using the regex crate (ripgrep's pure Rust regex engine). -//! -//! Provides ripgrep-compatible search. Uses the same regex engine as ripgrep. -//! POSIX grep/egrep/fgrep remain in lib.rs for BRE/ERE/fixed string compatibility. - -use std::collections::{HashSet, VecDeque}; -use std::ffi::OsString; -use std::io::{self, BufRead, Read, Write}; -use std::path::{Path, PathBuf}; - -use regex::{Regex, RegexBuilder}; - -const MAX_CONTEXT_LINES: usize = 100_000; -const MAX_CONTEXT_BYTES: usize = 16 * 1024 * 1024; -const MAX_FILE_RESULTS: usize = 1_000_000; -const MAX_INPUT_LINE_BYTES: usize = 16 * 1024 * 1024; -const MAX_PATTERN_BYTES: usize = 16 * 1024 * 1024; -const MAX_PATTERNS: usize = 100_000; - -/// Entry point for rg command. -pub fn rg(args: Vec) -> i32 { - let str_args: Vec = args - .iter() - .skip(1) - .map(|a| a.to_string_lossy().to_string()) - .collect(); - - match run(&str_args) { - Ok(code) => code, - Err(msg) => { - eprintln!("rg: {}", msg); - 2 - } - } -} - -struct Options { - patterns: Vec, - paths: Vec, - files_mode: bool, - ignore_case: bool, - smart_case: bool, - invert_match: bool, - count_only: bool, - files_with_matches: bool, - files_without_matches: bool, - line_numbers: Option, - word_regexp: bool, - line_regexp: bool, - fixed_strings: bool, - max_count: Option, - quiet: bool, - only_matching: bool, - after_context: usize, - before_context: usize, - show_filename: Option, - hidden: bool, - max_depth: Option, - sort_modified: bool, - glob_patterns: Vec, - pattern_bytes: usize, - type_include: Vec, - type_exclude: Vec, -} - -impl Options { - fn new() -> Self { - Self { - patterns: Vec::new(), - paths: Vec::new(), - files_mode: false, - ignore_case: false, - smart_case: true, - invert_match: false, - count_only: false, - files_with_matches: false, - files_without_matches: false, - line_numbers: None, - word_regexp: false, - line_regexp: false, - fixed_strings: false, - max_count: None, - quiet: false, - only_matching: false, - after_context: 0, - before_context: 0, - show_filename: None, - hidden: false, - max_depth: None, - sort_modified: false, - glob_patterns: Vec::new(), - pattern_bytes: 0, - type_include: Vec::new(), - type_exclude: Vec::new(), - } - } - - fn show_line_numbers(&self) -> bool { - self.line_numbers.unwrap_or(true) - } - - fn resolve_show_filename(&self, multi: bool) -> bool { - self.show_filename.unwrap_or(multi) - } - - fn has_context(&self) -> bool { - self.before_context > 0 || self.after_context > 0 - } -} - -fn run(args: &[String]) -> Result { - if args.len() == 1 && (args[0] == "--version" || args[0] == "-V") { - let stdout = io::stdout(); - let mut out = stdout.lock(); - writeln!(out, "ripgrep 14.1.0 (secure-exec)").map_err(|e| e.to_string())?; - out.flush().map_err(|e| e.to_string())?; - return Ok(0); - } - - let opts = parse_args(args)?; - - if opts.files_mode { - let paths = if opts.paths.is_empty() { - vec![".".to_string()] - } else { - opts.paths.clone() - }; - - let files = collect_files_from_paths(&paths, &opts).map_err(|e| e.to_string())?; - let stdout = io::stdout(); - let mut out = stdout.lock(); - for path in files { - writeln!(out, "{}", path.to_string_lossy()).map_err(|e| e.to_string())?; - } - out.flush().map_err(|e| e.to_string())?; - return Ok(0); - } - - if opts.patterns.is_empty() { - return Err("no pattern provided".to_string()); - } - - let regex = build_regex(&opts)?; - - if opts.paths.is_empty() { - // No paths: read from stdin - let stdin = io::stdin(); - let stdout = io::stdout(); - let mut out = stdout.lock(); - let result = search_stream(stdin.lock(), ®ex, &opts, None, false, &mut out) - .map_err(|e| e.to_string())?; - if opts.quiet { - return Ok(if result.matches > 0 { 0 } else { 1 }); - } - print_file_result(None, &result, &opts, &mut out).map_err(|e| e.to_string())?; - out.flush().map_err(|e| e.to_string())?; - return Ok(if result.matches > 0 { 0 } else { 1 }); - } - - let files = collect_files_from_paths(&opts.paths, &opts).map_err(|e| e.to_string())?; - let multi = files.len() > 1; - let show_fn = opts.resolve_show_filename(multi); - let mut any_match = false; - let mut had_error = false; - let stdout = io::stdout(); - let mut out = stdout.lock(); - - for path in &files { - match std::fs::File::open(path) { - Ok(f) => { - let reader = io::BufReader::new(f); - let fname = if show_fn { - Some(path.to_string_lossy().to_string()) - } else { - None - }; - let result = - match search_stream(reader, ®ex, &opts, fname.as_deref(), show_fn, &mut out) - { - Ok(result) => result, - Err(e) => { - eprintln!("rg: {}: {}", path.display(), e); - had_error = true; - continue; - } - }; - if result.matches > 0 { - any_match = true; - } - if opts.quiet && any_match { - return Ok(0); - } - if !opts.quiet { - print_file_result(fname.as_deref(), &result, &opts, &mut out) - .map_err(|e| e.to_string())?; - } - } - Err(e) => { - eprintln!("rg: {}: {}", path.display(), e); - had_error = true; - } - } - } - out.flush().map_err(|e| e.to_string())?; - - if had_error { - Ok(2) - } else if any_match { - Ok(0) - } else { - Ok(1) - } -} - -// --- Argument parsing --- - -fn parse_args(args: &[String]) -> Result { - let mut opts = Options::new(); - let mut i = 0; - let mut explicit_pattern = false; - - while i < args.len() { - let arg = &args[i]; - - if arg == "--" { - i += 1; - break; - } - - // Long options - if arg.starts_with("--") { - match arg.as_str() { - "--ignore-case" => opts.ignore_case = true, - "--case-sensitive" => { - opts.ignore_case = false; - opts.smart_case = false; - } - "--smart-case" => opts.smart_case = true, - "--invert-match" => opts.invert_match = true, - "--count" => opts.count_only = true, - "--files" => opts.files_mode = true, - "--files-with-matches" => opts.files_with_matches = true, - "--files-without-match" => opts.files_without_matches = true, - "--line-number" => opts.line_numbers = Some(true), - "--no-line-number" => opts.line_numbers = Some(false), - "--word-regexp" => opts.word_regexp = true, - "--line-regexp" => opts.line_regexp = true, - "--fixed-strings" => opts.fixed_strings = true, - "--quiet" | "--silent" => opts.quiet = true, - "--only-matching" => opts.only_matching = true, - "--hidden" | "--no-ignore" => opts.hidden = true, - "--follow" | "--no-ignore-vcs" | "--no-ignore-parent" => {} - "--with-filename" => opts.show_filename = Some(true), - "--no-filename" => opts.show_filename = Some(false), - "--no-heading" | "--heading" => {} // no-op (we always use inline format) - "--color=auto" | "--color=always" | "--color=never" => {} // no-op in WASI - "--no-color" => {} - _ if arg.starts_with("--color=") => {} - _ if arg.starts_with("--max-depth=") => { - opts.max_depth = Some( - arg[12..] - .parse() - .map_err(|_| format!("invalid number: '{}'", &arg[12..]))?, - ); - } - _ if arg.starts_with("--sort=") => match &arg[7..] { - "modified" => opts.sort_modified = true, - value => return Err(format!("unsupported sort: '{}'", value)), - }, - _ if arg.starts_with("--threads=") => {} - _ if arg.starts_with("--regexp=") => { - push_pattern(&mut opts, arg[9..].to_string())?; - explicit_pattern = true; - } - _ if arg.starts_with("--max-count=") => { - opts.max_count = Some( - arg[12..] - .parse() - .map_err(|_| format!("invalid number: '{}'", &arg[12..]))?, - ); - } - _ if arg.starts_with("--after-context=") => { - opts.after_context = parse_context_count(&arg[16..])?; - } - _ if arg.starts_with("--before-context=") => { - opts.before_context = parse_context_count(&arg[17..])?; - } - _ if arg.starts_with("--context=") => { - let n = parse_context_count(&arg[10..])?; - opts.before_context = n; - opts.after_context = n; - } - _ if arg.starts_with("--glob=") => { - opts.glob_patterns.push(arg[7..].to_string()); - } - _ if arg.starts_with("--type=") => { - opts.type_include.push(arg[7..].to_string()); - } - _ if arg.starts_with("--type-not=") => { - opts.type_exclude.push(arg[11..].to_string()); - } - "--regexp" | "--max-count" | "--after-context" | "--before-context" - | "--context" | "--glob" | "--type" | "--type-not" | "--file" | "--color" - | "--max-depth" | "--threads" => { - i += 1; - if i >= args.len() { - return Err(format!("{} requires an argument", arg)); - } - match arg.as_str() { - "--regexp" => { - push_pattern(&mut opts, args[i].clone())?; - explicit_pattern = true; - } - "--max-count" => { - opts.max_count = Some( - args[i] - .parse() - .map_err(|_| format!("invalid number: '{}'", args[i]))?, - ); - } - "--after-context" => { - opts.after_context = parse_context_count(&args[i])?; - } - "--before-context" => { - opts.before_context = parse_context_count(&args[i])?; - } - "--context" => { - let n = parse_context_count(&args[i])?; - opts.before_context = n; - opts.after_context = n; - } - "--glob" => opts.glob_patterns.push(args[i].clone()), - "--type" => opts.type_include.push(args[i].clone()), - "--type-not" => opts.type_exclude.push(args[i].clone()), - "--file" => { - read_patterns_from_file(&mut opts, &args[i])?; - explicit_pattern = true; - } - "--color" => {} // no-op - "--max-depth" => { - opts.max_depth = Some( - args[i] - .parse() - .map_err(|_| format!("invalid number: '{}'", args[i]))?, - ); - } - "--threads" => {} // no-op - _ => unreachable!(), - } - } - _ => return Err(format!("unrecognized option '{}'", arg)), - } - i += 1; - continue; - } - - // Short options - if arg.starts_with('-') && arg.len() > 1 { - let chars: Vec = arg[1..].chars().collect(); - let mut j = 0; - while j < chars.len() { - match chars[j] { - 'i' => opts.ignore_case = true, - 's' => { - opts.ignore_case = false; - opts.smart_case = false; - } - 'S' => opts.smart_case = true, - 'v' => opts.invert_match = true, - 'c' => opts.count_only = true, - 'l' => opts.files_with_matches = true, - 'n' => opts.line_numbers = Some(true), - 'N' => opts.line_numbers = Some(false), - 'w' => opts.word_regexp = true, - 'x' => opts.line_regexp = true, - 'F' => opts.fixed_strings = true, - 'q' => opts.quiet = true, - 'o' => opts.only_matching = true, - 'H' => opts.show_filename = Some(true), - '.' => opts.hidden = true, - 'e' => { - let rest: String = chars[j + 1..].iter().collect(); - if !rest.is_empty() { - push_pattern(&mut opts, rest)?; - } else { - i += 1; - if i >= args.len() { - return Err("option requires an argument -- 'e'".to_string()); - } - push_pattern(&mut opts, args[i].clone())?; - } - explicit_pattern = true; - j = chars.len(); - continue; - } - 'f' => { - i += 1; - if i >= args.len() { - return Err("option requires an argument -- 'f'".to_string()); - } - read_patterns_from_file(&mut opts, &args[i])?; - explicit_pattern = true; - j = chars.len(); - continue; - } - 'm' => { - i += 1; - if i >= args.len() { - return Err("option requires an argument -- 'm'".to_string()); - } - opts.max_count = Some( - args[i] - .parse() - .map_err(|_| format!("invalid number: '{}'", args[i]))?, - ); - j = chars.len(); - continue; - } - 'j' => { - i += 1; - if i >= args.len() { - return Err("option requires an argument -- 'j'".to_string()); - } - j = chars.len(); - continue; - } - 'A' => { - i += 1; - if i >= args.len() { - return Err("option requires an argument -- 'A'".to_string()); - } - opts.after_context = parse_context_count(&args[i])?; - j = chars.len(); - continue; - } - 'B' => { - i += 1; - if i >= args.len() { - return Err("option requires an argument -- 'B'".to_string()); - } - opts.before_context = parse_context_count(&args[i])?; - j = chars.len(); - continue; - } - 'C' => { - i += 1; - if i >= args.len() { - return Err("option requires an argument -- 'C'".to_string()); - } - let n = parse_context_count(&args[i])?; - opts.before_context = n; - opts.after_context = n; - j = chars.len(); - continue; - } - 'g' => { - i += 1; - if i >= args.len() { - return Err("option requires an argument -- 'g'".to_string()); - } - opts.glob_patterns.push(args[i].clone()); - j = chars.len(); - continue; - } - 't' => { - i += 1; - if i >= args.len() { - return Err("option requires an argument -- 't'".to_string()); - } - opts.type_include.push(args[i].clone()); - j = chars.len(); - continue; - } - 'T' => { - i += 1; - if i >= args.len() { - return Err("option requires an argument -- 'T'".to_string()); - } - opts.type_exclude.push(args[i].clone()); - j = chars.len(); - continue; - } - _ => return Err(format!("invalid option -- '{}'", chars[j])), - } - j += 1; - } - i += 1; - continue; - } - - // Positional argument - if opts.files_mode { - opts.paths.push(arg.clone()); - } else if !explicit_pattern && opts.patterns.is_empty() { - push_pattern(&mut opts, arg.clone())?; - explicit_pattern = true; - } else { - opts.paths.push(arg.clone()); - } - i += 1; - } - - // Remaining args after -- - while i < args.len() { - if opts.files_mode { - opts.paths.push(args[i].clone()); - } else if !explicit_pattern && opts.patterns.is_empty() { - push_pattern(&mut opts, args[i].clone())?; - explicit_pattern = true; - } else { - opts.paths.push(args[i].clone()); - } - i += 1; - } - - Ok(opts) -} - -fn parse_context_count(value: &str) -> Result { - let count: usize = value - .parse() - .map_err(|_| format!("invalid number: '{}'", value))?; - if count > MAX_CONTEXT_LINES { - return Err(format!("context count '{}' exceeds size limit", value)); - } - Ok(count) -} - -fn push_pattern(opts: &mut Options, pattern: String) -> Result<(), String> { - if opts.patterns.len() >= MAX_PATTERNS { - return Err("too many patterns".to_string()); - } - let next_bytes = opts - .pattern_bytes - .checked_add(pattern.len()) - .ok_or_else(|| "pattern data too large".to_string())?; - if next_bytes > MAX_PATTERN_BYTES { - return Err("pattern data exceeds size limit".to_string()); - } - opts.pattern_bytes = next_bytes; - opts.patterns.push(pattern); - Ok(()) -} - -fn read_patterns_from_file(opts: &mut Options, path: &str) -> Result<(), String> { - let metadata = std::fs::metadata(path).map_err(|e| format!("{}: {}", path, e))?; - if metadata.len() > MAX_PATTERN_BYTES as u64 { - return Err(format!("{}: pattern file exceeds size limit", path)); - } - let file = std::fs::File::open(path).map_err(|e| format!("{}: {}", path, e))?; - let limit = MAX_PATTERN_BYTES - .checked_add(1) - .ok_or_else(|| "pattern file size limit is too large".to_string())?; - let mut content = String::new(); - file.take(limit as u64) - .read_to_string(&mut content) - .map_err(|e| format!("{}: {}", path, e))?; - if content.len() > MAX_PATTERN_BYTES { - return Err(format!("{}: pattern file exceeds size limit", path)); - } - for line in content.lines() { - if !line.is_empty() { - push_pattern(opts, line.to_string())?; - } - } - Ok(()) -} - -// --- Pattern building --- - -fn build_regex(opts: &Options) -> Result { - let combined = if opts.patterns.len() == 1 { - prepare_pattern(&opts.patterns[0], opts) - } else { - let parts: Vec = opts - .patterns - .iter() - .map(|p| format!("(?:{})", prepare_pattern(p, opts))) - .collect(); - parts.join("|") - }; - - let case_insensitive = if opts.ignore_case { - true - } else if opts.smart_case { - // Smart case: insensitive unless pattern has uppercase - !combined.chars().any(|c| c.is_uppercase()) - } else { - false - }; - - RegexBuilder::new(&combined) - .case_insensitive(case_insensitive) - .build() - .map_err(|e| format!("regex error: {}", e)) -} - -fn prepare_pattern(pattern: &str, opts: &Options) -> String { - let base = if opts.fixed_strings { - regex::escape(pattern) - } else { - pattern.to_string() - }; - - if opts.word_regexp { - format!(r"\b(?:{})\b", base) - } else if opts.line_regexp { - format!("^(?:{})$", base) - } else { - base - } -} - -// --- File collection --- - -fn collect_files_from_paths(paths: &[String], opts: &Options) -> io::Result> { - let mut files = Vec::new(); - for path_str in paths { - let path = Path::new(path_str); - let metadata = std::fs::symlink_metadata(path)?; - let file_type = metadata.file_type(); - if file_type.is_dir() { - let mut active_dirs = HashSet::new(); - walk_dir(path, path, opts, &mut files, 0, &mut active_dirs)?; - } else if file_type.is_file() { - if should_include(path, path, false, opts) { - push_collected_file(&mut files, path.to_path_buf())?; - } - } else { - continue; - } - } - if opts.sort_modified { - files.sort_by_key(|path| { - std::fs::metadata(path) - .and_then(|meta| meta.modified()) - .ok() - }); - } else { - files.sort(); - } - Ok(files) -} - -fn push_collected_file(out: &mut Vec, path: PathBuf) -> io::Result<()> { - if out.len() >= MAX_FILE_RESULTS { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "file result count exceeds size limit", - )); - } - out.push(path); - Ok(()) -} - -fn walk_dir( - root: &Path, - dir: &Path, - opts: &Options, - out: &mut Vec, - depth: usize, - active_dirs: &mut HashSet, -) -> io::Result<()> { - let canonical = std::fs::canonicalize(dir)?; - if !active_dirs.insert(canonical.clone()) { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("recursive directory cycle at {}", dir.display()), - )); - } - - let result = (|| { - for entry in std::fs::read_dir(dir)? { - let entry = entry?; - let path = entry.path(); - let name = entry.file_name(); - let name_str = name.to_string_lossy(); - let file_type = entry.file_type()?; - let is_dir = file_type.is_dir(); - - // Skip hidden files/dirs unless --hidden - if !opts.hidden && name_str.starts_with('.') { - continue; - } - - if !should_include(root, &path, is_dir, opts) { - continue; - } - - if is_dir { - if opts.max_depth.map(|max| depth < max).unwrap_or(true) { - walk_dir(root, &path, opts, out, depth + 1, active_dirs)?; - } - } else if file_type.is_file() { - push_collected_file(out, path)?; - } - } - Ok(()) - })(); - - active_dirs.remove(&canonical); - result -} - -fn should_include(root: &Path, path: &Path, is_dir: bool, opts: &Options) -> bool { - let ext = path - .extension() - .map(|e| e.to_string_lossy().to_string()) - .unwrap_or_default(); - let relative_path = path - .strip_prefix(root) - .unwrap_or(path) - .to_string_lossy() - .replace('\\', "/"); - let file_name = path - .file_name() - .map(|n| n.to_string_lossy().to_string()) - .unwrap_or_default(); - - // Type include filters - if !opts.type_include.is_empty() { - let included = opts.type_include.iter().any(|t| { - type_extensions(t) - .map(|exts| exts.iter().any(|e| ext == *e)) - .unwrap_or(false) - }); - if !included { - return false; - } - } - - // Type exclude filters - if !opts.type_exclude.is_empty() { - let excluded = opts.type_exclude.iter().any(|t| { - type_extensions(t) - .map(|exts| exts.iter().any(|e| ext == *e)) - .unwrap_or(false) - }); - if excluded { - return false; - } - } - - // Glob filters - if !opts.glob_patterns.is_empty() { - for pattern in &opts.glob_patterns { - let (negated, pat) = if let Some(rest) = pattern.strip_prefix('!') { - (true, rest) - } else { - (false, pattern.as_str()) - }; - let matches = glob_matches(pat, &relative_path, &file_name, is_dir); - if negated && matches { - return false; - } - if !negated && !matches { - return false; - } - } - } - - true -} - -fn type_extensions(type_name: &str) -> Option<&'static [&'static str]> { - match type_name { - "rust" | "rs" => Some(&["rs"]), - "py" | "python" => Some(&["py", "pyi"]), - "js" | "javascript" => Some(&["js", "jsx", "mjs"]), - "ts" | "typescript" => Some(&["ts", "tsx", "mts"]), - "c" => Some(&["c", "h"]), - "cpp" | "c++" => Some(&["cpp", "cxx", "cc", "hpp", "hxx", "h"]), - "java" => Some(&["java"]), - "go" => Some(&["go"]), - "html" => Some(&["html", "htm"]), - "css" => Some(&["css"]), - "json" => Some(&["json"]), - "yaml" | "yml" => Some(&["yml", "yaml"]), - "toml" => Some(&["toml"]), - "md" | "markdown" => Some(&["md", "markdown"]), - "txt" | "text" => Some(&["txt"]), - "sh" | "shell" | "bash" => Some(&["sh", "bash"]), - "xml" => Some(&["xml"]), - "sql" => Some(&["sql"]), - "lua" => Some(&["lua"]), - "ruby" | "rb" => Some(&["rb"]), - "php" => Some(&["php"]), - "swift" => Some(&["swift"]), - "kotlin" | "kt" => Some(&["kt", "kts"]), - _ => None, - } -} - -fn glob_matches(pattern: &str, relative_path: &str, file_name: &str, is_dir: bool) -> bool { - let normalized = pattern.trim_end_matches('/'); - if pattern.ends_with('/') { - return is_dir - && relative_path - .split('/') - .any(|segment| segment == normalized || segment == file_name); - } - - let target = if pattern.contains('/') { - relative_path - } else { - file_name - }; - - let mut regex_pattern = String::from("^"); - let chars: Vec = normalized.chars().collect(); - let mut i = 0; - while i < chars.len() { - match chars[i] { - '*' => { - if i + 1 < chars.len() && chars[i + 1] == '*' { - regex_pattern.push_str(".*"); - i += 2; - } else { - regex_pattern.push_str("[^/]*"); - i += 1; - } - } - '?' => { - regex_pattern.push_str("[^/]"); - i += 1; - } - '{' => { - if let Some(end) = chars[i + 1..].iter().position(|c| *c == '}') { - let group: String = chars[i + 1..i + 1 + end].iter().collect(); - regex_pattern.push('('); - regex_pattern.push_str( - &group - .split(',') - .map(regex::escape) - .collect::>() - .join("|"), - ); - regex_pattern.push(')'); - i += end + 2; - } else { - regex_pattern.push_str("\\{"); - i += 1; - } - } - '.' | '+' | '(' | ')' | '|' | '^' | '$' | '[' | ']' | '\\' => { - regex_pattern.push('\\'); - regex_pattern.push(chars[i]); - i += 1; - } - other => { - regex_pattern.push(other); - i += 1; - } - } - } - regex_pattern.push('$'); - - Regex::new(®ex_pattern) - .map(|regex| regex.is_match(target)) - .unwrap_or(false) -} - -// --- Search --- - -struct FileResult { - matches: usize, - is_binary: bool, -} - -enum ResultLine { - Match(usize, String), - Context(usize, String), - Separator, -} - -fn search_stream( - mut reader: R, - regex: &Regex, - opts: &Options, - filename: Option<&str>, - show_filename: bool, - out: &mut W, -) -> io::Result { - let mut result = FileResult { - matches: 0, - is_binary: false, - }; - - let collect_lines = - !opts.quiet && !opts.files_with_matches && !opts.files_without_matches && !opts.count_only; - - let mut before_buf: VecDeque<(usize, String)> = VecDeque::new(); - let mut before_buf_bytes: usize = 0; - let mut after_remaining: usize = 0; - let mut last_printed: usize = 0; - let mut line_buf = Vec::new(); - let mut lineno: usize = 0; - - while let Some(line) = read_line_bounded(&mut reader, &mut line_buf)? { - lineno = lineno - .checked_add(1) - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "line number overflow"))?; - - // Binary detection: null bytes in line data - if line.as_bytes().contains(&0) { - result.is_binary = true; - break; - } - - let is_match = regex.is_match(&line) != opts.invert_match; - - if is_match { - result.matches += 1; - - if opts.quiet || opts.files_with_matches { - break; - } - - if collect_lines { - // Separator for non-contiguous match groups - if opts.has_context() && last_printed > 0 { - let first_before = before_buf.front().map(|(n, _)| *n).unwrap_or(lineno); - if first_before > last_printed + 1 { - print_result_line( - out, - filename, - opts, - show_filename, - &ResultLine::Separator, - )?; - } - } - - // Flush before-context buffer - for (bno, btext) in before_buf.drain(..) { - if bno > last_printed { - print_result_line( - out, - filename, - opts, - show_filename, - &ResultLine::Context(bno, btext), - )?; - last_printed = bno; - } - } - before_buf_bytes = 0; - - // Emit match - if opts.only_matching && !opts.invert_match { - for mat in regex.find_iter(&line) { - print_result_line( - out, - filename, - opts, - show_filename, - &ResultLine::Match(lineno, mat.as_str().to_string()), - )?; - } - } else { - print_result_line( - out, - filename, - opts, - show_filename, - &ResultLine::Match(lineno, line), - )?; - } - last_printed = lineno; - after_remaining = opts.after_context; - } - - if let Some(max) = opts.max_count { - if result.matches >= max { - break; - } - } - } else if collect_lines { - if after_remaining > 0 { - print_result_line( - out, - filename, - opts, - show_filename, - &ResultLine::Context(lineno, line), - )?; - last_printed = lineno; - after_remaining -= 1; - } else if opts.before_context > 0 { - before_buf_bytes = before_buf_bytes.checked_add(line.len()).ok_or_else(|| { - io::Error::new(io::ErrorKind::InvalidData, "context buffer too large") - })?; - if before_buf_bytes > MAX_CONTEXT_BYTES { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "context buffer exceeds size limit", - )); - } - before_buf.push_back((lineno, line)); - if before_buf.len() > opts.before_context { - if let Some((_, removed)) = before_buf.pop_front() { - before_buf_bytes = before_buf_bytes.saturating_sub(removed.len()); - } - } - } - } - } - - Ok(result) -} - -// --- Output --- - -fn print_file_result( - filename: Option<&str>, - result: &FileResult, - opts: &Options, - out: &mut W, -) -> io::Result<()> { - if result.is_binary { - if result.matches > 0 { - if let Some(name) = filename { - writeln!(out, "Binary file {} matches.", name)?; - } - } - return Ok(()); - } - - if opts.files_with_matches { - if result.matches > 0 { - let name = filename.unwrap_or("(standard input)"); - writeln!(out, "{}", name)?; - } - return Ok(()); - } - - if opts.files_without_matches { - if result.matches == 0 { - let name = filename.unwrap_or("(standard input)"); - writeln!(out, "{}", name)?; - } - return Ok(()); - } - - if opts.count_only { - if let Some(name) = filename { - writeln!(out, "{}:{}", name, result.matches)?; - } else { - writeln!(out, "{}", result.matches)?; - } - return Ok(()); - } - - Ok(()) -} - -fn print_result_line( - out: &mut W, - filename: Option<&str>, - opts: &Options, - show_filename: bool, - line: &ResultLine, -) -> io::Result<()> { - match line { - ResultLine::Match(lineno, text) => { - let mut prefix = String::new(); - if show_filename { - if let Some(name) = filename { - prefix.push_str(name); - prefix.push(':'); - } - } - if opts.show_line_numbers() { - prefix.push_str(&lineno.to_string()); - prefix.push(':'); - } - writeln!(out, "{}{}", prefix, text) - } - ResultLine::Context(lineno, text) => { - let mut prefix = String::new(); - if show_filename { - if let Some(name) = filename { - prefix.push_str(name); - prefix.push('-'); - } - } - if opts.show_line_numbers() { - prefix.push_str(&lineno.to_string()); - prefix.push('-'); - } - writeln!(out, "{}{}", prefix, text) - } - ResultLine::Separator => writeln!(out, "--"), - } -} - -fn read_line_bounded( - reader: &mut R, - line_buf: &mut Vec, -) -> io::Result> { - line_buf.clear(); - - loop { - let available = reader.fill_buf()?; - if available.is_empty() { - if line_buf.is_empty() { - return Ok(None); - } - break; - } - - let newline = available.iter().position(|&b| b == b'\n'); - let take = newline.map_or(available.len(), |pos| pos + 1); - let next_len = line_buf - .len() - .checked_add(take) - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "input line too long"))?; - if next_len > MAX_INPUT_LINE_BYTES { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "input line exceeds size limit", - )); - } - - line_buf.extend_from_slice(&available[..take]); - reader.consume(take); - if newline.is_some() { - break; - } - } - - if line_buf.ends_with(b"\n") { - line_buf.pop(); - if line_buf.ends_with(b"\r") { - line_buf.pop(); - } - } - - String::from_utf8(line_buf.clone()) - .map(Some) - .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e)) -} diff --git a/toolchain/crates/libs/shims/src/grep_alias.rs b/toolchain/crates/libs/shims/src/grep_alias.rs new file mode 100644 index 0000000000..bf2c83cc4b --- /dev/null +++ b/toolchain/crates/libs/shims/src/grep_alias.rs @@ -0,0 +1,36 @@ +//! GNU grep obsolescent alias launchers. +//! +//! Upstream GNU grep installs `egrep` and `fgrep` as scripts that warn and exec +//! `grep -E` / `grep -F`. Registry commands are WASM modules, so these compiled +//! launchers preserve the upstream script behavior through the process broker. + +use std::ffi::OsString; +use std::process::Stdio; + +pub fn egrep(args: Vec) -> i32 { + run_alias(args, "egrep", "-E") +} + +pub fn fgrep(args: Vec) -> i32 { + run_alias(args, "fgrep", "-F") +} + +fn run_alias(args: Vec, name: &str, option: &str) -> i32 { + eprintln!("{name}: warning: {name} is obsolescent; using grep {option}"); + + let mut command = std::process::Command::new("grep"); + command + .arg(option) + .args(args.into_iter().skip(1)) + .stdin(Stdio::inherit()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()); + + match command.status() { + Ok(status) => status.code().unwrap_or(1), + Err(error) => { + eprintln!("{name}: failed to run grep: {error}"); + 127 + } + } +} diff --git a/toolchain/crates/libs/shims/src/lib.rs b/toolchain/crates/libs/shims/src/lib.rs index 9e2358b3e8..7bbc25a050 100644 --- a/toolchain/crates/libs/shims/src/lib.rs +++ b/toolchain/crates/libs/shims/src/lib.rs @@ -5,6 +5,7 @@ //! rather than using uutils versions. pub mod env; +pub mod grep_alias; pub mod nice; pub mod nohup; pub mod stdbuf; diff --git a/toolchain/scripts/build-codex-wasi.sh b/toolchain/scripts/build-codex-wasi.sh new file mode 100755 index 0000000000..2c45a57579 --- /dev/null +++ b/toolchain/scripts/build-codex-wasi.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'USAGE' +Usage: + CODEX_REPO=/path/to/codex-rs/codex-rs toolchain/scripts/build-codex-wasi.sh + +Builds the external Codex WASI fork and installs the real codex-exec artifact +into software/codex/wasm/{codex-exec,codex}, where @agentos-software/codex-cli +stages commands from. + +Env: + CODEX_REPO Required. Checkout of the Codex fork with scripts/build-wasi-codex-exec.sh. + WASI_SDK_DIR Optional. Defaults to toolchain/c/vendor/wasi-sdk. + DEST_DIR Optional. Defaults to software/codex/wasm. +USAGE +} + +if [ "${1:-}" = "--help" ]; then + usage + exit 0 +fi + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TOOLCHAIN_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +AGENTOS_ROOT="$(cd "$TOOLCHAIN_DIR/.." && pwd)" + +CODEX_REPO="${CODEX_REPO:-}" +WASI_SDK_DIR="${WASI_SDK_DIR:-$TOOLCHAIN_DIR/c/vendor/wasi-sdk}" +DEST_DIR="${DEST_DIR:-$AGENTOS_ROOT/software/codex/wasm}" + +if [ -z "$CODEX_REPO" ]; then + echo "ERROR: CODEX_REPO is required." >&2 + echo " Set it to the wasi-port-codex-core checkout, for example:" >&2 + echo " CODEX_REPO=/path/to/codex-rs/codex-rs make -C toolchain codex-required" >&2 + exit 1 +fi + +BUILD_SCRIPT="$CODEX_REPO/scripts/build-wasi-codex-exec.sh" +if [ ! -x "$BUILD_SCRIPT" ]; then + echo "ERROR: codex fork build script not found or not executable: $BUILD_SCRIPT" >&2 + exit 1 +fi + +if [ ! -x "$WASI_SDK_DIR/bin/clang" ]; then + echo "ERROR: wasi-sdk clang not found at $WASI_SDK_DIR/bin/clang" >&2 + echo " Run: make -C toolchain/c wasi-sdk" >&2 + exit 1 +fi + +echo "== codex repo: $CODEX_REPO ==" +echo "== wasi-sdk: $WASI_SDK_DIR ==" +echo "== dest: $DEST_DIR ==" + +WASI_SDK_DIR="$(cd "$WASI_SDK_DIR" && pwd)" INSTALL=0 "$BUILD_SCRIPT" + +OUT="$CODEX_REPO/target/wasm32-wasip1/release/codex-exec.opt.wasm" +if [ ! -f "$OUT" ]; then + echo "ERROR: expected build output missing: $OUT" >&2 + exit 1 +fi + +mkdir -p "$DEST_DIR" +cp "$OUT" "$DEST_DIR/codex-exec" +cp "$OUT" "$DEST_DIR/codex" + +echo "== installed $(wc -c < "$OUT") bytes to $DEST_DIR/{codex-exec,codex} ==" diff --git a/toolchain/scripts/clone-and-build-codex-wasi.sh b/toolchain/scripts/clone-and-build-codex-wasi.sh new file mode 100755 index 0000000000..fd9993a4fb --- /dev/null +++ b/toolchain/scripts/clone-and-build-codex-wasi.sh @@ -0,0 +1,240 @@ +#!/usr/bin/env bash +# +# clone-and-build-codex-wasi.sh — REPRODUCIBLE wasm32-wasip1 build of the codex-exec +# `--session-turn` engine from the pinned codex WASI fork. +# +# Unlike scripts/build-codex-wasi.sh (which builds an EXISTING local checkout and +# relies on that machine's [patch.crates-io] paths + a pre-patched crates.io cache), +# this script is self-contained and reproducible from a clean environment: +# +# 1. Read the pin from toolchain/codex-ref ("/@"). +# 2. Shallow-clone the fork at that exact SHA into a gitignored scratch dir. +# 3. Rewrite the clone's [patch.crates-io] to point at the toolchain's reproducible +# stubs (toolchain/stubs/{reqwest-shim,portable-pty-wasi,ctrlc}); drop tokio's +# machine-path patch so tokio 1.52.3 comes through the vendored+patched tree. +# 4. Strip the hardcoded `-L .../self-contained` from the clone's .cargo/config.toml +# (that absolute rustup path is machine-specific; it is recomputed and passed via +# RUSTFLAGS at build time instead). +# 5. `cargo vendor` the workspace (+ the std library deps needed by -Z build-std) and +# apply toolchain/std-patches/crates/* to the vendored sources (tokio wasi-process, +# path-dedot, rustls-native-certs, socket2, ...) via scripts/patch-vendor.sh. +# 6. Build codex-exec for wasm32-wasip1 by reusing the fork's own +# scripts/build-wasi-codex-exec.sh (sysroot massaging + build-std + wasm-opt). +# 7. Install the optimized artifact to software/codex/wasm/{codex,codex-exec}. +# +# Usage: +# toolchain/scripts/clone-and-build-codex-wasi.sh +# +# Env (all optional; sensible defaults): +# CODEX_BUILD_DIR scratch root for the clone/vendor/target (default: toolchain/.codex-build) +# WASI_SDK_DIR wasi-sdk C toolchain (default: toolchain/c/vendor/wasi-sdk) +# DEST_DIR install destination (default: software/codex/wasm) +# TOOLCHAIN rust toolchain (default: nightly-2026-03-01) +# CODEX_GIT_BASE git host base (default: https://github.com) +# STOP_AFTER "vendor" -> stop right after vendor+patch (bounded verification: +# proves clone + patch-injection + vendor/patch succeed without the +# full ~29MB build). Unset -> full build + install. +# KEEP_SYSROOT forwarded to the fork build script (1 = skip sysroot restore). +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TOOLCHAIN_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +AGENTOS_ROOT="$(cd "$TOOLCHAIN_DIR/.." && pwd)" + +STUBS="$TOOLCHAIN_DIR/stubs" +CODEX_REF_FILE="$TOOLCHAIN_DIR/codex-ref" + +BUILD_ROOT="${CODEX_BUILD_DIR:-$TOOLCHAIN_DIR/.codex-build}" +CHECKOUT="$BUILD_ROOT/checkout" # git repo root (rivet-dev/codex) +WORKSPACE="$CHECKOUT/codex-rs" # cargo workspace (the fork nests it here) +WASI_SDK_DIR="${WASI_SDK_DIR:-$TOOLCHAIN_DIR/c/vendor/wasi-sdk}" +DEST_DIR="${DEST_DIR:-$AGENTOS_ROOT/software/codex/wasm}" +TOOLCHAIN="${TOOLCHAIN:-nightly-2026-03-01}" +CODEX_GIT_BASE="${CODEX_GIT_BASE:-https://github.com}" +STOP_AFTER="${STOP_AFTER:-}" + +# --- preflight --------------------------------------------------------------- +[ -f "$CODEX_REF_FILE" ] || { echo "ERROR: pin file not found: $CODEX_REF_FILE" >&2; exit 1; } +[ -x "$WASI_SDK_DIR/bin/clang" ] || { + echo "ERROR: wasi-sdk clang not found at $WASI_SDK_DIR/bin/clang" >&2 + echo " Run: make -C toolchain/c wasi-sdk" >&2 + exit 1 +} +command -v cargo >/dev/null 2>&1 || { echo "ERROR: cargo not on PATH" >&2; exit 1; } + +# --- 1. parse the pin -------------------------------------------------------- +REF="$(tr -d '[:space:]' < "$CODEX_REF_FILE")" +REPO="${REF%@*}" # owner/repo +SHA="${REF#*@}" # commit sha +[ -n "$REPO" ] && [ -n "$SHA" ] && [ "$REPO" != "$SHA" ] || { + echo "ERROR: malformed codex-ref '$REF' (expected '/@')" >&2; exit 1; } +URL="$CODEX_GIT_BASE/$REPO.git" + +echo "== codex-ref: $REPO @ $SHA ==" +echo "== clone url: $URL ==" +echo "== scratch: $BUILD_ROOT ==" + +# --- 2. shallow clone at the exact SHA (idempotent by SHA) ------------------- +STAMP="$CHECKOUT/.codex-built-sha" +if [ ! -f "$STAMP" ] || [ "$(cat "$STAMP" 2>/dev/null)" != "$SHA" ]; then + echo "== fetching fork at $SHA ==" + rm -rf "$CHECKOUT" + mkdir -p "$CHECKOUT" + git -C "$CHECKOUT" init -q + git -C "$CHECKOUT" remote add origin "$URL" + # Prefer a shallow fetch of the exact commit (GitHub allows reachable SHA-1 + # wants). Fall back to an unshallow fetch if the host rejects arbitrary SHAs. + if ! git -C "$CHECKOUT" fetch --depth 1 origin "$SHA" 2>/dev/null; then + echo " (shallow SHA fetch unsupported; fetching history)" + git -C "$CHECKOUT" fetch origin + fi + git -C "$CHECKOUT" checkout -q "$SHA" + echo "$SHA" > "$STAMP" +else + echo "== reusing existing clone at $SHA ==" +fi +[ -f "$WORKSPACE/Cargo.toml" ] || { + echo "ERROR: expected cargo workspace at $WORKSPACE/Cargo.toml" >&2; exit 1; } + +# --- 3. rewrite [patch.crates-io] -> reproducible toolchain stubs ------------ +echo "== rewriting [patch.crates-io] -> toolchain/stubs/* ==" +python3 - "$WORKSPACE/Cargo.toml" "$STUBS" <<'PY' +import re, sys +cargo, stubs = sys.argv[1], sys.argv[2] +lines = open(cargo).read().split('\n') +# Crates whose patch we own: reqwest/portable-pty/ctrlc get repointed at the +# reproducible toolchain stubs; tokio's patch is dropped so tokio 1.52.3 resolves +# through the vendored+patched sources (std-patches/crates/tokio/0001). +OWNED = ('portable-pty', 'reqwest', 'ctrlc', 'tokio') +owned_re = re.compile(r'^\s*#?\s*(?:%s)\s*=' % '|'.join(map(re.escape, OWNED))) +inject = [ + '# [patch.crates-io] injected by clone-and-build-codex-wasi.sh (reproducible stubs)', + 'portable-pty = {{ path = "{}/portable-pty-wasi" }}'.format(stubs), + 'reqwest = {{ path = "{}/reqwest-shim" }}'.format(stubs), + 'ctrlc = {{ path = "{}/ctrlc" }}'.format(stubs), + '# tokio: vendored+patched tokio 1.52.3 (std-patches/crates/tokio), not a path patch', +] +in_patch = False +injected = False +out = [] +for ln in lines: + s = ln.strip() + if s.startswith('[') and s.endswith(']'): + in_patch = (s == '[patch.crates-io]') + out.append(ln) + if in_patch: # inject our lines right after the header + out.extend(inject); injected = True + continue + if in_patch and owned_re.match(ln): # drop any prior line for an owned crate + continue + out.append(ln) +if not injected: # no [patch.crates-io] in the fork: add one + out += ['', '[patch.crates-io]'] + inject +open(cargo, 'w').write('\n'.join(out)) +PY +echo " --- resulting [patch.crates-io] head ---" +sed -n '/^\[patch.crates-io\]/,/^\[/p' "$WORKSPACE/Cargo.toml" | sed 's/^/ /' + +# --- 4. strip the hardcoded -L .../self-contained from .cargo/config.toml ---- +CLONE_CARGO_CFG="$WORKSPACE/.cargo/config.toml" +if [ -f "$CLONE_CARGO_CFG" ]; then + echo "== stripping machine -L from .cargo/config.toml ==" + python3 - "$CLONE_CARGO_CFG" <<'PY' +import re, sys +p = sys.argv[1] +s = open(p).read() +# Drop the "-C", "link-arg=-L/self-contained" pair anywhere in a rustflags +# array; the self-contained dir is recomputed and passed via RUSTFLAGS at build. +s = re.sub(r'"-C",\s*"link-arg=-L[^"]*self-contained",\s*', '', s) +open(p, 'w').write(s) +PY +fi + +# --- 5. vendor the workspace (+ std deps) and apply crate patches ------------ +RUST_STD_SRC="$(rustc "+$TOOLCHAIN" --print sysroot)/lib/rustlib/src/rust" +[ -d "$RUST_STD_SRC/library/std" ] || { + echo "ERROR: rust-src not found at $RUST_STD_SRC (rustup component add rust-src)" >&2; exit 1; } + +echo "== cargo vendor (workspace + std library deps for -Z build-std) ==" +cd "$WORKSPACE" +mkdir -p .cargo +cargo "+$TOOLCHAIN" vendor \ + --sync "$RUST_STD_SRC/library/std/Cargo.toml" \ + --sync "$RUST_STD_SRC/library/test/Cargo.toml" \ + "$WORKSPACE/vendor" > "$WORKSPACE/.cargo/vendor-sources.toml" + +# Append the source-replacement config so cargo builds from the patched vendor/ +# tree. Guard against double-append on a reused clone. +if ! grep -q 'source.crates-io' "$CLONE_CARGO_CFG" 2>/dev/null; then + printf '\n' >> "$CLONE_CARGO_CFG" + cat "$WORKSPACE/.cargo/vendor-sources.toml" >> "$CLONE_CARGO_CFG" +fi + +echo "== applying toolchain/std-patches/crates/* to vendored sources ==" +# patch-vendor.sh exits non-zero if ANY crate patch fails to apply. codex's +# dependency graph overlaps the command build's only partially, so some patches +# legitimately do not apply and are BENIGN for codex: +# - crossterm: codex uses the nornagon crossterm *git fork* (its own wasi +# support), not the crates.io crossterm the patch targets. +# - socket2 0.6.4: no longer carries the `not(target_env="p1")` exclusion the +# patch removes; codex builds against stock socket2 (verified in the fork). +# So don't let its exit code abort us; instead assert the patches codex REQUIRES. +VENDOR_DIR="$WORKSPACE/vendor" "$SCRIPT_DIR/patch-vendor.sh" || \ + echo " (patch-vendor reported failures; verifying codex-critical patches below)" + +echo "== verifying codex-critical crate patches applied ==" +assert_patched() { #