diff --git a/.gitignore b/.gitignore index 91949fe0cf..da80093777 100644 --- a/.gitignore +++ b/.gitignore @@ -90,3 +90,7 @@ crates/v8-runtime/assets/generated/ # staged local wasm command fixtures (11MB+ binaries; never commit) .local-cmds/ + +# Fetched Mozilla CA bundle (~230 KB). Pinned + fetched by `make -C toolchain/c +# ca-certificates`; staged into the sidecar at build time. Never committed. +crates/native-sidecar/assets/ca-certificates.crt diff --git a/CLAUDE.md b/CLAUDE.md index b2e3979007..86323f9ad1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -95,6 +95,15 @@ custom host-syscall imports. Treat that target as **native POSIX**; kernel/VFS syscalls, or the patched sysroot/libc. Do **not** paper over a Linux-deviating behavior in the package, its wrapper, or its test — chase it down into whichever runtime layer owns it and make that layer match Linux. +- **Cite the authoritative spec at the implementation site.** When you implement + against an external spec or wire/file format — `/proc`, network protocols, + syscall ABIs, on-disk layouts, TLS/crypto — put the reference **in a code + comment right where the format is emitted or parsed**: the man-page section + (e.g. `proc(5)`), the kernel source path (e.g. `fs/proc/array.c`), the RFC + (e.g. RFC 4253 for SSH transport), and/or the consumer's parser we must satisfy + (e.g. procps-ng `readproc.c`). A format emitter or protocol handler without a + doc link is incomplete. Conformance tests should name the captured real-Linux + fixture they validate against, so the chain spec → impl → test is explicit. ## Publishing diff --git a/crates/execution/assets/runners/wasi-module.js b/crates/execution/assets/runners/wasi-module.js index b861b77ba9..769b0be992 100644 --- a/crates/execution/assets/runners/wasi-module.js +++ b/crates/execution/assets/runners/wasi-module.js @@ -40,6 +40,7 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = const __agentOSWasiErrnoNoent = 44; const __agentOSWasiErrnoNosys = 52; const __agentOSWasiErrnoNotdir = 54; + const __agentOSWasiErrnoNotempty = 55; const __agentOSWasiErrnoPipe = 64; const __agentOSWasiErrnoRofs = 69; const __agentOSWasiErrnoNotcapable = 76; @@ -799,6 +800,8 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = return __agentOSWasiErrnoNoent; case "ENOTDIR": return __agentOSWasiErrnoNotdir; + case "ENOTEMPTY": + return __agentOSWasiErrnoNotempty; case "EEXIST": return __agentOSWasiErrnoExist; case "EINVAL": diff --git a/crates/execution/assets/runners/wasm-runner.mjs b/crates/execution/assets/runners/wasm-runner.mjs index f385336bd8..e0c5c3f707 100644 --- a/crates/execution/assets/runners/wasm-runner.mjs +++ b/crates/execution/assets/runners/wasm-runner.mjs @@ -32,11 +32,13 @@ const WASI_FILETYPE_UNKNOWN = 0; const WASI_FILETYPE_CHARACTER_DEVICE = 2; const WASI_FILETYPE_DIRECTORY = 3; const WASI_FILETYPE_REGULAR_FILE = 4; +const WASI_FILETYPE_SOCKET_STREAM = 6; const WASI_OFLAGS_CREAT = 1; const WASI_OFLAGS_DIRECTORY = 2; const WASI_OFLAGS_EXCL = 4; const WASI_OFLAGS_TRUNC = 8; const WASI_FDFLAGS_APPEND = 1; +const WASI_FDFLAGS_NONBLOCK = 4; const WASI_WHENCE_SET = 0; const WASI_WHENCE_CUR = 1; const WASI_WHENCE_END = 2; @@ -1414,7 +1416,12 @@ function seekGuestFileHandle(handle, offset, whence) { } else if (numericWhence === WASI_WHENCE_CUR) { base = BigInt(handle.position ?? 0); } else if (numericWhence === WASI_WHENCE_END) { - base = BigInt(Number(fsModule.fstatSync(handle.targetFd).size ?? 0)); + // Passthrough (read-only delegate) handles keep the real host fd in ioFd; + // targetFd is only a synthetic guest fd number and fstat'ing it reports + // size 0. Prefer ioFd so SEEK_END returns the true file size (e.g. mbedTLS + // sizing a CA bundle via fseek(SEEK_END)+ftell before reading it). + const sizeFd = typeof handle.ioFd === 'number' ? handle.ioFd : handle.targetFd; + base = BigInt(Number(fsModule.fstatSync(sizeFd).size ?? 0)); } else { return null; } @@ -2826,7 +2833,14 @@ function callSyncRpc(method, args = []) { } const hostNetSockets = new Map(); -let nextHostNetSocketFd = 4096; +// Host-net socket fds must stay BELOW the guests' FD_SETSIZE (1024 in the +// wasi-libc sysroot): libcurl's select-based Curl_poll / curl_multi_fdset +// guard every socket with `s < FD_SETSIZE` and silently drop larger fds from +// the pollset, which stalls any transfer that has to WAIT for socket +// readiness (non-blocking TLS handshakes, >16 KiB uploads). Real WASI fds are +// small integers and the synthetic fd space starts at 1 << 20, so a 600+ base +// keeps the ranges disjoint in practice while staying select()-compatible. +let nextHostNetSocketFd = 600; const HOST_NET_TIMEOUT_SENTINEL = '__agentos_net_timeout__'; const HOST_NET_MSG_PEEK = 0x0001; @@ -3042,12 +3056,30 @@ const HOST_NET_AF_INET = 2; const HOST_NET_AF_INET6 = 10; const HOST_NET_SOCK_DGRAM = 5; const HOST_NET_SOCKET_TYPE_MASK = 0xf; +// wasi-libc : SOCK_NONBLOCK / SOCK_CLOEXEC bits OR'd into the +// socket(2) type argument (Linux-style socket(..., SOCK_STREAM | SOCK_NONBLOCK)). +const HOST_NET_SOCK_NONBLOCK = 0x4000; const HOST_NET_SOL_SOCKET = 1; const HOST_NET_WASI_SOL_SOCKET = 0x7fffffff; const HOST_NET_SO_ERROR = 4; const HOST_NET_SO_RCVTIMEO_64 = 20; const HOST_NET_SO_RCVTIMEO_32 = 66; const HOST_NET_TIMEVAL_BYTES = 16; +// Performance/QoS socket options that guests may set but the host transport +// neither needs nor can honor per-socket: Node's net sockets already run +// with sensible defaults, and DSCP/traffic-class marking is not observable +// through the adapter. Accepted and ignored (values from the patched +// wasi-libc headers, matching Linux): setsockopt(2) succeeds, matching a +// Linux host where these are best-effort hints. OpenSSH sets all four on +// every connection (ssh_packet_set_tos / set_nodelay in opacket/misc) and +// treats failure as per-connection stderr noise. +const HOST_NET_SO_KEEPALIVE = 9; // SOL_SOCKET, socket(7) +const HOST_NET_IPPROTO_IP = 0; +const HOST_NET_IP_TOS = 1; // ip(7) +const HOST_NET_IPPROTO_TCP = 6; +const HOST_NET_TCP_NODELAY = 1; // tcp(7) +const HOST_NET_IPPROTO_IPV6 = 41; +const HOST_NET_IPV6_TCLASS = 67; // ipv6(7) function hostNetSocketBaseType(socket) { return Number(socket?.sockType ?? 0) & HOST_NET_SOCKET_TYPE_MASK; @@ -3057,6 +3089,35 @@ function hostNetSockoptKind(level, optname, optvalLen) { const normalizedLevel = Number(level) >>> 0; const normalizedOptname = Number(optname) >>> 0; const normalizedOptvalLen = Number(optvalLen) >>> 0; + // Accept-and-ignore QoS/keepalive/nagle hints (see constant block above). + // Option values are plain ints; accept any sane small buffer. + if (normalizedOptvalLen >= 1 && normalizedOptvalLen <= 16) { + if ( + (normalizedLevel === HOST_NET_SOL_SOCKET || + normalizedLevel === HOST_NET_WASI_SOL_SOCKET) && + normalizedOptname === HOST_NET_SO_KEEPALIVE + ) { + return 'ignore'; + } + if ( + normalizedLevel === HOST_NET_IPPROTO_TCP && + normalizedOptname === HOST_NET_TCP_NODELAY + ) { + return 'ignore'; + } + if ( + normalizedLevel === HOST_NET_IPPROTO_IP && + normalizedOptname === HOST_NET_IP_TOS + ) { + return 'ignore'; + } + if ( + normalizedLevel === HOST_NET_IPPROTO_IPV6 && + normalizedOptname === HOST_NET_IPV6_TCLASS + ) { + return 'ignore'; + } + } if ( normalizedLevel !== HOST_NET_SOL_SOCKET && normalizedLevel !== HOST_NET_WASI_SOL_SOCKET @@ -3238,14 +3299,16 @@ const hostNetImport = { net_poll(fdsPtr, nfds, timeoutMs, retReadyPtr) { const n = Number(nfds) >>> 0; const base0 = Number(fdsPtr) >>> 0; - // The patched wasi sysroot's effective poll bits (bits/poll.h): POLLIN=POLLRDNORM=0x1, - // POLLOUT=POLLWRNORM=0x2 (NOT the 0x004 in legacy poll.h). Guests (X server + libxcb) use - // these, so net_poll must match or POLLOUT readiness is never reported and writers block. + // Match the owned sysroot ABI in __header_poll.h exactly. Its WASI event + // representation uses POLLIN=0x1, POLLOUT=0x2 and the widened exceptional + // bits below, rather than Linux's numeric values. poll(2) defines the + // behavior; the sysroot header defines the guest-visible wire values. + // https://man7.org/linux/man-pages/man2/poll.2.html const POLLIN = 0x001; const POLLOUT = 0x002; - const POLLERR = 0x008; - const POLLHUP = 0x010; - const POLLNVAL = 0x020; + const POLLERR = 0x1000; + const POLLHUP = 0x2000; + const POLLNVAL = 0x4000; const t = Number(timeoutMs) | 0; const deadline = t < 0 ? null : Date.now() + Math.max(0, t); const kernelManagedStdio = @@ -3261,6 +3324,7 @@ const hostNetImport = { // comes from a batched __kernel_poll below, which doubles as the wait slice. const kernelTargets = []; const kernelEntries = []; + let hasHostNetWaitTarget = false; for (let i = 0; i < n; i++) { const base = base0 + i * 8; const fd = view.getInt32(base, true); @@ -3269,6 +3333,7 @@ const hostNetImport = { const socket = getHostNetSocket(fd); const handle = fd >= 0 ? lookupFdHandle(fd >>> 0) : undefined; if (socket && !socket.closed) { + hasHostNetWaitTarget = true; if (socket.serverId) { if (events & POLLIN) { // Report the listener readable only when a connection is actually pending. @@ -3283,6 +3348,16 @@ const hostNetImport = { if (events & POLLIN && socket.readChunks && socket.readChunks.length > 0) { revents |= POLLIN; } + // poll(2) reports peer shutdown as POLLHUP even when it was not + // requested, and a read after the queued data drains must return + // EOF without blocking. OpenSSH waits on this transition before + // exiting after the remote command closes its connection. + // https://man7.org/linux/man-pages/man2/poll.2.html + if (socket.readableEnded) { + revents |= POLLHUP; + if (events & POLLIN) revents |= POLLIN; + } + if (socket.lastError) revents |= POLLERR; if (events & POLLOUT) revents |= POLLOUT; } } else if (handle?.kind === 'pipe-read') { @@ -3299,21 +3374,28 @@ const hostNetImport = { } } else if (handle?.kind === 'pipe-write') { if (events & POLLOUT) revents |= POLLOUT; - } else if ( - fd >= 0 && - fd <= 2 && - kernelManagedStdio && - (!handle || (handle.kind === 'passthrough' && handle.targetFd === fd)) - ) { - // Kernel-managed stdio (PTY slave / stdio pipes): ask the kernel, like a - // native poll(2) on the terminal fd. + } else if (fd >= 0 && kernelManagedStdio && ( + (!handle && fd <= 2) || + (handle?.kind === 'passthrough' && Number(handle.targetFd) >= 0 && + Number(handle.targetFd) <= 2) + )) { + // poll(2): readiness means the requested operation will not block. + // https://man7.org/linux/man-pages/man2/poll.2.html + // Kernel-managed stdio (PTY slave / stdio pipes), including dup'd + // aliases: ask the kernel instead of treating a high alias like a + // regular file that is always ready. A false POLLIN here makes a + // guest block on an empty stdin pipe before it services another + // ready fd (for example OpenSSH flushing an exec request). + const kernelFd = handle?.kind === 'passthrough' + ? Number(handle.targetFd) >>> 0 + : fd; kernelTargets.push({ - fd, + fd: kernelFd, events: ((events & POLLIN) !== 0 ? KERNEL_POLLIN : 0) | ((events & POLLOUT) !== 0 ? KERNEL_POLLOUT : 0), }); - kernelEntries.push({ base, fd, events }); + kernelEntries.push({ base, fd, kernelFd, events }); } else if (handle) { // Regular files / other VFS-backed fds: always ready, as on Linux. revents |= events & (POLLIN | POLLOUT); @@ -3330,10 +3412,13 @@ const hostNetImport = { if (kernelTargets.length > 0) { // If something is already ready (or this is a non-blocking poll), probe the - // kernel without waiting; otherwise let the kernel wait one slice for us. + // kernel without waiting. Mixed host-net + kernel polls must also keep + // this probe nonblocking: __kernel_poll cannot wake for a host socket, + // so sleeping here starves each queued SSH packet for a full 10s slice. + // The socket pump below supplies the bounded wait in that case. const remaining = deadline == null ? Infinity : deadline - Date.now(); const sliceMs = - ready > 0 || t === 0 + ready > 0 || t === 0 || hasHostNetWaitTarget ? 0 : Math.max(0, Math.min(KERNEL_WAIT_SLICE_MS, remaining)); let response = null; @@ -3345,7 +3430,7 @@ const hostNetImport = { const responseEntries = Array.isArray(response?.fds) ? response.fds : []; for (const entry of kernelEntries) { const responseEntry = responseEntries.find( - (item) => (Number(item?.fd) >>> 0) === (entry.fd >>> 0), + (item) => (Number(item?.fd) >>> 0) === (entry.kernelFd >>> 0), ); const kernelRevents = Number(responseEntry?.revents) >>> 0; let revents = 0; @@ -3406,6 +3491,13 @@ const hostNetImport = { readableEnded: false, closed: false, lastError: null, + // Honor Linux-style socket(..., type | SOCK_NONBLOCK): guests like + // libcurl rely on O_NONBLOCK semantics (EAGAIN instead of blocking + // reads) to interleave send/recv on one connection. Dropping this bit + // deadlocks any upload larger than one TLS record: curl checks for an + // early server response mid-upload, and a blocking recv() waits on a + // server that is itself waiting for the rest of the request body. + nonblock: (numericType & HOST_NET_SOCK_NONBLOCK) !== 0, }); return writeGuestUint32(retFdPtr, fd); } catch { @@ -3874,6 +3966,9 @@ const hostNetImport = { if (sockoptKind == null) { return WASI_ERRNO_INVAL; } + if (sockoptKind === 'ignore') { + return WASI_ERRNO_SUCCESS; + } try { const timeoutMs = parseHostNetTimevalMs(readGuestBytes(optvalPtr, optvalLen)); if (timeoutMs == null && readGuestBytes(optvalPtr, optvalLen).some((byte) => byte !== 0)) { @@ -5186,11 +5281,14 @@ wasiImport.fd_read = (fd, iovs, iovsLen, nreadPtr) => { } if ( - numericFd === 0 && handle?.kind === 'passthrough' && - handle.targetFd === 0 && - passthroughHandles.get(0) === handle + handle.targetFd === 0 ) { + // dup(2) aliases share the same open file description as fd 0. In a + // sidecar-managed process they must therefore read the kernel stdin pipe, + // not the runner process's unrelated host stdin. OpenSSH duplicates stdin + // before its poll/read loop, so splitting these paths loses pipe EOF. + // https://man7.org/linux/man-pages/man2/dup.2.html const sidecarManagedProcess = typeof process?.env?.AGENTOS_SANDBOX_ROOT === 'string' && process.env.AGENTOS_SANDBOX_ROOT.length > 0; @@ -5476,6 +5574,26 @@ wasiImport.fd_tell = (fd, offsetPtr) => { }; wasiImport.fd_fdstat_get = (fd, statPtr) => { + // Host-net sockets (curl/wget/git TLS transports): report a stream-socket + // fdstat with the current O_NONBLOCK state so guest fcntl(F_GETFL) works. + // Without this, fcntl-based non-blocking setup fails with EBADF and guests + // that expect EAGAIN semantics (libcurl mid-upload reads) block forever. + { + const hostNetSocket = getHostNetSocket(fd); + if (hostNetSocket && !hostNetSocket.closed) { + return writeGuestFdstat( + statPtr, + WASI_FILETYPE_SOCKET_STREAM, + hostNetSocket.nonblock ? WASI_FDFLAGS_NONBLOCK : 0, + WASI_RIGHT_FD_READ | + WASI_RIGHT_FD_WRITE | + WASI_RIGHT_FD_FDSTAT_SET_FLAGS | + WASI_RIGHT_FD_FILESTAT_GET | + WASI_RIGHT_POLL_FD_READWRITE, + 0n, + ); + } + } const handle = __agentOSWasiMeasurePhase('fd_fdstat_get', 'lookup_handle', () => lookupFdHandle(fd) ); @@ -5601,6 +5719,16 @@ wasiImport.fd_fdstat_get = (fd, statPtr) => { }; wasiImport.fd_fdstat_set_flags = (fd, flags) => { + // Host-net sockets: honor O_NONBLOCK (guest fcntl F_SETFL). net_recv/net_send + // consult `socket.nonblock` to return EAGAIN instead of blocking, which + // non-blocking clients like libcurl rely on to interleave send/recv. + { + const hostNetSocket = getHostNetSocket(fd); + if (hostNetSocket && !hostNetSocket.closed) { + hostNetSocket.nonblock = (Number(flags) & WASI_FDFLAGS_NONBLOCK) !== 0; + return WASI_ERRNO_SUCCESS; + } + } const handle = lookupFdHandle(fd); if (handle && handle.kind !== 'passthrough') { return WASI_ERRNO_BADF; diff --git a/crates/execution/src/node_import_cache.rs b/crates/execution/src/node_import_cache.rs index fbe47a1c4e..8a37976c2c 100644 --- a/crates/execution/src/node_import_cache.rs +++ b/crates/execution/src/node_import_cache.rs @@ -15,7 +15,7 @@ const NODE_IMPORT_CACHE_PATH_ENV: &str = "AGENTOS_NODE_IMPORT_CACHE_PATH"; const NODE_IMPORT_CACHE_LOADER_PATH_ENV: &str = "AGENTOS_NODE_IMPORT_CACHE_LOADER_PATH"; const NODE_IMPORT_CACHE_SCHEMA_VERSION: &str = "1"; const NODE_IMPORT_CACHE_LOADER_VERSION: &str = "8"; -const NODE_IMPORT_CACHE_ASSET_VERSION: &str = "96"; +const NODE_IMPORT_CACHE_ASSET_VERSION: &str = "104"; const NODE_IMPORT_CACHE_DIR_PREFIX: &str = "agentos-node-import-cache"; const DEFAULT_NODE_IMPORT_CACHE_MATERIALIZE_TIMEOUT: Duration = Duration::from_secs(30); const PYODIDE_DIST_DIR: &str = "pyodide-dist"; diff --git a/crates/kernel/src/fd_table.rs b/crates/kernel/src/fd_table.rs index 3e57be8ec9..f42eb2b109 100644 --- a/crates/kernel/src/fd_table.rs +++ b/crates/kernel/src/fd_table.rs @@ -583,6 +583,15 @@ impl ProcessFdTable { child.next_fd = self.next_fd; for (fd, entry) in &self.entries { + // Kernel process creation is spawn (fork + exec combined), so + // close-on-exec descriptors must not leak into the child. This + // matters for pipe write ends: an inherited writer keeps the + // pipe's writer refcount above zero forever, so a blocked reader + // (for example a grandchild sharing the parent's stdin pipe) + // would never observe EOF. + if entry.fd_flags & FD_CLOEXEC != 0 { + continue; + } entry.description.increment_ref_count(); child.entries.insert( *fd, diff --git a/crates/kernel/src/kernel.rs b/crates/kernel/src/kernel.rs index abb5ab2481..4f98b784fe 100644 --- a/crates/kernel/src/kernel.rs +++ b/crates/kernel/src/kernel.rs @@ -426,12 +426,14 @@ enum ProcNode { CpuInfoFile, MemInfoFile, LoadAvgFile, + StatFile, UptimeFile, VersionFile, SelfLink { pid: u32 }, PidDir { pid: u32 }, PidFdDir { pid: u32 }, PidCmdline { pid: u32 }, + PidComm { pid: u32 }, PidEnviron { pid: u32 }, PidCwdLink { pid: u32 }, PidStatFile { pid: u32 }, @@ -3767,15 +3769,20 @@ impl KernelVm { self.filesystem .check_virtual_path(FsOperation::Read, path) .map_err(KernelError::from)?; - return Ok(self - .proc_read_dir(current_pid, &proc_node)? - .into_iter() - .map(|name| VirtualDirEntry { + let mut typed = Vec::new(); + for name in self.proc_read_dir(current_pid, &proc_node)? { + let child_path = join_child_path(&normalize_path(path), &name); + let child = self + .resolve_proc_node(&child_path, current_pid)? + .ok_or_else(|| proc_not_found_error(&child_path))?; + let filetype = proc_filetype(&child); + typed.push(VirtualDirEntry { name, - is_directory: false, - is_symbolic_link: false, - }) - .collect()); + is_directory: filetype == FILETYPE_DIRECTORY, + is_symbolic_link: filetype == FILETYPE_SYMBOLIC_LINK, + }); + } + return Ok(typed); } Ok(self.filesystem.read_dir_with_types(path)?) @@ -3819,6 +3826,7 @@ impl KernelVm { ["cpuinfo"] => Some(ProcNode::CpuInfoFile), ["meminfo"] => Some(ProcNode::MemInfoFile), ["loadavg"] => Some(ProcNode::LoadAvgFile), + ["stat"] => Some(ProcNode::StatFile), ["uptime"] => Some(ProcNode::UptimeFile), ["version"] => Some(ProcNode::VersionFile), _ => None, @@ -3840,6 +3848,7 @@ impl KernelVm { [_pid] => ProcNode::PidDir { pid }, [_pid, "fd"] => ProcNode::PidFdDir { pid }, [_pid, "cmdline"] => ProcNode::PidCmdline { pid }, + [_pid, "comm"] => ProcNode::PidComm { pid }, [_pid, "environ"] => ProcNode::PidEnviron { pid }, [_pid, "cwd"] => ProcNode::PidCwdLink { pid }, [_pid, "stat"] => ProcNode::PidStatFile { pid }, @@ -3887,9 +3896,11 @@ impl KernelVm { ProcNode::CpuInfoFile => Ok(self.proc_cpuinfo_bytes()), ProcNode::MemInfoFile => Ok(self.proc_meminfo_bytes()), ProcNode::LoadAvgFile => Ok(self.proc_loadavg_bytes()), + ProcNode::StatFile => Ok(self.proc_system_stat_bytes()), ProcNode::UptimeFile => Ok(self.proc_uptime_bytes()), ProcNode::VersionFile => Ok(self.proc_version_bytes()), ProcNode::PidCmdline { pid } => Ok(self.proc_cmdline_bytes(*pid)), + ProcNode::PidComm { pid } => Ok(self.proc_comm_bytes(*pid)), ProcNode::PidEnviron { pid } => Ok(self.proc_environ_bytes(*pid)), ProcNode::PidStatFile { pid } => Ok(self.proc_stat_bytes(*pid)), ProcNode::PidStatusFile { pid } => Ok(self.proc_status_bytes(*pid)), @@ -3942,6 +3953,10 @@ impl KernelVm { proc_inode(node), self.proc_loadavg_bytes().len() as u64, )), + ProcNode::StatFile => Ok(proc_file_stat( + proc_inode(node), + self.proc_system_stat_bytes().len() as u64, + )), ProcNode::UptimeFile => Ok(proc_file_stat( proc_inode(node), self.proc_uptime_bytes().len() as u64, @@ -3954,6 +3969,10 @@ impl KernelVm { proc_inode(node), self.proc_cmdline_bytes(*pid).len() as u64, )), + ProcNode::PidComm { pid } => Ok(proc_file_stat( + proc_inode(node), + self.proc_comm_bytes(*pid).len() as u64, + )), ProcNode::PidEnviron { pid } => Ok(proc_file_stat( proc_inode(node), self.proc_environ_bytes(*pid).len() as u64, @@ -4014,6 +4033,7 @@ impl KernelVm { entries.push(String::from("meminfo")); entries.push(String::from("mounts")); entries.push(String::from("self")); + entries.push(String::from("stat")); entries.push(String::from("uptime")); entries.push(String::from("version")); entries.sort(); @@ -4021,6 +4041,7 @@ impl KernelVm { } ProcNode::PidDir { .. } => Ok(vec![ String::from("cmdline"), + String::from("comm"), String::from("cwd"), String::from("environ"), String::from("fd"), @@ -4080,12 +4101,14 @@ impl KernelVm { ProcNode::CpuInfoFile => String::from("/proc/cpuinfo"), ProcNode::MemInfoFile => String::from("/proc/meminfo"), ProcNode::LoadAvgFile => String::from("/proc/loadavg"), + ProcNode::StatFile => String::from("/proc/stat"), ProcNode::UptimeFile => String::from("/proc/uptime"), ProcNode::VersionFile => String::from("/proc/version"), ProcNode::SelfLink { pid } => format!("/proc/{pid}"), ProcNode::PidDir { pid } => format!("/proc/{pid}"), ProcNode::PidFdDir { pid } => format!("/proc/{pid}/fd"), ProcNode::PidCmdline { pid } => format!("/proc/{pid}/cmdline"), + ProcNode::PidComm { pid } => format!("/proc/{pid}/comm"), ProcNode::PidEnviron { pid } => format!("/proc/{pid}/environ"), ProcNode::PidCwdLink { pid } => format!("/proc/{pid}/cwd"), ProcNode::PidStatFile { pid } => format!("/proc/{pid}/stat"), @@ -4118,24 +4141,86 @@ impl KernelVm { ) } + fn proc_comm_bytes(&self, pid: u32) -> Vec { + let entry = self + .processes + .get(pid) + .expect("process must exist while procfs path is resolved"); + let mut bytes = proc_task_comm(&entry.command).into_bytes(); + bytes.push(b'\n'); + bytes + } + fn proc_stat_bytes(&self, pid: u32) -> Vec { let entry = self .processes .get(pid) .expect("process must exist while procfs path is resolved"); - let command = entry.command.replace(')', "]"); + let command = proc_task_comm(&entry.command); let state = match entry.status { ProcessStatus::Running => 'R', ProcessStatus::Stopped => 'T', ProcessStatus::Exited => 'Z', }; - format!( - "{pid} ({command}) {state} {ppid} {pgid} {sid} 0 0 0 0 0 0 0 0 0 0 20 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0", - ppid = entry.ppid, - pgid = entry.pgid, - sid = entry.sid, - ) - .into_bytes() + // Linux's exact field order and unescaped parenthesized comm come from + // proc_pid_stat(5) and fs/proc/array.c::do_task_stat(). procps-ng's + // library/readproc.c consumes this positionally and finds the final + // ") " delimiter, so rewriting ')' inside comm corrupts real names. + // https://man7.org/linux/man-pages/man5/proc_pid_stat.5.html + // https://github.com/torvalds/linux/blob/master/fs/proc/array.c + // https://gitlab.com/procps-ng/procps/-/blob/master/library/readproc.c + let tail = [ + entry.ppid.to_string(), // 4 ppid + entry.pgid.to_string(), // 5 pgrp + entry.sid.to_string(), // 6 session + String::from("0"), // 7 tty_nr + String::from("-1"), // 8 tpgid + String::from("0"), // 9 flags + String::from("0"), // 10 minflt + String::from("0"), // 11 cminflt + String::from("0"), // 12 majflt + String::from("0"), // 13 cmajflt + String::from("0"), // 14 utime + String::from("0"), // 15 stime + String::from("0"), // 16 cutime + String::from("0"), // 17 cstime + String::from("20"), // 18 priority + String::from("0"), // 19 nice + String::from("1"), // 20 num_threads + String::from("0"), // 21 itrealvalue + String::from("0"), // 22 starttime + String::from("0"), // 23 vsize + String::from("0"), // 24 rss + String::from("0"), // 25 rsslim + String::from("0"), // 26 startcode + String::from("0"), // 27 endcode + String::from("0"), // 28 startstack + String::from("0"), // 29 kstkesp + String::from("0"), // 30 kstkeip + String::from("0"), // 31 signal + String::from("0"), // 32 blocked + String::from("0"), // 33 sigignore + String::from("0"), // 34 sigcatch + String::from("0"), // 35 wchan + String::from("0"), // 36 nswap + String::from("0"), // 37 cnswap + String::from("17"), // 38 exit_signal (SIGCHLD) + String::from("0"), // 39 processor + String::from("0"), // 40 rt_priority + String::from("0"), // 41 policy + String::from("0"), // 42 delayacct_blkio_ticks + String::from("0"), // 43 guest_time + String::from("0"), // 44 cguest_time + String::from("0"), // 45 start_data + String::from("0"), // 46 end_data + String::from("0"), // 47 start_brk + String::from("0"), // 48 arg_start + String::from("0"), // 49 arg_end + String::from("0"), // 50 env_start + String::from("0"), // 51 env_end + entry.exit_code.unwrap_or(0).to_string(), // 52 exit_code + ]; + format!("{pid} ({command}) {state} {}\n", tail.join(" ")).into_bytes() } fn proc_mounts_bytes(&self) -> Vec { @@ -4207,6 +4292,31 @@ impl KernelVm { format!("0.00 0.00 0.00 {running}/{total} {last_pid}\n").into_bytes() } + fn proc_system_stat_bytes(&self) -> Vec { + // proc_stat(5) defines USER_HZ CPU columns, btime, process counters, + // and runnable/blocked counts. procps consumes these labels directly. + // https://man7.org/linux/man-pages/man5/proc_stat.5.html + // https://gitlab.com/procps-ng/procps/-/blob/master/library/stat.c + let cpu_count = self.proc_cpu_count(); + let idle_ticks = (self.boot_instant.elapsed().as_secs_f64() * 100.0) as u64; + let aggregate_idle = idle_ticks.saturating_mul(cpu_count as u64); + let processes = self.processes.list_processes(); + let running = processes + .values() + .filter(|process| process.status == ProcessStatus::Running) + .count(); + let mut body = format!("cpu 0 0 0 {aggregate_idle} 0 0 0 0 0 0\n"); + for cpu in 0..cpu_count { + body.push_str(&format!("cpu{cpu} 0 0 0 {idle_ticks} 0 0 0 0 0 0\n")); + } + body.push_str(&format!( + "intr 0\nctxt 0\nbtime {}\nprocesses {}\nprocs_running {running}\nprocs_blocked 0\nsoftirq 0 0 0 0 0 0 0 0 0 0 0\n", + self.boot_time_ms / 1000, + processes.len(), + )); + body.into_bytes() + } + fn proc_uptime_bytes(&self) -> Vec { let uptime = self.boot_instant.elapsed().as_secs_f64(); format!("{uptime:.2} {uptime:.2}\n").into_bytes() @@ -4230,11 +4340,18 @@ impl KernelVm { ProcessStatus::Stopped => ('T', "stopped"), ProcessStatus::Exited => ('Z', "zombie"), }; + // Linux emits both Tgid and Pid in /proc//status. procps-ng's + // library/readproc.c treats Threads as proof of the modern format and + // then assigns its process id from Tgid, so omitting Tgid rewrites an + // otherwise valid PID to zero. + // https://man7.org/linux/man-pages/man5/proc_pid_status.5.html + // https://github.com/torvalds/linux/blob/master/fs/proc/array.c + // https://gitlab.com/procps-ng/procps/-/blob/master/library/readproc.c format!( - "Name:\t{name}\nState:\t{state_code} ({state_name})\nPid:\t{pid}\nPPid:\t{ppid}\nUid:\t{uid}\t{euid}\t{euid}\t{euid}\nGid:\t{gid}\t{egid}\t{egid}\t{egid}\nVmSize:\t{:>8} kB\nVmRSS:\t{:>9} kB\nThreads:\t1\n", + "Name:\t{name}\nState:\t{state_code} ({state_name})\nTgid:\t{pid}\nPid:\t{pid}\nPPid:\t{ppid}\nUid:\t{uid}\t{euid}\t{euid}\t{euid}\nGid:\t{gid}\t{egid}\t{egid}\t{egid}\nVmSize:\t{:>8} kB\nVmRSS:\t{:>9} kB\nThreads:\t1\n", 0, 0, - name = entry.command, + name = proc_status_task_name(&entry.command), ppid = entry.ppid, uid = entry.identity.uid, euid = entry.identity.euid, @@ -5101,9 +5218,11 @@ fn proc_filetype(node: &ProcNode) -> u8 { | ProcNode::CpuInfoFile | ProcNode::MemInfoFile | ProcNode::LoadAvgFile + | ProcNode::StatFile | ProcNode::UptimeFile | ProcNode::VersionFile | ProcNode::PidCmdline { .. } + | ProcNode::PidComm { .. } | ProcNode::PidEnviron { .. } | ProcNode::PidStatFile { .. } | ProcNode::PidStatusFile { .. } => FILETYPE_REGULAR_FILE, @@ -5117,20 +5236,37 @@ fn proc_inode(node: &ProcNode) -> u64 { ProcNode::CpuInfoFile => 0xfffe_0003, ProcNode::MemInfoFile => 0xfffe_0004, ProcNode::LoadAvgFile => 0xfffe_0005, - ProcNode::UptimeFile => 0xfffe_0006, - ProcNode::VersionFile => 0xfffe_0007, + ProcNode::StatFile => 0xfffe_0006, + ProcNode::UptimeFile => 0xfffe_0007, + ProcNode::VersionFile => 0xfffe_0008, ProcNode::SelfLink { pid } => 0xfffe_1000 + u64::from(*pid), ProcNode::PidDir { pid } => 0xfffe_2000 + u64::from(*pid), ProcNode::PidFdDir { pid } => 0xfffe_3000 + u64::from(*pid), ProcNode::PidCmdline { pid } => 0xfffe_4000 + u64::from(*pid), - ProcNode::PidEnviron { pid } => 0xfffe_5000 + u64::from(*pid), - ProcNode::PidCwdLink { pid } => 0xfffe_6000 + u64::from(*pid), - ProcNode::PidStatFile { pid } => 0xfffe_7000 + u64::from(*pid), - ProcNode::PidStatusFile { pid } => 0xfffe_8000 + u64::from(*pid), + ProcNode::PidComm { pid } => 0xfffe_5000 + u64::from(*pid), + ProcNode::PidEnviron { pid } => 0xfffe_6000 + u64::from(*pid), + ProcNode::PidCwdLink { pid } => 0xfffe_7000 + u64::from(*pid), + ProcNode::PidStatFile { pid } => 0xfffe_8000 + u64::from(*pid), + ProcNode::PidStatusFile { pid } => 0xfffe_9000 + u64::from(*pid), ProcNode::PidFdLink { pid, fd } => 0xffff_0000 + ((u64::from(*pid)) << 8) + u64::from(*fd), } } +fn proc_task_comm(command: &str) -> String { + let basename = command.rsplit('/').next().unwrap_or(command); + let mut end = basename.len().min(15); + while !basename.is_char_boundary(end) { + end -= 1; + } + basename[..end].to_owned() +} + +fn proc_status_task_name(command: &str) -> String { + proc_task_comm(command) + .replace('\\', "\\\\") + .replace('\n', "\\n") +} + fn null_separated_bytes(parts: Vec) -> Vec { if parts.is_empty() { return Vec::new(); diff --git a/crates/kernel/tests/api_surface.rs b/crates/kernel/tests/api_surface.rs index 23d7651670..6fa6e0f22f 100644 --- a/crates/kernel/tests/api_surface.rs +++ b/crates/kernel/tests/api_surface.rs @@ -1023,6 +1023,16 @@ fn proc_filesystem_exposes_live_process_metadata_and_fd_symlinks() { assert!(proc_entries.contains(&String::from("mounts"))); assert!(proc_entries.contains(&process.pid().to_string())); + let proc_typed = kernel + .read_dir_with_types_for_process("shell", process.pid(), "/proc") + .expect("read typed /proc"); + assert!(proc_typed + .iter() + .any(|entry| { entry.name == process.pid().to_string() && entry.is_directory })); + assert!(proc_typed + .iter() + .any(|entry| entry.name == "self" && entry.is_symbolic_link)); + assert_eq!( kernel .read_link_for_process("shell", process.pid(), "/proc/self") diff --git a/crates/kernel/tests/fixtures/procfs/generate_linux_task_name.py b/crates/kernel/tests/fixtures/procfs/generate_linux_task_name.py new file mode 100755 index 0000000000..8a16b6cf2a --- /dev/null +++ b/crates/kernel/tests/fixtures/procfs/generate_linux_task_name.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +"""Regenerate linux-*-task-name.json on a real Linux host.""" + +import ctypes +import json +import os +import platform + + +TASK_NAME = "agent)os\nproc" +PR_SET_NAME = 15 + +libc = ctypes.CDLL(None) +if libc.prctl(PR_SET_NAME, TASK_NAME.encode(), 0, 0, 0) != 0: + raise OSError(ctypes.get_errno(), "prctl(PR_SET_NAME) failed") + +pid = os.getpid() +with open(f"/proc/{pid}/comm", encoding="utf-8") as source: + comm = source.read() +with open(f"/proc/{pid}/stat", encoding="utf-8") as source: + stat = source.read() +with open(f"/proc/{pid}/status", encoding="utf-8") as source: + status = source.read() + +right_paren = stat.rfind(") ") +status_name = next( + line.split("\t", 1)[1] for line in status.splitlines() if line.startswith("Name:\t") +) +print( + json.dumps( + { + "source": f"{platform.system()} {platform.release()} {platform.machine()}", + "task_name_input": TASK_NAME, + "comm": comm, + "stat_comm": stat[stat.find(" (") + 2 : right_paren], + "status_name": status_name, + }, + indent=2, + ) +) diff --git a/crates/kernel/tests/fixtures/procfs/linux-6.1-task-name.json b/crates/kernel/tests/fixtures/procfs/linux-6.1-task-name.json new file mode 100644 index 0000000000..9a035cdaba --- /dev/null +++ b/crates/kernel/tests/fixtures/procfs/linux-6.1-task-name.json @@ -0,0 +1,7 @@ +{ + "source": "Linux 6.1.0-41-amd64 x86_64", + "task_name_input": "agent)os\nproc", + "comm": "agent)os\nproc\n", + "stat_comm": "agent)os\nproc", + "status_name": "agent)os\\nproc" +} diff --git a/crates/kernel/tests/identity.rs b/crates/kernel/tests/identity.rs index 541245114c..60db59c92d 100644 --- a/crates/kernel/tests/identity.rs +++ b/crates/kernel/tests/identity.rs @@ -159,6 +159,7 @@ fn procfs_exposes_linux_like_identity_and_system_files() { assert!(proc_entries.contains(&String::from("meminfo"))); assert!(proc_entries.contains(&String::from("mounts"))); assert!(proc_entries.contains(&String::from("self"))); + assert!(proc_entries.contains(&String::from("stat"))); assert!(proc_entries.contains(&String::from("uptime"))); assert!(proc_entries.contains(&String::from("version"))); assert!(proc_entries.contains(&pid.to_string())); @@ -167,6 +168,7 @@ fn procfs_exposes_linux_like_identity_and_system_files() { .read_dir(&format!("/proc/{pid}")) .expect("read /proc/"); assert!(pid_entries.contains(&String::from("status"))); + assert!(pid_entries.contains(&String::from("comm"))); let status = read_utf8(&mut kernel, &format!("/proc/{pid}/status")); let self_status = @@ -177,6 +179,7 @@ fn procfs_exposes_linux_like_identity_and_system_files() { assert_eq!(status_fields["Name"], "identity-check"); assert_eq!(status_fields["State"], "R (running)"); assert_eq!(status_fields["Pid"], pid.to_string()); + assert_eq!(status_fields["Tgid"], pid.to_string()); assert_eq!(status_fields["PPid"], "0"); assert_eq!(status_fields["Uid"], "501\t700\t700\t700"); assert_eq!(status_fields["Gid"], "502\t701\t701\t701"); @@ -213,4 +216,57 @@ fn procfs_exposes_linux_like_identity_and_system_files() { .stat(&format!("/proc/{pid}/status")) .expect("stat proc status"); assert_eq!(status_stat.size, status.len() as u64); + + let system_stat = read_utf8(&mut kernel, "/proc/stat"); + assert!(system_stat.starts_with("cpu ")); + assert!(system_stat.contains("\ncpu0 ")); + assert!(system_stat.contains("\nbtime ")); + assert!(system_stat.contains("\nprocs_running 1\n")); +} + +#[test] +fn procfs_task_names_match_captured_real_linux_fixture() { + // Captured by fixtures/procfs/generate_linux_task_name.py on the real-Linux + // host named in the fixture. This covers fs/proc/array.c's intentionally + // different escaping rules for comm, stat, and status. + let fixture: serde_json::Value = + serde_json::from_str(include_str!("fixtures/procfs/linux-6.1-task-name.json")) + .expect("parse captured Linux procfs fixture"); + let task_name = fixture["task_name_input"] + .as_str() + .expect("fixture task name"); + let mut kernel = configured_kernel(); + let process = kernel + .create_virtual_process( + "identity-driver", + "identity-driver", + task_name, + Vec::new(), + VirtualProcessOptions::default(), + ) + .expect("create fixture-named process"); + let pid = process.pid(); + + assert_eq!( + read_utf8(&mut kernel, &format!("/proc/{pid}/comm")), + fixture["comm"].as_str().expect("fixture comm") + ); + let stat = read_utf8(&mut kernel, &format!("/proc/{pid}/stat")); + let (stat_comm, stat_tail) = stat + .strip_prefix(&format!("{pid} (")) + .and_then(|body| body.rsplit_once(") ")) + .expect("parse emitted stat comm"); + assert_eq!( + stat_comm, + fixture["stat_comm"].as_str().expect("fixture stat comm") + ); + assert_eq!(stat_tail.split_whitespace().count(), 50); + + let status = read_utf8(&mut kernel, &format!("/proc/{pid}/status")); + assert_eq!( + parse_status_fields(&status)["Name"], + fixture["status_name"] + .as_str() + .expect("fixture status name") + ); } diff --git a/crates/native-sidecar/build.rs b/crates/native-sidecar/build.rs index e1d617fbd1..31399eb30f 100644 --- a/crates/native-sidecar/build.rs +++ b/crates/native-sidecar/build.rs @@ -1,4 +1,7 @@ -use std::{env, fs, path::PathBuf}; +use std::{ + env, fs, + path::{Path, PathBuf}, +}; // Stage the base filesystem fixture into OUT_DIR. In-tree builds use the // canonical AgentOS runtime-core fixture from the current workspace; the @@ -31,4 +34,41 @@ fn main() { error ) }); + + stage_ca_bundle(&manifest_dir, &out_dir); +} + +/// Stage the Mozilla CA bundle into OUT_DIR so it can be embedded via +/// `include_bytes!` and seeded into the VM at `/etc/ssl/certs/ca-certificates.crt`. +/// +/// The ~230 KB PEM blob is never committed. It is fetched into +/// `assets/ca-certificates.crt` by `make -C toolchain/c ca-certificates`. When +/// the asset is absent (fresh checkout, or `cargo publish` verification with no +/// network) we stage an empty placeholder — the sidecar then simply skips +/// seeding the bundle, matching the Pyodide "asset unavailable" pattern. Runtime +/// TLS still works via `--cacert`/`SSL_CERT_FILE` overrides in that case. +fn stage_ca_bundle(manifest_dir: &Path, out_dir: &Path) { + let asset = manifest_dir.join("assets/ca-certificates.crt"); + println!("cargo:rerun-if-changed={}", asset.display()); + + let dest = out_dir.join("ca-certificates.crt"); + if asset.exists() { + fs::copy(&asset, &dest).unwrap_or_else(|error| { + panic!( + "failed to stage ca-certificates.crt from {} to {}: {}", + asset.display(), + dest.display(), + error + ) + }); + } else { + // Empty placeholder keeps the include_bytes! of the OUT_DIR copy valid. + fs::write(&dest, b"").unwrap_or_else(|error| { + panic!( + "failed to stage empty ca-certificates.crt placeholder to {}: {}", + dest.display(), + error + ) + }); + } } diff --git a/crates/native-sidecar/src/execution.rs b/crates/native-sidecar/src/execution.rs index fc7a8d2921..68979f9430 100644 --- a/crates/native-sidecar/src/execution.rs +++ b/crates/native-sidecar/src/execution.rs @@ -41,7 +41,8 @@ use crate::state::{ JavascriptTlsDataValue, JavascriptTlsMaterial, JavascriptUdpFamily, JavascriptUdpSocketEvent, JavascriptUnixListenerEvent, KernelSocketReadinessEvent, KernelSocketReadinessRegistry, KernelSocketReadinessTarget, LoopbackTlsPendingWriteHandle, LoopbackTlsPendingWriteState, - NetworkResourceCounts, PendingTcpSocket, PendingUnixSocket, ProcNetEntry, ProcessEventEnvelope, + NetworkResourceCounts, PendingKernelStdin, PendingTcpSocket, PendingUnixSocket, ProcNetEntry, + ProcessEventEnvelope, PythonHostSocket, ResolvedChildProcessExecution, ResolvedTcpConnectAddr, SharedBridge, SharedSidecarRequestClient, SidecarKernel, SocketQueryKind, ToolExecution, VmDnsConfig, VmListenPolicy, VmState, DEFAULT_JAVASCRIPT_NET_BACKLOG, EXECUTION_DRIVER_NAME, @@ -471,6 +472,7 @@ impl ActiveProcess { kernel_pid, kernel_handle, kernel_stdin_writer_fd: None, + pending_kernel_stdin: PendingKernelStdin::default(), tty_master_fd: None, runtime, detached: false, @@ -7885,6 +7887,11 @@ where { return Ok(false); } + // Top off the child's stdin pipe from any host-side backlog before + // probing readiness: the child draining the pipe is exactly what frees + // capacity for the next queued stdin bytes (and, once the backlog is + // empty, executes a deferred stdin close so EOF arrives in order). + flush_pending_kernel_stdin(kernel, child)?; let now = Instant::now(); let requested_timeout_ms = match request.method.as_str() { "__kernel_stdin_read" => parse_kernel_stdin_read_args(request)?.1, @@ -9932,8 +9939,7 @@ pub(crate) fn sync_active_process_host_writes_to_kernel( vm: &mut VmState, ) -> Result<(), SidecarError> { if vm.root_filesystem_mode != RootFilesystemMode::ReadOnly { - let shadow_root = vm.cwd.clone(); - sync_host_directory_tree_to_kernel(vm, &shadow_root, "/")?; + sync_vm_shadow_root_to_kernel(vm)?; } let normalized_vm_root = normalize_host_path(&vm.cwd); @@ -9945,6 +9951,90 @@ pub(crate) fn sync_active_process_host_writes_to_kernel( Ok(()) } +/// Syncs the VM shadow root into the kernel VFS and reconciles deletions. +/// +/// The walk itself is additive (it copies shadow entries into the kernel), so +/// on its own it can never express a guest deletion performed directly on the +/// shadow tree — a removed file would be resurrected forever. To get real +/// Linux semantics the walk records every guest path it sees and diffs that +/// set against the previous walk's inventory: paths that were present in the +/// shadow before and are gone now are removed from the kernel VFS as well. +/// +/// Deletion propagation is scoped to the guest-writable shadow region: it only +/// runs for the VM shadow root walk (never for external host roots), re-checks +/// the protected/mount skip list, never touches the POSIX bootstrap skeleton, +/// and removes directories non-recursively so a kernel directory that still +/// has kernel-only content (for example a mount point or a path that was never +/// materialized into the shadow) is left in place with a warning. +fn sync_vm_shadow_root_to_kernel(vm: &mut VmState) -> Result<(), SidecarError> { + let shadow_root = normalize_host_path(&vm.cwd); + if host_sync_root_is_filesystem_root(&shadow_root) { + tracing::warn!("skipping host shadow sync rooted at the host filesystem root"); + return Ok(()); + } + let mut synced_file_times = BTreeMap::new(); + let mut inventory = BTreeSet::new(); + sync_host_directory_tree_to_kernel_inner( + vm, + &shadow_root, + &shadow_root, + "/", + &mut synced_file_times, + Some(&mut inventory), + )?; + propagate_shadow_deletions_to_kernel(vm, &inventory); + vm.shadow_sync_inventory = inventory; + Ok(()) +} + +/// Removes kernel paths whose shadow copy disappeared since the last walk. +/// +/// Best-effort by design: a failure to reconcile one stale path must not +/// poison the guest filesystem operation that triggered the sync, so failures +/// are surfaced as host-visible warnings instead of errors. Children sort +/// after their parents in the `BTreeSet`, so the reverse iteration removes +/// leaves before the directories that contain them. +fn propagate_shadow_deletions_to_kernel(vm: &mut VmState, current: &BTreeSet) { + if vm.shadow_sync_inventory.is_empty() { + return; + } + let stale: Vec = vm + .shadow_sync_inventory + .iter() + .rev() + .filter(|path| !current.contains(*path)) + .cloned() + .collect(); + for path in stale { + if path == "/" + || should_skip_shadow_sync_path(vm, &path) + || is_shadow_bootstrap_dir(&path) + { + continue; + } + let stat = match vm.kernel.lstat(&path) { + Ok(stat) => stat, + // Already gone (for example removed through the kernel-direct + // guest fs path, which mirrors deletions into the shadow itself). + Err(_) => continue, + }; + let result = if stat.is_directory && !stat.is_symbolic_link { + vm.kernel.remove_dir(&path) + } else { + vm.kernel.remove_file(&path) + }; + if let Err(error) = result { + if error.code() != "ENOENT" { + tracing::warn!( + path = %path, + error = %error, + "failed to propagate guest shadow deletion into the kernel VFS" + ); + } + } + } +} + fn collect_active_process_host_sync_roots( vm: &VmState, normalized_vm_root: &Path, @@ -9991,8 +10081,7 @@ fn sync_process_host_roots_to_kernel( process_guest_cwd: &str, ) -> Result<(), SidecarError> { if vm.root_filesystem_mode != RootFilesystemMode::ReadOnly { - let shadow_root = vm.cwd.clone(); - sync_host_directory_tree_to_kernel(vm, &shadow_root, "/")?; + sync_vm_shadow_root_to_kernel(vm)?; } if !path_is_within_root( @@ -10031,6 +10120,7 @@ fn sync_host_directory_tree_to_kernel( &normalized_host_root, &normalized_guest_root, &mut synced_file_times, + None, ) } @@ -10040,6 +10130,7 @@ fn sync_host_directory_tree_to_kernel_inner( current_host_dir: &Path, guest_root: &str, synced_file_times: &mut BTreeMap<(u64, u64), (u64, u64)>, + mut inventory: Option<&mut BTreeSet>, ) -> Result<(), SidecarError> { let entries = match fs::read_dir(current_host_dir) { Ok(entries) => entries, @@ -10101,6 +10192,15 @@ fn sync_host_directory_tree_to_kernel_inner( continue; } + // Record every shadow entry we can represent in the kernel (dirs, + // files, symlinks) so the deletion reconcile can tell "was in the + // shadow, now gone" apart from "never came from the shadow". + if let Some(inventory) = inventory.as_deref_mut() { + if file_type.is_dir() || file_type.is_file() || file_type.is_symlink() { + inventory.insert(guest_path.clone()); + } + } + if file_type.is_dir() { let metadata = entry.metadata().map_err(|error| { SidecarError::Io(format!( @@ -10136,6 +10236,7 @@ fn sync_host_directory_tree_to_kernel_inner( &host_path, guest_root, synced_file_times, + inventory.as_deref_mut(), )?; continue; } @@ -19493,6 +19594,36 @@ fn install_kernel_stdin_pipe(kernel: &mut SidecarKernel, pid: u32) -> Result git remote-https -> git-remote-https). + kernel + .fd_fcntl( + EXECUTION_DRIVER_NAME, + pid, + write_fd, + agentos_kernel::fd_table::F_SETFD, + agentos_kernel::fd_table::FD_CLOEXEC, + ) + .map_err(kernel_error)?; + // Non-blocking writes only: the sidecar dispatch thread feeds this pipe + // from `write_kernel_process_stdin` / `flush_pending_kernel_stdin`, and a + // blocking pipe write would deadlock — the child's pipe reads are parked + // sync RPCs serviced by this same thread. A full pipe surfaces EAGAIN and + // the remainder is queued in `ActiveProcess::pending_kernel_stdin`. + kernel + .fd_fcntl( + EXECUTION_DRIVER_NAME, + pid, + write_fd, + agentos_kernel::fd_table::F_SETFL, + agentos_kernel::fd_table::O_NONBLOCK, + ) + .map_err(kernel_error)?; Ok(write_fd) } @@ -19517,6 +19648,13 @@ fn javascript_child_process_stdin_mode(request: &JavascriptChildProcessSpawnRequ .unwrap_or("pipe") } +/// Host-side cap on stdin bytes queued for a child's kernel pipe. Matches the +/// kernel's default per-write bound (`resource.max_fd_write_bytes`); a parent +/// cannot park more than this behind a slow child before getting a typed +/// error. +const MAX_PENDING_KERNEL_STDIN_BYTES: usize = + agentos_kernel::resource_accounting::DEFAULT_MAX_FD_WRITE_BYTES; + pub(crate) fn write_kernel_process_stdin( kernel: &mut SidecarKernel, process: &mut ActiveProcess, @@ -19532,17 +19670,103 @@ pub(crate) fn write_kernel_process_stdin( let Some(writer_fd) = process.kernel_stdin_writer_fd else { return Ok(()); }; - kernel - .fd_write(EXECUTION_DRIVER_NAME, process.kernel_pid, writer_fd, chunk) - .map_err(kernel_error)?; - // For a TTY process the master write above drives line-discipline echo into - // the master output buffer. Drain it now and surface it as the single - // ordered Stdout stream so typed-character echo reaches the host even while - // the guest is blocked in read() and not producing output of its own. - if let Some(echo) = drain_tty_master_output(kernel, process)? { - process.queue_pending_execution_event(ActiveExecutionEvent::Stdout(echo))?; - } - forward_tty_slave_input_to_javascript(kernel, process)?; + if process.tty_master_fd.is_some() { + kernel + .fd_write(EXECUTION_DRIVER_NAME, process.kernel_pid, writer_fd, chunk) + .map_err(kernel_error)?; + // For a TTY process the master write above drives line-discipline echo + // into the master output buffer. Drain it now and surface it as the + // single ordered Stdout stream so typed-character echo reaches the + // host even while the guest is blocked in read() and not producing + // output of its own. + if let Some(echo) = drain_tty_master_output(kernel, process)? { + process.queue_pending_execution_event(ActiveExecutionEvent::Stdout(echo))?; + } + forward_tty_slave_input_to_javascript(kernel, process)?; + return Ok(()); + } + // Pipe-backed stdin. The kernel pipe caps at MAX_PIPE_BUFFER_BYTES and + // fd_write reports POSIX partial writes, so anything the pipe cannot take + // right now is queued and flushed as the child drains (see + // `flush_pending_kernel_stdin`). Silently dropping the remainder is how + // multi-buffer stdin payloads (e.g. git's spooled pack piped into + // `index-pack --stdin`) used to truncate at 64 KiB. + let pending_total = process.pending_kernel_stdin.total; + if pending_total.saturating_add(chunk.len()) > MAX_PENDING_KERNEL_STDIN_BYTES { + return Err(SidecarError::InvalidState(format!( + "child process stdin backlog limit exceeded: {pending_total} pending + {} new bytes \ + > {MAX_PENDING_KERNEL_STDIN_BYTES} (MAX_PENDING_KERNEL_STDIN_BYTES); the child is \ + not draining stdin — write smaller chunks after the child consumes them, or raise \ + this bound together with the kernel resource.max_fd_write_bytes limit", + chunk.len(), + ))); + } + process.pending_kernel_stdin.push(chunk); + flush_pending_kernel_stdin(kernel, process) +} + +/// Write as much queued stdin as the child's kernel pipe can take right now. +/// Called on every stdin write and from the child kernel-wait servicing path +/// (each parked `__kernel_stdin_read` / `__kernel_poll` re-check), so the +/// backlog drains in lockstep with the child's reads. Executes the deferred +/// stdin close once the backlog is empty. +pub(crate) fn flush_pending_kernel_stdin( + kernel: &mut SidecarKernel, + process: &mut ActiveProcess, +) -> Result<(), SidecarError> { + if process.tty_master_fd.is_some() { + return Ok(()); + } + let Some(writer_fd) = process.kernel_stdin_writer_fd else { + process.pending_kernel_stdin.clear(); + process.pending_kernel_stdin.close_requested = false; + return Ok(()); + }; + while let Some(front) = process.pending_kernel_stdin.chunks.pop_front() { + let offset = process.pending_kernel_stdin.front_offset; + let slice = &front[offset..]; + match kernel.fd_write(EXECUTION_DRIVER_NAME, process.kernel_pid, writer_fd, slice) { + Ok(written) if written >= slice.len() => { + process.pending_kernel_stdin.total = + process.pending_kernel_stdin.total.saturating_sub(slice.len()); + process.pending_kernel_stdin.front_offset = 0; + } + Ok(written) => { + // Pipe is full mid-chunk; keep the remainder queued. + process.pending_kernel_stdin.total = + process.pending_kernel_stdin.total.saturating_sub(written); + process.pending_kernel_stdin.front_offset = offset + written; + process.pending_kernel_stdin.chunks.push_front(front); + break; + } + Err(error) if error.code() == "EAGAIN" => { + process.pending_kernel_stdin.chunks.push_front(front); + break; + } + Err(error) if error.code() == "EPIPE" => { + // Reader side is gone (child exited or closed stdin): drop the + // backlog and release the writer, mirroring a SIGPIPE'd POSIX + // writer. + process.pending_kernel_stdin.clear(); + process.pending_kernel_stdin.close_requested = false; + process.kernel_stdin_writer_fd = None; + let _ = kernel.fd_close(EXECUTION_DRIVER_NAME, process.kernel_pid, writer_fd); + return Ok(()); + } + Err(error) => { + process.pending_kernel_stdin.chunks.push_front(front); + return Err(kernel_error(error)); + } + } + } + if process.pending_kernel_stdin.is_empty() && process.pending_kernel_stdin.close_requested { + process.pending_kernel_stdin.close_requested = false; + if let Some(writer_fd) = process.kernel_stdin_writer_fd.take() { + kernel + .fd_close(EXECUTION_DRIVER_NAME, process.kernel_pid, writer_fd) + .map_err(kernel_error)?; + } + } Ok(()) } @@ -19589,6 +19813,13 @@ pub(crate) fn close_kernel_process_stdin( kernel: &mut SidecarKernel, process: &mut ActiveProcess, ) -> Result<(), SidecarError> { + if !process.pending_kernel_stdin.is_empty() && process.kernel_stdin_writer_fd.is_some() { + // Queued stdin has not reached the pipe yet; closing now would hand + // the child a premature EOF. `flush_pending_kernel_stdin` performs the + // close once the backlog drains. + process.pending_kernel_stdin.close_requested = true; + return Ok(()); + } let Some(writer_fd) = process.kernel_stdin_writer_fd.take() else { return Ok(()); }; diff --git a/crates/native-sidecar/src/state.rs b/crates/native-sidecar/src/state.rs index 3004ec7b36..9cf857c76d 100644 --- a/crates/native-sidecar/src/state.rs +++ b/crates/native-sidecar/src/state.rs @@ -357,6 +357,15 @@ pub(crate) struct VmState { /// packages ship no `agentos-package.json`, so agent enumeration and /// resolution read this instead of the guest filesystem. pub(crate) projected_agent_launch: BTreeMap, + /// Guest paths that were present in the VM shadow root during the last + /// shadow->kernel sync walk. The next walk diffs the current shadow tree + /// against this set so guest deletions performed directly on the shadow + /// (host-side runtimes, WASI passthrough writes) propagate into the kernel + /// VFS instead of being resurrected by the otherwise additive sync. + /// Memory is bounded by the shadow tree itself, which is capped by the + /// kernel filesystem inode/byte resource limits that bound what the walk + /// can materialize. + pub(crate) shadow_sync_inventory: BTreeSet, } /// Launch parameters for one projected agent package. @@ -449,11 +458,49 @@ impl Default for VmListenPolicy { // Active process state // --------------------------------------------------------------------------- +/// Stdin bytes accepted from a parent's `child_process.write_stdin` but not +/// yet written into the child's kernel stdin pipe. The kernel pipe holds at +/// most `MAX_PIPE_BUFFER_BYTES` (64 KiB) and `fd_write` reports partial +/// writes with POSIX pipe semantics, so multi-buffer stdin payloads (for +/// example git's spooled pack fed to `index-pack --stdin`) must be queued +/// host-side and flushed as the child drains its pipe. `close_requested` +/// defers the writer-fd close until the backlog fully drains so the child +/// never observes an early EOF. +#[derive(Default)] +pub(crate) struct PendingKernelStdin { + pub(crate) chunks: VecDeque>, + /// Bytes of the front chunk already written into the pipe. + pub(crate) front_offset: usize, + /// Total unwritten bytes across all queued chunks. + pub(crate) total: usize, + pub(crate) close_requested: bool, +} + +impl PendingKernelStdin { + pub(crate) fn is_empty(&self) -> bool { + self.chunks.is_empty() + } + + pub(crate) fn push(&mut self, chunk: &[u8]) { + self.total += chunk.len(); + self.chunks.push_back(chunk.to_vec()); + } + + pub(crate) fn clear(&mut self) { + self.chunks.clear(); + self.front_offset = 0; + self.total = 0; + } +} + #[allow(dead_code)] pub(crate) struct ActiveProcess { pub(crate) kernel_pid: u32, pub(crate) kernel_handle: KernelProcessHandle, pub(crate) kernel_stdin_writer_fd: Option, + /// Backlog for pipe-backed kernel stdin awaiting pipe capacity; see + /// [`PendingKernelStdin`]. + pub(crate) pending_kernel_stdin: PendingKernelStdin, /// For a TTY (PTY-backed) process, the master-end fd whose output buffer /// carries cooked-mode echo plus ONLCR-processed guest output. When set, /// this master output is the single ordered output stream surfaced to the diff --git a/crates/native-sidecar/src/vm.rs b/crates/native-sidecar/src/vm.rs index be3afd17c3..300ad28dc3 100644 --- a/crates/native-sidecar/src/vm.rs +++ b/crates/native-sidecar/src/vm.rs @@ -109,6 +109,21 @@ const SHADOW_ROOT_BOOTSTRAP_DIRS: &[(&str, u32)] = &[ ("/workspace", 0o755), ]; +/// Mozilla CA bundle (the set Debian's `ca-certificates` produces), staged by +/// `build.rs` into `OUT_DIR`. It is empty when the `assets/ca-certificates.crt` +/// blob was absent at build time (fresh checkout / offline publish verify — run +/// `make -C toolchain/c ca-certificates` to populate it); an empty bundle is +/// simply not seeded. When present it is written into the VM shadow tree so +/// in-guest TLS (curl/wget's mbedTLS backend) resolves trust the Debian way via +/// `/etc/ssl/certs/ca-certificates.crt`. +const CA_CERTIFICATES_BUNDLE: &[u8] = + include_bytes!(concat!(env!("OUT_DIR"), "/ca-certificates.crt")); +const CA_CERTIFICATES_GUEST_PATH: &str = "/etc/ssl/certs/ca-certificates.crt"; +/// Conventional `/etc/ssl/cert.pem` -> `certs/ca-certificates.crt` symlink that +/// OpenSSL's default `OPENSSLDIR` layout expects. +const CA_CERTIFICATES_SYMLINK_PATH: &str = "/etc/ssl/cert.pem"; +const CA_CERTIFICATES_SYMLINK_TARGET: &str = "certs/ca-certificates.crt"; + fn send_kernel_socket_readiness_event( target: KernelSocketReadinessTarget, readiness: SocketReadiness, @@ -358,6 +373,7 @@ where signal_states: BTreeMap::new(), packages_staging_root: None, projected_agent_launch: BTreeMap::new(), + shadow_sync_inventory: BTreeSet::new(), }, ); @@ -1753,6 +1769,49 @@ fn bootstrap_shadow_root(root: &Path) -> Result<(), SidecarError> { )) })?; } + seed_ca_certificates_bundle(root)?; + Ok(()) +} + +/// Seed the Mozilla CA bundle into the shadow root at +/// `/etc/ssl/certs/ca-certificates.crt` (plus the conventional +/// `/etc/ssl/cert.pem` symlink) so guest TLS clients resolve trust the standard +/// Debian way. No-op when the bundle was not staged at build time. +fn seed_ca_certificates_bundle(root: &Path) -> Result<(), SidecarError> { + if CA_CERTIFICATES_BUNDLE.is_empty() { + return Ok(()); + } + + let bundle_path = shadow_path_for_guest(root, CA_CERTIFICATES_GUEST_PATH); + if let Some(parent) = bundle_path.parent() { + fs::create_dir_all(parent).map_err(|error| { + SidecarError::Io(format!( + "failed to create shadow CA certs directory {}: {error}", + parent.display() + )) + })?; + } + fs::write(&bundle_path, CA_CERTIFICATES_BUNDLE).map_err(|error| { + SidecarError::Io(format!( + "failed to seed CA bundle {}: {error}", + bundle_path.display() + )) + })?; + fs::set_permissions(&bundle_path, fs::Permissions::from_mode(0o644)).map_err(|error| { + SidecarError::Io(format!( + "failed to set CA bundle mode on {}: {error}", + bundle_path.display() + )) + })?; + + let symlink_path = shadow_path_for_guest(root, CA_CERTIFICATES_SYMLINK_PATH); + let _ = fs::remove_file(&symlink_path); + std::os::unix::fs::symlink(CA_CERTIFICATES_SYMLINK_TARGET, &symlink_path).map_err(|error| { + SidecarError::Io(format!( + "failed to seed CA bundle symlink {}: {error}", + symlink_path.display() + )) + })?; Ok(()) } @@ -2150,8 +2209,11 @@ mod tests { use super::{ bootstrap_native_root_filesystem, bootstrap_shadow_root, materialize_shadow_root_snapshot_entries, native_root_plugin_from_config, - prune_kernel_command_stub, shadow_path_for_guest, KERNEL_COMMAND_STUB, + prune_kernel_command_stub, shadow_path_for_guest, CA_CERTIFICATES_BUNDLE, + CA_CERTIFICATES_GUEST_PATH, CA_CERTIFICATES_SYMLINK_PATH, CA_CERTIFICATES_SYMLINK_TARGET, + KERNEL_COMMAND_STUB, }; + use std::path::Path; use crate::plugins::chunked_local::ChunkedLocalMountPlugin; use crate::protocol::{ RootFilesystemDescriptor, RootFilesystemEntry, RootFilesystemEntryKind, @@ -2207,6 +2269,44 @@ mod tests { fs::remove_dir_all(&root).expect("temp shadow root should be removed"); } + #[test] + fn bootstrap_shadow_root_seeds_ca_bundle_when_present() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock should be monotonic") + .as_nanos(); + let root = + std::env::temp_dir().join(format!("agentos-native-sidecar-ca-test-{unique}")); + fs::create_dir_all(&root).expect("temp shadow root should be created"); + + bootstrap_shadow_root(&root).expect("shadow bootstrap should succeed"); + + let bundle = shadow_path_for_guest(&root, CA_CERTIFICATES_GUEST_PATH); + let symlink = shadow_path_for_guest(&root, CA_CERTIFICATES_SYMLINK_PATH); + + if CA_CERTIFICATES_BUNDLE.is_empty() { + // No asset staged at build time — seeding is intentionally skipped. + assert!( + !bundle.exists(), + "CA bundle must not be seeded when the build asset is absent" + ); + } else { + let seeded = fs::read(&bundle).expect("CA bundle should be seeded"); + assert_eq!( + seeded, CA_CERTIFICATES_BUNDLE, + "seeded CA bundle should match the embedded asset" + ); + let target = fs::read_link(&symlink).expect("cert.pem symlink should be seeded"); + assert_eq!( + target, + Path::new(CA_CERTIFICATES_SYMLINK_TARGET), + "cert.pem should point at certs/ca-certificates.crt" + ); + } + + fs::remove_dir_all(&root).expect("temp shadow root should be removed"); + } + #[test] fn native_root_config_opens_chunked_local_as_persistent_root() { let unique = SystemTime::now() diff --git a/crates/native-sidecar/tests/filesystem.rs b/crates/native-sidecar/tests/filesystem.rs index e414eff247..ad0329d229 100644 --- a/crates/native-sidecar/tests/filesystem.rs +++ b/crates/native-sidecar/tests/filesystem.rs @@ -463,6 +463,171 @@ mod shadow_root { } } + fn base_guest_filesystem_request( + operation: GuestFilesystemOperation, + path: &str, + ) -> GuestFilesystemCallRequest { + GuestFilesystemCallRequest { + operation, + path: String::from(path), + destination_path: None, + target: None, + content: None, + encoding: None, + recursive: false, + max_depth: None, + mode: None, + uid: None, + gid: None, + atime_ms: None, + mtime_ms: None, + len: None, + offset: None, + } + } + + fn guest_path_exists( + sidecar: &mut agentos_native_sidecar::NativeSidecar, + connection_id: &str, + session_id: &str, + vm_id: &str, + request_id: i64, + path: &str, + ) -> bool { + let response = sidecar + .dispatch_wire_blocking(wire_request( + request_id, + wire_vm(connection_id, session_id, vm_id), + RequestPayload::GuestFilesystemCallRequest(base_guest_filesystem_request( + GuestFilesystemOperation::Exists, + path, + )), + )) + .expect("dispatch guest filesystem exists"); + match response.response.payload { + ResponsePayload::GuestFilesystemResultResponse(result) => { + result.exists.unwrap_or(false) + } + other => panic!("expected guest_filesystem_result response, got {other:?}"), + } + } + + /// Deleting a path directly from the VM shadow root (the way host-side + /// guest runtimes delete files, without a kernel-direct unlink) must + /// propagate into the kernel VFS on the next shadow sync walk instead of + /// being resurrected by the additive copy-in. + #[test] + fn shadow_direct_deletions_propagate_into_kernel_vfs() { + let mut sidecar = create_test_sidecar(); + let (connection_id, session_id) = authenticate_and_open_session(&mut sidecar); + // Guest filesystem calls need no command mounts; create the VM bare so + // this regression test runs even without built registry commands. + let cwd = temp_dir("filesystem-shadow-reconcile-cwd"); + let (vm_id, _) = create_vm_wire( + &mut sidecar, + 3, + &connection_id, + &session_id, + GuestRuntimeKind::JavaScript, + &cwd, + ); + + let nonce = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("system clock") + .as_nanos(); + let doomed_root = format!("/workspace/reconcile-{nonce}"); + let doomed_dir = format!("{doomed_root}/nested"); + let doomed_file = format!("{doomed_dir}/probe.txt"); + let survivor_file = format!("/workspace/reconcile-survivor-{nonce}.txt"); + + let mut mkdir = + base_guest_filesystem_request(GuestFilesystemOperation::CreateDir, &doomed_dir); + mkdir.recursive = true; + guest_filesystem_call(&mut sidecar, &connection_id, &session_id, &vm_id, 20, mkdir); + + let mut write = + base_guest_filesystem_request(GuestFilesystemOperation::WriteFile, &doomed_file); + write.content = Some(String::from("doomed\n")); + write.encoding = Some(RootFilesystemEntryEncoding::Utf8); + guest_filesystem_call(&mut sidecar, &connection_id, &session_id, &vm_id, 21, write); + + let mut survivor = + base_guest_filesystem_request(GuestFilesystemOperation::WriteFile, &survivor_file); + survivor.content = Some(String::from("survivor\n")); + survivor.encoding = Some(RootFilesystemEntryEncoding::Utf8); + guest_filesystem_call(&mut sidecar, &connection_id, &session_id, &vm_id, 22, survivor); + + // This read-side call walks the shadow tree and records the inventory + // the deletion reconcile diffs against. + assert!(guest_path_exists( + &mut sidecar, + &connection_id, + &session_id, + &vm_id, + 23, + &doomed_file, + )); + + // Locate this VM's shadow root through the mirrored unique file, then + // delete the subtree exactly as a host-side guest runtime would: on + // the shadow filesystem only, with no kernel-direct unlink. + let marker_rel = format!( + "workspace/reconcile-{nonce}/nested/probe.txt" + ); + let shadow_root = std::fs::read_dir(std::env::temp_dir()) + .expect("list temp dir") + .filter_map(|entry| entry.ok().map(|entry| entry.path())) + .find(|candidate| { + candidate + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.starts_with("agentos-native-sidecar-shadow-")) + && candidate.join(&marker_rel).is_file() + }) + .expect("locate VM shadow root through mirrored probe file"); + fs::remove_dir_all(shadow_root.join(format!("workspace/reconcile-{nonce}"))) + .expect("delete subtree from the shadow root"); + + // The next host filesystem call re-walks the shadow; the kernel must + // drop the deleted subtree instead of resurrecting it forever. + assert!( + !guest_path_exists( + &mut sidecar, + &connection_id, + &session_id, + &vm_id, + 24, + &doomed_file, + ), + "kernel resurrected a file deleted from the shadow root" + ); + assert!( + !guest_path_exists( + &mut sidecar, + &connection_id, + &session_id, + &vm_id, + 25, + &doomed_root, + ), + "kernel resurrected a directory deleted from the shadow root" + ); + assert!( + guest_path_exists( + &mut sidecar, + &connection_id, + &session_id, + &vm_id, + 26, + &survivor_file, + ), + "deletion reconcile must not remove shadow-backed paths that still exist" + ); + + dispose_vm_and_close_session(&mut sidecar, &connection_id, &session_id, &vm_id); + } + #[allow(clippy::too_many_arguments)] fn execute_command( sidecar: &mut agentos_native_sidecar::NativeSidecar, diff --git a/crates/native-sidecar/tests/fixtures/limits-inventory.json b/crates/native-sidecar/tests/fixtures/limits-inventory.json index 65f5098b8a..08e26cf1a1 100644 --- a/crates/native-sidecar/tests/fixtures/limits-inventory.json +++ b/crates/native-sidecar/tests/fixtures/limits-inventory.json @@ -1114,5 +1114,11 @@ "path": "crates/execution/src/wasm.rs", "class": "invariant", "rationale": "Host-side LRU entry cap for cached wasm module bytes; process-wide cache sizing, not per-VM policy." + }, + { + "name": "MAX_PENDING_KERNEL_STDIN_BYTES", + "path": "crates/native-sidecar/src/execution.rs", + "class": "policy-deferred", + "rationale": "Host-side backlog cap for child stdin awaiting kernel pipe capacity; mirrors resource.max_fd_write_bytes, not yet wired to VmLimits." } ] diff --git a/crates/vfs/src/posix/overlay_fs.rs b/crates/vfs/src/posix/overlay_fs.rs index 9fd0600587..653d18c819 100644 --- a/crates/vfs/src/posix/overlay_fs.rs +++ b/crates/vfs/src/posix/overlay_fs.rs @@ -1563,8 +1563,13 @@ impl VirtualFileSystem for OverlayFileSystem { if self.is_whited_out(path) { return Err(Self::entry_not_found(path)); } - let lower_exists = self.find_lower_by_exists(path).is_some(); - let upper_exists = self.exists_in_upper(path); + // POSIX unlink(2) removes the directory entry itself and never follows + // a symlink leaf, so existence must be checked with lstat semantics. + // `exists()` resolves symlinks, which made dangling symlinks (for + // example git's `.git/tXXXXXX` symlink-support probe) unremovable: + // the entry stayed listed by readdir while unlink reported ENOENT. + let lower_exists = self.find_lower_by_entry(path).is_some(); + let upper_exists = self.has_entry_in_upper(path); if !lower_exists && !upper_exists { return Err(Self::entry_not_found(path)); } @@ -2013,6 +2018,55 @@ mod tests { assert!(!overlay.exists("/workspace-temp")); } + #[test] + fn remove_file_unlinks_dangling_symlink() { + // git's symlink-support probe creates `.git/tXXXXXX -> testing` (a + // dangling symlink), lstats it, and unlinks it. unlink(2) must remove + // the link itself without resolving it. + let lower = MemoryFileSystem::new(); + let mut overlay = OverlayFileSystem::new(vec![lower], OverlayMode::Ephemeral); + overlay.mkdir("/repo", true).expect("create directory"); + overlay + .symlink("testing", "/repo/probe") + .expect("create dangling symlink"); + assert!(overlay + .lstat("/repo/probe") + .expect("lstat dangling symlink") + .is_symbolic_link); + + overlay + .remove_file("/repo/probe") + .expect("unlink dangling symlink"); + + assert!(overlay.lstat("/repo/probe").is_err()); + assert_eq!( + overlay.read_dir("/repo").expect("read emptied directory"), + Vec::::new() + ); + overlay + .remove_dir("/repo") + .expect("rmdir emptied directory"); + } + + #[test] + fn remove_file_unlinks_dangling_symlink_from_lower_layer() { + let mut lower = MemoryFileSystem::new(); + lower.mkdir("/repo", true).expect("create lower directory"); + lower + .symlink("missing-target", "/repo/probe") + .expect("seed lower dangling symlink"); + + let mut overlay = OverlayFileSystem::new(vec![lower], OverlayMode::Ephemeral); + overlay + .remove_file("/repo/probe") + .expect("whiteout lower dangling symlink"); + + assert!(overlay.lstat("/repo/probe").is_err()); + overlay + .remove_dir("/repo") + .expect("rmdir merged-empty directory"); + } + #[test] fn remove_dir_still_rejects_visible_children() { let mut lower = MemoryFileSystem::new(); diff --git a/docs-internal/networking-parity-spec.md b/docs-internal/networking-parity-spec.md new file mode 100644 index 0000000000..3d05964988 --- /dev/null +++ b/docs-internal/networking-parity-spec.md @@ -0,0 +1,234 @@ +# Networking Parity Spec — curl / wget / git full TLS + network support + +Status: spec · Owner: registry/runtime · Last updated: 2026-07-08 + +> **2026-07-09 convergence addendum:** this document records and justifies the +> original curl/wget/git milestone, which correctly shipped in-guest mbedTLS +> instead of host-brokered TLS. The later Node-stdlib decision does not undo +> that proof, but it supersedes mbedTLS as the **final shared backend**: Node +> v24.15.0 requires real OpenSSL 3.5.5 behavior, so `toolchain/c` will build one +> pinned OpenSSL wasm archive set and migrate the registry consumers forward to +> it. Read “OpenSSL in-guest — rejected” below as “rejected for the original +> curl-only milestone,” not as a prohibition on the now-decided cross-runtime +> unification. Normative current state and order live in +> `registry-networking-handoff.md` §5 and the Node replacement spec §7.4. + +## Goal + +`curl`, `wget`, and `git` must have **full networking with real TLS/SSL, using +native Linux semantics** — not host-brokered shims. Concretely: + +- `curl https://…` and `wget https://…` verify certificates against a real CA + bundle, honor `--cacert`/`--ca-certificate`, fail with the **real exit codes** + (curl exit 60 = cert verify failure), and support `--compressed` (gzip, brotli, + zstd). +- `git clone/fetch/push` over **HTTPS smart-HTTP** works against GitHub/GitLab. +- Certificate trust comes from a **CA bundle inside the VM** (`/etc/ssl/certs/…`), + the way Debian's `ca-certificates` package + apt/curl/openssl/python all resolve + it — not from the host machine's trust store. + +## Current state (verified) + +Sockets are **already real** and are not the problem. The patched wasi-libc +sysroot implements `socket()/connect()/getaddrinfo()/send()/recv()` over +`host_net` WASM imports (`toolchain/std-patches/wasi-libc/0008-sockets.patch`, +`0023-host-net-read-write-sockets.patch`; Rust mirror `toolchain/crates/wasi-ext`). +The runner forwards them to the sidecar socket table (`crates/execution/assets/ +runners/wasm-runner.mjs`). So curl/wget/git already do their own DNS, TCP and HTTP +byte-for-byte. **Only TLS and decompression are shimmed or missing.** + +| Tool | Networking today | TLS today | Gap | +|---|---|---|---| +| **curl** | real (8.11.1) | host-brokered `wasi_tls.c` → `net_tls_connect` → sidecar rustls + **host** cert store | no real verify, `--cacert`/exit-60/`-v` chain all wrong; no gzip/brotli/zstd | +| **wget** | real (1.24.5) | **none** (`--without-ssl`) — HTTP only | no HTTPS at all; no gzip | +| **git** | real, local-only (2.55.0, `NO_CURL`) | **none** | `git-remote-https` is a dead symlink→git binary; no HTTPS clone/fetch/push | + +The current TLS shim (`software/curl/native/c/overlay/lib/vtls/wasi_tls.c`) does +**no handshake and no verification in-guest**: it hands the TCP fd to the host +(`net_tls_connect(fd, host, flags)`), which terminates TLS with rustls + +`rustls_native_certs::load_native_certs()` — **the host machine's** trust store. +`--cacert`/`--capath`/`CURL_CA_BUNDLE`/`SSL_CERT_FILE` are silently ignored; any +TLS error returns curl exit 35 instead of 60; `curl -v` cannot print the chain. + +## The decision: in-guest TLS (mbedTLS) + a VM CA bundle + +Two directions were researched. **We choose in-guest TLS** because it is the only +one that delivers the acceptance criterion (native Linux semantics + a CA bundle +that mimics apt). + +### ✅ Chosen — in-guest mbedTLS 3.6 LTS + `/etc/ssl/certs/ca-certificates.crt` + +- **mbedTLS** is pure portable C99, zero platform deps, TLS 1.3, and curl has a + first-class maintained `USE_MBEDTLS` backend in 8.11.1. Entropy = `getentropy()` + (already proven in `wasi_tls.c`); clocks/time work; build single-threaded. +- Verification happens **in curl's own code path**: `--cacert` → + `mbedtls_x509_crt_parse_file` via the VFS, `-k` → generic `verifypeer`, verify + failure → `CURLE_PEER_FAILED_VERIFICATION` = **exit 60** with the real message, + `curl -v` prints version/cipher/cert. Exactly Linux curl-with-mbedTLS. +- The sidecar becomes a **dumb ciphertext pipe** — strictly better for the trust + model (the untrusted executor no longer asks the trusted host to authenticate + servers on its behalf), and hermetic (no dependence on the host's cert store). + +### ❌ Rejected for parity — host-brokered TLS (the current `net_tls_connect` path) + +Reusing the existing bridge is minimal work (wget = ~100-line shim, git = build +with the overlaid libcurl), but it **cannot reach native semantics**: it uses the +*host's* cert store (non-hermetic), ignores `--cacert`/`--capath`, returns the +wrong exit-code taxonomy, can't print the cert chain, and every future TLS flag +(client certs, `--pinnedpubkey`, TLS-version pinning) needs a new bridge hop — a +permanent shim treadmill. It also contradicts the "real tool, patch the sysroot" +philosophy the rest of the toolchain follows. Keep the host `net.socket_upgrade_tls` +path only for the **Node/JS runtime**, which is a separate surface. + +> Note: the current sidecar TLS path uses `rustls_native_certs` = the **host +> machine's** trust store (`crates/native-sidecar/src/execution.rs`). That is a +> latent hermeticity bug even for the JS runtime — it should read the VM's +> `/etc/ssl` bundle instead. Tracked here; fix alongside. + +## CA bundle — ship Debian-shaped trust inside the VM + +- Ship the **Mozilla CA bundle** (`curl.se/ca/cacert.pem`, i.e. the set Debian's + `ca-certificates` produces) at **`/etc/ssl/certs/ca-certificates.crt`**, with the + conventional `/etc/ssl/cert.pem` symlink. A `ca-certificates` registry package + owns the payload; VM bootstrap links it into the standard tree (the bootstrap + already seeds `/etc` in the shadow root — `crates/native-sidecar/src/vm.rs`). +- This one file at that one path is what makes the **whole class** of TLS tools + "just work": curl's compile-time `CURL_CA_BUNDLE` default, OpenSSL's `OPENSSLDIR` + (`/usr/lib/ssl` → `/etc/ssl/certs`), apt, python, wget — all resolve there on + Debian. `SSL_CERT_FILE`/`SSL_CERT_DIR`/`--cacert` env overrides then work for + free once the backend does real verification. + +## Per-tool plans + +### curl +1. Vendor + build **mbedTLS 3.6** for `wasm32-wasip1` in `toolchain/c/Makefile` + (same pattern as the existing zlib target). +2. In `toolchain/c/scripts/build-curl-upstream.sh`: drop `--without-ssl` and the + `USE_WASI_TLS` injection; add `--with-mbedtls`, `--with-zlib`, `--with-brotli`, + `--with-zstd`, `--with-ca-bundle=/etc/ssl/certs/ca-certificates.crt`; retire the + `wasi_tls.c/h` + `vtls.c` overlay. +3. Tests: assert **exit 60** + real verify message on self-signed without `-k`; + `--cacert` acceptance of a test CA; `--compressed` gzip/br/zstd round-trips. + +### wget +GNU Wget has no mbedTLS backend, so give it a real TLS backend against the same +mbedTLS + CA bundle (its SSL abstraction is just 4 functions in `src/ssl.h`: +`ssl_init`, `ssl_cleanup`, `ssl_connect_wget`, `ssl_check_certificate`). +1. Add `wasi_ssl.c` implementing those 4 functions over **mbedTLS** (handshake + + `mbedtls_x509_crt_parse_file` on `/etc/ssl/certs/ca-certificates.crt`; + `--no-check-certificate`/`--ca-certificate`/`--ca-directory` map to real + mbedTLS verify config, matching Linux wget). Overlay dir mirrors curl's + (`software/wget/native/c/overlay/…`). +2. In `build-wget-upstream.sh`: keep `./configure --without-ssl` (so it doesn't + probe GnuTLS/OpenSSL), then post-configure patch `src/wget.h`'s `HAVE_SSL` + condition to add `|| defined HAVE_WASI_TLS`, pass `-DHAVE_WASI_TLS`, copy the + overlay, append a `wasi_ssl.o` compile/link rule to the generated `src/Makefile` + (the exact playbook the curl script uses). This lights up the `https` scheme, + all `--secure-protocol/--ca-certificate/--no-check-certificate` options, and + HSTS. +3. Enable **zlib** (`--with-zlib` via `ZLIB_CFLAGS/ZLIB_LIBS` at the existing + `build/zlib/libz.a`). Leave `--without-libpsl` (cosmetic — cookie public-suffix + only; wget's built-in heuristic covers single-URL HTTPS). +4. Tests: mirror curl's HTTPS cases (verify-fail on self-signed, + `--no-check-certificate` success, keep-alive), + gzip via `--compression`. + +### git — HTTPS smart-HTTP (clone/fetch/**push**) +Git's HTTP transport lives entirely in the **`git-remote-https` remote helper**, +which links **libcurl in-process** (git never shells out to a `curl` binary). The +current symlink `git-remote-https → git` is structurally broken (`git`'s +`cmd_main` dies "cannot handle remote-https as a builtin"). +1. **Produce a reusable libcurl** artifact (`make -C lib install` the overlaid, + mbedTLS-linked libcurl into a prefix under `toolchain/c/build/curl-upstream/ + install`). +2. In `toolchain/c/scripts/build-git-upstream.sh`: drop `NO_CURL=1`; add + `CURL_CFLAGS/CURL_LDFLAGS` pointing at that libcurl (defining them skips + `curl-config`); keep `NO_EXPAT=1`; build targets **`git git-remote-http`**; + wasm-opt both. (Expect a handful of small WASI compile fixes in + `http.c`/`remote-curl.c`, same class the curl script already makes; the + existing `git_compat.c` shims cover most.) +3. **Packaging:** install `git-remote-http` as a **real second command** (replace + the wrong symlink loop in `toolchain/c/Makefile`); in + `software/git/agentos-package.json` move `git-remote-http` into `commands` and + re-point `git-remote-https` → `git-remote-http` (keep `git-upload-pack`/ + `git-receive-pack`/`git-upload-archive` → `git`; those *are* builtins). +4. **Push** is the same libcurl POST plumbing as fetch (`remote-curl.c` spawns + `git send-pack` and streams the pack via chunked `Transfer-Encoding` + + `Expect: 100-continue` — all inside libcurl, no new host surface). HTTP Basic + auth (token/`user:pass` URLs, `GIT_ASKPASS`) works. (`credential-cache` + unavailable — `NO_UNIX_SOCKETS`; NTLM/Negotiate unsupported — irrelevant for + GitHub/GitLab.) +5. **Nothing new on the host side** — TLS/DNS/TCP/spawn/permission-tiers are all + in place. Spawn already resolves `git-remote-https` by name on the guest PATH + via the `proc_spawn` broker (proven: `git clone` already spawns + `git upload-pack` locally). Permission tiers already grant `git-remote-http(s)` + tier `full`. +6. **ssh transport (secondary):** `git@host:` spawns `ssh` — needs an in-guest + `ssh` command (its own OpenSSH port, needs a real in-guest crypto stack, unlike + host-brokered TLS). HTTPS stays primary; ssh is a separate future project. +7. Tests: flip `hasGitHttpHelper` in `software/git/test/git.test.ts` (the + smart-HTTP clone/fetch suite is already written but skipped); add a push case + (small + >1 MiB chunked POST). + +## Compression (curl) + +- **zlib 1.3.1 is already vendored** and built against this sysroot (git links it): + `--with-zlib` via `CPPFLAGS/LDFLAGS` at `build/zlib`. +- **brotli** — decoder (`common`+`dec`) is dependency-free portable C; add a + Makefile fetch/build target mirroring zlib; `--with-brotli` (decode only). +- **zstd** — already compiles under this sysroot inside the duckdb build + (`toolchain/c/build/duckdb-cmake/third_party/zstd/`); build libzstd + single-threaded; `--with-zstd`. +- Out of scope here: nghttp2 (HTTP/2), libpsl. + +## Refuted / dead-end theses (documented so we don't re-try them) + +- **Host-brokered TLS for parity** — works but never reaches native semantics + (host cert store, ignored `--cacert`, wrong exit codes, no `-v` chain, shim + treadmill). Rejected as the parity path; kept only for the JS runtime. +- **OpenSSL in-guest for the original curl-only milestone** — its + config/ENGINE/provider machinery expects `dlopen`, threads, and platform RNG; + wasip1 builds were more work than curl needed, while mbedTLS already satisfied + that milestone. **Superseded for final convergence:** Node 24 requires the + OpenSSL 3.5 contract, the sysroot is ours to extend, and the M0 OpenSSL build + + handshake spike now owns proving or refuting the remaining port gaps. Do not + reuse this historical rationale to introduce a second backend. +- **GnuTLS in-guest** (wget's default) — drags nettle → GMP (asm) → libtasn1 → + p11-kit (dlopen). Non-starter on WASI. +- **rustls + aws-lc-rs in guest** — aws-lc-rs is C+asm, does **not** build for + wasm32-wasip1. `ring` support is unofficial/spotty. Mixing a Rust staticlib into + the clang-LTO curl link is fiddly. No fidelity gain over mbedTLS. +- **rustls-native-certs (as the trust story)** — needs an OS cert store; the guest + has none, and host-side it depends on the host machine's store (non-hermetic). +- **BearSSL** — curl **removed** BearSSL support in 8.12.0 and it lacks TLS 1.3. + Dead end for any curl upgrade. +- **Reusing curl's `wasi_tls.c` for wget** — impossible; it's written against + libcurl-internal `Curl_cfilter` APIs. The shareable unit is the **host import + contract / the mbedTLS backend**, not the C file. +- **git: symlink `git-remote-https` → git binary** (current state) — git's + `cmd_main` hard-dies for a non-builtin `git-*` argv[0]. Structurally impossible. +- **git: spawn the `curl` CLI from a shell helper** — the remote helper is a + long-lived, stateful, bidirectional protocol-v2 program (auth retry, gzip, + full-duplex chunked POST). The CLI can't provide it. +- **git: gitoxide / libgit2 / reimplement smart protocol in-process** — not real + upstream git; diverges on CLI/config/hooks; would need its own WASI net port + anyway. Rejected. +- **git: dumb-HTTP only** — GitHub/GitLab/Gitea serve smart-HTTP only for fetch + and never support dumb push. Non-starter (dumb fetch rides along free). +- **Anything threaded** — the runner is single-threaded wasip1; all TLS/compression + libs build single-threaded (mbedTLS/zstd support it). + +## Work breakdown & sequencing + +1. **mbedTLS 3.6** target in `toolchain/c/Makefile` (shared by curl, wget, git's + libcurl). ← foundational, do first. +2. **CA bundle**: `ca-certificates` payload at `/etc/ssl/certs/ca-certificates.crt` + + VM bootstrap seeding + sidecar reads the VM bundle (not the host store). ← + foundational. +3. **curl**: mbedTLS + zlib/brotli/zstd + CA bundle; retire `wasi_tls.c`; tests. +4. **libcurl install artifact** (from step 3's build) for git to link. +5. **git**: build-with-curl, real `git-remote-http` helper, packaging, push; tests. +6. **wget**: `wasi_ssl.c` mbedTLS backend + zlib + CA bundle; tests. + +Steps 3/5/6 all sit on 1+2. One jj rev per tool. Real e2e tests against a TLS +server (verify-fail on self-signed, `--cacert` success, real clone/fetch/push) are +the deliverable — no host-store shortcuts. diff --git a/docs-internal/registry-networking-handoff.md b/docs-internal/registry-networking-handoff.md new file mode 100644 index 0000000000..ba734648ac --- /dev/null +++ b/docs-internal/registry-networking-handoff.md @@ -0,0 +1,276 @@ +# Handoff — Registry networking (TLS) + ssh + /proc + remaining work + +Status: execution handoff (state reconciled) · Last updated: 2026-07-09 (PST) · Read this first, then +`docs-internal/networking-parity-spec.md` and `docs-internal/registry-parity-worklist.md`. + +This picks up an in-flight effort: give curl/wget/git **real HTTPS with native +Linux semantics**, make the **codex build reproducible**, and then continue into +**ssh**, **/proc**, and the Node stdlib migration. Current proof state: +- curl/wget/git HTTPS is **done and green** on in-guest mbedTLS. +- ssh is built and its default suite is **8/8 green**, including real + git-over-SSH clone + push through the runtime's synthetic-pipe/host-net poll + path (§4). +- the kernel procfs runtime surface and real procps-ng/psmisc consumers are + complete. `ps`, `pgrep`, `pkill`, `pstree`, `killall`, and `prtstat` pass live + guest-process e2e against the pinned upstream builds (§5). +- the Node replacement spec is ready in the separate `node-stdlib` workspace; + its final backend decision is real OpenSSL 3.5.5 wasm, not an invented + mbedTLS-backed OpenSSL façade (§5). + +--- + +## 0. The two original acceptance criteria (BOTH MET) + +1. **Forklift cleanly pushed** — the whole stack is on GitHub as PRs **#1661–#1717** + (52 revs: all real-tool replacements, codex reproducible build, the networking + spec, mbedTLS+CA, curl/wget/git TLS, and the VFS-deletion fix). +2. **curl + wget + git have real HTTPS with native Linux semantics** — verified: + - **curl 32/32** — `libcurl/8.11.1 mbedTLS/3.6.2 zlib/1.3.1 brotli/1.1.0 zstd/1.5.6`; cert-fail → **exit 60**, `--cacert`, `--compressed` (gzip/br/zstd). + - **wget 12/12** — `+ssl/mbedtls`; cert-fail → **exit 5**, `--ca-certificate`, gzip. + - **git 25/25 (smart-HTTP 8/8)** — HTTPS clone/fetch/**push (small & >1 MiB)**/cert-verify/`sslVerify`/`sslCAInfo`/clone-after-push all pass. + +Everything after this (ssh, /proc, etc.) is **additional** work the user directed. + +--- + +## 1. Architecture you must understand + +**Sockets/DNS/HTTP were already real** (BSD sockets over the `host_net` WASM-import +bridge — `toolchain/std-patches/wasi-libc/0008-sockets.patch`, +`0023-host-net-*`; runner `crates/execution/assets/runners/wasm-runner.mjs` +forwards to the sidecar socket table). The only gaps were **TLS + decompression**. + +**The implemented TLS model for the original registry milestone (do NOT revert +to host-brokered):** in-guest **mbedTLS 3.6** ++ a Debian-shaped **CA bundle at `/etc/ssl/certs/ca-certificates.crt`**. Real +verification happens *in the tool's own code* → correct exit codes, `--cacert`, +`-v` cert chain. The old host-brokered `wasi_tls.c` shim (host cert store, exit 35, +ignored `--cacert`) is **deleted** and documented as refuted-for-parity in +`networking-parity-spec.md`. + +**Final convergence target (decided after the Node review):** preserve the +in-guest model and VM CA bundle, but migrate TLS/crypto consumers forward to +one real OpenSSL 3.5.5 wasm build owned by `toolchain/c`. Node 24.15.0 requires +OpenSSL 3 EVP/provider/error behavior; mbedTLS 3.6 does not ship the proposed +OpenSSL 3 compatibility surface. The existing mbedTLS revisions remain valid +historical proof and are not rewritten. The convergence revs rebuild +curl/libcurl, wget, git, and OpenSSH against the shared OpenSSL archives, remove +`wasi_ssl.c` and the host `rustls-native-certs` path, and prove the shared CA +behavior cross-runtime (§5 and the Node spec §7.4). + +**ssh is different from TLS:** TLS is a *library linked in-process*; **ssh is a +standalone command other tools spawn as a subprocess** (git execs `/opt/agentos/ +bin/ssh`). There is no "link ssh into git." + +**Foundation artifacts (built, in the stack):** +- `make -C toolchain/c mbedtls` → `toolchain/c/build/mbedtls/{libmbedtls,libmbedx509,libmbedcrypto}.a` (TLS 1.3, `getentropy` entropy, `mbedtls_x509_crt_parse_file`). +- `make -C toolchain/c ca-certificates` → Mozilla bundle; VM bootstrap + (`crates/native-sidecar/src/vm.rs`) seeds it at `/etc/ssl/certs/ca-certificates.crt` (+ `/etc/ssl/cert.pem` symlink). +- zlib already built (`toolchain/c/build/zlib/libz.a`); brotli + zstd added for curl. + +--- + +## 2. The bug cascade we fixed (so you don't re-discover them) + +Real e2e (not "build verified") is what caught these. All fixed + tested: +- **git `fork()` in fetch-pack sideband demux** (no fork on WASI) → synchronous + spool-to-tempfile + index-pack (`toolchain/c/patches/git/0002-*.patch`). +- **kernel fd-CLOEXEC inheritance** — a grandchild inherited its own stdin-pipe + writer → no EOF → hung (`crates/kernel/src/fd_table.rs`, `execution.rs`). +- **spawn cwd** resolved PWD-first → stale cwd (`toolchain/std-patches/wasi-libc/0012-posix-spawn-cwd.patch`). +- **runner dropped `SOCK_NONBLOCK`** → curl `recv()` blocked → deadlock on uploads + >16 KiB (one TLS record); + socket fds above `FD_SETSIZE`. Fixed in + `wasm-runner.mjs` (parse `SOCK_NONBLOCK`, `fd_fdstat_get/set_flags` for sockets, + fd base 4096→600). **This same fix is why ssh's select loop works.** +- **stack overflow in `recv_sideband`** (64 KiB buffer on git's default 64 KiB wasm + stack) → link git with `-Wl,-z,stack-size=8388608 -Wl,--stack-first`. +- **sidecar truncated >64 KiB child stdin** (ignored `fd_write` partial writes) → + non-blocking stdin + a 64 MiB backlog queue (`execution.rs`, `state.rs`). +- **VFS deletions never propagated** (systemic): `OverlayFs::remove_file` followed + symlinks instead of lstat (git's dangling symlink probe → `.git` never emptied); + `rmdir` mapped `ENOTEMPTY`→`EIO`; shadow→kernel sync was additive-only. Fixed with + lstat removal + `ENOTEMPTY` mapping + an inventory-diff deletion-reconcile + (`crates/vfs/src/posix/overlay_fs.rs`, `wasi-module.js`, `execution.rs`). +- **curl `-o/-O` "failures" were a red herring** — stub `cat`/`wc` test-programs + shadowing real coreutils, now renamed `c-*`. + +--- + +## 3. State of the jj stack (CRITICAL — read carefully) + +The whole thing is one linear jj stack managed with **forklift** (`forklift submit -y` +rebases onto trunk + pushes each rev as a stacked PR). `reg-tests` is the workspace. + +**Pushed (PRs #1661–#1717):** all tool replacements, codex reproducible build +(#1708), networking spec (#1709), mbedTLS+CA (#1710), curl (#1711), git (#1712), +wget (#1713), git-fork-fix (#1714), SOCK_NONBLOCK (#1715), index-pack/stack/stdin +(#1716), VFS-deletion (#1717). + +**Committed locally but NOT yet forklifted (do this first):** +- `yvxuokln` — `docs(CLAUDE): require citing authoritative specs at the implementation site` +- `opqnkltx` — `feat(ssh): real OpenSSH client …` (**built + 7/7 e2e; see §4**) +- `lnqvznlo` — this handoff doc. + +**To push them:** `cd` to the workspace, ensure `@` is at the top rev, then +`forklift submit -y`. If forklift reports many conflicts on rebase, **they're +almost always just 2 workflow files** (`.github/workflows/ci.yml` + `publish.yaml`) +conflicting from the flatten vs main's ci changes — resolve at rev `wktwwvso` +(keep the flatten's `toolchain` paths + main's `cache-workspace-crates: true`) and +all descendant conflicts collapse. (We hit "47 conflicts" that were really these 2.) + +**jj discipline (shared workspace, ~100 workspaces + concurrent agents):** one +fix per rev; `jj describe` each; never `jj edit`/`rebase`/`abandon` to *inspect*; +verify `jj diff -r @ --summary` has **no build-artifact leakage** +(`toolchain/c/build/**`, `**/target`, `.cache` extracted trees, `node_modules`, +`.a`) before describing — those are gitignored, only source/patch/test/vendored-cmd +files belong in a rev. + +--- + +## 4. ssh — BUILT + VERIFIED, including git-over-SSH + +Rev `opqnkltx` (`feat(ssh)…`). **Real OpenSSH portable 10.4p1, `--without-openssl`**, +built for wasm32-wasip1 (845 KB, at `software/ssh/bin/ssh`, `packages/runtime-core/ +commands/ssh`, `toolchain/c/build/ssh`). Files: `toolchain/c/scripts/build-ssh- +upstream.sh`, `toolchain/c/Makefile` target, `toolchain/std-patches/wasi-libc/ +0029-openssh-compat-header-surface.patch`, `toolchain/std-patches/wasi-libc- +overrides/openssh_compat.c` (the no-op `closefrom` + ENOSYS `socketpair`), +`software/ssh/*`, `wasm-runner.mjs` setsockopt polish. + +**Verified — ssh e2e 8/8** +(`~/progress/agent-os/2026-07-09-registry-networking/2026-07-09T17-00-20-0700-ssh-full-suite-pass.log`): +the original seven direct cases plus real git-over-SSH clone + push. The runtime +fix teaches mixed host-net/kernel polling to probe kernel stdio without starving +the socket, routes duplicated stdio aliases to the kernel fd, and emits the owned +sysroot's actual poll exceptional-bit ABI. The in-test SSH server also preserves +the channel until it sends the child exit status. **git 25/25 unregressed** +(`~/progress/agent-os/2026-07-09-registry-networking/2026-07-09T17-01-10-0700-git-full-suite-pass.log`). + +**Build/test commands:** +``` +make -C toolchain/c build/ssh # deps: mbedtls not needed (no openssl); zlib + patched sysroot +pnpm --dir software/ssh test # with AGENTOS_SIDECAR_BIN pinned (see §6) +``` + +--- + +## 5. Remaining queue (normative order) + +### 5.1 Finish the reg-tests stack + +1. **DONE — close git-over-SSH:** the runtime synthetic-pipe + host-net poll/flush + bug is fixed, `AGENTOS_SSH_GIT_E2E` is removed, and real clone + push pass by + default with a pinned `AGENTOS_SIDECAR_BIN` (§4). +2. **DONE — finish `/proc` from the existing partial implementation:** the + process-table-backed design now provides `/proc//comm`, `/proc/stat`, + typed directory entries, all 52 `stat` fields in order, and Linux-compatible + task-name escaping. Emitters cite `proc(5)`, Linux `fs/proc/array.c`, and + procps-ng; a Linux 6.1 fixture and regeneration script cover raw `comm` edge + cases. Full kernel crate proof: + `~/progress/agent-os/2026-07-09-registry-networking/2026-07-09T17-18-40-0700-proc-full-kernel-suite.log`. +3. **DONE — prove and ship the consumers:** pinned procps-ng 4.0.6 and psmisc + 23.7 build against the owned sysroot and ship `ps`, `pgrep`, `pkill`, + `pstree`, `killall`, and `prtstat`. A captured Linux fixture has a regeneration + script. Live VM e2e proves process inspection and signaling through every + shipped command: + `~/progress/agent-os/2026-07-09-registry-networking/2026-07-09T18-09-17-0700-proc-consumer-live-e2e-program-name.log`. +4. Keep the handoff/worklist status current after every rev, audit for build + artifact leakage, and `forklift submit -y` each clean change before leaving + this workspace. + +### 5.2 Drive the Node-stdlib/wasm migration in its own workspace + +Spec: `~/.herdr/workspaces/agent-os/node-stdlib/docs-internal/ +node-stdlib-replacement-spec.md`. Do not implement it in this reg-tests working +copy. The Node program begins after §5.1 is green and forklifted. + +**Final crypto/TLS decision:** build the exact OpenSSL 3.5.5 source bundled by +Node v24.15.0 once in `toolchain/c`, producing content-addressed wasm32 +`libcrypto.a`/`libssl.a` archives. Node adapters and registry consumers link +those same archive/source hashes. Migrate curl/libcurl, wget, git, and OpenSSH +forward from the proven mbedTLS/no-OpenSSL state; do not rewrite the published +history and do not retain a second TLS backend or `wasi_ssl.c`-style adapter. + +**Required unification proof:** +1. Captured build/link manifests show every consumer linking the same + OpenSSL archive/source hashes (each final static wasm may embed its copy). +2. One cross-runtime e2e drives curl and Node `https.get`/`tls.connect` against + the same trusted and self-signed servers: trusted succeeds, self-signed + fails in the same class, and `--cacert`/`NODE_EXTRA_CA_CERTS` work against + the same `/etc/ssl/certs/ca-certificates.crt`. +3. The host `rustls-native-certs`/TLS-upgrade path, mbedTLS TLS backend, + per-tool adapters, and RustCrypto bridge crypto are retired when the shared + OpenSSL path becomes the only one. +4. OpenSSH is rebuilt with the shared OpenSSL for its normal full-crypto + configuration. OpenSSH-owned protocol primitives are not a second TLS + backend. + +**Explicitly out of scope:** wasm threads and the disabled browser runtime. +If real OpenSSL hits a concrete wall, exhaust sysroot and upstream OpenSSL +patches first, save the failing e2e/build evidence, and update both specs with +the refuted thesis before requesting a different architecture. + +**Doc-citation convention (now in CLAUDE.md, enforce it):** every format emitter / +protocol handler cites its authoritative reference **in a code comment at the site** +— man page, kernel source path, RFC, and/or the consumer's parser. For /proc: +`proc(5)`, `fs/proc/array.c`, procps `readproc.c`. For ssh patches: RFC 4251–4254. + +--- + +## 6. Operational knowledge you WILL need + +**The shared workspace is churned in real time** by other sessions — +`node_modules`, `target/`, and the cargo `cc-1.2.66` crate (`src/target/`) get wiped +mid-run, breaking vitest (`@vitest/utils`/`tinypool` not found) and racing the +sidecar `cargo build`. **This is not your code.** Workarounds that worked: +- Before any e2e: `pnpm install --frozen-lockfile`; if `cc` is broken, re-extract its `.crate`. +- **Pin a prebuilt sidecar** so tests don't trigger a racing `cargo build`: + `AGENTOS_SIDECAR_BIN=`. Recent good binaries in the scratchpad: + `/tmp/claude-1000/-home-nathan--herdr-workspaces-agent-os-reg-tests/91e4450c-fb78-4e8b-a128-eff26631dc40/scratchpad/{sidecar-vfs-deletion-fix,agentos-native-sidecar}`. + If you change Rust, rebuild the sidecar once (`cargo build -p agentos-native-sidecar`) + to a stable copy and pin THAT. +- The git HTTPS/ssh suites stand up a loopback server (git: `git http-backend`; + ssh: the `ssh2` npm package) and use `createGitKernelWithNet` — mirror that pattern. + +**Build gotcha:** wasi-sdk 25's clang auto-runs `wasm-opt` from PATH after link, +stripping the wasm name section (kills symbolized traps). Build with +`PATH=/usr/bin:/bin` when you need to debug a wasm crash address. + +**codex reproducible build:** `make -C toolchain codex` (CODEX_REPO unset → clones +`rivet-dev/codex@` + injects committed patches). Verified the +hard crates compile; **one open blocker for the full 29 MB artifact**: `rmcp` 0.15 +oauth transport wants a `Send` future the reqwest-shim doesn't provide (scoped fix: +make the shim future `Send` or cfg-gate rmcp oauth off wasi). + +--- + +## 7. Key file map + +- TLS/CA foundation today: `toolchain/c/Makefile` + (mbedtls/brotli/zstd/ca-certificates targets), + `crates/native-sidecar/src/vm.rs` (CA seeding), `build.rs`. Final shared + OpenSSL ownership/layout is normative in the Node replacement spec §9.4. +- curl: `toolchain/c/scripts/build-curl-upstream.sh`, `software/curl/{test,native}`. +- wget: `toolchain/c/scripts/build-wget-upstream.sh`, `software/wget/native/c/overlay/src/wasi_ssl.c` (mbedTLS backend), `software/wget/test`. +- git: `toolchain/c/scripts/build-git-upstream.sh`, `toolchain/c/patches/git/000{1,2}-*.patch`, `software/git/{test,agentos-package.json}`. +- ssh: `toolchain/c/scripts/build-ssh-upstream.sh`, `toolchain/std-patches/wasi-libc-overrides/openssh_compat.c`, `toolchain/std-patches/wasi-libc/0029-*.patch`, `software/ssh/*`. +- Runtime fixes: `crates/execution/assets/runners/wasm-runner.mjs` (sockets/stdio/spawn), `crates/native-sidecar/src/execution.rs` (stdin backlog, TLS), `crates/kernel/src/{fd_table,process_table,kernel}.rs`, `crates/vfs/src/posix/overlay_fs.rs`. +- Procfs baseline: `crates/kernel/src/kernel.rs`, + `crates/kernel/src/process_table.rs`, `crates/kernel/tests/{identity,api_surface}.rs`. +- Specs/state: `docs-internal/networking-parity-spec.md`, `docs-internal/registry-parity-worklist.md`, this file. +- Friction log (root causes + repros): `~/.agents/friction/agentos.md`. +- Proof logs: `~/progress/agent-os/2026-07-0{8,9}-*/`. + +--- + +## 8. Immediate next steps for the picking-up agent + +1. Forklift the completed procps-ng/psmisc revision. +2. Switch to the existing `node-stdlib` workspace without moving `reg-tests`'s + `@`; execute the Node spec M0→M5 with the shared real-OpenSSL decision. +3. Optional after the ordered work: git credential-cache note and the codex + `rmcp` OAuth `Send` blocker. + +Everything real-tool must stay **real upstream, patch the sysroot not the app** +(CLAUDE.md → Software Build). Prove every claim with a **real e2e**, not a build +check — that discipline is what caught every bug above. diff --git a/docs-internal/registry-parity-worklist.md b/docs-internal/registry-parity-worklist.md index 02ac4b61a9..ea71146b09 100644 --- a/docs-internal/registry-parity-worklist.md +++ b/docs-internal/registry-parity-worklist.md @@ -1,6 +1,6 @@ # Registry Linux-Parity Worklist -Status: worklist · Owner: registry · Last updated: 2026-07-08 +Status: worklist · Owner: registry · Last updated: 2026-07-09 (PST) ## Goal (hand this to the driver agent) @@ -119,6 +119,15 @@ external build (tracked separately in #9). **Objective:** replace each ❌ with a real/established implementation built to `wasm32-wasip1` and patched only where WASI forces it. The ✅ rows stay. +**✅ Original networking gap is closed; final backend convergence remains.** +curl/wget/git now perform real in-guest HTTPS with mbedTLS, the VM CA bundle, +decompression, and smart-HTTP clone/fetch/push. The Node-stdlib program later +selected one real OpenSSL 3.5.5 wasm build as the final backend for Node and the +registry consumers. Preserve the green mbedTLS proof while migrating forward; +do not reintroduce host-brokered TLS or retain mbedTLS as a second final backend. +Current status, proof, and normative order are in +`docs-internal/registry-networking-handoff.md`. + **Approach:** one command at a time, one jj rev each: swap our custom code for the established source (fetched + pinned like sqlite/duckdb), wire into the toolchain, patch for WASI, prove parity with real e2e tests. @@ -805,7 +814,7 @@ breadth; (2) `git` is **rare inside agent turns** (harnesses extract the diff out-of-band) but stays essential; (3) a **long tail of project-specific CLIs** (`dvc`, `sqlglot`, `sanic`, …) comes from pip/npm install, not the registry. -**Requested (add):** ssh 🟡, rsync 🟡, tmux/screen 🟡 (PTY — session persistence), +**Requested (add):** ssh ✅, rsync 🟡, tmux/screen 🟡 (PTY — session persistence), gpg 🟡, ffmpeg 🟡 (media transcode — heavy but headless), jj 🟢, dig 🟡, nslookup 🟡, less ⭐🟡 (pager), openssl ⭐🟡 (TLS/certs/keys/hashing). tail/head/cat are already in coreutils — confirm present. @@ -814,7 +823,7 @@ tail/head/cat are already in coreutils — confirm present. big C runtime but real; 563 uses in history), miller `mlr` 🟢 (CSV/JSON), xmlstarlet 🟢, pcre2grep 🟢. (jq/yq/sed/awk/grep/head/tail already covered.) -**Networking (host TCP/DNS bridge only):** openssl ⭐🟡, ssh 🟡, nc/netcat 🟡 +**Networking (host TCP/DNS bridge only):** openssl ⭐🟡, ssh ✅, nc/netcat 🟡 (TCP/UDP), socat 🟡, whois 🟢, dig/nslookup 🟡, redis-cli / psql client 🟡, aria2 🟡 (C++ downloader), sshpass 🟢 (ssh password helper). @@ -832,21 +841,22 @@ above). **Session:** tmux/screen 🟡 (PTY). -**Process management (add — real procps-ng + psmisc, C; ps/pkill/pgrep ≈ 3K uses):** -- **Need the `/proc` prerequisite (below):** ps, pgrep, pkill, pidof, pstree, - killall, uptime, free, vmstat, w, pwdx, pmap 🟡. +**Process management (real procps-ng + psmisc, C; ps/pkill/pgrep ≈ 3K uses):** +- **Shipped:** procps-ng 4.0.6 `ps`, `pgrep`, `pkill`; psmisc 23.7 `pstree`, + `killall`, `prtstat` ✅. +- **Still available to add from these suites:** pidof, uptime, free, vmstat, w, + pwdx, pmap 🟡. - **Signal-only — already work via the kernel (no /proc):** kill, killall-by-PID. (kill, sleep, timeout, env, nohup, nproc, nice/renice are coreutils — confirm they're shipped via uutils rather than re-adding.) -**⚙️ Runtime prerequisite — implement `/proc` (process-table-backed):** procps -reads `/proc//{stat,cmdline,status,comm}` and enumerates `/proc//`. The -**kernel already owns the process table** (`crates/kernel/src/process_table.rs`), -so expose a read-only procfs view of it to the guest (per-PID stat/cmdline/status -+ directory enumeration). Scope it minimal — just the fields procps parses, backed -by the existing process table, not a full Linux procfs. Unlocks the whole -ps/pkill/pgrep family (and top/htop later if ever wanted). **This is a runtime/VFS -item, do it before the procps packages.** +**✅ `/proc` runtime and first real consumers complete.** The +process-table-backed procfs now includes per-PID `stat`, `status`, `comm`, +`cmdline`, `environ`, `cwd`, and `fd`, plus `/proc/stat` and the prior system +files. The emitters cite `proc(5)`, Linux `fs/proc/array.c`, and procps-ng; a +captured Linux 6.1 fixture proves raw/escaped task-name behavior. Unskipped live +VM e2e covers all six shipped procps-ng/psmisc commands, including name-based +signals terminating real guest processes. **Excluded — not worth it / not possible here:** - **TUI / visual-only:** gitui, lazygit, eza, dust, ncdu, bat, delta, broot, k9s, diff --git a/packages/agentos/src/generated/actor-actions.generated.ts b/packages/agentos/src/generated/actor-actions.generated.ts index 9671fd7470..a66ee3822d 100644 --- a/packages/agentos/src/generated/actor-actions.generated.ts +++ b/packages/agentos/src/generated/actor-actions.generated.ts @@ -1,6 +1,7 @@ // @generated by agentos-actor-plugin. Do not edit. -// This file is committed so package builds do not need to compile the native -// plugin just to regenerate TypeScript action types. +// This file is generated locally and is intentionally not committed to Git. +// Regenerated by the agentos-actor-plugin build script; build the crate +// (e.g. `cargo build -p agentos-actor-plugin`) to refresh it. import type { ExecResult, PermissionReply, diff --git a/packages/runtime-core/commands/git b/packages/runtime-core/commands/git index 869f2fae78..0a577332dc 100644 Binary files a/packages/runtime-core/commands/git and b/packages/runtime-core/commands/git differ diff --git a/packages/runtime-core/commands/git-receive-pack b/packages/runtime-core/commands/git-receive-pack index 869f2fae78..0a577332dc 100644 Binary files a/packages/runtime-core/commands/git-receive-pack and b/packages/runtime-core/commands/git-receive-pack differ diff --git a/packages/runtime-core/commands/git-remote-http b/packages/runtime-core/commands/git-remote-http index 869f2fae78..8c78d028ca 100644 Binary files a/packages/runtime-core/commands/git-remote-http and b/packages/runtime-core/commands/git-remote-http differ diff --git a/packages/runtime-core/commands/git-remote-https b/packages/runtime-core/commands/git-remote-https index 869f2fae78..8c78d028ca 100644 Binary files a/packages/runtime-core/commands/git-remote-https and b/packages/runtime-core/commands/git-remote-https differ diff --git a/packages/runtime-core/commands/git-upload-archive b/packages/runtime-core/commands/git-upload-archive index 869f2fae78..0a577332dc 100644 Binary files a/packages/runtime-core/commands/git-upload-archive and b/packages/runtime-core/commands/git-upload-archive differ diff --git a/packages/runtime-core/commands/git-upload-pack b/packages/runtime-core/commands/git-upload-pack index 869f2fae78..0a577332dc 100644 Binary files a/packages/runtime-core/commands/git-upload-pack and b/packages/runtime-core/commands/git-upload-pack differ diff --git a/packages/runtime-core/commands/killall b/packages/runtime-core/commands/killall new file mode 100644 index 0000000000..b999eff03d Binary files /dev/null and b/packages/runtime-core/commands/killall differ diff --git a/packages/runtime-core/commands/pgrep b/packages/runtime-core/commands/pgrep new file mode 100644 index 0000000000..71cab40294 Binary files /dev/null and b/packages/runtime-core/commands/pgrep differ diff --git a/packages/runtime-core/commands/pkill b/packages/runtime-core/commands/pkill new file mode 100644 index 0000000000..71cab40294 Binary files /dev/null and b/packages/runtime-core/commands/pkill differ diff --git a/packages/runtime-core/commands/prtstat b/packages/runtime-core/commands/prtstat new file mode 100644 index 0000000000..bc333f5a3a Binary files /dev/null and b/packages/runtime-core/commands/prtstat differ diff --git a/packages/runtime-core/commands/ps b/packages/runtime-core/commands/ps new file mode 100644 index 0000000000..a0158fffef Binary files /dev/null and b/packages/runtime-core/commands/ps differ diff --git a/packages/runtime-core/commands/pstree b/packages/runtime-core/commands/pstree new file mode 100644 index 0000000000..fb9290299b Binary files /dev/null and b/packages/runtime-core/commands/pstree differ diff --git a/packages/runtime-core/commands/ssh b/packages/runtime-core/commands/ssh new file mode 100644 index 0000000000..30ed5ab00e Binary files /dev/null and b/packages/runtime-core/commands/ssh differ diff --git a/packages/runtime-core/tests/integration/vfs-consistency.test.ts b/packages/runtime-core/tests/integration/vfs-consistency.test.ts index 19939a6ff8..128ee05196 100644 --- a/packages/runtime-core/tests/integration/vfs-consistency.test.ts +++ b/packages/runtime-core/tests/integration/vfs-consistency.test.ts @@ -109,4 +109,67 @@ describeIf(!skipReason, 'cross-runtime VFS consistency', () => { ); expect(nodeResult.exitCode).not.toBe(0); }); + + // Guest deletions must propagate into the kernel VFS instead of being + // resurrected by the (otherwise additive) shadow->kernel sync. This is the + // failure mode behind git's failed-clone junk surviving `remove_junk`. + it('guest rm -r removes the tree from the kernel VFS', async () => { + ctx = await createIntegrationKernel(); + + const create = await ctx.kernel.exec( + 'sh -c "mkdir -p /tmp/doomed/nested && echo payload > /tmp/doomed/nested/file.txt"', + ); + expect(create.exitCode).toBe(0); + expect(await ctx.vfs.exists('/tmp/doomed/nested/file.txt')).toBe(true); + + const remove = await ctx.kernel.exec('rm -r /tmp/doomed'); + expect(remove.exitCode).toBe(0); + + expect(await ctx.vfs.exists('/tmp/doomed/nested/file.txt')).toBe(false); + expect(await ctx.vfs.exists('/tmp/doomed/nested')).toBe(false); + expect(await ctx.vfs.exists('/tmp/doomed')).toBe(false); + + const ls = await ctx.kernel.exec('ls /tmp/doomed'); + expect(ls.exitCode).not.toBe(0); + }); + + it('guest rmdir on an empty directory succeeds and propagates', async () => { + ctx = await createIntegrationKernel(); + + const mkdir = await ctx.kernel.exec('mkdir /tmp/empty-dir'); + expect(mkdir.exitCode).toBe(0); + expect(await ctx.vfs.exists('/tmp/empty-dir')).toBe(true); + + const rmdir = await ctx.kernel.exec('rmdir /tmp/empty-dir'); + expect(rmdir.exitCode, rmdir.stderr).toBe(0); + expect(await ctx.vfs.exists('/tmp/empty-dir')).toBe(false); + }); + + it('guest unlink removes a dangling symlink so its directory can be emptied', async () => { + // git's symlink-support probe: create `tXXXXXX -> testing` (dangling), + // lstat it, unlink it, then remove the directory. unlink(2) must not + // resolve the symlink leaf, and rmdir must not report EIO afterwards. + ctx = await createIntegrationKernel(); + + const probe = await ctx.kernel.exec( + 'sh -c "mkdir /tmp/probe-dir && ln -s missing-target /tmp/probe-dir/probe && rm /tmp/probe-dir/probe && rmdir /tmp/probe-dir"', + ); + expect(probe.exitCode, probe.stderr).toBe(0); + expect(await ctx.vfs.exists('/tmp/probe-dir')).toBe(false); + }); + + it('guest rmdir on a non-empty directory reports ENOTEMPTY, not EIO', async () => { + ctx = await createIntegrationKernel(); + + const setup = await ctx.kernel.exec( + 'sh -c "mkdir /tmp/full-dir && echo keep > /tmp/full-dir/keep.txt"', + ); + expect(setup.exitCode).toBe(0); + + const rmdir = await ctx.kernel.exec('rmdir /tmp/full-dir'); + expect(rmdir.exitCode).not.toBe(0); + expect(rmdir.stderr.toLowerCase()).toContain('not empty'); + expect(rmdir.stderr.toLowerCase()).not.toContain('i/o error'); + expect(await ctx.vfs.exists('/tmp/full-dir/keep.txt')).toBe(true); + }); }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b7c1814c38..8309001e48 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -3287,6 +3287,12 @@ importers: '@agentos-software/jq': specifier: workspace:* version: link:../jq + '@agentos-software/procps': + specifier: workspace:* + version: link:../procps + '@agentos-software/psmisc': + specifier: workspace:* + version: link:../psmisc '@agentos-software/ripgrep': specifier: workspace:* version: link:../ripgrep @@ -3585,6 +3591,48 @@ importers: specifier: ^2.1.9 version: 2.1.9(@types/node@22.19.15) + software/procps: + devDependencies: + '@agentos-software/manifest': + specifier: workspace:* + version: link:../../packages/manifest + '@agentos/test-harness': + specifier: workspace:* + version: link:../../test-harness + '@rivet-dev/agentos-toolchain': + specifier: workspace:* + version: link:../../packages/agentos-toolchain + '@types/node': + specifier: ^22.10.2 + version: 22.19.15 + typescript: + specifier: ^5.9.2 + version: 5.9.3 + vitest: + specifier: ^2.1.9 + version: 2.1.9(@types/node@22.19.15) + + software/psmisc: + devDependencies: + '@agentos-software/manifest': + specifier: workspace:* + version: link:../../packages/manifest + '@agentos/test-harness': + specifier: workspace:* + version: link:../../test-harness + '@rivet-dev/agentos-toolchain': + specifier: workspace:* + version: link:../../packages/agentos-toolchain + '@types/node': + specifier: ^22.10.2 + version: 22.19.15 + typescript: + specifier: ^5.9.2 + version: 5.9.3 + vitest: + specifier: ^2.1.9 + version: 2.1.9(@types/node@22.19.15) + software/ripgrep: devDependencies: '@agentos-software/manifest': @@ -3648,6 +3696,33 @@ importers: specifier: ^2.1.9 version: 2.1.9(@types/node@22.19.15) + software/ssh: + devDependencies: + '@agentos-software/manifest': + specifier: workspace:* + version: link:../../packages/manifest + '@agentos/test-harness': + specifier: workspace:* + version: link:../../test-harness + '@rivet-dev/agentos-toolchain': + specifier: workspace:* + version: link:../../packages/agentos-toolchain + '@types/node': + specifier: ^22.10.2 + version: 22.19.15 + '@types/ssh2': + specifier: ^1.15.1 + version: 1.15.5 + ssh2: + specifier: ^1.16.0 + version: 1.17.0 + typescript: + specifier: ^5.9.2 + version: 5.9.3 + vitest: + specifier: ^2.1.9 + version: 2.1.9(@types/node@22.19.15) + software/tar: devDependencies: '@agentos-software/manifest': @@ -6024,6 +6099,9 @@ packages: '@types/semver@7.7.1': resolution: {integrity: sha512-FmgJfu+MOcQ370SD0ev7EI8TlCAfKYU+B4m5T3yXc1CiRN94g/SZPtsCkk506aUDtlMnFZvasDwHHUcZUEaYuA==} + '@types/ssh2@1.15.5': + resolution: {integrity: sha512-N1ASjp/nXH3ovBHddRJpli4ozpk6UdDYIX4RJWFa9L1YKnzdhTlVmiGHm4DZnj/jLbqZpes4aeR30EFGQtvhQQ==} + '@types/uuid@10.0.0': resolution: {integrity: sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==} @@ -11925,6 +12003,10 @@ snapshots: '@types/semver@7.7.1': {} + '@types/ssh2@1.15.5': + dependencies: + '@types/node': 18.19.130 + '@types/uuid@10.0.0': {} '@types/yauzl@2.10.3': diff --git a/software/curl/native/c/overlay/lib/curl_setup.h b/software/curl/native/c/overlay/lib/curl_setup.h index 8edcf809eb..50b4479f23 100644 --- a/software/curl/native/c/overlay/lib/curl_setup.h +++ b/software/curl/native/c/overlay/lib/curl_setup.h @@ -742,7 +742,7 @@ #if defined(USE_GNUTLS) || defined(USE_OPENSSL) || defined(USE_MBEDTLS) || \ defined(USE_WOLFSSL) || defined(USE_SCHANNEL) || defined(USE_SECTRANSP) || \ - defined(USE_BEARSSL) || defined(USE_RUSTLS) || defined(USE_WASI_TLS) + defined(USE_BEARSSL) || defined(USE_RUSTLS) #define USE_SSL /* SSL support has been enabled */ #endif diff --git a/software/curl/native/c/overlay/lib/vtls/vtls.c b/software/curl/native/c/overlay/lib/vtls/vtls.c index 21b758f2e8..61b407ab22 100644 --- a/software/curl/native/c/overlay/lib/vtls/vtls.c +++ b/software/curl/native/c/overlay/lib/vtls/vtls.c @@ -64,7 +64,6 @@ #include "mbedtls.h" /* mbedTLS versions */ #include "bearssl.h" /* BearSSL versions */ #include "rustls.h" /* Rustls versions */ -#include "wasi_tls.h" /* WASI host TLS */ #include "slist.h" #include "sendf.h" @@ -1380,8 +1379,6 @@ static const struct Curl_ssl Curl_ssl_multi = { const struct Curl_ssl *Curl_ssl = #if defined(CURL_WITH_MULTI_SSL) &Curl_ssl_multi; -#elif defined(USE_WASI_TLS) - &Curl_ssl_wasi_tls; #elif defined(USE_WOLFSSL) &Curl_ssl_wolfssl; #elif defined(USE_GNUTLS) @@ -1426,9 +1423,6 @@ static const struct Curl_ssl *available_backends[] = { #endif #if defined(USE_RUSTLS) &Curl_ssl_rustls, -#endif -#if defined(USE_WASI_TLS) - &Curl_ssl_wasi_tls, #endif NULL }; diff --git a/software/curl/native/c/overlay/lib/vtls/wasi_tls.c b/software/curl/native/c/overlay/lib/vtls/wasi_tls.c deleted file mode 100644 index a6a59e1ee7..0000000000 --- a/software/curl/native/c/overlay/lib/vtls/wasi_tls.c +++ /dev/null @@ -1,187 +0,0 @@ -/* - * WASI TLS backend for libcurl - * - * Uses the host_net WASM import net_tls_connect() to upgrade a TCP - * socket to TLS. The host runtime (Node.js tls.connect()) handles all - * cryptographic operations transparently — after the upgrade, regular - * send()/recv() carry encrypted traffic. This backend therefore just - * calls the host import during connect and passes data through to the - * socket layer for send/recv. - */ - -#include "curl_setup.h" - -#ifdef USE_WASI_TLS - -#include - -#include "urldata.h" -#include "sendf.h" -#include "vtls.h" -#include "vtls_int.h" -#include "connect.h" -#include "select.h" -#include "curl_printf.h" - -/* The last #include files should be: */ -#include "curl_memory.h" -#include "memdebug.h" - -/* WASM import: upgrade a TCP socket FD to TLS. - * flags: 0 = verify peer certificate (default) - * 1 = skip peer certificate verification (-k / --insecure) - * Returns 0 on success, errno on failure. */ -__attribute__((import_module("host_net"), import_name("net_tls_connect"))) -extern int __wasi_net_tls_connect(int fd, const char *hostname_ptr, - int hostname_len, int flags); - -/* Per-connection backend data (minimal — host handles all TLS state) */ -struct wasi_tls_backend_data { - int dummy; /* struct must be non-empty */ -}; - -static size_t wasi_tls_version(char *buffer, size_t size) -{ - return msnprintf(buffer, size, "WASI-TLS/host"); -} - -static CURLcode wasi_tls_connect_blocking(struct Curl_cfilter *cf, - struct Curl_easy *data) -{ - struct ssl_connect_data *connssl = cf->ctx; - curl_socket_t sockfd = Curl_conn_cf_get_socket(cf, data); - const char *hostname = connssl->peer.hostname; - int flags = 0; - - /* Check if peer verification is disabled (curl -k / --insecure) */ - struct ssl_primary_config *conn_config = Curl_ssl_cf_get_primary_config(cf); - if(conn_config && !conn_config->verifypeer) { - flags = 1; - } - - int ret = __wasi_net_tls_connect((int)sockfd, hostname, - (int)strlen(hostname), flags); - if(ret != 0) { - failf(data, "WASI TLS: host TLS connect failed (errno %d)", ret); - return CURLE_SSL_CONNECT_ERROR; - } - - connssl->state = ssl_connection_complete; - return CURLE_OK; -} - -static CURLcode wasi_tls_connect_nonblocking(struct Curl_cfilter *cf, - struct Curl_easy *data, - bool *done) -{ - /* Our host TLS connect is synchronous — just call blocking version */ - CURLcode result = wasi_tls_connect_blocking(cf, data); - *done = (result == CURLE_OK); - return result; -} - -/* recv: pass through to the socket layer — host handles decryption */ -static ssize_t wasi_tls_recv(struct Curl_cfilter *cf, - struct Curl_easy *data, - char *buf, size_t len, CURLcode *err) -{ - return Curl_conn_cf_recv(cf->next, data, buf, len, err); -} - -/* send: pass through to the socket layer — host handles encryption */ -static ssize_t wasi_tls_send(struct Curl_cfilter *cf, - struct Curl_easy *data, - const void *buf, size_t len, CURLcode *err) -{ - return Curl_conn_cf_send(cf->next, data, buf, len, FALSE, err); -} - -static CURLcode wasi_tls_shutdown(struct Curl_cfilter *cf, - struct Curl_easy *data, - bool send_shutdown, bool *done) -{ - (void)cf; - (void)data; - (void)send_shutdown; - *done = TRUE; - return CURLE_OK; -} - -static bool wasi_tls_data_pending(struct Curl_cfilter *cf, - const struct Curl_easy *data) -{ - (void)cf; - (void)data; - return FALSE; -} - -/* data may be NULL */ -static CURLcode wasi_tls_random(struct Curl_easy *data, - unsigned char *entropy, size_t length) -{ - size_t offset = 0; - (void)data; - - while(offset < length) { - size_t chunk = length - offset; - if(chunk > 256) - chunk = 256; - - if(getentropy(entropy + offset, chunk) != 0) - return CURLE_FAILED_INIT; - - offset += chunk; - } - - return CURLE_OK; -} - -static void wasi_tls_close(struct Curl_cfilter *cf, struct Curl_easy *data) -{ - (void)cf; - (void)data; - /* Host-side TLS is cleaned up when the socket is closed */ -} - -static void *wasi_tls_get_internals(struct ssl_connect_data *connssl, - CURLINFO info) -{ - (void)connssl; - (void)info; - return NULL; -} - -const struct Curl_ssl Curl_ssl_wasi_tls = { - { CURLSSLBACKEND_NONE, "wasi-tls" }, /* info */ - - 0, /* supports — no special features */ - - sizeof(struct wasi_tls_backend_data), - - Curl_none_init, /* init */ - Curl_none_cleanup, /* cleanup */ - wasi_tls_version, /* version */ - Curl_none_check_cxn, /* check_cxn */ - wasi_tls_shutdown, /* shutdown */ - wasi_tls_data_pending, /* data_pending */ - wasi_tls_random, /* random */ - Curl_none_cert_status_request, /* cert_status_request */ - wasi_tls_connect_blocking, /* connect */ - wasi_tls_connect_nonblocking, /* connect_nonblocking */ - Curl_ssl_adjust_pollset, /* adjust_pollset */ - wasi_tls_get_internals, /* get_internals */ - wasi_tls_close, /* close */ - Curl_none_close_all, /* close_all */ - Curl_none_set_engine, /* set_engine */ - Curl_none_set_engine_default, /* set_engine_default */ - Curl_none_engines_list, /* engines_list */ - Curl_none_false_start, /* false_start */ - NULL, /* sha256sum */ - NULL, /* associate_connection */ - NULL, /* disassociate_connection */ - wasi_tls_recv, /* recv decrypted data */ - wasi_tls_send, /* send data to encrypt */ - NULL, /* get_channel_binding */ -}; - -#endif /* USE_WASI_TLS */ diff --git a/software/curl/native/c/overlay/lib/vtls/wasi_tls.h b/software/curl/native/c/overlay/lib/vtls/wasi_tls.h deleted file mode 100644 index d75fdf1d93..0000000000 --- a/software/curl/native/c/overlay/lib/vtls/wasi_tls.h +++ /dev/null @@ -1,18 +0,0 @@ -#ifndef HEADER_CURL_WASI_TLS_H -#define HEADER_CURL_WASI_TLS_H -/* - * WASI TLS backend for libcurl - * - * Delegates TLS to the host runtime via the host_net WASM import - * net_tls_connect(). After the host upgrades the socket, regular - * send()/recv() transparently carry encrypted traffic, so the - * backend's recv_plain/send_plain are simple pass-throughs. - */ - -#include "curl_setup.h" - -#ifdef USE_WASI_TLS -extern const struct Curl_ssl Curl_ssl_wasi_tls; -#endif - -#endif /* HEADER_CURL_WASI_TLS_H */ diff --git a/software/curl/test/curl.test.ts b/software/curl/test/curl.test.ts index 1097b12022..d79395baf1 100644 --- a/software/curl/test/curl.test.ts +++ b/software/curl/test/curl.test.ts @@ -38,9 +38,17 @@ import { type Server as TcpServer, } from 'node:net'; import { execSync } from 'node:child_process'; -import { existsSync, unlinkSync, writeFileSync } from 'node:fs'; +import { + existsSync, + mkdtempSync, + readFileSync, + rmSync, + unlinkSync, + writeFileSync, +} from 'node:fs'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; +import { brotliCompressSync, gzipSync, zstdCompressSync } from 'node:zlib'; // The upstream curl parity assertions below only hold for the C-built curl // artifact; the Rust fallback in COMMANDS_DIR intentionally supports a smaller @@ -150,6 +158,44 @@ const externalNetworkSkipReason = runExternalNetwork })() : 'set AGENTOS_E2E_NETWORK=1 to enable external-network coverage'; +// A known, highly compressible payload used by the --compressed round-trip +// tests. Long + repetitive so every codec produces a body distinct from the +// plaintext (proving curl actually inflated it, not just passed it through). +const COMPRESSION_PAYLOAD = + 'agentos-compression-parity ' + 'the quick brown fox jumps over the lazy dog. '.repeat(64); + +// Build a real CA and a leaf server certificate signed by it, with a SAN that +// covers the 127.0.0.1 loopback endpoint the tests connect to. This lets curl's +// mbedTLS backend perform genuine chain + hostname verification, exactly like +// Linux curl against a private CA. +function makeCaSignedCert(caCommonName: string): { + caPem: string; + serverKey: string; + serverCert: string; +} { + const dir = mkdtempSync(join(tmpdir(), 'curl-ca-')); + try { + execSync(`openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "${dir}/ca.key" 2>/dev/null`); + execSync( + `openssl req -x509 -new -key "${dir}/ca.key" -days 3650 -subj "/CN=${caCommonName}" -out "${dir}/ca.crt" 2>/dev/null`, + ); + execSync(`openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "${dir}/srv.key" 2>/dev/null`); + execSync(`openssl req -new -key "${dir}/srv.key" -subj "/CN=localhost" -out "${dir}/srv.csr" 2>/dev/null`); + writeFileSync(`${dir}/ext.cnf`, 'subjectAltName=DNS:localhost,IP:127.0.0.1\n'); + execSync( + `openssl x509 -req -in "${dir}/srv.csr" -CA "${dir}/ca.crt" -CAkey "${dir}/ca.key" ` + + `-CAcreateserial -days 3650 -extfile "${dir}/ext.cnf" -out "${dir}/srv.crt" 2>/dev/null`, + ); + return { + caPem: readFileSync(`${dir}/ca.crt`, 'utf8'), + serverKey: readFileSync(`${dir}/srv.key`, 'utf8'), + serverCert: readFileSync(`${dir}/srv.crt`, 'utf8'), + }; + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + function generateSelfSignedCert(): { key: string; cert: string } { const keyPath = join(tmpdir(), `curl-test-key-${process.pid}-${Date.now()}.pem`); try { @@ -176,10 +222,19 @@ describeIf(hasCurl || hasHttpGetTest, 'curl and socket layer', () => { let kernel: Kernel; let httpServer: HttpServer; let httpsServer: HttpsServer; + let validHttpsServer: HttpsServer; + let caHttpsServer: HttpsServer; let keepAliveServer: TcpServer; let httpPort: number; let httpsPort: number; + let validHttpsPort: number; + let caHttpsPort: number; let keepAlivePort: number; + // CA (PEM) trusted by the seeded /etc/ssl/certs/ca-certificates.crt bundle — + // it signs validHttpsServer's leaf. caOnlyPem signs caHttpsServer's leaf and + // is deliberately NOT in the bundle, so it verifies only via --cacert. + let seededCaPem = ''; + let caOnlyPem = ''; let flakyRequestCount = 0; beforeAll(async () => { @@ -333,6 +388,29 @@ describeIf(hasCurl || hasHttpGetTest, 'curl and socket layer', () => { return; } + if (url === '/gzip' || url === '/brotli' || url === '/zstd') { + const raw = Buffer.from(COMPRESSION_PAYLOAD); + let encoding: string; + let body: Buffer; + if (url === '/gzip') { + encoding = 'gzip'; + body = gzipSync(raw); + } else if (url === '/brotli') { + encoding = 'br'; + body = brotliCompressSync(raw); + } else { + encoding = 'zstd'; + body = zstdCompressSync(raw); + } + res.writeHead(200, { + 'Content-Type': 'text/plain', + 'Content-Encoding': encoding, + 'Content-Length': String(body.length), + }); + res.end(body); + return; + } + res.writeHead(404, { 'Content-Type': 'text/plain' }); res.end('not found'); }); @@ -372,6 +450,37 @@ describeIf(hasCurl || hasHttpGetTest, 'curl and socket layer', () => { httpsServer.listen(0, '127.0.0.1', resolveListen); }); httpsPort = (httpsServer.address() as import('node:net').AddressInfo).port; + + // HTTPS server whose leaf chains to a CA seeded into the guest's + // /etc/ssl/certs/ca-certificates.crt — verified with NO -k / --cacert. + const trusted = makeCaSignedCert('AgentOS Test Root CA'); + seededCaPem = trusted.caPem; + validHttpsServer = createHttpsServer( + { key: trusted.serverKey, cert: trusted.serverCert }, + (req, res) => { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ verified: true, path: req.url })); + }, + ); + await new Promise((resolveListen) => { + validHttpsServer.listen(0, '127.0.0.1', resolveListen); + }); + validHttpsPort = (validHttpsServer.address() as import('node:net').AddressInfo).port; + + // HTTPS server whose CA is provided ONLY via --cacert (not in the bundle). + const caOnly = makeCaSignedCert('AgentOS Cacert-Only CA'); + caOnlyPem = caOnly.caPem; + caHttpsServer = createHttpsServer( + { key: caOnly.serverKey, cert: caOnly.serverCert }, + (req, res) => { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ cacert: true, path: req.url })); + }, + ); + await new Promise((resolveListen) => { + caHttpsServer.listen(0, '127.0.0.1', resolveListen); + }); + caHttpsPort = (caHttpsServer.address() as import('node:net').AddressInfo).port; } keepAliveServer = createTcpServer((socket) => { @@ -403,6 +512,12 @@ describeIf(hasCurl || hasHttpGetTest, 'curl and socket layer', () => { if (httpsServer) { await new Promise((resolveClose) => httpsServer.close(() => resolveClose())); } + if (validHttpsServer) { + await new Promise((resolveClose) => validHttpsServer.close(() => resolveClose())); + } + if (caHttpsServer) { + await new Promise((resolveClose) => caHttpsServer.close(() => resolveClose())); + } if (keepAliveServer) { await new Promise((resolveClose) => keepAliveServer.close(() => resolveClose())); } @@ -418,9 +533,23 @@ describeIf(hasCurl || hasHttpGetTest, 'curl and socket layer', () => { kernel = createKernel({ filesystem, permissions: allowAll, - loopbackExemptPorts: [httpPort, httpsPort, keepAlivePort], + loopbackExemptPorts: [ + httpPort, + httpsPort, + validHttpsPort, + caHttpsPort, + keepAlivePort, + ], }); await kernel.mount(createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] })); + + // Seed the Debian-shaped trust store the way the native VM bootstrap does, + // so curl's compile-time default CA bundle resolves in-guest. Only the + // "trusted" CA is placed here; the cacert-only CA is intentionally absent. + if (seededCaPem) { + await filesystem.mkdir('/etc/ssl/certs', { recursive: true }); + await kernel.writeFile('/etc/ssl/certs/ca-certificates.crt', seededCaPem); + } return kernel; } @@ -669,18 +798,67 @@ describeIf(hasCurl || hasHttpGetTest, 'curl and socket layer', () => { expect(Date.now() - startedAt).toBeLessThan(8000); }, 15000); - itIf(hasCurl && hasOpenssl, 'curl -k performs an HTTPS request through the WASI TLS backend', async () => { + itIf(hasCurl, 'curl --version reports the mbedTLS backend', async () => { + await createKernelWithNet(); + const result = await kernel.exec('curl --version'); + expect(result.exitCode).toBe(0); + // Real in-guest TLS: the SSL version line must name mbedTLS, and the + // retired host shim string must be gone. + expect(result.stdout).toMatch(/mbedTLS/i); + expect(result.stdout).not.toMatch(/WASI-TLS|wasi-tls/i); + expect(result.stdout).toMatch(/^Features:.*\bSSL\b/m); + expect(result.stdout).toMatch(/^Features:.*\bbrotli\b/m); + expect(result.stdout).toMatch(/^Features:.*\bzstd\b/m); + }, 15000); + + itIf(hasCurl && hasOpenssl, 'curl verifies a CA-signed cert against the seeded CA bundle', async () => { + await createKernelWithNet(); + // No -k, no --cacert: trust comes solely from the seeded + // /etc/ssl/certs/ca-certificates.crt, exactly like Debian curl. + const result = await kernel.exec(`curl -sS https://127.0.0.1:${validHttpsPort}/json`); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain('"verified":true'); + }, 15000); + + itIf(hasCurl && hasOpenssl, 'curl fails with exit 60 and a real verify message on an untrusted cert', async () => { + await createKernelWithNet(); + const result = await kernel.exec(`curl -sS https://127.0.0.1:${httpsPort}/json`); + // CURLE_PEER_FAILED_VERIFICATION == 60, the native Linux taxonomy. NOT 35 + // (SSL connect error, the old host-shim behavior). + expect(result.exitCode).toBe(60); + expect(result.stderr).toMatch(/certificate|verify|self[- ]?signed|CA/i); + expect(result.stderr).not.toMatch(/WASI TLS|wasi-tls/i); + expect(result.stdout).toBe(''); + }, 15000); + + itIf(hasCurl && hasOpenssl, 'curl -k skips verification and succeeds on an untrusted cert', async () => { await createKernelWithNet(); const result = await kernel.exec(`curl -ks https://127.0.0.1:${httpsPort}/json`); expect(result.exitCode).toBe(0); expect(result.stdout).toContain('"secure":true'); }, 15000); - itIf(hasCurl && hasOpenssl, 'curl fails TLS verification without -k on a self-signed endpoint', async () => { + itIf(hasCurl && hasOpenssl, 'curl --cacert accepts a server signed by that CA', async () => { await createKernelWithNet(); - const result = await kernel.exec(`curl -sS https://127.0.0.1:${httpsPort}/json`); - expect(result.exitCode).not.toBe(0); - expect(result.stderr).toMatch(/certificate|tls|ssl|verify/i); + // caHttpsServer's CA is NOT in the seeded bundle, so this only passes if + // --cacert is honored (real file read + chain build in-guest). + await kernel.writeFile('/tmp/cacert-only.pem', caOnlyPem); + const result = await kernel.exec( + `curl -sS --cacert /tmp/cacert-only.pem https://127.0.0.1:${caHttpsPort}/json`, + ); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain('"cacert":true'); + }, 15000); + + itIf(hasCurl && hasOpenssl, 'curl --cacert with the wrong CA still fails verification (exit 60)', async () => { + await createKernelWithNet(); + // Point --cacert at the seeded CA, which did NOT sign caHttpsServer's leaf. + await kernel.writeFile('/tmp/wrong-ca.pem', seededCaPem); + const result = await kernel.exec( + `curl -sS --cacert /tmp/wrong-ca.pem https://127.0.0.1:${caHttpsPort}/json`, + ); + expect(result.exitCode).toBe(60); + expect(result.stderr).toMatch(/certificate|verify|CA/i); }, 15000); itIf(hasCurl && hasOpenssl, 'curl -k exits promptly after an HTTPS keep-alive response', async () => { @@ -692,6 +870,27 @@ describeIf(hasCurl || hasHttpGetTest, 'curl and socket layer', () => { expect(Date.now() - startedAt).toBeLessThan(8000); }, 15000); + itIf(hasCurl, 'curl --compressed round-trips a gzip response body', async () => { + await createKernelWithNet(); + const result = await kernel.exec(`curl -s --compressed http://127.0.0.1:${httpPort}/gzip`); + expect(result.exitCode).toBe(0); + expect(result.stdout).toBe(COMPRESSION_PAYLOAD); + }, 15000); + + itIf(hasCurl, 'curl --compressed round-trips a brotli response body', async () => { + await createKernelWithNet(); + const result = await kernel.exec(`curl -s --compressed http://127.0.0.1:${httpPort}/brotli`); + expect(result.exitCode).toBe(0); + expect(result.stdout).toBe(COMPRESSION_PAYLOAD); + }, 15000); + + itIf(hasCurl, 'curl --compressed round-trips a zstd response body', async () => { + await createKernelWithNet(); + const result = await kernel.exec(`curl -s --compressed http://127.0.0.1:${httpPort}/zstd`); + expect(result.exitCode).toBe(0); + expect(result.stdout).toBe(COMPRESSION_PAYLOAD); + }, 15000); + itIf(hasHttpGetTest && !externalNetworkSkipReason, 'http_get_test reaches an external host over real TCP', async () => { await createKernelWithNet(); const result = await execWithRetry(`http_get_test ${EXTERNAL_HOST} ${EXTERNAL_TCP_PORT} /`); diff --git a/software/everything/package.json b/software/everything/package.json index 95ec6ff1ea..76b29c4fbc 100644 --- a/software/everything/package.json +++ b/software/everything/package.json @@ -44,6 +44,8 @@ "@agentos-software/ripgrep": "workspace:*", "@agentos-software/fd": "workspace:*", "@agentos-software/tree": "workspace:*", + "@agentos-software/procps": "workspace:*", + "@agentos-software/psmisc": "workspace:*", "@agentos-software/file": "workspace:*", "@agentos-software/yq": "workspace:*", "@agentos-software/codex-cli": "workspace:*" diff --git a/software/everything/src/index.ts b/software/everything/src/index.ts index c45ab24679..4bd7c44ca2 100644 --- a/software/everything/src/index.ts +++ b/software/everything/src/index.ts @@ -19,6 +19,8 @@ import jq from "@agentos-software/jq"; import ripgrep from "@agentos-software/ripgrep"; import fd from "@agentos-software/fd"; import tree from "@agentos-software/tree"; +import procps from "@agentos-software/procps"; +import psmisc from "@agentos-software/psmisc"; import file from "@agentos-software/file"; import yq from "@agentos-software/yq"; import codex from "@agentos-software/codex-cli"; @@ -45,6 +47,8 @@ const everything = [ ripgrep, fd, tree, + procps, + psmisc, file, yq, codex, @@ -73,6 +77,8 @@ export { ripgrep, fd, tree, + procps, + psmisc, file, yq, codex, diff --git a/software/everything/test/everything.test.ts b/software/everything/test/everything.test.ts index f9d1ea24af..f2ab689dbf 100644 --- a/software/everything/test/everything.test.ts +++ b/software/everything/test/everything.test.ts @@ -21,6 +21,8 @@ import everything, { sqlite3, tar, tree, + procps, + psmisc, unzip, vim, wget, @@ -52,6 +54,8 @@ const expectedMembers = [ ripgrep, fd, tree, + procps, + psmisc, file, yq, codex, diff --git a/software/git/agentos-package.json b/software/git/agentos-package.json index 31c38e66e4..b1c4f9c536 100644 --- a/software/git/agentos-package.json +++ b/software/git/agentos-package.json @@ -1,10 +1,10 @@ { "commands": [ - "git" + "git", + "git-remote-http" ], "aliases": { - "git-remote-http": "git", - "git-remote-https": "git", + "git-remote-https": "git-remote-http", "git-upload-pack": "git", "git-receive-pack": "git", "git-upload-archive": "git" diff --git a/software/git/test/git.test.ts b/software/git/test/git.test.ts index 2516b7dc14..00e65d4e90 100644 --- a/software/git/test/git.test.ts +++ b/software/git/test/git.test.ts @@ -6,11 +6,11 @@ */ import { describe, it, expect, afterEach, beforeAll, afterAll, vi } from 'vitest'; -import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { resolve, join } from 'node:path'; import { tmpdir } from 'node:os'; -import { createServer, type Server as HttpServer } from 'node:http'; -import { spawn, spawnSync } from 'node:child_process'; +import { createServer as createHttpsServer, type Server as HttpsServer } from 'node:https'; +import { spawn, spawnSync, execSync } from 'node:child_process'; import { createWasmVmRuntime } from '@agentos/test-harness'; import { allowAll, @@ -27,8 +27,14 @@ vi.setConfig({ testTimeout: 30_000 }); /** Check git binary exists in addition to base WASM binaries */ const hasGit = hasWasmBinaries && existsSync(resolve(COMMANDS_DIR, 'git')); const hasHostGit = spawnSync('git', ['--version'], { stdio: 'ignore' }).status === 0; -// Smart HTTP needs Git's libcurl-backed remote helpers; this WASM build is NO_CURL. -const hasGitHttpHelper = false; +// Smart HTTP needs Git's libcurl-backed remote helper. It is now a real second +// WASM binary (git-remote-http links the overlaid mbedTLS libcurl in-process); +// git-remote-https aliases to it. +const hasGitHttpHelper = + hasGit && existsSync(resolve(COMMANDS_DIR, 'git-remote-http')); +// The real OpenSSH client (software/ssh) lights up git-over-ssh; its presence +// changes how ssh:// remotes fail when unreachable. +const hasSshClient = existsSync(resolve(COMMANDS_DIR, 'ssh')); const gitConfig = [ '-c safe.directory=*', @@ -53,7 +59,7 @@ async function createGitKernel() { return { kernel, vfs, dispose: () => kernel.dispose() }; } -async function createGitKernelWithNet(loopbackExemptPorts: number[]) { +async function createGitKernelWithNet(loopbackExemptPorts: number[], seededCaPem?: string) { const vfs = createInMemoryFileSystem(); await (vfs as any).chmod('/', 0o1777); await vfs.mkdir('/tmp', { recursive: true }); @@ -65,9 +71,48 @@ async function createGitKernelWithNet(loopbackExemptPorts: number[]) { syncFilesystemOnDispose: false, }); await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); + // Seed the Debian-shaped trust store the way the native VM bootstrap does, so + // libcurl's compile-time default CA bundle (/etc/ssl/certs/ca-certificates.crt) + // resolves in-guest for git-remote-http's mbedTLS backend. + if (seededCaPem) { + await vfs.mkdir('/etc/ssl/certs', { recursive: true }); + await kernel.writeFile('/etc/ssl/certs/ca-certificates.crt', seededCaPem); + } return { kernel, vfs, dispose: () => kernel.dispose() }; } +// Build a real CA and a leaf server certificate signed by it, with a SAN that +// covers the 127.0.0.1 loopback endpoint the VM connects to. This lets +// git-remote-http's mbedTLS backend perform genuine chain + hostname +// verification, exactly like Linux git against a private CA. +function makeCaSignedCert(caCommonName: string): { + caPem: string; + serverKey: string; + serverCert: string; +} { + const dir = mkdtempSync(join(tmpdir(), 'git-ca-')); + try { + execSync(`openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "${dir}/ca.key" 2>/dev/null`); + execSync( + `openssl req -x509 -new -key "${dir}/ca.key" -days 3650 -subj "/CN=${caCommonName}" -out "${dir}/ca.crt" 2>/dev/null`, + ); + execSync(`openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "${dir}/srv.key" 2>/dev/null`); + execSync(`openssl req -new -key "${dir}/srv.key" -subj "/CN=localhost" -out "${dir}/srv.csr" 2>/dev/null`); + writeFileSync(`${dir}/ext.cnf`, 'subjectAltName=DNS:localhost,IP:127.0.0.1\n'); + execSync( + `openssl x509 -req -in "${dir}/srv.csr" -CA "${dir}/ca.crt" -CAkey "${dir}/ca.key" ` + + `-CAcreateserial -days 3650 -extfile "${dir}/ext.cnf" -out "${dir}/srv.crt" 2>/dev/null`, + ); + return { + caPem: readFileSync(`${dir}/ca.crt`, 'utf8'), + serverKey: readFileSync(`${dir}/srv.key`, 'utf8'), + serverCert: readFileSync(`${dir}/srv.crt`, 'utf8'), + }; + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + function runHostGit(args: string[], cwd?: string) { const result = spawnSync('git', args, { cwd, @@ -386,73 +431,49 @@ describeIf(hasGit, 'git command', () => { expect(result.stderr).not.toContain('GitSubcommandUnsupported'); }); - it('clone rejects SSH-style remotes with a real Git spawn failure', async () => { - ({ kernel, dispose } = await createGitKernel()); - - const result = await kernel.exec(git('clone git@github.com:rivet-dev/agentos.git /tmp/clone')); - expect(result.exitCode).not.toBe(0); - expect(result.stderr).toMatch(/cannot run ssh|unable to fork|ssh|fatal/i); - expect(result.stderr).not.toContain('GitSubcommandUnsupported'); - }); - - it('clone rejects HTTPS remotes until Git is linked with libcurl remote helpers', async () => { + it('clone over ssh:// reaches the real ssh client and surfaces its transport error', async () => { ({ kernel, dispose } = await createGitKernel()); - const result = await kernel.exec( - git('clone https://private@example.com/owner/repo.git /tmp/clone'), - { env: { GIT_AUTH_TOKEN: 'test-token' } }, - ); + // Port 1 on loopback is never exempted for this kernel, so the real + // OpenSSH client (now on PATH — git connect.c execs `ssh`) fails with a + // genuine connection error instead of a spawn failure. Full git-over-ssh + // success coverage lives in software/ssh/test/ssh.test.ts. + const result = await kernel.exec(git('clone ssh://git@127.0.0.1:1/repo.git /tmp/clone')); expect(result.exitCode).not.toBe(0); - expect(result.stderr).toMatch(/remote-https|remote helper|fatal|unable/i); + if (hasSshClient) { + // The error must come from ssh's transport, proving git spawned it. + expect(result.stderr).toMatch(/ssh:|Connection closed|Could not read from remote repository/i); + expect(result.stderr).not.toMatch(/cannot run ssh|unable to fork/i); + } else { + expect(result.stderr).toMatch(/cannot run ssh|unable to fork|ssh|fatal/i); + } expect(result.stderr).not.toContain('GitSubcommandUnsupported'); }); - describeIf(hasHostGit && hasGitHttpHelper, 'remote clone over smart HTTP', () => { + // Real smart-HTTP over TLS: git-remote-http (libcurl + in-guest mbedTLS) + // clones/fetches/pushes against `git http-backend` behind a Node HTTPS + // endpoint. Certificate trust comes from a private CA seeded into the guest's + // /etc/ssl/certs bundle (the "trusted" server) — exactly the Debian trust + // path — while a second server signed by a CA absent from the bundle exercises + // verify-fail, http.sslVerify=false, GIT_SSL_NO_VERIFY, and http.sslCAInfo. + describeIf(hasHostGit && hasGitHttpHelper, 'smart-HTTP clone/fetch/push over TLS', () => { let repoRoot: string; - let httpServer: HttpServer; - let httpPort: number; - - beforeAll(async () => { - repoRoot = mkdtempSync(join(tmpdir(), 'agentos-git-http-')); - const worktree = join(repoRoot, 'worktree'); - const origin = join(repoRoot, 'origin.git'); - - runHostGit(['-c', 'init.defaultBranch=main', 'init', worktree]); - writeFileSync(join(worktree, 'README.md'), 'remote smart clone\n'); - runHostGit(['-C', worktree, 'add', 'README.md']); - runHostGit([ - '-C', worktree, - '-c', 'user.name=secure-exec', - '-c', 'user.email=agent@example.com', - 'commit', - '-m', - 'seed', - ]); - - runHostGit(['-C', worktree, 'checkout', '-b', 'feature/deep']); - writeFileSync(join(worktree, 'feature.txt'), 'remote branch payload\n'); - runHostGit(['-C', worktree, 'add', 'feature.txt']); - runHostGit([ - '-C', worktree, - '-c', 'user.name=secure-exec', - '-c', 'user.email=agent@example.com', - 'commit', - '-m', - 'feature branch', - ]); - - runHostGit(['-C', worktree, 'checkout', 'main']); - runHostGit(['clone', '--bare', worktree, origin]); - runHostGit(['-C', origin, 'repack', '-a', '-d', '-f', '--depth=50', '--window=50']); - - httpServer = createServer((req, res) => { - const url = new URL(req.url ?? '/', 'http://127.0.0.1'); + let trustedServer: HttpsServer; + let untrustedServer: HttpsServer; + let trustedPort: number; + let untrustedPort: number; + let trustedCaPem = ''; + let untrustedCaPem = ''; + + // A CGI bridge to `git http-backend`. receive-pack is enabled on the origin + // (below) so pushes are accepted; GIT_HTTP_EXPORT_ALL allows anonymous read. + function makeBackendHandler() { + return (req: import('node:http').IncomingMessage, res: import('node:http').ServerResponse) => { + const url = new URL(req.url ?? '/', 'https://127.0.0.1'); const bodyChunks: Buffer[] = []; - req.on('data', (chunk) => { bodyChunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); }); - req.on('end', () => { const requestBody = Buffer.concat(bodyChunks); const gitProtocol = req.headers['git-protocol']; @@ -473,7 +494,6 @@ describeIf(hasGit, 'git command', () => { const child = spawn('git', ['http-backend'], { env }); const stdout: Buffer[] = []; const stderr: Buffer[] = []; - child.stdout.on('data', (chunk) => { stdout.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); }); @@ -490,24 +510,20 @@ describeIf(hasGit, 'git command', () => { const altSep = output.indexOf(Buffer.from('\n\n')); const sepIndex = headerSep >= 0 ? headerSep : altSep; const sepLen = headerSep >= 0 ? 4 : altSep >= 0 ? 2 : 0; - if (code !== 0 && sepIndex === -1) { res.writeHead(500, { 'Content-Type': 'text/plain' }); res.end(Buffer.concat(stderr)); return; } - if (sepIndex === -1) { res.writeHead(500, { 'Content-Type': 'text/plain' }); res.end(output); return; } - const headerText = output.subarray(0, sepIndex).toString('utf8'); const responseBody = output.subarray(sepIndex + sepLen); let status = 200; const headers: Record = {}; - for (const line of headerText.split(/\r?\n/)) { if (!line) continue; const colon = line.indexOf(':'); @@ -520,34 +536,84 @@ describeIf(hasGit, 'git command', () => { headers[name] = value; } } - res.writeHead(status, headers); res.end(responseBody); }); - child.stdin.end(requestBody); }); - }); + }; + } - await new Promise((resolveListen) => { - httpServer.listen(0, '127.0.0.1', resolveListen); - }); - httpPort = (httpServer.address() as import('node:net').AddressInfo).port; + async function listen(server: HttpsServer): Promise { + await new Promise((r) => server.listen(0, '127.0.0.1', r)); + return (server.address() as import('node:net').AddressInfo).port; + } + + beforeAll(async () => { + repoRoot = mkdtempSync(join(tmpdir(), 'agentos-git-https-')); + const worktree = join(repoRoot, 'worktree'); + const origin = join(repoRoot, 'origin.git'); + + runHostGit(['-c', 'init.defaultBranch=main', 'init', worktree]); + writeFileSync(join(worktree, 'README.md'), 'remote smart clone\n'); + runHostGit(['-C', worktree, 'add', 'README.md']); + runHostGit([ + '-C', worktree, + '-c', 'user.name=secure-exec', '-c', 'user.email=agent@example.com', + 'commit', '-m', 'seed', + ]); + runHostGit(['-C', worktree, 'checkout', '-b', 'feature/deep']); + writeFileSync(join(worktree, 'feature.txt'), 'remote branch payload\n'); + runHostGit(['-C', worktree, 'add', 'feature.txt']); + runHostGit([ + '-C', worktree, + '-c', 'user.name=secure-exec', '-c', 'user.email=agent@example.com', + 'commit', '-m', 'feature branch', + ]); + runHostGit(['-C', worktree, 'checkout', 'main']); + runHostGit(['clone', '--bare', worktree, origin]); + runHostGit(['-C', origin, 'repack', '-a', '-d', '-f', '--depth=50', '--window=50']); + // Accept anonymous pushes over smart HTTP. + runHostGit(['-C', origin, 'config', 'http.receivepack', 'true']); + + const trusted = makeCaSignedCert('AgentOS Git Test Root CA'); + trustedCaPem = trusted.caPem; + trustedServer = createHttpsServer( + { key: trusted.serverKey, cert: trusted.serverCert }, + makeBackendHandler(), + ); + trustedPort = await listen(trustedServer); + + const untrusted = makeCaSignedCert('AgentOS Git Untrusted CA'); + untrustedCaPem = untrusted.caPem; + untrustedServer = createHttpsServer( + { key: untrusted.serverKey, cert: untrusted.serverCert }, + makeBackendHandler(), + ); + untrustedPort = await listen(untrustedServer); }); afterAll(async () => { - await new Promise((resolveClose) => httpServer.close(() => resolveClose())); + if (trustedServer) await new Promise((r) => trustedServer.close(() => r())); + if (untrustedServer) await new Promise((r) => untrustedServer.close(() => r())); rmSync(repoRoot, { recursive: true, force: true }); }); - it('clone fetches refs and worktree contents from a smart HTTP remote', async () => { - ({ kernel, vfs, dispose } = await createGitKernelWithNet([httpPort])); + const trustedUrl = () => `https://127.0.0.1:${trustedPort}/origin.git`; + const untrustedUrl = () => `https://127.0.0.1:${untrustedPort}/origin.git`; + + it('clone fetches refs and worktree contents over HTTPS with a trusted CA', async () => { + ({ kernel, vfs, dispose } = await createGitKernelWithNet([trustedPort], trustedCaPem)); - await run(kernel, git(`clone http://127.0.0.1:${httpPort}/origin.git /tmp/clone`)); + const res = await kernel.exec(git(`clone ${trustedUrl()} /tmp/clone`), { + env: { GIT_CURL_VERBOSE: '1' }, + }); + expect(res.exitCode).toBe(0); + // Proof a real TLS handshake happened in-guest (mbedTLS via libcurl). + expect(res.stderr).toMatch(/SSL connection|TLS|SSL certificate|CAfile/i); const head = new TextDecoder().decode(await kernel.readFile('/tmp/clone/.git/HEAD')); expect(head.trim()).toBe('ref: refs/heads/main'); - const readme = new TextDecoder().decode(await kernel.readFile('/tmp/clone/README.md')); expect(readme).toBe('remote smart clone\n'); await expectGitRef(kernel, '/tmp/clone', 'refs/remotes/origin/feature/deep'); @@ -556,5 +622,111 @@ describeIf(hasGit, 'git command', () => { const feature = new TextDecoder().decode(await kernel.readFile('/tmp/clone/feature.txt')); expect(feature).toBe('remote branch payload\n'); }); + + it('fetch picks up a new remote branch over HTTPS', async () => { + ({ kernel, vfs, dispose } = await createGitKernelWithNet([trustedPort], trustedCaPem)); + await run(kernel, git(`clone ${trustedUrl()} /tmp/clone`)); + + // Add a new branch on the origin (host side), then fetch it in-guest. + const bareBranch = 'fetched-branch'; + runHostGit(['-C', join(repoRoot, 'origin.git'), 'branch', bareBranch, 'main']); + + await run(kernel, git('-C /tmp/clone fetch origin')); + await expectGitRef(kernel, '/tmp/clone', `refs/remotes/origin/${bareBranch}`); + }); + + it('push sends a small commit over HTTPS smart-HTTP', async () => { + ({ kernel, vfs, dispose } = await createGitKernelWithNet([trustedPort], trustedCaPem)); + await run(kernel, git(`clone ${trustedUrl()} /tmp/clone`)); + + await kernel.writeFile('/tmp/clone/pushed.txt', 'pushed over https\n'); + await run(kernel, git('-C /tmp/clone add pushed.txt')); + await run(kernel, git("-C /tmp/clone commit -m 'push small'")); + const pushed = await kernel.exec(git('-C /tmp/clone push origin HEAD:refs/heads/small-push')); + expect(pushed.exitCode).toBe(0); + + // Verify the ref really landed in the origin bare repo (host side). + const originRef = spawnSync( + 'git', + ['-C', join(repoRoot, 'origin.git'), 'rev-parse', '--verify', 'refs/heads/small-push'], + { encoding: 'utf8' }, + ); + expect(originRef.status).toBe(0); + expect(originRef.stdout.trim()).toMatch(/^[0-9a-f]{40,64}$/); + }); + + it('push streams a >1 MiB commit over HTTPS (chunked POST)', async () => { + ({ kernel, vfs, dispose } = await createGitKernelWithNet([trustedPort], trustedCaPem)); + await run(kernel, git(`clone ${trustedUrl()} /tmp/clone`)); + + // Incompressible >1 MiB payload so the pack exceeds http.postBuffer (1 MiB) + // and libcurl must use chunked Transfer-Encoding + Expect: 100-continue. + const { randomBytes } = await import('node:crypto'); + const big = randomBytes(2 * 1024 * 1024); + await kernel.writeFile('/tmp/clone/big.bin', big); + await run(kernel, git('-C /tmp/clone add big.bin')); + await run(kernel, git("-C /tmp/clone commit -m 'push large'")); + const pushed = await kernel.exec(git('-C /tmp/clone push origin HEAD:refs/heads/large-push')); + expect(pushed.exitCode).toBe(0); + + const originRef = spawnSync( + 'git', + ['-C', join(repoRoot, 'origin.git'), 'rev-parse', '--verify', 'refs/heads/large-push'], + { encoding: 'utf8' }, + ); + expect(originRef.status).toBe(0); + // Confirm the large object is actually present in the origin object store. + const cat = spawnSync( + 'git', + ['-C', join(repoRoot, 'origin.git'), 'cat-file', '-s', `${originRef.stdout.trim()}^{tree}`], + { encoding: 'utf8' }, + ); + expect(cat.status).toBe(0); + }); + + it('clone fails with a real certificate-verification error on an untrusted CA', async () => { + // Only the trusted CA is seeded; the untrusted server's CA is absent. + ({ kernel, vfs, dispose } = await createGitKernelWithNet([untrustedPort], trustedCaPem)); + + const res = await kernel.exec(git(`clone ${untrustedUrl()} /tmp/clone`)); + expect(res.exitCode).not.toBe(0); + expect(res.stderr).toMatch(/certificate|SSL|TLS|verify|CAfile|unable to (access|get local)/i); + expect(res.stderr).not.toContain('GitSubcommandUnsupported'); + expect(await vfs.exists('/tmp/clone/.git')).toBe(false); + }); + + it('http.sslVerify=false bypasses verification for an untrusted CA', async () => { + ({ kernel, vfs, dispose } = await createGitKernelWithNet([untrustedPort], trustedCaPem)); + + const res = await kernel.exec(git(`-c http.sslVerify=false clone ${untrustedUrl()} /tmp/clone`)); + expect(res.exitCode).toBe(0); + const readme = new TextDecoder().decode(await kernel.readFile('/tmp/clone/README.md')); + expect(readme).toBe('remote smart clone\n'); + }); + + it('GIT_SSL_NO_VERIFY bypasses verification for an untrusted CA', async () => { + ({ kernel, vfs, dispose } = await createGitKernelWithNet([untrustedPort], trustedCaPem)); + + const res = await kernel.exec(git(`clone ${untrustedUrl()} /tmp/clone`), { + env: { GIT_SSL_NO_VERIFY: '1' }, + }); + expect(res.exitCode).toBe(0); + expect(await vfs.exists('/tmp/clone/.git/HEAD')).toBe(true); + }); + + it('http.sslCAInfo trusts an explicitly supplied CA bundle', async () => { + // Seed only the trusted CA in the default bundle; supply the untrusted + // server's CA via a VFS file referenced with http.sslCAInfo. + ({ kernel, vfs, dispose } = await createGitKernelWithNet([untrustedPort], trustedCaPem)); + await vfs.mkdir('/tmp/ca', { recursive: true }); + await kernel.writeFile('/tmp/ca/untrusted.pem', untrustedCaPem); + + const res = await kernel.exec( + git(`-c http.sslCAInfo=/tmp/ca/untrusted.pem clone ${untrustedUrl()} /tmp/clone`), + ); + expect(res.exitCode).toBe(0); + const readme = new TextDecoder().decode(await kernel.readFile('/tmp/clone/README.md')); + expect(readme).toBe('remote smart clone\n'); + }); }); }); diff --git a/software/procps/agentos-package.json b/software/procps/agentos-package.json new file mode 100644 index 0000000000..0cb34d3c76 --- /dev/null +++ b/software/procps/agentos-package.json @@ -0,0 +1,9 @@ +{ + "commands": ["ps", "pgrep", "pkill"], + "registry": { + "title": "procps", + "description": "Inspect and signal live VM processes with upstream procps-ng.", + "priority": 60, + "category": "core" + } +} diff --git a/software/procps/bin/pgrep b/software/procps/bin/pgrep new file mode 100644 index 0000000000..71cab40294 Binary files /dev/null and b/software/procps/bin/pgrep differ diff --git a/software/procps/bin/pkill b/software/procps/bin/pkill new file mode 100644 index 0000000000..71cab40294 Binary files /dev/null and b/software/procps/bin/pkill differ diff --git a/software/procps/bin/ps b/software/procps/bin/ps new file mode 100644 index 0000000000..a0158fffef Binary files /dev/null and b/software/procps/bin/ps differ diff --git a/software/procps/package.json b/software/procps/package.json new file mode 100644 index 0000000000..a6c80c254b --- /dev/null +++ b/software/procps/package.json @@ -0,0 +1,29 @@ +{ + "name": "@agentos-software/procps", + "version": "0.0.1", + "type": "module", + "license": "GPL-2.0-or-later", + "description": "Upstream procps-ng process inspection and selection commands for AgentOS VMs", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "files": ["dist", "!dist/package", "!dist/package.tar"], + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "scripts": { + "build": "agentos-toolchain stage --commands-dir ../../toolchain/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", + "check-types": "tsc --noEmit", + "test": "vitest run test/ --passWithNoTests" + }, + "devDependencies": { + "@agentos-software/manifest": "workspace:*", + "@rivet-dev/agentos-toolchain": "workspace:*", + "@types/node": "^22.10.2", + "typescript": "^5.9.2", + "@agentos/test-harness": "workspace:*", + "vitest": "^2.1.9" + } +} diff --git a/software/procps/src/index.ts b/software/procps/src/index.ts new file mode 100644 index 0000000000..dba100e2e1 --- /dev/null +++ b/software/procps/src/index.ts @@ -0,0 +1,5 @@ +import type { SoftwarePackageRef } from "@agentos-software/manifest"; + +const packagePath = new URL("./package.aospkg", import.meta.url).pathname; + +export default { packagePath } satisfies SoftwarePackageRef; diff --git a/software/procps/test/fixtures/generate-linux-process-consumers.py b/software/procps/test/fixtures/generate-linux-process-consumers.py new file mode 100755 index 0000000000..ac10d01379 --- /dev/null +++ b/software/procps/test/fixtures/generate-linux-process-consumers.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""Capture a real-Linux ps/pgrep fixture for the procps-ng VM e2e.""" + +import json +import os +import platform +import signal +import shutil +import subprocess +import tempfile +from datetime import datetime +from pathlib import Path +from zoneinfo import ZoneInfo + + +def version_line(*command: str) -> str: + result = subprocess.run(command, check=True, capture_output=True, text=True) + return (result.stdout or result.stderr).splitlines()[0] + + +def main() -> None: + with tempfile.TemporaryDirectory() as temp_dir: + name = "aos-procfx" + executable = Path(temp_dir) / name + executable.symlink_to(shutil.which("sleep")) + child = subprocess.Popen([executable, "30"]) + try: + ps = subprocess.run( + ["ps", "-o", "pid,ppid,stat,comm,args", "-p", str(child.pid)], + check=True, + capture_output=True, + text=True, + ) + pgrep = subprocess.run( + ["pgrep", "-x", name], + check=True, + capture_output=True, + text=True, + ) + pstree = subprocess.run( + ["pstree", "-p", str(child.pid)], + check=True, + capture_output=True, + text=True, + ) + prtstat = subprocess.run( + ["prtstat", str(child.pid)], + check=True, + capture_output=True, + text=True, + ) + lines = ps.stdout.rstrip("\n").splitlines() + fixture = { + "captured_at_pst": datetime.now(ZoneInfo("America/Los_Angeles")).isoformat(), + "linux": platform.uname()._asdict(), + "procps_version": version_line("ps", "--version"), + "psmisc_version": version_line("pstree", "--version"), + "ps_command": "ps -o pid,ppid,stat,comm,args -p ", + "ps_header": lines[0], + "ps_row": lines[1], + "pgrep_command": f"pgrep -x {name}", + "pgrep_pids": [int(value) for value in pgrep.stdout.split()], + "target_pid": child.pid, + "target_found": child.pid in {int(value) for value in pgrep.stdout.split()}, + "pstree_command": "pstree -p ", + "pstree_output": pstree.stdout.rstrip("\n"), + "prtstat_command": "prtstat ", + "prtstat_output": prtstat.stdout.rstrip("\n"), + } + output = Path(__file__).with_name("linux-procps-ng-4.0.6.json") + output.write_text(json.dumps(fixture, indent=2) + "\n") + finally: + os.kill(child.pid, signal.SIGTERM) + child.wait() + + +if __name__ == "__main__": + main() diff --git a/software/procps/test/fixtures/linux-procps-ng-4.0.6.json b/software/procps/test/fixtures/linux-procps-ng-4.0.6.json new file mode 100644 index 0000000000..4710f9ba17 --- /dev/null +++ b/software/procps/test/fixtures/linux-procps-ng-4.0.6.json @@ -0,0 +1,26 @@ +{ + "captured_at_pst": "2026-07-09T18:12:25.441793-07:00", + "linux": { + "system": "Linux", + "node": "nathan-dev", + "release": "6.1.0-41-amd64", + "version": "#1 SMP PREEMPT_DYNAMIC Debian 6.1.158-1 (2025-11-09)", + "machine": "x86_64", + "processor": "" + }, + "procps_version": "ps from procps-ng 4.0.2", + "psmisc_version": "pstree (PSmisc) 23.6", + "ps_command": "ps -o pid,ppid,stat,comm,args -p ", + "ps_header": " PID PPID STAT COMMAND COMMAND", + "ps_row": "2562887 2562885 S aos-procfx /tmp/tmpzgarw79f/aos-procfx 30", + "pgrep_command": "pgrep -x aos-procfx", + "pgrep_pids": [ + 2562887 + ], + "target_pid": 2562887, + "target_found": true, + "pstree_command": "pstree -p ", + "pstree_output": "aos-procfx(2562887)", + "prtstat_command": "prtstat ", + "prtstat_output": "Process: aos-procfx \t\tState: S (sleeping)\n CPU#: 2 \t\tTTY: 0:0\tThreads: 1\nProcess, Group and Session IDs\n Process ID: 2562887\t\t Parent ID: 2562885\n Group ID: 2562882\t\t Session ID: 2562882\n T Group ID: -1\n\nPage Faults\n This Process (minor major): 77 0\n Child Processes (minor major): 0 0\nCPU Times\n This Process (user system guest blkio): 0.00 0.00 0.00 0.00\n Child processes (user system guest): 0.00 0.00 0.00\nMemory\n Vsize: 6033 kB \n RSS: 946 kB \t\t RSS Limit: 18446744073709 MB\n Code Start: 0x55ee56cb2000\t\t Code Stop: 0x55ee56cb6609\n Stack Start: 0x7ffed5c84910\n Stack Pointer (ESP): 0\t Inst Pointer (EIP): 0\nScheduling\n Policy: normal\n Nice: 0 \t\t RT Priority: 0 (non RT)" +} diff --git a/software/procps/test/procps.test.ts b/software/procps/test/procps.test.ts new file mode 100644 index 0000000000..906f2d627b --- /dev/null +++ b/software/procps/test/procps.test.ts @@ -0,0 +1,69 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { existsSync, readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { + C_BUILD_DIR, + COMMANDS_DIR, + createIntegrationKernel, + createWasmVmRuntime, + describeIf, +} from "@agentos/test-harness"; +import type { IntegrationKernelResult } from "@agentos/test-harness"; + +const hasCommands = ["ps", "pgrep", "pkill"].every((command) => + existsSync(resolve(C_BUILD_DIR, command)), +); + +describe("captured real-Linux process-consumer fixture", () => { + it("records procps-ng and psmisc parsing the same live process", () => { + const fixture = JSON.parse( + readFileSync( + new URL("./fixtures/linux-procps-ng-4.0.6.json", import.meta.url), + "utf8", + ), + ); + + expect(fixture.procps_version).toMatch(/^ps from procps-ng /); + expect(fixture.psmisc_version).toMatch(/^pstree \(PSmisc\) /); + expect(fixture.target_found).toBe(true); + expect(fixture.ps_header).toMatch(/PID\s+PPID\s+STAT\s+COMMAND/); + expect(fixture.pstree_output).toBe(`aos-procfx(${fixture.target_pid})`); + expect(fixture.prtstat_output).toContain("Process: aos-procfx"); + }); +}); + +describeIf(hasCommands, "upstream procps-ng against live VM processes", () => { + let ctx: IntegrationKernelResult; + + afterEach(async () => { + await ctx?.dispose().catch(() => {}); + }); + + it("ps and pgrep identify a live child, then pkill terminates it", async () => { + ctx = await createIntegrationKernel({ runtimes: [] }); + await ctx.kernel.mount( + createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] }), + ); + + const sleeper = ctx.kernel.spawn("sleep", ["60"]); + await new Promise((resolveReady) => setTimeout(resolveReady, 100)); + + const pgrep = await ctx.kernel.exec("pgrep -x sleep"); + expect(pgrep.exitCode, pgrep.stderr).toBe(0); + const guestPid = Number.parseInt(pgrep.stdout.trim(), 10); + expect(guestPid).toBeGreaterThan(0); + + const ps = await ctx.kernel.exec("ps -eo pid,ppid,stat,comm,args"); + expect(ps.exitCode, ps.stderr).toBe(0); + expect(ps.stdout).toMatch(/\bPID\b.*\bPPID\b.*\bSTAT\b.*\bCOMMAND\b/); + expect(ps.stdout).toMatch(new RegExp(`^\\s*${guestPid}\\s+`, "m")); + expect(ps.stdout).toContain("sleep 60"); + + const pkill = await ctx.kernel.exec("pkill -TERM -x sleep"); + expect(pkill.exitCode, pkill.stderr).toBe(0); + await expect(sleeper.wait()).resolves.not.toBe(0); + + const gone = await ctx.kernel.exec("pgrep -x sleep"); + expect(gone.exitCode).toBe(1); + }, 60_000); +}); diff --git a/software/procps/tsconfig.json b/software/procps/tsconfig.json new file mode 100644 index 0000000000..db523bef56 --- /dev/null +++ b/software/procps/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"] +} diff --git a/software/psmisc/agentos-package.json b/software/psmisc/agentos-package.json new file mode 100644 index 0000000000..b92d8c61dd --- /dev/null +++ b/software/psmisc/agentos-package.json @@ -0,0 +1,9 @@ +{ + "commands": ["pstree", "killall", "prtstat"], + "registry": { + "title": "psmisc", + "description": "Display process trees, inspect process state, and signal by name.", + "priority": 61, + "category": "core" + } +} diff --git a/software/psmisc/bin/killall b/software/psmisc/bin/killall new file mode 100644 index 0000000000..b999eff03d Binary files /dev/null and b/software/psmisc/bin/killall differ diff --git a/software/psmisc/bin/prtstat b/software/psmisc/bin/prtstat new file mode 100644 index 0000000000..bc333f5a3a Binary files /dev/null and b/software/psmisc/bin/prtstat differ diff --git a/software/psmisc/bin/pstree b/software/psmisc/bin/pstree new file mode 100644 index 0000000000..fb9290299b Binary files /dev/null and b/software/psmisc/bin/pstree differ diff --git a/software/psmisc/package.json b/software/psmisc/package.json new file mode 100644 index 0000000000..c979553f85 --- /dev/null +++ b/software/psmisc/package.json @@ -0,0 +1,29 @@ +{ + "name": "@agentos-software/psmisc", + "version": "0.0.1", + "type": "module", + "license": "GPL-2.0-or-later", + "description": "Upstream psmisc process tree, signaling, and stat commands for AgentOS VMs", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "files": ["dist", "!dist/package", "!dist/package.tar"], + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "scripts": { + "build": "agentos-toolchain stage --commands-dir ../../toolchain/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", + "check-types": "tsc --noEmit", + "test": "vitest run test/ --passWithNoTests" + }, + "devDependencies": { + "@agentos-software/manifest": "workspace:*", + "@rivet-dev/agentos-toolchain": "workspace:*", + "@types/node": "^22.10.2", + "typescript": "^5.9.2", + "@agentos/test-harness": "workspace:*", + "vitest": "^2.1.9" + } +} diff --git a/software/psmisc/src/index.ts b/software/psmisc/src/index.ts new file mode 100644 index 0000000000..dba100e2e1 --- /dev/null +++ b/software/psmisc/src/index.ts @@ -0,0 +1,5 @@ +import type { SoftwarePackageRef } from "@agentos-software/manifest"; + +const packagePath = new URL("./package.aospkg", import.meta.url).pathname; + +export default { packagePath } satisfies SoftwarePackageRef; diff --git a/software/psmisc/test/psmisc.test.ts b/software/psmisc/test/psmisc.test.ts new file mode 100644 index 0000000000..87785fdd5e --- /dev/null +++ b/software/psmisc/test/psmisc.test.ts @@ -0,0 +1,50 @@ +import { afterEach, expect, it } from "vitest"; +import { existsSync } from "node:fs"; +import { resolve } from "node:path"; +import { + C_BUILD_DIR, + COMMANDS_DIR, + createIntegrationKernel, + createWasmVmRuntime, + describeIf, +} from "@agentos/test-harness"; +import type { IntegrationKernelResult } from "@agentos/test-harness"; + +const hasCommands = ["pstree", "killall", "prtstat"].every((command) => + existsSync(resolve(C_BUILD_DIR, command)), +); + +describeIf(hasCommands, "upstream psmisc against live VM processes", () => { + let ctx: IntegrationKernelResult; + + afterEach(async () => { + await ctx?.dispose().catch(() => {}); + }); + + it("pstree and prtstat inspect a child, then killall terminates it", async () => { + ctx = await createIntegrationKernel({ runtimes: [] }); + await ctx.kernel.mount( + createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] }), + ); + + const sleeper = ctx.kernel.spawn("sleep", ["60"]); + await new Promise((resolveReady) => setTimeout(resolveReady, 100)); + + const tree = await ctx.kernel.exec("pstree -p"); + expect(tree.exitCode, tree.stderr).toBe(0); + const guestPid = Number.parseInt( + /sleep\((\d+)\)/.exec(tree.stdout)?.[1] ?? "0", + 10, + ); + expect(guestPid).toBeGreaterThan(0); + + const stat = await ctx.kernel.exec(`prtstat ${guestPid}`); + expect(stat.exitCode, stat.stderr).toBe(0); + expect(stat.stdout).toContain(`Process: sleep`); + expect(stat.stdout).toContain(`State:`); + + const killed = await ctx.kernel.exec("killall sleep"); + expect(killed.exitCode, killed.stderr).toBe(0); + await expect(sleeper.wait()).resolves.not.toBe(0); + }, 60_000); +}); diff --git a/software/psmisc/tsconfig.json b/software/psmisc/tsconfig.json new file mode 100644 index 0000000000..db523bef56 --- /dev/null +++ b/software/psmisc/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"] +} diff --git a/software/ssh/agentos-package.json b/software/ssh/agentos-package.json new file mode 100644 index 0000000000..ddb3783e30 --- /dev/null +++ b/software/ssh/agentos-package.json @@ -0,0 +1,12 @@ +{ + "commands": [ + "ssh" + ], + "registry": { + "title": "ssh", + "description": "OpenSSH client for remote command execution and git-over-ssh.", + "priority": 55, + "image": "/images/registry/ssh.svg", + "category": "networking" + } +} diff --git a/software/ssh/package.json b/software/ssh/package.json new file mode 100644 index 0000000000..bad5ce0988 --- /dev/null +++ b/software/ssh/package.json @@ -0,0 +1,35 @@ +{ + "name": "@agentos-software/ssh", + "version": "0.0.1", + "type": "module", + "license": "Apache-2.0", + "description": "OpenSSH client for AgentOS VMs (batch/key-based; also powers git-over-ssh)", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "files": [ + "dist", + "!dist/package", + "!dist/package.tar" + ], + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "scripts": { + "build": "agentos-toolchain stage --commands-dir ../../toolchain/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", + "check-types": "tsc --noEmit", + "test": "vitest run test/ --passWithNoTests" + }, + "devDependencies": { + "@agentos-software/manifest": "workspace:*", + "@rivet-dev/agentos-toolchain": "workspace:*", + "@types/node": "^22.10.2", + "@types/ssh2": "^1.15.1", + "typescript": "^5.9.2", + "@agentos/test-harness": "workspace:*", + "ssh2": "^1.16.0", + "vitest": "^2.1.9" + } +} diff --git a/software/ssh/src/index.ts b/software/ssh/src/index.ts new file mode 100644 index 0000000000..dba100e2e1 --- /dev/null +++ b/software/ssh/src/index.ts @@ -0,0 +1,5 @@ +import type { SoftwarePackageRef } from "@agentos-software/manifest"; + +const packagePath = new URL("./package.aospkg", import.meta.url).pathname; + +export default { packagePath } satisfies SoftwarePackageRef; diff --git a/software/ssh/test/ssh.test.ts b/software/ssh/test/ssh.test.ts new file mode 100644 index 0000000000..13731cc825 --- /dev/null +++ b/software/ssh/test/ssh.test.ts @@ -0,0 +1,455 @@ +/** + * Integration tests for the real OpenSSH ssh client (10.4p1, built + * --without-openssl: ed25519 + curve25519-sha256 + chacha20-poly1305). + * + * Mirrors the git HTTPS suite's loopback-server harness: an in-test SSH + * server (the pure-JS `ssh2` package) listens on a host loopback port that + * the kernel exempts, and the WASM ssh client connects out through the + * kernel's host_net path. Covers batch/key-based exec, host-key + * verification (known_hosts, StrictHostKeyChecking=accept-new), publickey + * auth failure, and git-over-ssh (clone + push against host + * git-upload-pack/git-receive-pack). + */ + +import { describe, it, expect, afterEach, beforeAll, afterAll, vi } from 'vitest'; +import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { resolve, join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { spawn, spawnSync } from 'node:child_process'; +import { Server as SshServer, utils as sshUtils } from 'ssh2'; +import type { Connection } from 'ssh2'; +import { createWasmVmRuntime } from '@agentos/test-harness'; +import { + allowAll, + COMMANDS_DIR, + createInMemoryFileSystem, + createKernel, + describeIf, + hasWasmBinaries, +} from '@agentos/test-harness'; +import type { Kernel } from '@agentos/test-harness'; + +vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); + +const hasSsh = hasWasmBinaries && existsSync(resolve(COMMANDS_DIR, 'ssh')); +const hasGit = hasWasmBinaries && existsSync(resolve(COMMANDS_DIR, 'git')); +const hasHostGit = spawnSync('git', ['--version'], { stdio: 'ignore' }).status === 0; + +const SSH_USER = 'agentos'; + +interface TestKeys { + hostKey: ReturnType; + clientKey: ReturnType; + /** A second client keypair the server does NOT authorize. */ + wrongClientKey: ReturnType; + /** A second host key used to simulate a changed/unknown server identity. */ + otherHostKey: ReturnType; +} + +function generateKeys(): TestKeys { + return { + hostKey: sshUtils.generateKeyPairSync('ed25519'), + clientKey: sshUtils.generateKeyPairSync('ed25519'), + wrongClientKey: sshUtils.generateKeyPairSync('ed25519'), + otherHostKey: sshUtils.generateKeyPairSync('ed25519'), + }; +} + +/** Standard ssh2 publickey-auth handler restricted to one authorized key. */ +function installAuthHandler(client: Connection, authorizedPublicKey: string) { + const allowed = sshUtils.parseKey(authorizedPublicKey); + if (allowed instanceof Error) throw allowed; + client.on('authentication', (ctx) => { + if (ctx.method !== 'publickey') { + return ctx.reject(['publickey']); + } + const matches = + ctx.key.algo === allowed.type && + ctx.key.data.equals(allowed.getPublicSSH()); + if (!matches) { + return ctx.reject(['publickey']); + } + if (ctx.signature && ctx.blob) { + if (allowed.verify(ctx.blob, ctx.signature, ctx.hashAlgo) === true) { + return ctx.accept(); + } + return ctx.reject(['publickey']); + } + // pk-check phase (no signature yet): tell the client the key is OK. + return ctx.accept(); + }); +} + +/** exec handler: `echo hello`-style canned command execution. */ +function installEchoExecHandler(client: Connection) { + client.on('ready', () => { + client.on('session', (acceptSession) => { + const session = acceptSession(); + session.on('exec', (acceptExec, _reject, info) => { + const stream = acceptExec(); + if (info.command === 'echo hello') { + stream.write('hello\n'); + stream.exit(0); + } else { + stream.stderr.write(`unknown test command: ${info.command}\n`); + stream.exit(127); + } + stream.end(); + }); + }); + }); +} + +/** + * exec handler bridging `git-upload-pack '/x.git'` / `git-receive-pack ...` + * to the host git against a bare repo root — an in-test stand-in for a real + * SSH git host (what git-shell does on a server). + */ +function installGitExecHandler(client: Connection, repoRoot: string) { + client.on('ready', () => { + client.on('session', (acceptSession) => { + const session = acceptSession(); + session.on('exec', (acceptExec, reject, info) => { + const match = /^(git-upload-pack|git-receive-pack|git-upload-archive) '(.*)'$/.exec( + info.command, + ); + if (!match) { + const stream = acceptExec(); + stream.stderr.write(`unsupported command: ${info.command}\n`); + stream.exit(128); + stream.end(); + return; + } + const [, service, requestedPath] = match; + const repoPath = join(repoRoot, requestedPath.replace(/^\/+/, '')); + const stream = acceptExec(); + const child = spawn('git', [service.replace(/^git-/, ''), repoPath]); + stream.pipe(child.stdin); + child.stdout.pipe(stream, { end: false }); + child.stderr.pipe(stream.stderr, { end: false }); + child.on('close', (code) => { + stream.exit(code ?? 1); + stream.end(); + }); + child.on('error', () => { + stream.exit(127); + stream.end(); + }); + }); + }); + }); +} + +async function listen(server: SshServer): Promise { + await new Promise((r) => server.listen(0, '127.0.0.1', r)); + return (server.address() as import('node:net').AddressInfo).port; +} + +async function createSshKernel(loopbackExemptPorts: number[]) { + const vfs = createInMemoryFileSystem(); + await (vfs as any).chmod('/', 0o1777); + await vfs.mkdir('/tmp', { recursive: true }); + await (vfs as any).chmod('/tmp', 0o1777); + const kernel = createKernel({ + filesystem: vfs, + permissions: allowAll, + loopbackExemptPorts, + syncFilesystemOnDispose: false, + }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); + return { kernel, vfs, dispose: () => kernel.dispose() }; +} + +async function run( + kernel: Kernel, + cmd: string, +): Promise<{ stdout: string; stderr: string; exitCode: number }> { + const r = await kernel.exec(cmd); + if (r.exitCode !== 0) { + throw new Error( + `Command failed (exit ${r.exitCode}): ${cmd}\nstdout: ${r.stdout}\nstderr: ${r.stderr}`, + ); + } + return r; +} + +/** + * Resolve the guest user's home directory (ssh resolves `~` through + * getpwuid(getuid())->pw_dir, which the runtime keeps aligned with $HOME). + */ +async function guestHome(kernel: Kernel): Promise { + const r = await run(kernel, "sh -c 'echo $HOME'"); + const home = r.stdout.trim(); + expect(home).toMatch(/^\//); + return home; +} + +/** Seed ~/.ssh with an identity and (optionally) a known_hosts line. */ +async function seedSshDir( + kernel: Kernel, + vfs: any, + home: string, + privateKey: string, + knownHostsLine?: string, +): Promise { + const sshDir = `${home}/.ssh`; + await vfs.mkdir(sshDir, { recursive: true }); + await vfs.chmod(sshDir, 0o700); + await kernel.writeFile(`${sshDir}/id_ed25519`, `${privateKey}\n`); + await vfs.chmod(`${sshDir}/id_ed25519`, 0o600); + if (knownHostsLine !== undefined) { + await kernel.writeFile(`${sshDir}/known_hosts`, `${knownHostsLine}\n`); + await vfs.chmod(`${sshDir}/known_hosts`, 0o600); + } + return sshDir; +} + +function knownHostsEntry(port: number, hostPublicKey: string): string { + // `[host]:port` hashing syntax from sshd(8) AUTHORIZED_KEYS/known_hosts + // format; non-default ports always use the bracketed form. + return `[127.0.0.1]:${port} ${hostPublicKey}`; +} + +// TODO(P6): requires the ssh WASM artifact, intentionally excluded from the +// fast software-build gate (same as git). +describeIf(hasSsh, 'ssh command', () => { + let kernel: Kernel; + let vfs: any; + let dispose: (() => Promise) | undefined; + + afterEach(async () => { + await dispose?.(); + dispose = undefined; + }); + + it('ssh -V reports the real OpenSSH version without OpenSSL', async () => { + ({ kernel, vfs, dispose } = await createSshKernel([])); + const r = await kernel.exec('ssh -V'); + expect(r.exitCode).toBe(0); + const banner = `${r.stdout}${r.stderr}`; + expect(banner).toMatch(/OpenSSH_10\.4/); + expect(banner).toMatch(/without OpenSSL/i); + }); + + describe('against an in-test ssh2 server', () => { + let keys: TestKeys; + let server: SshServer; + let port: number; + + beforeAll(async () => { + keys = generateKeys(); + server = new SshServer({ hostKeys: [keys.hostKey.private] }, (client) => { + installAuthHandler(client, keys.clientKey.public); + installEchoExecHandler(client); + }); + port = await listen(server); + }); + + afterAll(async () => { + await new Promise((r) => server.close(() => r())); + }); + + const sshCmd = (extra: string) => + `ssh -T -o BatchMode=yes ${extra} -p ${port} ${SSH_USER}@127.0.0.1 echo hello`; + + it('runs a remote command with ed25519 publickey auth and known_hosts', async () => { + ({ kernel, vfs, dispose } = await createSshKernel([port])); + const home = await guestHome(kernel); + await seedSshDir( + kernel, + vfs, + home, + keys.clientKey.private, + knownHostsEntry(port, keys.hostKey.public), + ); + + const r = await kernel.exec(sshCmd('')); + expect(r.stderr).not.toMatch(/setsockopt/i); + expect(r.stdout).toBe('hello\n'); + expect(r.exitCode).toBe(0); + }); + + it('propagates the remote exit status', async () => { + ({ kernel, vfs, dispose } = await createSshKernel([port])); + const home = await guestHome(kernel); + await seedSshDir( + kernel, + vfs, + home, + keys.clientKey.private, + knownHostsEntry(port, keys.hostKey.public), + ); + + const r = await kernel.exec( + `ssh -T -o BatchMode=yes -p ${port} ${SSH_USER}@127.0.0.1 false-command`, + ); + expect(r.exitCode).toBe(127); + expect(r.stderr).toContain('unknown test command'); + }); + + it('fails publickey auth with an unauthorized client key', async () => { + ({ kernel, vfs, dispose } = await createSshKernel([port])); + const home = await guestHome(kernel); + await seedSshDir( + kernel, + vfs, + home, + keys.wrongClientKey.private, + knownHostsEntry(port, keys.hostKey.public), + ); + + const r = await kernel.exec(sshCmd('')); + expect(r.exitCode).not.toBe(0); + expect(r.stderr).toMatch(/Permission denied \(publickey\)/i); + expect(r.stdout).not.toContain('hello'); + }); + + it('fails host key verification when known_hosts pins a different key', async () => { + ({ kernel, vfs, dispose } = await createSshKernel([port])); + const home = await guestHome(kernel); + await seedSshDir( + kernel, + vfs, + home, + keys.clientKey.private, + knownHostsEntry(port, keys.otherHostKey.public), + ); + + const r = await kernel.exec(sshCmd('')); + expect(r.exitCode).not.toBe(0); + expect(r.stderr).toMatch( + /REMOTE HOST IDENTIFICATION HAS CHANGED|Host key verification failed/i, + ); + expect(r.stdout).not.toContain('hello'); + }); + + it('fails closed in BatchMode when the host key is unknown', async () => { + ({ kernel, vfs, dispose } = await createSshKernel([port])); + const home = await guestHome(kernel); + await seedSshDir(kernel, vfs, home, keys.clientKey.private); + + const r = await kernel.exec(sshCmd('')); + expect(r.exitCode).not.toBe(0); + expect(r.stderr).toMatch(/Host key verification failed/i); + }); + + it('StrictHostKeyChecking=accept-new succeeds and records the host key', async () => { + ({ kernel, vfs, dispose } = await createSshKernel([port])); + const home = await guestHome(kernel); + const sshDir = await seedSshDir(kernel, vfs, home, keys.clientKey.private); + + const r = await kernel.exec(sshCmd('-o StrictHostKeyChecking=accept-new')); + expect(r.stdout).toBe('hello\n'); + expect(r.exitCode).toBe(0); + expect(r.stderr).toMatch(/Permanently added/i); + + const knownHosts = new TextDecoder().decode( + await kernel.readFile(`${sshDir}/known_hosts`), + ); + const hostKeyBlob = keys.hostKey.public.split(/\s+/)[1]; + expect(knownHosts).toContain(`[127.0.0.1]:${port}`); + expect(knownHosts).toContain(hostKeyBlob); + }); + }); + + // git-over-ssh: the WASM git execs the WASM ssh from PATH (git connect.c), + // which tunnels git-upload-pack / git-receive-pack to the host-side bare + // repo behind the ssh2 server. + // + // This also regresses mixed polling in the runtime: ssh polls a dup'd stdin + // pipe alongside its host-net socket while Git waits for the remote helper. + describeIf(hasGit && hasHostGit, 'git-over-ssh clone/push', () => { + let keys: TestKeys; + let server: SshServer; + let port: number; + let repoRoot: string; + + const gitConfig = [ + '-c safe.directory=*', + '-c init.defaultBranch=main', + '-c user.name=agentos', + '-c user.email=agentos@example.invalid', + ].join(' '); + const git = (args: string) => `git ${gitConfig} ${args}`; + + function runHostGit(args: string[], cwd?: string) { + const result = spawnSync('git', args, { cwd, encoding: 'utf8' }); + if (result.status !== 0) { + throw new Error( + `host git failed: git ${args.join(' ')}\nstdout: ${result.stdout}\nstderr: ${result.stderr}`, + ); + } + } + + beforeAll(async () => { + keys = generateKeys(); + repoRoot = mkdtempSync(join(tmpdir(), 'agentos-git-ssh-')); + const worktree = join(repoRoot, 'worktree'); + const origin = join(repoRoot, 'origin.git'); + + runHostGit(['-c', 'init.defaultBranch=main', 'init', worktree]); + writeFileSync(join(worktree, 'README.md'), 'remote ssh clone\n'); + runHostGit(['-C', worktree, 'add', 'README.md']); + runHostGit([ + '-C', worktree, + '-c', 'user.name=agentos', '-c', 'user.email=agentos@example.invalid', + 'commit', '-m', 'seed', + ]); + runHostGit(['clone', '--bare', worktree, origin]); + + server = new SshServer({ hostKeys: [keys.hostKey.private] }, (client) => { + installAuthHandler(client, keys.clientKey.public); + installGitExecHandler(client, repoRoot); + }); + port = await listen(server); + }); + + afterAll(async () => { + if (server) await new Promise((r) => server.close(() => r())); + rmSync(repoRoot, { recursive: true, force: true }); + }); + + it('clones and pushes over ssh://', async () => { + ({ kernel, vfs, dispose } = await createSshKernel([port])); + const home = await guestHome(kernel); + await seedSshDir( + kernel, + vfs, + home, + keys.clientKey.private, + knownHostsEntry(port, keys.hostKey.public), + ); + + const url = `ssh://${SSH_USER}@127.0.0.1:${port}/origin.git`; + + const cloned = await kernel.exec(git(`clone ${url} /tmp/clone`)); + expect(cloned.exitCode, cloned.stderr).toBe(0); + const readme = new TextDecoder().decode( + await kernel.readFile('/tmp/clone/README.md'), + ); + expect(readme).toBe('remote ssh clone\n'); + const head = new TextDecoder().decode( + await kernel.readFile('/tmp/clone/.git/HEAD'), + ); + expect(head.trim()).toBe('ref: refs/heads/main'); + + // Push a new commit back over the same transport. + await kernel.writeFile('/tmp/clone/pushed.txt', 'pushed over ssh\n'); + await run(kernel, git('-C /tmp/clone add pushed.txt')); + await run(kernel, git("-C /tmp/clone commit -m 'push over ssh'")); + const pushed = await kernel.exec( + git('-C /tmp/clone push origin HEAD:refs/heads/ssh-push'), + ); + expect(pushed.exitCode).toBe(0); + + // Verify the ref really landed in the host-side bare repo. + const originRef = spawnSync( + 'git', + ['-C', join(repoRoot, 'origin.git'), 'rev-parse', '--verify', 'refs/heads/ssh-push'], + { encoding: 'utf8' }, + ); + expect(originRef.status).toBe(0); + expect(originRef.stdout.trim()).toMatch(/^[0-9a-f]{40,64}$/); + }); + }); +}); diff --git a/software/ssh/tsconfig.json b/software/ssh/tsconfig.json new file mode 100644 index 0000000000..03ce790ab7 --- /dev/null +++ b/software/ssh/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"] +} diff --git a/software/wget/native/c/overlay/src/wasi_ssl.c b/software/wget/native/c/overlay/src/wasi_ssl.c new file mode 100644 index 0000000000..222ad09429 --- /dev/null +++ b/software/wget/native/c/overlay/src/wasi_ssl.c @@ -0,0 +1,535 @@ +/* In-guest TLS backend for GNU Wget on wasm32-wasip1, built on mbedTLS. + + GNU Wget has no upstream mbedTLS backend; its TLS abstraction is the four + functions declared in src/ssl.h (ssl_init, ssl_cleanup, ssl_connect_wget, + ssl_check_certificate). This file implements those four against mbedTLS, + performing a real TLS handshake and X.509 chain + hostname verification + entirely inside the guest -- the sidecar is a dumb ciphertext pipe. + + The already-connected TCP file descriptor handed to ssl_connect_wget is a + real socket carried by the patched wasi-libc sysroot (host_net imports), so + the mbedTLS BIO callbacks simply read()/write() that fd. On success the fd is + registered with Wget's transport layer (fd_register_transport) so that all + subsequent fd_read/fd_write/fd_peek calls flow through the TLS session. + + Trust configuration mirrors Linux Wget: certificates are verified against + /etc/ssl/certs/ca-certificates.crt by default, --ca-certificate + (opt.ca_cert) and --ca-directory (opt.ca_directory) override the trust + anchors, --crl-file (opt.crl_file) adds a revocation list, and + --no-check-certificate (opt.check_cert != CHECK_CERT_ON) downgrades a + verification failure to a warning. A failed handshake or a rejected + certificate makes the relevant function return false, so http.c reports + CONSSLERR / VERIFCERTERR exactly as with the GnuTLS/OpenSSL backends. */ + +#include "wget.h" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "connect.h" +#include "log.h" +#include "ssl.h" +#include "url.h" +#include "utils.h" + +/* Debian-shaped default trust store, seeded into the VM by the native + bootstrap. Matches curl's compile-time CA bundle default and OpenSSL's + OPENSSLDIR resolution on Debian. */ +#ifndef WASI_TLS_DEFAULT_CA_BUNDLE +# define WASI_TLS_DEFAULT_CA_BUNDLE "/etc/ssl/certs/ca-certificates.crt" +#endif + +/* The TLS transport singleton, defined after the I/O callbacks below. */ +static struct transport_implementation wasi_tls_transport; + +struct wasi_ssl_context +{ + mbedtls_ssl_context ssl; + mbedtls_ssl_config conf; + mbedtls_x509_crt cacert; + mbedtls_x509_crl crl; + bool have_crl; + mbedtls_ctr_drbg_context ctr_drbg; + mbedtls_entropy_context entropy; + int fd; + int last_err; /* last mbedTLS error, for errstr */ + unsigned char *peekbuf; /* buffered-but-unconsumed plaintext */ + int peeklen; + int peekcap; +}; + +/* mbedTLS BIO send/recv over the raw, already-connected socket fd. Sockets are + blocking on wasip1 (Wget only flips them non-blocking on Windows), but we map + EAGAIN/EINTR anyway so the handshake and I/O paths stay correct if a socket + is ever non-blocking. */ + +static int +wasi_bio_send (void *arg, const unsigned char *buf, size_t len) +{ + struct wasi_ssl_context *ctx = arg; + ssize_t n; + + do + n = write (ctx->fd, buf, len); + while (n < 0 && errno == EINTR); + + if (n >= 0) + return (int) n; + if (errno == EAGAIN || errno == EWOULDBLOCK) + return MBEDTLS_ERR_SSL_WANT_WRITE; + return MBEDTLS_ERR_NET_SEND_FAILED; +} + +static int +wasi_bio_recv (void *arg, unsigned char *buf, size_t len) +{ + struct wasi_ssl_context *ctx = arg; + ssize_t n; + + do + n = read (ctx->fd, buf, len); + while (n < 0 && errno == EINTR); + + if (n >= 0) + return (int) n; + if (errno == EAGAIN || errno == EWOULDBLOCK) + return MBEDTLS_ERR_SSL_WANT_READ; + return MBEDTLS_ERR_NET_RECV_FAILED; +} + +/* Global init. mbedTLS keeps no process-wide state we must set up, so this is + a no-op that simply reports readiness, like the GnuTLS backend. */ +bool +ssl_init (void) +{ + return true; +} + +void +ssl_cleanup (void) +{ +} + +/* Map --secure-protocol onto the mbedTLS min/max TLS version knobs. mbedTLS + 3.6 only speaks TLS 1.2 and 1.3, so the legacy SSLv3/TLS1.0/1.1 selectors + pin the floor at TLS 1.2 (the lowest still supported), matching how a modern + Wget build behaves. */ +static void +wasi_apply_secure_protocol (mbedtls_ssl_config *conf) +{ + switch (opt.secure_protocol) + { + case secure_protocol_tlsv1_3: + mbedtls_ssl_conf_min_tls_version (conf, MBEDTLS_SSL_VERSION_TLS1_3); + mbedtls_ssl_conf_max_tls_version (conf, MBEDTLS_SSL_VERSION_TLS1_3); + break; + case secure_protocol_tlsv1_2: + mbedtls_ssl_conf_min_tls_version (conf, MBEDTLS_SSL_VERSION_TLS1_2); + mbedtls_ssl_conf_max_tls_version (conf, MBEDTLS_SSL_VERSION_TLS1_2); + break; + case secure_protocol_tlsv1: + case secure_protocol_tlsv1_1: + case secure_protocol_sslv2: + case secure_protocol_sslv3: + /* Not supported by mbedTLS 3.6; fall back to the lowest available. */ + mbedtls_ssl_conf_min_tls_version (conf, MBEDTLS_SSL_VERSION_TLS1_2); + break; + case secure_protocol_auto: + case secure_protocol_pfs: + default: + /* Library defaults: TLS 1.2 .. 1.3. */ + break; + } +} + +/* Load the trust anchors exactly like Linux Wget: --ca-certificate and + --ca-directory when given, otherwise the seeded Debian bundle. Returns the + number of certificates parsed (>= 0) or a negative mbedTLS error. */ +static int +wasi_load_trust (struct wasi_ssl_context *ctx) +{ + int loaded = 0; + int ret; + + if (opt.ca_cert) + { + ret = mbedtls_x509_crt_parse_file (&ctx->cacert, opt.ca_cert); + if (ret < 0) + return ret; + loaded += 1; + } + + if (opt.ca_directory && 0 != strcmp (opt.ca_directory, "")) + { + ret = mbedtls_x509_crt_parse_path (&ctx->cacert, opt.ca_directory); + /* parse_path returns the number of files that failed to parse as a + positive value; only a negative value is a hard error. */ + if (ret < 0) + return ret; + loaded += 1; + } + + if (loaded == 0) + { + ret = mbedtls_x509_crt_parse_file (&ctx->cacert, WASI_TLS_DEFAULT_CA_BUNDLE); + if (ret < 0) + return ret; + loaded += 1; + } + + return loaded; +} + +/* Perform the TLS handshake on FD and, on success, register the TLS transport + so Wget's fd_* helpers use it. CONTINUE_SESSION (session resumption) is not + supported here; http.c always passes NULL. Returns true on success. */ +bool +ssl_connect_wget (int fd, const char *hostname, int *continue_session) +{ + struct wasi_ssl_context *ctx; + int ret; + + (void) continue_session; + + DEBUGP (("Initiating SSL handshake (mbedTLS).\n")); + + ctx = xnew0 (struct wasi_ssl_context); + ctx->fd = fd; + + mbedtls_ssl_init (&ctx->ssl); + mbedtls_ssl_config_init (&ctx->conf); + mbedtls_x509_crt_init (&ctx->cacert); + mbedtls_x509_crl_init (&ctx->crl); + mbedtls_ctr_drbg_init (&ctx->ctr_drbg); + mbedtls_entropy_init (&ctx->entropy); + + ret = mbedtls_ctr_drbg_seed (&ctx->ctr_drbg, mbedtls_entropy_func, + &ctx->entropy, + (const unsigned char *) "agentos-wget-tls", 16); + if (ret != 0) + goto error; + + /* Load trust anchors. A failure to read the bundle is only fatal when the + user asked us to verify; with --no-check-certificate we proceed with an + empty trust store (verification is skipped in ssl_check_certificate). */ + ret = wasi_load_trust (ctx); + if (ret < 0 && opt.check_cert == CHECK_CERT_ON) + { + char errbuf[128]; + mbedtls_strerror (ret, errbuf, sizeof errbuf); + logprintf (LOG_NOTQUIET, + _("Could not load CA certificates: %s\n"), errbuf); + goto error; + } + + if (opt.crl_file) + { + ret = mbedtls_x509_crl_parse_file (&ctx->crl, opt.crl_file); + if (ret != 0 && opt.check_cert == CHECK_CERT_ON) + { + char errbuf[128]; + mbedtls_strerror (ret, errbuf, sizeof errbuf); + logprintf (LOG_NOTQUIET, + _("Could not load CRL from %s: %s\n"), + opt.crl_file, errbuf); + goto error; + } + if (ret == 0) + ctx->have_crl = true; + } + + ret = mbedtls_ssl_config_defaults (&ctx->conf, MBEDTLS_SSL_IS_CLIENT, + MBEDTLS_SSL_TRANSPORT_STREAM, + MBEDTLS_SSL_PRESET_DEFAULT); + if (ret != 0) + goto error; + + wasi_apply_secure_protocol (&ctx->conf); + + /* Verify optionally: the handshake always completes and records the chain + + hostname result, which ssl_check_certificate reads and turns into a + pass/fail decision honoring opt.check_cert -- the same split the OpenSSL + backend uses (SSL_VERIFY_NONE at handshake, manual check afterwards). */ + mbedtls_ssl_conf_authmode (&ctx->conf, MBEDTLS_SSL_VERIFY_OPTIONAL); + mbedtls_ssl_conf_ca_chain (&ctx->conf, &ctx->cacert, + ctx->have_crl ? &ctx->crl : NULL); + mbedtls_ssl_conf_rng (&ctx->conf, mbedtls_ctr_drbg_random, &ctx->ctr_drbg); + + ret = mbedtls_ssl_setup (&ctx->ssl, &ctx->conf); + if (ret != 0) + goto error; + + /* SNI + the name checked during verification. Covers DNS and IP-address + SANs (mbedTLS matches an IP literal against iPAddress SAN entries). */ + ret = mbedtls_ssl_set_hostname (&ctx->ssl, hostname); + if (ret != 0) + goto error; + + mbedtls_ssl_set_bio (&ctx->ssl, ctx, wasi_bio_send, wasi_bio_recv, NULL); + + do + { + ret = mbedtls_ssl_handshake (&ctx->ssl); + if (ret == MBEDTLS_ERR_SSL_WANT_READ) + select_fd (fd, opt.read_timeout, WAIT_FOR_READ); + else if (ret == MBEDTLS_ERR_SSL_WANT_WRITE) + select_fd (fd, opt.read_timeout, WAIT_FOR_WRITE); + } + while (ret == MBEDTLS_ERR_SSL_WANT_READ || ret == MBEDTLS_ERR_SSL_WANT_WRITE); + + if (ret != 0) + { + char errbuf[128]; + mbedtls_strerror (ret, errbuf, sizeof errbuf); + DEBUGP (("SSL handshake failed: %s\n", errbuf)); + logprintf (LOG_NOTQUIET, _("SSL handshake failed: %s\n"), errbuf); + goto error; + } + + /* Register FD with Wget's transport layer so fd_read/fd_write/fd_peek use + our TLS callbacks from here on. */ + fd_register_transport (fd, &wasi_tls_transport, ctx); + DEBUGP (("Handshake successful; TLS registered on socket %d\n", fd)); + + return true; + + error: + mbedtls_ssl_free (&ctx->ssl); + mbedtls_ssl_config_free (&ctx->conf); + mbedtls_x509_crt_free (&ctx->cacert); + mbedtls_x509_crl_free (&ctx->crl); + mbedtls_ctr_drbg_free (&ctx->ctr_drbg); + mbedtls_entropy_free (&ctx->entropy); + xfree (ctx); + return false; +} + +/* --- Wget transport implementation over the TLS session --- */ + +static int +wasi_tls_read (int fd, char *buf, int bufsize, void *arg, double timeout) +{ + struct wasi_ssl_context *ctx = arg; + int ret; + + /* Serve any peeked-but-unconsumed plaintext first. */ + if (ctx->peeklen > 0) + { + int n = ctx->peeklen < bufsize ? ctx->peeklen : bufsize; + memcpy (buf, ctx->peekbuf, n); + if (n < ctx->peeklen) + memmove (ctx->peekbuf, ctx->peekbuf + n, ctx->peeklen - n); + ctx->peeklen -= n; + return n; + } + + if (timeout == -1) + timeout = opt.read_timeout; + if (timeout && mbedtls_ssl_get_bytes_avail (&ctx->ssl) == 0) + { + int sel = select_fd (fd, timeout, WAIT_FOR_READ); + if (sel <= 0) + return sel; /* 0 = timeout, -1 = error; matches Wget's expectation */ + } + + do + ret = mbedtls_ssl_read (&ctx->ssl, (unsigned char *) buf, bufsize); + while (ret == MBEDTLS_ERR_SSL_WANT_READ || ret == MBEDTLS_ERR_SSL_WANT_WRITE); + + if (ret == MBEDTLS_ERR_SSL_PEER_CLOSE_NOTIFY) + return 0; + if (ret < 0) + { + ctx->last_err = ret; + return -1; + } + return ret; +} + +static int +wasi_tls_write (int fd _GL_UNUSED, char *buf, int bufsize, void *arg) +{ + struct wasi_ssl_context *ctx = arg; + int written = 0; + + while (written < bufsize) + { + int ret = mbedtls_ssl_write (&ctx->ssl, + (const unsigned char *) buf + written, + bufsize - written); + if (ret == MBEDTLS_ERR_SSL_WANT_READ || ret == MBEDTLS_ERR_SSL_WANT_WRITE) + continue; + if (ret < 0) + { + ctx->last_err = ret; + return -1; + } + written += ret; + } + return written; +} + +static int +wasi_tls_poll (int fd, double timeout, int wait_for, void *arg) +{ + struct wasi_ssl_context *ctx = arg; + + if ((wait_for & WAIT_FOR_READ) + && (ctx->peeklen > 0 || mbedtls_ssl_get_bytes_avail (&ctx->ssl) > 0)) + return 1; + if (timeout == -1) + timeout = opt.read_timeout; + return select_fd (fd, timeout, wait_for); +} + +static int +wasi_tls_peek (int fd, char *buf, int bufsize, void *arg, double timeout) +{ + struct wasi_ssl_context *ctx = arg; + int n; + + /* Mirror recv(MSG_PEEK)/SSL_peek semantics: preview data without consuming + it, returning as soon as *some* data is available -- one TLS record's + worth. Never try to fill BUFSIZE (that would block on a keep-alive + connection once the whole response fits in one record). Wget's + fd_read_hunk consumes each preview and re-peeks for the next record, so + returning one chunk at a time is sufficient and cannot deadlock. + + Data that mbedtls_ssl_read pulls off the wire here is retained in peekbuf + so wasi_tls_read drains it first -- i.e. the "peek" does not lose it. */ + if (ctx->peeklen == 0) + { + int ret; + + if (ctx->peekcap < bufsize) + { + ctx->peekbuf = xrealloc (ctx->peekbuf, bufsize); + ctx->peekcap = bufsize; + } + + if (timeout == -1) + timeout = opt.read_timeout; + if (timeout && mbedtls_ssl_get_bytes_avail (&ctx->ssl) == 0) + { + int sel = select_fd (fd, timeout, WAIT_FOR_READ); + if (sel <= 0) + return sel; /* 0 = timeout, -1 = error */ + } + + do + ret = mbedtls_ssl_read (&ctx->ssl, ctx->peekbuf, bufsize); + while (ret == MBEDTLS_ERR_SSL_WANT_READ + || ret == MBEDTLS_ERR_SSL_WANT_WRITE); + + if (ret == MBEDTLS_ERR_SSL_PEER_CLOSE_NOTIFY) + return 0; + if (ret < 0) + { + ctx->last_err = ret; + return -1; + } + ctx->peeklen = ret; + } + + n = ctx->peeklen < bufsize ? ctx->peeklen : bufsize; + if (n > 0) + memcpy (buf, ctx->peekbuf, n); + return n; +} + +static const char * +wasi_tls_errstr (int fd _GL_UNUSED, void *arg) +{ + struct wasi_ssl_context *ctx = arg; + static char errbuf[160]; + + if (ctx->last_err == 0) + return NULL; + mbedtls_strerror (ctx->last_err, errbuf, sizeof errbuf); + return errbuf; +} + +static void +wasi_tls_close (int fd, void *arg) +{ + struct wasi_ssl_context *ctx = arg; + + mbedtls_ssl_close_notify (&ctx->ssl); + mbedtls_ssl_free (&ctx->ssl); + mbedtls_ssl_config_free (&ctx->conf); + mbedtls_x509_crt_free (&ctx->cacert); + mbedtls_x509_crl_free (&ctx->crl); + mbedtls_ctr_drbg_free (&ctx->ctr_drbg); + mbedtls_entropy_free (&ctx->entropy); + xfree (ctx->peekbuf); + xfree (ctx); + + close (fd); + DEBUGP (("Closed %d/SSL (mbedTLS)\n", fd)); +} + +static struct transport_implementation wasi_tls_transport = { + wasi_tls_read, wasi_tls_write, wasi_tls_poll, + wasi_tls_peek, wasi_tls_errstr, wasi_tls_close +}; + +/* Verify the peer certificate against the configured trust anchors and check + that it matches HOST. Reads the verify result recorded during the handshake. + Returns false only when verification failed AND the user requested checking + (opt.check_cert == CHECK_CERT_ON); otherwise it warns and returns true, + matching Wget's --no-check-certificate semantics. */ +bool +ssl_check_certificate (int fd, const char *host) +{ + struct wasi_ssl_context *ctx = fd_transport_context (fd); + uint32_t flags; + bool success; + const char *severity = opt.check_cert ? _("ERROR") : _("WARNING"); + + if (!ctx) + return opt.check_cert != CHECK_CERT_ON; + + flags = mbedtls_ssl_get_verify_result (&ctx->ssl); + success = (flags == 0); + + if (!success) + { + char vbuf[512]; + int n = mbedtls_x509_crt_verify_info (vbuf, sizeof vbuf, " ", flags); + if (n < 0) + { + vbuf[0] = '\0'; + n = 0; + } + + logprintf (LOG_NOTQUIET, + _("%s: cannot verify %s's certificate:\n"), + severity, quote (host)); + if (n > 0) + logprintf (LOG_NOTQUIET, "%s", vbuf); + + if (opt.check_cert == CHECK_CERT_ON) + logprintf (LOG_NOTQUIET, + _("To connect to %s insecurely, use `--no-check-certificate'.\n"), + quote (host)); + } + else + DEBUGP (("X509 certificate successfully verified and matches host %s\n", + quote (host))); + + return opt.check_cert == CHECK_CERT_ON ? success : true; +} + +/* + * vim: tabstop=2 shiftwidth=2 softtabstop=2 + */ diff --git a/software/wget/test/wget.test.ts b/software/wget/test/wget.test.ts index c5c13ff20b..05d20d0e72 100644 --- a/software/wget/test/wget.test.ts +++ b/software/wget/test/wget.test.ts @@ -1,12 +1,26 @@ import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; -import { existsSync } from "node:fs"; +import { execSync } from "node:child_process"; +import { + existsSync, + mkdtempSync, + readFileSync, + rmSync, + unlinkSync, + writeFileSync, +} from "node:fs"; import { createServer, type IncomingMessage, type Server, type ServerResponse, } from "node:http"; -import { resolve } from "node:path"; +import { + createServer as createHttpsServer, + type Server as HttpsServer, +} from "node:https"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { gzipSync } from "node:zlib"; import { createWasmVmRuntime } from "@agentos/test-harness"; import { allowAll, @@ -15,6 +29,7 @@ import { createInMemoryFileSystem, createKernel, describeIf, + itIf, } from "@agentos/test-harness"; import type { Kernel } from "@agentos/test-harness"; @@ -26,10 +41,96 @@ const hasWgetBinary = WGET_COMMAND_DIRS.some((dir) => ); const WGET_EXEC_TIMEOUT_MS = 10_000; +let hasOpenssl = false; +try { + execSync("openssl version", { stdio: "pipe" }); + hasOpenssl = true; +} catch { + hasOpenssl = false; +} + +// A long, highly compressible payload so the gzipped body is clearly distinct +// from the plaintext (proving wget actually inflated it via zlib). +const COMPRESSION_PAYLOAD = + "agentos-wget-compression " + + "the quick brown fox jumps over the lazy dog. ".repeat(64); + +// Build a real CA and a leaf cert signed by it, with a SAN covering the +// 127.0.0.1 loopback endpoint the tests connect to. This lets wget's mbedTLS +// backend perform genuine chain + hostname verification, exactly like Linux +// wget against a private CA. +function makeCaSignedCert(caCommonName: string): { + caPem: string; + serverKey: string; + serverCert: string; +} { + const dir = mkdtempSync(join(tmpdir(), "wget-ca-")); + try { + execSync( + `openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "${dir}/ca.key" 2>/dev/null`, + ); + execSync( + `openssl req -x509 -new -key "${dir}/ca.key" -days 3650 -subj "/CN=${caCommonName}" -out "${dir}/ca.crt" 2>/dev/null`, + ); + execSync( + `openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "${dir}/srv.key" 2>/dev/null`, + ); + execSync( + `openssl req -new -key "${dir}/srv.key" -subj "/CN=localhost" -out "${dir}/srv.csr" 2>/dev/null`, + ); + writeFileSync(`${dir}/ext.cnf`, "subjectAltName=DNS:localhost,IP:127.0.0.1\n"); + execSync( + `openssl x509 -req -in "${dir}/srv.csr" -CA "${dir}/ca.crt" -CAkey "${dir}/ca.key" ` + + `-CAcreateserial -days 3650 -extfile "${dir}/ext.cnf" -out "${dir}/srv.crt" 2>/dev/null`, + ); + return { + caPem: readFileSync(`${dir}/ca.crt`, "utf8"), + serverKey: readFileSync(`${dir}/srv.key`, "utf8"), + serverCert: readFileSync(`${dir}/srv.crt`, "utf8"), + }; + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +function generateSelfSignedCert(): { key: string; cert: string } { + const keyPath = join(tmpdir(), `wget-test-key-${process.pid}-${Date.now()}.pem`); + try { + const key = execSync( + "openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 2>/dev/null", + { encoding: "utf8" }, + ); + writeFileSync(keyPath, key); + const cert = execSync( + `openssl req -new -x509 -key "${keyPath}" -days 1 -subj "/CN=localhost" -addext "subjectAltName=DNS:localhost,IP:127.0.0.1" 2>/dev/null`, + { encoding: "utf8" }, + ); + return { key, cert }; + } finally { + try { + unlinkSync(keyPath); + } catch { + // Best effort cleanup for test temp files. + } + } +} + describeIf(hasWgetBinary, "wget command", () => { let kernel: Kernel; let server: Server; + let selfSignedServer: HttpsServer; + let validHttpsServer: HttpsServer; + let caHttpsServer: HttpsServer; let port: number; + let selfSignedPort: number; + let validHttpsPort: number; + let caHttpsPort: number; + // CA (PEM) trusted by the seeded /etc/ssl/certs/ca-certificates.crt bundle; + // it signs validHttpsServer's leaf. caOnlyPem signs caHttpsServer's leaf and + // is deliberately NOT in the bundle, so it verifies only via + // --ca-certificate. + let seededCaPem = ""; + let caOnlyPem = ""; beforeAll(async () => { server = createServer((req: IncomingMessage, res: ServerResponse) => { @@ -47,6 +148,17 @@ describeIf(hasWgetBinary, "wget command", () => { return; } + if (url === "/gzip") { + const body = gzipSync(Buffer.from(COMPRESSION_PAYLOAD)); + res.writeHead(200, { + "Content-Type": "text/plain", + "Content-Encoding": "gzip", + "Content-Length": String(body.length), + }); + res.end(body); + return; + } + if (url === "/redirect") { res.writeHead(302, { Location: `http://127.0.0.1:${port}/redirected`, @@ -69,12 +181,72 @@ describeIf(hasWgetBinary, "wget command", () => { server.listen(0, "127.0.0.1", resolveListen), ); port = (server.address() as import("node:net").AddressInfo).port; + + if (hasOpenssl) { + // Self-signed leaf: no chain to any trusted CA -> must fail verify. + const selfSigned = generateSelfSignedCert(); + selfSignedServer = createHttpsServer( + { key: selfSigned.key, cert: selfSigned.cert }, + (req, res) => { + res.writeHead(200, { "Content-Type": "text/plain" }); + res.end("self-signed secure content"); + }, + ); + await new Promise((resolveListen) => + selfSignedServer.listen(0, "127.0.0.1", resolveListen), + ); + selfSignedPort = ( + selfSignedServer.address() as import("node:net").AddressInfo + ).port; + + // Leaf chaining to a CA seeded into the guest's bundle -> verifies + // with no --no-check-certificate / --ca-certificate. + const trusted = makeCaSignedCert("AgentOS Wget Test Root CA"); + seededCaPem = trusted.caPem; + validHttpsServer = createHttpsServer( + { key: trusted.serverKey, cert: trusted.serverCert }, + (req, res) => { + res.writeHead(200, { "Content-Type": "text/plain" }); + res.end("verified https content"); + }, + ); + await new Promise((resolveListen) => + validHttpsServer.listen(0, "127.0.0.1", resolveListen), + ); + validHttpsPort = ( + validHttpsServer.address() as import("node:net").AddressInfo + ).port; + + // Leaf whose CA is provided ONLY via --ca-certificate (not in bundle). + const caOnly = makeCaSignedCert("AgentOS Wget Cacert-Only CA"); + caOnlyPem = caOnly.caPem; + caHttpsServer = createHttpsServer( + { key: caOnly.serverKey, cert: caOnly.serverCert }, + (req, res) => { + res.writeHead(200, { "Content-Type": "text/plain" }); + res.end("cacert https content"); + }, + ); + await new Promise((resolveListen) => + caHttpsServer.listen(0, "127.0.0.1", resolveListen), + ); + caHttpsPort = ( + caHttpsServer.address() as import("node:net").AddressInfo + ).port; + } }); afterAll(async () => { - await new Promise((resolveClose) => - server.close(() => resolveClose()), - ); + for (const s of [ + server, + selfSignedServer, + validHttpsServer, + caHttpsServer, + ]) { + if (s) { + await new Promise((resolveClose) => s.close(() => resolveClose())); + } + } }); afterEach(async () => { @@ -86,9 +258,23 @@ describeIf(hasWgetBinary, "wget command", () => { kernel = createKernel({ filesystem, permissions: allowAll, - loopbackExemptPorts: [port], + loopbackExemptPorts: [ + port, + selfSignedPort, + validHttpsPort, + caHttpsPort, + ].filter((p) => typeof p === "number"), }); await kernel.mount(createWasmVmRuntime({ commandDirs: WGET_COMMAND_DIRS })); + + // Seed the Debian-shaped trust store the way the native VM bootstrap + // does, so wget's default CA bundle resolves in-guest. Only the + // "trusted" CA is placed here; the cacert-only CA is intentionally + // absent. + if (seededCaPem) { + await filesystem.mkdir("/etc/ssl/certs", { recursive: true }); + await kernel.writeFile("/etc/ssl/certs/ca-certificates.crt", seededCaPem); + } return filesystem; } @@ -159,4 +345,105 @@ describeIf(hasWgetBinary, "wget command", () => { "arrived after redirect", ); }, 15_000); + + it("--version reports the mbedTLS HTTPS backend", async () => { + await mountKernel(); + + const result = await kernel.exec("wget --version", { + timeout: WGET_EXEC_TIMEOUT_MS, + }); + + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(result.stdout).toContain("GNU Wget 1.24.5"); + // Real in-guest TLS: HTTPS is compiled in and the backend is mbedTLS. + expect(result.stdout).toMatch(/\+https/); + expect(result.stdout).toMatch(/ssl\/mbedtls/i); + }, 15_000); + + it("--compression=auto inflates a gzip response body", async () => { + const filesystem = await mountKernel(); + + const result = await kernel.exec( + `wget --compression=auto -O /gz.txt http://127.0.0.1:${port}/gzip`, + { timeout: WGET_EXEC_TIMEOUT_MS }, + ); + + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(await filesystem.readTextFile("/gz.txt")).toBe(COMPRESSION_PAYLOAD); + }, 15_000); + + itIf(hasOpenssl, "downloads over HTTPS verifying against the seeded CA bundle", async () => { + const filesystem = await mountKernel(); + + // No --no-check-certificate, no --ca-certificate: trust comes solely + // from the seeded /etc/ssl/certs/ca-certificates.crt, like Debian wget. + const result = await kernel.exec( + `wget -O /secure.txt https://127.0.0.1:${validHttpsPort}/file`, + { timeout: WGET_EXEC_TIMEOUT_MS }, + ); + + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(await filesystem.readTextFile("/secure.txt")).toBe( + "verified https content", + ); + }, 15_000); + + itIf(hasOpenssl, "fails with a real cert error on an untrusted (self-signed) server", async () => { + await mountKernel(); + + const result = await kernel.exec( + `wget -O /nope.txt https://127.0.0.1:${selfSignedPort}/file`, + { timeout: WGET_EXEC_TIMEOUT_MS }, + ); + + // VERIFCERTERR -> WGET_EXIT_SSL_AUTH_FAIL == 5, the native taxonomy. + expect(result.exitCode).toBe(5); + expect(result.stderr).toMatch(/cannot verify|certificate|not trusted/i); + }, 15_000); + + itIf(hasOpenssl, "--no-check-certificate accepts a self-signed server", async () => { + const filesystem = await mountKernel(); + + const result = await kernel.exec( + `wget --no-check-certificate -O /insecure.txt https://127.0.0.1:${selfSignedPort}/file`, + { timeout: WGET_EXEC_TIMEOUT_MS }, + ); + + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(await filesystem.readTextFile("/insecure.txt")).toBe( + "self-signed secure content", + ); + }, 15_000); + + itIf(hasOpenssl, "--ca-certificate trusts a server signed by that CA", async () => { + const filesystem = await mountKernel(); + + // caHttpsServer's CA is NOT in the seeded bundle, so this only passes if + // --ca-certificate is honored (real file read + chain build in-guest). + await kernel.writeFile("/tmp/cacert-only.pem", caOnlyPem); + const result = await kernel.exec( + `wget --ca-certificate=/tmp/cacert-only.pem -O /cacert.txt https://127.0.0.1:${caHttpsPort}/file`, + { timeout: WGET_EXEC_TIMEOUT_MS }, + ); + + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(await filesystem.readTextFile("/cacert.txt")).toBe( + "cacert https content", + ); + }, 15_000); + + itIf(hasOpenssl, "--ca-certificate with the wrong CA still fails verification", async () => { + await mountKernel(); + + // Point --ca-certificate at the seeded CA, which did NOT sign + // caHttpsServer's leaf. + await kernel.writeFile("/tmp/wrong-ca.pem", seededCaPem); + const result = await kernel.exec( + `wget --ca-certificate=/tmp/wrong-ca.pem -O /wrong.txt https://127.0.0.1:${caHttpsPort}/file`, + { timeout: WGET_EXEC_TIMEOUT_MS }, + ); + + expect(result.exitCode).toBe(5); + expect(result.stderr).toMatch(/cannot verify|certificate|not trusted/i); + }, 15_000); }); diff --git a/toolchain/Makefile b/toolchain/Makefile index cff2c8ea86..0ff9c0f897 100644 --- a/toolchain/Makefile +++ b/toolchain/Makefile @@ -242,7 +242,7 @@ wasm: c/sysroot/lib/wasm32-wasi/libc.a vendor patch-vendor patch-std wasm-opt-ch # codex the codex fork build (codex, codex-exec) # `just toolchain-cmd ` calls this. `make wasm` builds the fast # Rust set; `make -C c programs install` builds/installs the fast C set. -C_COMMANDS := zip unzip envsubst sqlite3 curl wget grep git duckdb vim +C_COMMANDS := zip unzip envsubst sqlite3 curl wget grep git duckdb vim ssh ps pgrep pkill pstree killall prtstat .PHONY: cmd/% cmd/%: diff --git a/toolchain/c/Makefile b/toolchain/c/Makefile index 0190b360f2..31d4b0ea37 100644 --- a/toolchain/c/Makefile +++ b/toolchain/c/Makefile @@ -68,10 +68,10 @@ COMMANDS_DIR ?= ../target/wasm32-wasip1/release/commands # Fast real commands installed by the default software gate. Slow/heavy commands # (duckdb, vim) remain available through explicit parent `make cmd/`. -COMMANDS := zip unzip envsubst sqlite3 curl wget grep git tree +COMMANDS := zip unzip envsubst sqlite3 curl wget grep git git-remote-http tree ssh ps pgrep pkill pstree killall prtstat # Programs requiring patched sysroot (Tier 2+ custom host imports) -PATCHED_PROGRAMS := http_get_test isatty_test getpid_test getppid_test getppid_verify userinfo pipe_test dup_test spawn_child spawn_exit_code pipeline kill_child waitpid_return waitpid_edge syscall_coverage getpwuid_test signal_tests sigaction_self sigaction_behavior delayed_tcp_echo delayed_kill pipe_edge tcp_accept_spawn tcp_echo tcp_server http_server udp_echo unix_socket signal_handler dns_lookup sqlite3 curl wget grep tree zip unzip fs_probe +PATCHED_PROGRAMS := http_get_test isatty_test getpid_test getppid_test getppid_verify userinfo pipe_test dup_test spawn_child spawn_exit_code pipeline kill_child waitpid_return waitpid_edge syscall_coverage getpwuid_test signal_tests sigaction_self sigaction_behavior delayed_tcp_echo delayed_kill pipe_edge tcp_accept_spawn tcp_echo tcp_server http_server udp_echo unix_socket signal_handler dns_lookup sqlite3 curl wget grep tree zip unzip ps pgrep pkill pstree killall prtstat fs_probe # Discover all package command and test-program C source files. ALL_SOURCES := $(foreach dir,$(C_SOURCE_DIRS),$(wildcard $(dir)/*.c)) @@ -89,7 +89,7 @@ endif ifeq ($(wildcard $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a),) CUSTOM_WASM_PROG_NAMES := else - CUSTOM_WASM_PROG_NAMES := curl wget grep sqlite3 tree zip unzip + CUSTOM_WASM_PROG_NAMES := curl wget grep sqlite3 tree zip unzip ps pgrep pkill pstree killall prtstat endif WASM_PROG_NAMES := $(sort $(basename $(notdir $(SOURCES))) $(CUSTOM_WASM_PROG_NAMES)) @@ -148,20 +148,64 @@ CURL_RELEASE_VERSION := 8.11.1 CURL_RELEASE_TAG := 8_11_1 CURL_RELEASE_URL := https://github.com/curl/curl/releases/download/curl-$(CURL_RELEASE_TAG)/curl-$(CURL_RELEASE_VERSION).tar.xz CURL_UPSTREAM_BUILD_DIR := $(BUILD_DIR)/curl-upstream +# Reusable libcurl artifact (headers + libcurl.a) installed by the curl build so +# git's git-remote-http links the same overlaid, mbedTLS-linked libcurl. +CURL_LIBCURL_PREFIX := $(CURL_UPSTREAM_BUILD_DIR)/install +CURL_LIBCURL_ARTIFACT := $(CURL_LIBCURL_PREFIX)/lib/libcurl.a CURL_UPSTREAM_OVERLAY_DIR := ../../software/curl/native/c/overlay CURL_UPSTREAM_OVERLAY_FILES := $(wildcard $(CURL_UPSTREAM_OVERLAY_DIR)/lib/*.c $(CURL_UPSTREAM_OVERLAY_DIR)/lib/*.h $(CURL_UPSTREAM_OVERLAY_DIR)/lib/vtls/*.c $(CURL_UPSTREAM_OVERLAY_DIR)/lib/vtls/*.h) ZLIB_VERSION := 1.3.1 ZLIB_URL := https://zlib.net/zlib-$(ZLIB_VERSION).tar.gz ZLIB_BUILD_DIR := $(BUILD_DIR)/zlib +# mbedTLS 3.6 LTS — in-guest TLS for curl/wget/git (see +# docs-internal/networking-parity-spec.md). Use the prepared release tarball +# (`mbedtls-.tar.bz2`), which bundles the generated sources and the +# `framework` submodule; the bare GitHub source archive does not. Everything +# ships under library/ in the 3.6 LTS line (crypto is not yet split into a +# separate tf-psa-crypto tree as on 4.x/master), so `make lib` needs no +# submodule fetch. We cross-compile every library/*.c against the wasi-sdk +# sysroot, mirroring the zlib rule, plus a getentropy() entropy shim. +MBEDTLS_VERSION := 3.6.2 +MBEDTLS_URL := https://github.com/Mbed-TLS/mbedtls/releases/download/mbedtls-$(MBEDTLS_VERSION)/mbedtls-$(MBEDTLS_VERSION).tar.bz2 +# Deferred `=`: LIBS_CACHE is defined further down, so expand it lazily. +MBEDTLS_SRC_DIR = $(LIBS_CACHE)/mbedtls-$(MBEDTLS_VERSION) +MBEDTLS_BUILD_DIR := $(BUILD_DIR)/mbedtls +MBEDTLS_CONFIG_DIR := mbedtls +MBEDTLS_LIBS := $(MBEDTLS_BUILD_DIR)/libmbedtls.a $(MBEDTLS_BUILD_DIR)/libmbedx509.a $(MBEDTLS_BUILD_DIR)/libmbedcrypto.a +# Mozilla CA bundle (the set Debian's ca-certificates produces) shipped in the +# VM at /etc/ssl/certs/ca-certificates.crt. Pinned URL + fetch rule only; the +# ~230 KB PEM blob is never committed (the sidecar stages it via build.rs into +# OUT_DIR, falling back to an empty placeholder when absent — see +# crates/native-sidecar/build.rs). `make ca-certificates` populates the asset. +CACERT_URL := https://curl.se/ca/cacert.pem +CACERT_ASSET := ../../crates/native-sidecar/assets/ca-certificates.crt WGET_VERSION := 1.24.5 WGET_URL := https://ftp.gnu.org/gnu/wget/wget-$(WGET_VERSION).tar.gz WGET_UPSTREAM_BUILD_DIR := $(BUILD_DIR)/wget-upstream WGET_UPSTREAM_OVERLAY_INCLUDE_DIR := overlays/wget/include -WGET_UPSTREAM_OVERLAY_FILES := $(wildcard $(WGET_UPSTREAM_OVERLAY_INCLUDE_DIR)/*.h) +# mbedTLS backend overlay (mirrors the curl overlay layout): src/wasi_ssl.c +# is copied into the wget source tree at configure time. +WGET_UPSTREAM_OVERLAY_DIR := ../../software/wget/native/c/overlay +WGET_UPSTREAM_OVERLAY_FILES := $(wildcard $(WGET_UPSTREAM_OVERLAY_INCLUDE_DIR)/*.h) \ + $(shell find $(WGET_UPSTREAM_OVERLAY_DIR) -type f 2>/dev/null) GIT_VERSION := 2.55.0 GIT_URL := https://www.kernel.org/pub/software/scm/git/git-$(GIT_VERSION).tar.xz GIT_UPSTREAM_BUILD_DIR := $(BUILD_DIR)/git-upstream GIT_PATCH_DIR := patches/git +# OpenSSH portable — real upstream ssh client (batch/key-based), built +# --without-openssl (internal ed25519/curve25519/chacha20-poly1305 crypto). +OPENSSH_VERSION := 10.4p1 +OPENSSH_URL := https://cdn.openbsd.org/pub/OpenBSD/OpenSSH/portable/openssh-$(OPENSSH_VERSION).tar.gz +OPENSSH_UPSTREAM_BUILD_DIR := $(BUILD_DIR)/ssh-upstream +PROCPS_VERSION := 4.0.6 +PROCPS_URL := https://downloads.sourceforge.net/project/procps-ng/Production/procps-ng-$(PROCPS_VERSION).tar.xz +PROCPS_SHA256 := 67bea6fbc3a42a535a0230c9e891e5ddfb4d9d39422d46565a2990d1ace15216 +PROCPS_UPSTREAM_BUILD_DIR := $(BUILD_DIR)/procps-upstream +PSMISC_VERSION := 23.7 +PSMISC_URL := https://downloads.sourceforge.net/project/psmisc/psmisc/psmisc-$(PSMISC_VERSION).tar.xz +PSMISC_SHA256 := 58c55d9c1402474065adae669511c191de374b0871eec781239ab400b907c327 +PSMISC_UPSTREAM_BUILD_DIR := $(BUILD_DIR)/psmisc-upstream +PSMISC_PATCH_DIR := patches/psmisc GREP_VERSION := 3.12 GREP_URL := https://ftp.gnu.org/gnu/grep/grep-$(GREP_VERSION).tar.xz GREP_UPSTREAM_BUILD_DIR := $(BUILD_DIR)/grep-upstream @@ -440,6 +484,12 @@ $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a: wasi-sdk ../scripts/patch-wasi-libc.s sysroot: $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a +# psmisc's upstream configure requires the conventional terminfo link name. +# The owned compatibility symbols live in libc; create the empty marker archive +# on demand so generated sysroot archives do not need to be committed. +$(PATCHED_SYSROOT)/lib/wasm32-wasi/libtinfo.a: $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a $(WASI_SDK_DIR)/bin/llvm-ar + $(WASI_SDK_DIR)/bin/llvm-ar rcs $@ + # --- wasm-opt check --- wasm-opt-check: @@ -545,6 +595,143 @@ $(ZLIB_BUILD_DIR)/libz.a: $(LIBS_CACHE)/zlib-$(ZLIB_VERSION)/zlib.h $(PATCHED_SY done $(WASI_SDK_DIR)/bin/llvm-ar rcs $@ $(ZLIB_BUILD_DIR)/*.o +# --- mbedTLS 3.6 LTS for wasm32-wasip1 --- +# Fetch/extract the prepared release tarball, then cross-compile every +# library/*.c (plus the getentropy entropy shim) against the wasi-sdk sysroot, +# splitting objects into the three canonical archives by filename the same way +# mbedTLS's own Makefile does (crypto / x509 / tls). Single-threaded; entropy +# via getentropy() (MBEDTLS_ENTROPY_HARDWARE_ALT); TLS 1.3 on (default in 3.6). +$(MBEDTLS_SRC_DIR)/library/ssl_tls.c: + @echo "Fetching mbedTLS $(MBEDTLS_VERSION)..." + @mkdir -p $(LIBS_CACHE) + @curl -fSL "$(MBEDTLS_URL)" -o "$(LIBS_CACHE)/mbedtls-$(MBEDTLS_VERSION).tar.bz2" + @rm -rf "$(MBEDTLS_SRC_DIR)" + @tar -xjf "$(LIBS_CACHE)/mbedtls-$(MBEDTLS_VERSION).tar.bz2" -C "$(LIBS_CACHE)" + +.PHONY: mbedtls +mbedtls: $(MBEDTLS_LIBS) + +$(MBEDTLS_LIBS): $(MBEDTLS_SRC_DIR)/library/ssl_tls.c \ + $(MBEDTLS_CONFIG_DIR)/wasi_user_config.h \ + $(MBEDTLS_CONFIG_DIR)/mbedtls_wasi_entropy.c \ + $(WASI_SDK_DIR)/bin/clang + @echo "=== Building mbedTLS $(MBEDTLS_VERSION) for wasm32-wasip1 ===" + @mkdir -p $(MBEDTLS_BUILD_DIR)/crypto $(MBEDTLS_BUILD_DIR)/x509 $(MBEDTLS_BUILD_DIR)/tls + @for src in $(MBEDTLS_SRC_DIR)/library/*.c $(MBEDTLS_CONFIG_DIR)/mbedtls_wasi_entropy.c; do \ + name=$$(basename "$${src%.c}"); \ + case "$$name" in \ + x509*|pkcs7) grp=x509 ;; \ + ssl_*|ssl|mps_*|debug|net_sockets) grp=tls ;; \ + *) grp=crypto ;; \ + esac; \ + $(CC) --target=wasm32-wasip1 --sysroot=$(SYSROOT) -O2 \ + -I$(MBEDTLS_SRC_DIR)/include -I$(MBEDTLS_CONFIG_DIR) \ + -DMBEDTLS_USER_CONFIG_FILE='"wasi_user_config.h"' \ + -c "$$src" -o "$(MBEDTLS_BUILD_DIR)/$$grp/$$name.o" || exit 1; \ + done + @rm -f $(MBEDTLS_LIBS) + $(WASI_SDK_DIR)/bin/llvm-ar rcs $(MBEDTLS_BUILD_DIR)/libmbedcrypto.a $(MBEDTLS_BUILD_DIR)/crypto/*.o + $(WASI_SDK_DIR)/bin/llvm-ar rcs $(MBEDTLS_BUILD_DIR)/libmbedx509.a $(MBEDTLS_BUILD_DIR)/x509/*.o + $(WASI_SDK_DIR)/bin/llvm-ar rcs $(MBEDTLS_BUILD_DIR)/libmbedtls.a $(MBEDTLS_BUILD_DIR)/tls/*.o + @echo "=== mbedTLS libs built ===" + @ls -l $(MBEDTLS_LIBS) + +# --- brotli 1.1.0 (decoder) for wasm32-wasip1 --- +# curl's --compressed brotli decode. Only the dependency-free portable C +# `common` + `dec` sources are needed (no encoder). Mirrors the zlib rule: +# cross-compile each .c against the wasi-sdk sysroot and archive into the two +# canonical libs (libbrotlicommon.a, libbrotlidec.a; dec depends on common). +BROTLI_VERSION := 1.1.0 +BROTLI_URL := https://github.com/google/brotli/archive/refs/tags/v$(BROTLI_VERSION).tar.gz +BROTLI_SRC_DIR = $(LIBS_CACHE)/brotli-$(BROTLI_VERSION) +BROTLI_BUILD_DIR := $(BUILD_DIR)/brotli +BROTLI_INCLUDE_DIR = $(BROTLI_SRC_DIR)/c/include +BROTLI_COMMON_SRCS := constants.c context.c dictionary.c platform.c shared_dictionary.c transform.c +BROTLI_DEC_SRCS := bit_reader.c decode.c huffman.c state.c +BROTLI_LIBS := $(BROTLI_BUILD_DIR)/libbrotlidec.a $(BROTLI_BUILD_DIR)/libbrotlicommon.a + +$(BROTLI_SRC_DIR)/c/dec/decode.c: + @echo "Fetching brotli $(BROTLI_VERSION)..." + @mkdir -p $(LIBS_CACHE) + @curl -fSL "$(BROTLI_URL)" -o "$(LIBS_CACHE)/brotli-$(BROTLI_VERSION).tar.gz" + @rm -rf "$(BROTLI_SRC_DIR)" + @tar -xzf "$(LIBS_CACHE)/brotli-$(BROTLI_VERSION).tar.gz" -C "$(LIBS_CACHE)" + +.PHONY: brotli +brotli: $(BROTLI_LIBS) + +$(BROTLI_LIBS): $(BROTLI_SRC_DIR)/c/dec/decode.c $(WASI_SDK_DIR)/bin/clang + @echo "=== Building brotli $(BROTLI_VERSION) (dec) for wasm32-wasip1 ===" + @mkdir -p $(BROTLI_BUILD_DIR)/common $(BROTLI_BUILD_DIR)/dec + @for src in $(BROTLI_COMMON_SRCS); do \ + $(CC) --target=wasm32-wasip1 --sysroot=$(SYSROOT) -O2 \ + -I$(BROTLI_INCLUDE_DIR) \ + -c "$(BROTLI_SRC_DIR)/c/common/$$src" \ + -o "$(BROTLI_BUILD_DIR)/common/$${src%.c}.o" || exit 1; \ + done + @for src in $(BROTLI_DEC_SRCS); do \ + $(CC) --target=wasm32-wasip1 --sysroot=$(SYSROOT) -O2 \ + -I$(BROTLI_INCLUDE_DIR) \ + -c "$(BROTLI_SRC_DIR)/c/dec/$$src" \ + -o "$(BROTLI_BUILD_DIR)/dec/$${src%.c}.o" || exit 1; \ + done + @rm -f $(BROTLI_LIBS) + $(WASI_SDK_DIR)/bin/llvm-ar rcs $(BROTLI_BUILD_DIR)/libbrotlicommon.a $(BROTLI_BUILD_DIR)/common/*.o + $(WASI_SDK_DIR)/bin/llvm-ar rcs $(BROTLI_BUILD_DIR)/libbrotlidec.a $(BROTLI_BUILD_DIR)/dec/*.o + @echo "=== brotli libs built ===" + @ls -l $(BROTLI_LIBS) + +# --- zstd 1.5.6 (decoder) for wasm32-wasip1 --- +# curl's --compressed zstd decode. Single-threaded (ZSTD_MULTITHREAD off), +# ZSTD_DISABLE_ASM=1 so the portable C huffman path is used instead of the +# amd64 .S. Only lib/common + lib/decompress are needed for ZSTD_*DStream. +ZSTD_VERSION := 1.5.6 +ZSTD_URL := https://github.com/facebook/zstd/releases/download/v$(ZSTD_VERSION)/zstd-$(ZSTD_VERSION).tar.gz +ZSTD_SRC_DIR = $(LIBS_CACHE)/zstd-$(ZSTD_VERSION) +ZSTD_BUILD_DIR := $(BUILD_DIR)/zstd +ZSTD_INCLUDE_DIR = $(ZSTD_SRC_DIR)/lib +ZSTD_LIB := $(ZSTD_BUILD_DIR)/libzstd.a +# NB: leave ZSTD_MULTITHREAD *undefined* (not =0). zstd's threading.h keys the +# pthread path off `#if defined(ZSTD_MULTITHREAD)`, so even =0 pulls in +# pthread_create/join, which wasi-libc static-asserts away without threads. +ZSTD_CFLAGS := -O2 -DZSTD_DISABLE_ASM=1 \ + -I$(ZSTD_SRC_DIR)/lib -I$(ZSTD_SRC_DIR)/lib/common + +$(ZSTD_SRC_DIR)/lib/zstd.h: + @echo "Fetching zstd $(ZSTD_VERSION)..." + @mkdir -p $(LIBS_CACHE) + @curl -fSL "$(ZSTD_URL)" -o "$(LIBS_CACHE)/zstd-$(ZSTD_VERSION).tar.gz" + @rm -rf "$(ZSTD_SRC_DIR)" + @tar -xzf "$(LIBS_CACHE)/zstd-$(ZSTD_VERSION).tar.gz" -C "$(LIBS_CACHE)" + +.PHONY: zstd +zstd: $(ZSTD_LIB) + +$(ZSTD_LIB): $(ZSTD_SRC_DIR)/lib/zstd.h $(WASI_SDK_DIR)/bin/clang + @echo "=== Building zstd $(ZSTD_VERSION) (decompress) for wasm32-wasip1 ===" + @mkdir -p $(ZSTD_BUILD_DIR)/obj + @for src in $(ZSTD_SRC_DIR)/lib/common/*.c $(ZSTD_SRC_DIR)/lib/decompress/*.c; do \ + name=$$(basename "$${src%.c}"); \ + $(CC) --target=wasm32-wasip1 --sysroot=$(SYSROOT) $(ZSTD_CFLAGS) \ + -c "$$src" -o "$(ZSTD_BUILD_DIR)/obj/$$name.o" || exit 1; \ + done + @rm -f $(ZSTD_LIB) + $(WASI_SDK_DIR)/bin/llvm-ar rcs $(ZSTD_LIB) $(ZSTD_BUILD_DIR)/obj/*.o + @echo "=== zstd lib built ===" + @ls -l $(ZSTD_LIB) + +# --- CA bundle (Mozilla / Debian ca-certificates) --- +# Fetch the pinned Mozilla CA bundle into the sidecar asset staged by build.rs. +# The blob is gitignored; run this once to enable real HTTPS verification. +.PHONY: ca-certificates +ca-certificates: $(CACERT_ASSET) + +$(CACERT_ASSET): + @echo "Fetching Mozilla CA bundle from $(CACERT_URL)..." + @mkdir -p $(dir $(CACERT_ASSET)) + @curl -fSL "$(CACERT_URL)" -o "$(CACERT_ASSET)" + @echo "CA bundle staged at $(CACERT_ASSET) ($$(wc -c < $(CACERT_ASSET)) bytes)" + $(NATIVE_DIR)/sqlite3_cli: $(NATIVE_DIR)/sqlite3 cp $< $@ @@ -583,7 +770,7 @@ $(BUILD_DIR)/unzip: scripts/build-infozip-upstream.sh $(PATCHED_SYSROOT)/lib/was --cc "$(abspath $(CC)) --target=wasm32-wasip1 --sysroot=$(abspath $(SYSROOT))" \ --output "$(abspath $@)" -$(BUILD_DIR)/git: scripts/build-git-upstream.sh $(ZLIB_BUILD_DIR)/libz.a $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a $(WASI_SDK_DIR)/bin/clang +$(BUILD_DIR)/git: scripts/build-git-upstream.sh $(CURL_LIBCURL_ARTIFACT) $(MBEDTLS_LIBS) $(BROTLI_LIBS) $(ZSTD_LIB) $(ZLIB_BUILD_DIR)/libz.a $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a $(WASI_SDK_DIR)/bin/clang @mkdir -p $(BUILD_DIR) bash scripts/build-git-upstream.sh \ --version "$(GIT_VERSION)" \ @@ -593,12 +780,78 @@ $(BUILD_DIR)/git: scripts/build-git-upstream.sh $(ZLIB_BUILD_DIR)/libz.a $(PATCH --patch-dir "$(abspath $(GIT_PATCH_DIR))" \ --zlib-dir "$(abspath $(LIBS_CACHE)/zlib-$(ZLIB_VERSION))" \ --zlib-build-dir "$(abspath $(ZLIB_BUILD_DIR))" \ + --curl-prefix "$(abspath $(CURL_LIBCURL_PREFIX))" \ + --mbedtls-libdir "$(abspath $(MBEDTLS_BUILD_DIR))" \ + --brotli-libdir "$(abspath $(BROTLI_BUILD_DIR))" \ + --zstd-libdir "$(abspath $(ZSTD_BUILD_DIR))" \ --cc "$(abspath $(CC))" \ --ar "$(abspath $(WASI_SDK_DIR)/bin/llvm-ar)" \ --ranlib "$(abspath $(WASI_SDK_DIR)/bin/llvm-ranlib)" \ --sysroot "$(abspath $(SYSROOT))" \ + --remote-http-output "$(abspath $(BUILD_DIR)/git-remote-http)" \ --output "$(abspath $@)" +# git-remote-http is produced by the same git build invocation. +$(BUILD_DIR)/git-remote-http: $(BUILD_DIR)/git + @test -f $@ || { echo "Error: git-remote-http missing at $@ (git build did not produce it)"; exit 1; } + +# ssh: upstream OpenSSH client built --without-openssl against the patched +# sysroot (closefrom/socketpair stubs live in wasi-libc-overrides). Also +# lights up git-over-ssh: git's connect.c execs `ssh` from PATH. +$(BUILD_DIR)/ssh: scripts/build-ssh-upstream.sh $(ZLIB_BUILD_DIR)/libz.a $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a $(WASI_SDK_DIR)/bin/clang + @mkdir -p $(BUILD_DIR) + bash scripts/build-ssh-upstream.sh \ + --version "$(OPENSSH_VERSION)" \ + --url "$(OPENSSH_URL)" \ + --cache-dir "$(abspath $(LIBS_CACHE))" \ + --build-dir "$(abspath $(OPENSSH_UPSTREAM_BUILD_DIR))" \ + --zlib-include "$(abspath $(LIBS_CACHE)/zlib-$(ZLIB_VERSION))" \ + --zlib-libdir "$(abspath $(ZLIB_BUILD_DIR))" \ + --overlay-include-dir "$(abspath include)" \ + --cc "$(abspath $(CC)) --target=wasm32-wasip1 --sysroot=$(abspath $(SYSROOT))" \ + --ar "$(abspath $(WASI_SDK_DIR)/bin/llvm-ar)" \ + --ranlib "$(abspath $(WASI_SDK_DIR)/bin/llvm-ranlib)" \ + --output "$(abspath $@)" + +# procps-ng: process-table consumers backed by the kernel's Linux-shaped /proc. +$(BUILD_DIR)/ps: scripts/build-procps-upstream.sh $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a $(WASI_SDK_DIR)/bin/clang + @mkdir -p $(BUILD_DIR) + bash scripts/build-procps-upstream.sh \ + --version "$(PROCPS_VERSION)" \ + --url "$(PROCPS_URL)" \ + --sha256 "$(PROCPS_SHA256)" \ + --cache-dir "$(abspath $(LIBS_CACHE))" \ + --build-dir "$(abspath $(PROCPS_UPSTREAM_BUILD_DIR))" \ + --overlay-include-dir "$(abspath include)" \ + --cc "$(abspath $(CC))" \ + --ar "$(abspath $(WASI_SDK_DIR)/bin/llvm-ar)" \ + --ranlib "$(abspath $(WASI_SDK_DIR)/bin/llvm-ranlib)" \ + --sysroot "$(abspath $(SYSROOT))" \ + --output-dir "$(abspath $(BUILD_DIR))" + +$(BUILD_DIR)/pgrep $(BUILD_DIR)/pkill: $(BUILD_DIR)/ps + @test -f $@ || { echo "Error: $@ missing from procps-ng build"; exit 1; } + +# psmisc: non-TUI process tree, name-based signaling, and stat inspection. +$(BUILD_DIR)/pstree: scripts/build-psmisc-upstream.sh $(wildcard $(PSMISC_PATCH_DIR)/*.patch) $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a $(PATCHED_SYSROOT)/lib/wasm32-wasi/libtinfo.a $(WASI_SDK_DIR)/bin/clang + @mkdir -p $(BUILD_DIR) + bash scripts/build-psmisc-upstream.sh \ + --version "$(PSMISC_VERSION)" \ + --url "$(PSMISC_URL)" \ + --sha256 "$(PSMISC_SHA256)" \ + --cache-dir "$(abspath $(LIBS_CACHE))" \ + --build-dir "$(abspath $(PSMISC_UPSTREAM_BUILD_DIR))" \ + --patch-dir "$(abspath $(PSMISC_PATCH_DIR))" \ + --overlay-include-dir "$(abspath include)" \ + --cc "$(abspath $(CC))" \ + --ar "$(abspath $(WASI_SDK_DIR)/bin/llvm-ar)" \ + --ranlib "$(abspath $(WASI_SDK_DIR)/bin/llvm-ranlib)" \ + --sysroot "$(abspath $(SYSROOT))" \ + --output-dir "$(abspath $(BUILD_DIR))" + +$(BUILD_DIR)/killall $(BUILD_DIR)/prtstat: $(BUILD_DIR)/pstree + @test -f $@ || { echo "Error: $@ missing from psmisc build"; exit 1; } + # curl_test: links libcurl (HTTP/HTTPS build for WASM via host_net + host_tls) CURL_SRCS := $(wildcard libs/curl/lib/*.c) $(wildcard libs/curl/lib/vauth/*.c) \ $(wildcard libs/curl/lib/vtls/*.c) $(wildcard libs/curl/lib/vquic/*.c) \ @@ -606,7 +859,7 @@ CURL_SRCS := $(wildcard libs/curl/lib/*.c) $(wildcard libs/curl/lib/vauth/*.c) \ CURL_INCLUDES := -Ilibs/curl/include -Ilibs/curl/lib -include libs/curl/lib/curl_setup.h -include libs/curl/lib/curl_printf.h CURL_LIB_DEFS := -DHAVE_CONFIG_H -DBUILDING_LIBCURL -D_WASI_EMULATED_SIGNAL -DHAVE_BASENAME -DHAVE_LIBGEN_H -$(BUILD_DIR)/curl: scripts/build-curl-upstream.sh $(CURL_UPSTREAM_OVERLAY_FILES) $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a $(WASI_SDK_DIR)/bin/clang +$(BUILD_DIR)/curl: scripts/build-curl-upstream.sh $(CURL_UPSTREAM_OVERLAY_FILES) $(MBEDTLS_LIBS) $(ZLIB_BUILD_DIR)/libz.a $(BROTLI_LIBS) $(ZSTD_LIB) $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a $(WASI_SDK_DIR)/bin/clang @mkdir -p $(BUILD_DIR) bash scripts/build-curl-upstream.sh \ --version "$(CURL_RELEASE_VERSION)" \ @@ -618,9 +871,24 @@ $(BUILD_DIR)/curl: scripts/build-curl-upstream.sh $(CURL_UPSTREAM_OVERLAY_FILES) --cc "$(abspath $(CC)) --target=wasm32-wasip1 --sysroot=$(abspath $(SYSROOT))" \ --ar "$(abspath $(WASI_SDK_DIR)/bin/llvm-ar)" \ --ranlib "$(abspath $(WASI_SDK_DIR)/bin/llvm-ranlib)" \ + --mbedtls-include "$(abspath $(MBEDTLS_SRC_DIR)/include)" \ + --mbedtls-libdir "$(abspath $(MBEDTLS_BUILD_DIR))" \ + --zlib-include "$(abspath $(LIBS_CACHE)/zlib-$(ZLIB_VERSION))" \ + --zlib-libdir "$(abspath $(ZLIB_BUILD_DIR))" \ + --brotli-include "$(abspath $(BROTLI_INCLUDE_DIR))" \ + --brotli-libdir "$(abspath $(BROTLI_BUILD_DIR))" \ + --zstd-include "$(abspath $(ZSTD_INCLUDE_DIR))" \ + --zstd-libdir "$(abspath $(ZSTD_BUILD_DIR))" \ + --ca-bundle "/etc/ssl/certs/ca-certificates.crt" \ + --install-prefix "$(abspath $(CURL_LIBCURL_PREFIX))" \ --output "$(abspath $@)" -$(BUILD_DIR)/wget: scripts/build-wget-upstream.sh $(WGET_UPSTREAM_OVERLAY_FILES) $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a $(WASI_SDK_DIR)/bin/clang +# The libcurl install artifact is produced as a side effect of the curl build. +# Declare it so git can depend on it; verify it exists after the curl target ran. +$(CURL_LIBCURL_ARTIFACT): $(BUILD_DIR)/curl + @test -f $@ || { echo "Error: libcurl artifact missing at $@ (curl build did not install it)"; exit 1; } + +$(BUILD_DIR)/wget: scripts/build-wget-upstream.sh $(WGET_UPSTREAM_OVERLAY_FILES) $(MBEDTLS_LIBS) $(ZLIB_BUILD_DIR)/libz.a $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a $(WASI_SDK_DIR)/bin/clang @mkdir -p $(BUILD_DIR) bash scripts/build-wget-upstream.sh \ --version "$(WGET_VERSION)" \ @@ -628,6 +896,11 @@ $(BUILD_DIR)/wget: scripts/build-wget-upstream.sh $(WGET_UPSTREAM_OVERLAY_FILES) --cache-dir "$(abspath $(LIBS_CACHE))" \ --build-dir "$(abspath $(WGET_UPSTREAM_BUILD_DIR))" \ --overlay-include-dir "$(abspath $(WGET_UPSTREAM_OVERLAY_INCLUDE_DIR))" \ + --overlay-dir "$(abspath $(WGET_UPSTREAM_OVERLAY_DIR))" \ + --mbedtls-include "$(abspath $(MBEDTLS_SRC_DIR)/include)" \ + --mbedtls-libdir "$(abspath $(MBEDTLS_BUILD_DIR))" \ + --zlib-include "$(abspath $(LIBS_CACHE)/zlib-$(ZLIB_VERSION))" \ + --zlib-libdir "$(abspath $(ZLIB_BUILD_DIR))" \ --cc "$(abspath $(CC)) --target=wasm32-wasip1 --sysroot=$(abspath $(SYSROOT))" \ --ar "$(abspath $(WASI_SDK_DIR)/bin/llvm-ar)" \ --ranlib "$(abspath $(WASI_SDK_DIR)/bin/llvm-ranlib)" \ @@ -695,12 +968,18 @@ install: fi; \ done; \ echo "Installed $$INSTALLED C command(s) to $(COMMANDS_DIR)" - @# Create symlinks for Git helper entry points used by upstream Git. + @# Git helper entry points. upload-pack/receive-pack/upload-archive ARE git + @# builtins, so they alias the git binary. The HTTPS transport lives in the + @# real git-remote-http binary (libcurl in-process); git-remote-https is an + @# alias to it (same binary, ftp/ftps schemes unsupported and unused). @if [ -f "$(COMMANDS_DIR)/git" ]; then \ - for alias in git-remote-http git-remote-https git-upload-pack git-receive-pack git-upload-archive; do \ + for alias in git-upload-pack git-receive-pack git-upload-archive; do \ ln -sf git "$(COMMANDS_DIR)/$$alias"; \ done; \ fi + @if [ -f "$(COMMANDS_DIR)/git-remote-http" ]; then \ + ln -sf git-remote-http "$(COMMANDS_DIR)/git-remote-https"; \ + fi # --- Clean --- diff --git a/toolchain/c/build/cat b/toolchain/c/build/cat deleted file mode 100644 index f59943b868..0000000000 Binary files a/toolchain/c/build/cat and /dev/null differ diff --git a/toolchain/c/build/env b/toolchain/c/build/env deleted file mode 100644 index 401e013931..0000000000 Binary files a/toolchain/c/build/env and /dev/null differ diff --git a/toolchain/c/build/sort b/toolchain/c/build/sort deleted file mode 100644 index 95605b0e06..0000000000 Binary files a/toolchain/c/build/sort and /dev/null differ diff --git a/toolchain/c/build/wc b/toolchain/c/build/wc deleted file mode 100644 index 40bcb6891d..0000000000 Binary files a/toolchain/c/build/wc and /dev/null differ diff --git a/toolchain/c/include/curses.h b/toolchain/c/include/curses.h new file mode 100644 index 0000000000..e3d3802bfd --- /dev/null +++ b/toolchain/c/include/curses.h @@ -0,0 +1,11 @@ +#ifndef AGENTOS_C_INCLUDE_CURSES_H +#define AGENTOS_C_INCLUDE_CURSES_H + +#define OK 0 +#define ERR (-1) + +int setupterm(const char *term, int filedes, int *errret); +char *tigetstr(const char *capname); +int tputs(const char *str, int affcnt, int (*putc_fn)(int)); + +#endif diff --git a/toolchain/c/include/net/if.h b/toolchain/c/include/net/if.h index 0a03c72f2d..67f0f10cad 100644 --- a/toolchain/c/include/net/if.h +++ b/toolchain/c/include/net/if.h @@ -9,4 +9,18 @@ #define IF_NAMESIZE IFNAMSIZ #endif +/* Interface flag bits, values as in Linux (netdevice(7)) and musl. + * The runtime's getifaddrs() (see ifaddrs.h in this overlay) reports a + * loopback-only interface set; ported tools (e.g. OpenSSH's BindInterface + * handling in sshconnect.c and Match-address handling in readconf.c) test + * IFF_UP / IFF_LOOPBACK on those entries. */ +#ifndef IFF_UP +#define IFF_UP 0x1 +#define IFF_BROADCAST 0x2 +#define IFF_LOOPBACK 0x8 +#define IFF_POINTOPOINT 0x10 +#define IFF_RUNNING 0x40 +#define IFF_MULTICAST 0x1000 +#endif + #endif diff --git a/toolchain/c/include/term.h b/toolchain/c/include/term.h new file mode 100644 index 0000000000..51f5954849 --- /dev/null +++ b/toolchain/c/include/term.h @@ -0,0 +1,9 @@ +#ifndef AGENTOS_C_INCLUDE_TERM_H +#define AGENTOS_C_INCLUDE_TERM_H + +#include + +int tgetent(char *buffer, const char *termtype); +char *tgetstr(const char *id, char **area); + +#endif diff --git a/toolchain/c/mbedtls/mbedtls_wasi_entropy.c b/toolchain/c/mbedtls/mbedtls_wasi_entropy.c new file mode 100644 index 0000000000..ac76210218 --- /dev/null +++ b/toolchain/c/mbedtls/mbedtls_wasi_entropy.c @@ -0,0 +1,60 @@ +/* + * Hardware entropy shim for the wasm32-wasip1 (WASI) mbedTLS build. + * + * The WASI build sets MBEDTLS_NO_PLATFORM_ENTROPY (no /dev/urandom fopen path) + * and MBEDTLS_ENTROPY_HARDWARE_ALT, which routes the entropy module through + * this mbedtls_hardware_poll() implementation. We satisfy it with getentropy(), + * the WASI random primitive (backed by the kernel's random device layer) that + * the repo's host-side wasi_tls.c already uses. getentropy() fills at most 256 + * bytes per call, so we loop for larger requests. + */ + +#include +#include /* clock_gettime(), CLOCK_MONOTONIC */ +#include /* getentropy() — declared by wasi-libc */ + +#include "mbedtls/build_info.h" +#include "mbedtls/platform_util.h" /* mbedtls_ms_time_t */ + +/* + * WASI mbedtls_ms_time() over clock_gettime(CLOCK_MONOTONIC). Enabled by + * MBEDTLS_PLATFORM_MS_TIME_ALT in the WASI user config (see wasi_user_config.h), + * which suppresses mbedTLS's own platform-detected definition. + */ +#if defined(MBEDTLS_PLATFORM_MS_TIME_ALT) +mbedtls_ms_time_t mbedtls_ms_time(void) +{ + struct timespec tv; + if (clock_gettime(CLOCK_MONOTONIC, &tv) != 0) { + return (mbedtls_ms_time_t) time(NULL) * 1000; + } + return (mbedtls_ms_time_t) tv.tv_sec * 1000 + + (mbedtls_ms_time_t) (tv.tv_nsec / 1000000); +} +#endif /* MBEDTLS_PLATFORM_MS_TIME_ALT */ + +int mbedtls_hardware_poll(void *data, unsigned char *output, size_t len, + size_t *olen) +{ + (void)data; + + size_t filled = 0; + while (filled < len) { + size_t chunk = len - filled; + if (chunk > 256) { + chunk = 256; /* getentropy() rejects requests larger than 256. */ + } + if (getentropy(output + filled, chunk) != 0) { + if (olen != NULL) { + *olen = filled; + } + return -1; /* MBEDTLS_ERR_ENTROPY_SOURCE_FAILED at the call site. */ + } + filled += chunk; + } + + if (olen != NULL) { + *olen = len; + } + return 0; +} diff --git a/toolchain/c/mbedtls/wasi_user_config.h b/toolchain/c/mbedtls/wasi_user_config.h new file mode 100644 index 0000000000..88b22cb823 --- /dev/null +++ b/toolchain/c/mbedtls/wasi_user_config.h @@ -0,0 +1,44 @@ +/* + * mbedTLS user-config overrides for the wasm32-wasip1 (WASI) build. + * + * This file is included via -DMBEDTLS_USER_CONFIG_FILE after mbedTLS's own + * default `mbedtls_config.h`, so it only needs to describe the deltas required + * to cross-compile cleanly against the wasi-sdk sysroot. The default 3.6 config + * already enables TLS 1.2 + TLS 1.3, X.509, PK, PSA crypto, and MBEDTLS_FS_IO + * (needed for curl/wget's mbedtls_x509_crt_parse_file on /etc/ssl/certs). + * + * Deltas: + * - Single-threaded: MBEDTLS_THREADING_C stays off (default) — no pthreads. + * - Entropy: WASI has no /dev/urandom fopen path, so disable platform entropy + * and route the entropy module through mbedtls_hardware_poll(), which we + * implement over getentropy() (see mbedtls_wasi_entropy.c). getentropy() is + * the same primitive the repo's host-side wasi_tls.c already relies on. + * - Networking: MBEDTLS_NET_C pulls in BSD select()/socket() + * that wasi-libc does not fully provide. curl/wget own the transport and + * only need the TLS record + X.509 layers, so drop mbedTLS's own sockets. + * - Timing: MBEDTLS_TIMING_C uses gettimeofday()/select() hardware timers that + * are unnecessary here (no DTLS retransmission timers in the curl path). + */ + +#ifndef MBEDTLS_WASI_USER_CONFIG_H +#define MBEDTLS_WASI_USER_CONFIG_H + +/* Entropy: replace the POSIX /dev/urandom source with a getentropy() poll. */ +#undef MBEDTLS_PLATFORM_ENTROPY +#define MBEDTLS_NO_PLATFORM_ENTROPY +#define MBEDTLS_ENTROPY_HARDWARE_ALT + +/* + * mbedtls_ms_time(): mbedTLS's built-in POSIX branch only compiles when it can + * see _POSIX_VERSION, but it guards the include behind unix/__unix + * macros that clang does not define for wasm32, so the platform detection falls + * through to `#error "No mbedtls_ms_time available"`. Provide it ourselves over + * clock_gettime(CLOCK_MONOTONIC) (available in wasi-libc) via the ALT hook. + */ +#define MBEDTLS_PLATFORM_MS_TIME_ALT + +/* Drop mbedTLS's own BSD-sockets and hardware-timing modules for WASI. */ +#undef MBEDTLS_NET_C +#undef MBEDTLS_TIMING_C + +#endif /* MBEDTLS_WASI_USER_CONFIG_H */ diff --git a/toolchain/c/patches/git/0002-wasi-synchronous-sideband-demux.patch b/toolchain/c/patches/git/0002-wasi-synchronous-sideband-demux.patch new file mode 100644 index 0000000000..57fc156011 --- /dev/null +++ b/toolchain/c/patches/git/0002-wasi-synchronous-sideband-demux.patch @@ -0,0 +1,225 @@ +Demultiplex pack-transfer sidebands synchronously on WASI. + +fetch-pack and send-pack normally run the sideband demultiplexer as an +async task via start_async(). With NO_PTHREADS that falls back to +fork(), which WASI cannot provide, so smart-HTTP clone/fetch/push died +with "fetch-pack: unable to fork off sideband demultiplexer". + +Neither concurrency primitive exists on WASI, but none is required for +correctness: by the time the demultiplexer output is consumed, the +request is fully on the wire and the multiplexed response can be +drained in the main flow. + +- fetch-pack get_pack(): drain the sideband stream synchronously with + recv_sideband() before spawning the pack consumer. Band#2 progress + and band#3 errors reach stderr inline; band#1 (the pack data) is + spooled to a temporary file under objects/pack/, which is then fed + to index-pack/unpack-objects as stdin and unlinked afterwards. +- send-pack: after the commands + pack have been fully written (and + flushed for stateless RPC), drain the response sideband the same + way, spooling band#1 (the ref-status report) to a temporary file + that receive_status() then parses. On the pack_objects() failure + path the demultiplexer has not run and the connection is already + broken, so no status is collected. + +The pack still transfers over the exact same sideband protocol; +the only behavioral difference is that the pack is spooled to disk +before indexing instead of being indexed while downloading. + +--- a/fetch-pack.c ++++ b/fetch-pack.c +@@ -892,6 +892,7 @@ + return retval; + } + ++#ifndef __wasi__ + static int sideband_demux(int in UNUSED, int out, void *data) + { + int *xd = data; +@@ -901,6 +902,7 @@ + close(out); + return ret; + } ++#endif + + static void create_promisor_file(const char *keep_name, + struct ref **sought, int nr_sought) +@@ -969,9 +971,13 @@ + struct child_process cmd = CHILD_PROCESS_INIT; + int fsck_objects = 0; + int ret; ++#ifdef __wasi__ ++ struct strbuf spool_path = STRBUF_INIT; ++#endif + + memset(&demux, 0, sizeof(demux)); + if (use_sideband) { ++#ifndef __wasi__ + /* xd[] is talking with upload-pack; subprocess reads from + * xd[0], spits out band#2 to stderr, and feeds us band#1 + * through demux->out. +@@ -982,6 +988,25 @@ + demux.isolate_sigpipe = 1; + if (start_async(&demux)) + die(_("fetch-pack: unable to fork off sideband demultiplexer")); ++#else ++ /* ++ * WASI has neither fork() nor threads, so the sideband ++ * demultiplexer cannot run concurrently with the pack ++ * indexer. Demultiplex synchronously instead: drain the ++ * sideband stream here (band#2 progress and band#3 errors ++ * reach stderr inline) and spool band#1 (the pack data) to ++ * a temporary file in the object database, then feed the ++ * spooled pack to index-pack/unpack-objects as its stdin. ++ */ ++ int spool_fd = odb_mkstemp(the_repository->objects, ++ &spool_path, ++ "pack/tmp_sideband_XXXXXX"); ++ if (recv_sideband("fetch-pack", xd[0], spool_fd)) ++ die(_("error in sideband demultiplexer")); ++ if (lseek(spool_fd, 0, SEEK_SET) == -1) ++ die_errno(_("could not rewind spooled pack")); ++ demux.out = spool_fd; ++#endif + } + else + demux.out = xd[0]; +@@ -1098,8 +1123,16 @@ + ret == 0; + else + die(_("%s failed"), cmd_name); ++#ifndef __wasi__ + if (use_sideband && finish_async(&demux)) + die(_("error in sideband demultiplexer")); ++#else ++ if (use_sideband) { ++ /* recv_sideband() already ran to completion above. */ ++ unlink(spool_path.buf); ++ strbuf_release(&spool_path); ++ } ++#endif + + sigchain_pop(SIGPIPE); + +--- a/send-pack.c ++++ b/send-pack.c +@@ -281,6 +281,7 @@ + return ret; + } + ++#ifndef __wasi__ + static int sideband_demux(int in UNUSED, int out, void *data) + { + int *fd = data, ret; +@@ -290,6 +291,7 @@ + close(out); + return ret; + } ++#endif + + static int advertise_shallow_grafts_cb(const struct commit_graft *graft, void *cb) + { +@@ -538,6 +540,10 @@ + char *push_cert_nonce = NULL; + struct packet_reader reader; + int use_bitmaps; ++#ifdef __wasi__ ++ struct strbuf spool_path = STRBUF_INIT; ++ int demux_ret = 0; ++#endif + + if (!remote_refs) { + fprintf(stderr, "No refs in common and none specified; doing nothing.\n" +@@ -729,6 +735,7 @@ + + if (use_sideband && cmds_sent) { + memset(&demux, 0, sizeof(demux)); ++#ifndef __wasi__ + demux.proc = sideband_demux; + demux.data = fd; + demux.out = -1; +@@ -736,6 +743,12 @@ + if (start_async(&demux)) + die("send-pack: unable to fork off sideband demultiplexer"); + in = demux.out; ++#endif ++ /* ++ * On WASI (no fork(), no threads) the demultiplexer instead ++ * runs synchronously once the request has been fully written; ++ * see below, right before receive_status(). ++ */ + } + + packet_reader_init(&reader, in, NULL, 0, +@@ -755,6 +768,7 @@ + * as well as marking refs with their remote status (if + * we get one). + */ ++#ifndef __wasi__ + if (status_report) + receive_status(r, &reader, remote_refs); + +@@ -762,6 +776,13 @@ + close(demux.out); + finish_async(&demux); + } ++#else ++ /* ++ * The synchronous WASI demultiplexer has not run yet ++ * and the connection is already broken; there is no ++ * status to collect. ++ */ ++#endif + fd[1] = -1; + + ret = -1; +@@ -774,6 +795,27 @@ + if (args->stateless_rpc && cmds_sent) + packet_flush(out); + ++#ifdef __wasi__ ++ if (use_sideband && cmds_sent) { ++ /* ++ * The request (commands + pack) is fully on the wire, so the ++ * response can be demultiplexed synchronously now: band#2 ++ * progress and band#3 errors reach stderr here, and band#1 ++ * (the status report) is spooled to a temporary file that ++ * receive_status() then parses. ++ */ ++ int spool_fd = odb_mkstemp(r->objects, &spool_path, ++ "pack/tmp_sideband_XXXXXX"); ++ demux_ret = recv_sideband("send-pack", fd[0], spool_fd); ++ if (lseek(spool_fd, 0, SEEK_SET) == -1) ++ die_errno("could not rewind spooled status report"); ++ in = spool_fd; ++ packet_reader_init(&reader, in, NULL, 0, ++ PACKET_READ_CHOMP_NEWLINE | ++ PACKET_READ_DIE_ON_ERR_PACKET); ++ } ++#endif ++ + if (status_report && cmds_sent) + ret = receive_status(r, &reader, remote_refs); + else +@@ -782,11 +824,21 @@ + packet_flush(out); + + if (use_sideband && cmds_sent) { ++#ifndef __wasi__ + close(demux.out); + if (finish_async(&demux)) { + error("error in sideband demultiplexer"); + ret = -1; + } ++#else ++ close(in); ++ unlink(spool_path.buf); ++ strbuf_release(&spool_path); ++ if (demux_ret) { ++ error("error in sideband demultiplexer"); ++ ret = -1; ++ } ++#endif + } + + if (ret < 0) diff --git a/toolchain/c/patches/psmisc/0001-include-stdbool.patch b/toolchain/c/patches/psmisc/0001-include-stdbool.patch new file mode 100644 index 0000000000..3aba1c56f6 --- /dev/null +++ b/toolchain/c/patches/psmisc/0001-include-stdbool.patch @@ -0,0 +1,13 @@ +pstree uses bool even when both SELinux and AppArmor support are disabled. +Include the standard C boolean header directly instead of relying on an +optional security-library header to provide it. + +--- a/src/pstree.c ++++ b/src/pstree.c +@@ -43,6 +43,7 @@ + #include + #include ++#include + + #include "i18n.h" + #include "comm.h" diff --git a/toolchain/c/scripts/build-curl-upstream.sh b/toolchain/c/scripts/build-curl-upstream.sh index d2ffe82c2e..106393a959 100644 --- a/toolchain/c/scripts/build-curl-upstream.sh +++ b/toolchain/c/scripts/build-curl-upstream.sh @@ -13,7 +13,17 @@ Usage: build-curl-upstream.sh \ --cc \ --ar \ --ranlib \ - --output + --mbedtls-include \ + --mbedtls-libdir \ + --zlib-include \ + --zlib-libdir \ + --brotli-include \ + --brotli-libdir \ + --zstd-include \ + --zstd-libdir \ + --ca-bundle \ + --output \ + [--install-prefix ] EOF } @@ -26,50 +36,40 @@ OVERLAY_DIR="" CC_CMD="" AR_CMD="" RANLIB_CMD="" +MBEDTLS_INCLUDE="" +MBEDTLS_LIBDIR="" +ZLIB_INCLUDE="" +ZLIB_LIBDIR="" +BROTLI_INCLUDE="" +BROTLI_LIBDIR="" +ZSTD_INCLUDE="" +ZSTD_LIBDIR="" +CA_BUNDLE="" OUTPUT="" +INSTALL_PREFIX="" while [[ $# -gt 0 ]]; do case "$1" in - --version) - VERSION="$2" - shift 2 - ;; - --tag) - TAG="$2" - shift 2 - ;; - --url) - URL="$2" - shift 2 - ;; - --cache-dir) - CACHE_DIR="$2" - shift 2 - ;; - --build-dir) - BUILD_DIR="$2" - shift 2 - ;; - --overlay-dir) - OVERLAY_DIR="$2" - shift 2 - ;; - --cc) - CC_CMD="$2" - shift 2 - ;; - --ar) - AR_CMD="$2" - shift 2 - ;; - --ranlib) - RANLIB_CMD="$2" - shift 2 - ;; - --output) - OUTPUT="$2" - shift 2 - ;; + --install-prefix) INSTALL_PREFIX="$2"; shift 2 ;; + --version) VERSION="$2"; shift 2 ;; + --tag) TAG="$2"; shift 2 ;; + --url) URL="$2"; shift 2 ;; + --cache-dir) CACHE_DIR="$2"; shift 2 ;; + --build-dir) BUILD_DIR="$2"; shift 2 ;; + --overlay-dir) OVERLAY_DIR="$2"; shift 2 ;; + --cc) CC_CMD="$2"; shift 2 ;; + --ar) AR_CMD="$2"; shift 2 ;; + --ranlib) RANLIB_CMD="$2"; shift 2 ;; + --mbedtls-include) MBEDTLS_INCLUDE="$2"; shift 2 ;; + --mbedtls-libdir) MBEDTLS_LIBDIR="$2"; shift 2 ;; + --zlib-include) ZLIB_INCLUDE="$2"; shift 2 ;; + --zlib-libdir) ZLIB_LIBDIR="$2"; shift 2 ;; + --brotli-include) BROTLI_INCLUDE="$2"; shift 2 ;; + --brotli-libdir) BROTLI_LIBDIR="$2"; shift 2 ;; + --zstd-include) ZSTD_INCLUDE="$2"; shift 2 ;; + --zstd-libdir) ZSTD_LIBDIR="$2"; shift 2 ;; + --ca-bundle) CA_BUNDLE="$2"; shift 2 ;; + --output) OUTPUT="$2"; shift 2 ;; *) echo "Unknown argument: $1" >&2 usage >&2 @@ -78,7 +78,7 @@ while [[ $# -gt 0 ]]; do esac done -if [[ -z "$VERSION" || -z "$TAG" || -z "$URL" || -z "$CACHE_DIR" || -z "$BUILD_DIR" || -z "$OVERLAY_DIR" || -z "$CC_CMD" || -z "$AR_CMD" || -z "$RANLIB_CMD" || -z "$OUTPUT" ]]; then +if [[ -z "$VERSION" || -z "$TAG" || -z "$URL" || -z "$CACHE_DIR" || -z "$BUILD_DIR" || -z "$OVERLAY_DIR" || -z "$CC_CMD" || -z "$AR_CMD" || -z "$RANLIB_CMD" || -z "$MBEDTLS_INCLUDE" || -z "$MBEDTLS_LIBDIR" || -z "$ZLIB_INCLUDE" || -z "$ZLIB_LIBDIR" || -z "$BROTLI_INCLUDE" || -z "$BROTLI_LIBDIR" || -z "$ZSTD_INCLUDE" || -z "$ZSTD_LIBDIR" || -z "$CA_BUNDLE" || -z "$OUTPUT" ]]; then usage >&2 exit 1 fi @@ -122,6 +122,27 @@ while IFS= read -r -d '' file; do cp "$file" "$SRC_DIR/$rel" done < <(find "$OVERLAY_DIR" -type f -print0) +# Stage a single dependency prefix (include/ + lib/) so curl's configure can +# find mbedTLS/zlib/brotli/zstd with plain --with-=. PKG_CONFIG is +# disabled (no .pc files for our cross-built statics), so configure derives +# -I/include -L/lib and the correct -l flags from the prefix. +DEPS="$BUILD_DIR/deps" +rm -rf "$DEPS" +mkdir -p "$DEPS/include" "$DEPS/lib" + +# Headers: mbedtls/ + psa/, zlib.h/zconf.h, brotli/, zstd.h — merged into one tree. +cp -a "$MBEDTLS_INCLUDE/." "$DEPS/include/" +cp -a "$ZLIB_INCLUDE/zlib.h" "$ZLIB_INCLUDE/zconf.h" "$DEPS/include/" +cp -a "$BROTLI_INCLUDE/." "$DEPS/include/" +cp -a "$ZSTD_INCLUDE/zstd.h" "$DEPS/include/" + +# Static archives. +cp -a "$MBEDTLS_LIBDIR/libmbedtls.a" "$MBEDTLS_LIBDIR/libmbedx509.a" \ + "$MBEDTLS_LIBDIR/libmbedcrypto.a" "$DEPS/lib/" +cp -a "$ZLIB_LIBDIR/libz.a" "$DEPS/lib/" +cp -a "$BROTLI_LIBDIR/libbrotlidec.a" "$BROTLI_LIBDIR/libbrotlicommon.a" "$DEPS/lib/" +cp -a "$ZSTD_LIBDIR/libzstd.a" "$DEPS/lib/" + pushd "$SRC_DIR" >/dev/null echo "Patching WASI-incompatible signal/setjmp includes..." @@ -168,56 +189,30 @@ for rel_path, edits in replacements.items(): path.write_text(updated) PY -echo "Configuring upstream curl for wasm32-wasip1..." +echo "Configuring upstream curl for wasm32-wasip1 (in-guest mbedTLS + zlib/brotli/zstd)..." +# mbedTLS 3.x removed the legacy havege RNG, so curl's configure runs its +# "mbedtls_ssl_init in -lmbedtls" link probe — which needs the three archives +# in the correct static order plus brotlicommon (brotlidec depends on it). +# LIBS carries brotlicommon (configure only appends -lbrotlidec on its own). CC="$CC_CMD" \ AR="$AR_CMD" \ RANLIB="$RANLIB_CMD" \ +PKG_CONFIG="false" \ CFLAGS="-O2 -flto" \ +CPPFLAGS="-I$DEPS/include" \ +LDFLAGS="-L$DEPS/lib" \ +LIBS="-lbrotlicommon" \ ./configure \ --host=wasm32-unknown-wasi \ --disable-shared \ --disable-threaded-resolver \ --disable-ldap \ - --without-zlib \ - --without-brotli \ - --without-zstd \ --without-libpsl \ - --without-ca-bundle \ - --without-ca-path \ - --without-ssl - -echo "Enabling the secure-exec WASI TLS backend in generated curl config..." -python3 - <<'PY' -from pathlib import Path - -config = Path("lib/curl_config.h") -text = config.read_text() - -updates = { - "#define CURL_DISABLE_HSTS 1": "/* #undef CURL_DISABLE_HSTS */", -} -for old, new in updates.items(): - text = text.replace(old, new) - -if "#define USE_WASI_TLS 1" not in text: - text += "\n#define USE_WASI_TLS 1\n" -if "#define CURL_DISABLE_OPENSSL_AUTO_LOAD_CONFIG 1" not in text: - text += "#define CURL_DISABLE_OPENSSL_AUTO_LOAD_CONFIG 1\n" - -config.write_text(text) -PY - -echo "Patching generated lib/Makefile to compile wasi_tls.c..." -cat >> lib/Makefile <<'EOF' - -am_libcurl_la_OBJECTS += vtls/libcurl_la-wasi_tls.lo -libcurl_la_LIBADD += vtls/libcurl_la-wasi_tls.lo -libcurl.la: vtls/libcurl_la-wasi_tls.lo - -vtls/libcurl_la-wasi_tls.lo: vtls/$(am__dirstamp) vtls/$(DEPDIR)/$(am__dirstamp) vtls/wasi_tls.c - $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libcurl_la_CPPFLAGS) $(CPPFLAGS) $(libcurl_la_CFLAGS) $(CFLAGS) -MT vtls/libcurl_la-wasi_tls.lo -MD -MP -MF vtls/$(DEPDIR)/libcurl_la-wasi_tls.Tpo -c -o vtls/libcurl_la-wasi_tls.lo `test -f 'vtls/wasi_tls.c' || echo '$(srcdir)/'`vtls/wasi_tls.c - $(AM_V_at)$(am__mv) vtls/$(DEPDIR)/libcurl_la-wasi_tls.Tpo vtls/$(DEPDIR)/libcurl_la-wasi_tls.Plo -EOF + --with-mbedtls="$DEPS" \ + --with-zlib="$DEPS" \ + --with-brotli="$DEPS" \ + --with-zstd="$DEPS" \ + --with-ca-bundle="$CA_BUNDLE" echo "Building upstream libcurl..." make -C lib libcurl.la @@ -225,6 +220,33 @@ make -C lib libcurl.la echo "Building upstream curl tool..." make -C src curl +# Reusable libcurl artifact: install the overlaid, mbedTLS-linked libcurl into a +# prefix (headers + libcurl.a) so git can link it in-process. Same overlay and +# configure as above, so git's git-remote-http gets the identical TLS backend. +if [[ -n "$INSTALL_PREFIX" ]]; then + echo "Installing libcurl to $INSTALL_PREFIX ..." + rm -rf "$INSTALL_PREFIX" + mkdir -p "$INSTALL_PREFIX/include/curl" "$INSTALL_PREFIX/lib/pkgconfig" + make -C lib install prefix="$INSTALL_PREFIX" >/dev/null 2>&1 || true + make -C include install prefix="$INSTALL_PREFIX" >/dev/null 2>&1 || true + # Guarantee headers + the static archive land regardless of libtool install + # quirks under a cross toolchain. + cp -a include/curl/*.h "$INSTALL_PREFIX/include/curl/" 2>/dev/null || true + if [[ -f lib/.libs/libcurl.a && ! -f "$INSTALL_PREFIX/lib/libcurl.a" ]]; then + cp -a lib/.libs/libcurl.a "$INSTALL_PREFIX/lib/libcurl.a" + fi + cp -a libcurl.pc "$INSTALL_PREFIX/lib/pkgconfig/" 2>/dev/null || true + if [[ ! -f "$INSTALL_PREFIX/lib/libcurl.a" ]]; then + echo "libcurl.a missing after install into $INSTALL_PREFIX" >&2 + exit 1 + fi + if [[ ! -f "$INSTALL_PREFIX/include/curl/curl.h" ]]; then + echo "curl/curl.h missing after install into $INSTALL_PREFIX" >&2 + exit 1 + fi + echo "Installed libcurl artifact ($(wc -c < "$INSTALL_PREFIX/lib/libcurl.a") bytes) at $INSTALL_PREFIX" +fi + BIN="" for candidate in "src/.libs/curl" "src/curl" "src/curl.wasm"; do if [[ -f "$candidate" ]]; then diff --git a/toolchain/c/scripts/build-git-upstream.sh b/toolchain/c/scripts/build-git-upstream.sh index f5268b6943..40d343a530 100755 --- a/toolchain/c/scripts/build-git-upstream.sh +++ b/toolchain/c/scripts/build-git-upstream.sh @@ -11,11 +11,16 @@ Usage: build-git-upstream.sh \ --patch-dir \ --zlib-dir \ --zlib-build-dir \ + --curl-prefix \ + --mbedtls-libdir \ + --brotli-libdir \ + --zstd-libdir \ --cc \ --ar \ --ranlib \ --sysroot \ - --output + --output \ + --remote-http-output EOF } @@ -25,12 +30,17 @@ CACHE_DIR="" BUILD_DIR="" ZLIB_DIR="" ZLIB_BUILD_DIR="" +CURL_PREFIX="" +MBEDTLS_LIBDIR="" +BROTLI_LIBDIR="" +ZSTD_LIBDIR="" PATCH_DIR="" CC_CMD="" AR_CMD="" RANLIB_CMD="" SYSROOT="" OUTPUT="" +REMOTE_HTTP_OUTPUT="" while [[ $# -gt 0 ]]; do case "$1" in @@ -41,16 +51,21 @@ while [[ $# -gt 0 ]]; do --patch-dir) PATCH_DIR="$2"; shift 2 ;; --zlib-dir) ZLIB_DIR="$2"; shift 2 ;; --zlib-build-dir) ZLIB_BUILD_DIR="$2"; shift 2 ;; + --curl-prefix) CURL_PREFIX="$2"; shift 2 ;; + --mbedtls-libdir) MBEDTLS_LIBDIR="$2"; shift 2 ;; + --brotli-libdir) BROTLI_LIBDIR="$2"; shift 2 ;; + --zstd-libdir) ZSTD_LIBDIR="$2"; shift 2 ;; --cc) CC_CMD="$2"; shift 2 ;; --ar) AR_CMD="$2"; shift 2 ;; --ranlib) RANLIB_CMD="$2"; shift 2 ;; --sysroot) SYSROOT="$2"; shift 2 ;; --output) OUTPUT="$2"; shift 2 ;; + --remote-http-output) REMOTE_HTTP_OUTPUT="$2"; shift 2 ;; *) echo "Unknown argument: $1" >&2; usage >&2; exit 1 ;; esac done -if [[ -z "$VERSION" || -z "$URL" || -z "$CACHE_DIR" || -z "$BUILD_DIR" || -z "$PATCH_DIR" || -z "$ZLIB_DIR" || -z "$ZLIB_BUILD_DIR" || -z "$CC_CMD" || -z "$AR_CMD" || -z "$RANLIB_CMD" || -z "$SYSROOT" || -z "$OUTPUT" ]]; then +if [[ -z "$VERSION" || -z "$URL" || -z "$CACHE_DIR" || -z "$BUILD_DIR" || -z "$PATCH_DIR" || -z "$ZLIB_DIR" || -z "$ZLIB_BUILD_DIR" || -z "$CURL_PREFIX" || -z "$MBEDTLS_LIBDIR" || -z "$BROTLI_LIBDIR" || -z "$ZSTD_LIBDIR" || -z "$CC_CMD" || -z "$AR_CMD" || -z "$RANLIB_CMD" || -z "$SYSROOT" || -z "$OUTPUT" || -z "$REMOTE_HTTP_OUTPUT" ]]; then usage >&2 exit 1 fi @@ -97,7 +112,24 @@ if [[ -d "$PATCH_DIR" ]]; then done fi -echo "Building upstream Git ${VERSION} for wasm32-wasip1..." +# libcurl + its TLS/compression backends, linked in-process by git-remote-http. +# Defining CURL_CFLAGS/CURL_LDFLAGS makes Git's Makefile skip the curl-config +# probe entirely and use these flags verbatim. Static link order matters: +# libcurl first, then mbedTLS (tls -> x509 -> crypto), then zlib and the brotli / +# zstd decoders curl's --compressed path pulls in. +CURL_CFLAGS="-I$CURL_PREFIX/include" +CURL_LDFLAGS="-L$CURL_PREFIX/lib -lcurl -L$MBEDTLS_LIBDIR -lmbedtls -lmbedx509 -lmbedcrypto -L$ZLIB_BUILD_DIR -lz -L$BROTLI_LIBDIR -lbrotlidec -lbrotlicommon -L$ZSTD_LIBDIR -lzstd" + +echo "Building upstream Git ${VERSION} for wasm32-wasip1 (with in-process libcurl)..." +# Stack layout: wasm-ld defaults to a 64 KiB stack placed *above* the data +# segment. Git assumes native-sized stacks (recv_sideband alone puts a +# LARGE_PACKET_MAX (~64 KiB) buffer on the stack, reached several frames deep +# on every sideband pack transfer), so with the default layout the buffer +# silently overwrites static data and clones/fetches die with wild wasm traps +# once packs carry payloads bigger than a few KiB. Give git an 8 MiB stack +# (glibc-default sized) and --stack-first so any future overflow traps at the +# faulting frame instead of corrupting the data segment. +GIT_WASM_STACK_FLAGS="-Wl,-z,stack-size=8388608 -Wl,--stack-first" make -j"${MAKE_JOBS:-2}" \ uname_S=WASI \ CC="$CC_CMD" \ @@ -105,7 +137,9 @@ make -j"${MAKE_JOBS:-2}" \ AR="$AR_CMD" \ RANLIB="$RANLIB_CMD" \ CFLAGS="--target=wasm32-wasip1 --sysroot=$SYSROOT -I$ZLIB_DIR -O2 -D_WASI_EMULATED_PROCESS_CLOCKS -D_WASI_EMULATED_MMAN" \ - LDFLAGS="--target=wasm32-wasip1 --sysroot=$SYSROOT -L$ZLIB_BUILD_DIR -lwasi-emulated-process-clocks -lwasi-emulated-mman" \ + LDFLAGS="--target=wasm32-wasip1 --sysroot=$SYSROOT -L$ZLIB_BUILD_DIR -lwasi-emulated-process-clocks -lwasi-emulated-mman $GIT_WASM_STACK_FLAGS" \ + CURL_CFLAGS="$CURL_CFLAGS" \ + CURL_LDFLAGS="$CURL_LDFLAGS" \ CSPRNG_METHOD=getentropy \ HAVE_PATHS_H=YesPlease \ HAVE_DEV_TTY=YesPlease \ @@ -114,7 +148,6 @@ make -j"${MAKE_JOBS:-2}" \ HAVE_GETDELIM=YesPlease \ NO_RUST=1 \ NO_OPENSSL=1 \ - NO_CURL=1 \ NO_EXPAT=1 \ NO_GETTEXT=1 \ NO_TCLTK=1 \ @@ -128,16 +161,29 @@ make -j"${MAKE_JOBS:-2}" \ NO_UNIX_SOCKETS=1 \ NO_SYS_POLL_H=1 \ NO_NSEC=1 \ - git - -mkdir -p "$(dirname "$OUTPUT")" -if command -v wasm-opt >/dev/null 2>&1; then - echo "Optimizing Git WASM binary..." - wasm-opt -O3 --strip-debug --all-features git -o "$OUTPUT" -else - cp git "$OUTPUT" + git git-remote-http + +if [[ ! -f git-remote-http ]]; then + echo "Expected git-remote-http binary was not produced" >&2 + exit 1 fi +emit() { + local src="$1" + local out="$2" + mkdir -p "$(dirname "$out")" + if command -v wasm-opt >/dev/null 2>&1; then + echo "Optimizing $src WASM binary -> $out..." + wasm-opt -O3 --strip-debug --all-features "$src" -o "$out" + else + cp "$src" "$out" + fi +} + +emit git "$OUTPUT" +emit git-remote-http "$REMOTE_HTTP_OUTPUT" + popd >/dev/null echo "Built upstream Git at $OUTPUT" +echo "Built git-remote-http at $REMOTE_HTTP_OUTPUT" diff --git a/toolchain/c/scripts/build-procps-upstream.sh b/toolchain/c/scripts/build-procps-upstream.sh new file mode 100755 index 0000000000..33f2fd70d2 --- /dev/null +++ b/toolchain/c/scripts/build-procps-upstream.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +set -euo pipefail + +VERSION="" +URL="" +SHA256="" +CACHE_DIR="" +BUILD_DIR="" +OVERLAY_INCLUDE_DIR="" +CC_CMD="" +AR_CMD="" +RANLIB_CMD="" +SYSROOT="" +OUTPUT_DIR="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --version) VERSION="$2"; shift 2 ;; + --url) URL="$2"; shift 2 ;; + --sha256) SHA256="$2"; shift 2 ;; + --cache-dir) CACHE_DIR="$2"; shift 2 ;; + --build-dir) BUILD_DIR="$2"; shift 2 ;; + --overlay-include-dir) OVERLAY_INCLUDE_DIR="$2"; shift 2 ;; + --cc) CC_CMD="$2"; shift 2 ;; + --ar) AR_CMD="$2"; shift 2 ;; + --ranlib) RANLIB_CMD="$2"; shift 2 ;; + --sysroot) SYSROOT="$2"; shift 2 ;; + --output-dir) OUTPUT_DIR="$2"; shift 2 ;; + *) echo "Unknown argument: $1" >&2; exit 1 ;; + esac +done + +for required in VERSION URL SHA256 CACHE_DIR BUILD_DIR OVERLAY_INCLUDE_DIR CC_CMD AR_CMD RANLIB_CMD SYSROOT OUTPUT_DIR; do + if [[ -z "${!required}" ]]; then + echo "Missing required argument: $required" >&2 + exit 1 + fi +done + +fetch() { + local url="$1" out="$2" + if command -v curl >/dev/null 2>&1; then + curl --retry 3 --retry-all-errors -fSL "$url" -o "$out" + else + wget -q "$url" -O "$out" + fi +} + +mkdir -p "$CACHE_DIR" +tarball="$CACHE_DIR/procps-ng-$VERSION.tar.xz" +[[ -f "$tarball" ]] || fetch "$URL" "$tarball" +printf '%s %s\n' "$SHA256" "$tarball" | sha256sum -c - + +rm -rf "$BUILD_DIR" +mkdir -p "$BUILD_DIR/source" "$BUILD_DIR/out" +tar -xJf "$tarball" -C "$BUILD_DIR/source" +src_dir="$BUILD_DIR/source/procps-ng-$VERSION" +[[ -d "$src_dir" ]] || { echo "Expected source at $src_dir" >&2; exit 1; } + +common_flags="--target=wasm32-wasip1 --sysroot=$SYSROOT -I$OVERLAY_INCLUDE_DIR -O2 -D_GNU_SOURCE -D_WASI_EMULATED_PROCESS_CLOCKS -D_WASI_EMULATED_MMAN" +link_flags="--target=wasm32-wasip1 --sysroot=$SYSROOT -Wl,-z,stack-size=8388608 -Wl,--stack-first" + +pushd "$BUILD_DIR/out" >/dev/null +CC="$CC_CMD" \ +CPP="$CC_CMD $common_flags -E" \ +AR="$AR_CMD" \ +RANLIB="$RANLIB_CMD" \ +CFLAGS="$common_flags" \ +LDFLAGS="$link_flags" \ +LIBS="-lwasi-emulated-process-clocks -lwasi-emulated-mman" \ +ac_cv_func_fork=no \ +ac_cv_func_vfork=no \ +ac_cv_func_dup2=yes \ +ac_cv_func_gethostname=no \ +ac_cv_func_malloc_0_nonnull=yes \ +ac_cv_func_realloc_0_nonnull=yes \ +"$src_dir/configure" \ + --host=wasm32-unknown-wasi \ + --disable-shared \ + --enable-static \ + --disable-nls \ + --without-ncurses \ + --disable-pidof \ + --disable-pidwait \ + --disable-kill \ + --disable-w + +make -j"${MAKE_JOBS:-2}" src/ps/pscommand src/pgrep + +emit() { + local source="$1" target="$2" + mkdir -p "$OUTPUT_DIR" + if command -v wasm-opt >/dev/null 2>&1; then + wasm-opt -O3 --strip-debug --all-features "$source" -o "$OUTPUT_DIR/$target" + else + cp "$source" "$OUTPUT_DIR/$target" + fi +} + +emit src/ps/pscommand ps +emit src/pgrep pgrep +emit src/pgrep pkill +popd >/dev/null + +echo "Built upstream procps-ng $VERSION: ps pgrep pkill" diff --git a/toolchain/c/scripts/build-psmisc-upstream.sh b/toolchain/c/scripts/build-psmisc-upstream.sh new file mode 100755 index 0000000000..9be34287f4 --- /dev/null +++ b/toolchain/c/scripts/build-psmisc-upstream.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +set -euo pipefail + +VERSION="" +URL="" +SHA256="" +CACHE_DIR="" +BUILD_DIR="" +PATCH_DIR="" +OVERLAY_INCLUDE_DIR="" +CC_CMD="" +AR_CMD="" +RANLIB_CMD="" +SYSROOT="" +OUTPUT_DIR="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --version) VERSION="$2"; shift 2 ;; + --url) URL="$2"; shift 2 ;; + --sha256) SHA256="$2"; shift 2 ;; + --cache-dir) CACHE_DIR="$2"; shift 2 ;; + --build-dir) BUILD_DIR="$2"; shift 2 ;; + --patch-dir) PATCH_DIR="$2"; shift 2 ;; + --overlay-include-dir) OVERLAY_INCLUDE_DIR="$2"; shift 2 ;; + --cc) CC_CMD="$2"; shift 2 ;; + --ar) AR_CMD="$2"; shift 2 ;; + --ranlib) RANLIB_CMD="$2"; shift 2 ;; + --sysroot) SYSROOT="$2"; shift 2 ;; + --output-dir) OUTPUT_DIR="$2"; shift 2 ;; + *) echo "Unknown argument: $1" >&2; exit 1 ;; + esac +done + +for required in VERSION URL SHA256 CACHE_DIR BUILD_DIR PATCH_DIR OVERLAY_INCLUDE_DIR CC_CMD AR_CMD RANLIB_CMD SYSROOT OUTPUT_DIR; do + if [[ -z "${!required}" ]]; then + echo "Missing required argument: $required" >&2 + exit 1 + fi +done + +fetch() { + local url="$1" out="$2" + if command -v curl >/dev/null 2>&1; then + curl --retry 3 --retry-all-errors -fSL "$url" -o "$out" + else + wget -q "$url" -O "$out" + fi +} + +mkdir -p "$CACHE_DIR" +tarball="$CACHE_DIR/psmisc-$VERSION.tar.xz" +[[ -f "$tarball" ]] || fetch "$URL" "$tarball" +printf '%s %s\n' "$SHA256" "$tarball" | sha256sum -c - + +rm -rf "$BUILD_DIR" +mkdir -p "$BUILD_DIR/source" "$BUILD_DIR/out" +tar -xJf "$tarball" -C "$BUILD_DIR/source" +src_dir="$BUILD_DIR/source/psmisc-$VERSION" +[[ -d "$src_dir" ]] || { echo "Expected source at $src_dir" >&2; exit 1; } +for patch_file in "$PATCH_DIR"/*.patch; do + [[ -e "$patch_file" ]] || continue + patch -d "$src_dir" -p1 < "$patch_file" +done + +common_flags="--target=wasm32-wasip1 --sysroot=$SYSROOT -I$OVERLAY_INCLUDE_DIR -O2 -D_GNU_SOURCE -D_WASI_EMULATED_PROCESS_CLOCKS -D_WASI_EMULATED_MMAN" +link_flags="--target=wasm32-wasip1 --sysroot=$SYSROOT -Wl,-z,stack-size=8388608 -Wl,--stack-first" + +pushd "$BUILD_DIR/out" >/dev/null +CC="$CC_CMD" \ +CPP="$CC_CMD $common_flags -E" \ +AR="$AR_CMD" \ +RANLIB="$RANLIB_CMD" \ +CFLAGS="$common_flags" \ +LDFLAGS="$link_flags" \ +LIBS="-lwasi-emulated-process-clocks -lwasi-emulated-mman" \ +ac_cv_func_fork=no \ +ac_cv_func_vfork=no \ +ac_cv_func_malloc_0_nonnull=yes \ +ac_cv_func_realloc_0_nonnull=yes \ +ac_cv_func_memcmp_working=yes \ +ac_cv_func_lstat_dereferences_slashed_symlink=yes \ +ac_cv_func_lstat_empty_string_bug=no \ +ac_cv_func_stat_empty_string_bug=no \ +"$src_dir/configure" \ + --host=wasm32-unknown-wasi \ + --disable-nls \ + --disable-selinux \ + --disable-statx \ + --disable-ipv6 \ + --disable-harden-flags + +make -j"${MAKE_JOBS:-2}" src/pstree src/killall src/prtstat + +emit() { + local source="$1" target="$2" + mkdir -p "$OUTPUT_DIR" + if command -v wasm-opt >/dev/null 2>&1; then + wasm-opt -O3 --strip-debug --all-features "$source" -o "$OUTPUT_DIR/$target" + else + cp "$source" "$OUTPUT_DIR/$target" + fi +} + +emit src/pstree pstree +emit src/killall killall +emit src/prtstat prtstat +popd >/dev/null + +echo "Built upstream psmisc $VERSION: pstree killall prtstat" diff --git a/toolchain/c/scripts/build-ssh-upstream.sh b/toolchain/c/scripts/build-ssh-upstream.sh new file mode 100755 index 0000000000..1f5a7dd389 --- /dev/null +++ b/toolchain/c/scripts/build-ssh-upstream.sh @@ -0,0 +1,176 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: build-ssh-upstream.sh \ + --version \ + --url \ + --cache-dir \ + --build-dir \ + --zlib-include \ + --zlib-libdir \ + --overlay-include-dir \ + --cc \ + --ar \ + --ranlib \ + --output +EOF +} + +VERSION="" +URL="" +CACHE_DIR="" +BUILD_DIR="" +ZLIB_INCLUDE="" +ZLIB_LIBDIR="" +OVERLAY_INCLUDE_DIR="" +CC_CMD="" +AR_CMD="" +RANLIB_CMD="" +OUTPUT="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --version) VERSION="$2"; shift 2 ;; + --url) URL="$2"; shift 2 ;; + --cache-dir) CACHE_DIR="$2"; shift 2 ;; + --build-dir) BUILD_DIR="$2"; shift 2 ;; + --zlib-include) ZLIB_INCLUDE="$2"; shift 2 ;; + --zlib-libdir) ZLIB_LIBDIR="$2"; shift 2 ;; + --overlay-include-dir) OVERLAY_INCLUDE_DIR="$2"; shift 2 ;; + --cc) CC_CMD="$2"; shift 2 ;; + --ar) AR_CMD="$2"; shift 2 ;; + --ranlib) RANLIB_CMD="$2"; shift 2 ;; + --output) OUTPUT="$2"; shift 2 ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +if [[ -z "$VERSION" || -z "$URL" || -z "$CACHE_DIR" || -z "$BUILD_DIR" || -z "$ZLIB_INCLUDE" || -z "$ZLIB_LIBDIR" || -z "$OVERLAY_INCLUDE_DIR" || -z "$CC_CMD" || -z "$AR_CMD" || -z "$RANLIB_CMD" || -z "$OUTPUT" ]]; then + usage >&2 + exit 1 +fi + +fetch() { + local url="$1" + local out="$2" + if command -v curl >/dev/null 2>&1; then + curl --retry 3 --retry-all-errors -fSL "$url" -o "$out" + elif command -v wget >/dev/null 2>&1; then + wget -q "$url" -O "$out" + else + echo "Neither curl nor wget is available to fetch $url" >&2 + exit 1 + fi +} + +mkdir -p "$CACHE_DIR" +rm -rf "$BUILD_DIR" +mkdir -p "$BUILD_DIR" + +TARBALL="$CACHE_DIR/openssh-${VERSION}.tar.gz" +if [[ ! -f "$TARBALL" ]]; then + echo "Fetching upstream OpenSSH ${VERSION} release tarball..." + fetch "$URL" "$TARBALL" +fi + +echo "Extracting upstream OpenSSH ${VERSION}..." +tar -xzf "$TARBALL" -C "$BUILD_DIR" + +SRC_DIR="$BUILD_DIR/openssh-${VERSION}" +if [[ ! -d "$SRC_DIR" ]]; then + echo "Expected extracted source at $SRC_DIR" >&2 + exit 1 +fi + +# OpenSSH's --with-zlib=PATH expects a normal install prefix (PATH/include, +# PATH/lib). Our zlib artifacts live in the source tree + a separate build +# dir, so stage a prefix out of symlinks. +ZLIB_PREFIX="$BUILD_DIR/zlib-prefix" +mkdir -p "$ZLIB_PREFIX/include" "$ZLIB_PREFIX/lib" +ln -sf "$ZLIB_INCLUDE/zlib.h" "$ZLIB_PREFIX/include/zlib.h" +ln -sf "$ZLIB_INCLUDE/zconf.h" "$ZLIB_PREFIX/include/zconf.h" +ln -sf "$ZLIB_LIBDIR/libz.a" "$ZLIB_PREFIX/lib/libz.a" + +pushd "$SRC_DIR" >/dev/null + +# Batch/key-based OpenSSH client (RFC 4251 architecture, RFC 4253 transport, +# RFC 4252 auth, RFC 4254 connection) built --without-openssl: the internal +# crypto set is ed25519 signatures, curve25519-sha256 KEX and +# chacha20-poly1305@openssh.com (see upstream README + `ssh -Q` on such a +# build). No RSA/ECDSA, no PKCS#11, no FIDO/security keys. +# +# Stack layout mirrors the git build: wasm-ld's default 64 KiB stack above the +# data segment is far too small for OpenSSH (packet buffers and kex state live +# on the stack several frames deep), so give it an 8 MiB stack and +# --stack-first so overflows trap instead of corrupting static data. +SSH_WASM_STACK_FLAGS="-Wl,-z,stack-size=8388608 -Wl,--stack-first" + +# Autoconf cache seeds for cross-compilation gaps (mirrors the wget build's +# gl_cv_*/ac_cv_* seeding): +# - ac_cv_func_setrlimit=no: the patched sysroot exports a getrlimit-only +# surface (std-patches/wasi-libc/0017-resource-limits-and-groups.patch); +# OpenSSH only uses setrlimit to zero core dumps / fd limits, which the +# kernel already enforces per-VM. +# - ac_cv_have_broken_snprintf/vsnprintf=no: run tests can't execute under +# cross builds; the patched musl snprintf is C99-conformant, so declare it +# instead of letting configure assume the worst and pull in replacements. +# +# CFLAGS carries -I$OVERLAY_INCLUDE_DIR (toolchain/c/include): the shared +# include_next overlays used by every C command build. OpenSSH needs +# 's struct winsize + TIOCGWINSZ (tty(4) window-size ioctl) from +# that overlay for channels.c's window-change tracking; at runtime the +# patched sysroot ioctl() reports ENOTTY on non-PTY fds and the client just +# skips the update. +echo "Configuring upstream OpenSSH for wasm32-wasip1 (--without-openssl)..." +CC="$CC_CMD" \ +AR="$AR_CMD" \ +RANLIB="$RANLIB_CMD" \ +CFLAGS="-O2 -D_WASI_EMULATED_PROCESS_CLOCKS -D_WASI_EMULATED_MMAN -I$OVERLAY_INCLUDE_DIR" \ +LDFLAGS="$SSH_WASM_STACK_FLAGS" \ +LIBS="-lwasi-emulated-process-clocks -lwasi-emulated-mman" \ +ac_cv_func_setrlimit=no \ +ac_cv_have_broken_snprintf=no \ +ac_cv_have_broken_vsnprintf=no \ +./configure \ + --host=wasm32-unknown-wasi \ + --without-openssl \ + --with-zlib="$ZLIB_PREFIX" \ + --without-pam \ + --without-selinux \ + --without-libedit \ + --without-audit \ + --disable-utmp \ + --disable-wtmp \ + --disable-lastlog \ + --disable-btmp \ + --disable-pkcs11 \ + --disable-security-key \ + --without-stackprotect \ + --without-hardening \ + --sysconfdir=/etc/ssh + +echo "Building upstream OpenSSH ssh client..." +make -j"${MAKE_JOBS:-2}" ssh + +if [[ ! -f ssh ]]; then + echo "Expected ssh binary was not produced" >&2 + exit 1 +fi + +mkdir -p "$(dirname "$OUTPUT")" +if command -v wasm-opt >/dev/null 2>&1; then + echo "Optimizing OpenSSH ssh WASM binary..." + wasm-opt -O3 --strip-debug --all-features ssh -o "$OUTPUT" +else + cp ssh "$OUTPUT" +fi + +popd >/dev/null + +echo "Built upstream OpenSSH ssh at $OUTPUT" diff --git a/toolchain/c/scripts/build-wget-upstream.sh b/toolchain/c/scripts/build-wget-upstream.sh index 312937df63..93fa629140 100644 --- a/toolchain/c/scripts/build-wget-upstream.sh +++ b/toolchain/c/scripts/build-wget-upstream.sh @@ -9,6 +9,11 @@ Usage: build-wget-upstream.sh \ --cache-dir \ --build-dir \ --overlay-include-dir \ + --overlay-dir \ + --mbedtls-include \ + --mbedtls-libdir \ + --zlib-include \ + --zlib-libdir \ --cc \ --ar \ --ranlib \ @@ -21,6 +26,11 @@ URL="" CACHE_DIR="" BUILD_DIR="" OVERLAY_INCLUDE_DIR="" +OVERLAY_DIR="" +MBEDTLS_INCLUDE="" +MBEDTLS_LIBDIR="" +ZLIB_INCLUDE="" +ZLIB_LIBDIR="" CC_CMD="" AR_CMD="" RANLIB_CMD="" @@ -28,42 +38,20 @@ OUTPUT="" while [[ $# -gt 0 ]]; do case "$1" in - --version) - VERSION="$2" - shift 2 - ;; - --url) - URL="$2" - shift 2 - ;; - --cache-dir) - CACHE_DIR="$2" - shift 2 - ;; - --build-dir) - BUILD_DIR="$2" - shift 2 - ;; - --overlay-include-dir) - OVERLAY_INCLUDE_DIR="$2" - shift 2 - ;; - --cc) - CC_CMD="$2" - shift 2 - ;; - --ar) - AR_CMD="$2" - shift 2 - ;; - --ranlib) - RANLIB_CMD="$2" - shift 2 - ;; - --output) - OUTPUT="$2" - shift 2 - ;; + --version) VERSION="$2"; shift 2 ;; + --url) URL="$2"; shift 2 ;; + --cache-dir) CACHE_DIR="$2"; shift 2 ;; + --build-dir) BUILD_DIR="$2"; shift 2 ;; + --overlay-include-dir) OVERLAY_INCLUDE_DIR="$2"; shift 2 ;; + --overlay-dir) OVERLAY_DIR="$2"; shift 2 ;; + --mbedtls-include) MBEDTLS_INCLUDE="$2"; shift 2 ;; + --mbedtls-libdir) MBEDTLS_LIBDIR="$2"; shift 2 ;; + --zlib-include) ZLIB_INCLUDE="$2"; shift 2 ;; + --zlib-libdir) ZLIB_LIBDIR="$2"; shift 2 ;; + --cc) CC_CMD="$2"; shift 2 ;; + --ar) AR_CMD="$2"; shift 2 ;; + --ranlib) RANLIB_CMD="$2"; shift 2 ;; + --output) OUTPUT="$2"; shift 2 ;; *) echo "Unknown argument: $1" >&2 usage >&2 @@ -72,7 +60,7 @@ while [[ $# -gt 0 ]]; do esac done -if [[ -z "$VERSION" || -z "$URL" || -z "$CACHE_DIR" || -z "$BUILD_DIR" || -z "$OVERLAY_INCLUDE_DIR" || -z "$CC_CMD" || -z "$AR_CMD" || -z "$RANLIB_CMD" || -z "$OUTPUT" ]]; then +if [[ -z "$VERSION" || -z "$URL" || -z "$CACHE_DIR" || -z "$BUILD_DIR" || -z "$OVERLAY_INCLUDE_DIR" || -z "$OVERLAY_DIR" || -z "$MBEDTLS_INCLUDE" || -z "$MBEDTLS_LIBDIR" || -z "$ZLIB_INCLUDE" || -z "$ZLIB_LIBDIR" || -z "$CC_CMD" || -z "$AR_CMD" || -z "$RANLIB_CMD" || -z "$OUTPUT" ]]; then usage >&2 exit 1 fi @@ -111,7 +99,15 @@ fi pushd "$SRC_DIR" >/dev/null -echo "Configuring upstream GNU Wget for wasm32-wasip1..." +# GNU Wget has no upstream mbedTLS backend. We keep ./configure --without-ssl so +# it never probes GnuTLS/OpenSSL, then light up its SSL abstraction manually: +# patch src/wget.h's HAVE_SSL derivation to trigger on HAVE_WASI_TLS, compile +# -DHAVE_WASI_TLS everywhere (so http.c/main.c/init.c enable the https scheme, +# --secure-protocol/--ca-certificate/--no-check-certificate and HSTS), overlay +# our mbedTLS backend (src/wasi_ssl.c), and append its object + the mbedTLS +# archives to the generated src/Makefile link. This mirrors the curl build's +# playbook. zlib is enabled for gzip Content-Encoding via ZLIB_CFLAGS/ZLIB_LIBS. +echo "Configuring upstream GNU Wget for wasm32-wasip1 (in-guest mbedTLS + zlib)..." CC="$CC_CMD" \ AR="$AR_CMD" \ RANLIB="$RANLIB_CMD" \ @@ -122,7 +118,9 @@ PCRE2_CFLAGS="" \ PCRE2_LIBS="" \ PCRE_CFLAGS="" \ PCRE_LIBS="" \ -CPPFLAGS="-I$OVERLAY_INCLUDE_DIR" \ +ZLIB_CFLAGS="-I$ZLIB_INCLUDE" \ +ZLIB_LIBS="-L$ZLIB_LIBDIR -lz" \ +CPPFLAGS="-I$OVERLAY_INCLUDE_DIR -I$MBEDTLS_INCLUDE" \ gl_cv_func_posix_spawn_works=yes \ gl_cv_func_posix_spawn_secure_exec=yes \ gl_cv_func_posix_spawnp_secure_exec=yes \ @@ -134,7 +132,7 @@ gl_cv_func_select_detects_ebadf=yes \ gl_cv_func_pselect_detects_ebadf=yes \ ac_cv_func_clock_getres=no \ ac_cv_func_clock_gettime=no \ -CFLAGS="-O2 -flto -D_WASI_EMULATED_SIGNAL -D_WASI_EMULATED_MMAN -D_WASI_EMULATED_PROCESS_CLOCKS -DFD_SETSIZE=8192" \ +CFLAGS="-O2 -flto -DHAVE_WASI_TLS -D_WASI_EMULATED_SIGNAL -D_WASI_EMULATED_MMAN -D_WASI_EMULATED_PROCESS_CLOCKS -DFD_SETSIZE=8192" \ LIBS="-lwasi-emulated-signal -lwasi-emulated-mman -lwasi-emulated-process-clocks" \ ./configure \ --host=wasm32-unknown-wasi \ @@ -148,10 +146,86 @@ LIBS="-lwasi-emulated-signal -lwasi-emulated-mman -lwasi-emulated-process-clocks --disable-pcre2 \ --without-ssl \ --without-libpsl \ - --without-zlib \ --without-libuuid \ --without-metalink +# Patch HAVE_SSL derivation: wget only defines HAVE_SSL when a probed backend +# (OpenSSL/GnuTLS) is present. Add our in-guest backend to the condition. +echo "Patching src/wget.h HAVE_SSL derivation for HAVE_WASI_TLS..." +python3 - <<'PY' +from pathlib import Path + +path = Path("src/wget.h") +text = path.read_text() +needle = "#if defined HAVE_LIBSSL || defined HAVE_LIBSSL32 || defined HAVE_LIBGNUTLS" +replacement = needle + " || defined HAVE_WASI_TLS" +if "defined HAVE_WASI_TLS" not in text: + if needle not in text: + raise SystemExit("Expected HAVE_SSL derivation not found in src/wget.h") + text = text.replace(needle, replacement, 1) + path.write_text(text) + print(" patched src/wget.h") +else: + print(" src/wget.h already patched") + +# Report the real backend in `wget --version` (otherwise it prints "-ssl", +# since neither HAVE_LIBSSL nor HAVE_LIBGNUTLS is defined). +bi = Path("src/build_info.c") +bitext = bi.read_text() +ssl_needle = ('#if defined HAVE_LIBSSL || defined HAVE_LIBSSL32\n' + ' "+ssl/openssl",\n' + '#elif defined HAVE_LIBGNUTLS\n' + ' "+ssl/gnutls",\n' + '#else\n' + ' "-ssl",\n' + '#endif') +ssl_repl = ('#if defined HAVE_LIBSSL || defined HAVE_LIBSSL32\n' + ' "+ssl/openssl",\n' + '#elif defined HAVE_LIBGNUTLS\n' + ' "+ssl/gnutls",\n' + '#elif defined HAVE_WASI_TLS\n' + ' "+ssl/mbedtls",\n' + '#else\n' + ' "-ssl",\n' + '#endif') +if '"+ssl/mbedtls"' not in bitext: + if ssl_needle not in bitext: + raise SystemExit("Expected ssl feature block not found in src/build_info.c") + bi.write_text(bitext.replace(ssl_needle, ssl_repl, 1)) + print(" patched src/build_info.c") +else: + print(" src/build_info.c already patched") +PY + +# Overlay the mbedTLS backend (src/wasi_ssl.c) and any other overlay files. +echo "Applying wget overlay from $OVERLAY_DIR ..." +while IFS= read -r -d '' file; do + rel="${file#"$OVERLAY_DIR"/}" + mkdir -p "$(dirname "$rel")" + cp "$file" "$rel" + echo " overlaid $rel" +done < <(find "$OVERLAY_DIR" -type f -print0) + +# Append the wasi_ssl.o object + mbedTLS archives to the generated src/Makefile. +# wget_OBJECTS/wget_LDADD are recursively-expanded, so += appends cleanly and +# the object is picked up by automake's built-in .c.o rule (which already +# carries our CPPFLAGS mbedTLS include and -DHAVE_WASI_TLS from CFLAGS). The +# archives are ordered tls -> x509 -> crypto for the static link. +echo "Appending wasi_ssl.o + mbedTLS link rules to src/Makefile..." +cat >> src/Makefile < spells the constants AF_*; the PF_* names are the + * historical BSD "protocol family" aliases (4.4BSD; on Linux and musl + * PF_UNIX == AF_UNIX, PF_LOCAL/AF_LOCAL are additional POSIX.1-2008 spellings + * of the same family). Ported tools still use them (e.g. OpenSSH misc.c's + * unix_listener uses PF_UNIX), so alias them to the AF_* values above. */ +#define AF_LOCAL AF_UNIX +#define PF_UNIX AF_UNIX +#define PF_LOCAL AF_UNIX #ifdef __cplusplus extern "C" { diff --git a/toolchain/c/sysroot/include/wasm32-wasi/fcntl.h b/toolchain/c/sysroot/include/wasm32-wasi/fcntl.h index f622311b9a..00a7a5444d 100644 --- a/toolchain/c/sysroot/include/wasm32-wasi/fcntl.h +++ b/toolchain/c/sysroot/include/wasm32-wasi/fcntl.h @@ -51,6 +51,13 @@ int posix_fallocate(int, off_t, off_t); #define O_TEXT 0 #endif +#ifndef O_PATH +/* WASI O_SEARCH is the descriptor-only path capability corresponding to + * Linux O_PATH for directory-relative lookups. + * https://man7.org/linux/man-pages/man2/open.2.html */ +#define O_PATH O_SEARCH +#endif + #ifndef F_RDLCK #define F_RDLCK 0 #endif diff --git a/toolchain/c/sysroot/include/wasm32-wasi/netdb.h b/toolchain/c/sysroot/include/wasm32-wasi/netdb.h index eecb4824fc..bc35d70b53 100644 --- a/toolchain/c/sysroot/include/wasm32-wasi/netdb.h +++ b/toolchain/c/sysroot/include/wasm32-wasi/netdb.h @@ -160,6 +160,44 @@ int getservbyname_r(const char *, const char *, struct servent *, char *, size_t #endif + +/* OpenBSD getrrsetbyname(3) surface (also on FreeBSD/NetBSD libc): DNS RRset + * query API used by OpenSSH's dns.c for SSHFP host-key verification + * (VerifyHostKeyDNS, RFC 4255). Struct layout and constants match OpenBSD + * . The agentOS sysroot has no DNS resolver library (lookups route + * through the host getaddrinfo import), so the libc definition + * (std-patches/wasi-libc-overrides/openssh_compat.c) always fails with + * ERRSET_FAIL and callers degrade to a normal "DNS lookup error" path. */ +struct rdatainfo { + unsigned int rdi_length; /* length of data */ + unsigned char *rdi_data; /* record data */ +}; + +struct rrsetinfo { + unsigned int rri_flags; /* RRSET_VALIDATED... */ + unsigned int rri_rdclass; /* class number */ + unsigned int rri_rdtype; /* RR type number */ + unsigned int rri_ttl; /* time to live */ + unsigned int rri_nrdatas; /* size of rdatas array */ + unsigned int rri_nsigs; /* size of sigs array */ + char *rri_name; /* canonical name */ + struct rdatainfo *rri_rdatas; /* individual records */ + struct rdatainfo *rri_sigs; /* individual signatures */ +}; + +#define RRSET_VALIDATED 1 /* DNSSEC AD bit was set in the reply */ + +#define ERRSET_SUCCESS 0 +#define ERRSET_NOMEMORY 1 +#define ERRSET_FAIL 2 +#define ERRSET_INVAL 3 +#define ERRSET_NONAME 4 +#define ERRSET_NODATA 5 + +int getrrsetbyname(const char *, unsigned int, unsigned int, unsigned int, + struct rrsetinfo **); +void freerrset(struct rrsetinfo *); + #ifdef __cplusplus } #endif diff --git a/toolchain/c/sysroot/include/wasm32-wasi/signal.h b/toolchain/c/sysroot/include/wasm32-wasi/signal.h index dde7ce9d07..2d3c09c655 100644 --- a/toolchain/c/sysroot/include/wasm32-wasi/signal.h +++ b/toolchain/c/sysroot/include/wasm32-wasi/signal.h @@ -62,6 +62,80 @@ typedef struct sigaltstack stack_t; #include +#ifndef __wasilibc_unmodified_upstream +/* POSIX.1-2024 sigval/siginfo_t surface. The layout follows + * musl's Linux ABI definition below. AgentOS currently delivers only the + * signal number, but upstream tools may construct siginfo_t before an + * optional sigqueue/pidfd probe that returns ENOSYS. + * https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html */ +#define __NEED_uid_t +#define __NEED_clock_t +#include + +#define SI_QUEUE (-1) +/* Linux reserves 32..64 for real-time signals; AgentOS exposes the same + * 64-signal namespace. https://man7.org/linux/man-pages/man7/signal.7.html */ +#define SIGRTMIN 32 +#define SIGRTMAX 64 + +union sigval { + int sival_int; + void *sival_ptr; +}; + +typedef struct { +#ifdef __SI_SWAP_ERRNO_CODE + int si_signo, si_code, si_errno; +#else + int si_signo, si_errno, si_code; +#endif + union { + char __pad[128 - 2*sizeof(int) - sizeof(long)]; + struct { + union { + struct { + pid_t si_pid; + uid_t si_uid; + } __piduid; + struct { + int si_timerid; + int si_overrun; + } __timer; + } __first; + union { + union sigval si_value; + struct { + int si_status; + clock_t si_utime, si_stime; + } __sigchld; + } __second; + } __si_common; + struct { + void *si_addr; + short si_addr_lsb; + } __sigfault; + struct { + long si_band; + int si_fd; + } __sigpoll; + } __si_fields; +} siginfo_t; +#define si_pid __si_fields.__si_common.__first.__piduid.si_pid +#define si_uid __si_fields.__si_common.__first.__piduid.si_uid +#define si_status __si_fields.__si_common.__second.__sigchld.si_status +#define si_utime __si_fields.__si_common.__second.__sigchld.si_utime +#define si_stime __si_fields.__si_common.__second.__sigchld.si_stime +#define si_value __si_fields.__si_common.__second.si_value +#define si_addr __si_fields.__sigfault.si_addr +#define si_addr_lsb __si_fields.__sigfault.si_addr_lsb +#define si_band __si_fields.__sigpoll.si_band +#define si_fd __si_fields.__sigpoll.si_fd +#define si_timerid __si_fields.__si_common.__first.__timer.si_timerid +#define si_overrun __si_fields.__si_common.__first.__timer.si_overrun +#define si_ptr si_value.sival_ptr +#define si_int si_value.sival_int +#endif + #ifndef __wasilibc_unmodified_upstream struct sigaction { union { @@ -266,6 +340,7 @@ int sigdelset(sigset_t *, int); int sigismember(const sigset_t *, int); int sigaction(int, const struct sigaction *__restrict, struct sigaction *__restrict); int sigprocmask(int, const sigset_t *__restrict, sigset_t *__restrict); +int sigqueue(pid_t, int, union sigval); #ifdef __wasilibc_unmodified_upstream /* WASI has no signal sets */ int sigsuspend(const sigset_t *); diff --git a/toolchain/c/sysroot/include/wasm32-wasi/sys/socket.h b/toolchain/c/sysroot/include/wasm32-wasi/sys/socket.h index b1957d4d5c..1233b7db9b 100644 --- a/toolchain/c/sysroot/include/wasm32-wasi/sys/socket.h +++ b/toolchain/c/sysroot/include/wasm32-wasi/sys/socket.h @@ -400,9 +400,13 @@ struct sockaddr_storage { int socket (int, int, int); -#ifdef __wasilibc_unmodified_upstream /* WASI has no socketpair */ +/* Declared unconditionally for the agentOS sysroot: POSIX socketpair(3p) + * (POSIX.1-2024, XSH). The kernel has no socketpair syscall, so the libc + * definition (std-patches/wasi-libc-overrides/openssh_compat.c) fails with + * ENOSYS; the declaration exists so ported tools such as OpenSSH that + * reference socketpair() on cold paths (e.g. ProxyUseFdpass in sshconnect.c) + * compile and link against the honest ENOSYS stub. */ int socketpair (int, int, int, int [2]); -#endif int shutdown (int, int); diff --git a/toolchain/c/sysroot/include/wasm32-wasi/sys/syscall.h b/toolchain/c/sysroot/include/wasm32-wasi/sys/syscall.h index ceed6b59cb..29249baa46 100644 --- a/toolchain/c/sysroot/include/wasm32-wasi/sys/syscall.h +++ b/toolchain/c/sysroot/include/wasm32-wasi/sys/syscall.h @@ -4,7 +4,10 @@ #ifdef __wasilibc_unmodified_upstream /* WASI has no syscall */ #include #else -/* The generic syscall funtion is not yet implemented. */ +/* Linux syscall numbers are not part of the WASI ABI. The AgentOS libc + * definition returns ENOSYS so upstream feature probes can take their normal + * portable fallback. See syscall(2): https://man7.org/linux/man-pages/man2/syscall.2.html */ +long syscall(long, ...); #endif #endif diff --git a/toolchain/c/sysroot/include/wasm32-wasi/time.h b/toolchain/c/sysroot/include/wasm32-wasi/time.h index 919ae5d21d..53477dd27b 100644 --- a/toolchain/c/sysroot/include/wasm32-wasi/time.h +++ b/toolchain/c/sysroot/include/wasm32-wasi/time.h @@ -119,6 +119,12 @@ struct itimerspec { #define CLOCK_TAI 11 #define TIMER_ABSTIME 1 +#else +/* Linux clock_gettime(2) includes suspended time in CLOCK_BOOTTIME. AgentOS + * VMs have no distinct system-suspend interval, so its owned POSIX clock maps + * to the WASI monotonic clock. + * https://man7.org/linux/man-pages/man2/clock_gettime.2.html */ +#define CLOCK_BOOTTIME CLOCK_MONOTONIC #endif int nanosleep (const struct timespec *, struct timespec *); diff --git a/toolchain/c/sysroot/include/wasm32-wasi/unistd.h b/toolchain/c/sysroot/include/wasm32-wasi/unistd.h index 8b8d692200..5939408916 100644 --- a/toolchain/c/sysroot/include/wasm32-wasi/unistd.h +++ b/toolchain/c/sysroot/include/wasm32-wasi/unistd.h @@ -51,6 +51,13 @@ int pipe2(int [2], int); int pipe(int [2]); int close(int); int posix_close(int, int); +/* closefrom(2) as on Solaris/FreeBSD/OpenBSD (close all fds >= lowfd) and + * glibc >= 2.34. Declared unconditionally in the agentOS sysroot; the + * definition (std-patches/wasi-libc-overrides/openssh_compat.c) is a + * deliberate NO-OP because WASI preopened directory capabilities start at + * fd 3 and closing them would sever all path resolution. See OpenSSH + * ssh.c:main() which calls closefrom(STDERR_FILENO + 1) at startup. */ +void closefrom(int); int dup(int); int dup2(int, int); #ifdef __wasilibc_unmodified_upstream /* WASI has no dup3 */ @@ -154,6 +161,7 @@ pid_t getppid(void); pid_t getpgrp(void); pid_t setsid(void); pid_t getpgid(pid_t); +pid_t getsid(pid_t); #ifdef __wasilibc_unmodified_upstream /* WASI has no full process groups */ pid_t getpgid(pid_t); int setpgid(pid_t, pid_t); @@ -173,12 +181,18 @@ gid_t getgid(void); gid_t getegid(void); int getgroups(int, gid_t []); -#ifdef __wasilibc_unmodified_upstream /* WASI has no setuid etc. */ +/* Declared for the agentOS sysroot. Guest process identity is fixed by the + * kernel (uid/gid come from the host_user import; see + * std-patches/wasi-libc/0005-user-identity.patch), so the definitions in + * std-patches/wasi-libc-overrides/openssh_compat.c implement the + * unprivileged-process subset of POSIX setuid(3p)/setgid(3p): setting an id + * to a value the process already holds succeeds (no-op), anything else fails + * with EPERM. Ported tools (e.g. OpenSSH uidswap.c) call + * setuid(getuid())-style self-assignments to drop privileges they never had. */ int setuid(uid_t); int seteuid(uid_t); int setgid(gid_t); int setegid(gid_t); -#endif char *getlogin(void); int getlogin_r(char *, size_t); @@ -199,10 +213,10 @@ size_t confstr(int, char *, size_t); #define F_LOCK 1 #define F_TLOCK 2 #define F_TEST 3 -#ifdef __wasilibc_unmodified_upstream /* WASI has no setreuid */ +/* See the setuid() comment above: same fixed-identity EPERM semantics + * (POSIX setreuid(3p)); defined in wasi-libc-overrides/openssh_compat.c. */ int setreuid(uid_t, uid_t); int setregid(gid_t, gid_t); -#endif #ifdef __wasilibc_unmodified_upstream /* WASI has no POSIX file locking */ int lockf(int, int, off_t); #endif @@ -259,12 +273,15 @@ extern int optreset; #ifdef _GNU_SOURCE extern char **environ; -#ifdef __wasilibc_unmodified_upstream /* WASI has no get/setresuid */ +/* See the setuid() comment above: setresuid/setresgid (Linux + * setresuid(2)) follow the same fixed-identity semantics (-1 keeps a field, + * self-assignment succeeds, everything else is EPERM); getresuid/getresgid + * report the single kernel identity for all three ids. Defined in + * wasi-libc-overrides/openssh_compat.c. */ int setresuid(uid_t, uid_t, uid_t); int setresgid(gid_t, gid_t, gid_t); int getresuid(uid_t *, uid_t *, uid_t *); int getresgid(gid_t *, gid_t *, gid_t *); -#endif #ifdef __wasilibc_unmodified_upstream /* WASI has no cwd */ char *get_current_dir_name(void); #endif diff --git a/toolchain/c/sysroot/lib/wasm32-wasi/libc.a b/toolchain/c/sysroot/lib/wasm32-wasi/libc.a index aa37ca6c17..858299add7 100644 Binary files a/toolchain/c/sysroot/lib/wasm32-wasi/libc.a and b/toolchain/c/sysroot/lib/wasm32-wasi/libc.a differ diff --git a/toolchain/std-patches/wasi-libc-overrides/fcntl.c b/toolchain/std-patches/wasi-libc-overrides/fcntl.c index ad938ca653..24291ecd6d 100644 --- a/toolchain/std-patches/wasi-libc-overrides/fcntl.c +++ b/toolchain/std-patches/wasi-libc-overrides/fcntl.c @@ -14,6 +14,7 @@ #include #include #include +#include #include /* WASI headers omit F_DUPFD and F_DUPFD_CLOEXEC — define with Linux values */ @@ -171,5 +172,21 @@ int fcntl(int fd, int cmd, ...) { } va_end(ap); - return result; + return result; +} + +/* flock(2) is specified in terms of whole-file advisory locks. Route it to + * the owned fcntl lock surface so callers observe the same VM semantics and + * LOCK_NB selects the nonblocking command. + * https://man7.org/linux/man-pages/man2/flock.2.html */ +int flock(int fd, int operation) { + struct flock lock = { + .l_type = (operation & LOCK_UN) ? F_UNLCK + : (operation & LOCK_EX) ? F_WRLCK + : F_RDLCK, + .l_whence = SEEK_SET, + .l_start = 0, + .l_len = 0, + }; + return fcntl(fd, (operation & LOCK_NB) ? F_SETLK : F_SETLKW, &lock); } diff --git a/toolchain/std-patches/wasi-libc-overrides/openssh_compat.c b/toolchain/std-patches/wasi-libc-overrides/openssh_compat.c new file mode 100644 index 0000000000..73fcbb9908 --- /dev/null +++ b/toolchain/std-patches/wasi-libc-overrides/openssh_compat.c @@ -0,0 +1,373 @@ +/* + * OpenSSH compatibility stubs for the agentOS wasm32-wasip1 sysroot. + * + * Declarations for both functions live in the patched sysroot headers + * (std-patches/wasi-libc/0029-openssh-compat-header-surface.patch). + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * closefrom(2) — Solaris/FreeBSD/OpenBSD extension, glibc >= 2.34: + * "closefrom() closes all open file descriptors greater than or equal to + * lowfd" (closefrom(2), FreeBSD man pages). + * + * This MUST be a no-op here. OpenSSH's ssh.c:main() calls + * closefrom(STDERR_FILENO + 1) unconditionally at startup to drop leaked + * descriptors before doing anything else. On WASI, however, the preopened + * directory capability handles handed to the module by the runtime start at + * fd 3 (see WASI preview1 `preopens`; wasi-libc + * libc-bottom-half/sources/preopens.c populates its preopen table from + * fd_prestat_get on fds >= 3). Closing fds 3+ would discard every preopen + * and with them all path resolution — every subsequent open() fails with + * ENOTCAPABLE. A faithful "close everything" implementation is therefore + * strictly worse than doing nothing: guest processes in this runtime are + * spawned with exactly the descriptors the kernel intends them to have, so + * there are no inherited stray fds to drop and the security intent of the + * call is already satisfied by construction. + * + * Providing the symbol also makes OpenSSH's configure AC_CHECK_FUNCS link + * probe succeed (HAVE_CLOSEFROM), which keeps its openbsd-compat + * bsd-closefrom.c replacement — a close() loop up to sysconf(_SC_OPEN_MAX) + * that would destroy the preopens — out of the build entirely. + */ +void closefrom(int lowfd) { + (void)lowfd; +} + +/* + * socketpair(3p) — POSIX.1-2024: "the socketpair() function shall create an + * unnamed pair of connected sockets". The agentOS kernel has no socketpair + * syscall (no AF_UNIX datagram plumbing between two anonymous fds), so this + * honestly fails with ENOSYS, which POSIX permits via the EOPNOTSUPP/EAFNOSUPPORT + * family for unsupported domains; ENOSYS names the real condition ("the + * function is not supported by this implementation"). + * + * The stub exists so ported tools link: OpenSSH references socketpair() from + * cold paths the batch client never takes — sshconnect.c + * ssh_proxy_fdpass_connect() (ProxyCommand/ProxyUseFdpass, RFC 4251 §4.4 + * "proxies and gateways" territory) and channels/mux code. If one of those + * paths is ever reached, the caller gets a real errno instead of a link + * failure or a silent fake socket. + */ +int socketpair(int domain, int type, int protocol, int sv[2]) { + (void)domain; + (void)type; + (void)protocol; + (void)sv; + errno = ENOSYS; + return -1; +} + +/* + * ppoll(2) — Linux/ppoll(3p, POSIX.1-2024): poll with a struct timespec + * timeout and an atomically-applied signal mask. The patched sysroot routes + * poll() through the host_net.net_poll import + * (std-patches/wasi-libc/0023-host-net-read-write-sockets.patch), which + * understands host-net sockets, kernel pipes/PTYs, and the runtime's + * high-numbered dup'd fds. Implement ppoll as a wrapper over that poll so + * link probes (AC_CHECK_FUNCS(ppoll)) succeed and ported tools skip their + * own replacements — OpenSSH's openbsd-compat/bsd-poll.c fallback is built + * on pselect() and fails with EINVAL for any fd >= FD_SETSIZE, which the + * runtime's virtual dup fds (>= 0x100000, see the fd_dup_min host import) + * always are. + * + * The sigmask parameter is deliberately ignored: this runtime has no + * asynchronous signal delivery to race against — signals are cooperative + * and dispatched at poll boundaries inside net_poll (see + * std-patches/wasi-libc/0011-sigaction.patch), so the atomic + * "swap-mask-and-wait" that ppoll(2) exists to provide has no observable + * effect here. + */ +int ppoll(struct pollfd *fds, nfds_t nfds, const struct timespec *timeout, + const sigset_t *sigmask) { + int timeout_ms; + + (void)sigmask; + if (timeout == NULL) { + timeout_ms = -1; + } else if (timeout->tv_sec < 0 || timeout->tv_nsec < 0 || + timeout->tv_nsec > 999999999L) { + errno = EINVAL; + return -1; + } else if (timeout->tv_sec > (time_t)(INT_MAX / 1000 - 1)) { + timeout_ms = INT_MAX; + } else { + timeout_ms = (int)(timeout->tv_sec * 1000 + + (timeout->tv_nsec + 999999L) / 1000000L); + } + return poll(fds, nfds, timeout_ms); +} + +/* + * set*id family — unprivileged-process subset of POSIX setuid(3p) / + * setgid(3p) / setreuid(3p) and Linux setresuid(2). Guest process identity is + * fixed by the kernel (getuid()/getgid() come from the host_user import, see + * std-patches/wasi-libc/0005-user-identity.patch); there is no privilege to + * raise or drop. POSIX: an unprivileged process may set its ids only to + * values it already holds (real, effective, or saved — all identical here), + * and "the setuid() function shall fail [EPERM] ... if uid does not match the + * real user ID or the saved set-user-ID". So self-assignment succeeds as a + * no-op and any other target fails with EPERM. OpenSSH's uidswap.c relies on + * exactly the self-assignment case (setuid(getuid()) etc.) to guarantee it + * holds no elevated privileges. + */ +static int set_fixed_id(unsigned int requested, unsigned int current) { + if (requested == current) + return 0; + errno = EPERM; + return -1; +} + +/* (uid_t)-1 / (gid_t)-1 means "leave unchanged" per setreuid(3p)/setresuid(2). */ +static int set_fixed_id_opt(unsigned int requested, unsigned int current) { + if (requested == (unsigned int)-1) + return 0; + return set_fixed_id(requested, current); +} + +int setuid(uid_t uid) { + return set_fixed_id(uid, getuid()); +} + +int seteuid(uid_t euid) { + return set_fixed_id(euid, geteuid()); +} + +int setgid(gid_t gid) { + return set_fixed_id(gid, getgid()); +} + +int setegid(gid_t egid) { + return set_fixed_id(egid, getegid()); +} + +int setreuid(uid_t ruid, uid_t euid) { + if (set_fixed_id_opt(ruid, getuid()) < 0) + return -1; + return set_fixed_id_opt(euid, geteuid()); +} + +int setregid(gid_t rgid, gid_t egid) { + if (set_fixed_id_opt(rgid, getgid()) < 0) + return -1; + return set_fixed_id_opt(egid, getegid()); +} + +int setresuid(uid_t ruid, uid_t euid, uid_t suid) { + if (set_fixed_id_opt(ruid, getuid()) < 0) + return -1; + if (set_fixed_id_opt(euid, geteuid()) < 0) + return -1; + return set_fixed_id_opt(suid, getuid()); +} + +int setresgid(gid_t rgid, gid_t egid, gid_t sgid) { + if (set_fixed_id_opt(rgid, getgid()) < 0) + return -1; + if (set_fixed_id_opt(egid, getegid()) < 0) + return -1; + return set_fixed_id_opt(sgid, getgid()); +} + +int getresuid(uid_t *ruid, uid_t *euid, uid_t *suid) { + if (ruid) + *ruid = getuid(); + if (euid) + *euid = geteuid(); + if (suid) + *suid = getuid(); + return 0; +} + +int getresgid(gid_t *rgid, gid_t *egid, gid_t *sgid) { + if (rgid) + *rgid = getgid(); + if (egid) + *egid = getegid(); + if (sgid) + *sgid = getgid(); + return 0; +} + +/* + * getgrent(3p) / setgrent(3p) / endgrent(3p) — POSIX group database + * enumeration. The agentOS runtime resolves groups by id/name through host + * imports (std-patches/wasi-libc/0025-group-lookup-compat.patch provides + * getgrgid/getgrnam); there is no enumerable /etc/group database, so + * enumeration reports immediate end-of-database: POSIX getgrent(3p) "shall + * return a null pointer ... on end-of-file". setgrent/endgrent rewind/close + * that (empty) enumeration and are no-ops. Having real symbols keeps ported + * tools off their compat macro replacements (OpenSSH's bsd-misc.h + * `#define endgrent() do { } while(0)` collides with the grp.h prototype). + */ +struct group *getgrent(void) { + return NULL; +} + +void setgrent(void) { +} + +void endgrent(void) { +} + +/* + * setgroups(2) / initgroups(3) — supplementary group management. The guest's + * supplementary group set is fixed by the kernel (getgroups() reports it via + * the host import from + * std-patches/wasi-libc/0017-resource-limits-and-groups.patch), and POSIX + * reserves changing it to privileged processes ("appropriate privileges", + * setgroups(2): "EPERM The calling process ... does not have the CAP_SETGID + * capability"). Requests that restate the current single-group set succeed + * as no-ops; anything else fails with EPERM. initgroups(3) is glibc/BSD + * "read /etc/group and call setgroups" — with the enumerable group database + * empty (see getgrent above) its computed set is exactly {basegid}. OpenSSH + * misc.c subprocess() only calls initgroups when geteuid() == 0. + */ +int setgroups(size_t size, const gid_t *list) { + if (size == 0 || (size == 1 && list != NULL && list[0] == getgid())) + return 0; + errno = EPERM; + return -1; +} + +int initgroups(const char *user, gid_t group) { + gid_t set[1]; + + (void)user; + set[0] = group; + return setgroups(1, set); +} + +/* + * getnameinfo(3p) — POSIX.1-2024 / RFC 3493 §6.2 ("Socket Address Structure + * to Node Name and Service Name"). wasi-libc's (un-omitted by the + * sockets patch) declares getnameinfo but the sysroot never defined it; the + * host_net-backed host_socket.o only implements getaddrinfo. This is the + * numeric subset: it formats addresses with inet_ntop and ports with + * snprintf. Reverse DNS is not available in the runtime (there is no + * host-side reverse-lookup import), so NI_NAMEREQD fails with EAI_NONAME — + * exactly what POSIX prescribes when "the name of the host cannot be + * located" — and name-preferred lookups degrade to the numeric form, the + * same observable behavior as a Linux host with no working reverse DNS. + * OpenSSH uses getnameinfo() pervasively (canonical host strings for + * known_hosts, log messages) with NI_NUMERICHOST/NI_NUMERICSERV on the paths + * the batch client exercises. + */ +int getnameinfo(const struct sockaddr *restrict sa, socklen_t salen, + char *restrict host, socklen_t hostlen, + char *restrict serv, socklen_t servlen, int flags) { + char buf[INET6_ADDRSTRLEN]; + unsigned short port; + + if (sa == NULL) + return EAI_FAIL; + + switch (sa->sa_family) { + case AF_INET: { + const struct sockaddr_in *sin = (const struct sockaddr_in *)sa; + if (salen < (socklen_t)sizeof(*sin)) + return EAI_FAMILY; + if (host != NULL && hostlen > 0) { + if (flags & NI_NAMEREQD) + return EAI_NONAME; + if (inet_ntop(AF_INET, &sin->sin_addr, buf, + sizeof(buf)) == NULL) + return EAI_FAIL; + if (strlen(buf) >= (size_t)hostlen) + return EAI_OVERFLOW; + memcpy(host, buf, strlen(buf) + 1); + } + port = ntohs(sin->sin_port); + break; + } + case AF_INET6: { + const struct sockaddr_in6 *sin6 = + (const struct sockaddr_in6 *)sa; + if (salen < (socklen_t)sizeof(*sin6)) + return EAI_FAMILY; + if (host != NULL && hostlen > 0) { + if (flags & NI_NAMEREQD) + return EAI_NONAME; + if (inet_ntop(AF_INET6, &sin6->sin6_addr, buf, + sizeof(buf)) == NULL) + return EAI_FAIL; + if (strlen(buf) >= (size_t)hostlen) + return EAI_OVERFLOW; + memcpy(host, buf, strlen(buf) + 1); + } + port = ntohs(sin6->sin6_port); + break; + } + default: + return EAI_FAMILY; + } + + if (serv != NULL && servlen > 0) { + /* No services database in the VM; numeric only (as if + * NI_NUMERICSERV were always set). */ + char portbuf[8]; + int n = snprintf(portbuf, sizeof(portbuf), "%u", + (unsigned int)port); + if (n < 0 || n >= (int)servlen) + return EAI_OVERFLOW; + memcpy(serv, portbuf, (size_t)n + 1); + } + + return 0; +} + +/* + * getrrsetbyname(3) / freerrset(3) — OpenBSD DNS RRset query API, declared in + * the patched . OpenSSH's dns.c uses it to fetch SSHFP records for + * VerifyHostKeyDNS (RFC 4255 "Using DNS to Securely Publish Secure Shell + * (SSH) Key Fingerprints"). This sysroot has no resolver library (res_query + * and friends are not built; DNS goes through the host getaddrinfo import), + * so raw RRset queries are unsupported: fail with ERRSET_FAIL ("general + * failure", getrrsetbyname(3) RETURN VALUES). dns.c maps that to its normal + * "DNS lookup error" diagnostic and host-key verification proceeds via + * known_hosts, exactly like a host whose resolver cannot reach a nameserver. + */ +int getrrsetbyname(const char *hostname, unsigned int rdclass, + unsigned int rdtype, unsigned int flags, struct rrsetinfo **res) { + (void)hostname; + (void)rdclass; + (void)rdtype; + (void)flags; + if (res) + *res = NULL; + return ERRSET_FAIL; +} + +void freerrset(struct rrsetinfo *rrset) { + unsigned int i; + + if (rrset == NULL) + return; + /* Mirrors OpenBSD lib/libc/net/getrrsetbyname.c freerrset(). */ + if (rrset->rri_rdatas) { + for (i = 0; i < rrset->rri_nrdatas; i++) + free(rrset->rri_rdatas[i].rdi_data); + free(rrset->rri_rdatas); + } + if (rrset->rri_sigs) { + for (i = 0; i < rrset->rri_nsigs; i++) + free(rrset->rri_sigs[i].rdi_data); + free(rrset->rri_sigs); + } + free(rrset->rri_name); + free(rrset); +} diff --git a/toolchain/std-patches/wasi-libc-overrides/proc_compat.c b/toolchain/std-patches/wasi-libc-overrides/proc_compat.c new file mode 100644 index 0000000000..44545f0b9a --- /dev/null +++ b/toolchain/std-patches/wasi-libc-overrides/proc_compat.c @@ -0,0 +1,111 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Linux syscall(2) numbers have no meaning in a wasm32-wasip1 VM. Returning + * ENOSYS is the documented contract for an unimplemented syscall and lets + * procps-ng's pidfd path fall back to the host-backed kill(2) implementation. + * https://man7.org/linux/man-pages/man2/syscall.2.html */ +long syscall(long number, ...) { + (void)number; + errno = ENOSYS; + return -1; +} + +/* getsid(2) is declared so portable consumers compile. Session selection is + * optional in procps-ng; until the host_process ABI exposes session lookup, + * fail explicitly instead of inventing an id. + * https://man7.org/linux/man-pages/man2/getsid.2.html */ +pid_t getsid(pid_t pid) { + (void)pid; + errno = ENOSYS; + return -1; +} + +/* AgentOS' cooperative signal ABI currently transports a signal number, not + * POSIX queued payload metadata. Report that limit to callers exactly as + * sigqueue(3) permits for an unsupported operation. + * https://man7.org/linux/man-pages/man3/sigqueue.3.html */ +int sigqueue(pid_t pid, int sig, union sigval value) { + (void)pid; + (void)sig; + (void)value; + errno = ENOSYS; + return -1; +} + +/* VMs do not maintain a host-login accounting database. getutent(3)'s normal + * end-of-file result therefore represents the complete empty guest database; + * set/end remain valid cursor lifecycle calls. + * https://man7.org/linux/man-pages/man3/getutent.3.html */ +void setutent(void) { +} + +struct utmp *getutent(void) { + return NULL; +} + +void endutent(void) { +} + +/* BSD err(3) is a libc compatibility API used by procps-ng. Keep diagnostics + * on stderr and preserve errno only for the warn/err variants. + * https://man7.org/linux/man-pages/man3/err.3.html */ +void vwarn(const char *format, va_list args) { + fprintf(stderr, "%s: ", program_invocation_short_name); + if (format != NULL) { + vfprintf(stderr, format, args); + fputs(": ", stderr); + } + perror(NULL); +} + +void vwarnx(const char *format, va_list args) { + fprintf(stderr, "%s: ", program_invocation_short_name); + if (format != NULL) + vfprintf(stderr, format, args); + fputc('\n', stderr); +} + +void warn(const char *format, ...) { + va_list args; + va_start(args, format); + vwarn(format, args); + va_end(args); +} + +void warnx(const char *format, ...) { + va_list args; + va_start(args, format); + vwarnx(format, args); + va_end(args); +} + +_Noreturn void verr(int status, const char *format, va_list args) { + vwarn(format, args); + exit(status); +} + +_Noreturn void verrx(int status, const char *format, va_list args) { + vwarnx(format, args); + exit(status); +} + +_Noreturn void err(int status, const char *format, ...) { + va_list args; + va_start(args, format); + verr(status, format, args); +} + +_Noreturn void errx(int status, const char *format, ...) { + va_list args; + va_start(args, format); + verrx(status, format, args); +} diff --git a/toolchain/std-patches/wasi-libc-overrides/termcap_compat.c b/toolchain/std-patches/wasi-libc-overrides/termcap_compat.c new file mode 100644 index 0000000000..c9862148f9 --- /dev/null +++ b/toolchain/std-patches/wasi-libc-overrides/termcap_compat.c @@ -0,0 +1,44 @@ +#include +#include +#include +#include + +/* AgentOS VMs do not project a terminfo/termcap database. These APIs report + * capability absence, which makes consumers use their documented ASCII path; + * tputs still preserves explicitly supplied control strings. + * https://man7.org/linux/man-pages/man5/termcap.5.html */ +int setupterm(const char *term, int filedes, int *errret) { + (void)term; + (void)filedes; + if (errret != NULL) + *errret = -1; + return ERR; +} + +char *tigetstr(const char *capname) { + (void)capname; + return NULL; +} + +int tgetent(char *buffer, const char *termtype) { + (void)buffer; + (void)termtype; + return 0; +} + +char *tgetstr(const char *id, char **area) { + (void)id; + (void)area; + return NULL; +} + +int tputs(const char *str, int affcnt, int (*putc_fn)(int)) { + (void)affcnt; + if (str == NULL || putc_fn == NULL) + return ERR; + for (; *str != '\0'; str++) { + if (putc_fn((unsigned char)*str) == EOF) + return ERR; + } + return OK; +} diff --git a/toolchain/std-patches/wasi-libc/0012-posix-spawn-cwd.patch b/toolchain/std-patches/wasi-libc/0012-posix-spawn-cwd.patch index bce70048b7..06c430628e 100644 --- a/toolchain/std-patches/wasi-libc/0012-posix-spawn-cwd.patch +++ b/toolchain/std-patches/wasi-libc/0012-posix-spawn-cwd.patch @@ -26,7 +26,7 @@ uint32_t stdin_fd = 0, stdout_fd = 1, stderr_fd = 2; if (fa && fa->__actions) { for (struct __fdop *op = fa->__actions; op; op = op->next) { -@@ -259,16 +271,27 @@ +@@ -259,16 +271,30 @@ else close(opened); break; } @@ -37,13 +37,16 @@ } } -+ // Resolve cwd: explicit chdir action > inherited PWD > getcwd() > empty ++ // Resolve cwd: explicit chdir action > current cwd > inherited PWD > ++ // empty. getcwd() lazily initializes from PWD (see getcwd.c), so it is ++ // always at least as fresh as the environment and, unlike PWD, tracks ++ // in-process chdir() calls (e.g. `git -C ` spawning helpers). + char cwd_buf[1024]; + const char *cwd_str = spawn_cwd; -+ if (!cwd_str) cwd_str = find_pwd_in_env(env); + if (!cwd_str && getcwd(cwd_buf, sizeof(cwd_buf))) { + cwd_str = cwd_buf; + } ++ if (!cwd_str) cwd_str = find_pwd_in_env(env); + uint32_t child_pid; uint32_t err = __host_proc_spawn( diff --git a/toolchain/std-patches/wasi-libc/0029-openssh-compat-header-surface.patch b/toolchain/std-patches/wasi-libc/0029-openssh-compat-header-surface.patch new file mode 100644 index 0000000000..5b1b763f4a --- /dev/null +++ b/toolchain/std-patches/wasi-libc/0029-openssh-compat-header-surface.patch @@ -0,0 +1,196 @@ +diff --git a/libc-bottom-half/headers/public/__errno_values.h b/libc-bottom-half/headers/public/__errno_values.h +index 6d6d412..df0a65d 100644 +--- a/libc-bottom-half/headers/public/__errno_values.h ++++ b/libc-bottom-half/headers/public/__errno_values.h +@@ -83,4 +83,12 @@ + #define EOPNOTSUPP ENOTSUP + #define EWOULDBLOCK EAGAIN + ++/* Linux-compat errno constant for ported tools (agentOS sysroot). errno(3): ++ * "EPFNOSUPPORT 96 Protocol family not supported". No WASI errno maps to it ++ * and the kernel never raises it; it exists so upstream code that tests for ++ * it (e.g. OpenSSH openbsd-compat/bindresvport.c) compiles. It gets a ++ * distinct value (Linux's 96, above the WASI errno range) rather than an ++ * alias of EAFNOSUPPORT so duplicate switch-case labels stay legal. */ ++#define EPFNOSUPPORT 96 ++ + #endif +diff --git a/libc-bottom-half/headers/public/__header_sys_socket.h b/libc-bottom-half/headers/public/__header_sys_socket.h +index 3965794..e4cea06 100644 +--- a/libc-bottom-half/headers/public/__header_sys_socket.h ++++ b/libc-bottom-half/headers/public/__header_sys_socket.h +@@ -143,6 +143,14 @@ + #define AF_INET PF_INET + #define AF_INET6 PF_INET6 + #define AF_UNIX 3 ++/* POSIX spells the constants AF_*; the PF_* names are the ++ * historical BSD "protocol family" aliases (4.4BSD; on Linux and musl ++ * PF_UNIX == AF_UNIX, PF_LOCAL/AF_LOCAL are additional POSIX.1-2008 spellings ++ * of the same family). Ported tools still use them (e.g. OpenSSH misc.c's ++ * unix_listener uses PF_UNIX), so alias them to the AF_* values above. */ ++#define AF_LOCAL AF_UNIX ++#define PF_UNIX AF_UNIX ++#define PF_LOCAL AF_UNIX + + #ifdef __cplusplus + extern "C" { +diff --git a/libc-top-half/musl/include/netdb.h b/libc-top-half/musl/include/netdb.h +index eecb482..bc35d70 100644 +--- a/libc-top-half/musl/include/netdb.h ++++ b/libc-top-half/musl/include/netdb.h +@@ -160,6 +160,44 @@ int getservbyname_r(const char *, const char *, struct servent *, char *, size_t + #endif + + ++ ++/* OpenBSD getrrsetbyname(3) surface (also on FreeBSD/NetBSD libc): DNS RRset ++ * query API used by OpenSSH's dns.c for SSHFP host-key verification ++ * (VerifyHostKeyDNS, RFC 4255). Struct layout and constants match OpenBSD ++ * . The agentOS sysroot has no DNS resolver library (lookups route ++ * through the host getaddrinfo import), so the libc definition ++ * (std-patches/wasi-libc-overrides/openssh_compat.c) always fails with ++ * ERRSET_FAIL and callers degrade to a normal "DNS lookup error" path. */ ++struct rdatainfo { ++ unsigned int rdi_length; /* length of data */ ++ unsigned char *rdi_data; /* record data */ ++}; ++ ++struct rrsetinfo { ++ unsigned int rri_flags; /* RRSET_VALIDATED... */ ++ unsigned int rri_rdclass; /* class number */ ++ unsigned int rri_rdtype; /* RR type number */ ++ unsigned int rri_ttl; /* time to live */ ++ unsigned int rri_nrdatas; /* size of rdatas array */ ++ unsigned int rri_nsigs; /* size of sigs array */ ++ char *rri_name; /* canonical name */ ++ struct rdatainfo *rri_rdatas; /* individual records */ ++ struct rdatainfo *rri_sigs; /* individual signatures */ ++}; ++ ++#define RRSET_VALIDATED 1 /* DNSSEC AD bit was set in the reply */ ++ ++#define ERRSET_SUCCESS 0 ++#define ERRSET_NOMEMORY 1 ++#define ERRSET_FAIL 2 ++#define ERRSET_INVAL 3 ++#define ERRSET_NONAME 4 ++#define ERRSET_NODATA 5 ++ ++int getrrsetbyname(const char *, unsigned int, unsigned int, unsigned int, ++ struct rrsetinfo **); ++void freerrset(struct rrsetinfo *); ++ + #ifdef __cplusplus + } + #endif +diff --git a/libc-top-half/musl/include/sys/socket.h b/libc-top-half/musl/include/sys/socket.h +index b1957d4..1233b7d 100644 +--- a/libc-top-half/musl/include/sys/socket.h ++++ b/libc-top-half/musl/include/sys/socket.h +@@ -400,9 +400,13 @@ struct sockaddr_storage { + + int socket (int, int, int); + +-#ifdef __wasilibc_unmodified_upstream /* WASI has no socketpair */ ++/* Declared unconditionally for the agentOS sysroot: POSIX socketpair(3p) ++ * (POSIX.1-2024, XSH). The kernel has no socketpair syscall, so the libc ++ * definition (std-patches/wasi-libc-overrides/openssh_compat.c) fails with ++ * ENOSYS; the declaration exists so ported tools such as OpenSSH that ++ * reference socketpair() on cold paths (e.g. ProxyUseFdpass in sshconnect.c) ++ * compile and link against the honest ENOSYS stub. */ + int socketpair (int, int, int, int [2]); +-#endif + + int shutdown (int, int); + +diff --git a/libc-top-half/musl/include/unistd.h b/libc-top-half/musl/include/unistd.h +index 8b8d692..490d57c 100644 +--- a/libc-top-half/musl/include/unistd.h ++++ b/libc-top-half/musl/include/unistd.h +@@ -51,6 +51,13 @@ int pipe2(int [2], int); + int pipe(int [2]); + int close(int); + int posix_close(int, int); ++/* closefrom(2) as on Solaris/FreeBSD/OpenBSD (close all fds >= lowfd) and ++ * glibc >= 2.34. Declared unconditionally in the agentOS sysroot; the ++ * definition (std-patches/wasi-libc-overrides/openssh_compat.c) is a ++ * deliberate NO-OP because WASI preopened directory capabilities start at ++ * fd 3 and closing them would sever all path resolution. See OpenSSH ++ * ssh.c:main() which calls closefrom(STDERR_FILENO + 1) at startup. */ ++void closefrom(int); + int dup(int); + int dup2(int, int); + #ifdef __wasilibc_unmodified_upstream /* WASI has no dup3 */ +@@ -173,12 +180,18 @@ gid_t getgid(void); + gid_t getegid(void); + int getgroups(int, gid_t []); + +-#ifdef __wasilibc_unmodified_upstream /* WASI has no setuid etc. */ ++/* Declared for the agentOS sysroot. Guest process identity is fixed by the ++ * kernel (uid/gid come from the host_user import; see ++ * std-patches/wasi-libc/0005-user-identity.patch), so the definitions in ++ * std-patches/wasi-libc-overrides/openssh_compat.c implement the ++ * unprivileged-process subset of POSIX setuid(3p)/setgid(3p): setting an id ++ * to a value the process already holds succeeds (no-op), anything else fails ++ * with EPERM. Ported tools (e.g. OpenSSH uidswap.c) call ++ * setuid(getuid())-style self-assignments to drop privileges they never had. */ + int setuid(uid_t); + int seteuid(uid_t); + int setgid(gid_t); + int setegid(gid_t); +-#endif + + char *getlogin(void); + int getlogin_r(char *, size_t); +@@ -199,10 +212,10 @@ size_t confstr(int, char *, size_t); + #define F_LOCK 1 + #define F_TLOCK 2 + #define F_TEST 3 +-#ifdef __wasilibc_unmodified_upstream /* WASI has no setreuid */ ++/* See the setuid() comment above: same fixed-identity EPERM semantics ++ * (POSIX setreuid(3p)); defined in wasi-libc-overrides/openssh_compat.c. */ + int setreuid(uid_t, uid_t); + int setregid(gid_t, gid_t); +-#endif + #ifdef __wasilibc_unmodified_upstream /* WASI has no POSIX file locking */ + int lockf(int, int, off_t); + #endif +@@ -259,12 +272,15 @@ extern int optreset; + + #ifdef _GNU_SOURCE + extern char **environ; +-#ifdef __wasilibc_unmodified_upstream /* WASI has no get/setresuid */ ++/* See the setuid() comment above: setresuid/setresgid (Linux ++ * setresuid(2)) follow the same fixed-identity semantics (-1 keeps a field, ++ * self-assignment succeeds, everything else is EPERM); getresuid/getresgid ++ * report the single kernel identity for all three ids. Defined in ++ * wasi-libc-overrides/openssh_compat.c. */ + int setresuid(uid_t, uid_t, uid_t); + int setresgid(gid_t, gid_t, gid_t); + int getresuid(uid_t *, uid_t *, uid_t *); + int getresgid(gid_t *, gid_t *, gid_t *); +-#endif + #ifdef __wasilibc_unmodified_upstream /* WASI has no cwd */ + char *get_current_dir_name(void); + #endif +diff --git a/scripts/install-include-headers.sh b/scripts/install-include-headers.sh +index 8813360..9a740ad 100755 +--- a/scripts/install-include-headers.sh ++++ b/scripts/install-include-headers.sh +@@ -63,11 +63,15 @@ MUSL_OMIT_HEADERS+=("sys/procfs.h" "sys/user.h" "sys/kd.h" "sys/vt.h" \ + "sys/fsuid.h" "sys/io.h" "sys/prctl.h" "sys/mtio.h" "sys/mount.h" \ + "sys/fanotify.h" "sys/personality.h" "elf.h" "link.h" "bits/link.h" \ + "scsi/scsi.h" "scsi/scsi_ioctl.h" "scsi/sg.h" "sys/auxv.h" \ +- "shadow.h" "mntent.h" "resolv.h" "pty.h" "ulimit.h" "sys/xattr.h" \ ++ "shadow.h" "mntent.h" "pty.h" "ulimit.h" "sys/xattr.h" \ + "wordexp.h" "sys/membarrier.h" "sys/signalfd.h" \ + "net/if.h" "net/if_arp.h" \ + "net/ethernet.h" "net/route.h" "netinet/if_ether.h" "netinet/ether.h" \ + "sys/timerfd.h" "libintl.h" "sys/sysmacros.h" "aio.h") ++# NOTE: resolv.h un-omitted by the agentOS 0029-openssh-compat patch — musl's ++# resolv.h is declaration-only and ported tools (OpenSSH sshkey.c, ++# sshbuf-misc.c) include unconditionally. No resolver code is ++# built; linking res_* still fails, which is the honest surface. + # Exclude `netdb.h` from all of the p1 targets. + # NOTE: commented out by the secure-exec 0008-sockets patch — we provide getaddrinfo via host_net + #if [[ $TARGET_TRIPLE == *"wasi" || $TARGET_TRIPLE == *"wasi-threads" || \ diff --git a/toolchain/std-patches/wasi-libc/0030-proc-consumer-header-surface.patch b/toolchain/std-patches/wasi-libc/0030-proc-consumer-header-surface.patch new file mode 100644 index 0000000000..bf6f886a1a --- /dev/null +++ b/toolchain/std-patches/wasi-libc/0030-proc-consumer-header-surface.patch @@ -0,0 +1,173 @@ +Expose the POSIX metadata used by procps-ng in the AgentOS-owned sysroot. + +procps-ng uses Linux pidfds opportunistically, then falls back to kill(2) when +the generic syscall returns ENOSYS. Keep that probe and the associated signal +types in unmodified upstream code. AgentOS VMs have no suspend interval, so +CLOCK_BOOTTIME is the same monotonic clock as CLOCK_MONOTONIC. + +--- a/libc-top-half/musl/include/fcntl.h ++++ b/libc-top-half/musl/include/fcntl.h +@@ -51,6 +51,13 @@ + #define O_TEXT 0 + #endif + ++#ifndef O_PATH ++/* WASI O_SEARCH is the descriptor-only path capability corresponding to ++ * Linux O_PATH for directory-relative lookups. ++ * https://man7.org/linux/man-pages/man2/open.2.html */ ++#define O_PATH O_SEARCH ++#endif ++ + #ifndef F_RDLCK + #define F_RDLCK 0 + #endif +--- a/libc-top-half/musl/include/signal.h ++++ b/libc-top-half/musl/include/signal.h +@@ -63,6 +63,80 @@ + #include + + #ifndef __wasilibc_unmodified_upstream ++/* POSIX.1-2024 sigval/siginfo_t surface. The layout follows ++ * musl's Linux ABI definition below. AgentOS currently delivers only the ++ * signal number, but upstream tools may construct siginfo_t before an ++ * optional sigqueue/pidfd probe that returns ENOSYS. ++ * https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/signal.h.html */ ++#define __NEED_uid_t ++#define __NEED_clock_t ++#include ++ ++#define SI_QUEUE (-1) ++/* Linux reserves 32..64 for real-time signals; AgentOS exposes the same ++ * 64-signal namespace. https://man7.org/linux/man-pages/man7/signal.7.html */ ++#define SIGRTMIN 32 ++#define SIGRTMAX 64 ++ ++union sigval { ++ int sival_int; ++ void *sival_ptr; ++}; ++ ++typedef struct { ++#ifdef __SI_SWAP_ERRNO_CODE ++ int si_signo, si_code, si_errno; ++#else ++ int si_signo, si_errno, si_code; ++#endif ++ union { ++ char __pad[128 - 2*sizeof(int) - sizeof(long)]; ++ struct { ++ union { ++ struct { ++ pid_t si_pid; ++ uid_t si_uid; ++ } __piduid; ++ struct { ++ int si_timerid; ++ int si_overrun; ++ } __timer; ++ } __first; ++ union { ++ union sigval si_value; ++ struct { ++ int si_status; ++ clock_t si_utime, si_stime; ++ } __sigchld; ++ } __second; ++ } __si_common; ++ struct { ++ void *si_addr; ++ short si_addr_lsb; ++ } __sigfault; ++ struct { ++ long si_band; ++ int si_fd; ++ } __sigpoll; ++ } __si_fields; ++} siginfo_t; ++#define si_pid __si_fields.__si_common.__first.__piduid.si_pid ++#define si_uid __si_fields.__si_common.__first.__piduid.si_uid ++#define si_status __si_fields.__si_common.__second.__sigchld.si_status ++#define si_utime __si_fields.__si_common.__second.__sigchld.si_utime ++#define si_stime __si_fields.__si_common.__second.__sigchld.si_stime ++#define si_value __si_fields.__si_common.__second.si_value ++#define si_addr __si_fields.__sigfault.si_addr ++#define si_addr_lsb __si_fields.__sigfault.si_addr_lsb ++#define si_band __si_fields.__sigpoll.si_band ++#define si_fd __si_fields.__sigpoll.si_fd ++#define si_timerid __si_fields.__si_common.__first.__timer.si_timerid ++#define si_overrun __si_fields.__si_common.__first.__timer.si_overrun ++#define si_ptr si_value.sival_ptr ++#define si_int si_value.sival_int ++#endif ++ ++#ifndef __wasilibc_unmodified_upstream + struct sigaction { + union { + void (*sa_handler)(int); +@@ -266,6 +336,7 @@ + int sigismember(const sigset_t *, int); + int sigaction(int, const struct sigaction *__restrict, struct sigaction *__restrict); + int sigprocmask(int, const sigset_t *__restrict, sigset_t *__restrict); ++int sigqueue(pid_t, int, union sigval); + + #ifdef __wasilibc_unmodified_upstream /* WASI has no signal sets */ + int sigsuspend(const sigset_t *); +--- a/libc-top-half/musl/include/sys/syscall.h ++++ b/libc-top-half/musl/include/sys/syscall.h +@@ -4,7 +4,10 @@ + #ifdef __wasilibc_unmodified_upstream /* WASI has no syscall */ + #include + #else +-/* The generic syscall funtion is not yet implemented. */ ++/* Linux syscall numbers are not part of the WASI ABI. The AgentOS libc ++ * definition returns ENOSYS so upstream feature probes can take their normal ++ * portable fallback. See syscall(2): https://man7.org/linux/man-pages/man2/syscall.2.html */ ++long syscall(long, ...); + #endif + + #endif +--- a/libc-top-half/musl/include/time.h ++++ b/libc-top-half/musl/include/time.h +@@ -119,6 +119,12 @@ + #define CLOCK_TAI 11 + + #define TIMER_ABSTIME 1 ++#else ++/* Linux clock_gettime(2) includes suspended time in CLOCK_BOOTTIME. AgentOS ++ * VMs have no distinct system-suspend interval, so its owned POSIX clock maps ++ * to the WASI monotonic clock. ++ * https://man7.org/linux/man-pages/man2/clock_gettime.2.html */ ++#define CLOCK_BOOTTIME CLOCK_MONOTONIC + #endif + + int nanosleep (const struct timespec *, struct timespec *); +--- a/libc-top-half/musl/include/unistd.h ++++ b/libc-top-half/musl/include/unistd.h +@@ -161,6 +161,7 @@ + pid_t getpgrp(void); + pid_t setsid(void); + pid_t getpgid(pid_t); ++pid_t getsid(pid_t); + #ifdef __wasilibc_unmodified_upstream /* WASI has no full process groups */ + pid_t getpgid(pid_t); + int setpgid(pid_t, pid_t); +--- a/scripts/install-include-headers.sh ++++ b/scripts/install-include-headers.sh +@@ -57,7 +57,7 @@ + "sys/ptrace.h" "sys/statfs.h" "bits/kd.h" "bits/vt.h" "bits/soundcard.h" \ + "bits/sem.h" "bits/shm.h" "bits/msg.h" "bits/ipc.h" "bits/ptrace.h" \ + "bits/statfs.h" "sys/vfs.h" "sys/syslog.h" "wait.h" \ +- "ucontext.h" "sys/ucontext.h" "utmp.h" "utmpx.h" \ ++ "ucontext.h" "sys/ucontext.h" \ + "lastlog.h" "sys/acct.h" "sys/cachectl.h" "sys/epoll.h" "sys/reboot.h" \ + "sys/swap.h" "sys/sendfile.h" "sys/inotify.h" "sys/quota.h" "sys/klog.h" \ + "sys/fsuid.h" "sys/io.h" "sys/prctl.h" "sys/mtio.h" "sys/mount.h" \ +@@ -67,7 +67,7 @@ + "wordexp.h" "sys/membarrier.h" "sys/signalfd.h" \ + "net/if.h" "net/if_arp.h" \ + "net/ethernet.h" "net/route.h" "netinet/if_ether.h" "netinet/ether.h" \ +- "sys/timerfd.h" "libintl.h" "sys/sysmacros.h" "aio.h") ++ "sys/timerfd.h" "libintl.h" "aio.h") + # NOTE: resolv.h un-omitted by the agentOS 0029-openssh-compat patch — musl's + # resolv.h is declaration-only and ported tools (OpenSSH sshkey.c, + # sshbuf-misc.c) include unconditionally. No resolver code is diff --git a/toolchain/std-patches/wasi-libc/0031-initialize-program-invocation-names.patch b/toolchain/std-patches/wasi-libc/0031-initialize-program-invocation-names.patch new file mode 100644 index 0000000000..fdb37788d0 --- /dev/null +++ b/toolchain/std-patches/wasi-libc/0031-initialize-program-invocation-names.patch @@ -0,0 +1,41 @@ +Initialize the GNU/BSD program-name globals from the WASI argv vector. + +WASI commands bypass musl's __libc_start_main, which normally initializes +__progname and its program_invocation_* aliases. Populate them in the WASI +argument startup path so real multi-call binaries can select behavior from +argv[0], as they do on Linux. + +--- a/libc-bottom-half/sources/__main_void.c ++++ b/libc-bottom-half/sources/__main_void.c +@@ -11,6 +11,9 @@ + __attribute__((__weak__)) + int __main_argc_argv(int argc, char *argv[]); + ++extern char *__progname; ++extern char *__progname_full; ++ + // If the user's `main` function expects arguments, the compiler will rename + // it to `__main_argc_argv`, and this version will get linked in, which + // initializes the argument data and calls `__main_argc_argv`. +@@ -50,6 +53,21 @@ int __main_void(void) { + _Exit(EX_OSERR); + } + ++ /* musl normally initializes these aliases in __libc_start_main, but WASI ++ * commands enter through __main_void instead. Keep both the full argv[0] ++ * and its final path component for program_invocation_name(3) consumers. ++ * https://man7.org/linux/man-pages/man3/program_invocation_name.3.html ++ * https://git.musl-libc.org/cgit/musl/tree/src/env/__libc_start_main.c */ ++ if (argc != 0 && argv[0] != NULL) { ++ __progname_full = argv[0]; ++ __progname = argv[0]; ++ for (char *cursor = argv[0]; *cursor != '\0'; ++cursor) { ++ if (*cursor == '/') { ++ __progname = cursor + 1; ++ } ++ } ++ } ++ + // Call `__main_argc_argv` with the arguments! + return __main_argc_argv(argc, argv); + } diff --git a/toolchain/test-programs/cat.c b/toolchain/test-programs/c-cat.c similarity index 100% rename from toolchain/test-programs/cat.c rename to toolchain/test-programs/c-cat.c diff --git a/toolchain/test-programs/env.c b/toolchain/test-programs/c-env.c similarity index 100% rename from toolchain/test-programs/env.c rename to toolchain/test-programs/c-env.c diff --git a/toolchain/test-programs/sort.c b/toolchain/test-programs/c-sort.c similarity index 100% rename from toolchain/test-programs/sort.c rename to toolchain/test-programs/c-sort.c diff --git a/toolchain/test-programs/wc.c b/toolchain/test-programs/c-wc.c similarity index 100% rename from toolchain/test-programs/wc.c rename to toolchain/test-programs/c-wc.c