diff --git a/crates/client/tests/common/mod.rs b/crates/client/tests/common/mod.rs index e91cd0a9be..ca01e86829 100644 --- a/crates/client/tests/common/mod.rs +++ b/crates/client/tests/common/mod.rs @@ -149,8 +149,8 @@ fn wasm_command_mounts() -> Vec { /// Locate the materialized coreutils package under the in-repo registry build. pub fn coreutils_package_dir() -> Option { - let registry_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("../../software/coreutils/dist/package"); + let registry_dir = + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../software/coreutils/dist/package"); if registry_dir.join("agentos-package.json").is_file() { return std::fs::canonicalize(registry_dir).ok(); } diff --git a/crates/client/tests/shell_pty_packages_e2e.rs b/crates/client/tests/shell_pty_packages_e2e.rs index 7df9ff3af4..b057fdd5c6 100644 --- a/crates/client/tests/shell_pty_packages_e2e.rs +++ b/crates/client/tests/shell_pty_packages_e2e.rs @@ -26,8 +26,7 @@ fn coreutils_package_path() -> Option { } } for dir in [ - PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("../../software/coreutils/dist/package"), + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../software/coreutils/dist/package"), PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("../../node_modules/@agentos-software/coreutils/dist/package"), ] { diff --git a/crates/execution/assets/runners/wasm-runner.mjs b/crates/execution/assets/runners/wasm-runner.mjs index 47e83c2178..f385336bd8 100644 --- a/crates/execution/assets/runners/wasm-runner.mjs +++ b/crates/execution/assets/runners/wasm-runner.mjs @@ -30,6 +30,7 @@ const WASI_ERRNO_FAULT = 21; const WASI_RIGHT_FD_WRITE = 64n; const WASI_FILETYPE_UNKNOWN = 0; const WASI_FILETYPE_CHARACTER_DEVICE = 2; +const WASI_FILETYPE_DIRECTORY = 3; const WASI_FILETYPE_REGULAR_FILE = 4; const WASI_OFLAGS_CREAT = 1; const WASI_OFLAGS_DIRECTORY = 2; @@ -428,7 +429,8 @@ const passthroughHandles = new Map([ ]); const retainedSyntheticHandlesByDisplayFd = new Map(); const retainedSpawnOutputHandlesByFd = new Map(); -let nextSyntheticFd = 64; +const FIRST_SYNTHETIC_FD = 1 << 20; +let nextSyntheticFd = FIRST_SYNTHETIC_FD; let nextSyntheticPipeId = 1; const syntheticWaitArray = new Int32Array(new SharedArrayBuffer(4)); let delegateWriteScratch = { base: 0, capacity: 0 }; @@ -1160,13 +1162,19 @@ function fsOpenFlagForPathOpen(oflags, rightsBase, fdflags) { return 'r+'; } -function allocateSyntheticFd() { - let fd = nextSyntheticFd; - while ( +function syntheticFdInUse(fd) { + return ( syntheticFdEntries.has(fd) || passthroughHandles.has(fd) || + retainedSpawnOutputHandlesByFd.has(fd) || + retainedSyntheticHandlesByDisplayFd.has(fd) || delegateManagedFdRefCounts.has(fd) - ) { + ); +} + +function allocateSyntheticFd(minFd = nextSyntheticFd) { + let fd = Math.max(FIRST_SYNTHETIC_FD, Number(minFd) >>> 0); + while (syntheticFdInUse(fd)) { fd += 1; } nextSyntheticFd = fd + 1; @@ -1226,8 +1234,9 @@ function openManagedPathIoFd(guestPath, rightsBase, fdflags) { return null; } try { + const hostPath = resolveHostFsPath(guestPath) ?? guestPath; return fsModule.openSync( - guestPath, + hostPath, fsOpenNumericFlagsForManagedPath(rightsBase, fdflags), 0o666, ); @@ -1243,16 +1252,35 @@ function retainPathOpenDelegateFd(openedFdPtr, guestPath, fdflags, rightsBase) { try { const openedFd = new DataView(instanceMemory.buffer).getUint32(Number(openedFdPtr), true); + let retainedFd = openedFd; + if (openedFd > 2 && syntheticFdInUse(openedFd)) { + if (typeof delegateManagedFdRenumber !== 'function') { + return WASI_ERRNO_FAULT; + } + retainedFd = allocateSyntheticFd(openedFd + 1); + const renumberResult = delegateManagedFdRenumber(openedFd, retainedFd); + if (renumberResult !== WASI_ERRNO_SUCCESS) { + return renumberResult; + } + const writeResult = writeGuestUint32(openedFdPtr, retainedFd); + if (writeResult !== WASI_ERRNO_SUCCESS) { + return writeResult; + } + traceHostProcess('path-open-delegate-renumber', { + openedFd, + retainedFd, + }); + } const append = (Number(fdflags) & WASI_FDFLAGS_APPEND) !== 0; - retainDelegateFd(openedFd); - if (openedFd > 2 && !passthroughHandles.has(openedFd)) { + retainDelegateFd(retainedFd); + if (retainedFd > 2 && !passthroughHandles.has(retainedFd)) { const ioFd = openManagedPathIoFd(guestPath, rightsBase, fdflags); - closedPassthroughFds.delete(openedFd); - passthroughHandles.set(openedFd, { + closedPassthroughFds.delete(retainedFd); + passthroughHandles.set(retainedFd, { kind: 'passthrough', - targetFd: openedFd, + targetFd: retainedFd, ioFd, - displayFd: openedFd, + displayFd: retainedFd, refCount: 0, open: true, readOnly: @@ -1331,6 +1359,19 @@ function writeGuestFilestat(ptr, stats, filetype = WASI_FILETYPE_REGULAR_FILE) { } } +function wasiFiletypeFromStats(stats) { + if (typeof stats?.isDirectory === 'function' && stats.isDirectory()) { + return WASI_FILETYPE_DIRECTORY; + } + if (typeof stats?.isCharacterDevice === 'function' && stats.isCharacterDevice()) { + return WASI_FILETYPE_CHARACTER_DEVICE; + } + if (typeof stats?.isFile === 'function' && stats.isFile()) { + return WASI_FILETYPE_REGULAR_FILE; + } + return WASI_FILETYPE_UNKNOWN; +} + function writeGuestFdstat(ptr, filetype, flags, rightsBase, rightsInheriting) { if (!(instanceMemory instanceof WebAssembly.Memory)) { return WASI_ERRNO_FAULT; @@ -4226,8 +4267,8 @@ const hostProcessImport = { readHandleCount: 0, writeHandleCount: 0, }; - const readFd = nextSyntheticFd++; - const writeFd = nextSyntheticFd++; + const readFd = allocateSyntheticFd(); + const writeFd = allocateSyntheticFd(); syntheticFdEntries.set(readFd, createPipeHandle('pipe-read', pipe, readFd)); syntheticFdEntries.set(writeFd, createPipeHandle('pipe-write', pipe, writeFd)); if (writeGuestUint32(retReadFdPtr, readFd) !== WASI_ERRNO_SUCCESS) { @@ -4244,20 +4285,7 @@ const hostProcessImport = { if (!handle) { return WASI_ERRNO_BADF; } - let duplicatedFd = 0; - while ( - duplicatedFd <= 2 && - ( - syntheticFdEntries.has(duplicatedFd) || - passthroughHandles.has(duplicatedFd) || - delegateManagedFdRefCounts.has(duplicatedFd) - ) - ) { - duplicatedFd += 1; - } - if (duplicatedFd > 2) { - duplicatedFd = nextSyntheticFd++; - } + const duplicatedFd = allocateSyntheticFd(0); syntheticFdEntries.set(duplicatedFd, handle); traceHostProcess('fd-dup', { fd: Number(fd) >>> 0, @@ -4329,15 +4357,7 @@ const hostProcessImport = { return WASI_ERRNO_BADF; } - let duplicatedFd = minimumFdNumber >>> 0; - while ( - syntheticFdEntries.has(duplicatedFd) || - passthroughHandles.has(duplicatedFd) || - delegateManagedFdRefCounts.has(duplicatedFd) - ) { - duplicatedFd += 1; - } - nextSyntheticFd = Math.max(nextSyntheticFd, duplicatedFd + 1); + const duplicatedFd = allocateSyntheticFd(minimumFdNumber); syntheticFdEntries.set(duplicatedFd, handle); traceHostProcess('fd-dup-min', { @@ -5213,11 +5233,13 @@ wasiImport.fd_read = (fd, iovs, iovsLen, nreadPtr) => { } if (handle?.kind === 'passthrough') { - return delegateManagedFdRead - ? __agentOSWasiMeasurePhase('fd_read', 'delegate_call', () => - delegateManagedFdRead(handle.targetFd, iovs, iovsLen, nreadPtr) - ) - : WASI_ERRNO_BADF; + if (!delegateManagedFdRead) { + return WASI_ERRNO_BADF; + } + const result = __agentOSWasiMeasurePhase('fd_read', 'delegate_call', () => + delegateManagedFdRead(handle.targetFd, iovs, iovsLen, nreadPtr) + ); + return result; } if (rejectClosedPassthroughFd(numericFd)) { @@ -5291,9 +5313,10 @@ wasiImport.fd_pread = (fd, iovs, iovsLen, offset, nreadPtr) => { return mapSyntheticFsError(error); } } - return delegateFdPread - ? delegateFdPread(handle.targetFd, iovs, iovsLen, offset, nreadPtr) - : WASI_ERRNO_BADF; + if (!delegateFdPread) { + return WASI_ERRNO_BADF; + } + return delegateFdPread(handle.targetFd, iovs, iovsLen, offset, nreadPtr); } if (rejectClosedPassthroughFd(fd)) { @@ -5308,6 +5331,9 @@ wasiImport.fd_pread = (fd, iovs, iovsLen, offset, nreadPtr) => { wasiImport.fd_pwrite = (fd, iovs, iovsLen, offset, nwrittenPtr) => { const handle = lookupFdHandle(fd); if (handle?.kind === 'guest-file') { + if (handle.readOnly === true) { + return WASI_ERRNO_ROFS; + } try { const bytes = collectGuestIovBytes(iovs, iovsLen); const written = fsModule.writeSync( @@ -5512,11 +5538,50 @@ wasiImport.fd_fdstat_get = (fd, statPtr) => { ); } + if (handle?.kind === 'guest-file') { + try { + const stat = fsModule.fstatSync(handle.targetFd); + return writeGuestFdstat( + statPtr, + wasiFiletypeFromStats(stat), + 0, + WASI_RIGHT_FD_READ | + WASI_RIGHT_FD_SEEK | + WASI_RIGHT_FD_TELL | + WASI_RIGHT_FD_FILESTAT_GET | + WASI_RIGHT_FD_WRITE | + WASI_RIGHT_FD_SYNC, + 0n, + ); + } catch (error) { + return mapSyntheticFsError(error); + } + } + if (handle && handle.kind !== 'passthrough') { return WASI_ERRNO_BADF; } if (handle?.kind === 'passthrough') { + if (typeof handle.ioFd === 'number') { + try { + const stat = fsModule.fstatSync(handle.ioFd); + return writeGuestFdstat( + statPtr, + wasiFiletypeFromStats(stat), + 0, + WASI_RIGHT_FD_READ | + WASI_RIGHT_FD_SEEK | + WASI_RIGHT_FD_TELL | + WASI_RIGHT_FD_FILESTAT_GET | + WASI_RIGHT_FD_WRITE | + WASI_RIGHT_FD_SYNC, + 0n, + ); + } catch (error) { + return mapSyntheticFsError(error); + } + } return delegateManagedFdFdstatGet ? __agentOSWasiMeasurePhase('fd_fdstat_get', 'delegate_call', () => delegateManagedFdFdstatGet(handle.targetFd, statPtr) @@ -5574,9 +5639,10 @@ wasiImport.fd_filestat_get = (fd, statPtr) => { return mapSyntheticFsError(error); } } - return delegateManagedFdFilestatGet - ? delegateManagedFdFilestatGet(handle.targetFd, statPtr) - : WASI_ERRNO_BADF; + if (!delegateManagedFdFilestatGet) { + return WASI_ERRNO_BADF; + } + return delegateManagedFdFilestatGet(handle.targetFd, statPtr); } if (rejectClosedPassthroughFd(fd)) { @@ -5726,6 +5792,9 @@ wasiImport.fd_write = (fd, iovs, iovsLen, nwrittenPtr) => { } if (handle?.kind === 'guest-file') { + if (handle.readOnly === true) { + return WASI_ERRNO_ROFS; + } try { const bytes = __agentOSWasiMeasurePhase('fd_write', 'guest_iov_collect', () => collectGuestIovBytes(iovs, iovsLen) diff --git a/crates/native-sidecar/tests/projection_bench.rs b/crates/native-sidecar/tests/projection_bench.rs index 1dc4e6b72b..034b4830f8 100644 --- a/crates/native-sidecar/tests/projection_bench.rs +++ b/crates/native-sidecar/tests/projection_bench.rs @@ -614,14 +614,8 @@ fn projection_bench() { "PROJ_BENCH_COREUTILS_TAR", "software/coreutils/dist/package.tar", ); - let tar_tar = source_tar_path( - "PROJ_BENCH_TAR_TAR", - "software/tar/dist/package.tar", - ); - let git_tar = source_tar_path( - "PROJ_BENCH_GIT_TAR", - "software/git/dist/package.tar", - ); + let tar_tar = source_tar_path("PROJ_BENCH_TAR_TAR", "software/tar/dist/package.tar"); + let git_tar = source_tar_path("PROJ_BENCH_GIT_TAR", "software/git/dist/package.tar"); println!("\n# agentOS package load benchmark (.aospkg)"); println!( diff --git a/crates/vfs/src/posix/overlay_fs.rs b/crates/vfs/src/posix/overlay_fs.rs index 3c4ce17454..9fd0600587 100644 --- a/crates/vfs/src/posix/overlay_fs.rs +++ b/crates/vfs/src/posix/overlay_fs.rs @@ -2005,7 +2005,9 @@ mod tests { .expect("rename replacement over whiteout"); assert_eq!( - overlay.read_file("/archive.zip").expect("read renamed file"), + overlay + .read_file("/archive.zip") + .expect("read renamed file"), b"new" ); assert!(!overlay.exists("/workspace-temp")); diff --git a/docs-internal/registry-parity-worklist.md b/docs-internal/registry-parity-worklist.md index 32e448a0a5..72c6918889 100644 --- a/docs-internal/registry-parity-worklist.md +++ b/docs-internal/registry-parity-worklist.md @@ -102,7 +102,7 @@ actual backing: | **curl** | DONE | our custom driver over a libcurl fork | real `curl` CLI (upstream `src/tool_*.c`) | | **wget** | DONE | our 174-line `wget.c` (dropped) | real GNU Wget vs our sysroot — stub `getrlimit`/`getgroups`, then build | | **http-get** | DONE | our 95-line `http_get.c` | dropped; real curl covers HTTP fetches | -| **git** | TODO | our hand-rolled git from `sha1`+`flate2` | **real git** (upstream C), patched for WASI — **NOT gitoxide** | +| **git** | DONE | our hand-rolled git from `sha1`+`flate2` | **real git** (upstream C), patched for WASI — **NOT gitoxide** | | **fd** | DONE | our `secureexec-fd` on raw `regex` (not sharkdp/fd) | real **fd** (sharkdp) | | **findutils** (`find`,`xargs`) | TODO | our hand-rolled on `regex`/shims | real GNU findutils, or `uutils/findutils` | | **tree** | DONE | our hand-rolled, zero deps | real `tree`, or an established one | @@ -165,6 +165,23 @@ ripgrep crates); **(b) gnulib `getrlimit`/`getgroups`** → sysroot stubs, alrea documented for wget (item #8) (hits GNU grep/findutils). Subprocess spawn already works (`wasi-spawn` broker), so `xargs` is not a blocker. +- **git — DONE.** Replaced the custom Rust `sha1`+`flate2` implementation with real + upstream Git 2.55.0 built by the C toolchain and staged as `git` plus helper + command aliases. The WASI changes stayed below Git's behavior surface: + `run-command.c` uses `posix_spawn` so helper subprocesses go through the existing + wasi-spawn broker, the sysroot exposes Git's missing C compatibility surface, and + the runner now allocates synthetic fds in a high range so managed pipe/file fds + cannot collide with delegate-opened WASI fds. Smart HTTP remains intentionally + disabled in this build (`NO_CURL`), so HTTPS clone fails with the real Git helper + error instead of a custom transport. Proof: clean upstream Git rebuild passes in + `2026-07-08T11-28-00-0700-git-clean-rebuild-after-high-synthetic-fd.log`; + package build stages 6 commands in + `2026-07-08T11-33-00-0700-git-package-build-clean-binary-after-install.log`; + native sidecar rebuild passes in + `2026-07-08T11-34-00-0700-sidecar-rebuild-after-git-clean-package.log`; + full Git e2e passes 18/18 in + `2026-07-08T11-51-00-0700-git-full-e2e-high-synthetic-fd-clean-binary-after-test-fix.log`. + Rev: `tmvlxlvk` — `fix(git): build upstream git`. - **sqlite3 CLI — DONE.** Engine was already the real amalgamation (`libs/sqlite3/sqlite3.c`); the command now builds the official upstream `shell.c` from the same fetched zip as `sqlite3`. The local 558-line diff --git a/packages/runtime-core/commands/git b/packages/runtime-core/commands/git index 1865311237..869f2fae78 100644 Binary files a/packages/runtime-core/commands/git and b/packages/runtime-core/commands/git differ diff --git a/packages/runtime-core/commands/git-receive-pack b/packages/runtime-core/commands/git-receive-pack new file mode 100644 index 0000000000..869f2fae78 Binary files /dev/null and b/packages/runtime-core/commands/git-receive-pack differ diff --git a/packages/runtime-core/commands/git-remote-http b/packages/runtime-core/commands/git-remote-http index 1865311237..869f2fae78 100644 Binary files a/packages/runtime-core/commands/git-remote-http and b/packages/runtime-core/commands/git-remote-http differ diff --git a/packages/runtime-core/commands/git-remote-https b/packages/runtime-core/commands/git-remote-https index 1865311237..869f2fae78 100644 Binary files a/packages/runtime-core/commands/git-remote-https and b/packages/runtime-core/commands/git-remote-https differ diff --git a/packages/runtime-core/commands/git-upload-archive b/packages/runtime-core/commands/git-upload-archive new file mode 100644 index 0000000000..869f2fae78 Binary files /dev/null and b/packages/runtime-core/commands/git-upload-archive differ diff --git a/packages/runtime-core/commands/git-upload-pack b/packages/runtime-core/commands/git-upload-pack new file mode 100644 index 0000000000..869f2fae78 Binary files /dev/null and b/packages/runtime-core/commands/git-upload-pack differ diff --git a/software/git/agentos-package.json b/software/git/agentos-package.json index c58e5c4f32..31c38e66e4 100644 --- a/software/git/agentos-package.json +++ b/software/git/agentos-package.json @@ -4,11 +4,14 @@ ], "aliases": { "git-remote-http": "git", - "git-remote-https": "git" + "git-remote-https": "git", + "git-upload-pack": "git", + "git-receive-pack": "git", + "git-upload-archive": "git" }, "registry": { "title": "git", - "description": "Git version control (clone, commit, push, pull).", + "description": "Upstream Git version control for AgentOS VMs.", "priority": 90, "image": "/images/registry/git.svg", "category": "developer-tools" diff --git a/software/git/bin/git b/software/git/bin/git index 1865311237..8c7211cc97 100644 Binary files a/software/git/bin/git and b/software/git/bin/git differ diff --git a/software/git/bin/git-remote-http b/software/git/bin/git-remote-http index 1865311237..8c7211cc97 100644 Binary files a/software/git/bin/git-remote-http and b/software/git/bin/git-remote-http differ diff --git a/software/git/bin/git-remote-https b/software/git/bin/git-remote-https index 1865311237..8c7211cc97 100644 Binary files a/software/git/bin/git-remote-https and b/software/git/bin/git-remote-https differ diff --git a/software/git/native/crates/cmd-git/Cargo.toml b/software/git/native/crates/cmd-git/Cargo.toml deleted file mode 100644 index 83f16f93c4..0000000000 --- a/software/git/native/crates/cmd-git/Cargo.toml +++ /dev/null @@ -1,14 +0,0 @@ -[package] -workspace = "../../../../../toolchain" -name = "cmd-git" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "git standalone binary for secure-exec VM" - -[[bin]] -name = "git" -path = "src/main.rs" - -[dependencies] -secureexec-git = { path = "../git" } diff --git a/software/git/native/crates/cmd-git/src/main.rs b/software/git/native/crates/cmd-git/src/main.rs deleted file mode 100644 index 83886647f0..0000000000 --- a/software/git/native/crates/cmd-git/src/main.rs +++ /dev/null @@ -1,4 +0,0 @@ -fn main() { - let args: Vec = std::env::args_os().collect(); - std::process::exit(secureexec_git::main(args)); -} diff --git a/software/git/native/crates/git/Cargo.toml b/software/git/native/crates/git/Cargo.toml deleted file mode 100644 index 5185e88b15..0000000000 --- a/software/git/native/crates/git/Cargo.toml +++ /dev/null @@ -1,16 +0,0 @@ -[package] -workspace = "../../../../../toolchain" -name = "secureexec-git" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "Minimal git implementation for secure-exec VM" - -[lib] -name = "secureexec_git" -path = "src/lib.rs" - -[dependencies] -sha1 = "0.10.6" -flate2 = "1.1" -wasi-http = { package = "secureexec-wasi-http", path = "../../../../../toolchain/crates/libs/wasi-http" } diff --git a/software/git/native/crates/git/README.md b/software/git/native/crates/git/README.md deleted file mode 100644 index 26fcb37043..0000000000 --- a/software/git/native/crates/git/README.md +++ /dev/null @@ -1,36 +0,0 @@ -# secure-exec VM git compatibility - -The `git` command in secure-exec is a clean-room, Apache-2.0 reimplementation of a -small git subset for WasmVM guests. - -## Supported commands - -- `git init` -- `git add` -- `git commit -m ...` -- `git rev-parse ` -- `git branch` -- `git checkout` -- `git clone ` -- `git clone http://...` -- `git clone https://...` - -Remote clone support is limited to unauthenticated smart-HTTP fetches. The -implementation can read from `http://` and `https://` remotes, but it does not -support credentials, push, or SSH transport. - -## Unsupported commands and transports - -The guest command exits with a typed `GitSubcommandUnsupported` error for: - -- `git push` -- `git fetch` -- `git pull` -- `git remote` -- SSH-style clone URLs such as `git@host:owner/repo.git` and `ssh://...` -- `git://...` clone URLs -- Authenticated HTTP(S) remotes such as `https://user@host/repo.git` -- Submodules, hooks, GPG signing, and other advanced porcelain workflows - -If you need behavior outside this subset, keep using host git or extend the -clean-room implementation deliberately. diff --git a/software/git/native/crates/git/src/lib.rs b/software/git/native/crates/git/src/lib.rs deleted file mode 100644 index 62a273b237..0000000000 --- a/software/git/native/crates/git/src/lib.rs +++ /dev/null @@ -1,2127 +0,0 @@ -//! Minimal clean-room git implementation (Apache-2.0). -//! -//! Supports: init, add, commit, branch, checkout (with DWIM), local clone, -//! and smart-HTTP clone over http:// and https://. - -use flate2::bufread::ZlibDecoder as BufZlibDecoder; -use flate2::read::ZlibDecoder; -use flate2::write::ZlibEncoder; -use flate2::Compression; -use sha1::{Digest, Sha1}; -use std::collections::{BTreeMap, HashMap, HashSet}; -use std::ffi::{OsStr, OsString}; -use std::fmt; -use std::fs; -use std::io::{self, Cursor, Read, Write}; -use std::path::{Component, Path, PathBuf}; -use wasi_http::{HttpClient, Method, Request}; - -// ─── Hex utilities ────────────────────────────────────────────────────────── - -fn hex(bytes: &[u8]) -> String { - bytes.iter().map(|b| format!("{:02x}", b)).collect() -} - -fn unhex(s: &str) -> io::Result<[u8; 20]> { - if s.len() != 40 { - return Err(err(&format!("invalid hash length: {}", s.len()))); - } - let bytes: Vec = (0..s.len()) - .step_by(2) - .map(|i| { - u8::from_str_radix(&s[i..i + 2], 16).map_err(|e| err(&format!("invalid hex: {}", e))) - }) - .collect::>>()?; - let mut hash = [0u8; 20]; - hash.copy_from_slice(&bytes); - Ok(hash) -} - -fn err(msg: &str) -> io::Error { - io::Error::new(io::ErrorKind::Other, msg.to_string()) -} - -fn print_stdout_line(args: fmt::Arguments<'_>) -> io::Result<()> { - let mut stdout = io::stdout().lock(); - stdout.write_fmt(args)?; - stdout.write_all(b"\n")?; - stdout.flush() -} - -const SUPPORT_DOC_PATH: &str = "software/git/native/crates/git/README.md"; -const MAX_GIT_OBJECT_BYTES: usize = 128 * 1024 * 1024; -const MAX_GIT_OBJECT_HEADER_BYTES: usize = 128; -const MAX_INDEX_ENTRIES: usize = 1_000_000; -const MAX_INDEX_BYTES: usize = 256 * 1024 * 1024; -const MAX_ADVERTISED_REFS: usize = 100_000; -const MAX_HTTP_ERROR_BODY_BYTES: usize = 8192; -const MAX_PKT_LINES: usize = 100_000; -const MAX_PKT_LINE_PAYLOAD_BYTES: usize = 16 * 1024 * 1024; -const MAX_REF_ADVERTISEMENT_BYTES: usize = 16 * 1024 * 1024; -const MAX_PACK_INFLATED_BYTES: usize = 256 * 1024 * 1024; -const MAX_PACK_BYTES: usize = 256 * 1024 * 1024; -const MAX_PACK_OBJECTS: usize = 1_000_000; -const MAX_PACK_RESOLVED_BYTES: usize = 256 * 1024 * 1024; - -fn unsupported(subcommand: &str, detail: &str) -> io::Error { - err(&format!( - "GitSubcommandUnsupported: git {} {} See {} for supported transports and commands.", - subcommand, detail, SUPPORT_DOC_PATH - )) -} - -/// WASI-safe mkdir -p: create_dir_all has issues with WASI permission checks -/// on already-existing directories, so we create one level at a time. -/// We also treat PermissionDenied as "already exists" because some WASI runtimes -/// return EACCES instead of EEXIST for existing directories. -fn mkdirs(path: &Path) -> io::Result<()> { - let mut ancestors: Vec<&Path> = path.ancestors().collect(); - ancestors.reverse(); // root first - for dir in ancestors { - if dir == Path::new("/") || dir == Path::new("") { - continue; - } - match fs::create_dir(dir) { - Ok(()) => {} - Err(e) if e.kind() == io::ErrorKind::AlreadyExists => {} - // WASI kernels may return EACCES for existing directories - Err(e) if e.kind() == io::ErrorKind::PermissionDenied => { - // Verify it actually exists; propagate if it truly doesn't - if !dir.exists() { - return Err(e); - } - } - Err(e) => return Err(e), - } - } - Ok(()) -} - -fn dir_is_empty(path: &Path) -> io::Result { - if !path.exists() { - return Ok(true); - } - if !path.is_dir() { - return Ok(false); - } - Ok(fs::read_dir(path)?.next().is_none()) -} - -fn read_to_end_limited(reader: R, limit: usize, context: &str) -> io::Result> { - let limit_plus_one = limit - .checked_add(1) - .ok_or_else(|| err(&format!("{} size limit is too large", context)))?; - let mut out = Vec::new(); - reader.take(limit_plus_one as u64).read_to_end(&mut out)?; - if out.len() > limit { - return Err(err(&format!("{} exceeds size limit", context))); - } - Ok(out) -} - -fn read_file_limited(path: &Path, limit: usize, context: &str) -> io::Result> { - let metadata = fs::metadata(path).map_err(|e| { - err(&format!( - "cannot stat {} '{}': {}", - context, - path.display(), - e - )) - })?; - if metadata.len() > limit as u64 { - return Err(err(&format!( - "{} '{}' exceeds size limit", - context, - path.display() - ))); - } - fs::read(path).map_err(|e| { - err(&format!( - "cannot read {} '{}': {}", - context, - path.display(), - e - )) - }) -} - -fn try_reserve_exact(vec: &mut Vec, additional: usize, context: &str) -> io::Result<()> { - vec.try_reserve_exact(additional) - .map_err(|_| err(&format!("{} allocation failed", context))) -} - -fn append_pack_bytes(pack: &mut Vec, bytes: &[u8]) -> io::Result<()> { - let new_len = pack - .len() - .checked_add(bytes.len()) - .ok_or_else(|| err("packfile size overflow"))?; - if new_len > MAX_PACK_BYTES { - return Err(err("packfile exceeds size limit")); - } - try_reserve_exact(pack, bytes.len(), "packfile")?; - pack.extend_from_slice(bytes); - Ok(()) -} - -fn add_pack_inflated_bytes(total: &mut usize, bytes: usize) -> io::Result<()> { - *total = total - .checked_add(bytes) - .ok_or_else(|| err("inflated pack size overflow"))?; - if *total > MAX_PACK_INFLATED_BYTES { - return Err(err("inflated pack data exceeds size limit")); - } - Ok(()) -} - -fn add_pack_resolved_bytes(total: &mut usize, bytes: usize) -> io::Result<()> { - *total = total - .checked_add(bytes) - .ok_or_else(|| err("resolved pack size overflow"))?; - if *total > MAX_PACK_RESOLVED_BYTES { - return Err(err("resolved pack data exceeds size limit")); - } - Ok(()) -} - -fn add_pkt_payload_bytes(total: &mut usize, bytes: usize) -> io::Result<()> { - *total = total - .checked_add(bytes) - .ok_or_else(|| err("pkt-line payload size overflow"))?; - if *total > MAX_PKT_LINE_PAYLOAD_BYTES { - return Err(err("pkt-line payload exceeds size limit")); - } - Ok(()) -} - -fn add_advertised_ref_count(total: usize) -> io::Result<()> { - if total > MAX_ADVERTISED_REFS { - return Err(err("remote advertised too many refs")); - } - Ok(()) -} - -fn response_body_preview(body: &[u8]) -> String { - let limit = body.len().min(MAX_HTTP_ERROR_BODY_BYTES); - let mut preview = String::from_utf8_lossy(&body[..limit]).trim().to_string(); - if body.len() > limit { - preview.push_str("..."); - } - preview -} - -fn normalize_repo_path(path: &str) -> io::Result { - if path.is_empty() || path.as_bytes().contains(&0) { - return Err(err("invalid empty or nul-containing repository path")); - } - - let mut parts = Vec::new(); - for component in Path::new(path).components() { - match component { - Component::Normal(part) if part == OsStr::new(".git") => { - return Err(err("repository path must not enter .git")); - } - Component::Normal(part) => { - let part = part - .to_str() - .ok_or_else(|| err("repository path must be utf-8"))?; - if part.is_empty() { - return Err(err("repository path contains an empty component")); - } - parts.push(part); - } - Component::CurDir - | Component::ParentDir - | Component::RootDir - | Component::Prefix(_) => { - return Err(err("repository path must be relative and normalized")); - } - } - } - - if parts.is_empty() { - return Err(err("repository path must name a file")); - } - - Ok(parts.join("/")) -} - -fn worktree_path(workdir: &Path, repo_path: &str) -> io::Result { - Ok(workdir.join(normalize_repo_path(repo_path)?)) -} - -fn validate_ref_tail(name: &str) -> io::Result<&str> { - if name.is_empty() - || name.as_bytes().contains(&0) - || name.starts_with('/') - || name.ends_with('/') - || name.contains('\\') - { - return Err(err("invalid git ref name")); - } - - for part in name.split('/') { - if part.is_empty() || part == "." || part == ".." { - return Err(err("invalid git ref name")); - } - } - - Ok(name) -} - -fn validate_branch_name(name: &str) -> io::Result<&str> { - validate_ref_tail(name) -} - -fn validate_refname(refname: &str) -> io::Result<&str> { - if refname == "HEAD" { - return Ok(refname); - } - - for prefix in ["refs/heads/", "refs/remotes/", "refs/tags/"] { - if let Some(tail) = refname.strip_prefix(prefix) { - validate_ref_tail(tail)?; - return Ok(refname); - } - } - - Err(err("unsupported git ref name")) -} - -fn hash_bytes(obj_type: &str, data: &[u8]) -> [u8; 20] { - let header = format!("{} {}\0", obj_type, data.len()); - let mut hasher = Sha1::new(); - hasher.update(header.as_bytes()); - hasher.update(data); - hasher.finalize().into() -} - -fn is_zero_oid(hash: &[u8; 20]) -> bool { - hash.iter().all(|b| *b == 0) -} - -fn is_remote_source(source: &str) -> bool { - source.starts_with("http://") || source.starts_with("https://") -} - -fn is_ssh_clone_source(source: &str) -> bool { - source.starts_with("git@") - || source.starts_with("ssh://") - || source.starts_with("git://") - || source.starts_with("ssh+git://") -} - -fn has_http_auth(source: &str) -> bool { - if !is_remote_source(source) { - return false; - } - - let Some(scheme_sep) = source.find("://") else { - return false; - }; - let authority = &source[scheme_sep + 3..] - .split_once('/') - .map(|(authority, _)| authority) - .unwrap_or(&source[scheme_sep + 3..]); - - authority.contains('@') -} - -fn infer_clone_destination(source: &str) -> io::Result { - let basename = if is_remote_source(source) || is_ssh_clone_source(source) { - let trimmed = source.trim_end_matches('/'); - let leaf = if let Some((_, scp_path)) = trimmed.rsplit_once(':') { - if is_ssh_clone_source(source) && !scp_path.contains('/') { - scp_path - } else { - trimmed - .rsplit('/') - .next() - .filter(|segment| !segment.is_empty()) - .ok_or_else(|| err("could not determine destination path from source"))? - } - } else { - trimmed - .rsplit('/') - .next() - .filter(|segment| !segment.is_empty()) - .ok_or_else(|| err("could not determine destination path from source"))? - }; - leaf.to_string() - } else { - Path::new(source) - .file_name() - .map(|name| name.to_string_lossy().to_string()) - .filter(|name| !name.is_empty()) - .ok_or_else(|| err("could not determine destination path from source"))? - }; - - Ok(basename - .strip_suffix(".git") - .unwrap_or(&basename) - .to_string()) -} - -fn prepare_clone_destination( - dest: &Path, - source_label: &str, - default_branch: &str, -) -> io::Result { - validate_branch_name(default_branch)?; - mkdirs(dest)?; - - let dst_git = dest.join(".git"); - mkdirs(&dst_git.join("objects"))?; - mkdirs(&dst_git.join("refs/heads"))?; - mkdirs(&dst_git.join("refs/tags"))?; - mkdirs(&dst_git.join("refs/remotes/origin"))?; - - fs::write( - dst_git.join("HEAD"), - format!("ref: refs/heads/{}\n", default_branch), - )?; - fs::write( - dst_git.join("config"), - format!( - "[core]\n\trepositoryformatversion = 0\n\tfilemode = true\n\tbare = false\n\ - [remote \"origin\"]\n\turl = {}\n\tfetch = +refs/heads/*:refs/remotes/origin/*\n", - source_label - ), - )?; - - Ok(dst_git) -} - -#[derive(Debug)] -enum PktLine { - Data(Vec), - Flush, -} - -fn parse_pkt_lines(data: &[u8]) -> io::Result> { - let mut pos = 0usize; - let mut lines = Vec::new(); - let mut payload_bytes = 0usize; - - while pos + 4 <= data.len() { - if lines.len() >= MAX_PKT_LINES { - return Err(err("too many pkt-lines")); - } - let len_str = - std::str::from_utf8(&data[pos..pos + 4]).map_err(|_| err("invalid pkt-line"))?; - let len = usize::from_str_radix(len_str, 16).map_err(|_| err("invalid pkt-line length"))?; - pos += 4; - - if len == 0 { - lines.push(PktLine::Flush); - continue; - } - if len < 4 || pos + len - 4 > data.len() { - return Err(err("truncated pkt-line")); - } - - let payload_len = len - 4; - add_pkt_payload_bytes(&mut payload_bytes, payload_len)?; - lines.push(PktLine::Data(data[pos..pos + payload_len].to_vec())); - pos += len - 4; - } - - if pos != data.len() { - return Err(err("trailing bytes after pkt-lines")); - } - - Ok(lines) -} - -#[derive(Debug, Default)] -struct RemoteAdvertisement { - head_hash: Option<[u8; 20]>, - head_target: Option, - branches: BTreeMap, - tags: BTreeMap, -} - -fn parse_advertised_ref(line: &[u8], adv: &mut RemoteAdvertisement) -> io::Result<()> { - let (ref_part, capability_part) = if let Some(nul) = line.iter().position(|b| *b == 0) { - (&line[..nul], Some(&line[nul + 1..])) - } else { - (line, None) - }; - - let ref_text = std::str::from_utf8(ref_part) - .map_err(|_| err("invalid advertised ref"))? - .trim_end_matches('\n'); - if ref_text.is_empty() { - return Ok(()); - } - - let (hash_hex, refname) = ref_text - .split_once(' ') - .ok_or_else(|| err("malformed advertised ref"))?; - let hash = unhex(hash_hex)?; - - if refname == "HEAD" { - adv.head_hash = Some(hash); - } else if refname.starts_with("refs/heads/") { - validate_refname(refname)?; - adv.branches.insert(refname.to_string(), hash); - add_advertised_ref_count(adv.branches.len() + adv.tags.len())?; - } else if refname.starts_with("refs/tags/") { - validate_refname(refname)?; - adv.tags.insert(refname.to_string(), hash); - add_advertised_ref_count(adv.branches.len() + adv.tags.len())?; - } - - if let Some(capabilities) = capability_part { - let caps = std::str::from_utf8(capabilities).map_err(|_| err("invalid capability list"))?; - for cap in caps.split_whitespace() { - if let Some(target) = cap.strip_prefix("symref=HEAD:") { - validate_refname(target)?; - adv.head_target = Some(target.to_string()); - } - } - } - - Ok(()) -} - -fn fetch_remote_advertisement(url: &str) -> io::Result { - let info_refs_url = format!( - "{}/info/refs?service=git-upload-pack", - url.trim_end_matches('/') - ); - let req = Request::new(Method::Get, &info_refs_url) - .map_err(|e| err(&format!("bad info/refs URL: {}", e)))? - .header("Accept", "application/x-git-upload-pack-advertisement") - .header("Git-Protocol", "version=0"); - - let resp = HttpClient::new() - .send(&req) - .map_err(|e| err(&format!("fetch info/refs failed: {}", e)))?; - if resp.status != 200 { - let body = response_body_preview(&resp.body); - return Err(err(&format!( - "remote advertised refs request failed (HTTP {}): {}", - resp.status, body - ))); - } - if resp.body.len() > MAX_REF_ADVERTISEMENT_BYTES { - return Err(err("remote advertised refs response exceeds size limit")); - } - - let lines = parse_pkt_lines(&resp.body)?; - let mut adv = RemoteAdvertisement::default(); - let mut saw_service = false; - let mut in_refs = false; - - for line in lines { - match line { - PktLine::Flush => { - if saw_service { - in_refs = true; - } - } - PktLine::Data(data) => { - if !saw_service { - let service = std::str::from_utf8(&data) - .map_err(|_| err("invalid git service header"))?; - if service.trim_end_matches('\n') != "# service=git-upload-pack" { - return Err(err("unexpected git service advertisement")); - } - saw_service = true; - continue; - } - if !in_refs { - continue; - } - parse_advertised_ref(&data, &mut adv)?; - } - } - } - - if !saw_service { - return Err(err("missing git-upload-pack service header")); - } - - Ok(adv) -} - -fn branch_name_from_ref(refname: &str) -> io::Result { - refname - .strip_prefix("refs/heads/") - .map(validate_branch_name) - .transpose()? - .map(|name| name.to_string()) - .ok_or_else(|| err("expected refs/heads/* ref")) -} - -fn default_branch_ref(adv: &RemoteAdvertisement) -> Option<(String, [u8; 20])> { - if let Some(target) = adv.head_target.as_ref() { - if let Some(hash) = adv.branches.get(target) { - return Some((target.clone(), *hash)); - } - } - - if let Some(head_hash) = adv.head_hash { - if let Some((name, hash)) = adv.branches.iter().find(|(_, hash)| **hash == head_hash) { - return Some((name.clone(), *hash)); - } - } - - adv.branches - .iter() - .next() - .map(|(name, hash)| (name.clone(), *hash)) -} - -fn pkt_line_bytes(payload: &[u8]) -> Vec { - let mut out = format!("{:04x}", payload.len() + 4).into_bytes(); - out.extend_from_slice(payload); - out -} - -fn fetch_remote_pack(url: &str, wants: &[String]) -> io::Result> { - if wants.is_empty() { - return Ok(Vec::new()); - } - - let upload_pack_url = format!("{}/git-upload-pack", url.trim_end_matches('/')); - let mut body = Vec::new(); - for (i, want) in wants.iter().enumerate() { - let line = if i == 0 { - format!( - "want {} side-band-64k ofs-delta no-progress include-tag agent=agentos/1.0\n", - want - ) - } else { - format!("want {}\n", want) - }; - body.extend_from_slice(&pkt_line_bytes(line.as_bytes())); - } - body.extend_from_slice(b"0000"); - body.extend_from_slice(&pkt_line_bytes(b"done\n")); - - let req = Request::new(Method::Post, &upload_pack_url) - .map_err(|e| err(&format!("bad upload-pack URL: {}", e)))? - .header("Content-Type", "application/x-git-upload-pack-request") - .header("Accept", "application/x-git-upload-pack-result") - .header("Git-Protocol", "version=0") - .body(body); - - let resp = HttpClient::new() - .send(&req) - .map_err(|e| err(&format!("git-upload-pack failed: {}", e)))?; - if resp.status != 200 { - let body = response_body_preview(&resp.body); - return Err(err(&format!( - "git-upload-pack returned HTTP {}: {}", - resp.status, body - ))); - } - - if resp.body.len() > MAX_PACK_BYTES { - return Err(err("packfile exceeds size limit")); - } - if resp.body.starts_with(b"PACK") { - return Ok(resp.body); - } - - let mut pack = Vec::new(); - for line in parse_pkt_lines(&resp.body)? { - match line { - PktLine::Flush => {} - PktLine::Data(payload) => { - if payload == b"NAK\n" || payload.starts_with(b"ACK ") { - continue; - } - if payload.starts_with(b"ERR ") { - let msg = String::from_utf8_lossy(&payload[4..]); - return Err(err(&format!("remote upload-pack error: {}", msg.trim()))); - } - if payload.starts_with(b"PACK") { - append_pack_bytes(&mut pack, &payload)?; - continue; - } - if payload.is_empty() { - continue; - } - match payload[0] { - 1 => append_pack_bytes(&mut pack, &payload[1..])?, - 2 => {} - 3 => { - let msg = String::from_utf8_lossy(&payload[1..]); - return Err(err(&format!("remote upload-pack error: {}", msg.trim()))); - } - _ => return Err(err("unexpected upload-pack response payload")), - } - } - } - } - - if pack.is_empty() { - return Err(err("git-upload-pack response did not include a packfile")); - } - Ok(pack) -} - -#[derive(Clone, Debug)] -enum PackedObjectKind { - Full { obj_type: String, data: Vec }, - OfsDelta { base_offset: usize, delta: Vec }, - RefDelta { base_hash: [u8; 20], delta: Vec }, -} - -#[derive(Clone, Debug)] -struct PackedObject { - offset: usize, - kind: PackedObjectKind, -} - -#[derive(Clone, Debug)] -struct ResolvedObject { - obj_type: String, - data: Vec, - hash: [u8; 20], -} - -fn parse_pack_object_header(pack: &[u8], offset: &mut usize) -> io::Result<(u8, usize)> { - if *offset >= pack.len() { - return Err(err("truncated pack object header")); - } - - let mut byte = pack[*offset]; - *offset += 1; - - let obj_type = (byte >> 4) & 0x7; - let mut size = (byte & 0x0f) as usize; - let mut shift = 4usize; - - while byte & 0x80 != 0 { - if *offset >= pack.len() { - return Err(err("truncated pack object size")); - } - byte = pack[*offset]; - *offset += 1; - if shift >= usize::BITS as usize { - return Err(err("pack object size is too large")); - } - size |= ((byte & 0x7f) as usize) - .checked_shl(shift as u32) - .ok_or_else(|| err("pack object size is too large"))?; - shift += 7; - } - - Ok((obj_type, size)) -} - -fn parse_ofs_delta_base( - pack: &[u8], - offset: &mut usize, - object_offset: usize, -) -> io::Result { - if *offset >= pack.len() { - return Err(err("truncated ofs-delta base")); - } - - let mut byte = pack[*offset]; - *offset += 1; - let mut distance = (byte & 0x7f) as usize; - - while byte & 0x80 != 0 { - if *offset >= pack.len() { - return Err(err("truncated ofs-delta base")); - } - byte = pack[*offset]; - *offset += 1; - distance = distance - .checked_add(1) - .and_then(|value| value.checked_shl(7)) - .map(|value| value | ((byte & 0x7f) as usize)) - .ok_or_else(|| err("ofs-delta base distance is too large"))?; - } - - object_offset - .checked_sub(distance) - .ok_or_else(|| err("invalid ofs-delta base distance")) -} - -fn inflate_pack_stream(data: &[u8], expected_size: usize) -> io::Result<(Vec, usize)> { - if expected_size > MAX_GIT_OBJECT_BYTES { - return Err(err("pack object exceeds size limit")); - } - let cursor = Cursor::new(data); - let mut decoder = BufZlibDecoder::new(cursor); - let out = read_to_end_limited(&mut decoder, expected_size, "pack object")?; - if out.len() != expected_size { - return Err(err("pack object size mismatch")); - } - let consumed = decoder.get_ref().position() as usize; - Ok((out, consumed)) -} - -fn parse_packfile(pack: &[u8]) -> io::Result> { - if pack.len() < 12 + 20 { - return Err(err("packfile too small")); - } - if &pack[..4] != b"PACK" { - return Err(err("invalid packfile signature")); - } - - let version = u32::from_be_bytes(pack[4..8].try_into().unwrap()); - if version != 2 && version != 3 { - return Err(err(&format!("unsupported packfile version {}", version))); - } - let object_count = u32::from_be_bytes(pack[8..12].try_into().unwrap()) as usize; - let pack_end = pack.len() - 20; - let mut offset = 12usize; - let max_count_by_bytes = pack_end.saturating_sub(offset); - if object_count > MAX_PACK_OBJECTS || object_count > max_count_by_bytes { - return Err(err("packfile object count is too large")); - } - let mut objects = Vec::new(); - try_reserve_exact(&mut objects, object_count, "pack object table")?; - let mut inflated_bytes = 0usize; - - for _ in 0..object_count { - if offset >= pack_end { - return Err(err("truncated packfile")); - } - - let object_offset = offset; - let (obj_type, object_size) = parse_pack_object_header(pack, &mut offset)?; - - let kind = match obj_type { - 1 | 2 | 3 | 4 => { - let obj_type = match obj_type { - 1 => "commit", - 2 => "tree", - 3 => "blob", - 4 => "tag", - _ => unreachable!(), - }; - let (data, consumed) = inflate_pack_stream(&pack[offset..pack_end], object_size)?; - offset += consumed; - add_pack_inflated_bytes(&mut inflated_bytes, data.len())?; - PackedObjectKind::Full { - obj_type: obj_type.to_string(), - data, - } - } - 6 => { - let base_offset = parse_ofs_delta_base(pack, &mut offset, object_offset)?; - let (delta, consumed) = inflate_pack_stream(&pack[offset..pack_end], object_size)?; - offset += consumed; - add_pack_inflated_bytes(&mut inflated_bytes, delta.len())?; - PackedObjectKind::OfsDelta { base_offset, delta } - } - 7 => { - if offset + 20 > pack_end { - return Err(err("truncated ref-delta base")); - } - let mut base_hash = [0u8; 20]; - base_hash.copy_from_slice(&pack[offset..offset + 20]); - offset += 20; - let (delta, consumed) = inflate_pack_stream(&pack[offset..pack_end], object_size)?; - offset += consumed; - add_pack_inflated_bytes(&mut inflated_bytes, delta.len())?; - PackedObjectKind::RefDelta { base_hash, delta } - } - _ => return Err(err(&format!("unsupported pack object type {}", obj_type))), - }; - - objects.push(PackedObject { - offset: object_offset, - kind, - }); - } - - Ok(objects) -} - -fn read_delta_varint(data: &[u8], pos: &mut usize) -> io::Result { - let mut value = 0usize; - let mut shift = 0usize; - - loop { - if *pos >= data.len() { - return Err(err("truncated delta header")); - } - let byte = data[*pos]; - *pos += 1; - - if shift >= usize::BITS as usize { - return Err(err("delta varint is too large")); - } - value |= ((byte & 0x7f) as usize) - .checked_shl(shift as u32) - .ok_or_else(|| err("delta varint is too large"))?; - if byte & 0x80 == 0 { - return Ok(value); - } - shift += 7; - } -} - -fn ensure_delta_output_room( - current_len: usize, - additional_len: usize, - result_size: usize, -) -> io::Result<()> { - let next_len = current_len - .checked_add(additional_len) - .ok_or_else(|| err("delta result size overflow"))?; - if next_len > result_size { - return Err(err("delta result exceeds declared size")); - } - Ok(()) -} - -fn apply_delta(base: &[u8], delta: &[u8]) -> io::Result> { - let mut pos = 0usize; - let base_size = read_delta_varint(delta, &mut pos)?; - if base_size != base.len() { - return Err(err("delta base size mismatch")); - } - let result_size = read_delta_varint(delta, &mut pos)?; - if result_size > MAX_GIT_OBJECT_BYTES { - return Err(err("delta result exceeds size limit")); - } - let mut out = Vec::new(); - try_reserve_exact(&mut out, result_size, "delta result")?; - - while pos < delta.len() { - let opcode = delta[pos]; - pos += 1; - - if opcode & 0x80 != 0 { - let mut copy_offset = 0usize; - let mut copy_size = 0usize; - - if opcode & 0x01 != 0 { - copy_offset |= delta - .get(pos) - .copied() - .ok_or_else(|| err("truncated delta copy"))? - as usize; - pos += 1; - } - if opcode & 0x02 != 0 { - copy_offset |= (delta - .get(pos) - .copied() - .ok_or_else(|| err("truncated delta copy"))? - as usize) - << 8; - pos += 1; - } - if opcode & 0x04 != 0 { - copy_offset |= (delta - .get(pos) - .copied() - .ok_or_else(|| err("truncated delta copy"))? - as usize) - << 16; - pos += 1; - } - if opcode & 0x08 != 0 { - copy_offset |= (delta - .get(pos) - .copied() - .ok_or_else(|| err("truncated delta copy"))? - as usize) - << 24; - pos += 1; - } - - if opcode & 0x10 != 0 { - copy_size |= delta - .get(pos) - .copied() - .ok_or_else(|| err("truncated delta copy"))? - as usize; - pos += 1; - } - if opcode & 0x20 != 0 { - copy_size |= (delta - .get(pos) - .copied() - .ok_or_else(|| err("truncated delta copy"))? - as usize) - << 8; - pos += 1; - } - if opcode & 0x40 != 0 { - copy_size |= (delta - .get(pos) - .copied() - .ok_or_else(|| err("truncated delta copy"))? - as usize) - << 16; - pos += 1; - } - if copy_size == 0 { - copy_size = 0x10000; - } - - let end = copy_offset - .checked_add(copy_size) - .ok_or_else(|| err("delta copy overflow"))?; - if end > base.len() { - return Err(err("delta copy exceeds base object")); - } - ensure_delta_output_room(out.len(), copy_size, result_size)?; - out.extend_from_slice(&base[copy_offset..end]); - } else if opcode != 0 { - let insert_len = opcode as usize; - let end = pos - .checked_add(insert_len) - .ok_or_else(|| err("delta insert overflow"))?; - if end > delta.len() { - return Err(err("truncated delta insert")); - } - ensure_delta_output_room(out.len(), insert_len, result_size)?; - out.extend_from_slice(&delta[pos..end]); - pos = end; - } else { - return Err(err("invalid delta opcode")); - } - } - - if out.len() != result_size { - return Err(err("delta result size mismatch")); - } - - Ok(out) -} - -fn maybe_read_local_object(git_dir: &Path, hash: &[u8; 20]) -> io::Result> { - let h = hex(hash); - let path = git_dir.join("objects").join(&h[..2]).join(&h[2..]); - if !path.exists() { - return Ok(None); - } - let (obj_type, data) = read_object(git_dir, hash)?; - Ok(Some(ResolvedObject { - obj_type, - data, - hash: *hash, - })) -} - -fn find_entry_by_hash( - target: &[u8; 20], - git_dir: &Path, - objects: &[PackedObject], - offset_to_index: &HashMap, - memo: &mut [Option], - visiting: &mut [bool], - resolved_bytes: &mut usize, -) -> io::Result> { - for idx in 0..objects.len() { - if visiting[idx] { - continue; - } - let resolved = resolve_packed_object( - idx, - git_dir, - objects, - offset_to_index, - memo, - visiting, - resolved_bytes, - )?; - if resolved.hash == *target { - return Ok(Some(idx)); - } - } - - Ok(None) -} - -fn resolve_packed_object( - idx: usize, - git_dir: &Path, - objects: &[PackedObject], - offset_to_index: &HashMap, - memo: &mut [Option], - visiting: &mut [bool], - resolved_bytes: &mut usize, -) -> io::Result { - if let Some(resolved) = memo[idx].as_ref() { - return Ok(resolved.clone()); - } - if visiting[idx] { - return Err(err("cyclic pack delta dependency")); - } - visiting[idx] = true; - - let resolved = match &objects[idx].kind { - PackedObjectKind::Full { obj_type, data } => ResolvedObject { - obj_type: obj_type.clone(), - data: data.clone(), - hash: hash_bytes(obj_type, data), - }, - PackedObjectKind::OfsDelta { base_offset, delta } => { - let base_idx = *offset_to_index - .get(base_offset) - .ok_or_else(|| err("missing ofs-delta base object"))?; - let base = resolve_packed_object( - base_idx, - git_dir, - objects, - offset_to_index, - memo, - visiting, - resolved_bytes, - )?; - let data = apply_delta(&base.data, delta)?; - let hash = hash_bytes(&base.obj_type, &data); - ResolvedObject { - obj_type: base.obj_type, - data, - hash, - } - } - PackedObjectKind::RefDelta { base_hash, delta } => { - let base = if let Some(local) = maybe_read_local_object(git_dir, base_hash)? { - local - } else if let Some(base_idx) = find_entry_by_hash( - base_hash, - git_dir, - objects, - offset_to_index, - memo, - visiting, - resolved_bytes, - )? { - resolve_packed_object( - base_idx, - git_dir, - objects, - offset_to_index, - memo, - visiting, - resolved_bytes, - )? - } else { - return Err(err("missing ref-delta base object")); - }; - - let data = apply_delta(&base.data, delta)?; - let hash = hash_bytes(&base.obj_type, &data); - ResolvedObject { - obj_type: base.obj_type, - data, - hash, - } - } - }; - - visiting[idx] = false; - add_pack_resolved_bytes(resolved_bytes, resolved.data.len())?; - memo[idx] = Some(resolved.clone()); - Ok(resolved) -} - -fn store_pack_objects(git_dir: &Path, pack: &[u8]) -> io::Result<()> { - if pack.is_empty() { - return Ok(()); - } - - let objects = parse_packfile(pack)?; - let offset_to_index: HashMap = objects - .iter() - .enumerate() - .map(|(idx, obj)| (obj.offset, idx)) - .collect(); - let mut memo: Vec> = vec![None; objects.len()]; - let mut visiting = vec![false; objects.len()]; - let mut resolved_bytes = 0usize; - - for idx in 0..objects.len() { - let resolved = resolve_packed_object( - idx, - git_dir, - &objects, - &offset_to_index, - &mut memo, - &mut visiting, - &mut resolved_bytes, - )?; - let stored = hash_object(git_dir, &resolved.obj_type, &resolved.data)?; - if stored != resolved.hash { - return Err(err("pack object hash mismatch")); - } - } - Ok(()) -} - -fn cmd_clone_remote(source: &str, dest: &Path) -> io::Result<()> { - if dest.exists() && !dir_is_empty(dest)? { - return Err(err(&format!( - "destination path '{}' already exists and is not an empty directory", - dest.display() - ))); - } - - let advertisement = fetch_remote_advertisement(source)?; - let mut wants: Vec = Vec::new(); - let mut seen = HashSet::new(); - for hash in advertisement.branches.values() { - if !is_zero_oid(hash) { - let hash_hex = hex(hash); - if seen.insert(hash_hex.clone()) { - wants.push(hash_hex); - } - } - } - if wants.is_empty() { - if let Some(head_hash) = advertisement.head_hash { - if !is_zero_oid(&head_hash) { - wants.push(hex(&head_hash)); - } - } - } - - let default_ref = if let Some((refname, hash)) = default_branch_ref(&advertisement) { - Some((refname.clone(), branch_name_from_ref(&refname)?, hash)) - } else { - None - }; - let default_branch = default_ref - .as_ref() - .map(|(_, branch, _)| branch.clone()) - .or_else(|| { - advertisement - .head_target - .as_ref() - .and_then(|target| branch_name_from_ref(target).ok()) - }) - .unwrap_or_else(|| "main".to_string()); - - let pack = fetch_remote_pack(source, &wants)?; - - let dst_git = prepare_clone_destination(dest, source, &default_branch)?; - store_pack_objects(&dst_git, &pack)?; - - for (refname, hash) in &advertisement.branches { - if is_zero_oid(hash) { - continue; - } - let branch = branch_name_from_ref(refname)?; - update_ref(&dst_git, &format!("refs/remotes/origin/{}", branch), hash)?; - } - - for (refname, hash) in &advertisement.tags { - if is_zero_oid(hash) { - continue; - } - update_ref(&dst_git, refname, hash)?; - } - - let default_hash = default_ref - .as_ref() - .map(|(_, _, hash)| *hash) - .or(advertisement.head_hash) - .filter(|hash| !is_zero_oid(hash)); - - if let Some(hash) = default_hash { - update_ref(&dst_git, &format!("refs/heads/{}", default_branch), &hash)?; - cmd_checkout(dest, &default_branch, false)?; - } - - Ok(()) -} - -// ─── Object store ─────────────────────────────────────────────────────────── - -fn hash_object(git_dir: &Path, obj_type: &str, data: &[u8]) -> io::Result<[u8; 20]> { - if data.len() > MAX_GIT_OBJECT_BYTES { - return Err(err("git object exceeds size limit")); - } - let header = format!("{} {}\0", obj_type, data.len()); - let mut hasher = Sha1::new(); - hasher.update(header.as_bytes()); - hasher.update(data); - let hash: [u8; 20] = hasher.finalize().into(); - - let h = hex(&hash); - let dir = git_dir.join("objects").join(&h[..2]); - let path = dir.join(&h[2..]); - if !path.exists() { - mkdirs(&dir)?; - let mut enc = ZlibEncoder::new(Vec::new(), Compression::default()); - enc.write_all(header.as_bytes())?; - enc.write_all(data)?; - fs::write(&path, enc.finish()?)?; - } - Ok(hash) -} - -fn read_object(git_dir: &Path, hash: &[u8; 20]) -> io::Result<(String, Vec)> { - let h = hex(hash); - let path = git_dir.join("objects").join(&h[..2]).join(&h[2..]); - let read_limit = MAX_GIT_OBJECT_BYTES - .checked_add(MAX_GIT_OBJECT_HEADER_BYTES) - .ok_or_else(|| err("git object size limit is too large"))?; - let compressed = read_file_limited(&path, read_limit, "git object")?; - let mut dec = ZlibDecoder::new(&compressed[..]); - let buf = read_to_end_limited(&mut dec, read_limit, "git object")?; - - let nul = buf - .iter() - .position(|&b| b == 0) - .ok_or_else(|| err("no nul in object"))?; - let header = - std::str::from_utf8(&buf[..nul]).map_err(|_| err("invalid object header encoding"))?; - let (typ, size) = header - .split_once(' ') - .ok_or_else(|| err("malformed object header"))?; - let size: usize = size.parse().map_err(|_| err("invalid object size"))?; - if size > MAX_GIT_OBJECT_BYTES { - return Err(err("git object exceeds size limit")); - } - if buf.len() - nul - 1 != size { - return Err(err("git object size mismatch")); - } - Ok((typ.to_string(), buf[nul + 1..].to_vec())) -} - -// ─── Index ────────────────────────────────────────────────────────────────── - -#[derive(Clone, Debug)] -struct IndexEntry { - mode: u32, - sha1: [u8; 20], - name: String, -} - -fn read_index(git_dir: &Path) -> io::Result> { - let path = git_dir.join("index"); - if !path.exists() { - return Ok(Vec::new()); - } - let data = read_file_limited(&path, MAX_INDEX_BYTES, "git index")?; - if data.len() < 12 || &data[0..4] != b"DIRC" { - return Err(err("invalid index file")); - } - let version = u32::from_be_bytes(data[4..8].try_into().unwrap()); - if version != 2 { - return Err(err(&format!("unsupported index version {}", version))); - } - let count = u32::from_be_bytes(data[8..12].try_into().unwrap()) as usize; - let max_count_by_bytes = data.len().saturating_sub(12) / 62; - if count > MAX_INDEX_ENTRIES || count > max_count_by_bytes { - return Err(err("index entry count is too large")); - } - - let mut entries = Vec::new(); - try_reserve_exact(&mut entries, count, "index entry table")?; - let mut pos = 12; - - for _ in 0..count { - if pos + 62 > data.len() { - return Err(err("truncated index")); - } - // Stat fields at pos+0..pos+24 (ctime, mtime, dev, ino) - skip - let mode = u32::from_be_bytes(data[pos + 24..pos + 28].try_into().unwrap()); - // Skip uid(4), gid(4), size(4) at pos+28..pos+40 - let mut sha1 = [0u8; 20]; - sha1.copy_from_slice(&data[pos + 40..pos + 60]); - // Flags at pos+60..pos+62 - // Find name (NUL-terminated after fixed 62 bytes) - let name_start = pos + 62; - let nul_offset = data[name_start..] - .iter() - .position(|&b| b == 0) - .ok_or_else(|| err("unterminated index entry name"))?; - let name = String::from_utf8(data[name_start..name_start + nul_offset].to_vec()) - .map_err(|_| err("invalid entry name"))?; - let name = normalize_repo_path(&name)?; - - entries.push(IndexEntry { mode, sha1, name }); - - // Advance past padding: entry padded to 8-byte boundary - let entry_len = 62 + nul_offset; - pos += (entry_len + 8) & !7; - } - - Ok(entries) -} - -fn write_index(git_dir: &Path, entries: &[IndexEntry]) -> io::Result<()> { - let entry_count = u32::try_from(entries.len()).map_err(|_| err("too many index entries"))?; - let mut buf = Vec::new(); - buf.extend_from_slice(b"DIRC"); - buf.extend_from_slice(&2u32.to_be_bytes()); - buf.extend_from_slice(&entry_count.to_be_bytes()); - - for entry in entries { - let name = normalize_repo_path(&entry.name)?; - if name.len() > 0xFFF { - return Err(err("index entry name is too long")); - } - let entry_start = buf.len(); - // ctime(8) + mtime(8) + dev(4) + ino(4) = 24 bytes of zeros - buf.extend_from_slice(&[0u8; 24]); - buf.extend_from_slice(&entry.mode.to_be_bytes()); - // uid(4) + gid(4) + size(4) = 12 bytes of zeros - buf.extend_from_slice(&[0u8; 12]); - buf.extend_from_slice(&entry.sha1); - // Flags: name length in lower 12 bits - buf.extend_from_slice(&(name.len() as u16).to_be_bytes()); - buf.extend_from_slice(name.as_bytes()); - // Pad to 8-byte boundary (1-8 NUL bytes) - let entry_len = buf.len() - entry_start; - let padded = (entry_len + 8) & !7; - buf.resize(entry_start + padded, 0); - } - - // SHA-1 checksum of entire index - let checksum: [u8; 20] = Sha1::digest(&buf).into(); - buf.extend_from_slice(&checksum); - - fs::write(git_dir.join("index"), &buf) -} - -// ─── Refs ─────────────────────────────────────────────────────────────────── - -fn resolve_ref(git_dir: &Path, refname: &str) -> io::Result> { - let refname = validate_refname(refname)?; - let ref_path = git_dir.join(refname); - if !ref_path.exists() { - return Ok(None); - } - let content = fs::read_to_string(&ref_path)?; - let content = content.trim(); - if let Some(target) = content.strip_prefix("ref: ") { - resolve_ref(git_dir, target) - } else { - Ok(Some(unhex(content)?)) - } -} - -fn resolve_head(git_dir: &Path) -> io::Result> { - resolve_ref(git_dir, "HEAD") -} - -fn head_branch(git_dir: &Path) -> io::Result> { - let head = fs::read_to_string(git_dir.join("HEAD"))?; - let head = head.trim(); - Ok(head.strip_prefix("ref: refs/heads/").map(|s| s.to_string())) -} - -fn update_ref(git_dir: &Path, refname: &str, hash: &[u8; 20]) -> io::Result<()> { - let refname = validate_refname(refname)?; - let ref_path = git_dir.join(refname); - if let Some(parent) = ref_path.parent() { - mkdirs(parent)?; - } - fs::write(&ref_path, format!("{}\n", hex(hash))) -} - -// ─── Tree operations ──────────────────────────────────────────────────────── - -fn build_tree(git_dir: &Path, entries: &[IndexEntry]) -> io::Result<[u8; 20]> { - build_tree_at(git_dir, entries, "") -} - -fn build_tree_at(git_dir: &Path, entries: &[IndexEntry], prefix: &str) -> io::Result<[u8; 20]> { - struct TreeEntry { - mode: String, - name: String, - hash: [u8; 20], - } - - let mut tree_entries: Vec = Vec::new(); - let mut subdirs: BTreeMap> = BTreeMap::new(); - - for entry in entries { - let relative = if prefix.is_empty() { - &entry.name[..] - } else if let Some(rest) = entry.name.strip_prefix(prefix) { - rest - } else { - continue; - }; - - if let Some(slash) = relative.find('/') { - let dir = &relative[..slash]; - subdirs - .entry(dir.to_string()) - .or_default() - .push(entry.clone()); - } else { - tree_entries.push(TreeEntry { - mode: format!("{:o}", entry.mode), - name: relative.to_string(), - hash: entry.sha1, - }); - } - } - - for (dir, sub_entries) in &subdirs { - let sub_prefix = if prefix.is_empty() { - format!("{}/", dir) - } else { - format!("{}{}/", prefix, dir) - }; - let subtree_hash = build_tree_at(git_dir, sub_entries, &sub_prefix)?; - tree_entries.push(TreeEntry { - mode: "40000".to_string(), - name: dir.clone(), - hash: subtree_hash, - }); - } - - // Sort: directories get trailing / for comparison - tree_entries.sort_by(|a, b| { - let ak = if a.mode == "40000" { - format!("{}/", a.name) - } else { - a.name.clone() - }; - let bk = if b.mode == "40000" { - format!("{}/", b.name) - } else { - b.name.clone() - }; - ak.cmp(&bk) - }); - - let mut data = Vec::new(); - for te in &tree_entries { - data.extend_from_slice(te.mode.as_bytes()); - data.push(b' '); - data.extend_from_slice(te.name.as_bytes()); - data.push(0); - data.extend_from_slice(&te.hash); - } - - hash_object(git_dir, "tree", &data) -} - -/// Read a tree recursively into flat (path, mode, hash) entries. -fn read_tree_entries(git_dir: &Path, hash: &[u8; 20], prefix: &str) -> io::Result> { - let (typ, data) = read_object(git_dir, hash)?; - if typ != "tree" { - return Err(err("expected tree object")); - } - - let mut entries = Vec::new(); - let mut pos = 0; - - while pos < data.len() { - let space = data[pos..] - .iter() - .position(|&b| b == b' ') - .ok_or_else(|| err("bad tree entry"))?; - let mode_str = - std::str::from_utf8(&data[pos..pos + space]).map_err(|_| err("bad tree mode"))?; - let mode = u32::from_str_radix(mode_str, 8).map_err(|_| err("bad tree mode number"))?; - pos += space + 1; - - let nul = data[pos..] - .iter() - .position(|&b| b == 0) - .ok_or_else(|| err("bad tree entry name"))?; - let name = std::str::from_utf8(&data[pos..pos + nul]).map_err(|_| err("bad name"))?; - if name.contains('/') { - return Err(err("tree entry name must not contain '/'")); - } - if normalize_repo_path(name)? != name { - return Err(err("tree entry name must be normalized")); - } - pos += nul + 1; - - if pos + 20 > data.len() { - return Err(err("truncated tree hash")); - } - let mut hash_buf = [0u8; 20]; - hash_buf.copy_from_slice(&data[pos..pos + 20]); - pos += 20; - - let full_name = if prefix.is_empty() { - name.to_string() - } else { - format!("{}{}", prefix, name) - }; - - if mode == 0o40000 { - let sub = read_tree_entries(git_dir, &hash_buf, &format!("{}/", full_name))?; - entries.extend(sub); - } else { - let full_name = normalize_repo_path(&full_name)?; - entries.push(IndexEntry { - mode, - sha1: hash_buf, - name: full_name, - }); - } - } - - Ok(entries) -} - -/// Extract the tree hash from a commit object. -fn commit_tree(git_dir: &Path, commit_hash: &[u8; 20]) -> io::Result<[u8; 20]> { - let (typ, data) = read_object(git_dir, commit_hash)?; - if typ != "commit" { - return Err(err("not a commit object")); - } - let text = String::from_utf8_lossy(&data); - let tree_hex = text - .lines() - .find_map(|l| l.strip_prefix("tree ")) - .ok_or_else(|| err("no tree line in commit"))?; - unhex(tree_hex) -} - -// ─── Commands ─────────────────────────────────────────────────────────────── - -fn cmd_init(path: &Path, quiet: bool) -> io::Result<()> { - let git_dir = path.join(".git"); - // Create directories one at a time to avoid create_dir_all issues on WASI - for dir in &[ - path.to_path_buf(), - git_dir.clone(), - git_dir.join("objects"), - git_dir.join("refs"), - git_dir.join("refs/heads"), - git_dir.join("refs/tags"), - ] { - match fs::create_dir(dir) { - Ok(()) => {} - Err(e) if e.kind() == io::ErrorKind::AlreadyExists => {} - Err(e) => return Err(err(&format!("mkdir {}: {}", dir.display(), e))), - } - } - fs::write(git_dir.join("HEAD"), "ref: refs/heads/main\n") - .map_err(|e| err(&format!("write HEAD: {}", e)))?; - fs::write( - git_dir.join("config"), - "[core]\n\trepositoryformatversion = 0\n\tfilemode = true\n\tbare = false\n", - ) - .map_err(|e| err(&format!("write config: {}", e)))?; - if !quiet { - print_stdout_line(format_args!( - "Initialized empty Git repository in {}/.git/", - path.display() - ))?; - } - Ok(()) -} - -fn collect_add_paths(workdir: &Path, rel_path: &str, out: &mut Vec) -> io::Result<()> { - if rel_path == ".git" || rel_path.starts_with(".git/") || rel_path.ends_with("/.git") { - return Ok(()); - } - - if rel_path == "." { - for entry in fs::read_dir(workdir)? { - let entry = entry?; - let name = entry - .file_name() - .to_str() - .ok_or_else(|| err("repository path must be utf-8"))? - .to_owned(); - if name == ".git" { - continue; - } - collect_add_paths(workdir, &name, out)?; - } - return Ok(()); - } - - let repo_path = normalize_repo_path(rel_path)?; - let file_path = workdir.join(&repo_path); - if file_path.is_dir() { - for entry in fs::read_dir(&file_path)? { - let entry = entry?; - let name = entry - .file_name() - .to_str() - .ok_or_else(|| err("repository path must be utf-8"))? - .to_owned(); - if name == ".git" { - continue; - } - collect_add_paths(workdir, &format!("{repo_path}/{name}"), out)?; - } - return Ok(()); - } - - if file_path.exists() { - out.push(repo_path); - } - Ok(()) -} - -fn cmd_add(workdir: &Path, paths: &[String]) -> io::Result<()> { - let git_dir = workdir.join(".git"); - let mut entries = read_index(&git_dir).map_err(|e| { - err(&format!( - "cannot read index at {}: {}", - git_dir.display(), - e - )) - })?; - - for rel_path in paths { - let mut repo_paths = Vec::new(); - collect_add_paths(workdir, rel_path, &mut repo_paths)?; - if repo_paths.is_empty() { - return Err(err(&format!( - "pathspec '{}' did not match any files", - rel_path - ))); - } - for repo_path in repo_paths { - let file_path = workdir.join(&repo_path); - if !file_path.exists() { - return Err(err(&format!( - "pathspec '{}' did not match any files (looked at {})", - rel_path, - file_path.display() - ))); - } - let content = read_file_limited(&file_path, MAX_GIT_OBJECT_BYTES, "file")?; - let hash = hash_object(&git_dir, "blob", &content)?; - - entries.retain(|e| e.name != repo_path); - entries.push(IndexEntry { - mode: 0o100644, - sha1: hash, - name: repo_path, - }); - } - } - - entries.sort_by(|a, b| a.name.cmp(&b.name)); - write_index(&git_dir, &entries) -} - -fn cmd_commit(workdir: &Path, message: &str) -> io::Result<()> { - let git_dir = workdir.join(".git"); - let entries = read_index(&git_dir)?; - if entries.is_empty() { - return Err(err("nothing to commit")); - } - - let tree_hash = build_tree(&git_dir, &entries)?; - let parent = resolve_head(&git_dir)?; - - let name = std::env::var("GIT_AUTHOR_NAME") - .or_else(|_| std::env::var("GIT_COMMITTER_NAME")) - .unwrap_or_else(|_| "secure-exec".to_string()); - let email = std::env::var("GIT_AUTHOR_EMAIL") - .or_else(|_| std::env::var("GIT_COMMITTER_EMAIL")) - .unwrap_or_else(|_| "agent@os".to_string()); - let timestamp = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_secs()) - .unwrap_or(1700000000); - - let mut commit = String::new(); - commit.push_str(&format!("tree {}\n", hex(&tree_hash))); - if let Some(p) = parent { - commit.push_str(&format!("parent {}\n", hex(&p))); - } - commit.push_str(&format!( - "author {} <{}> {} +0000\n", - name, email, timestamp - )); - commit.push_str(&format!( - "committer {} <{}> {} +0000\n", - name, email, timestamp - )); - commit.push_str(&format!("\n{}\n", message)); - - let commit_hash = hash_object(&git_dir, "commit", commit.as_bytes())?; - - // Update the ref that HEAD points to - let head_content = fs::read_to_string(git_dir.join("HEAD"))?; - let head_content = head_content.trim(); - if let Some(refname) = head_content.strip_prefix("ref: ") { - update_ref(&git_dir, refname, &commit_hash)?; - } else { - fs::write(git_dir.join("HEAD"), format!("{}\n", hex(&commit_hash)))?; - } - - Ok(()) -} - -fn cmd_rev_parse(workdir: &Path, args: &[String]) -> io::Result<()> { - let git_dir = workdir.join(".git"); - if args.len() != 1 { - return Err(err("usage: git rev-parse ")); - } - - let hash = resolve_ref(&git_dir, &args[0])?.ok_or_else(|| { - err(&format!( - "unknown revision or path not in the working tree: {}", - args[0] - )) - })?; - print_stdout_line(format_args!("{}", hex(&hash))) -} - -fn cmd_branch(workdir: &Path) -> io::Result<()> { - let git_dir = workdir.join(".git"); - let heads_dir = git_dir.join("refs/heads"); - let current = head_branch(&git_dir)?; - - let mut branches: Vec = Vec::new(); - if heads_dir.exists() { - for entry in fs::read_dir(&heads_dir)? { - let entry = entry?; - let name = entry.file_name().to_string_lossy().to_string(); - if name == "." || name == ".." { - continue; - } - if entry.file_type()?.is_file() { - branches.push(name); - } - } - } - branches.sort(); - - for branch in &branches { - if Some(branch.as_str()) == current.as_deref() { - print_stdout_line(format_args!("* {}", branch))?; - } else { - print_stdout_line(format_args!(" {}", branch))?; - } - } - - Ok(()) -} - -fn cmd_checkout(workdir: &Path, target: &str, create_branch: bool) -> io::Result<()> { - let git_dir = workdir.join(".git"); - - if create_branch { - validate_branch_name(target)?; - let head = resolve_head(&git_dir)?.ok_or_else(|| err("HEAD not found for new branch"))?; - update_ref(&git_dir, &format!("refs/heads/{}", target), &head)?; - fs::write( - git_dir.join("HEAD"), - format!("ref: refs/heads/{}\n", target), - )?; - return Ok(()); - } - - // Resolve target: local branch first, then DWIM remote tracking - validate_branch_name(target)?; - let branch_ref = format!("refs/heads/{}", target); - let commit_hash = if let Some(h) = resolve_ref(&git_dir, &branch_ref)? { - fs::write(git_dir.join("HEAD"), format!("ref: {}\n", branch_ref))?; - h - } else { - let remote_ref = format!("refs/remotes/origin/{}", target); - if let Some(h) = resolve_ref(&git_dir, &remote_ref)? { - update_ref(&git_dir, &branch_ref, &h)?; - fs::write(git_dir.join("HEAD"), format!("ref: {}\n", branch_ref))?; - h - } else { - return Err(err(&format!( - "pathspec '{}' did not match any branch", - target - ))); - } - }; - - // Read target tree - let tree_hash = commit_tree(&git_dir, &commit_hash)?; - let new_entries = read_tree_entries(&git_dir, &tree_hash, "")?; - - // Clean up files from current index that aren't in target - let old_entries = read_index(&git_dir)?; - let new_names: HashSet<&str> = new_entries.iter().map(|e| e.name.as_str()).collect(); - for old in &old_entries { - if !new_names.contains(old.name.as_str()) { - let p = worktree_path(workdir, &old.name)?; - if p.exists() { - fs::remove_file(&p).map_err(|e| err(&format!("remove {}: {}", p.display(), e)))?; - } - } - } - - // Write files from target tree - for entry in &new_entries { - let p = worktree_path(workdir, &entry.name)?; - if let Some(parent) = p.parent() { - mkdirs(parent)?; - } - let (_, blob) = read_object(&git_dir, &entry.sha1)?; - fs::write(&p, &blob)?; - } - - // Update index - write_index(&git_dir, &new_entries)?; - - Ok(()) -} - -fn cmd_clone_local(source: &Path, dest: &Path) -> io::Result<()> { - let src_git = source.join(".git"); - if !src_git.is_dir() { - return Err(err(&format!( - "'{}' does not appear to be a git repository", - source.display() - ))); - } - - if dest.exists() && !dir_is_empty(dest)? { - return Err(err(&format!( - "destination path '{}' already exists and is not an empty directory", - dest.display() - ))); - } - - mkdirs(dest)?; - - let dst_git = dest.join(".git"); - - // Init destination - mkdirs(&dst_git.join("objects"))?; - mkdirs(&dst_git.join("refs/heads"))?; - mkdirs(&dst_git.join("refs/tags"))?; - mkdirs(&dst_git.join("refs/remotes/origin"))?; - - // Copy all objects - copy_dir_recursive(&src_git.join("objects"), &dst_git.join("objects"))?; - - // Create remote tracking branches from source heads - let src_heads = src_git.join("refs/heads"); - if src_heads.exists() { - copy_dir_recursive(&src_heads, &dst_git.join("refs/remotes/origin"))?; - } - - // Determine default branch from source HEAD - let src_head = fs::read_to_string(src_git.join("HEAD"))?; - let src_head = src_head.trim(); - let default_branch = src_head - .strip_prefix("ref: refs/heads/") - .unwrap_or("main") - .to_string(); - validate_branch_name(&default_branch)?; - - // Create local branch for default - let remote_ref = format!("refs/remotes/origin/{}", default_branch); - let mut has_default_branch = false; - if let Some(hash) = resolve_ref(&dst_git, &remote_ref)? { - update_ref(&dst_git, &format!("refs/heads/{}", default_branch), &hash)?; - has_default_branch = true; - } - - // Set HEAD and write config - fs::write( - dst_git.join("HEAD"), - format!("ref: refs/heads/{}\n", default_branch), - )?; - fs::write( - dst_git.join("config"), - format!( - "[core]\n\trepositoryformatversion = 0\n\tfilemode = true\n\tbare = false\n\ - [remote \"origin\"]\n\turl = {}\n\tfetch = +refs/heads/*:refs/remotes/origin/*\n", - source.display() - ), - )?; - - // Checkout working tree - if has_default_branch { - cmd_checkout(dest, &default_branch, false)?; - } - - Ok(()) -} - -fn copy_dir_recursive(src: &Path, dst: &Path) -> io::Result<()> { - mkdirs(dst)?; - if !src.exists() { - return Ok(()); - } - for entry in fs::read_dir(src)? { - let entry = entry?; - let name = entry.file_name().to_string_lossy().to_string(); - if name == "." || name == ".." { - continue; - } - let dst_path = dst.join(entry.file_name()); - if entry.file_type()?.is_dir() { - copy_dir_recursive(&entry.path(), &dst_path)?; - } else { - let content = read_file_limited( - &entry.path(), - MAX_GIT_OBJECT_BYTES + MAX_GIT_OBJECT_HEADER_BYTES, - "git repository file", - )?; - fs::write(&dst_path, content)?; - } - } - Ok(()) -} - -// ─── Entry point ──────────────────────────────────────────────────────────── - -pub fn main(args: Vec) -> i32 { - let str_args: Vec = args - .iter() - .map(|a| a.to_string_lossy().to_string()) - .collect(); - match run(&str_args) { - Ok(()) => 0, - Err(e) => { - eprintln!("fatal: {}", e); - 128 - } - } -} - -fn run(args: &[String]) -> io::Result<()> { - let mut i = 1; // skip argv[0] - let mut workdir = std::env::current_dir().unwrap_or_else(|_| PathBuf::from("/")); - - // Parse global options - while i < args.len() { - if args[i] == "-C" { - i += 1; - if i >= args.len() { - return Err(err("-C requires a directory argument")); - } - let p = PathBuf::from(&args[i]); - workdir = if p.is_absolute() { p } else { workdir.join(p) }; - i += 1; - } else if args[i] == "-c" { - i += 1; - if i >= args.len() { - return Err(err("-c requires a config assignment")); - } - i += 1; - } else { - break; - } - } - - if i >= args.len() { - eprintln!("usage: git []"); - return Err(err("no subcommand")); - } - - let subcmd = &args[i]; - let sub_args = &args[i + 1..]; - - match subcmd.as_str() { - "init" => { - let mut path_arg = None; - let mut quiet = false; - for arg in sub_args { - if arg == "-q" || arg == "--quiet" { - quiet = true; - continue; - } - if arg.starts_with('-') { - return Err(unsupported("init", &format!("does not support `{}`.", arg))); - } - if path_arg.replace(arg).is_some() { - return Err(err("usage: git init []")); - } - } - let path = if let Some(path_arg) = path_arg { - let p = PathBuf::from(path_arg); - if p.is_absolute() { - p - } else { - workdir.join(p) - } - } else { - workdir - }; - cmd_init(&path, quiet) - } - "add" => { - if sub_args.is_empty() { - return Err(err("nothing specified, nothing added")); - } - let paths: Vec = sub_args.iter().map(|s| s.to_string()).collect(); - cmd_add(&workdir, &paths) - } - "commit" => { - let mut message = None; - let mut j = 0; - while j < sub_args.len() { - if sub_args[j] == "-m" && j + 1 < sub_args.len() { - message = Some(sub_args[j + 1].clone()); - j += 2; - } else { - j += 1; - } - } - let msg = message.ok_or_else(|| err("no commit message (-m)"))?; - cmd_commit(&workdir, &msg) - } - "branch" => cmd_branch(&workdir), - "rev-parse" => cmd_rev_parse(&workdir, sub_args), - "checkout" => { - let mut create = false; - let mut target = None; - for arg in sub_args { - if arg == "-b" { - create = true; - } else if !arg.starts_with('-') { - target = Some(arg.clone()); - } - } - let t = target.ok_or_else(|| err("no branch name specified"))?; - cmd_checkout(&workdir, &t, create) - } - "clone" => { - if sub_args.is_empty() || sub_args.len() > 2 { - return Err(err("usage: git clone []")); - } - let src_arg = &sub_args[0]; - if is_ssh_clone_source(src_arg) { - return Err(unsupported( - "clone", - &format!("does not support SSH or git:// remotes (`{}`).", src_arg), - )); - } - if has_http_auth(src_arg) { - return Err(unsupported( - "clone", - &format!( - "does not support authenticated HTTP(S) remotes (`{}`).", - src_arg - ), - )); - } - let dst_arg = if sub_args.len() == 2 { - sub_args[1].clone() - } else { - infer_clone_destination(src_arg)? - }; - let dst = PathBuf::from(&dst_arg); - let dst = if dst.is_absolute() { - dst - } else { - workdir.join(dst) - }; - print_stdout_line(format_args!("Cloning into '{}'...", dst.display()))?; - if is_remote_source(src_arg) { - cmd_clone_remote(src_arg, &dst) - } else { - let src = PathBuf::from(src_arg); - let src = if src.is_absolute() { - src - } else { - workdir.join(src) - }; - cmd_clone_local(&src, &dst) - } - } - other => Err(unsupported( - other, - "is not implemented in the secure-exec VM git command.", - )), - } -} diff --git a/software/git/package.json b/software/git/package.json index 02d3316064..c37a93b30f 100644 --- a/software/git/package.json +++ b/software/git/package.json @@ -2,8 +2,8 @@ "name": "@agentos-software/git", "version": "0.3.3", "type": "module", - "license": "Apache-2.0", - "description": "git version control for secure-exec VMs (planned)", + "license": "GPL-2.0-only", + "description": "Upstream Git version control for AgentOS VMs", "main": "./dist/index.js", "types": "./dist/index.d.ts", "files": [ diff --git a/software/git/test/git.test.ts b/software/git/test/git.test.ts index b2d4aed682..2516b7dc14 100644 --- a/software/git/test/git.test.ts +++ b/software/git/test/git.test.ts @@ -27,6 +27,19 @@ vi.setConfig({ testTimeout: 30_000 }); /** Check git binary exists in addition to base WASM binaries */ const hasGit = hasWasmBinaries && existsSync(resolve(COMMANDS_DIR, 'git')); const hasHostGit = spawnSync('git', ['--version'], { stdio: 'ignore' }).status === 0; +// Smart HTTP needs Git's libcurl-backed remote helpers; this WASM build is NO_CURL. +const hasGitHttpHelper = false; + +const gitConfig = [ + '-c safe.directory=*', + '-c init.defaultBranch=main', + '-c user.name=agentos', + '-c user.email=agentos@example.invalid', +].join(' '); + +function git(args: string) { + return `git ${gitConfig} ${args}`; +} /** Create a kernel with a world-writable in-memory filesystem */ async function createGitKernel() { @@ -76,6 +89,11 @@ async function run(kernel: Kernel, cmd: string): Promise<{ stdout: string; stder return r; } +async function expectGitRef(kernel: Kernel, repo: string, ref: string) { + const result = await run(kernel, git(`-C ${repo} rev-parse --verify ${ref}`)); + expect(result.stdout.trim()).toMatch(/^[0-9a-f]{40,64}$/); +} + // TODO(P6): requires git WASM artifact, intentionally excluded from the fast software-build gate. describeIf(hasGit, 'git command', () => { let kernel: Kernel; @@ -89,7 +107,7 @@ describeIf(hasGit, 'git command', () => { it('init creates .git directory structure', async () => { ({ kernel, vfs, dispose } = await createGitKernel()); - const result = await run(kernel, 'git init /repo'); + const result = await run(kernel, git('init /repo')); expect(result.stdout).toContain('Initialized empty Git repository'); expect(await vfs.exists('/repo/.git/HEAD')).toBe(true); @@ -103,10 +121,10 @@ describeIf(hasGit, 'git command', () => { it('add + commit creates objects and updates ref', async () => { ({ kernel, vfs, dispose } = await createGitKernel()); - await run(kernel, 'git init /repo'); + await run(kernel, git('init /repo')); await kernel.writeFile('/repo/hello.txt', 'hello world\n'); - await run(kernel, 'git -C /repo add hello.txt'); - await run(kernel, "git -C /repo commit -m 'first commit'"); + await run(kernel, git('-C /repo add hello.txt')); + await run(kernel, git("-C /repo commit -m 'first commit'")); expect(await vfs.exists('/repo/.git/refs/heads/main')).toBe(true); }); @@ -114,26 +132,26 @@ describeIf(hasGit, 'git command', () => { it('branch lists branches with current marked', async () => { ({ kernel, vfs, dispose } = await createGitKernel()); - await run(kernel, 'git init /repo'); + await run(kernel, git('init /repo')); await kernel.writeFile('/repo/file.txt', 'content\n'); - await run(kernel, 'git -C /repo add file.txt'); - await run(kernel, "git -C /repo commit -m 'init'"); + await run(kernel, git('-C /repo add file.txt')); + await run(kernel, git("-C /repo commit -m 'init'")); - const result = await run(kernel, 'git -C /repo branch'); + const result = await run(kernel, git('-C /repo branch')); expect(result.stdout.trim()).toBe('* main'); }); it('checkout -b creates a new branch', async () => { ({ kernel, vfs, dispose } = await createGitKernel()); - await run(kernel, 'git init /repo'); + await run(kernel, git('init /repo')); await kernel.writeFile('/repo/file.txt', 'content\n'); - await run(kernel, 'git -C /repo add file.txt'); - await run(kernel, "git -C /repo commit -m 'init'"); + await run(kernel, git('-C /repo add file.txt')); + await run(kernel, git("-C /repo commit -m 'init'")); - await run(kernel, 'git -C /repo checkout -b feature'); + await run(kernel, git('-C /repo checkout -b feature')); - const result = await run(kernel, 'git -C /repo branch'); + const result = await run(kernel, git('-C /repo branch')); const lines = result.stdout.trim().split('\n').map((l: string) => l.trim()); expect(lines).toContain('* feature'); expect(lines).toContain('main'); @@ -143,36 +161,36 @@ describeIf(hasGit, 'git command', () => { ({ kernel, vfs, dispose } = await createGitKernel()); // Create origin repo - await run(kernel, 'git init /tmp/origin'); + await run(kernel, git('init /tmp/origin')); await kernel.writeFile('/tmp/origin/README.md', '# demo repo\n'); - await run(kernel, 'git -C /tmp/origin add README.md'); - await run(kernel, "git -C /tmp/origin commit -m 'initial commit'"); + await run(kernel, git('-C /tmp/origin add README.md')); + await run(kernel, git("-C /tmp/origin commit -m 'initial commit'")); // Check default branch - let r = await run(kernel, 'git -C /tmp/origin branch'); + let r = await run(kernel, git('-C /tmp/origin branch')); expect(r.stdout.trim()).toBe('* main'); // Create feature branch with a new file - await run(kernel, 'git -C /tmp/origin checkout -b feature'); + await run(kernel, git('-C /tmp/origin checkout -b feature')); await kernel.writeFile('/tmp/origin/feature.txt', 'checked out from feature\n'); - await run(kernel, 'git -C /tmp/origin add feature.txt'); - await run(kernel, "git -C /tmp/origin commit -m 'add feature file'"); + await run(kernel, git('-C /tmp/origin add feature.txt')); + await run(kernel, git("-C /tmp/origin commit -m 'add feature file'")); // Switch back to main - await run(kernel, 'git -C /tmp/origin checkout main'); + await run(kernel, git('-C /tmp/origin checkout main')); // Clone - await run(kernel, 'git clone /tmp/origin /tmp/clone'); + await run(kernel, git('clone /tmp/origin /tmp/clone')); // Clone should only show main branch initially - r = await run(kernel, 'git -C /tmp/clone branch'); + r = await run(kernel, git('-C /tmp/clone branch')); expect(r.stdout.trim()).toBe('* main'); // Checkout feature (DWIM from remote tracking) - await run(kernel, 'git -C /tmp/clone checkout feature'); + await run(kernel, git('-C /tmp/clone checkout feature')); // Now both branches should be listed - r = await run(kernel, 'git -C /tmp/clone branch'); + r = await run(kernel, git('-C /tmp/clone branch')); const branches = r.stdout.trim().split('\n').map((l: string) => l.trim()); expect(branches).toContain('* feature'); expect(branches).toContain('main'); @@ -189,13 +207,13 @@ describeIf(hasGit, 'git command', () => { it('clone without an explicit destination uses the source basename', async () => { ({ kernel, vfs, dispose } = await createGitKernel()); - await run(kernel, 'git init /tmp/origin'); + await run(kernel, git('init /tmp/origin')); await kernel.writeFile('/tmp/origin/README.md', 'default destination\n'); - await run(kernel, 'git -C /tmp/origin add README.md'); - await run(kernel, "git -C /tmp/origin commit -m 'seed'"); + await run(kernel, git('-C /tmp/origin add README.md')); + await run(kernel, git("-C /tmp/origin commit -m 'seed'")); await run(kernel, 'mkdir -p /work'); - await run(kernel, 'git -C /work clone /tmp/origin'); + await run(kernel, git('-C /work clone /tmp/origin')); expect(await vfs.exists('/work/origin/.git/HEAD')).toBe(true); const readmeContent = new TextDecoder().decode(await vfs.readFile('/work/origin/README.md')); @@ -205,13 +223,13 @@ describeIf(hasGit, 'git command', () => { it('clone without an explicit destination strips a trailing .git suffix', async () => { ({ kernel, vfs, dispose } = await createGitKernel()); - await run(kernel, 'git init /tmp/origin.git'); + await run(kernel, git('init /tmp/origin.git')); await kernel.writeFile('/tmp/origin.git/README.md', 'suffix destination\n'); - await run(kernel, 'git -C /tmp/origin.git add README.md'); - await run(kernel, "git -C /tmp/origin.git commit -m 'seed'"); + await run(kernel, git('-C /tmp/origin.git add README.md')); + await run(kernel, git("-C /tmp/origin.git commit -m 'seed'")); await run(kernel, 'mkdir -p /work'); - await run(kernel, 'git -C /work clone /tmp/origin.git'); + await run(kernel, git('-C /work clone /tmp/origin.git')); expect(await vfs.exists('/work/origin/.git/HEAD')).toBe(true); const readmeContent = new TextDecoder().decode(await vfs.readFile('/work/origin/README.md')); @@ -221,13 +239,13 @@ describeIf(hasGit, 'git command', () => { it('clone into an existing empty destination directory succeeds', async () => { ({ kernel, vfs, dispose } = await createGitKernel()); - await run(kernel, 'git init /tmp/origin'); + await run(kernel, git('init /tmp/origin')); await kernel.writeFile('/tmp/origin/README.md', 'empty destination\n'); - await run(kernel, 'git -C /tmp/origin add README.md'); - await run(kernel, "git -C /tmp/origin commit -m 'seed'"); + await run(kernel, git('-C /tmp/origin add README.md')); + await run(kernel, git("-C /tmp/origin commit -m 'seed'")); await run(kernel, 'mkdir -p /tmp/clone'); - await run(kernel, 'git clone /tmp/origin /tmp/clone'); + await run(kernel, git('clone /tmp/origin /tmp/clone')); expect(await vfs.exists('/tmp/clone/.git/HEAD')).toBe(true); const readmeContent = new TextDecoder().decode(await vfs.readFile('/tmp/clone/README.md')); @@ -237,15 +255,15 @@ describeIf(hasGit, 'git command', () => { it('clone rejects a non-empty destination directory', async () => { ({ kernel, vfs, dispose } = await createGitKernel()); - await run(kernel, 'git init /tmp/origin'); + await run(kernel, git('init /tmp/origin')); await kernel.writeFile('/tmp/origin/README.md', 'origin\n'); - await run(kernel, 'git -C /tmp/origin add README.md'); - await run(kernel, "git -C /tmp/origin commit -m 'seed'"); + await run(kernel, git('-C /tmp/origin add README.md')); + await run(kernel, git("-C /tmp/origin commit -m 'seed'")); await run(kernel, 'mkdir -p /tmp/clone'); await kernel.writeFile('/tmp/clone/existing.txt', 'keep me\n'); - const result = await kernel.exec('git clone /tmp/origin /tmp/clone'); + const result = await kernel.exec(git('clone /tmp/origin /tmp/clone')); expect(result.exitCode).not.toBe(0); expect(result.stderr).toMatch(/already exists|not an empty directory|destination/i); @@ -257,7 +275,7 @@ describeIf(hasGit, 'git command', () => { it('clone of a missing repository fails without leaving a partial destination', async () => { ({ kernel, vfs, dispose } = await createGitKernel()); - const result = await kernel.exec('git clone /tmp/missing /tmp/clone'); + const result = await kernel.exec(git('clone /tmp/missing /tmp/clone')); expect(result.exitCode).not.toBe(0); expect(result.stderr).toMatch(/not a git repository|missing|no such file|fatal/i); expect(await vfs.exists('/tmp/clone')).toBe(false); @@ -266,8 +284,8 @@ describeIf(hasGit, 'git command', () => { it('clone of an empty repository succeeds and leaves an empty worktree', async () => { ({ kernel, vfs, dispose } = await createGitKernel()); - await run(kernel, 'git init /tmp/origin'); - await run(kernel, 'git clone /tmp/origin /tmp/clone'); + await run(kernel, git('init /tmp/origin')); + await run(kernel, git('clone /tmp/origin /tmp/clone')); const head = new TextDecoder().decode(await vfs.readFile('/tmp/clone/.git/HEAD')); expect(head.trim()).toBe('ref: refs/heads/main'); @@ -279,14 +297,14 @@ describeIf(hasGit, 'git command', () => { it('clone preserves nested directory trees', async () => { ({ kernel, vfs, dispose } = await createGitKernel()); - await run(kernel, 'git init /tmp/origin'); + await run(kernel, git('init /tmp/origin')); await run(kernel, 'mkdir -p /tmp/origin/src/nested'); await kernel.writeFile('/tmp/origin/src/nested/file.txt', 'nested payload\n'); await kernel.writeFile('/tmp/origin/src/root.txt', 'root payload\n'); - await run(kernel, 'git -C /tmp/origin add src/nested/file.txt src/root.txt'); - await run(kernel, "git -C /tmp/origin commit -m 'nested tree'"); + await run(kernel, git('-C /tmp/origin add src/nested/file.txt src/root.txt')); + await run(kernel, git("-C /tmp/origin commit -m 'nested tree'")); - await run(kernel, 'git clone /tmp/origin /tmp/clone'); + await run(kernel, git('clone /tmp/origin /tmp/clone')); const nested = new TextDecoder().decode(await vfs.readFile('/tmp/clone/src/nested/file.txt')); const root = new TextDecoder().decode(await vfs.readFile('/tmp/clone/src/root.txt')); @@ -297,17 +315,17 @@ describeIf(hasGit, 'git command', () => { it('clone honors the source default branch when HEAD is not main', async () => { ({ kernel, vfs, dispose } = await createGitKernel()); - await run(kernel, 'git init /tmp/origin'); + await run(kernel, git('init /tmp/origin')); await kernel.writeFile('/tmp/origin/README.md', 'main branch\n'); - await run(kernel, 'git -C /tmp/origin add README.md'); - await run(kernel, "git -C /tmp/origin commit -m 'main'"); + await run(kernel, git('-C /tmp/origin add README.md')); + await run(kernel, git("-C /tmp/origin commit -m 'main'")); - await run(kernel, 'git -C /tmp/origin checkout -b trunk'); + await run(kernel, git('-C /tmp/origin checkout -b trunk')); await kernel.writeFile('/tmp/origin/trunk.txt', 'trunk branch\n'); - await run(kernel, 'git -C /tmp/origin add trunk.txt'); - await run(kernel, "git -C /tmp/origin commit -m 'trunk'"); + await run(kernel, git('-C /tmp/origin add trunk.txt')); + await run(kernel, git("-C /tmp/origin commit -m 'trunk'")); - await run(kernel, 'git clone /tmp/origin /tmp/clone'); + await run(kernel, git('clone /tmp/origin /tmp/clone')); const head = new TextDecoder().decode(await vfs.readFile('/tmp/clone/.git/HEAD')); expect(head.trim()).toBe('ref: refs/heads/trunk'); @@ -319,22 +337,22 @@ describeIf(hasGit, 'git command', () => { it('clone copies nested branch refs and checkout DWIM works for branch names with slashes', async () => { ({ kernel, vfs, dispose } = await createGitKernel()); - await run(kernel, 'git init /tmp/origin'); + await run(kernel, git('init /tmp/origin')); await kernel.writeFile('/tmp/origin/README.md', '# demo repo\n'); - await run(kernel, 'git -C /tmp/origin add README.md'); - await run(kernel, "git -C /tmp/origin commit -m 'initial commit'"); + await run(kernel, git('-C /tmp/origin add README.md')); + await run(kernel, git("-C /tmp/origin commit -m 'initial commit'")); - await run(kernel, 'git -C /tmp/origin checkout -b feature/deep'); + await run(kernel, git('-C /tmp/origin checkout -b feature/deep')); await kernel.writeFile('/tmp/origin/feature.txt', 'nested branch payload\n'); - await run(kernel, 'git -C /tmp/origin add feature.txt'); - await run(kernel, "git -C /tmp/origin commit -m 'nested branch'"); - await run(kernel, 'git -C /tmp/origin checkout main'); + await run(kernel, git('-C /tmp/origin add feature.txt')); + await run(kernel, git("-C /tmp/origin commit -m 'nested branch'")); + await run(kernel, git('-C /tmp/origin checkout main')); - await run(kernel, 'git clone /tmp/origin /tmp/clone'); + await run(kernel, git('clone /tmp/origin /tmp/clone')); - expect(await vfs.exists('/tmp/clone/.git/refs/remotes/origin/feature/deep')).toBe(true); + await expectGitRef(kernel, '/tmp/clone', 'refs/remotes/origin/feature/deep'); - await run(kernel, 'git -C /tmp/clone checkout feature/deep'); + await run(kernel, git('-C /tmp/clone checkout feature/deep')); const featureContent = new TextDecoder().decode(await vfs.readFile('/tmp/clone/feature.txt')); expect(featureContent).toBe('nested branch payload\n'); const head = new TextDecoder().decode(await vfs.readFile('/tmp/clone/.git/HEAD')); @@ -345,56 +363,51 @@ describeIf(hasGit, 'git command', () => { ({ kernel, vfs, dispose } = await createGitKernel()); await run(kernel, 'mkdir -p /tmp/work'); - await run(kernel, 'git init /tmp/work/origin'); + await run(kernel, git('init /tmp/work/origin')); await kernel.writeFile('/tmp/work/origin/README.md', 'relative clone\n'); - await run(kernel, 'git -C /tmp/work/origin add README.md'); - await run(kernel, "git -C /tmp/work/origin commit -m 'seed'"); + await run(kernel, git('-C /tmp/work/origin add README.md')); + await run(kernel, git("-C /tmp/work/origin commit -m 'seed'")); - await run(kernel, 'git -C /tmp/work clone ./origin ./clone'); + await run(kernel, git('-C /tmp/work clone ./origin ./clone')); expect(await vfs.exists('/tmp/work/clone/.git/HEAD')).toBe(true); const readmeContent = new TextDecoder().decode(await vfs.readFile('/tmp/work/clone/README.md')); expect(readmeContent).toBe('relative clone\n'); }); - it('push fails with a typed unsupported-subcommand error', async () => { + it('push fails with a real Git remote/ref error', async () => { ({ kernel, dispose } = await createGitKernel()); - await run(kernel, 'git init /tmp/repo'); + await run(kernel, git('init /tmp/repo')); - const result = await kernel.exec('git -C /tmp/repo push origin main'); + const result = await kernel.exec(git('-C /tmp/repo push origin main')); expect(result.exitCode).not.toBe(0); - expect(result.stderr).toContain('GitSubcommandUnsupported'); - expect(result.stderr).toContain('git push'); - expect(result.stderr).toContain('README.md'); + expect(result.stderr).toMatch(/fatal|error|origin|refspec|repository/i); + expect(result.stderr).not.toContain('GitSubcommandUnsupported'); }); - it('clone rejects SSH-style remotes with a typed unsupported-subcommand error', async () => { + it('clone rejects SSH-style remotes with a real Git spawn failure', async () => { ({ kernel, dispose } = await createGitKernel()); - const result = await kernel.exec('git clone git@github.com:rivet-dev/agentos.git /tmp/clone'); + const result = await kernel.exec(git('clone git@github.com:rivet-dev/agentos.git /tmp/clone')); expect(result.exitCode).not.toBe(0); - expect(result.stderr).toContain('GitSubcommandUnsupported'); - expect(result.stderr).toContain('git clone'); - expect(result.stderr).toContain('SSH'); - expect(result.stderr).toContain('README.md'); + expect(result.stderr).toMatch(/cannot run ssh|unable to fork|ssh|fatal/i); + expect(result.stderr).not.toContain('GitSubcommandUnsupported'); }); - it('clone rejects authenticated HTTPS remotes loudly instead of attempting a broken auth flow', async () => { + it('clone rejects HTTPS remotes until Git is linked with libcurl remote helpers', async () => { ({ kernel, dispose } = await createGitKernel()); const result = await kernel.exec( - 'git clone https://private@example.com/owner/repo.git /tmp/clone', + git('clone https://private@example.com/owner/repo.git /tmp/clone'), { env: { GIT_AUTH_TOKEN: 'test-token' } }, ); expect(result.exitCode).not.toBe(0); - expect(result.stderr).toContain('GitSubcommandUnsupported'); - expect(result.stderr).toContain('git clone'); - expect(result.stderr).toContain('authenticated HTTP(S) remotes'); - expect(result.stderr).toContain('README.md'); + expect(result.stderr).toMatch(/remote-https|remote helper|fatal|unable/i); + expect(result.stderr).not.toContain('GitSubcommandUnsupported'); }); - describeIf(hasHostGit, 'remote clone over smart HTTP', () => { + describeIf(hasHostGit && hasGitHttpHelper, 'remote clone over smart HTTP', () => { let repoRoot: string; let httpServer: HttpServer; let httpPort: number; @@ -530,16 +543,16 @@ describeIf(hasGit, 'git command', () => { it('clone fetches refs and worktree contents from a smart HTTP remote', async () => { ({ kernel, vfs, dispose } = await createGitKernelWithNet([httpPort])); - await run(kernel, `git clone http://127.0.0.1:${httpPort}/origin.git /tmp/clone`); + await run(kernel, git(`clone http://127.0.0.1:${httpPort}/origin.git /tmp/clone`)); const head = new TextDecoder().decode(await kernel.readFile('/tmp/clone/.git/HEAD')); expect(head.trim()).toBe('ref: refs/heads/main'); const readme = new TextDecoder().decode(await kernel.readFile('/tmp/clone/README.md')); expect(readme).toBe('remote smart clone\n'); - expect(await kernel.exists('/tmp/clone/.git/refs/remotes/origin/feature/deep')).toBe(true); + await expectGitRef(kernel, '/tmp/clone', 'refs/remotes/origin/feature/deep'); - await run(kernel, 'git -C /tmp/clone checkout feature/deep'); + await run(kernel, git('-C /tmp/clone checkout feature/deep')); const feature = new TextDecoder().decode(await kernel.readFile('/tmp/clone/feature.txt')); expect(feature).toBe('remote branch payload\n'); }); diff --git a/toolchain/Makefile b/toolchain/Makefile index 1654bfa784..f9e99e2991 100644 --- a/toolchain/Makefile +++ b/toolchain/Makefile @@ -212,7 +212,7 @@ wasm: vendor patch-vendor patch-std wasm-opt-check # codex the codex fork build (codex, codex-exec) # `just toolchain-cmd ` calls this. `make wasm` builds the fast # Rust set; `make -C c programs install` builds/installs the fast C set. -C_COMMANDS := zip unzip envsubst sqlite3 curl wget grep duckdb vim +C_COMMANDS := zip unzip envsubst sqlite3 curl wget grep git duckdb vim .PHONY: cmd/% cmd/%: diff --git a/toolchain/c/Makefile b/toolchain/c/Makefile index 17ad4c2cd4..3d0d9670a0 100644 --- a/toolchain/c/Makefile +++ b/toolchain/c/Makefile @@ -68,7 +68,7 @@ COMMANDS_DIR ?= ../target/wasm32-wasip1/release/commands # Fast real commands installed by the default software gate. Slow/heavy commands # (duckdb, vim) remain available through explicit parent `make cmd/`. -COMMANDS := zip unzip envsubst sqlite3 curl wget grep tree +COMMANDS := zip unzip envsubst sqlite3 curl wget grep git tree # Programs requiring patched sysroot (Tier 2+ custom host imports) PATCHED_PROGRAMS := http_get_test isatty_test getpid_test getppid_test getppid_verify userinfo pipe_test dup_test spawn_child spawn_exit_code pipeline kill_child waitpid_return waitpid_edge syscall_coverage getpwuid_test signal_tests sigaction_self sigaction_behavior delayed_tcp_echo delayed_kill pipe_edge tcp_accept_spawn tcp_echo tcp_server http_server udp_echo unix_socket signal_handler dns_lookup sqlite3 curl wget grep tree zip unzip fs_probe @@ -150,11 +150,18 @@ CURL_RELEASE_URL := https://github.com/curl/curl/releases/download/curl-$(CURL_R CURL_UPSTREAM_BUILD_DIR := $(BUILD_DIR)/curl-upstream CURL_UPSTREAM_OVERLAY_DIR := ../../software/curl/native/c/overlay CURL_UPSTREAM_OVERLAY_FILES := $(wildcard $(CURL_UPSTREAM_OVERLAY_DIR)/lib/*.c $(CURL_UPSTREAM_OVERLAY_DIR)/lib/*.h $(CURL_UPSTREAM_OVERLAY_DIR)/lib/vtls/*.c $(CURL_UPSTREAM_OVERLAY_DIR)/lib/vtls/*.h) +ZLIB_VERSION := 1.3.1 +ZLIB_URL := https://zlib.net/zlib-$(ZLIB_VERSION).tar.gz +ZLIB_BUILD_DIR := $(BUILD_DIR)/zlib WGET_VERSION := 1.24.5 WGET_URL := https://ftp.gnu.org/gnu/wget/wget-$(WGET_VERSION).tar.gz WGET_UPSTREAM_BUILD_DIR := $(BUILD_DIR)/wget-upstream WGET_UPSTREAM_OVERLAY_INCLUDE_DIR := overlays/wget/include WGET_UPSTREAM_OVERLAY_FILES := $(wildcard $(WGET_UPSTREAM_OVERLAY_INCLUDE_DIR)/*.h) +GIT_VERSION := 2.55.0 +GIT_URL := https://www.kernel.org/pub/software/scm/git/git-$(GIT_VERSION).tar.xz +GIT_UPSTREAM_BUILD_DIR := $(BUILD_DIR)/git-upstream +GIT_PATCH_DIR := patches/git GREP_VERSION := 3.12 GREP_URL := https://ftp.gnu.org/gnu/grep/grep-$(GREP_VERSION).tar.xz GREP_UPSTREAM_BUILD_DIR := $(BUILD_DIR)/grep-upstream @@ -521,6 +528,23 @@ $(NATIVE_DIR)/sqlite3: libs/sqlite3/shell.c libs/sqlite3/sqlite3.c $(NATIVE_CC) $(NATIVE_CFLAGS) $(SQLITE_SHELL_NATIVE) -Ilibs/sqlite3 -o $@ \ libs/sqlite3/shell.c libs/sqlite3/sqlite3.c -lm -lpthread +$(LIBS_CACHE)/zlib-$(ZLIB_VERSION)/zlib.h: + @echo "Fetching zlib $(ZLIB_VERSION)..." + @mkdir -p $(LIBS_CACHE) + @curl -fSL "$(ZLIB_URL)" -o "$(LIBS_CACHE)/zlib-$(ZLIB_VERSION).tar.gz" + @rm -rf "$(LIBS_CACHE)/zlib-$(ZLIB_VERSION)" + @tar -xzf "$(LIBS_CACHE)/zlib-$(ZLIB_VERSION).tar.gz" -C "$(LIBS_CACHE)" + +$(ZLIB_BUILD_DIR)/libz.a: $(LIBS_CACHE)/zlib-$(ZLIB_VERSION)/zlib.h $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a $(WASI_SDK_DIR)/bin/clang + @mkdir -p $(ZLIB_BUILD_DIR) + @for src in adler32.c crc32.c deflate.c infback.c inffast.c inflate.c inftrees.c trees.c zutil.c compress.c uncompr.c gzclose.c gzlib.c gzread.c gzwrite.c; do \ + $(CC) --target=wasm32-wasip1 --sysroot=$(SYSROOT) -O2 -DHAVE_UNISTD_H \ + -I$(LIBS_CACHE)/zlib-$(ZLIB_VERSION) \ + -c "$(LIBS_CACHE)/zlib-$(ZLIB_VERSION)/$$src" \ + -o "$(ZLIB_BUILD_DIR)/$${src%.c}.o"; \ + done + $(WASI_SDK_DIR)/bin/llvm-ar rcs $@ $(ZLIB_BUILD_DIR)/*.o + $(NATIVE_DIR)/sqlite3_cli: $(NATIVE_DIR)/sqlite3 cp $< $@ @@ -559,6 +583,22 @@ $(BUILD_DIR)/unzip: scripts/build-infozip-upstream.sh $(PATCHED_SYSROOT)/lib/was --cc "$(abspath $(CC)) --target=wasm32-wasip1 --sysroot=$(abspath $(SYSROOT))" \ --output "$(abspath $@)" +$(BUILD_DIR)/git: scripts/build-git-upstream.sh $(ZLIB_BUILD_DIR)/libz.a $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a $(WASI_SDK_DIR)/bin/clang + @mkdir -p $(BUILD_DIR) + bash scripts/build-git-upstream.sh \ + --version "$(GIT_VERSION)" \ + --url "$(GIT_URL)" \ + --cache-dir "$(abspath $(LIBS_CACHE))" \ + --build-dir "$(abspath $(GIT_UPSTREAM_BUILD_DIR))" \ + --patch-dir "$(abspath $(GIT_PATCH_DIR))" \ + --zlib-dir "$(abspath $(LIBS_CACHE)/zlib-$(ZLIB_VERSION))" \ + --zlib-build-dir "$(abspath $(ZLIB_BUILD_DIR))" \ + --cc "$(abspath $(CC))" \ + --ar "$(abspath $(WASI_SDK_DIR)/bin/llvm-ar)" \ + --ranlib "$(abspath $(WASI_SDK_DIR)/bin/llvm-ranlib)" \ + --sysroot "$(abspath $(SYSROOT))" \ + --output "$(abspath $@)" + # curl_test: links libcurl (HTTP/HTTPS build for WASM via host_net + host_tls) CURL_SRCS := $(wildcard libs/curl/lib/*.c) $(wildcard libs/curl/lib/vauth/*.c) \ $(wildcard libs/curl/lib/vtls/*.c) $(wildcard libs/curl/lib/vquic/*.c) \ @@ -655,10 +695,11 @@ install: fi; \ done; \ echo "Installed $$INSTALLED C command(s) to $(COMMANDS_DIR)" - @# Create symlinks for git remote helpers + @# Create symlinks for Git helper entry points used by upstream Git. @if [ -f "$(COMMANDS_DIR)/git" ]; then \ - ln -sf git "$(COMMANDS_DIR)/git-remote-http"; \ - ln -sf git "$(COMMANDS_DIR)/git-remote-https"; \ + for alias in git-remote-http git-remote-https git-upload-pack git-receive-pack git-upload-archive; do \ + ln -sf git "$(COMMANDS_DIR)/$$alias"; \ + done; \ fi # --- Clean --- diff --git a/toolchain/c/patches/git/0001-wasi-posix-spawn-run-command.patch b/toolchain/c/patches/git/0001-wasi-posix-spawn-run-command.patch new file mode 100644 index 0000000000..96a9f86e62 --- /dev/null +++ b/toolchain/c/patches/git/0001-wasi-posix-spawn-run-command.patch @@ -0,0 +1,137 @@ +Use posix_spawn for Git child processes on WASI. + +AgentOS provides POSIX process creation through the sysroot's posix_spawn +implementation, backed by the host_process broker. WASI cannot provide fork() +semantics, so route Git's central child-process path through posix_spawn while +leaving the rest of upstream Git's command machinery intact. + +--- a/run-command.c ++++ b/run-command.c +@@ -18,6 +18,10 @@ + #include "packfile.h" + #include "compat/nonblock.h" + ++#ifdef __wasi__ ++#include ++#endif ++ + void child_process_init(struct child_process *child) + { + struct child_process blank = CHILD_PROCESS_INIT; +@@ -747,12 +751,14 @@ + + #ifndef GIT_WINDOWS_NATIVE + { ++#ifndef __wasi__ + int notify_pipe[2]; ++ struct child_err cerr; ++ struct atfork_state as; ++#endif + int null_fd = -1; + char **childenv; + struct strvec argv = STRVEC_INIT; +- struct child_err cerr; +- struct atfork_state as; + + if (prepare_cmd(&argv, cmd) < 0) { + failed_errno = errno; +@@ -764,8 +770,10 @@ + + trace_argv_printf(&argv.v[1], "trace: start_command:"); + ++#ifndef __wasi__ + if (pipe(notify_pipe)) + notify_pipe[0] = notify_pipe[1] = -1; ++#endif + + if (cmd->no_stdin || cmd->no_stdout || cmd->no_stderr) { + null_fd = xopen("/dev/null", O_RDWR | O_CLOEXEC); +@@ -773,6 +781,79 @@ + } + + childenv = prep_childenv(cmd->env.v); ++#ifdef __wasi__ ++ { ++ posix_spawn_file_actions_t actions; ++ int spawn_err; ++ ++ spawn_err = posix_spawn_file_actions_init(&actions); ++ if (spawn_err) { ++ failed_errno = spawn_err; ++ cmd->pid = -1; ++ errno = spawn_err; ++ goto wasi_spawn_done; ++ } ++ ++ if (cmd->no_stdin) ++ spawn_err = posix_spawn_file_actions_adddup2(&actions, null_fd, 0); ++ else if (need_in) ++ spawn_err = posix_spawn_file_actions_adddup2(&actions, fdin[0], 0); ++ else if (cmd->in) ++ spawn_err = posix_spawn_file_actions_adddup2(&actions, cmd->in, 0); ++ if (spawn_err) ++ goto wasi_spawn_failed_action; ++ ++ if (cmd->no_stderr) ++ spawn_err = posix_spawn_file_actions_adddup2(&actions, null_fd, 2); ++ else if (need_err) ++ spawn_err = posix_spawn_file_actions_adddup2(&actions, fderr[1], 2); ++ else if (cmd->err > 1) ++ spawn_err = posix_spawn_file_actions_adddup2(&actions, cmd->err, 2); ++ if (spawn_err) ++ goto wasi_spawn_failed_action; ++ ++ if (cmd->no_stdout) ++ spawn_err = posix_spawn_file_actions_adddup2(&actions, null_fd, 1); ++ else if (cmd->stdout_to_stderr) ++ spawn_err = posix_spawn_file_actions_adddup2(&actions, 2, 1); ++ else if (need_out) ++ spawn_err = posix_spawn_file_actions_adddup2(&actions, fdout[1], 1); ++ else if (cmd->out > 1) ++ spawn_err = posix_spawn_file_actions_adddup2(&actions, cmd->out, 1); ++ if (spawn_err) ++ goto wasi_spawn_failed_action; ++ ++ if (cmd->dir) { ++ spawn_err = posix_spawn_file_actions_addchdir(&actions, cmd->dir); ++ if (spawn_err) ++ goto wasi_spawn_failed_action; ++ } ++ ++ spawn_err = posix_spawn(&cmd->pid, argv.v[1], &actions, NULL, ++ (char *const *)argv.v + 1, childenv); ++ if (spawn_err == ENOEXEC) ++ spawn_err = posix_spawn(&cmd->pid, argv.v[0], &actions, NULL, ++ (char *const *)argv.v, childenv); ++ ++wasi_spawn_failed_action: ++ posix_spawn_file_actions_destroy(&actions); ++ if (spawn_err) { ++ failed_errno = spawn_err; ++ cmd->pid = -1; ++ errno = spawn_err; ++ if (!cmd->silent_exec_failure || errno != ENOENT) ++ error_errno("cannot spawn %s", cmd->args.v[0]); ++ } else if (cmd->clean_on_exit) { ++ mark_child_for_cleanup(cmd->pid, cmd); ++ } ++ ++wasi_spawn_done: ++ if (null_fd >= 0) ++ close(null_fd); ++ strvec_clear(&argv); ++ free(childenv); ++ } ++#else + atfork_prepare(&as); + + /* +@@ -904,8 +985,9 @@ + if (null_fd >= 0) + close(null_fd); + strvec_clear(&argv); + free(childenv); ++#endif + } + end_of_spawn: + diff --git a/toolchain/c/scripts/build-git-upstream.sh b/toolchain/c/scripts/build-git-upstream.sh new file mode 100755 index 0000000000..f5268b6943 --- /dev/null +++ b/toolchain/c/scripts/build-git-upstream.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: build-git-upstream.sh \ + --version \ + --url \ + --cache-dir \ + --build-dir \ + --patch-dir \ + --zlib-dir \ + --zlib-build-dir \ + --cc \ + --ar \ + --ranlib \ + --sysroot \ + --output +EOF +} + +VERSION="" +URL="" +CACHE_DIR="" +BUILD_DIR="" +ZLIB_DIR="" +ZLIB_BUILD_DIR="" +PATCH_DIR="" +CC_CMD="" +AR_CMD="" +RANLIB_CMD="" +SYSROOT="" +OUTPUT="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --version) VERSION="$2"; shift 2 ;; + --url) URL="$2"; shift 2 ;; + --cache-dir) CACHE_DIR="$2"; shift 2 ;; + --build-dir) BUILD_DIR="$2"; shift 2 ;; + --patch-dir) PATCH_DIR="$2"; shift 2 ;; + --zlib-dir) ZLIB_DIR="$2"; shift 2 ;; + --zlib-build-dir) ZLIB_BUILD_DIR="$2"; shift 2 ;; + --cc) CC_CMD="$2"; shift 2 ;; + --ar) AR_CMD="$2"; shift 2 ;; + --ranlib) RANLIB_CMD="$2"; shift 2 ;; + --sysroot) SYSROOT="$2"; shift 2 ;; + --output) OUTPUT="$2"; shift 2 ;; + *) echo "Unknown argument: $1" >&2; usage >&2; exit 1 ;; + esac +done + +if [[ -z "$VERSION" || -z "$URL" || -z "$CACHE_DIR" || -z "$BUILD_DIR" || -z "$PATCH_DIR" || -z "$ZLIB_DIR" || -z "$ZLIB_BUILD_DIR" || -z "$CC_CMD" || -z "$AR_CMD" || -z "$RANLIB_CMD" || -z "$SYSROOT" || -z "$OUTPUT" ]]; then + usage >&2 + exit 1 +fi + +fetch() { + local url="$1" + local out="$2" + if command -v curl >/dev/null 2>&1; then + curl --retry 3 --retry-all-errors -fSL "$url" -o "$out" + elif command -v wget >/dev/null 2>&1; then + wget -q "$url" -O "$out" + else + echo "Neither curl nor wget is available to fetch $url" >&2 + exit 1 + fi +} + +mkdir -p "$CACHE_DIR" +rm -rf "$BUILD_DIR" +mkdir -p "$BUILD_DIR" + +TARBALL="$CACHE_DIR/git-${VERSION}.tar.xz" +if [[ ! -f "$TARBALL" ]]; then + echo "Fetching upstream Git ${VERSION} release tarball..." + fetch "$URL" "$TARBALL" +fi + +echo "Extracting upstream Git ${VERSION}..." +tar -xf "$TARBALL" -C "$BUILD_DIR" + +SRC_DIR="$BUILD_DIR/git-${VERSION}" +if [[ ! -d "$SRC_DIR" ]]; then + echo "Expected extracted source at $SRC_DIR" >&2 + exit 1 +fi + +pushd "$SRC_DIR" >/dev/null + +if [[ -d "$PATCH_DIR" ]]; then + for patch_file in "$PATCH_DIR"/*.patch; do + [[ -e "$patch_file" ]] || continue + echo "Applying $(basename "$patch_file")..." + patch -p1 < "$patch_file" + done +fi + +echo "Building upstream Git ${VERSION} for wasm32-wasip1..." +make -j"${MAKE_JOBS:-2}" \ + uname_S=WASI \ + CC="$CC_CMD" \ + HOSTCC="${HOSTCC:-cc}" \ + AR="$AR_CMD" \ + RANLIB="$RANLIB_CMD" \ + CFLAGS="--target=wasm32-wasip1 --sysroot=$SYSROOT -I$ZLIB_DIR -O2 -D_WASI_EMULATED_PROCESS_CLOCKS -D_WASI_EMULATED_MMAN" \ + LDFLAGS="--target=wasm32-wasip1 --sysroot=$SYSROOT -L$ZLIB_BUILD_DIR -lwasi-emulated-process-clocks -lwasi-emulated-mman" \ + CSPRNG_METHOD=getentropy \ + HAVE_PATHS_H=YesPlease \ + HAVE_DEV_TTY=YesPlease \ + HAVE_CLOCK_GETTIME=YesPlease \ + HAVE_CLOCK_MONOTONIC=YesPlease \ + HAVE_GETDELIM=YesPlease \ + NO_RUST=1 \ + NO_OPENSSL=1 \ + NO_CURL=1 \ + NO_EXPAT=1 \ + NO_GETTEXT=1 \ + NO_TCLTK=1 \ + NO_PERL=1 \ + NO_PYTHON=1 \ + NO_REGEX=NeedsStartEnd \ + NO_ICONV=1 \ + NO_PTHREADS=1 \ + NO_MMAP=1 \ + NO_IPV6=1 \ + NO_UNIX_SOCKETS=1 \ + NO_SYS_POLL_H=1 \ + NO_NSEC=1 \ + git + +mkdir -p "$(dirname "$OUTPUT")" +if command -v wasm-opt >/dev/null 2>&1; then + echo "Optimizing Git WASM binary..." + wasm-opt -O3 --strip-debug --all-features git -o "$OUTPUT" +else + cp git "$OUTPUT" +fi + +popd >/dev/null + +echo "Built upstream Git at $OUTPUT" diff --git a/toolchain/std-patches/git/README.md b/toolchain/std-patches/git/README.md deleted file mode 100644 index b8a55dffab..0000000000 --- a/toolchain/std-patches/git/README.md +++ /dev/null @@ -1,28 +0,0 @@ -# Git WASM Compatibility - -Git (GPL-2.0) cannot be vendored due to license restrictions. -This project uses a clean-room, Apache-2.0 licensed reimplementation -of git plumbing commands. - -## WASM-Incompatible Patterns in Git Source - -The following patterns in upstream git are incompatible with WASM/WASI -and were avoided in our clean-room implementation: - -### 1. fork+exec in run_command.c -Git's `run_command()` uses `fork()`+`exec()` extensively for spawning -child processes (hooks, filters, remotes). WASI has no `fork()`. -**Our approach:** Use `posix_spawn()` via `host_process` imports when -process spawning is needed in future commands. - -### 2. mmap in wrapper.c -Git uses `mmap()` for memory-mapped file I/O (packfiles, index). -WASI has limited `mmap()` support. -**Our approach:** Use standard `malloc()`+`read()` for all file I/O. -Git upstream supports this via `NO_MMAP=1`. - -### 3. Signal handlers (sigaction for SIGPIPE, SIGCHLD) -Git registers signal handlers for `SIGPIPE` (ignore broken pipe) and -`SIGCHLD` (reap child processes). WASI has no signal support. -**Our approach:** No signal handlers needed — WASI processes don't -receive signals, and our process model handles cleanup via the kernel. diff --git a/toolchain/std-patches/wasi-libc-overrides/git_compat.c b/toolchain/std-patches/wasi-libc-overrides/git_compat.c new file mode 100644 index 0000000000..f1a5734b7b --- /dev/null +++ b/toolchain/std-patches/wasi-libc-overrides/git_compat.c @@ -0,0 +1,200 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef h_errno +#undef h_errno +#endif + +void __SIG_IGN(int sig) { + (void)sig; +} + +void __SIG_ERR(int sig) { + (void)sig; +} + +int raise(int sig) { + (void)sig; + errno = ENOSYS; + return -1; +} + +pid_t getpgid(pid_t pid) { + return pid == 0 ? getpid() : pid; +} + +unsigned alarm(unsigned seconds) { + (void)seconds; + return 0; +} + +int getitimer(int which, struct itimerval *value) { + (void)which; + if (value) + memset(value, 0, sizeof(*value)); + return 0; +} + +int setitimer(int which, const struct itimerval *new_value, struct itimerval *old_value) { + (void)which; + (void)new_value; + if (old_value) + memset(old_value, 0, sizeof(*old_value)); + return 0; +} + +static int exec_spawn_wait(const char *path, char *const argv[], char *const envp[], int search_path) { + pid_t pid; + int status; + int err = search_path + ? posix_spawnp(&pid, path, NULL, NULL, argv, envp) + : posix_spawn(&pid, path, NULL, NULL, argv, envp); + + if (err != 0) { + errno = err; + return -1; + } + + while (waitpid(pid, &status, 0) < 0) { + if (errno != EINTR) + _exit(127); + } + + if (WIFEXITED(status)) + _exit(WEXITSTATUS(status)); + if (WIFSIGNALED(status)) + _exit(128 + WTERMSIG(status)); + _exit(127); +} + +static int exec_list(const char *path, const char *arg, va_list ap, int search_path) { + extern char **environ; + va_list count_ap; + size_t argc = 1; + char **argv; + const char *next; + + va_copy(count_ap, ap); + while ((next = va_arg(count_ap, const char *)) != NULL) + argc++; + va_end(count_ap); + + argv = calloc(argc + 1, sizeof(*argv)); + if (!argv) { + errno = ENOMEM; + return -1; + } + + argv[0] = (char *)arg; + for (size_t i = 1; i < argc; i++) + argv[i] = va_arg(ap, char *); + + int ret = exec_spawn_wait(path, argv, environ, search_path); + free(argv); + return ret; +} + +int execl(const char *path, const char *arg, ...) { + va_list ap; + va_start(ap, arg); + int ret = exec_list(path, arg, ap, 0); + va_end(ap); + return ret; +} + +int execlp(const char *file, const char *arg, ...) { + va_list ap; + va_start(ap, arg); + int ret = exec_list(file, arg, ap, 1); + va_end(ap); + return ret; +} + +int execv(const char *path, char *const argv[]) { + extern char **environ; + return exec_spawn_wait(path, argv, environ, 0); +} + +int execve(const char *path, char *const argv[], char *const envp[]) { + return exec_spawn_wait(path, argv, envp, 0); +} + +int execvp(const char *file, char *const argv[]) { + extern char **environ; + return exec_spawn_wait(file, argv, environ, 1); +} + +_Thread_local int h_errno; + +int *__h_errno_location(void) { + return &h_errno; +} + +const char *hstrerror(int err) { + (void)err; + return strerror(errno); +} + +struct hostent *gethostbyname(const char *name) { + (void)name; + h_errno = HOST_NOT_FOUND; + return NULL; +} + +struct servent *getservbyname(const char *name, const char *proto) { + (void)name; + (void)proto; + return NULL; +} + +struct passwd *getpwnam(const char *name) { + (void)name; + return NULL; +} + +char *inet_ntoa(struct in_addr in) { + static _Thread_local char buf[INET_ADDRSTRLEN]; + + if (inet_ntop(AF_INET, &in, buf, sizeof(buf)) == NULL) { + buf[0] = '0'; + buf[1] = '\0'; + } + return buf; +} + +void openlog(const char *ident, int option, int facility) { + (void)ident; + (void)option; + (void)facility; +} + +void syslog(int priority, const char *format, ...) { + (void)priority; + (void)format; +} + +void closelog(void) { +} + +int setlogmask(int mask) { + return mask; +} + +void vsyslog(int priority, const char *format, va_list ap) { + (void)priority; + (void)format; + (void)ap; +} diff --git a/toolchain/std-patches/wasi-libc/0028-git-compat-header-surface.patch b/toolchain/std-patches/wasi-libc/0028-git-compat-header-surface.patch new file mode 100644 index 0000000000..0f02dd4fd7 --- /dev/null +++ b/toolchain/std-patches/wasi-libc/0028-git-compat-header-surface.patch @@ -0,0 +1,100 @@ +Expose small POSIX compatibility surfaces needed by upstream Git. + +Git's process runner is patched to use the existing AgentOS posix_spawn +broker on WASI, but some linked translation units still reference standard +headers and functions such as syslog, paths, exec, alarm, network lookup, +and interval timers. Expose those declarations in the owned sysroot and +provide runtime shims in libc-bottom-half overrides. + +--- a/libc-top-half/musl/include/unistd.h ++++ b/libc-top-half/musl/include/unistd.h +@@ -116,6 +116,7 @@ + #define W_OK 2 + #define X_OK 1 + #endif ++unsigned alarm(unsigned); + + int access(const char *, int); + int faccessat(int, const char *, int, int); +@@ -137,6 +138,7 @@ + pid_t fork(void); + #ifdef __wasilibc_unmodified_upstream /* WASI has no fork/exec */ + pid_t _Fork(void); ++#endif + int execve(const char *, char *const [], char *const []); + int execv(const char *, char *const []); + int execle(const char *, const char *, ...); +@@ -144,7 +146,6 @@ + int execvp(const char *, char *const []); + int execlp(const char *, const char *, ...); + int fexecve(int, char *const [], char *const []); +-#endif + _Noreturn void _exit(int); + + pid_t getpid(void); +@@ -153,6 +154,7 @@ + pid_t getpgrp(void); + pid_t setsid(void); ++pid_t getpgid(pid_t); + #ifdef __wasilibc_unmodified_upstream /* WASI has no full process groups */ + pid_t getpgid(pid_t); + int setpgid(pid_t, pid_t); + pid_t getsid(pid_t); +--- a/libc-top-half/musl/include/sys/time.h ++++ b/libc-top-half/musl/include/sys/time.h +@@ -10,7 +10,6 @@ + + int gettimeofday (struct timeval *__restrict, void *__restrict); + +-#ifdef __wasilibc_unmodified_upstream /* WASI has no getitimer */ + #define ITIMER_REAL 0 + #define ITIMER_VIRTUAL 1 + #define ITIMER_PROF 2 +@@ -22,7 +21,6 @@ + + int getitimer (int, struct itimerval *); + int setitimer (int, const struct itimerval *__restrict, struct itimerval *__restrict); +-#endif + int utimes (const char *, const struct timeval [2]); + + #if defined(_GNU_SOURCE) || defined(_BSD_SOURCE) +--- a/libc-top-half/musl/include/arpa/inet.h ++++ b/libc-top-half/musl/include/arpa/inet.h +@@ -16,8 +16,8 @@ + #ifdef __wasilibc_unmodified_upstream /* WASI has no inet_addr */ + in_addr_t inet_addr (const char *); + in_addr_t inet_network (const char *); +-char *inet_ntoa (struct in_addr); + #endif ++char *inet_ntoa (struct in_addr); + int inet_pton (int, const char *__restrict, void *__restrict); + const char *inet_ntop (int, const void *__restrict, char *__restrict, socklen_t); + +--- a/libc-top-half/musl/include/netdb.h ++++ b/libc-top-half/musl/include/netdb.h +@@ -132,6 +132,12 @@ + #define NO_ADDRESS NO_DATA + #endif + ++#ifndef h_errno ++int *__h_errno_location(void); ++#define h_errno (*__h_errno_location()) ++#endif ++const char *hstrerror(int); ++ + #if defined(_GNU_SOURCE) || defined(_BSD_SOURCE) + void herror(const char *); + const char *hstrerror(int); +--- a/scripts/install-include-headers.sh ++++ b/scripts/install-include-headers.sh +@@ -56,8 +56,8 @@ + "sys/soundcard.h" "sys/sem.h" "sys/shm.h" "sys/msg.h" "sys/ipc.h" \ + "sys/ptrace.h" "sys/statfs.h" "bits/kd.h" "bits/vt.h" "bits/soundcard.h" \ + "bits/sem.h" "bits/shm.h" "bits/msg.h" "bits/ipc.h" "bits/ptrace.h" \ +- "bits/statfs.h" "sys/vfs.h" "syslog.h" "sys/syslog.h" "wait.h" \ +- "ucontext.h" "sys/ucontext.h" "paths.h" "utmp.h" "utmpx.h" \ ++ "bits/statfs.h" "sys/vfs.h" "sys/syslog.h" "wait.h" \ ++ "ucontext.h" "sys/ucontext.h" "utmp.h" "utmpx.h" \ + "lastlog.h" "sys/acct.h" "sys/cachectl.h" "sys/epoll.h" "sys/reboot.h" \ + "sys/swap.h" "sys/sendfile.h" "sys/inotify.h" "sys/quota.h" "sys/klog.h" \ + "sys/fsuid.h" "sys/io.h" "sys/prctl.h" "sys/mtio.h" "sys/mount.h" \