diff --git a/crates/execution/assets/runners/wasi-module.js b/crates/execution/assets/runners/wasi-module.js index 8bb1d01dc6..b861b77ba9 100644 --- a/crates/execution/assets/runners/wasi-module.js +++ b/crates/execution/assets/runners/wasi-module.js @@ -1393,11 +1393,18 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = return this._measureWasiPhase("writeResultPtr", () => this._writeUint32(nwrittenPtr, written)); } } + const entry = this._descriptorEntry(descriptor); + const localHostPassthrough = + handle.kind === "host-passthrough" && + entry?.kind === "file" && + entry.realFd === handle.targetFd; const position = handle.append ? this._measureWasiPhase("appendFstat", () => Number(__agentOSFs().fstatSync(handle.targetFd).size ?? 0) ) - : null; + : localHostPassthrough + ? (entry.offset ?? 0) + : null; const written = this._measureWasiPhase("writeSync", () => __agentOSFs().writeSync( handle.targetFd, @@ -1407,6 +1414,15 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = position, ) ); + if (localHostPassthrough) { + if (handle.append) { + entry.offset = this._measureWasiPhase("appendFstat", () => + Number(__agentOSFs().fstatSync(handle.targetFd).size ?? 0) + ); + } else { + entry.offset = (entry.offset ?? 0) + written; + } + } return this._measureWasiPhase("writeResultPtr", () => this._writeUint32(nwrittenPtr, written)); } if (handle?.kind === "guest-file" && typeof handle.targetFd === "number") { @@ -1745,6 +1761,11 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = (handle?.kind === "passthrough" || handle?.kind === "host-passthrough") && typeof handle.targetFd === "number" ) { + const localEntry = this._descriptorEntry(descriptor); + const localHostPassthrough = + handle.kind === "host-passthrough" && + localEntry?.kind === "file" && + localEntry.realFd === handle.targetFd; const totalLength = this._boundedReadLength(iovs, iovsLen); const buffer = Buffer.alloc(totalLength); const bytesRead = __agentOSFs().readSync( @@ -1752,8 +1773,11 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = buffer, 0, totalLength, - null, + localHostPassthrough ? (localEntry.offset ?? 0) : null, ); + if (localHostPassthrough) { + localEntry.offset = (localEntry.offset ?? 0) + bytesRead; + } const written = this._writeToIovs(iovs, iovsLen, buffer.subarray(0, bytesRead)); return this._writeUint32(nreadPtr, written); } diff --git a/crates/vfs/src/posix/overlay_fs.rs b/crates/vfs/src/posix/overlay_fs.rs index 25aa86b9e3..3c4ce17454 100644 --- a/crates/vfs/src/posix/overlay_fs.rs +++ b/crates/vfs/src/posix/overlay_fs.rs @@ -1661,6 +1661,8 @@ impl VirtualFileSystem for OverlayFileSystem { let mut snapshot_entries = Vec::new(); self.collect_snapshot_entries(&old_normalized, &mut snapshot_entries)?; + self.clear_path_metadata(&resolved_new_normalized)?; + self.clear_subtree_metadata(&resolved_new_normalized)?; if let Ok(destination_stat) = self.merged_lstat(&resolved_new_normalized) { if destination_stat.is_directory && !destination_stat.is_symbolic_link @@ -1678,7 +1680,6 @@ impl VirtualFileSystem for OverlayFileSystem { .remove_file(&resolved_new_normalized)?; } } - self.clear_subtree_metadata(&resolved_new_normalized)?; } self.stage_snapshot_entries_in_upper(&snapshot_entries)?; @@ -1985,6 +1986,31 @@ mod tests { assert_error_code(overlay.read_dir("/a"), "ENOENT"); } + #[test] + fn rename_clears_destination_whiteout() { + let lower = MemoryFileSystem::new(); + let mut overlay = OverlayFileSystem::new(vec![lower], OverlayMode::Ephemeral); + overlay + .write_file("/archive.zip", b"old".to_vec()) + .expect("create old archive"); + overlay + .remove_file("/archive.zip") + .expect("whiteout old archive"); + overlay + .write_file("/workspace-temp", b"new".to_vec()) + .expect("create replacement"); + + overlay + .rename("/workspace-temp", "/archive.zip") + .expect("rename replacement over whiteout"); + + assert_eq!( + overlay.read_file("/archive.zip").expect("read renamed file"), + b"new" + ); + assert!(!overlay.exists("/workspace-temp")); + } + #[test] fn remove_dir_still_rejects_visible_children() { let mut lower = MemoryFileSystem::new(); diff --git a/docs-internal/registry-parity-worklist.md b/docs-internal/registry-parity-worklist.md index 8c52a46a97..32e448a0a5 100644 --- a/docs-internal/registry-parity-worklist.md +++ b/docs-internal/registry-parity-worklist.md @@ -281,9 +281,21 @@ works (`wasi-spawn` broker), so `xargs` is not a blocker. `2026-07-08T06-58-56-0700-ripgrep-check-types-after-install.log`; e2e ripgrep tests pass 8/8 in `2026-07-08T07-02-00-0700-ripgrep-vitest-after-git-fixture.log`. Rev: `msypkqmo` — `fix(ripgrep): build upstream ripgrep`. -- **zip / unzip — moderate.** Real **Info-ZIP** source (fetch+pin like zlib/sqlite); - zlib is already vendored. Filesystem + `isatty`/`utime`/`chmod`/perms stubs; no - sockets/threads/spawn. Friction is Info-ZIP's crufty build, not syscalls. +- **zip / unzip — DONE.** Replaced both custom minizip-based C wrappers with real + upstream Info-ZIP Zip 3.0 and UnZip 6.0 release tarballs, built through the C + toolchain. Runtime/sysroot fixes stayed one layer down: temp-file, ownership, + `system`/`popen`/`pclose` compatibility in patched wasi-libc; overlay rename + over destination whiteouts; and WASI host-passthrough read/write offset + tracking after `fd_seek`. Proof: wasi-libc patch check passes in + `2026-07-08T08-34-29-0700-wasi-libc-patch-check-final.log`; native sidecar + build passes in `2026-07-08T08-34-07-0700-sidecar-build-final-runner-format.log`; + VFS rename regression passes in + `2026-07-08T08-34-29-0700-vfs-core-rename-whiteout-final.log`; final Zip e2e + passes 2/2 in + `2026-07-08T08-36-45-0700-zip-test-final-after-current-sysroot-copy-after-install.log`; + final UnZip e2e passes 6/6 in + `2026-07-08T08-37-12-0700-unzip-test-final-after-current-sysroot-copy-after-install.log`. + Rev: `nppnuxpr` — `fix(infozip): build upstream zip and unzip`. - **findutils — moderate→hard.** `find` is fs+regex (easy); `xargs` spawn already works. **uutils/findutils** (Rust) avoids gnulib; **GNU findutils** (C) hits the same gnulib cascade as wget. Prefer uutils unless GNU parity is required. @@ -396,19 +408,30 @@ so a reader sees the whole board at a glance. cookie-suffix checks, LDAP, and no CA bundle (cert trust is whatever `wasi_tls` enforces). Those are the 5 skipped tests. Verdict: solid for real HTTP(S). -### 7. zip / unzip — hostile-archive hardening cases fail (3 each) — DONE -- **Broken:** fallback parser doesn't reject a wrapping local offset, doesn't skip - empty-normalized-name entries, doesn't cap hostile uncompressed sizes before - allocating. -- **Objective:** unzip rejects/handles malformed & hostile archives the way a - hardened Linux unzip does (bounded allocation, typed errors), and zip↔unzip - roundtrips remain correct. -- **Proof:** `software/unzip/test/` passes 6/6 in - `2026-07-08T00-57-22-0700-item7-unzip-test-final-pass.txt`; `software/zip/test/` - passes 2/2 in `2026-07-08T00-57-22-0700-item7-zip-test-final-pass.txt`. - Package type checks pass in `2026-07-08T00-57-39-0700-item7-unzip-check-types.txt` - and `2026-07-08T00-57-39-0700-item7-zip-check-types.txt`. -- **rev:** `krxkqtnx` — `fix(unzip): harden fallback archive parsing` +### 7. zip / unzip — replace custom wrappers with real Info-ZIP — DONE +- **Broken:** the shipped commands were custom C wrappers over zlib/minizip, not + real Zip/UnZip. The old fallback parser also diverged from hardened Linux unzip + behavior on wrapping local offsets, empty normalized names, and hostile size + declarations. +- **Objective:** build real upstream Info-ZIP Zip and UnZip to `wasm32-wasip1`, + patching the AgentOS sysroot/runtime where needed, and prove real zip↔unzip + roundtrips plus malformed-archive rejection in VM e2e tests. +- **Resolved:** upstream Info-ZIP Zip 3.0 and UnZip 6.0 now build from release + tarballs through the C toolchain. The custom `software/zip/native/c/zip.c` and + `software/unzip/native/c/unzip.c` sources are deleted. Required compatibility + lives one layer down: patched wasi-libc temp-file, ownership, `system`, `popen`, + and `pclose` surfaces; VFS overlay rename-over-whiteout cleanup; and + host-passthrough `fd_seek` offset tracking in the WASI runner. +- **Proof:** wasi-libc patch check passes in + `2026-07-08T08-34-29-0700-wasi-libc-patch-check-final.log`; native sidecar + build passes in `2026-07-08T08-34-07-0700-sidecar-build-final-runner-format.log`; + VFS rename regression passes in + `2026-07-08T08-34-29-0700-vfs-core-rename-whiteout-final.log`; `software/zip` + e2e passes 2/2 in + `2026-07-08T08-36-45-0700-zip-test-final-after-current-sysroot-copy-after-install.log`; + `software/unzip` e2e passes 6/6 in + `2026-07-08T08-37-12-0700-unzip-test-final-after-current-sysroot-copy-after-install.log`. +- **rev:** `nppnuxpr` — `fix(infozip): build upstream zip and unzip` --- diff --git a/packages/runtime-core/commands/unzip b/packages/runtime-core/commands/unzip old mode 100644 new mode 100755 index ae4f0b9327..3bc6f3f020 Binary files a/packages/runtime-core/commands/unzip and b/packages/runtime-core/commands/unzip differ diff --git a/packages/runtime-core/commands/zip b/packages/runtime-core/commands/zip old mode 100644 new mode 100755 index 8324bdc9fe..edbd00f1c3 Binary files a/packages/runtime-core/commands/zip and b/packages/runtime-core/commands/zip differ diff --git a/software/unzip/bin/unzip b/software/unzip/bin/unzip old mode 100644 new mode 100755 index ae4f0b9327..3bc6f3f020 Binary files a/software/unzip/bin/unzip and b/software/unzip/bin/unzip differ diff --git a/software/unzip/native/c/unzip.c b/software/unzip/native/c/unzip.c deleted file mode 100644 index 1082a7e314..0000000000 --- a/software/unzip/native/c/unzip.c +++ /dev/null @@ -1,744 +0,0 @@ -/* unzip.c — Extract ZIP archives using zlib/minizip - * - * Usage: unzip archive.zip (extract all to cwd) - * unzip -d outdir archive.zip (extract to directory) - * unzip -l archive.zip (list contents) - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include "ioapi.h" -#include "unzip.h" - -#define MAX_PATH_LEN 4096 -#define WRITE_BUF_SIZE 8192 -#define MAX_ARCHIVE_BYTES (512u * 1024u * 1024u) - -/* Cap per-entry allocation in the fallback parser. Hostile central directory - * records can claim sizes up to 4 GiB; refuse anything above this bound. */ -#define MAX_UNCOMPRESSED_SIZE (256u * 1024u * 1024u) - -typedef struct { - FILE *file; - char *filename; - char mode[4]; - long position; - long size; -} unzip_file_stream; - -static voidpf ZCALLBACK unzip_open_file(voidpf opaque, const char *filename, int mode) { - unzip_file_stream *stream = NULL; - const char *mode_fopen = NULL; - (void)opaque; - - if ((mode & ZLIB_FILEFUNC_MODE_READWRITEFILTER) == ZLIB_FILEFUNC_MODE_READ) - mode_fopen = "rb"; - else if (mode & ZLIB_FILEFUNC_MODE_EXISTING) - mode_fopen = "r+b"; - else if (mode & ZLIB_FILEFUNC_MODE_CREATE) - mode_fopen = "wb"; - - if (filename == NULL || mode_fopen == NULL) - return NULL; - - stream = (unzip_file_stream *)calloc(1, sizeof(unzip_file_stream)); - if (!stream) - return NULL; - - stream->filename = (char *)malloc(strlen(filename) + 1); - if (!stream->filename) { - free(stream); - return NULL; - } - strcpy(stream->filename, filename); - strncpy(stream->mode, mode_fopen, sizeof(stream->mode) - 1); - stream->mode[sizeof(stream->mode) - 1] = '\0'; - - stream->file = fopen(filename, mode_fopen); - if (!stream->file) { - free(stream->filename); - free(stream); - return NULL; - } - struct stat st; - stream->size = stat(filename, &st) == 0 ? (long)st.st_size : 0; - stream->position = 0; - return stream; -} - -static uLong ZCALLBACK unzip_read_file(voidpf opaque, voidpf stream, void *buf, uLong size) { - unzip_file_stream *file_stream = (unzip_file_stream *)stream; - uLong got; - (void)opaque; - got = (uLong)fread(buf, 1, (size_t)size, file_stream->file); - file_stream->position += (long)got; - return got; -} - -static uLong ZCALLBACK unzip_write_file(voidpf opaque, voidpf stream, const void *buf, uLong size) { - unzip_file_stream *file_stream = (unzip_file_stream *)stream; - uLong wrote; - (void)opaque; - wrote = (uLong)fwrite(buf, 1, (size_t)size, file_stream->file); - file_stream->position += (long)wrote; - if (file_stream->position > file_stream->size) - file_stream->size = file_stream->position; - return wrote; -} - -static long ZCALLBACK unzip_tell_file(voidpf opaque, voidpf stream) { - unzip_file_stream *file_stream = (unzip_file_stream *)stream; - (void)opaque; - return file_stream->position; -} - -static long ZCALLBACK unzip_seek_file(voidpf opaque, voidpf stream, uLong offset, int origin) { - int fseek_origin = 0; - long seek_offset = (long)offset; - unzip_file_stream *file_stream = (unzip_file_stream *)stream; - (void)opaque; - - switch (origin) { - case ZLIB_FILEFUNC_SEEK_CUR: - seek_offset = file_stream->position + (long)offset; - fseek_origin = SEEK_SET; - break; - case ZLIB_FILEFUNC_SEEK_END: - seek_offset = file_stream->size + (long)offset; - fseek_origin = SEEK_SET; - break; - case ZLIB_FILEFUNC_SEEK_SET: - fseek_origin = SEEK_SET; - break; - default: - return -1; - } - - fclose(file_stream->file); - file_stream->file = fopen(file_stream->filename, file_stream->mode); - if (!file_stream->file) - return -1; - - if (fseek(file_stream->file, seek_offset, fseek_origin) != 0) - return -1; - clearerr(file_stream->file); - file_stream->position = seek_offset; - return 0; -} - -static int ZCALLBACK unzip_close_file(voidpf opaque, voidpf stream) { - unzip_file_stream *file_stream = (unzip_file_stream *)stream; - int ret; - (void)opaque; - ret = fclose(file_stream->file); - free(file_stream->filename); - free(file_stream); - return ret; -} - -static int ZCALLBACK unzip_error_file(voidpf opaque, voidpf stream) { - unzip_file_stream *file_stream = (unzip_file_stream *)stream; - (void)opaque; - return ferror(file_stream->file); -} - -static unzFile open_archive(const char *archive) { - zlib_filefunc_def filefunc = { - .zopen_file = unzip_open_file, - .zread_file = unzip_read_file, - .zwrite_file = unzip_write_file, - .ztell_file = unzip_tell_file, - .zseek_file = unzip_seek_file, - .zclose_file = unzip_close_file, - .zerror_file = unzip_error_file, - .opaque = NULL, - }; - return unzOpen2(archive, &filefunc); -} - -/* Ensure all parent directories of path exist */ -static int mkdirs(const char *path) { - char tmp[MAX_PATH_LEN]; - size_t len = strlen(path); - if (len >= sizeof(tmp)) return -1; - memcpy(tmp, path, len + 1); - - for (size_t i = 1; i < len; i++) { - if (tmp[i] == '/') { - tmp[i] = '\0'; - if (mkdir(tmp, 0755) != 0 && errno != EEXIST) - return -1; - tmp[i] = '/'; - } - } - return 0; -} - -static uint16_t read_le16(const unsigned char *p) { - return (uint16_t)p[0] | ((uint16_t)p[1] << 8); -} - -static uint32_t read_le32(const unsigned char *p) { - return (uint32_t)p[0] | ((uint32_t)p[1] << 8) | - ((uint32_t)p[2] << 16) | ((uint32_t)p[3] << 24); -} - -static int read_archive_bytes(const char *archive, unsigned char **out, size_t *out_len) { - int fd = open(archive, O_RDONLY); - size_t cap = WRITE_BUF_SIZE; - size_t len = 0; - unsigned char *data = NULL; - if (fd < 0) - return -1; - - data = (unsigned char *)malloc(cap); - if (!data) { - close(fd); - return -1; - } - - for (;;) { - if (len == cap) { - size_t next_cap = cap * 2; - unsigned char *next; - if (next_cap <= cap || next_cap > MAX_ARCHIVE_BYTES) { - free(data); - close(fd); - return -1; - } - next = (unsigned char *)realloc(data, next_cap); - if (!next) { - free(data); - close(fd); - return -1; - } - data = next; - cap = next_cap; - } - - ssize_t got = read(fd, data + len, cap - len); - if (got < 0) { - free(data); - close(fd); - return -1; - } - if (got == 0) - break; - len += (size_t)got; - } - - close(fd); - *out = data; - *out_len = len; - return 0; -} - -static int find_eocd(const unsigned char *data, size_t len, size_t *eocd_offset) { - size_t min = len > 0xffff + 22 ? len - (0xffff + 22) : 0; - if (len < 22) - return -1; - for (size_t pos = len - 22; pos + 4 <= len && pos >= min; pos--) { - if (read_le32(data + pos) == 0x06054b50) { - *eocd_offset = pos; - return 0; - } - if (pos == 0) - break; - } - return -1; -} - -static int locate_eocd(const unsigned char *data, size_t len, size_t *eocd_offset) { - if (len >= 22 && read_le32(data + len - 22) == 0x06054b50) { - *eocd_offset = len - 22; - return 0; - } - return find_eocd(data, len, eocd_offset); -} - -static const char *entry_output_name(const char *name, size_t name_len) { - const char *end = name + name_len; - while (name < end && *name == '/') - name++; - return name; -} - -static int inflate_raw_entry(const unsigned char *src, size_t src_len, unsigned char *dst, size_t dst_len) { - z_stream stream; - memset(&stream, 0, sizeof(stream)); - stream.next_in = (Bytef *)src; - stream.avail_in = (uInt)src_len; - stream.next_out = dst; - stream.avail_out = (uInt)dst_len; - if (inflateInit2(&stream, -MAX_WBITS) != Z_OK) - return -1; - int result = inflate(&stream, Z_FINISH); - inflateEnd(&stream); - return result == Z_STREAM_END && stream.total_out == dst_len ? 0 : -1; -} - -static int simple_archive_entries(const unsigned char *data, size_t len, size_t *cd_offset, uint16_t *entry_count) { - size_t eocd; - if (len < 22 || locate_eocd(data, len, &eocd) != 0 || eocd > len - 22) - return -1; - *entry_count = read_le16(data + eocd + 10); - *cd_offset = read_le32(data + eocd + 16); - return *cd_offset < len ? 0 : -1; -} - -static int archive_requires_fallback_parser(const char *archive) { - unsigned char *data = NULL; - size_t len = 0; - size_t eocd; - uint32_t cd_size; - uint32_t cd_offset; - int requires_fallback = 0; - - if (read_archive_bytes(archive, &data, &len) != 0) - return 0; - if (len >= 22) { - for (size_t pos = len - 22; pos + 22 <= len; pos--) { - if (read_le32(data + pos) == 0x06054b50 && - read_le32(data + pos + 12) == 0xffffffffu) { - requires_fallback = 1; - break; - } - if (pos == 0) - break; - } - } - if (len >= 22 && locate_eocd(data, len, &eocd) == 0 && eocd <= len - 22) { - cd_size = read_le32(data + eocd + 12); - cd_offset = read_le32(data + eocd + 16); - if ((size_t)cd_offset > eocd || - (size_t)cd_size > eocd - (size_t)cd_offset) { - requires_fallback = 1; - } - } - free(data); - return requires_fallback; -} - -static int simple_list_archive(const char *archive) { - unsigned char *data = NULL; - size_t len = 0; - size_t pos; - uint16_t entries; - unsigned long total_size = 0; - if (read_archive_bytes(archive, &data, &len) != 0 || - simple_archive_entries(data, len, &pos, &entries) != 0) { - free(data); - return 1; - } - - printf(" Length Name\n"); - printf("--------- ----\n"); - for (uint16_t i = 0; i < entries; i++) { - uint16_t name_len; - uint16_t extra_len; - uint16_t comment_len; - uint32_t uncompressed_size; - if (len < 46 || pos > len - 46 || read_le32(data + pos) != 0x02014b50) { - free(data); - return 1; - } - uncompressed_size = read_le32(data + pos + 24); - name_len = read_le16(data + pos + 28); - extra_len = read_le16(data + pos + 30); - comment_len = read_le16(data + pos + 32); - size_t header_len = 46 + (size_t)name_len + (size_t)extra_len + (size_t)comment_len; - if (header_len > len - pos) { - free(data); - return 1; - } - printf("%9lu %.*s\n", (unsigned long)uncompressed_size, name_len, data + pos + 46); - total_size += uncompressed_size; - pos += header_len; - } - printf("--------- ----\n"); - printf("%9lu %u file(s)\n", total_size, entries); - free(data); - return 0; -} - -static int simple_extract_archive(const char *archive, const char *outdir) { - unsigned char *data = NULL; - size_t len = 0; - size_t pos; - uint16_t entries; - int errors = 0; - if (read_archive_bytes(archive, &data, &len) != 0 || - simple_archive_entries(data, len, &pos, &entries) != 0) { - free(data); - return 1; - } - - if (outdir && mkdir(outdir, 0755) != 0 && errno != EEXIST) { - fprintf(stderr, "unzip: cannot create directory '%s': %s\n", outdir, strerror(errno)); - free(data); - return 1; - } - - for (uint16_t i = 0; i < entries; i++) { - uint16_t method; - uint16_t name_len; - uint16_t extra_len; - uint16_t comment_len; - uint16_t local_name_len; - uint16_t local_extra_len; - uint32_t compressed_size; - uint32_t uncompressed_size; - uint32_t local_offset; - size_t file_data_offset; - const char *name; - const char *safe_name; - char outpath[MAX_PATH_LEN]; - unsigned char *out = NULL; - - if (len < 46 || pos > len - 46 || read_le32(data + pos) != 0x02014b50) { - errors++; - break; - } - method = read_le16(data + pos + 10); - compressed_size = read_le32(data + pos + 20); - uncompressed_size = read_le32(data + pos + 24); - name_len = read_le16(data + pos + 28); - extra_len = read_le16(data + pos + 30); - comment_len = read_le16(data + pos + 32); - local_offset = read_le32(data + pos + 42); - size_t header_len = 46 + (size_t)name_len + (size_t)extra_len + (size_t)comment_len; - if (header_len > len - pos || len < 30 || (size_t)local_offset > len - 30) { - fprintf(stderr, "unzip: error reading local header for '%.*s'\n", - (int)name_len, data + pos + 46); - errors++; - break; - } - - name = (const char *)(data + pos + 46); - safe_name = entry_output_name(name, name_len); - size_t safe_len = (size_t)name_len - (size_t)(safe_name - name); - pos += header_len; - if (safe_len == 0) - continue; - snprintf(outpath, sizeof(outpath), "%s%s%.*s", - outdir ? outdir : "", outdir ? "/" : "", (int)safe_len, safe_name); - - size_t out_len = strlen(outpath); - if (out_len > 0 && outpath[out_len - 1] == '/') { - if (mkdir(outpath, 0755) != 0 && errno != EEXIST) - errors++; - continue; - } - if (mkdirs(outpath) != 0) { - errors++; - continue; - } - - if (read_le32(data + local_offset) != 0x04034b50) { - errors++; - continue; - } - local_name_len = read_le16(data + local_offset + 26); - local_extra_len = read_le16(data + local_offset + 28); - size_t local_header_len = 30 + (size_t)local_name_len + (size_t)local_extra_len; - if (local_header_len > len - (size_t)local_offset) { - errors++; - continue; - } - file_data_offset = (size_t)local_offset + local_header_len; - if ((size_t)compressed_size > len - file_data_offset) { - errors++; - continue; - } - - if (uncompressed_size > MAX_UNCOMPRESSED_SIZE) { - fprintf(stderr, "unzip: entry '%.*s' too large (%lu bytes)\n", - (int)safe_len, safe_name, (unsigned long)uncompressed_size); - errors++; - continue; - } - out = (unsigned char *)malloc(uncompressed_size > 0 ? uncompressed_size : 1); - if (!out) { - errors++; - continue; - } - if (method == 0) { - if (compressed_size != uncompressed_size) { - errors++; - free(out); - continue; - } - memcpy(out, data + file_data_offset, uncompressed_size); - } else if (method == Z_DEFLATED) { - if (inflate_raw_entry(data + file_data_offset, compressed_size, out, uncompressed_size) != 0) { - errors++; - free(out); - continue; - } - } else { - fprintf(stderr, "unzip: unsupported compression method %u for '%.*s'\n", method, name_len, name); - errors++; - free(out); - continue; - } - - int fd = open(outpath, O_WRONLY | O_CREAT | O_TRUNC, 0644); - if (fd < 0) { - fprintf(stderr, "unzip: cannot create '%s': %s\n", outpath, strerror(errno)); - errors++; - free(out); - continue; - } - size_t written = 0; - while (written < uncompressed_size) { - ssize_t n = write(fd, out + written, uncompressed_size - written); - if (n <= 0) { - errors++; - break; - } - written += (size_t)n; - } - close(fd); - free(out); - } - - free(data); - if (errors > 0) { - fprintf(stderr, "unzip: completed with %d error(s)\n", errors); - fflush(stderr); - return 1; - } - return 0; -} - -/* List archive contents */ -static int list_archive(const char *archive) { - if (archive_requires_fallback_parser(archive)) - return simple_list_archive(archive); - - unzFile uf = open_archive(archive); - if (!uf) { - return simple_list_archive(archive); - } - - unz_global_info gi; - if (unzGetGlobalInfo(uf, &gi) != UNZ_OK) { - fprintf(stderr, "unzip: cannot read archive info\n"); - unzClose(uf); - return 1; - } - - printf(" Length Name\n"); - printf("--------- ----\n"); - - unsigned long total_size = 0; - for (uLong i = 0; i < gi.number_entry; i++) { - char filename[MAX_PATH_LEN]; - unz_file_info fi; - if (unzGetCurrentFileInfo(uf, &fi, filename, sizeof(filename), - NULL, 0, NULL, 0) != UNZ_OK) { - fprintf(stderr, "unzip: error reading file info\n"); - unzClose(uf); - return 1; - } - - printf("%9lu %s\n", fi.uncompressed_size, filename); - total_size += fi.uncompressed_size; - - if (i + 1 < gi.number_entry) { - if (unzGoToNextFile(uf) != UNZ_OK) { - fprintf(stderr, "unzip: error iterating archive\n"); - unzClose(uf); - return 1; - } - } - } - - printf("--------- ----\n"); - printf("%9lu %lu file(s)\n", total_size, gi.number_entry); - - unzClose(uf); - return 0; -} - -/* Extract a single file from the archive */ -static int extract_current_file(unzFile uf, const char *outdir) { - char filename[MAX_PATH_LEN]; - unz_file_info fi; - if (unzGetCurrentFileInfo(uf, &fi, filename, sizeof(filename), - NULL, 0, NULL, 0) != UNZ_OK) { - fprintf(stderr, "unzip: error reading file info\n"); - return -1; - } - - /* Build output path */ - char outpath[MAX_PATH_LEN]; - if (outdir) { - snprintf(outpath, sizeof(outpath), "%s/%s", outdir, filename); - } else { - snprintf(outpath, sizeof(outpath), "%s", filename); - } - - /* Directory entry (trailing slash) */ - size_t namelen = strlen(outpath); - if (namelen > 0 && outpath[namelen - 1] == '/') { - if (mkdir(outpath, 0755) != 0 && errno != EEXIST) { - fprintf(stderr, "unzip: cannot create directory '%s': %s\n", - outpath, strerror(errno)); - return -1; - } - return 0; - } - - /* Ensure parent directory exists */ - if (mkdirs(outpath) != 0) { - fprintf(stderr, "unzip: cannot create parent directories for '%s'\n", outpath); - return -1; - } - - if (unzOpenCurrentFile(uf) != UNZ_OK) { - fprintf(stderr, "unzip: cannot open '%s' in archive\n", filename); - return -1; - } - - FILE *fout = fopen(outpath, "wb"); - if (!fout) { - fprintf(stderr, "unzip: cannot create '%s': %s\n", outpath, strerror(errno)); - unzCloseCurrentFile(uf); - return -1; - } - - unsigned char buf[WRITE_BUF_SIZE]; - int err = UNZ_OK; - int bytes; - while ((bytes = unzReadCurrentFile(uf, buf, sizeof(buf))) > 0) { - if (fwrite(buf, 1, (size_t)bytes, fout) != (size_t)bytes) { - fprintf(stderr, "unzip: error writing '%s'\n", outpath); - err = -1; - break; - } - } - if (bytes < 0) { - fprintf(stderr, "unzip: error reading '%s' from archive\n", filename); - err = -1; - } - - fclose(fout); - unzCloseCurrentFile(uf); - return err; -} - -/* Extract all files from the archive */ -static int extract_archive(const char *archive, const char *outdir) { - if (archive_requires_fallback_parser(archive)) - return simple_extract_archive(archive, outdir); - - unzFile uf = open_archive(archive); - if (!uf) { - return simple_extract_archive(archive, outdir); - } - - /* Create output directory if specified */ - if (outdir) { - if (mkdir(outdir, 0755) != 0 && errno != EEXIST) { - fprintf(stderr, "unzip: cannot create directory '%s': %s\n", - outdir, strerror(errno)); - unzClose(uf); - return 1; - } - } - - unz_global_info gi; - if (unzGetGlobalInfo(uf, &gi) != UNZ_OK) { - unzClose(uf); - return simple_extract_archive(archive, outdir); - } - - int errors = 0; - int retry_simple = 0; - for (uLong i = 0; i < gi.number_entry; i++) { - if (extract_current_file(uf, outdir) != 0) { - errors++; - retry_simple = 1; - break; - } - - if (i + 1 < gi.number_entry) { - if (unzGoToNextFile(uf) != UNZ_OK) { - unzClose(uf); - return simple_extract_archive(archive, outdir); - } - } - } - - unzClose(uf); - - if (retry_simple) - return simple_extract_archive(archive, outdir); - - if (errors > 0) { - fprintf(stderr, "unzip: completed with %d error(s)\n", errors); - return 1; - } - return 0; -} - -static void print_usage(void) { - fprintf(stderr, "Usage: unzip [-l] [-d dir] archive.zip\n"); - fprintf(stderr, " -l List archive contents\n"); - fprintf(stderr, " -d dir Extract to directory\n"); -} - -int main(int argc, char *argv[]) { - if (argc < 2) { - print_usage(); - return 1; - } - - int list_mode = 0; - const char *outdir = NULL; - const char *archive = NULL; - int i = 1; - - /* Parse flags */ - while (i < argc && argv[i][0] == '-') { - if (strcmp(argv[i], "-l") == 0) { - list_mode = 1; - i++; - } else if (strcmp(argv[i], "-d") == 0) { - if (i + 1 >= argc) { - fprintf(stderr, "unzip: -d requires a directory argument\n"); - return 1; - } - outdir = argv[i + 1]; - i += 2; - } else if (strcmp(argv[i], "--") == 0) { - i++; - break; - } else { - fprintf(stderr, "unzip: unknown option '%s'\n", argv[i]); - print_usage(); - return 1; - } - } - - if (i >= argc) { - fprintf(stderr, "unzip: no archive specified\n"); - print_usage(); - return 1; - } - - archive = argv[i]; - - if (list_mode) { - return list_archive(archive); - } else { - return extract_archive(archive, outdir); - } -} diff --git a/software/unzip/test/unzip.test.ts b/software/unzip/test/unzip.test.ts index b86c768dde..b6081dc2e2 100644 --- a/software/unzip/test/unzip.test.ts +++ b/software/unzip/test/unzip.test.ts @@ -137,7 +137,7 @@ describeIf( } }); - it("fallback parser rejects an entry with a wrapping local offset", async () => { + it("rejects an entry with a wrapping local offset", async () => { const vfs = createInMemoryFileSystem(); const bytes = buildFallbackArchive(new Uint8Array(0), [ { @@ -156,12 +156,12 @@ describeIf( ); const result = await kernel.exec("unzip -d /out /evil.zip"); - expect(result.exitCode, result.stderr).toBe(1); + expect(result.exitCode, result.stderr).not.toBe(0); expect(result.stderr).toMatch(/error/); expect(await vfs.exists("/out/evil.txt")).toBe(false); }); - it("fallback parser skips an entry whose normalized name is empty", async () => { + it("rejects an entry whose normalized name is empty", async () => { const vfs = createInMemoryFileSystem(); const bytes = buildFallbackArchive(new Uint8Array(0), [ { @@ -180,12 +180,11 @@ describeIf( ); const result = await kernel.exec("unzip /empty-name.zip"); - expect(result.exitCode, result.stderr).toBe(0); - expect(result.stdout).not.toMatch(/error/); - expect(result.stderr).not.toMatch(/error/); + expect(result.exitCode, result.stderr).not.toBe(0); + expect(result.stderr).toMatch(/error/); }); - it("fallback parser caps hostile uncompressed sizes before allocating", async () => { + it("rejects hostile uncompressed sizes before extracting", async () => { const vfs = createInMemoryFileSystem(); const prefix = new Uint8Array(31); const pdv = new DataView(prefix.buffer); @@ -211,8 +210,8 @@ describeIf( ); const result = await kernel.exec("unzip -d /cap-out /big.zip"); - expect(result.exitCode, result.stderr).toBe(1); - expect(result.stderr).toMatch(/too large/); + expect(result.exitCode, result.stderr).not.toBe(0); + expect(result.stderr).toMatch(/error/); expect(await vfs.exists("/cap-out/big.bin")).toBe(false); }); }, diff --git a/software/zip/bin/zip b/software/zip/bin/zip old mode 100644 new mode 100755 index 8324bdc9fe..edbd00f1c3 Binary files a/software/zip/bin/zip and b/software/zip/bin/zip differ diff --git a/software/zip/native/c/zip.c b/software/zip/native/c/zip.c deleted file mode 100644 index d8eca7a5bd..0000000000 --- a/software/zip/native/c/zip.c +++ /dev/null @@ -1,203 +0,0 @@ -/* zip.c — Create ZIP archives using zlib/minizip - * - * Usage: zip [-r] archive.zip file1 [file2 ...] - * -r Recurse into directories - */ - -#include -#include -#include -#include -#include -#include -#include "zip.h" - -#define MAX_PATH_LEN 4096 -#define READ_BUF_SIZE 8192 - -/* Add a single file to the zip archive */ -static int add_file_to_zip(zipFile zf, const char *filepath, const char *archivepath) { - FILE *fin = fopen(filepath, "rb"); - if (!fin) { - fprintf(stderr, "zip: cannot open '%s': ", filepath); - perror(""); - return -1; - } - - zip_fileinfo zi; - memset(&zi, 0, sizeof(zi)); - - /* Get file modification time */ - struct stat st; - if (stat(filepath, &st) == 0) { - struct tm *lt = localtime(&st.st_mtime); - if (lt) { - zi.tmz_date.tm_sec = lt->tm_sec; - zi.tmz_date.tm_min = lt->tm_min; - zi.tmz_date.tm_hour = lt->tm_hour; - zi.tmz_date.tm_mday = lt->tm_mday; - zi.tmz_date.tm_mon = lt->tm_mon; - zi.tmz_date.tm_year = lt->tm_year; - } - } - - int err = zipOpenNewFileInZip(zf, archivepath, &zi, - NULL, 0, NULL, 0, NULL, - Z_DEFLATED, Z_DEFAULT_COMPRESSION); - if (err != ZIP_OK) { - fprintf(stderr, "zip: error opening '%s' in archive\n", archivepath); - fclose(fin); - return -1; - } - - unsigned char buf[READ_BUF_SIZE]; - size_t n; - while ((n = fread(buf, 1, sizeof(buf), fin)) > 0) { - if (zipWriteInFileInZip(zf, buf, (unsigned int)n) != ZIP_OK) { - fprintf(stderr, "zip: error writing '%s' to archive\n", archivepath); - fclose(fin); - zipCloseFileInZip(zf); - return -1; - } - } - - fclose(fin); - zipCloseFileInZip(zf); - return 0; -} - -/* Recursively add directory contents to zip */ -static int add_dir_to_zip(zipFile zf, const char *dirpath, const char *archivebase) { - DIR *d = opendir(dirpath); - if (!d) { - fprintf(stderr, "zip: cannot open directory '%s': ", dirpath); - perror(""); - return -1; - } - - /* Add directory entry (trailing slash) */ - char direntry[MAX_PATH_LEN]; - snprintf(direntry, sizeof(direntry), "%s/", archivebase); - zip_fileinfo zi; - memset(&zi, 0, sizeof(zi)); - zipOpenNewFileInZip(zf, direntry, &zi, NULL, 0, NULL, 0, NULL, 0, 0); - zipCloseFileInZip(zf); - - struct dirent *entry; - int err = 0; - while ((entry = readdir(d)) != NULL) { - if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) - continue; - - char fullpath[MAX_PATH_LEN]; - char arcpath[MAX_PATH_LEN]; - snprintf(fullpath, sizeof(fullpath), "%s/%s", dirpath, entry->d_name); - snprintf(arcpath, sizeof(arcpath), "%s/%s", archivebase, entry->d_name); - - struct stat st; - if (stat(fullpath, &st) != 0) { - fprintf(stderr, "zip: cannot stat '%s': ", fullpath); - perror(""); - err = -1; - continue; - } - - if (S_ISDIR(st.st_mode)) { - if (add_dir_to_zip(zf, fullpath, arcpath) != 0) - err = -1; - } else { - if (add_file_to_zip(zf, fullpath, arcpath) != 0) - err = -1; - } - } - - closedir(d); - return err; -} - -static void print_usage(void) { - fprintf(stderr, "Usage: zip [-r] archive.zip file1 [file2 ...]\n"); - fprintf(stderr, " -r Recurse into directories\n"); -} - -int main(int argc, char *argv[]) { - if (argc < 3) { - print_usage(); - return 1; - } - - int recursive = 0; - int arg_start = 1; - - /* Parse flags */ - while (arg_start < argc && argv[arg_start][0] == '-') { - if (strcmp(argv[arg_start], "-r") == 0) { - recursive = 1; - arg_start++; - } else if (strcmp(argv[arg_start], "--") == 0) { - arg_start++; - break; - } else { - fprintf(stderr, "zip: unknown option '%s'\n", argv[arg_start]); - print_usage(); - return 1; - } - } - - if (argc - arg_start < 2) { - print_usage(); - return 1; - } - - const char *archive = argv[arg_start]; - arg_start++; - - zipFile zf = zipOpen(archive, APPEND_STATUS_CREATE); - if (!zf) { - fprintf(stderr, "zip: cannot create '%s'\n", archive); - return 1; - } - - int errors = 0; - for (int i = arg_start; i < argc; i++) { - const char *path = argv[i]; - - /* Strip trailing slashes for consistent archive paths */ - char cleanpath[MAX_PATH_LEN]; - strncpy(cleanpath, path, sizeof(cleanpath) - 1); - cleanpath[sizeof(cleanpath) - 1] = '\0'; - size_t len = strlen(cleanpath); - while (len > 1 && cleanpath[len - 1] == '/') - cleanpath[--len] = '\0'; - - struct stat st; - if (stat(cleanpath, &st) != 0) { - fprintf(stderr, "zip: cannot stat '%s': ", cleanpath); - perror(""); - errors++; - continue; - } - - if (S_ISDIR(st.st_mode)) { - if (!recursive) { - fprintf(stderr, "zip: '%s' is a directory (use -r to recurse)\n", cleanpath); - errors++; - continue; - } - if (add_dir_to_zip(zf, cleanpath, cleanpath) != 0) - errors++; - } else { - if (add_file_to_zip(zf, cleanpath, cleanpath) != 0) - errors++; - } - } - - zipClose(zf, NULL); - - if (errors > 0) { - fprintf(stderr, "zip: completed with %d error(s)\n", errors); - return 1; - } - - return 0; -} diff --git a/toolchain/c/Makefile b/toolchain/c/Makefile index 991118950b..17ad4c2cd4 100644 --- a/toolchain/c/Makefile +++ b/toolchain/c/Makefile @@ -71,7 +71,7 @@ COMMANDS_DIR ?= ../target/wasm32-wasip1/release/commands COMMANDS := zip unzip envsubst sqlite3 curl wget grep tree # Programs requiring patched sysroot (Tier 2+ custom host imports) -PATCHED_PROGRAMS := http_get_test isatty_test getpid_test getppid_test getppid_verify userinfo pipe_test dup_test spawn_child spawn_exit_code pipeline kill_child waitpid_return waitpid_edge syscall_coverage getpwuid_test signal_tests sigaction_self sigaction_behavior delayed_tcp_echo delayed_kill pipe_edge tcp_accept_spawn tcp_echo tcp_server http_server udp_echo unix_socket signal_handler dns_lookup sqlite3 curl wget grep tree fs_probe +PATCHED_PROGRAMS := http_get_test isatty_test getpid_test getppid_test getppid_verify userinfo pipe_test dup_test spawn_child spawn_exit_code pipeline kill_child waitpid_return waitpid_edge syscall_coverage getpwuid_test signal_tests sigaction_self sigaction_behavior delayed_tcp_echo delayed_kill pipe_edge tcp_accept_spawn tcp_echo tcp_server http_server udp_echo unix_socket signal_handler dns_lookup sqlite3 curl wget grep tree zip unzip fs_probe # Discover all package command and test-program C source files. ALL_SOURCES := $(foreach dir,$(C_SOURCE_DIRS),$(wildcard $(dir)/*.c)) @@ -89,7 +89,7 @@ endif ifeq ($(wildcard $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a),) CUSTOM_WASM_PROG_NAMES := else - CUSTOM_WASM_PROG_NAMES := curl wget grep sqlite3 tree + CUSTOM_WASM_PROG_NAMES := curl wget grep sqlite3 tree zip unzip endif WASM_PROG_NAMES := $(sort $(basename $(notdir $(SOURCES))) $(CUSTOM_WASM_PROG_NAMES)) @@ -136,7 +136,6 @@ $(WASI_SDK_DIR)/bin/clang: # All downloads cached in libs/ — add libs/ to .gitignore. SQLITE3_URL := https://www.sqlite.org/2024/sqlite-amalgamation-3470200.zip -ZLIB_URL := https://github.com/madler/zlib/archive/refs/tags/v1.3.1.zip CJSON_URL := https://github.com/DaveGamble/cJSON/archive/refs/tags/v1.7.18.zip CURL_COMMIT := main CURL_FORK_REPO_PREFIX := secure @@ -159,9 +158,14 @@ WGET_UPSTREAM_OVERLAY_FILES := $(wildcard $(WGET_UPSTREAM_OVERLAY_INCLUDE_DIR)/* GREP_VERSION := 3.12 GREP_URL := https://ftp.gnu.org/gnu/grep/grep-$(GREP_VERSION).tar.xz GREP_UPSTREAM_BUILD_DIR := $(BUILD_DIR)/grep-upstream +INFOZIP_ZIP_VERSION := 30 +INFOZIP_ZIP_URL := https://downloads.sourceforge.net/infozip/zip$(INFOZIP_ZIP_VERSION).tar.gz +INFOZIP_ZIP_BUILD_DIR := $(BUILD_DIR)/infozip-zip-upstream +INFOZIP_UNZIP_VERSION := 60 +INFOZIP_UNZIP_URL := https://downloads.sourceforge.net/infozip/unzip$(INFOZIP_UNZIP_VERSION).tar.gz +INFOZIP_UNZIP_BUILD_DIR := $(BUILD_DIR)/infozip-unzip-upstream TREE_VERSION := 2.3.2 TREE_URL := https://gitlab.com/OldManProgrammer/unix-tree/-/archive/$(TREE_VERSION)/unix-tree-$(TREE_VERSION).tar.gz -MINIZIP_URL := https://github.com/madler/zlib/archive/refs/tags/v1.3.1.zip # duckdb-wasm currently documents DuckDB v1.5.0 in its README. We build the # matching upstream DuckDB tag ourselves with our patched WASI/POSIX sysroot. DUCKDB_VERSION := v1.5.0 @@ -196,26 +200,6 @@ $(LIBS_DIR)/tree/.fetched: @cp $(addprefix $(LIBS_CACHE)/unix-tree-$(TREE_VERSION)/,$(TREE_SOURCES) tree.h) $(LIBS_DIR)/tree/ @touch $@ -libs/zlib/zutil.c: - @echo "Fetching zlib..." - @mkdir -p $(LIBS_CACHE) $(LIBS_DIR)/zlib - @curl -fSL "$(ZLIB_URL)" -o "$(LIBS_CACHE)/zlib.zip" - @cd $(LIBS_CACHE) && unzip -qo zlib.zip - @cp $(LIBS_CACHE)/zlib-*/*.c $(LIBS_CACHE)/zlib-*/*.h $(LIBS_DIR)/zlib/ - -libs/minizip/ioapi.c: - @echo "Fetching minizip (from zlib contrib)..." - @mkdir -p $(LIBS_CACHE) $(LIBS_DIR)/minizip - @if [ ! -f "$(LIBS_CACHE)/zlib.zip" ]; then \ - curl -fSL "$(MINIZIP_URL)" -o "$(LIBS_CACHE)/zlib.zip"; \ - cd $(LIBS_CACHE) && unzip -qo zlib.zip; \ - fi - @cp $(LIBS_CACHE)/zlib-*/contrib/minizip/ioapi.c $(LIBS_CACHE)/zlib-*/contrib/minizip/ioapi.h \ - $(LIBS_CACHE)/zlib-*/contrib/minizip/zip.c $(LIBS_CACHE)/zlib-*/contrib/minizip/zip.h \ - $(LIBS_CACHE)/zlib-*/contrib/minizip/unzip.c $(LIBS_CACHE)/zlib-*/contrib/minizip/unzip.h \ - $(LIBS_CACHE)/zlib-*/contrib/minizip/crypt.h \ - $(LIBS_DIR)/minizip/ - libs/cjson/cJSON.c: @echo "Fetching cJSON..." @mkdir -p $(LIBS_CACHE) $(LIBS_DIR)/cjson @@ -240,8 +224,8 @@ libs/duckdb/CMakeLists.txt: @tar -xzf "$(LIBS_CACHE)/duckdb.tar.gz" --strip-components=1 -C $(LIBS_DIR)/duckdb .PHONY: fetch-libs fetch-fast-libs clean-libs -fetch-libs: libs/sqlite3/sqlite3.c libs/zlib/zutil.c libs/minizip/ioapi.c libs/cjson/cJSON.c libs/curl/lib/easy.c libs/duckdb/CMakeLists.txt -fetch-fast-libs: libs/sqlite3/sqlite3.c libs/zlib/zutil.c libs/minizip/ioapi.c libs/cjson/cJSON.c libs/curl/lib/easy.c +fetch-libs: libs/sqlite3/sqlite3.c libs/cjson/cJSON.c libs/curl/lib/easy.c libs/duckdb/CMakeLists.txt +fetch-fast-libs: libs/sqlite3/sqlite3.c libs/cjson/cJSON.c libs/curl/lib/easy.c clean-libs: rm -rf $(LIBS_DIR) $(LIBS_CACHE) @@ -552,43 +536,28 @@ $(NATIVE_DIR)/tree: $(TREE_FILES) $(NATIVE_CC) $(NATIVE_CFLAGS) $(TREE_WASM_FLAGS) -I$(LIBS_DIR)/tree -o $@ \ $(addprefix $(LIBS_DIR)/tree/,$(TREE_SOURCES)) -# zip: links zlib + minizip -ZLIB_SRCS := libs/zlib/adler32.c libs/zlib/compress.c libs/zlib/crc32.c libs/zlib/deflate.c \ - libs/zlib/infback.c libs/zlib/inffast.c libs/zlib/inflate.c libs/zlib/inftrees.c \ - libs/zlib/trees.c libs/zlib/uncompr.c libs/zlib/zutil.c -MINIZIP_SRCS := libs/minizip/ioapi.c libs/minizip/zip.c -ZIP_INCLUDES := -Ilibs/zlib -Ilibs/minizip - -$(BUILD_DIR)/zip: $(call c_source,zip) $(ZLIB_SRCS) $(MINIZIP_SRCS) $(WASI_SDK_DIR)/bin/clang +# zip/unzip: upstream Info-ZIP releases built against the patched C sysroot. +$(BUILD_DIR)/zip: scripts/build-infozip-upstream.sh $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a $(WASI_SDK_DIR)/bin/clang @mkdir -p $(BUILD_DIR) - $(CC) $(WASM_CFLAGS) $(ZIP_INCLUDES) -o $@.wasm $(call c_source,zip) $(ZLIB_SRCS) $(MINIZIP_SRCS) - @if [ "$(HAS_WASM_OPT)" = "1" ]; then \ - wasm-opt -O3 --strip-debug $@.wasm -o $@; \ - else \ - cp $@.wasm $@; \ - fi - @rm -f $@.wasm - -$(NATIVE_DIR)/zip: $(call c_source,zip) $(ZLIB_SRCS) $(MINIZIP_SRCS) - @mkdir -p $(NATIVE_DIR) - $(NATIVE_CC) $(NATIVE_CFLAGS) $(ZIP_INCLUDES) -o $@ $(call c_source,zip) $(ZLIB_SRCS) $(MINIZIP_SRCS) - -# unzip: links zlib + minizip (unzip side) -MINIZIP_UNZIP_SRCS := libs/minizip/ioapi.c libs/minizip/unzip.c + bash scripts/build-infozip-upstream.sh \ + --tool zip \ + --version "$(INFOZIP_ZIP_VERSION)" \ + --url "$(INFOZIP_ZIP_URL)" \ + --cache-dir "$(abspath $(LIBS_CACHE))" \ + --build-dir "$(abspath $(INFOZIP_ZIP_BUILD_DIR))" \ + --cc "$(abspath $(CC)) --target=wasm32-wasip1 --sysroot=$(abspath $(SYSROOT))" \ + --output "$(abspath $@)" -$(BUILD_DIR)/unzip: $(call c_source,unzip) $(ZLIB_SRCS) $(MINIZIP_UNZIP_SRCS) $(WASI_SDK_DIR)/bin/clang +$(BUILD_DIR)/unzip: scripts/build-infozip-upstream.sh $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a $(WASI_SDK_DIR)/bin/clang @mkdir -p $(BUILD_DIR) - $(CC) $(WASM_CFLAGS) $(ZIP_INCLUDES) -o $@.wasm $(call c_source,unzip) $(ZLIB_SRCS) $(MINIZIP_UNZIP_SRCS) - @if [ "$(HAS_WASM_OPT)" = "1" ]; then \ - wasm-opt -O3 --strip-debug $@.wasm -o $@; \ - else \ - cp $@.wasm $@; \ - fi - @rm -f $@.wasm - -$(NATIVE_DIR)/unzip: $(call c_source,unzip) $(ZLIB_SRCS) $(MINIZIP_UNZIP_SRCS) - @mkdir -p $(NATIVE_DIR) - $(NATIVE_CC) $(NATIVE_CFLAGS) $(ZIP_INCLUDES) -o $@ $(call c_source,unzip) $(ZLIB_SRCS) $(MINIZIP_UNZIP_SRCS) + bash scripts/build-infozip-upstream.sh \ + --tool unzip \ + --version "$(INFOZIP_UNZIP_VERSION)" \ + --url "$(INFOZIP_UNZIP_URL)" \ + --cache-dir "$(abspath $(LIBS_CACHE))" \ + --build-dir "$(abspath $(INFOZIP_UNZIP_BUILD_DIR))" \ + --cc "$(abspath $(CC)) --target=wasm32-wasip1 --sysroot=$(abspath $(SYSROOT))" \ + --output "$(abspath $@)" # curl_test: links libcurl (HTTP/HTTPS build for WASM via host_net + host_tls) CURL_SRCS := $(wildcard libs/curl/lib/*.c) $(wildcard libs/curl/lib/vauth/*.c) \ diff --git a/toolchain/c/scripts/build-infozip-upstream.sh b/toolchain/c/scripts/build-infozip-upstream.sh new file mode 100755 index 0000000000..1e765bb04f --- /dev/null +++ b/toolchain/c/scripts/build-infozip-upstream.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: build-infozip-upstream.sh \ + --tool \ + --version \ + --url \ + --cache-dir \ + --build-dir \ + --cc \ + --output +EOF +} + +TOOL="" +VERSION="" +URL="" +CACHE_DIR="" +BUILD_DIR="" +CC_CMD="" +OUTPUT="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --tool) + TOOL="$2" + shift 2 + ;; + --version) + VERSION="$2" + shift 2 + ;; + --url) + URL="$2" + shift 2 + ;; + --cache-dir) + CACHE_DIR="$2" + shift 2 + ;; + --build-dir) + BUILD_DIR="$2" + shift 2 + ;; + --cc) + CC_CMD="$2" + shift 2 + ;; + --output) + OUTPUT="$2" + shift 2 + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +if [[ -z "$TOOL" || -z "$VERSION" || -z "$URL" || -z "$CACHE_DIR" || -z "$BUILD_DIR" || -z "$CC_CMD" || -z "$OUTPUT" ]]; then + usage >&2 + exit 1 +fi + +case "$TOOL" in + zip|unzip) ;; + *) + echo "Unsupported Info-ZIP tool: $TOOL" >&2 + usage >&2 + exit 1 + ;; +esac + +fetch() { + local url="$1" + local out="$2" + if command -v curl >/dev/null 2>&1; then + curl --retry 3 --retry-all-errors -fSL "$url" -o "$out" + elif command -v wget >/dev/null 2>&1; then + wget -q "$url" -O "$out" + else + echo "Neither curl nor wget is available to fetch $url" >&2 + exit 1 + fi +} + +mkdir -p "$CACHE_DIR" +rm -rf "$BUILD_DIR" +mkdir -p "$BUILD_DIR" + +TARBALL="$CACHE_DIR/infozip-${TOOL}-${VERSION}.tar.gz" +if [[ ! -f "$TARBALL" ]]; then + echo "Fetching upstream Info-ZIP ${TOOL} ${VERSION} release tarball..." + fetch "$URL" "$TARBALL" +fi + +echo "Extracting upstream Info-ZIP ${TOOL} ${VERSION}..." +tar -xzf "$TARBALL" -C "$BUILD_DIR" + +SRC_DIR="$BUILD_DIR/${TOOL}${VERSION}" +if [[ ! -d "$SRC_DIR" ]]; then + echo "Expected extracted source at $SRC_DIR" >&2 + exit 1 +fi + +pushd "$SRC_DIR" >/dev/null + +case "$TOOL" in + zip) + echo "Building upstream Info-ZIP Zip ${VERSION} for wasm32-wasip1..." + make -f unix/Makefile clean + make -f unix/Makefile zips \ + CC="$CC_CMD" \ + BIND="$CC_CMD" \ + CFLAGS="-O2 -I. -DUNIX -DNO_ASM -DNO_BZIP2 -DNO_LCHMOD -DNO_LCHOWN -DLARGE_FILE_SUPPORT" \ + LFLAGS2="" + BIN="zip" + ;; + unzip) + echo "Building upstream Info-ZIP UnZip ${VERSION} for wasm32-wasip1..." + make -f unix/Makefile clean + make -f unix/Makefile unzips \ + CC="$CC_CMD" \ + LD="$CC_CMD" \ + CF="-O2 -I. -DUNIX -DBSD4_4 -DNO_PARAM_H -DNO_LCHMOD -DNO_LCHOWN -DNO_SYMLINK -DLARGE_FILE_SUPPORT" \ + LF2="" \ + SL2="" \ + FL2="" + BIN="unzip" + ;; +esac + +if [[ ! -f "$BIN" ]]; then + echo "Unable to locate built Info-ZIP $TOOL binary" >&2 + exit 1 +fi + +mkdir -p "$(dirname "$OUTPUT")" +if command -v wasm-opt >/dev/null 2>&1; then + echo "Optimizing Info-ZIP $TOOL WASM binary..." + wasm-opt -O3 --strip-debug --all-features "$BIN" -o "$OUTPUT" +else + cp "$BIN" "$OUTPUT" +fi + +popd >/dev/null + +echo "Built upstream Info-ZIP $TOOL at $OUTPUT" diff --git a/toolchain/std-patches/wasi-libc-overrides/ownership.c b/toolchain/std-patches/wasi-libc-overrides/ownership.c new file mode 100644 index 0000000000..909e43f15d --- /dev/null +++ b/toolchain/std-patches/wasi-libc-overrides/ownership.c @@ -0,0 +1,55 @@ +/** + * Minimal ownership and umask surface for POSIX software on AgentOS WASI. + * + * AgentOS tracks file mode bits, but does not currently model uid/gid ownership + * mutation in the VFS. Expose the POSIX calls so upstream programs can build + * unchanged. Ownership changes fail as unsupported at runtime; umask is a + * process-local value used by programs that calculate extracted file modes. + */ + +#include +#include +#include +#include + +static mode_t current_umask = 0022; + +mode_t umask(mode_t mask) { + mode_t previous = current_umask; + current_umask = mask & 0777; + return previous; +} + +int chown(const char *path, uid_t owner, gid_t group) { + (void)path; + (void)owner; + (void)group; + errno = ENOSYS; + return -1; +} + +int fchown(int fd, uid_t owner, gid_t group) { + (void)fd; + (void)owner; + (void)group; + errno = ENOSYS; + return -1; +} + +int lchown(const char *path, uid_t owner, gid_t group) { + (void)path; + (void)owner; + (void)group; + errno = ENOSYS; + return -1; +} + +int fchownat(int fd, const char *path, uid_t owner, gid_t group, int flags) { + (void)fd; + (void)path; + (void)owner; + (void)group; + (void)flags; + errno = ENOSYS; + return -1; +} diff --git a/toolchain/std-patches/wasi-libc-overrides/system.c b/toolchain/std-patches/wasi-libc-overrides/system.c new file mode 100644 index 0000000000..c3a6f3a9c2 --- /dev/null +++ b/toolchain/std-patches/wasi-libc-overrides/system.c @@ -0,0 +1,24 @@ +#include +#include +#include + +int system(const char *command) { + if (command == NULL) { + return 0; + } + errno = ENOSYS; + return -1; +} + +FILE *popen(const char *command, const char *mode) { + (void)command; + (void)mode; + errno = ENOSYS; + return NULL; +} + +int pclose(FILE *stream) { + (void)stream; + errno = ENOSYS; + return -1; +} diff --git a/toolchain/std-patches/wasi-libc-overrides/tempfile.c b/toolchain/std-patches/wasi-libc-overrides/tempfile.c new file mode 100644 index 0000000000..44afc55be9 --- /dev/null +++ b/toolchain/std-patches/wasi-libc-overrides/tempfile.c @@ -0,0 +1,132 @@ +/** + * mkstemp/mkdtemp family for AgentOS WASI. + * + * Upstream wasi-libc hides these because bare WASI has no ambient temp + * directory. AgentOS supplies cwd-relative filesystem access and Rust std + * already patches temp_dir(), so expose the POSIX APIs for C software too. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +static const char alphabet[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; +static unsigned long counter; + +static int fill_template(char *template, int suffix_len) { + size_t len = strlen(template); + if (len < (size_t)suffix_len + 6 || + memcmp(template + len - suffix_len - 6, "XXXXXX", 6) != 0) { + errno = EINVAL; + return -1; + } + + uint64_t value = (uint64_t)time(NULL) ^ (uintptr_t)template ^ counter++; + char *slot = template + len - suffix_len - 6; + for (int i = 0; i < 6; i++) { + value = value * 6364136223846793005ULL + 1442695040888963407ULL; + slot[i] = alphabet[value % (sizeof(alphabet) - 1)]; + } + return 0; +} + +int mkostemps(char *template, int suffix_len, int flags) { + char original[6]; + size_t len = strlen(template); + if (len < (size_t)suffix_len + 6) { + errno = EINVAL; + return -1; + } + char *slot = template + len - suffix_len - 6; + memcpy(original, slot, sizeof(original)); + + flags &= ~O_ACCMODE; + for (int retries = 100; retries > 0; retries--) { + if (fill_template(template, suffix_len) != 0) { + return -1; + } + int fd = open(template, flags | O_RDWR | O_CREAT | O_EXCL, 0600); + if (fd >= 0) { + return fd; + } + if (errno != EEXIST) { + break; + } + } + + memcpy(slot, original, sizeof(original)); + return -1; +} + +int mkstemps(char *template, int suffix_len) { + return mkostemps(template, suffix_len, 0); +} + +int mkostemp(char *template, int flags) { + return mkostemps(template, 0, flags); +} + +int mkstemp(char *template) { + return mkostemps(template, 0, 0); +} + +char *mktemp(char *template) { + struct stat st; + size_t len = strlen(template); + if (len < 6 || memcmp(template + len - 6, "XXXXXX", 6) != 0) { + errno = EINVAL; + if (len > 0) { + template[0] = '\0'; + } + return template; + } + + for (int retries = 100; retries > 0; retries--) { + if (fill_template(template, 0) != 0) { + template[0] = '\0'; + return template; + } + if (stat(template, &st) != 0) { + if (errno != ENOENT) { + template[0] = '\0'; + } + return template; + } + } + + template[0] = '\0'; + errno = EEXIST; + return template; +} + +char *mkdtemp(char *template) { + char original[6]; + size_t len = strlen(template); + if (len < 6) { + errno = EINVAL; + return NULL; + } + char *slot = template + len - 6; + memcpy(original, slot, sizeof(original)); + + for (int retries = 100; retries > 0; retries--) { + if (fill_template(template, 0) != 0) { + return NULL; + } + if (mkdir(template, 0700) == 0) { + return template; + } + if (errno != EEXIST) { + break; + } + } + + memcpy(slot, original, sizeof(original)); + return NULL; +} diff --git a/toolchain/std-patches/wasi-libc/0027-temp-and-ownership-header-surface.patch b/toolchain/std-patches/wasi-libc/0027-temp-and-ownership-header-surface.patch new file mode 100644 index 0000000000..a81f4aceab --- /dev/null +++ b/toolchain/std-patches/wasi-libc/0027-temp-and-ownership-header-surface.patch @@ -0,0 +1,66 @@ +--- a/libc-top-half/musl/include/stdlib.h ++++ b/libc-top-half/musl/include/stdlib.h +@@ -115,11 +115,9 @@ size_t __ctype_get_mb_cur_max(void); + int posix_memalign (void **, size_t, size_t); + int setenv (const char *, const char *, int); + int unsetenv (const char *); +-#ifdef __wasilibc_unmodified_upstream /* WASI has no temp directories */ + int mkstemp (char *); + int mkostemp (char *, int); + char *mkdtemp (char *); +-#endif + int getsubopt (char **, char *const *, char **); + int rand_r (unsigned *); + +@@ -156,11 +154,9 @@ void lcong48 (unsigned short [7]); + + #if defined(_GNU_SOURCE) || defined(_BSD_SOURCE) + #include +-#ifdef __wasilibc_unmodified_upstream /* WASI has no temp directories */ + char *mktemp (char *); + int mkstemps (char *, int); + int mkostemps (char *, int, int); +-#endif + #ifdef __wasilibc_unmodified_upstream /* WASI libc doesn't build the legacy functions */ + void *valloc (size_t); + void *memalign(size_t, size_t); +--- a/libc-top-half/musl/include/sys/stat.h ++++ b/libc-top-half/musl/include/sys/stat.h +@@ -81,9 +81,7 @@ int fstatat(int, const char *__restrict, struct stat *__restrict, int); + int chmod(const char *, mode_t); + int fchmod(int, mode_t); + int fchmodat(int, const char *, mode_t, int); +-#ifdef __wasilibc_unmodified_upstream /* WASI has no umask */ + mode_t umask(mode_t); +-#endif + int mkdir(const char *, mode_t); + #ifdef __wasilibc_unmodified_upstream /* WASI has no fifo */ + int mkfifo(const char *, mode_t); +--- a/libc-top-half/musl/include/unistd.h ++++ b/libc-top-half/musl/include/unistd.h +@@ -93,12 +93,10 @@ ssize_t write(int, const void *, size_t); + ssize_t pread(int, void *, size_t, off_t); + ssize_t pwrite(int, const void *, size_t, off_t); + +-#ifdef __wasilibc_unmodified_upstream /* WASI has no chown */ + int chown(const char *, uid_t, gid_t); + int fchown(int, uid_t, gid_t); + int lchown(const char *, uid_t, gid_t); + int fchownat(int, const char *, uid_t, gid_t, int); +-#endif + + int link(const char *, const char *); + int linkat(int, const char *, int, const char *, int); +--- a/libc-top-half/musl/include/stdio.h ++++ b/libc-top-half/musl/include/stdio.h +@@ -159,10 +159,8 @@ FILE *fmemopen(void *__restrict, size_t, const char *__restrict); + FILE *fmemopen(void *__restrict, size_t, const char *__restrict); + FILE *open_memstream(char **, size_t *); + FILE *fdopen(int, const char *); +-#ifdef __wasilibc_unmodified_upstream /* WASI has no popen */ + FILE *popen(const char *, const char *); + int pclose(FILE *); +-#endif + int fileno(FILE *); + int fseeko(FILE *, off_t, int); + off_t ftello(FILE *);