diff --git a/docs-internal/registry-parity-worklist.md b/docs-internal/registry-parity-worklist.md index 5742b46e40..f7b844c866 100644 --- a/docs-internal/registry-parity-worklist.md +++ b/docs-internal/registry-parity-worklist.md @@ -104,7 +104,7 @@ actual backing: | **wget** | DONE | our 174-line `wget.c` (dropped) | real GNU Wget vs our sysroot — stub `getrlimit`/`getgroups`, then build | | **http-get** | DONE | our 95-line `http_get.c` | dropped; real curl covers HTTP fetches | | **git** | TODO | our hand-rolled git from `sha1`+`flate2` | **real git** (upstream C), patched for WASI — **NOT gitoxide** | -| **fd** | TODO | our `secureexec-fd` on raw `regex` (not sharkdp/fd) | real **fd** (sharkdp) | +| **fd** | DONE | our `secureexec-fd` on raw `regex` (not sharkdp/fd) | real **fd** (sharkdp) | | **findutils** (`find`,`xargs`) | TODO | our hand-rolled on `regex`/shims | real GNU findutils, or `uutils/findutils` | | **tree** | DONE | our hand-rolled, zero deps | real `tree`, or an established one | | **grep** | TODO | our `secureexec-grep` on raw `regex` (**not** an established grep pkg) | **real GNU grep**, or a popular established grep (ripgrep's `grep` crates) | @@ -226,9 +226,25 @@ works (`wasi-spawn` broker), so `xargs` is not a blocker. longer includes the deleted Rust tree crates in `2026-07-08T05-33-17-0700-tree-cargo-metadata-after-removing-empty-dirs.log`. Rev: `kpmrwxln` — `fix(tree): build upstream tree`. -- **fd — moderate.** Swap `cmd-fd` to depend on real `fd-find` (like - coreutils→`uu_*`; `fd-lock` is already patched). Only real issue: the parallel - `ignore`/crossbeam walker needs the **serial-thread patch** (uu_sort pattern). +- **fd — DONE.** Replaced the custom Rust `secureexec-fd` regex walker with + upstream sharkdp `fd-find` 10.4.2. Because `fd-find` is a bin-only crate, + `cmd-fd` now acts as the workspace build trigger while the toolchain builds + the upstream `fd` binary directly. WASI compatibility stays in dependency + patches: `fd-find` gets narrow file-type/path/receiver adjustments, and + `ignore` uses a serial walker on `wasm32-wasip1` instead of host threads. Proof: + clean patch dry-runs pass in + `2026-07-08T06-19-50-0700-fd-find-patch-clean-dry-run.log` and + `2026-07-08T06-26-52-0700-fd-ignore-patch-final-dry-run.log`; + clean upstream WASM build compiles `fd-find v10.4.2` in + `2026-07-08T06-21-54-0700-fd-upstream-wasm-rebuild-after-ignore-fix.log`; + runtime/package command hashes match in + `2026-07-08T06-22-42-0700-fd-copy-built-binary-hashes.log`; package build and + check-types pass in + `2026-07-08T06-24-27-0700-fd-package-build.log` and + `2026-07-08T06-25-41-0700-fd-check-types-final.log`; e2e fd tests pass 9/9, + including `fd --version` reporting `fd 10.4.2`, in + `2026-07-08T06-25-00-0700-fd-vitest-upstream-after-dir-format.log`. + Rev: `mrskpomv` — `fix(fd): build upstream fd-find`. - **grep — moderate.** Decision is real GNU grep (gnulib cascade → sysroot stubs) or ripgrep's `grep-*` crates (threads → serial patch). NB the current `secureexec-grep` also backs `rg`, so the shipped "ripgrep" is custom too. diff --git a/packages/runtime-core/commands/fd b/packages/runtime-core/commands/fd index b724ec24b1..137bb9d133 100644 Binary files a/packages/runtime-core/commands/fd and b/packages/runtime-core/commands/fd differ diff --git a/software/fd/bin/fd b/software/fd/bin/fd index b724ec24b1..137bb9d133 100644 Binary files a/software/fd/bin/fd and b/software/fd/bin/fd differ diff --git a/software/fd/native/crates/cmd-fd/Cargo.toml b/software/fd/native/crates/cmd-fd/Cargo.toml index 419fd7f396..ef6223d123 100644 --- a/software/fd/native/crates/cmd-fd/Cargo.toml +++ b/software/fd/native/crates/cmd-fd/Cargo.toml @@ -5,10 +5,10 @@ version.workspace = true edition.workspace = true license.workspace = true description = "fd standalone binary for secure-exec VM" - -[[bin]] -name = "fd" -path = "src/main.rs" +publish = false [dependencies] -secureexec-fd = { path = "../fd" } +# fd-find is a bin-only crate. `toolchain/Makefile` builds its upstream `fd` +# binary directly; this local package exists only so `cmd/fd` is discoverable +# and the upstream package is present in Cargo's resolved/vendor set. +fd-find = { version = "10.4.2", default-features = false } diff --git a/software/fd/native/crates/cmd-fd/src/lib.rs b/software/fd/native/crates/cmd-fd/src/lib.rs new file mode 100644 index 0000000000..a372ecb5e7 --- /dev/null +++ b/software/fd/native/crates/cmd-fd/src/lib.rs @@ -0,0 +1 @@ +//! Build trigger for the upstream `fd-find` command package. diff --git a/software/fd/native/crates/cmd-fd/src/main.rs b/software/fd/native/crates/cmd-fd/src/main.rs deleted file mode 100644 index e74e129d80..0000000000 --- a/software/fd/native/crates/cmd-fd/src/main.rs +++ /dev/null @@ -1,4 +0,0 @@ -fn main() { - let args: Vec = std::env::args_os().collect(); - std::process::exit(secureexec_fd::main(args)); -} diff --git a/software/fd/native/crates/fd/Cargo.toml b/software/fd/native/crates/fd/Cargo.toml deleted file mode 100644 index 47c01c9571..0000000000 --- a/software/fd/native/crates/fd/Cargo.toml +++ /dev/null @@ -1,10 +0,0 @@ -[package] -workspace = "../../../../../toolchain" -name = "secureexec-fd" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "fd-find compatible file finder for secure-exec standalone binaries" - -[dependencies] -regex = "1" diff --git a/software/fd/native/crates/fd/src/lib.rs b/software/fd/native/crates/fd/src/lib.rs deleted file mode 100644 index 8ab845214f..0000000000 --- a/software/fd/native/crates/fd/src/lib.rs +++ /dev/null @@ -1,353 +0,0 @@ -//! Minimal fd-find compatible file finder for WASM. -//! -//! Uses std::fs for directory traversal (no walkdir/rayon dependency) -//! and regex for pattern matching. Covers common fd patterns: -//! fd PATTERN, fd -e EXT, fd -t f/d, fd -H (hidden), fd -I (no-ignore). - -use std::collections::HashSet; -use std::ffi::OsString; -use std::fs; -use std::io::{self, Write}; -use std::path::{Path, PathBuf}; - -use regex::Regex; - -/// Entry point for fd command. -pub fn main(args: Vec) -> i32 { - let str_args: Vec = args - .iter() - .skip(1) // skip argv[0] - .map(|a| a.to_string_lossy().to_string()) - .collect(); - - match run(&str_args) { - Ok(code) => code, - Err(msg) => { - eprintln!("[fd error]: {}", msg); - 1 - } - } -} - -#[derive(Debug, Clone, Copy, PartialEq)] -enum TypeFilter { - File, - Directory, - Symlink, -} - -enum ParsedArgs { - Search(Options), - Help, - Version, -} - -struct Options { - pattern: Option, - extensions: Vec, - type_filter: Option, - max_depth: Option, - search_paths: Vec, - show_hidden: bool, - _case_insensitive: bool, - full_path: bool, - absolute_path: bool, -} - -fn run(args: &[String]) -> Result { - let parsed = parse_args(args)?; - let stdout = io::stdout(); - let mut out = stdout.lock(); - - let ParsedArgs::Search(opts) = parsed else { - match parsed { - ParsedArgs::Help => print_help(&mut out), - ParsedArgs::Version => writeln!(out, "fd 0.1.0 (secure-exec)"), - ParsedArgs::Search(_) => unreachable!(), - } - .map_err(|e| format!("failed to write output: {e}"))?; - out.flush() - .map_err(|e| format!("failed to write output: {e}"))?; - return Ok(0); - }; - - let mut found = false; - - for search_path in &opts.search_paths { - let base = PathBuf::from(search_path); - let mut visited_dirs = HashSet::new(); - walk( - &base, - search_path, - 0, - &opts, - &mut found, - &mut out, - &mut visited_dirs, - )?; - } - - out.flush() - .map_err(|e| format!("failed to write output: {e}"))?; - - Ok(if found { 0 } else { 1 }) -} - -fn walk( - full_path: &Path, - base_path: &str, - depth: usize, - opts: &Options, - found: &mut bool, - out: &mut W, - visited_dirs: &mut HashSet, -) -> Result<(), String> { - if let Some(max) = opts.max_depth { - if depth > max { - return Ok(()); - } - } - - // fd skips the root search directory itself (depth 0); only matches children - if depth > 0 { - if matches_entry(full_path, base_path, opts) { - *found = true; - let display = if opts.absolute_path { - match fs::canonicalize(full_path) { - Ok(abs) => abs.to_string_lossy().to_string(), - Err(_) => full_path.to_string_lossy().to_string(), - } - } else { - full_path.to_string_lossy().to_string() - }; - writeln!(out, "{}", display).map_err(|e| format!("failed to write output: {e}"))?; - } - } - - // Recurse into directories - if full_path.is_dir() { - let canonical_path = - fs::canonicalize(full_path).map_err(|e| format!("{}: {}", full_path.display(), e))?; - if !visited_dirs.insert(canonical_path) { - return Ok(()); - } - - if let Some(max) = opts.max_depth { - if depth >= max { - return Ok(()); - } - } - - let entries = - fs::read_dir(full_path).map_err(|e| format!("{}: {}", full_path.display(), e))?; - - let mut sorted: Vec<_> = entries - .collect::, _>>() - .map_err(|e| format!("{}: {}", full_path.display(), e))?; - sorted.sort_by_key(|e| e.file_name()); - - for entry in sorted { - let name = entry.file_name().to_string_lossy().to_string(); - - // Skip hidden files/directories unless -H is set - if !opts.show_hidden && name.starts_with('.') { - continue; - } - - let child = entry.path(); - walk(&child, base_path, depth + 1, opts, found, out, visited_dirs)?; - } - } - - Ok(()) -} - -fn matches_entry(path: &Path, _base_path: &str, opts: &Options) -> bool { - // Type filter - if let Some(ref tf) = opts.type_filter { - let ok = match tf { - TypeFilter::File => path.is_file(), - TypeFilter::Directory => path.is_dir(), - TypeFilter::Symlink => path - .symlink_metadata() - .map(|m| m.file_type().is_symlink()) - .unwrap_or(false), - }; - if !ok { - return false; - } - } - - // Extension filter - if !opts.extensions.is_empty() { - let ext = path - .extension() - .map(|e| e.to_string_lossy().to_string()) - .unwrap_or_default(); - let ext_lower = ext.to_lowercase(); - if !opts - .extensions - .iter() - .any(|e| e.to_lowercase() == ext_lower) - { - return false; - } - } - - // Pattern match - if let Some(ref re) = opts.pattern { - let target = if opts.full_path { - path.to_string_lossy().to_string() - } else { - path.file_name() - .map(|n| n.to_string_lossy().to_string()) - .unwrap_or_default() - }; - if !re.is_match(&target) { - return false; - } - } - - true -} - -fn parse_args(args: &[String]) -> Result { - let mut pattern: Option = None; - let mut extensions: Vec = Vec::new(); - let mut type_filter: Option = None; - let mut max_depth: Option = None; - let mut search_paths: Vec = Vec::new(); - let mut show_hidden = false; - let mut case_insensitive = false; - let mut full_path = false; - let mut absolute_path = false; - let mut i = 0; - - while i < args.len() { - let arg = &args[i]; - match arg.as_str() { - "-h" | "--help" => { - return Ok(ParsedArgs::Help); - } - "-V" | "--version" => { - return Ok(ParsedArgs::Version); - } - "-H" | "--hidden" => { - show_hidden = true; - } - "-I" | "--no-ignore" => { - // No .gitignore support in WASM VFS — this is a no-op - } - "-i" | "--ignore-case" => { - case_insensitive = true; - } - "-s" | "--case-sensitive" => { - case_insensitive = false; - } - "-p" | "--full-path" => { - full_path = true; - } - "-a" | "--absolute-path" => { - absolute_path = true; - } - "-e" | "--extension" => { - i += 1; - if i >= args.len() { - return Err("-e/--extension requires an argument".to_string()); - } - extensions.push(args[i].clone()); - } - "-t" | "--type" => { - i += 1; - if i >= args.len() { - return Err("-t/--type requires an argument".to_string()); - } - type_filter = Some(match args[i].as_str() { - "f" | "file" => TypeFilter::File, - "d" | "directory" => TypeFilter::Directory, - "l" | "symlink" => TypeFilter::Symlink, - other => return Err(format!("unknown type filter: {}", other)), - }); - } - "-d" | "--max-depth" => { - i += 1; - if i >= args.len() { - return Err("-d/--max-depth requires an argument".to_string()); - } - max_depth = Some( - args[i] - .parse() - .map_err(|_| format!("invalid max-depth: {}", args[i]))?, - ); - } - arg if arg.starts_with('-') => { - return Err(format!("unknown option: {}", arg)); - } - _ => { - // First positional arg is pattern, rest are search paths - if pattern.is_none() { - pattern = Some(arg.clone()); - } else { - search_paths.push(arg.clone()); - } - } - } - i += 1; - } - - if search_paths.is_empty() { - search_paths.push(".".to_string()); - } - - // Compile pattern regex - let compiled_pattern = match pattern { - Some(pat) => { - let re_str = if case_insensitive { - format!("(?i){}", pat) - } else { - pat - }; - Some(Regex::new(&re_str).map_err(|e| format!("invalid pattern: {}", e))?) - } - None => None, - }; - - Ok(ParsedArgs::Search(Options { - pattern: compiled_pattern, - extensions, - type_filter, - max_depth, - search_paths, - show_hidden, - _case_insensitive: case_insensitive, - full_path, - absolute_path, - })) -} - -fn print_help(out: &mut W) -> io::Result<()> { - writeln!( - out, - "fd - a simple, fast file finder - -USAGE: - fd [OPTIONS] [PATTERN] [PATH...] - -ARGS: - Regex search pattern - ... Search paths (default: current directory) - -OPTIONS: - -H, --hidden Include hidden files/directories - -I, --no-ignore Do not respect .gitignore (no-op in sandbox) - -i, --ignore-case Case-insensitive search - -s, --case-sensitive Case-sensitive search (default) - -p, --full-path Match pattern against full path - -a, --absolute-path Show absolute paths - -e, --extension EXT Filter by file extension - -t, --type TYPE Filter by type: f(ile), d(irectory), l(ink) - -d, --max-depth N Maximum search depth - -h, --help Print help - -V, --version Print version" - ) -} diff --git a/software/fd/test/fd.test.ts b/software/fd/test/fd.test.ts index b102e7595e..0b76049d2f 100644 --- a/software/fd/test/fd.test.ts +++ b/software/fd/test/fd.test.ts @@ -10,107 +10,20 @@ * Tests verify stdout correctness rather than exit code. */ +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; import { describe, it, expect, afterEach } from 'vitest'; import { createWasmVmRuntime } from '@agentos/test-harness'; -import { COMMANDS_DIR, createKernel, describeIf, hasWasmBinaries } from '@agentos/test-harness'; +import { COMMANDS_DIR, createKernel, describeIf, hasWasmBinaries, NodeFileSystem } from '@agentos/test-harness'; import type { Kernel } from '@agentos/test-harness'; -// Minimal in-memory VFS for kernel tests -class SimpleVFS { - private files = new Map(); - private dirs = new Set(['/']); - - async readFile(path: string): Promise { - const data = this.files.get(path); - if (!data) throw new Error(`ENOENT: ${path}`); - return data; - } - async readTextFile(path: string): Promise { - return new TextDecoder().decode(await this.readFile(path)); - } - async readDir(path: string): Promise { - const prefix = path === '/' ? '/' : path + '/'; - const entries: string[] = []; - for (const p of [...this.files.keys(), ...this.dirs]) { - if (p !== path && p.startsWith(prefix)) { - const rest = p.slice(prefix.length); - if (!rest.includes('/')) entries.push(rest); - } - } - return entries; - } - async readDirWithTypes(path: string) { - return (await this.readDir(path)).map(name => ({ - name, - isDirectory: this.dirs.has(path === '/' ? `/${name}` : `${path}/${name}`), - })); - } - async writeFile(path: string, content: string | Uint8Array): Promise { - const data = typeof content === 'string' ? new TextEncoder().encode(content) : content; - this.files.set(path, new Uint8Array(data)); - const parts = path.split('/').filter(Boolean); - for (let i = 1; i < parts.length; i++) { - this.dirs.add('/' + parts.slice(0, i).join('/')); - } - } - async createDir(path: string) { this.dirs.add(path); } - async mkdir(path: string, _options?: { recursive?: boolean }) { - this.dirs.add(path); - const parts = path.split('/').filter(Boolean); - for (let i = 1; i < parts.length; i++) { - this.dirs.add('/' + parts.slice(0, i).join('/')); - } - } - async exists(path: string): Promise { - return this.files.has(path) || this.dirs.has(path); - } - async stat(path: string) { - const isDir = this.dirs.has(path); - const data = this.files.get(path); - if (!isDir && !data) throw new Error(`ENOENT: ${path}`); - return { - mode: isDir ? 0o40755 : 0o100644, - size: data?.length ?? 0, - isDirectory: isDir, - isSymbolicLink: false, - atimeMs: Date.now(), - mtimeMs: Date.now(), - ctimeMs: Date.now(), - birthtimeMs: Date.now(), - ino: 0, - nlink: 1, - uid: 1000, - gid: 1000, - }; - } - async lstat(path: string) { return this.stat(path); } - async removeFile(path: string) { this.files.delete(path); } - async removeDir(path: string) { this.dirs.delete(path); } - async rename(oldPath: string, newPath: string) { - const data = this.files.get(oldPath); - if (data) { - this.files.set(newPath, data); - this.files.delete(oldPath); - } - } - async pread(path: string, offset: number, length: number): Promise { - const data = this.files.get(path); - if (!data) throw new Error(`ENOENT: ${path}`); - return data.slice(offset, offset + length); - } - async pwrite(path: string, offset: number, content: Uint8Array): Promise { - const data = this.files.get(path); - if (!data) throw new Error(`ENOENT: ${path}`); - const next = new Uint8Array(Math.max(data.length, offset + content.length)); - next.set(data); - next.set(content, offset); - this.files.set(path, next); - } -} +let tempRoot: string | undefined; /** Create a VFS pre-populated with a test directory structure */ -async function createTestVFS(): Promise { - const vfs = new SimpleVFS(); +async function createTestVFS(): Promise { + tempRoot = await mkdtemp(join(tmpdir(), 'agentos-fd-')); + // /project/ // src/ // main.js @@ -124,17 +37,24 @@ async function createTestVFS(): Promise { // secret.txt // .gitignore // config.json - await vfs.writeFile('/project/src/main.js', 'console.log("main")'); - await vfs.writeFile('/project/src/utils.js', 'export {}'); - await vfs.writeFile('/project/src/helpers.ts', 'export {}'); - await vfs.writeFile('/project/lib/parser.js', 'module.exports = {}'); - await vfs.writeFile('/project/docs/readme.md', '# Readme'); - await vfs.writeFile('/project/.hidden/secret.txt', 'secret'); - await vfs.writeFile('/project/.gitignore', 'node_modules'); - await vfs.writeFile('/project/config.json', '{}'); + await writeFixture('/project/src/main.js', 'console.log("main")'); + await writeFixture('/project/src/utils.js', 'export {}'); + await writeFixture('/project/src/helpers.ts', 'export {}'); + await writeFixture('/project/lib/parser.js', 'module.exports = {}'); + await writeFixture('/project/docs/readme.md', '# Readme'); + await writeFixture('/project/.hidden/secret.txt', 'secret'); + await writeFixture('/project/.gitignore', 'node_modules'); + await writeFixture('/project/config.json', '{}'); // /empty/ — empty directory - await vfs.mkdir('/empty', { recursive: true }); - return vfs; + await mkdir(join(tempRoot, 'empty'), { recursive: true }); + return new NodeFileSystem({ root: tempRoot }); +} + +async function writeFixture(path: string, contents: string): Promise { + if (!tempRoot) throw new Error('fixture root not initialized'); + const hostPath = join(tempRoot, path.replace(/^\/+/, '')); + await mkdir(dirname(hostPath), { recursive: true }); + await writeFile(hostPath, contents); } /** Parse fd output lines, sorted for deterministic comparison */ @@ -147,11 +67,24 @@ describeIf(hasWasmBinaries, 'fd-find command', { timeout: 10_000 }, () => { afterEach(async () => { await kernel?.dispose(); + if (tempRoot) { + await rm(tempRoot, { recursive: true, force: true }); + tempRoot = undefined; + } + }); + + it('reports the upstream fd-find version', async () => { + const vfs = await createTestVFS(); + kernel = createKernel({ filesystem: vfs }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); + + const result = await kernel.exec('fd --version', {}); + expect(result.stdout.trim()).toBe('fd 10.4.2'); }); it('finds files matching regex pattern in current directory', async () => { const vfs = await createTestVFS(); - kernel = createKernel({ filesystem: vfs as any }); + kernel = createKernel({ filesystem: vfs }); await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); const result = await kernel.exec('fd main /project', {}); @@ -161,7 +94,7 @@ describeIf(hasWasmBinaries, 'fd-find command', { timeout: 10_000 }, () => { it('finds all .js files with -e js', async () => { const vfs = await createTestVFS(); - kernel = createKernel({ filesystem: vfs as any }); + kernel = createKernel({ filesystem: vfs }); await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); const result = await kernel.exec('fd -e js . /project', {}); @@ -175,7 +108,7 @@ describeIf(hasWasmBinaries, 'fd-find command', { timeout: 10_000 }, () => { it('finds only files with -t f', async () => { const vfs = await createTestVFS(); - kernel = createKernel({ filesystem: vfs as any }); + kernel = createKernel({ filesystem: vfs }); await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); const result = await kernel.exec('fd -t f . /project', {}); @@ -192,7 +125,7 @@ describeIf(hasWasmBinaries, 'fd-find command', { timeout: 10_000 }, () => { it('finds only directories with -t d', async () => { const vfs = await createTestVFS(); - kernel = createKernel({ filesystem: vfs as any }); + kernel = createKernel({ filesystem: vfs }); await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); const result = await kernel.exec('fd -t d . /project', {}); @@ -203,14 +136,14 @@ describeIf(hasWasmBinaries, 'fd-find command', { timeout: 10_000 }, () => { expect(stat.isDirectory).toBe(true); } // Should include known directories (hidden skipped by default) - expect(lines).toContain('/project/src'); - expect(lines).toContain('/project/lib'); - expect(lines).toContain('/project/docs'); + expect(lines).toContain('/project/src/'); + expect(lines).toContain('/project/lib/'); + expect(lines).toContain('/project/docs/'); }); it('returns no results for empty directory', async () => { const vfs = await createTestVFS(); - kernel = createKernel({ filesystem: vfs as any }); + kernel = createKernel({ filesystem: vfs }); await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); const result = await kernel.exec('fd . /empty', {}); @@ -219,7 +152,7 @@ describeIf(hasWasmBinaries, 'fd-find command', { timeout: 10_000 }, () => { it('returns empty output when no files match pattern', async () => { const vfs = await createTestVFS(); - kernel = createKernel({ filesystem: vfs as any }); + kernel = createKernel({ filesystem: vfs }); await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); const result = await kernel.exec('fd zzzznonexistent /project', {}); @@ -228,7 +161,7 @@ describeIf(hasWasmBinaries, 'fd-find command', { timeout: 10_000 }, () => { it('skips hidden files and directories by default', async () => { const vfs = await createTestVFS(); - kernel = createKernel({ filesystem: vfs as any }); + kernel = createKernel({ filesystem: vfs }); await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); const result = await kernel.exec('fd . /project', {}); @@ -243,13 +176,13 @@ describeIf(hasWasmBinaries, 'fd-find command', { timeout: 10_000 }, () => { it('includes hidden files with -H flag', async () => { const vfs = await createTestVFS(); - kernel = createKernel({ filesystem: vfs as any }); + kernel = createKernel({ filesystem: vfs }); await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); const result = await kernel.exec('fd -H . /project', {}); const lines = parseLines(result.stdout); // Hidden items should now appear expect(lines).toContain('/project/.gitignore'); - expect(lines).toContain('/project/.hidden'); + expect(lines).toContain('/project/.hidden/'); }); }); diff --git a/toolchain/Cargo.lock b/toolchain/Cargo.lock index 0c8cfd808d..7fe76204f8 100644 --- a/toolchain/Cargo.lock +++ b/toolchain/Cargo.lock @@ -151,6 +151,17 @@ dependencies = [ "triomphe", ] +[[package]] +name = "argmax" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0144c58b55af0133ec3963ce5e4d07aad866e3bbcfdcddbf4590dbd7ad6ff557" +dependencies = [ + "libc", + "nix 0.30.1", + "once_cell", +] + [[package]] name = "arrayref" version = "0.3.9" @@ -163,20 +174,6 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" -[[package]] -name = "assert_fs" -version = "1.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ecf5c70ca07b7f80220bce936f0556a960ca6fb00fc2bd4125b5e581b218137" -dependencies = [ - "anstyle", - "globwalk", - "predicates", - "predicates-core", - "predicates-tree", - "tempfile", -] - [[package]] name = "async-recursion" version = "1.1.1" @@ -650,15 +647,6 @@ dependencies = [ "terminal_size", ] -[[package]] -name = "clap_complete" -version = "4.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0a7a9bfdb35811f9e59832f0f05975114d2251b415fb534108e6f34060fd772" -dependencies = [ - "clap", -] - [[package]] name = "clap_derive" version = "4.6.0" @@ -677,16 +665,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" -[[package]] -name = "clap_mangen" -version = "0.2.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e30ffc187e2e3aeafcd1c6e2aa416e29739454c0ccaa419226d5ecd181f2d78" -dependencies = [ - "clap", - "roff", -] - [[package]] name = "cmd-arch" version = "0.1.0" @@ -901,7 +879,7 @@ dependencies = [ name = "cmd-fd" version = "0.1.0" dependencies = [ - "secureexec-fd", + "fd-find", ] [[package]] @@ -1602,6 +1580,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-deque" version = "0.8.6" @@ -1688,22 +1675,6 @@ dependencies = [ "typenum", ] -[[package]] -name = "ctor" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "424e0138278faeb2b401f174ad17e715c829512d74f3d1e81eb43365c2e0590e" -dependencies = [ - "ctor-proc-macro", - "dtor", -] - -[[package]] -name = "ctor-proc-macro" -version = "0.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52560adf09603e58c9a7ee1fe1dcb95a16927b17c127f0ac02d6e768a0e25bc1" - [[package]] name = "ctrlc" version = "3.5.2" @@ -1825,12 +1796,6 @@ dependencies = [ "syn", ] -[[package]] -name = "difflib" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" - [[package]] name = "digest" version = "0.10.7" @@ -1861,21 +1826,6 @@ dependencies = [ "litrs", ] -[[package]] -name = "dtor" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "404d02eeb088a82cfd873006cb713fe411306c7d182c344905e101fb1167d301" -dependencies = [ - "dtor-proc-macro", -] - -[[package]] -name = "dtor-proc-macro" -version = "0.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f678cf4a922c215c63e0de95eb1ff08a958a81d47e485cf9da1e27bf6305cfa5" - [[package]] name = "dunce" version = "1.0.5" @@ -1916,6 +1866,27 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "etcetera" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de48cc4d1c1d97a20fd819def54b890cadde72ed3ad0c614822a0a433361be96" +dependencies = [ + "cfg-if", + "windows-sys 0.61.2", +] + +[[package]] +name = "faccess" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ae66425802d6a903e268ae1a08b8c38ba143520f227a205edf4e9c7e3e26d5" +dependencies = [ + "bitflags 1.3.2", + "libc", + "winapi", +] + [[package]] name = "fancy-regex" version = "0.16.2" @@ -1944,6 +1915,32 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "fd-find" +version = "10.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b95ed7d1f53e0446a7d47715801f6bee95f816c4aa33e25b5d89a2734ab00436" +dependencies = [ + "aho-corasick", + "anyhow", + "argmax", + "clap", + "crossbeam-channel", + "ctrlc", + "etcetera", + "faccess", + "globset", + "ignore", + "jiff", + "libc", + "lscolors", + "nix 0.31.3", + "normpath", + "nu-ansi-term", + "regex", + "regex-syntax", +] + [[package]] name = "fd-lock" version = "4.0.4" @@ -1993,15 +1990,6 @@ dependencies = [ "miniz_oxide", ] -[[package]] -name = "float-cmp" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b09cf3155332e944990140d967ff5eceb70df778b34f77d8075db46e4704e6d8" -dependencies = [ - "num-traits", -] - [[package]] name = "fluent" version = "0.17.0" @@ -2273,17 +2261,6 @@ dependencies = [ "regex-syntax", ] -[[package]] -name = "globwalk" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf760ebf69878d9fd8f110c89703d90ce35095324d1f1edcb595c63945ee757" -dependencies = [ - "bitflags 2.11.0", - "ignore", - "walkdir", -] - [[package]] name = "half" version = "2.7.1" @@ -2965,9 +2942,9 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "libc" -version = "0.2.183" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libm" @@ -3131,6 +3108,18 @@ dependencies = [ "libc", ] +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags 2.11.0", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "nom" version = "7.1.3" @@ -3150,18 +3139,21 @@ dependencies = [ "memchr", ] -[[package]] -name = "normalize-line-endings" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" - [[package]] name = "normalize-path" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f5438dd2b2ff4c6df6e1ce22d825ed2fa93ee2922235cc45186991717f0a892d" +[[package]] +name = "normpath" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9985ef7269fa99f3b12437bb698381da2428743ab90f20393f399fa14cab21a" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "notify" version = "8.2.0" @@ -3525,36 +3517,6 @@ dependencies = [ "zerocopy", ] -[[package]] -name = "predicates" -version = "3.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ada8f2932f28a27ee7b70dd6c1c39ea0675c55a36879ab92f3a715eaa1e63cfe" -dependencies = [ - "anstyle", - "difflib", - "float-cmp", - "normalize-line-endings", - "predicates-core", - "regex", -] - -[[package]] -name = "predicates-core" -version = "1.0.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cad38746f3166b4031b1a0d39ad9f954dd291e7854fcc0eed52ee41a0b50d144" - -[[package]] -name = "predicates-tree" -version = "1.0.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0de1b847b39c8131db0467e9df1ff60e6d0562ab8e9a16e568ad0fdb372e2f2" -dependencies = [ - "predicates-core", - "termtree", -] - [[package]] name = "prettyplease" version = "0.2.37" @@ -3816,12 +3778,6 @@ dependencies = [ "libc", ] -[[package]] -name = "roff" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "323c417e1d9665a65b263ec744ba09030cfb277e9daa0b018a4ab62e57bc8189" - [[package]] name = "rpds" version = "1.2.0" @@ -3942,13 +3898,6 @@ dependencies = [ "regex", ] -[[package]] -name = "secureexec-fd" -version = "0.1.0" -dependencies = [ - "regex", -] - [[package]] name = "secureexec-file-cmd" version = "0.1.0" @@ -4065,24 +4014,16 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f5a00cdbffacbccf02decb7577da5e298c091bdee492a2a1b87a3d8b0cc5b7c5" dependencies = [ - "assert_fs", "clap", - "clap_complete", - "clap_mangen", - "ctor", "fancy-regex 0.17.0", "memchr", "memmap2", "once_cell", "phf 0.13.1", "phf_codegen 0.13.1", - "predicates", "regex", - "sysinfo", "tempfile", - "terminal_size", - "textwrap", - "uucore 0.5.0", + "uucore 0.7.0", ] [[package]] @@ -4288,12 +4229,6 @@ version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" -[[package]] -name = "smawk" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8e2fb0f499abb4d162f2bedad68f5ef91a1682b5a03596ddb67efd37768d100" - [[package]] name = "spin" version = "0.10.0" @@ -4465,24 +4400,6 @@ dependencies = [ "phf_codegen 0.11.3", ] -[[package]] -name = "termtree" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" - -[[package]] -name = "textwrap" -version = "0.16.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057" -dependencies = [ - "smawk", - "terminal_size", - "unicode-linebreak", - "unicode-width 0.2.0", -] - [[package]] name = "thiserror" version = "1.0.69" @@ -4745,12 +4662,6 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" -[[package]] -name = "unicode-linebreak" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" - [[package]] name = "unicode-segmentation" version = "1.12.0" @@ -5804,26 +5715,6 @@ dependencies = [ "wild", ] -[[package]] -name = "uucore" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5eddd390f3fdef74f104a948559e6de29203f60f8f563c8c9f528cd4c88ee78" -dependencies = [ - "clap", - "fluent", - "fluent-bundle", - "fluent-syntax", - "libc", - "nix 0.30.1", - "os_display", - "phf 0.13.1", - "thiserror 2.0.18", - "unic-langid", - "uucore_procs 0.5.0", - "wild", -] - [[package]] name = "uucore" version = "0.7.0" @@ -5887,16 +5778,6 @@ dependencies = [ "quote", ] -[[package]] -name = "uucore_procs" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47148309a1f7a989d165dabbbc7f2bf156d7ff6affe7d69c1c5bfb822e663ae6" -dependencies = [ - "proc-macro2", - "quote", -] - [[package]] name = "uucore_procs" version = "0.7.0" diff --git a/toolchain/Makefile b/toolchain/Makefile index 06b41a7c08..a287e55630 100644 --- a/toolchain/Makefile +++ b/toolchain/Makefile @@ -239,11 +239,17 @@ wasm-cmd: vendor patch-vendor patch-std wasm-opt-check @crate_dir=$$(find ../software -path "*/native/crates/cmd-$(CMD)" -type d -print -quit); \ test -n "$$crate_dir" || test -d "crates/commands/$(CMD)" || { \ echo "ERROR: no software/*/native/crates/cmd-$(CMD) or crates/commands/$(CMD)"; exit 1; } + @cargo_pkg="cmd-$(CMD)"; cargo_bin="$(CMD)"; \ + if [ "$(CMD)" = "fd" ]; then \ + cargo_pkg="fd-find"; \ + cargo_bin="fd"; \ + fi; \ RUSTFLAGS="-C link-arg=-L$(WASI_SYSROOT_SELF_CONTAINED) --cfg tokio_unstable" \ cargo build --target $(WASM_TARGET) \ -Z build-std=std,panic_abort \ --release \ - -p cmd-$(CMD) + -p $$cargo_pkg \ + --bin $$cargo_bin @mkdir -p $(COMMANDS_DIR) @if command -v wasm-opt >/dev/null 2>&1; then \ wasm-opt -O3 --strip-debug --all-features "$(RELEASE_DIR)/$(CMD).wasm" -o "$(COMMANDS_DIR)/$(CMD)"; \ diff --git a/toolchain/std-patches/crates/fd-find/0001-wasi-compat.patch b/toolchain/std-patches/crates/fd-find/0001-wasi-compat.patch new file mode 100644 index 0000000000..6801b6ce07 --- /dev/null +++ b/toolchain/std-patches/crates/fd-find/0001-wasi-compat.patch @@ -0,0 +1,107 @@ +--- a/src/filesystem.rs ++++ b/src/filesystem.rs +@@ -64,7 +64,7 @@ pub fn is_block_device(ft: fs::FileType) -> bool { + ft.is_block_device() + } + +-#[cfg(windows)] ++#[cfg(any(windows, target_os = "wasi"))] + pub fn is_block_device(_: fs::FileType) -> bool { + false + } +@@ -74,7 +74,7 @@ pub fn is_char_device(ft: fs::FileType) -> bool { + ft.is_char_device() + } + +-#[cfg(windows)] ++#[cfg(any(windows, target_os = "wasi"))] + pub fn is_char_device(_: fs::FileType) -> bool { + false + } +@@ -84,7 +84,7 @@ pub fn is_socket(ft: fs::FileType) -> bool { + ft.is_socket() + } + +-#[cfg(windows)] ++#[cfg(any(windows, target_os = "wasi"))] + pub fn is_socket(_: fs::FileType) -> bool { + false + } +@@ -94,13 +94,16 @@ pub fn is_pipe(ft: fs::FileType) -> bool { + ft.is_fifo() + } + +-#[cfg(windows)] ++#[cfg(any(windows, target_os = "wasi"))] + pub fn is_pipe(_: fs::FileType) -> bool { + false + } + +-#[cfg(any(unix, target_os = "redox"))] ++#[cfg(any(unix, target_os = "redox", target_os = "wasi"))] + pub fn osstr_to_bytes(input: &OsStr) -> Cow<'_, [u8]> { ++ #[cfg(target_os = "wasi")] ++ use std::os::wasi::ffi::OsStrExt; ++ #[cfg(not(target_os = "wasi"))] + use std::os::unix::ffi::OsStrExt; + Cow::Borrowed(input.as_bytes()) + } +--- a/src/main.rs ++++ b/src/main.rs +@@ -257,6 +257,9 @@ fn config(mut opts: Opts) -> Result { + || opts.no_global_ignore_file), + follow_links: opts.follow, ++ #[cfg(any(unix, windows))] + one_file_system: opts.one_file_system, ++ #[cfg(not(any(unix, windows)))] ++ one_file_system: false, + null_separator: opts.null_separator, + quiet: opts.quiet, + max_depth: opts.max_depth(), +--- a/src/walk.rs ++++ b/src/walk.rs +@@ -10,6 +10,8 @@ use std::time::{Duration, Instant}; + + use anyhow::{Result, anyhow}; + use crossbeam_channel::{Receiver, RecvTimeoutError, SendError, Sender, bounded}; ++#[cfg(target_os = "wasi")] ++use crossbeam_channel::unbounded; + use etcetera::BaseStrategy; + use ignore::overrides::{Override, OverrideBuilder}; + use ignore::{WalkBuilder, WalkParallel, WalkState}; +@@ -656,17 +658,27 @@ impl WorkerState { + .unwrap(); + } + +- let (tx, rx) = bounded(2 * config.threads); ++ #[cfg(target_os = "wasi")] ++ let exit_code = { ++ let (tx, rx) = unbounded(); ++ self.spawn_senders(walker, tx); ++ self.receive(rx) ++ }; + +- let exit_code = thread::scope(|scope| { +- // Spawn the receiver thread(s) +- let receiver = scope.spawn(|| self.receive(rx)); ++ #[cfg(not(target_os = "wasi"))] ++ let exit_code = { ++ let (tx, rx) = bounded(2 * config.threads); + +- // Spawn the sender threads. +- self.spawn_senders(walker, tx); ++ thread::scope(|scope| { ++ // Spawn the receiver thread(s) ++ let receiver = scope.spawn(|| self.receive(rx)); + +- receiver.join().unwrap() +- }); ++ // Spawn the sender threads. ++ self.spawn_senders(walker, tx); ++ ++ receiver.join().unwrap() ++ }) ++ }; + + if self.interrupt_flag.load(Ordering::Relaxed) { + Ok(ExitCode::KilledBySigint) diff --git a/toolchain/std-patches/crates/ignore/0001-wasi-serial-walk.patch b/toolchain/std-patches/crates/ignore/0001-wasi-serial-walk.patch new file mode 100644 index 0000000000..89b7f8c8c2 --- /dev/null +++ b/toolchain/std-patches/crates/ignore/0001-wasi-serial-walk.patch @@ -0,0 +1,60 @@ +--- a/src/walk.rs ++++ b/src/walk.rs +@@ -1353,6 +1353,49 @@ + /// synchronization. Then, once traversal is complete, all of the results + /// can be merged together into a single data structure. + pub fn visit(mut self, builder: &mut dyn ParallelVisitorBuilder<'_>) { ++ #[cfg(target_os = "wasi")] ++ { ++ let mut visitor = builder.build(); ++ let mut paths = Vec::new().into_iter(); ++ std::mem::swap(&mut paths, &mut self.paths); ++ let its = paths ++ .map(|p| { ++ if p == Path::new("-") { ++ (p, None) ++ } else { ++ let mut wd = WalkDir::new(&p); ++ wd = wd.follow_links(self.follow_links || p.is_file()); ++ wd = wd.same_file_system(self.same_file_system); ++ if let Some(max_depth) = self.max_depth { ++ wd = wd.max_depth(max_depth); ++ } ++ if let Some(min_depth) = self.min_depth { ++ wd = wd.min_depth(min_depth); ++ } ++ (p, Some(WalkEventIter::from(wd))) ++ } ++ }) ++ .collect::>() ++ .into_iter(); ++ let mut walk = Walk { ++ its, ++ it: None, ++ ig_root: self.ig_root.clone(), ++ ig: self.ig_root, ++ max_filesize: self.max_filesize, ++ skip: self.skip, ++ filter: self.filter, ++ }; ++ for entry in &mut walk { ++ if visitor.visit(entry).is_quit() { ++ break; ++ } ++ } ++ return; ++ } ++ ++ #[cfg(not(target_os = "wasi"))] ++ { + let threads = self.threads(); + let mut stack = vec![]; + { +@@ -1428,6 +1471,7 @@ + handle.join().unwrap(); + } + }); ++ } + } + + fn threads(&self) -> usize {