diff --git a/docs-internal/registry-parity-worklist.md b/docs-internal/registry-parity-worklist.md index 72c6918889..c0e4f3b5eb 100644 --- a/docs-internal/registry-parity-worklist.md +++ b/docs-internal/registry-parity-worklist.md @@ -104,7 +104,7 @@ actual backing: | **http-get** | DONE | our 95-line `http_get.c` | dropped; real curl covers HTTP fetches | | **git** | DONE | our hand-rolled git from `sha1`+`flate2` | **real git** (upstream C), patched for WASI — **NOT gitoxide** | | **fd** | DONE | our `secureexec-fd` on raw `regex` (not sharkdp/fd) | real **fd** (sharkdp) | -| **findutils** (`find`,`xargs`) | TODO | our hand-rolled on `regex`/shims | real GNU findutils, or `uutils/findutils` | +| **findutils** (`find`,`xargs`) | DONE | our hand-rolled on `regex`/shims | replaced with `uutils/findutils` | | **tree** | DONE | our hand-rolled, zero deps | real `tree`, or an established one | | **grep** | DONE | our `secureexec-grep` on raw `regex` (**not** an established grep pkg) | real **GNU grep** | | **ripgrep** (`rg`) | DONE | our `secureexec-grep` recursive search shim, not real ripgrep | real upstream **ripgrep** | @@ -313,9 +313,22 @@ works (`wasi-spawn` broker), so `xargs` is not a blocker. final UnZip e2e passes 6/6 in `2026-07-08T08-37-12-0700-unzip-test-final-after-current-sysroot-copy-after-install.log`. Rev: `nppnuxpr` — `fix(infozip): build upstream zip and unzip`. -- **findutils — moderate→hard.** `find` is fs+regex (easy); `xargs` spawn already - works. **uutils/findutils** (Rust) avoids gnulib; **GNU findutils** (C) hits the - same gnulib cascade as wget. Prefer uutils unless GNU parity is required. +- **findutils — DONE.** Replaced the custom Rust `find` crate with upstream + `uutils/findutils` 0.9.0 entrypoints for both `find` and `xargs`. Kept the fix + in the toolchain/sysroot layer: Rust WASI metadata extensions and process + `ExitStatusExt::signal()` are exposed for crates that expect Unix-like std + surfaces, while `xargs` still uses normal `std::process::Command` spawning + through the existing VM broker. Proof: `find` wasm build passes in + `2026-07-08T12-42-00-0700-findutils-find-build-after-permissions-mode-patch.log`; + `xargs` wasm build passes in + `2026-07-08T12-45-00-0700-findutils-xargs-build-after-find-success.log`; + package build passes in + `2026-07-08T12-51-00-0700-findutils-package-build-after-install.log`; sidecar + validation build passes in + `2026-07-08T12-35-00-0700-native-sidecar-build-after-forced-pnpm.log`; final + package e2e passes 5/5, including `xargs -n 2 echo` spawn batching, in + `2026-07-08T12-44-00-0700-findutils-vitest-uutils-after-depth-test-fix.log`. + Rev: `msknmmps`. Ranked easiest→hardest: **sqlite3-CLI · http-get (drop) · tree · fd · grep · ripgrep · zip · unzip · findutils.** diff --git a/software/findutils/bin/find b/software/findutils/bin/find index e08e8b58a1..3693ed2473 100644 Binary files a/software/findutils/bin/find and b/software/findutils/bin/find differ diff --git a/software/findutils/bin/xargs b/software/findutils/bin/xargs index 613a7bb3ea..2b303eb53d 100644 Binary files a/software/findutils/bin/xargs and b/software/findutils/bin/xargs differ diff --git a/software/findutils/native/crates/cmd-find/Cargo.toml b/software/findutils/native/crates/cmd-find/Cargo.toml index f394612385..19cc8f38f4 100644 --- a/software/findutils/native/crates/cmd-find/Cargo.toml +++ b/software/findutils/native/crates/cmd-find/Cargo.toml @@ -11,4 +11,5 @@ name = "find" path = "src/main.rs" [dependencies] -secureexec-find = { path = "../find" } +findutils = { version = "0.9.0", default-features = false } +uucore = { version = "0.9.0", default-features = false } diff --git a/software/findutils/native/crates/cmd-find/src/main.rs b/software/findutils/native/crates/cmd-find/src/main.rs index 5887b1a7a4..8990e43353 100644 --- a/software/findutils/native/crates/cmd-find/src/main.rs +++ b/software/findutils/native/crates/cmd-find/src/main.rs @@ -1,4 +1,11 @@ fn main() { - let args: Vec = std::env::args_os().collect(); - std::process::exit(secureexec_find::main(args)); + uucore::panic::mute_sigpipe_panic(); + + let args = std::env::args().collect::>(); + let args = args + .iter() + .map(std::convert::AsRef::as_ref) + .collect::>(); + let deps = findutils::find::StandardDependencies::new(); + std::process::exit(findutils::find::find_main(&args, &deps)); } diff --git a/software/findutils/native/crates/cmd-xargs/Cargo.toml b/software/findutils/native/crates/cmd-xargs/Cargo.toml index e43b660a5b..1037c80a58 100644 --- a/software/findutils/native/crates/cmd-xargs/Cargo.toml +++ b/software/findutils/native/crates/cmd-xargs/Cargo.toml @@ -11,4 +11,4 @@ name = "xargs" path = "src/main.rs" [dependencies] -shims = { package = "secureexec-shims", path = "../../../../../toolchain/crates/libs/shims" } +findutils = { version = "0.9.0", default-features = false } diff --git a/software/findutils/native/crates/cmd-xargs/src/main.rs b/software/findutils/native/crates/cmd-xargs/src/main.rs index 0ef4547696..85586163c6 100644 --- a/software/findutils/native/crates/cmd-xargs/src/main.rs +++ b/software/findutils/native/crates/cmd-xargs/src/main.rs @@ -1,4 +1,8 @@ fn main() { - let args: Vec = std::env::args_os().collect(); - std::process::exit(shims::xargs::xargs(args)); + let args = std::env::args().collect::>(); + let args = args + .iter() + .map(std::convert::AsRef::as_ref) + .collect::>(); + std::process::exit(findutils::xargs::xargs_main(&args)); } diff --git a/software/findutils/native/crates/find/Cargo.toml b/software/findutils/native/crates/find/Cargo.toml deleted file mode 100644 index 4c4cd79502..0000000000 --- a/software/findutils/native/crates/find/Cargo.toml +++ /dev/null @@ -1,10 +0,0 @@ -[package] -workspace = "../../../../../toolchain" -name = "secureexec-find" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "find implementation for secure-exec standalone binaries" - -[dependencies] -regex = "1" diff --git a/software/findutils/native/crates/find/src/lib.rs b/software/findutils/native/crates/find/src/lib.rs deleted file mode 100644 index 059e80fa53..0000000000 --- a/software/findutils/native/crates/find/src/lib.rs +++ /dev/null @@ -1,593 +0,0 @@ -//! POSIX find implementation for WASM. -//! -//! Custom implementation using std::fs (WASI) for directory traversal -//! and glob-to-regex conversion for pattern matching. fd-find crate -//! cannot be used directly as it's a binary crate with platform-specific -//! dependencies incompatible with wasm32-wasip1. - -use std::collections::HashSet; -use std::ffi::OsString; -use std::fs; -use std::io::{self, Write}; -use std::path::{Path, PathBuf}; - -use regex::Regex; - -/// Entry point for find command. -pub fn main(args: Vec) -> i32 { - let str_args: Vec = args - .iter() - .skip(1) // skip argv[0] - .map(|a| a.to_string_lossy().to_string()) - .collect(); - - match run_find(&str_args) { - Ok(found) => { - if found { - 0 - } else { - 0 - } // find returns 0 even if no matches - } - Err(msg) => { - eprintln!("find: {}", msg); - 1 - } - } -} - -/// Parsed find expression node. -#[derive(Debug)] -enum Expr { - /// -name PATTERN (glob match on filename only) - Name(Regex), - /// -iname PATTERN (case-insensitive glob match on filename) - IName(Regex), - /// -path PATTERN (glob match on full path) - PathMatch(Regex), - /// -ipath PATTERN (case-insensitive glob match on full path) - IPathMatch(Regex), - /// -type TYPE (d, f, l) - Type(FileType), - /// -empty (file size 0 or empty directory) - Empty, - /// -maxdepth N (handled specially during traversal) - MaxDepth(usize), - /// -mindepth N (handled specially during traversal) - MinDepth(usize), - /// -print (explicit print action) - Print, - /// -not EXPR / ! EXPR - Not(Box), - /// EXPR -a EXPR / EXPR -and EXPR / implicit AND - And(Box, Box), - /// EXPR -o EXPR / EXPR -or EXPR - Or(Box, Box), -} - -#[derive(Debug, Clone, Copy, PartialEq)] -enum FileType { - Directory, - File, - Symlink, -} - -struct FindOptions { - paths: Vec, - expr: Option, - max_depth: Option, - min_depth: usize, -} - -fn run_find(args: &[String]) -> Result { - let opts = parse_args(args)?; - let mut found_any = false; - let stdout = io::stdout(); - let mut out = stdout.lock(); - - for path in &opts.paths { - let mut visited_dirs = HashSet::new(); - walk( - &PathBuf::from(path), - path, - 0, - &opts, - &mut found_any, - &mut out, - &mut visited_dirs, - )?; - } - - out.flush() - .map_err(|e| format!("failed to write output: {e}"))?; - - Ok(found_any) -} - -/// Recursive directory walk. -fn walk( - full_path: &Path, - display_path: &str, - depth: usize, - opts: &FindOptions, - found_any: &mut bool, - out: &mut W, - visited_dirs: &mut HashSet, -) -> Result<(), String> { - // Check max depth before processing - if let Some(max) = opts.max_depth { - if depth > max { - return Ok(()); - } - } - - // Evaluate expression against this entry (if at or below min_depth) - if depth >= opts.min_depth { - let matches = match &opts.expr { - Some(expr) => eval_expr(expr, full_path, display_path), - None => true, // no expression means match everything - }; - - if matches { - *found_any = true; - // Default action is print - writeln!(out, "{}", display_path) - .map_err(|e| format!("failed to write output: {e}"))?; - } - } - - // Recurse into directories - if full_path.is_dir() { - // Check max depth for children - if let Some(max) = opts.max_depth { - if depth >= max { - return Ok(()); - } - } - - let canonical_path = - fs::canonicalize(full_path).map_err(|e| format!("{}: {}", display_path, e))?; - if !visited_dirs.insert(canonical_path.clone()) { - return Ok(()); - } - - let entries = fs::read_dir(full_path).map_err(|e| format!("{}: {}", display_path, e))?; - - let mut sorted_entries: Vec<_> = entries - .collect::, _>>() - .map_err(|e| format!("{}: {}", display_path, e))?; - sorted_entries.sort_by_key(|e| e.file_name()); - - for entry in sorted_entries { - let child_name = entry.file_name().to_string_lossy().to_string(); - let child_display = if display_path == "/" { - format!("/{}", child_name) - } else { - format!("{}/{}", display_path, child_name) - }; - let child_full = entry.path(); - - walk( - &child_full, - &child_display, - depth + 1, - opts, - found_any, - out, - visited_dirs, - )?; - } - visited_dirs.remove(&canonical_path); - } - - Ok(()) -} - -/// Evaluate an expression against a path. -fn eval_expr(expr: &Expr, full_path: &Path, display_path: &str) -> bool { - match expr { - Expr::Name(re) => { - let name = full_path - .file_name() - .map(|n| n.to_string_lossy().to_string()) - .unwrap_or_default(); - re.is_match(&name) - } - Expr::IName(re) => { - let name = full_path - .file_name() - .map(|n| n.to_string_lossy().to_string()) - .unwrap_or_default(); - re.is_match(&name) - } - Expr::PathMatch(re) => re.is_match(display_path), - Expr::IPathMatch(re) => re.is_match(display_path), - Expr::Type(ft) => match ft { - FileType::Directory => full_path.is_dir(), - FileType::File => full_path.is_file(), - FileType::Symlink => full_path - .symlink_metadata() - .map(|m| m.file_type().is_symlink()) - .unwrap_or(false), - }, - Expr::Empty => { - if full_path.is_dir() { - fs::read_dir(full_path) - .map(|mut d| d.next().is_none()) - .unwrap_or(false) - } else if full_path.is_file() { - full_path.metadata().map(|m| m.len() == 0).unwrap_or(false) - } else { - false - } - } - Expr::MaxDepth(_) | Expr::MinDepth(_) => true, // handled in walk() - Expr::Print => true, // print is an action, always "matches" - Expr::Not(inner) => !eval_expr(inner, full_path, display_path), - Expr::And(left, right) => { - eval_expr(left, full_path, display_path) && eval_expr(right, full_path, display_path) - } - Expr::Or(left, right) => { - eval_expr(left, full_path, display_path) || eval_expr(right, full_path, display_path) - } - } -} - -/// Convert a shell glob pattern to a regex pattern. -/// Supports *, ?, [charset], and ** (for path matching). -fn glob_to_regex(glob: &str, case_insensitive: bool) -> Result { - let mut re = String::from("^"); - let chars: Vec = glob.chars().collect(); - let mut i = 0; - - while i < chars.len() { - match chars[i] { - '*' => { - if i + 1 < chars.len() && chars[i + 1] == '*' { - // ** matches everything including / - re.push_str(".*"); - i += 1; - } else { - // * matches everything except / - re.push_str("[^/]*"); - } - } - '?' => re.push_str("[^/]"), - '[' => { - re.push('['); - i += 1; - if i < chars.len() && chars[i] == '!' { - re.push('^'); - i += 1; - } else if i < chars.len() && chars[i] == '^' { - re.push('^'); - i += 1; - } - while i < chars.len() && chars[i] != ']' { - if chars[i] == '\\' && i + 1 < chars.len() { - re.push('\\'); - i += 1; - re.push(chars[i]); - } else { - re.push(chars[i]); - } - i += 1; - } - re.push(']'); - } - '.' | '+' | '(' | ')' | '{' | '}' | '|' | '^' | '$' | '\\' => { - re.push('\\'); - re.push(chars[i]); - } - _ => re.push(chars[i]), - } - i += 1; - } - - re.push('$'); - - let pattern = if case_insensitive { - format!("(?i){}", re) - } else { - re - }; - - Regex::new(&pattern).map_err(|e| format!("invalid pattern: {}", e)) -} - -/// Parse find command arguments. -fn parse_args(args: &[String]) -> Result { - let mut paths: Vec = Vec::new(); - let mut exprs: Vec = Vec::new(); - let mut max_depth: Option = None; - let mut min_depth: usize = 0; - let mut i = 0; - - // First, collect paths (arguments before any expression) - while i < args.len() { - let arg = &args[i]; - if arg.starts_with('-') || arg == "!" || arg == "(" { - break; - } - paths.push(arg.clone()); - i += 1; - } - - if paths.is_empty() { - paths.push(".".to_string()); - } - - // Parse expressions - while i < args.len() { - let arg = &args[i]; - - match arg.as_str() { - "-name" => { - i += 1; - if i >= args.len() { - return Err("-name requires an argument".to_string()); - } - let re = glob_to_regex(&args[i], false)?; - exprs.push(Expr::Name(re)); - } - "-iname" => { - i += 1; - if i >= args.len() { - return Err("-iname requires an argument".to_string()); - } - let re = glob_to_regex(&args[i], true)?; - exprs.push(Expr::IName(re)); - } - "-path" | "-wholename" => { - i += 1; - if i >= args.len() { - return Err(format!("{} requires an argument", arg)); - } - let re = glob_to_regex(&args[i], false)?; - exprs.push(Expr::PathMatch(re)); - } - "-ipath" | "-iwholename" => { - i += 1; - if i >= args.len() { - return Err(format!("{} requires an argument", arg)); - } - let re = glob_to_regex(&args[i], true)?; - exprs.push(Expr::IPathMatch(re)); - } - "-type" => { - i += 1; - if i >= args.len() { - return Err("-type requires an argument".to_string()); - } - let ft = match args[i].as_str() { - "d" => FileType::Directory, - "f" => FileType::File, - "l" => FileType::Symlink, - other => return Err(format!("unknown file type: {}", other)), - }; - exprs.push(Expr::Type(ft)); - } - "-empty" => { - exprs.push(Expr::Empty); - } - "-maxdepth" => { - i += 1; - if i >= args.len() { - return Err("-maxdepth requires an argument".to_string()); - } - let n: usize = args[i] - .parse() - .map_err(|_| format!("-maxdepth: {}: not a number", args[i]))?; - max_depth = Some(n); - } - "-mindepth" => { - i += 1; - if i >= args.len() { - return Err("-mindepth requires an argument".to_string()); - } - let n: usize = args[i] - .parse() - .map_err(|_| format!("-mindepth: {}: not a number", args[i]))?; - min_depth = n; - } - "-print" => { - exprs.push(Expr::Print); - } - "!" | "-not" => { - i += 1; - if i >= args.len() { - return Err("! requires an expression".to_string()); - } - // Parse the next single expression - let (next_expr, next_i) = parse_single_expr(&args, i)?; - if let Some((md, mi)) = extract_depth(&next_expr) { - if let Some(d) = md { - max_depth = Some(d); - } - min_depth = mi.unwrap_or(min_depth); - } - exprs.push(Expr::Not(Box::new(next_expr))); - i = next_i; - continue; // skip the i += 1 at end - } - "-a" | "-and" => { - // Implicit AND between consecutive expressions, -a is explicit - // Just skip it; AND is applied when combining exprs - } - "-o" | "-or" => { - // Combine everything so far as left side of OR - if exprs.is_empty() { - return Err("-or requires a preceding expression".to_string()); - } - let left = combine_and(exprs); - - // Parse remaining as right side - i += 1; - let mut right_exprs = Vec::new(); - while i < args.len() { - let (e, ni) = parse_single_expr(args, i)?; - if let Some((md, mi)) = extract_depth(&e) { - if let Some(d) = md { - max_depth = Some(d); - } - min_depth = mi.unwrap_or(min_depth); - } - right_exprs.push(e); - i = ni; - } - let right = combine_and(right_exprs); - exprs = vec![Expr::Or(Box::new(left), Box::new(right))]; - break; - } - "(" => { - // Grouped expression - find matching ) - i += 1; - let mut depth = 1; - let start = i; - while i < args.len() && depth > 0 { - if args[i] == "(" { - depth += 1; - } - if args[i] == ")" { - depth -= 1; - } - if depth > 0 { - i += 1; - } - } - if depth != 0 { - return Err("unmatched '('".to_string()); - } - let sub_args: Vec = args[start..i].to_vec(); - let sub_opts = parse_args_exprs(&sub_args)?; - if let Some(e) = sub_opts { - exprs.push(e); - } - } - other if other.starts_with('-') => { - return Err(format!("unknown option: {}", other)); - } - _ => { - // Treat as path if no expressions yet, otherwise error - return Err(format!("unexpected argument: {}", arg)); - } - } - i += 1; - } - - let expr = if exprs.is_empty() { - None - } else { - Some(combine_and(exprs)) - }; - - Ok(FindOptions { - paths, - expr, - max_depth, - min_depth, - }) -} - -/// Parse a single expression at position i, returning the expression and the next position. -fn parse_single_expr(args: &[String], i: usize) -> Result<(Expr, usize), String> { - if i >= args.len() { - return Err("unexpected end of expression".to_string()); - } - - let arg = &args[i]; - match arg.as_str() { - "-name" => { - if i + 1 >= args.len() { - return Err("-name requires an argument".to_string()); - } - let re = glob_to_regex(&args[i + 1], false)?; - Ok((Expr::Name(re), i + 2)) - } - "-iname" => { - if i + 1 >= args.len() { - return Err("-iname requires an argument".to_string()); - } - let re = glob_to_regex(&args[i + 1], true)?; - Ok((Expr::IName(re), i + 2)) - } - "-path" | "-wholename" => { - if i + 1 >= args.len() { - return Err(format!("{} requires an argument", arg)); - } - let re = glob_to_regex(&args[i + 1], false)?; - Ok((Expr::PathMatch(re), i + 2)) - } - "-type" => { - if i + 1 >= args.len() { - return Err("-type requires an argument".to_string()); - } - let ft = match args[i + 1].as_str() { - "d" => FileType::Directory, - "f" => FileType::File, - "l" => FileType::Symlink, - other => return Err(format!("unknown file type: {}", other)), - }; - Ok((Expr::Type(ft), i + 2)) - } - "-empty" => Ok((Expr::Empty, i + 1)), - "-maxdepth" => { - if i + 1 >= args.len() { - return Err("-maxdepth requires an argument".to_string()); - } - let n: usize = args[i + 1] - .parse() - .map_err(|_| format!("-maxdepth: {}: not a number", args[i + 1]))?; - Ok((Expr::MaxDepth(n), i + 2)) - } - "-mindepth" => { - if i + 1 >= args.len() { - return Err("-mindepth requires an argument".to_string()); - } - let n: usize = args[i + 1] - .parse() - .map_err(|_| format!("-mindepth: {}: not a number", args[i + 1]))?; - Ok((Expr::MinDepth(n), i + 2)) - } - "-print" => Ok((Expr::Print, i + 1)), - _ => Err(format!("unknown expression: {}", arg)), - } -} - -/// Parse expression-only args (for grouped expressions). -fn parse_args_exprs(args: &[String]) -> Result, String> { - let mut exprs = Vec::new(); - let mut i = 0; - - while i < args.len() { - let (expr, next_i) = parse_single_expr(args, i)?; - exprs.push(expr); - i = next_i; - } - - if exprs.is_empty() { - Ok(None) - } else { - Ok(Some(combine_and(exprs))) - } -} - -/// Extract depth settings from an expression. -fn extract_depth(expr: &Expr) -> Option<(Option, Option)> { - match expr { - Expr::MaxDepth(n) => Some((Some(*n), None)), - Expr::MinDepth(n) => Some((None, Some(*n))), - _ => None, - } -} - -/// Combine a list of expressions with AND. -fn combine_and(mut exprs: Vec) -> Expr { - if exprs.len() == 1 { - return exprs.remove(0); - } - let first = exprs.remove(0); - exprs - .into_iter() - .fold(first, |acc, e| Expr::And(Box::new(acc), Box::new(e))) -} diff --git a/software/findutils/test/findutils.test.ts b/software/findutils/test/findutils.test.ts index 030f38d208..f343e46864 100644 --- a/software/findutils/test/findutils.test.ts +++ b/software/findutils/test/findutils.test.ts @@ -60,6 +60,23 @@ describeIf(hasWasmBinaries, "findutils commands", { timeout: 10_000 }, () => { ]); }); + it("find supports depth limits", async () => { + const vfs = createInMemoryFileSystem(); + await vfs.mkdir("/project/src/nested", { recursive: true }); + await vfs.mkdir("/project/docs", { recursive: true }); + await vfs.writeFile("/project/src/main.js", "console.log('main')\n"); + await vfs.writeFile("/project/src/nested/helper.ts", "export {}\n"); + await vfs.writeFile("/project/docs/readme.md", "# Readme\n"); + + kernel = createKernel({ filesystem: vfs }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); + + const result = await kernel.exec( + "find /project -mindepth 2 -maxdepth 2 -type f -name '*.js'", + ); + expect(parseLines(result.stdout)).toEqual(["/project/src/main.js"]); + }); + it("xargs passes stdin arguments to a command", async () => { const vfs = createInMemoryFileSystem(); await vfs.writeFile("/args.txt", "alpha\nbeta\n"); @@ -70,4 +87,19 @@ describeIf(hasWasmBinaries, "findutils commands", { timeout: 10_000 }, () => { const result = await kernel.exec("xargs echo < /args.txt"); expect(result.stdout.trim()).toBe("alpha beta"); }); + + it("xargs batches arguments across spawned commands", async () => { + const vfs = createInMemoryFileSystem(); + await vfs.writeFile("/args.txt", "one two three four five\n"); + + kernel = createKernel({ filesystem: vfs }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); + + const result = await kernel.exec("xargs -n 2 echo < /args.txt"); + expect(result.stdout.trim().split("\n")).toEqual([ + "one two", + "three four", + "five", + ]); + }); }); diff --git a/toolchain/Cargo.lock b/toolchain/Cargo.lock index 4c6e2c57d6..7358c629ab 100644 --- a/toolchain/Cargo.lock +++ b/toolchain/Cargo.lock @@ -612,9 +612,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.44" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", "js-sys", @@ -907,7 +907,8 @@ dependencies = [ name = "cmd-find" version = "0.1.0" dependencies = [ - "secureexec-find", + "findutils", + "uucore 0.9.0", ] [[package]] @@ -924,13 +925,6 @@ dependencies = [ "uu_fold", ] -[[package]] -name = "cmd-git" -version = "0.1.0" -dependencies = [ - "secureexec-git", -] - [[package]] name = "cmd-gzip" version = "0.1.0" @@ -1411,7 +1405,7 @@ dependencies = [ name = "cmd-xargs" version = "0.1.0" dependencies = [ - "secureexec-shims", + "findutils", ] [[package]] @@ -1994,6 +1988,25 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "findutils" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "719cc0af0060da07d610f4faa20d2d46dca46ff6bdf778a4b4feec3a7c8af7ea" +dependencies = [ + "argmax", + "chrono", + "clap", + "faccess", + "itertools 0.14.0", + "nix 0.31.3", + "onig", + "regex", + "thiserror 2.0.18", + "uucore 0.9.0", + "walkdir", +] + [[package]] name = "fixed_decimal" version = "0.7.1" @@ -3411,6 +3424,28 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "onig" +version = "6.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc3cbf698f9438986c11a880c90a6d04b9de27575afd28bbf45b154b6c709e2" +dependencies = [ + "bitflags 2.11.0", + "libc", + "once_cell", + "onig_sys", +] + +[[package]] +name = "onig_sys" +version = "69.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e68317604e77e53b85896388e1a803c1d21b74c899ec9e5e1112db90735edd7" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "os_display" version = "0.1.4" @@ -3576,6 +3611,12 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + [[package]] name = "plain" version = "0.2.3" @@ -4033,22 +4074,6 @@ dependencies = [ "infer", ] -[[package]] -name = "secureexec-find" -version = "0.1.0" -dependencies = [ - "regex", -] - -[[package]] -name = "secureexec-git" -version = "0.1.0" -dependencies = [ - "flate2", - "secureexec-wasi-http", - "sha1", -] - [[package]] name = "secureexec-gzip" version = "0.1.0" @@ -5924,6 +5949,30 @@ dependencies = [ "z85", ] +[[package]] +name = "uucore" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "069b34217c27f611e1589f540f58118dbf226a9e407d38ab472052ff075a1dc2" +dependencies = [ + "bstr", + "clap", + "dunce", + "fluent", + "fluent-syntax", + "libc", + "nix 0.31.3", + "os_display", + "rustc-hash", + "rustix 1.1.4", + "thiserror 2.0.18", + "unic-langid", + "uucore_procs 0.9.0", + "wild", + "winapi-util", + "windows-sys 0.61.2", +] + [[package]] name = "uucore_procs" version = "0.4.0" @@ -5944,6 +5993,16 @@ dependencies = [ "quote", ] +[[package]] +name = "uucore_procs" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34337da211e7abfff7189b794afb3b5018fe356fb36474b73645458fc1201350" +dependencies = [ + "proc-macro2", + "quote", +] + [[package]] name = "uuid" version = "1.22.0" diff --git a/toolchain/Makefile b/toolchain/Makefile index f9e99e2991..236f7c8b7c 100644 --- a/toolchain/Makefile +++ b/toolchain/Makefile @@ -1,5 +1,8 @@ WASM_TARGET := wasm32-wasip1 RELEASE_DIR := target/$(WASM_TARGET)/release +WASI_C_SYSROOT := $(CURDIR)/c/sysroot +WASI_C_CC := $(CURDIR)/c/vendor/wasi-sdk/bin/clang +WASI_C_AR := $(CURDIR)/c/vendor/wasi-sdk/bin/llvm-ar # Standalone binary output directory (configurable) COMMANDS_DIR ?= $(RELEASE_DIR)/commands @@ -74,6 +77,9 @@ codex: c/vendor/wasi-sdk/bin/clang c/vendor/wasi-sdk/bin/clang: $(MAKE) -C c wasi-sdk +c/sysroot/lib/wasm32-wasi/libc.a: + $(MAKE) -C c sysroot + # Strict variant (fails if the fork is absent) for environments that require the codex artifact. .PHONY: codex-required codex-required: @@ -147,10 +153,13 @@ wasm-opt-check: fi # Build all standalone command binaries, optimize, strip .wasm extension, create symlinks -wasm: vendor patch-vendor patch-std wasm-opt-check +wasm: c/sysroot/lib/wasm32-wasi/libc.a vendor patch-vendor patch-std wasm-opt-check # --cfg tokio_unstable lifts tokio's wasm `compile_error!` so the patched # tokio process/net modules (std-patches/crates/tokio) compile on wasm32-wasip1. # Needed by codex (tokio::process/net). Harmless for commands not using tokio. + CC_wasm32_wasip1="$(WASI_C_CC)" \ + AR_wasm32_wasip1="$(WASI_C_AR)" \ + CFLAGS_wasm32_wasip1="--sysroot=$(WASI_C_SYSROOT)" \ RUSTFLAGS="-C link-arg=-L$(WASI_SYSROOT_SELF_CONTAINED) --cfg tokio_unstable" \ cargo build --target $(WASM_TARGET) \ -Z build-std=std,panic_abort \ @@ -233,7 +242,7 @@ cmd/%: # Build ONE command binary (cargo package cmd-$(CMD) -> $(COMMANDS_DIR)/$(CMD)). # Usage: make wasm-cmd CMD=sh .PHONY: wasm-cmd -wasm-cmd: vendor patch-vendor patch-std wasm-opt-check +wasm-cmd: c/sysroot/lib/wasm32-wasi/libc.a vendor patch-vendor patch-std wasm-opt-check @test -n "$(CMD)" || { echo "ERROR: pass CMD= (a dir under crates/commands/)"; exit 1; } @crate_dir=$$(find ../software -path "*/native/crates/cmd-$(CMD)" -type d -print -quit); \ test -n "$$crate_dir" || test -d "crates/commands/$(CMD)" || { \ @@ -246,6 +255,9 @@ wasm-cmd: vendor patch-vendor patch-std wasm-opt-check cargo_pkg="ripgrep"; \ cargo_bin="rg"; \ fi; \ + CC_wasm32_wasip1="$(WASI_C_CC)" \ + AR_wasm32_wasip1="$(WASI_C_AR)" \ + CFLAGS_wasm32_wasip1="--sysroot=$(WASI_C_SYSROOT)" \ RUSTFLAGS="-C link-arg=-L$(WASI_SYSROOT_SELF_CONTAINED) --cfg tokio_unstable" \ cargo build --target $(WASM_TARGET) \ -Z build-std=std,panic_abort \ diff --git a/toolchain/std-patches/0010-wasi-metadata-ext-fields.patch b/toolchain/std-patches/0010-wasi-metadata-ext-fields.patch new file mode 100644 index 0000000000..e472e0ab6e --- /dev/null +++ b/toolchain/std-patches/0010-wasi-metadata-ext-fields.patch @@ -0,0 +1,55 @@ +--- a/library/std/src/os/wasi/fs.rs ++++ b/library/std/src/os/wasi/fs.rs +@@ -316,6 +316,18 @@ + } + } + ++/// WASI-specific extensions to [`fs::Permissions`]. ++pub trait PermissionsExt { ++ /// Returns the underlying POSIX mode bits. ++ fn mode(&self) -> u32; ++} ++ ++impl PermissionsExt for fs::Permissions { ++ fn mode(&self) -> u32 { ++ self.as_inner().mode() ++ } ++} ++ + /// WASI-specific extensions to [`fs::Metadata`]. + pub trait MetadataExt { + /// Returns the `st_dev` field of the internal `filestat_t` +@@ -324,6 +336,14 @@ + fn ino(&self) -> u64; + /// Returns the `st_nlink` field of the internal `filestat_t` + fn nlink(&self) -> u64; ++ /// Returns the underlying POSIX mode bits. ++ fn mode(&self) -> u32; ++ /// Returns the file size in bytes. ++ fn size(&self) -> u64; ++ /// Returns the owning user ID. ++ fn uid(&self) -> u32; ++ /// Returns the owning group ID. ++ fn gid(&self) -> u32; + } + + impl MetadataExt for fs::Metadata { +@@ -336,6 +356,18 @@ + fn nlink(&self) -> u64 { + self.as_inner().as_inner().st_nlink + } ++ fn mode(&self) -> u32 { ++ self.as_inner().perm().mode() ++ } ++ fn size(&self) -> u64 { ++ self.as_inner().size() ++ } ++ fn uid(&self) -> u32 { ++ self.as_inner().as_inner().st_uid ++ } ++ fn gid(&self) -> u32 { ++ self.as_inner().as_inner().st_gid ++ } + } + + /// WASI-specific extensions for [`fs::FileType`]. diff --git a/toolchain/std-patches/0011-wasi-file-permissions-mode.patch b/toolchain/std-patches/0011-wasi-file-permissions-mode.patch new file mode 100644 index 0000000000..f4fd55bce2 --- /dev/null +++ b/toolchain/std-patches/0011-wasi-file-permissions-mode.patch @@ -0,0 +1,10 @@ +--- a/library/std/src/sys/fs/unix.rs ++++ b/library/std/src/sys/fs/unix.rs +@@ -778,7 +778,6 @@ impl FilePermissions { + self.mode |= 0o222; + } + } +- #[cfg(not(target_os = "wasi"))] + pub fn mode(&self) -> u32 { + self.mode as u32 + } diff --git a/toolchain/std-patches/crates/findutils/0001-wasi-compat.patch b/toolchain/std-patches/crates/findutils/0001-wasi-compat.patch new file mode 100644 index 0000000000..fbb306d446 --- /dev/null +++ b/toolchain/std-patches/crates/findutils/0001-wasi-compat.patch @@ -0,0 +1,331 @@ +--- a/src/lib.rs ++++ b/src/lib.rs +@@ -4,6 +4,8 @@ + // license that can be found in the LICENSE file or at + // https://opensource.org/licenses/MIT. + ++#![cfg_attr(target_os = "wasi", feature(wasi_ext))] ++ + pub mod find; + pub mod locate; + pub mod updatedb; +--- a/src/find/matchers/group.rs ++++ b/src/find/matchers/group.rs +@@ -9,6 +9,8 @@ + use nix::unistd::Group; + #[cfg(unix)] + use std::os::unix::fs::MetadataExt; ++#[cfg(target_os = "wasi")] ++use std::os::wasi::fs::MetadataExt; + + pub struct GroupMatcher { + gid: ComparableValue, +@@ -31,14 +33,14 @@ + Self { gid } + } + +- #[cfg(windows)] ++ #[cfg(not(unix))] + pub fn from_group_name(_group: &str) -> Option { + None + } + } + + impl Matcher for GroupMatcher { +- #[cfg(unix)] ++ #[cfg(any(unix, target_os = "wasi"))] + fn matches(&self, file_info: &WalkEntry, _: &mut MatcherIO) -> bool { + match file_info.metadata() { + Ok(metadata) => self.gid.matches(metadata.gid().into()), +@@ -46,7 +48,7 @@ + } + } + +- #[cfg(windows)] ++ #[cfg(not(any(unix, target_os = "wasi")))] + fn matches(&self, _file_info: &WalkEntry, _: &mut MatcherIO) -> bool { + // The user group acquisition function for Windows systems is not implemented in MetadataExt, + // so it is somewhat difficult to implement it. :( +@@ -80,7 +82,7 @@ + false + } + +- #[cfg(windows)] ++ #[cfg(not(unix))] + fn matches(&self, _file_info: &WalkEntry, _: &mut MatcherIO) -> bool { + false + } +--- a/src/find/matchers/ls.rs ++++ b/src/find/matchers/ls.rs +@@ -11,7 +11,7 @@ + + use super::{Matcher, MatcherIO, WalkEntry}; + +-#[cfg(unix)] ++#[cfg(any(unix, target_os = "wasi"))] + fn format_permissions(mode: uucore::libc::mode_t) -> String { + let file_type = match mode & (uucore::libc::S_IFMT as uucore::libc::mode_t) { + uucore::libc::S_IFDIR => "d", +@@ -118,7 +118,7 @@ + Self { output_file } + } + +- #[cfg(unix)] ++ #[cfg(any(unix, target_os = "wasi"))] + fn print( + &self, + file_info: &WalkEntry, +@@ -126,8 +126,12 @@ + mut out: impl Write, + print_error_message: bool, + ) { ++ #[cfg(unix)] + use nix::unistd::{Gid, Group, Uid, User}; ++ #[cfg(unix)] + use std::os::unix::fs::{MetadataExt, PermissionsExt}; ++ #[cfg(target_os = "wasi")] ++ use std::os::wasi::fs::MetadataExt; + + let metadata = file_info.metadata().unwrap(); + +@@ -147,17 +151,32 @@ + number_of_blocks + (4 - (remainder)) + } + }; +- let permission = +- { format_permissions(metadata.permissions().mode() as uucore::libc::mode_t) }; ++ #[cfg(unix)] ++ let mode = metadata.permissions().mode(); ++ #[cfg(target_os = "wasi")] ++ let mode = metadata.mode(); ++ let permission = { format_permissions(mode as uucore::libc::mode_t) }; + let hard_links = metadata.nlink(); ++ #[cfg(unix)] + let user = { + let uid = metadata.uid(); + User::from_uid(Uid::from_raw(uid)).unwrap().unwrap().name + }; ++ #[cfg(target_os = "wasi")] ++ let user = { ++ let uid = metadata.uid(); ++ uid.to_string() ++ }; ++ #[cfg(unix)] + let group = { + let gid = metadata.gid(); + Group::from_gid(Gid::from_raw(gid)).unwrap().unwrap().name + }; ++ #[cfg(target_os = "wasi")] ++ let group = { ++ let gid = metadata.gid(); ++ gid.to_string() ++ }; + let size = metadata.size(); + let last_modified = { + let system_time = metadata.modified().unwrap(); +@@ -195,7 +214,7 @@ + } + } + +- #[cfg(windows)] ++ #[cfg(not(any(unix, target_os = "wasi")))] + fn print( + &self, + file_info: &WalkEntry, +@@ -288,7 +307,7 @@ + #[cfg(test)] + mod tests { + #[test] +- #[cfg(unix)] ++ #[cfg(any(unix, target_os = "wasi"))] + fn test_format_permissions() { + use super::format_permissions; + +--- a/src/find/matchers/mod.rs ++++ b/src/find/matchers/mod.rs +@@ -25,7 +25,7 @@ + mod regex; + mod samefile; + mod size; +-#[cfg(unix)] ++#[cfg(any(unix, target_os = "wasi"))] + mod stat; + pub mod time; + mod type_matcher; +@@ -50,7 +50,7 @@ + use self::regex::RegexMatcher; + use self::samefile::SameFileMatcher; + use self::size::SizeMatcher; +-#[cfg(unix)] ++#[cfg(any(unix, target_os = "wasi"))] + use self::stat::{InodeMatcher, LinksMatcher}; + use self::time::{ + FileAgeRangeMatcher, FileTimeMatcher, FileTimeType, NewerMatcher, NewerOptionMatcher, +@@ -702,7 +702,7 @@ + .into_box(), + ) + } +- #[cfg(unix)] ++ #[cfg(any(unix, target_os = "wasi"))] + "-inum" => { + if i >= args.len() - 1 { + return Err(From::from(format!("missing argument to {}", args[i]))); +@@ -711,13 +711,13 @@ + i += 1; + Some(InodeMatcher::new(inum).into_box()) + } +- #[cfg(not(unix))] ++ #[cfg(not(any(unix, target_os = "wasi")))] + "-inum" => { + return Err(From::from( + "Inode numbers are not available on this platform", + )); + } +- #[cfg(unix)] ++ #[cfg(any(unix, target_os = "wasi"))] + "-links" => { + if i >= args.len() - 1 { + return Err(From::from(format!("missing argument to {}", args[i]))); +@@ -726,7 +726,7 @@ + i += 1; + Some(LinksMatcher::new(inum).into_box()) + } +- #[cfg(not(unix))] ++ #[cfg(not(any(unix, target_os = "wasi")))] + "-links" => { + return Err(From::from("Link counts are not available on this platform")); + } +--- a/src/find/matchers/name.rs ++++ b/src/find/matchers/name.rs +@@ -24,14 +24,14 @@ + fn matches(&self, file_info: &WalkEntry, _: &mut MatcherIO) -> bool { + let name = file_info.file_name().to_string_lossy(); + +- #[cfg(unix)] ++ #[cfg(any(unix, target_os = "wasi"))] + if name.len() > 1 && name.chars().all(|x| x == '/') { + self.pattern.matches("/") + } else { + self.pattern.matches(&name) + } + +- #[cfg(windows)] ++ #[cfg(not(any(unix, target_os = "wasi")))] + self.pattern.matches(&name) + } + } +--- a/src/find/matchers/stat.rs ++++ b/src/find/matchers/stat.rs +@@ -4,7 +4,10 @@ + // license that can be found in the LICENSE file or at + // https://opensource.org/licenses/MIT. + ++#[cfg(unix)] + use std::os::unix::fs::MetadataExt; ++#[cfg(target_os = "wasi")] ++use std::os::wasi::fs::MetadataExt; + + use super::{ComparableValue, Matcher, MatcherIO, WalkEntry}; + +--- a/src/find/matchers/user.rs ++++ b/src/find/matchers/user.rs +@@ -9,6 +9,8 @@ + use nix::unistd::User; + #[cfg(unix)] + use std::os::unix::fs::MetadataExt; ++#[cfg(target_os = "wasi")] ++use std::os::wasi::fs::MetadataExt; + + pub struct UserMatcher { + uid: ComparableValue, +@@ -31,14 +33,14 @@ + Self { uid } + } + +- #[cfg(windows)] ++ #[cfg(not(unix))] + pub fn from_user_name(_user: &str) -> Option { + None + } + } + + impl Matcher for UserMatcher { +- #[cfg(unix)] ++ #[cfg(any(unix, target_os = "wasi"))] + fn matches(&self, file_info: &WalkEntry, _: &mut MatcherIO) -> bool { + match file_info.metadata() { + Ok(metadata) => self.uid.matches(metadata.uid().into()), +@@ -46,7 +48,7 @@ + } + } + +- #[cfg(windows)] ++ #[cfg(not(any(unix, target_os = "wasi")))] + fn matches(&self, _file_info: &WalkEntry, _: &mut MatcherIO) -> bool { + false + } +@@ -78,7 +80,7 @@ + false + } + +- #[cfg(windows)] ++ #[cfg(not(unix))] + fn matches(&self, _file_info: &WalkEntry, _: &mut MatcherIO) -> bool { + false + } +--- a/src/xargs/mod.rs ++++ b/src/xargs/mod.rs +@@ -145,9 +145,12 @@ + s.encode_wide().count() + 1 + } + +-#[cfg(unix)] ++#[cfg(any(unix, target_os = "wasi"))] + fn count_osstr_chars_for_exec(s: &OsStr) -> usize { ++ #[cfg(unix)] + use std::os::unix::ffi::OsStrExt; ++ #[cfg(target_os = "wasi")] ++ use std::os::wasi::ffi::OsStrExt; + // Include +1 for the null terminator. + s.as_bytes().len() + 1 + } +@@ -173,12 +176,15 @@ + MaxCharsCommandSizeLimiter::new(MAX_CMDLINE) + } + +- #[cfg(unix)] ++ #[cfg(any(unix, target_os = "wasi"))] + fn new_system(env: &HashMap) -> Self { + // POSIX requires that we leave 2048 bytes of space so that the child processes + // can have room to set their own environment variables. + const ARG_HEADROOM: usize = 2048; ++ #[cfg(unix)] + let arg_max = unsafe { uucore::libc::sysconf(uucore::libc::_SC_ARG_MAX) } as usize; ++ #[cfg(target_os = "wasi")] ++ let arg_max = 131072usize; + + let env_size: usize = env + .iter() +@@ -454,9 +460,12 @@ + Ok(CommandResult::Failure) + } + } else { +- #[cfg(unix)] ++ #[cfg(any(unix, target_os = "wasi"))] + { ++ #[cfg(unix)] + use std::os::unix::process::ExitStatusExt; ++ #[cfg(target_os = "wasi")] ++ use std::os::wasi::process::ExitStatusExt; + if let Some(signal) = status.signal() { + Err(CommandExecutionError::Killed { signal }) + } else { +@@ -464,7 +473,7 @@ + } + } + +- #[cfg(not(unix))] ++ #[cfg(not(any(unix, target_os = "wasi")))] + Err(CommandExecutionError::Unknown) + } + } diff --git a/toolchain/std-patches/std/os/wasi/process.rs b/toolchain/std-patches/std/os/wasi/process.rs index 223daafd1b..3cd35eb328 100644 --- a/toolchain/std-patches/std/os/wasi/process.rs +++ b/toolchain/std-patches/std/os/wasi/process.rs @@ -60,6 +60,13 @@ pub trait ExitStatusExt { /// Construct an `ExitStatus` from the given raw code. #[stable(feature = "exit_status_from", since = "1.12.0")] fn from_raw(raw: i32) -> Self; + + /// If the process was terminated by a signal, returns that signal. + /// + /// AgentOS process statuses are reported as exit codes by the wasi-spawn + /// broker, so there is no POSIX signal payload to expose here. + #[stable(feature = "process_extensions", since = "1.2.0")] + fn signal(&self) -> Option; } #[stable(feature = "exit_status_from", since = "1.12.0")] @@ -67,4 +74,8 @@ impl ExitStatusExt for process::ExitStatus { fn from_raw(raw: i32) -> Self { process::ExitStatus::from_inner(crate::sys::process::ExitStatus::from(raw)) } + + fn signal(&self) -> Option { + None + } }