diff --git a/docs-internal/registry-parity-worklist.md b/docs-internal/registry-parity-worklist.md index 5275cc904c..86a297be21 100644 --- a/docs-internal/registry-parity-worklist.md +++ b/docs-internal/registry-parity-worklist.md @@ -609,6 +609,19 @@ real e2e tests that prove Linux-parity behavior — not smoke tests. Biome is not applicable for this package test path; it reported the file is ignored by config in `2026-07-08T13-25-12-0700-item12-gzip-biome-check.log`. Rev: `tlstlwvy`. + - **yq — DONE.** Added package-local VM e2e coverage for the staged `yq` + command and fixed the wrapper to accept file operands instead of only + stdin. The suite proves YAML filtering, YAML-to-JSON query output, + explicit JSON/TOML/XML input formats, and invalid YAML parse errors through + the packaged WASM command. Proof: + `2026-07-08T13-27-36-0700-item12-yq-toolchain-cmd-build.log`; + `2026-07-08T13-28-59-0700-item12-yq-package-build-after-file-operands.log`; + `2026-07-08T13-29-10-0700-item12-yq-package-e2e-final.log`; + `2026-07-08T13-29-10-0700-item12-yq-check-types-final.log`; + `2026-07-08T13-29-11-0700-item12-yq-cargo-fmt-check-final.log`. + Biome is not applicable for this package test path; it reported the file is + ignored by config in + `2026-07-08T13-29-23-0700-item12-yq-biome-check.log`. Rev: `znlmtymu`. - **jq — DONE.** Added package-local VM e2e coverage for the staged `jq` command and fixed the jaq-backed CLI wrapper to accept Linux-style file operands instead of only stdin. The suite now proves version output, diff --git a/software/yq/bin/yq b/software/yq/bin/yq index 98a2de41ee..3c58c4cbf8 100644 Binary files a/software/yq/bin/yq and b/software/yq/bin/yq differ diff --git a/software/yq/native/crates/yq/src/lib.rs b/software/yq/native/crates/yq/src/lib.rs index 871e3d7b85..e5332476bd 100644 --- a/software/yq/native/crates/yq/src/lib.rs +++ b/software/yq/native/crates/yq/src/lib.rs @@ -5,6 +5,7 @@ use std::ffi::OsString; use std::fmt; +use std::fs::File as FsFile; use std::io::{self, Read, Write}; use jaq_core::load::{Arena, File, Loader}; @@ -35,6 +36,7 @@ struct YqOptions { compact: bool, null_input: bool, slurp: bool, + input_paths: Vec, } /// Entry point for yq command. @@ -76,6 +78,7 @@ fn parse_args(args: &[String]) -> Result { compact: false, null_input: false, slurp: false, + input_paths: Vec::new(), }; let mut filter_set = false; @@ -85,6 +88,16 @@ fn parse_args(args: &[String]) -> Result { let arg = &args[i]; if arg == "--" { + let remaining = &args[i + 1..]; + if !filter_set { + if let Some(filter) = remaining.first() { + opts.filter = filter.clone(); + filter_set = true; + opts.input_paths.extend(remaining.iter().skip(1).cloned()); + } + } else { + opts.input_paths.extend(remaining.iter().cloned()); + } break; } @@ -151,7 +164,7 @@ fn parse_args(args: &[String]) -> Result { opts.filter = arg.clone(); filter_set = true; } else { - return Err(format!("unexpected argument: {}", arg)); + opts.input_paths.push(arg.clone()); } i += 1; @@ -404,17 +417,47 @@ fn record_output_value(output_count: &mut usize) -> Result<(), String> { } fn read_limited_string(reader: R) -> Result { + read_limited_source(reader, "stdin", MAX_INPUT_BYTES) +} + +fn read_limited_source(reader: R, label: &str, limit: usize) -> Result { let mut input = String::new(); reader - .take((MAX_INPUT_BYTES + 1) as u64) + .take((limit + 1) as u64) .read_to_string(&mut input) - .map_err(|e| format!("failed to read stdin: {}", e))?; - if input.len() > MAX_INPUT_BYTES { - return Err("stdin exceeds size limit".to_string()); + .map_err(|e| format!("failed to read {}: {}", label, e))?; + if input.len() > limit { + return Err(format!("{} exceeds size limit", label)); } Ok(input) } +fn read_sources(opts: &YqOptions) -> Result, String> { + if opts.input_paths.is_empty() { + return Ok(vec![read_limited_string(io::stdin())?]); + } + + let mut total = 0usize; + let mut sources = Vec::new(); + for path in &opts.input_paths { + let remaining = MAX_INPUT_BYTES.saturating_sub(total); + let data = if path == "-" { + read_limited_source(io::stdin(), "stdin", remaining)? + } else { + let file = FsFile::open(path).map_err(|e| format!("failed to open {}: {}", path, e))?; + read_limited_source(file, path, remaining)? + }; + total = total + .checked_add(data.len()) + .ok_or_else(|| "input exceeds size limit".to_string())?; + if total > MAX_INPUT_BYTES { + return Err("input exceeds size limit".to_string()); + } + sources.push(data); + } + Ok(sources) +} + fn insert_or_array( map: &mut serde_json::Map, key: String, @@ -810,37 +853,39 @@ fn write_toml_string(out: &mut LimitedString, s: &str) -> Result<(), String> { fn run_yq(args: &[String]) -> Result { let opts = parse_args(args)?; - // Read input - let stdin_data = if opts.null_input { - String::new() + let sources = if opts.null_input { + Vec::new() } else { - read_limited_string(io::stdin())? + read_sources(&opts)? }; - // Determine input format - let in_format = opts.input_format.unwrap_or_else(|| { - if opts.null_input { - Format::Yaml - } else { - detect_format(&stdin_data) - } - }); + let input_formats: Vec = if opts.null_input { + vec![opts.input_format.unwrap_or(Format::Yaml)] + } else { + sources + .iter() + .map(|source| opts.input_format.unwrap_or_else(|| detect_format(source))) + .collect() + }; + let default_input_format = input_formats.first().copied().unwrap_or(Format::Yaml); // Default output format: YAML for YAML input, otherwise matches input - let out_format = opts.output_format.unwrap_or(in_format); + let out_format = opts.output_format.unwrap_or(default_input_format); // Parse input to JSON, then convert to jaq Val let inputs = if opts.null_input { vec![Val::from(serde_json::Value::Null)] } else { - let json_val = parse_input(&stdin_data, in_format)?; + let json_values: Result, _> = sources + .iter() + .zip(input_formats.iter().copied()) + .map(|(source, format)| parse_input(source, format)) + .collect(); + let mut json_values = json_values?; if opts.slurp { - match json_val { - serde_json::Value::Array(_) => vec![Val::from(json_val)], - _ => vec![Val::from(serde_json::Value::Array(vec![json_val]))], - } + vec![Val::from(serde_json::Value::Array(json_values))] } else { - vec![Val::from(json_val)] + json_values.drain(..).map(Val::from).collect() } }; diff --git a/software/yq/test/yq.test.ts b/software/yq/test/yq.test.ts new file mode 100644 index 0000000000..c74fe1b1d7 --- /dev/null +++ b/software/yq/test/yq.test.ts @@ -0,0 +1,181 @@ +import { existsSync } from "node:fs"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { + NodeFileSystem, + createKernel, + createWasmVmRuntime, + describeIf, +} from "@agentos/test-harness"; +import type { Kernel } from "@agentos/test-harness"; +import { afterEach, expect, it } from "vitest"; + +const YQ_COMMAND_DIR = fileURLToPath(new URL("../bin", import.meta.url)); +const hasYqPackageBinary = existsSync(join(YQ_COMMAND_DIR, "yq")); + +let tempRoot: string | undefined; + +async function writeFixture(path: string, contents: string): Promise { + if (!tempRoot) throw new Error("fixture root not initialized"); + const hostPath = join(tempRoot, path.replace(/^\/+/, "")); + await mkdir(dirname(hostPath), { recursive: true }); + await writeFile(hostPath, contents); +} + +async function createTestVFS(): Promise { + tempRoot = await mkdtemp(join(tmpdir(), "agentos-yq-")); + await writeFixture( + "/project/services.yaml", + [ + "services:", + " - name: api", + " enabled: true", + " port: 8080", + " - name: worker", + " enabled: false", + " port: 9090", + ].join("\n") + "\n", + ); + await writeFixture( + "/project/services.json", + JSON.stringify({ services: [{ name: "api" }, { name: "worker" }] }) + "\n", + ); + await writeFixture( + "/project/config.toml", + ["[server]", 'name = "agentos"', "port = 7331", "enabled = true"].join("\n") + + "\n", + ); + await writeFixture( + "/project/inventory.xml", + 'hammernail\n', + ); + await writeFixture("/project/broken.yaml", "services:\n - name: ok\n bad"); + return new NodeFileSystem({ root: tempRoot }); +} + +function lines(stdout: string): string[] { + return stdout.split("\n").filter((line) => line.length > 0); +} + +describeIf(hasYqPackageBinary, "yq command", { timeout: 10_000 }, () => { + let kernel: Kernel | undefined; + + afterEach(async () => { + await kernel?.dispose(); + kernel = undefined; + if (tempRoot) { + await rm(tempRoot, { recursive: true, force: true }); + tempRoot = undefined; + } + }); + + async function mountFixture(): Promise { + const vfs = await createTestVFS(); + kernel = createKernel({ filesystem: vfs }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [YQ_COMMAND_DIR] })); + } + + async function runYq(args: string[]) { + if (!kernel) throw new Error("kernel not mounted"); + let stdout = ""; + let stderr = ""; + const proc = kernel.spawn("yq", args, { + onStdout: (chunk) => { + stdout += Buffer.from(chunk).toString("utf8"); + }, + onStderr: (chunk) => { + stderr += Buffer.from(chunk).toString("utf8"); + }, + }); + const exitCode = await proc.wait(); + await new Promise((resolve) => setTimeout(resolve, 0)); + return { stdout, stderr, exitCode }; + } + + it("filters YAML files and emits raw strings", async () => { + await mountFixture(); + + const result = await runYq([ + "-r", + ".services[] | select(.enabled) | .name", + "/project/services.yaml", + ]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(lines(result.stdout)).toEqual(["api"]); + }); + + it("converts YAML query results to compact JSON", async () => { + await mountFixture(); + + const result = await runYq([ + "-o", + "json", + "-c", + "{names: [.services[].name], ports: [.services[].port]}", + "/project/services.yaml", + ]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(JSON.parse(result.stdout.trim())).toEqual({ + names: ["api", "worker"], + ports: [8080, 9090], + }); + }); + + it("reads JSON files explicitly", async () => { + await mountFixture(); + + const result = await runYq([ + "-p", + "json", + "-r", + ".services[].name", + "/project/services.json", + ]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(lines(result.stdout)).toEqual(["api", "worker"]); + }); + + it("reads TOML files explicitly", async () => { + await mountFixture(); + + const result = await runYq([ + "-p", + "toml", + "-o", + "json", + "-c", + ".server", + "/project/config.toml", + ]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(JSON.parse(result.stdout.trim())).toEqual({ + name: "agentos", + port: 7331, + enabled: true, + }); + }); + + it("reads XML files explicitly", async () => { + await mountFixture(); + + const result = await runYq([ + "-p", + "xml", + "-r", + '.inventory.item[]["#text"]', + "/project/inventory.xml", + ]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(lines(result.stdout)).toEqual(["hammer", "nail"]); + }); + + it("fails with a parse error for invalid YAML", async () => { + await mountFixture(); + + const result = await runYq([".", "/project/broken.yaml"]); + expect(result.exitCode).not.toBe(0); + expect(result.stderr).toContain("invalid YAML"); + }); +});