test(duckdb): prove analytical csv e2e

docs-internal/registry-parity-worklist.md

@@ -536,8 +536,16 @@ real e2e tests that prove Linux-parity behavior — not smoke tests.
     eagerly requiring every registry package to be built. Proof:
     `2026-07-08T13-13-00-0700-item11-git-quickstart-final.log`.
     Rev: `svltqsmx`.
-  - **duckdb — TODO.** Needs real analytical SQL, CSV read/write, and file-backed
-    DB e2e kept runnable when the DuckDB package artifacts are present.
+  - **duckdb — DONE.** Rebuilt the upstream DuckDB package artifact from the
+    patched C sysroot and strengthened the package e2e so it validates only the
+    DuckDB/Curl packages it uses. The VM e2e now covers file-backed DML, real
+    analytical SQL, DuckDB CSV export, `read_csv_auto` re-import, and the
+    negative HTTP-URL path for DuckDB itself. Proof:
+    `2026-07-08T12-38-07-0700-item11-duckdb-package-build-after-install.log`;
+    `2026-07-08T12-46-18-0700-item11-duckdb-package-e2e-final-file-pass.log`;
+    `2026-07-08T12-46-11-0700-item11-core-tsc-duckdb-final-file-after-refresh.log`;
+    `2026-07-08T12-46-10-0700-item11-duckdb-biome-check-final-file-after-refresh.log`.
+    Rev: `qrwnvouk`.
   - **codex — TODO.** Needs the remaining real full-turn coverage un-skipped after
     #9.
 - **rev:** one per command, e.g. `test(duckdb): real analytical-SQL e2e; un-skip`

packages/core/tests/duckdb-package.test.ts

@@ -1,25 +1,60 @@
+import { existsSync, statSync } from "node:fs";
 import {
 	createServer,
 	type IncomingMessage,
 	type Server,
 	type ServerResponse,
 } from "node:http";
+import { dirname, join } from "node:path";
 import coreutils from "@agentos-software/coreutils";
 import curl from "@agentos-software/curl";
-import { afterEach, beforeEach, describe, expect, test } from "vitest";
 import duckdb from "@agentos-software/duckdb";
+import { afterEach, beforeEach, describe, expect, test } from "vitest";
 import { AgentOs } from "../dist/index.js";
-import { cSysrootPackageSkipReason } from "./helpers/registry-commands.js";
 
 const DUCKDB_PACKAGE = duckdb;
 const CURL_PACKAGE = curl;
 // C-sysroot packages are the ONE sanctioned skip: they need the patched wasi C
-// sysroot most checkouts don't build (see helpers/registry-commands.ts).
+// sysroot most checkouts don't build.
 const duckdbPackageSkipReason = cSysrootPackageSkipReason(
 	{ pkg: DUCKDB_PACKAGE, name: "duckdb" },
 	{ pkg: CURL_PACKAGE, name: "curl" },
 );
 
+interface RegistryPackageRef {
+	packagePath: string;
+}
+
+function isNonPlaceholderPackage(path: string): boolean {
+	try {
+		return existsSync(path) && statSync(path).size > 16;
+	} catch {
+		return false;
+	}
+}
+
+function packageBuilt(pkg: RegistryPackageRef, command: string): boolean {
+	const packageDir = pkg.packagePath.endsWith(".aospkg")
+		? join(dirname(pkg.packagePath), "package")
+		: pkg.packagePath;
+	return (
+		isNonPlaceholderPackage(pkg.packagePath) &&
+		existsSync(join(packageDir, "agentos-package.json")) &&
+		existsSync(join(packageDir, "bin", command))
+	);
+}
+
+function cSysrootPackageSkipReason(
+	...packages: Array<{ pkg: RegistryPackageRef; name: string }>
+): string | false {
+	const unbuilt = packages.filter(({ pkg, name }) => !packageBuilt(pkg, name));
+	if (unbuilt.length === 0) return false;
+	return (
+		`C-sysroot software packages not built: ${unbuilt.map(({ name }) => name).join(", ")} ` +
+		"(needs the patched wasi C sysroot: `make -C toolchain/c`, then `pnpm --dir software/<package> build`)"
+	);
+}
+
 function closeServer(server: Server) {
 	return new Promise<void>((resolve, reject) => {
 		server.close((err) => {
@@ -63,9 +98,7 @@ describe("duckdb registry package", () => {
 		await vm.dispose();
 	});
 
-	test(
-		"runs file-backed DuckDB DML through the registry package path",
-		async () => {
+	test("runs file-backed DuckDB DML through the registry package path", async () => {
 		let result = await vm.exec(
 			`duckdb -csv /tmp/app.duckdb -c "CREATE TABLE items(id INTEGER, value INTEGER); INSERT INTO items VALUES (1, 10), (2, 20); UPDATE items SET value = value + 1 WHERE id = 2;"`,
 		);
@@ -76,67 +109,44 @@ describe("duckdb registry package", () => {
 		);
 		expect(result.exitCode, result.stderr || result.stdout).toBe(0);
 		expect(result.stdout.trim()).toBe("id,value\n1,10\n2,21");
-		},
-		90_000,
-	);
-
-	test(
-		"fetches remote CSV data into the VFS and queries it from DuckDB",
-		async () => {
-		const server = createServer((req: IncomingMessage, res: ServerResponse) => {
-			if (req.url === "/remote.csv") {
-				res.writeHead(200, { "Content-Type": "text/csv" });
-				res.end("city,value\nsf,3\nla,5\n");
-				return;
-			}
-
-			res.writeHead(404, { "Content-Type": "text/plain" });
-			res.end("not found");
-		});
+	}, 90_000);
 
-		await new Promise<void>((resolve) =>
-			server.listen(0, "127.0.0.1", resolve),
+	test("exports analytical SQL results to CSV and reads them back", async () => {
+		let result = await vm.exec(
+			[
+				`duckdb /tmp/analytics.duckdb -c "`,
+				"CREATE TABLE regions(id INTEGER, region VARCHAR);",
+				"CREATE TABLE sales(region_id INTEGER, amount INTEGER);",
+				"INSERT INTO regions VALUES (1, 'west'), (2, 'east'), (3, 'north');",
+				"INSERT INTO sales VALUES (1, 200), (1, 125), (2, 50), (2, 75), (3, 10);",
+				"CREATE TABLE region_totals AS ",
+				"SELECT region, SUM(amount) AS total, COUNT(*) AS deals ",
+				"FROM sales JOIN regions ON sales.region_id = regions.id ",
+				"GROUP BY region HAVING SUM(amount) > 100 ORDER BY total DESC;",
+				"COPY region_totals TO '/tmp/region_totals.csv' (HEADER, DELIMITER ',');",
+				`"`,
+			].join(""),
 		);
+		expect(result.exitCode, result.stderr || result.stdout).toBe(0);
 
-		try {
-			const address = server.address();
-			if (!address || typeof address === "string") {
-				throw new Error("failed to bind test HTTP server");
-			}
-			await recreateVm({ loopbackExemptPorts: [address.port] });
-
-			let result = await vm.exec(
-				`curl -fsS -o /tmp/remote.csv http://127.0.0.1:${address.port}/remote.csv`,
-			);
-			expect(result.exitCode, result.stderr || result.stdout).toBe(0);
-
-			result = await vm.exec(
-				`duckdb -csv -c "SELECT SUM(value) AS total FROM read_csv_auto('/tmp/remote.csv');"`,
-			);
-			expect(result.exitCode, result.stderr || result.stdout).toBe(0);
-			expect(result.stdout.trim()).toBe("total\n8");
-		} finally {
-			await closeServer(server);
-		}
-		},
-		90_000,
-	);
+		result = await vm.exec(
+			`duckdb -csv /tmp/analytics.duckdb -c "SELECT region, total, deals FROM read_csv_auto('/tmp/region_totals.csv') ORDER BY total DESC;"`,
+		);
+		expect(result.exitCode, result.stderr || result.stdout).toBe(0);
+		expect(result.stdout.trim()).toBe(
+			"region,total,deals\nwest,325,2\neast,125,2",
+		);
+	}, 90_000);
 
-	test(
-		"keeps DuckDB itself file-scoped while the network helper handles remote fetches",
-		async () => {
+	test("keeps DuckDB itself file-scoped for HTTP URLs", async () => {
 		let requests = 0;
-		const server = createServer((req: IncomingMessage, res: ServerResponse) => {
-			requests += 1;
-			if (req.url === "/remote.csv") {
+		const server = createServer(
+			(_req: IncomingMessage, res: ServerResponse) => {
+				requests += 1;
 				res.writeHead(200, { "Content-Type": "text/csv" });
 				res.end("city,value\nsf,3\nla,5\n");
-				return;
-			}
-
-			res.writeHead(404, { "Content-Type": "text/plain" });
-			res.end("not found");
-		});
+			},
+		);
 
 		await new Promise<void>((resolve) =>
 			server.listen(0, "127.0.0.1", resolve),
@@ -157,25 +167,22 @@ describe("duckdb registry package", () => {
 		} finally {
 			await closeServer(server);
 		}
-		},
-		90_000,
-	);
+	}, 90_000);
 
-	test(
-		"propagates registry package command permission tiers into the runtime",
-		async () => {
+	test("propagates registry package command permission tiers into the runtime", async () => {
 		await vm.dispose();
 
-			const curlReadOnly = {
+		const curlReadOnly = {
 			...CURL_PACKAGE,
 			commands: [{ name: "curl", permissionTier: "read-only" as const }],
 		};
-		vm = await AgentOs.create({ software: [coreutils, curlReadOnly] });
 
-		const server = createServer((req: IncomingMessage, res: ServerResponse) => {
-			res.writeHead(200, { "Content-Type": "text/plain" });
-			res.end("ok");
-		});
+		const server = createServer(
+			(_req: IncomingMessage, res: ServerResponse) => {
+				res.writeHead(200, { "Content-Type": "text/plain" });
+				res.end("ok");
+			},
+		);
 
 		await new Promise<void>((resolve) =>
 			server.listen(0, "127.0.0.1", resolve),
@@ -198,7 +205,5 @@ describe("duckdb registry package", () => {
 		} finally {
 			await closeServer(server);
 		}
-		},
-		90_000,
-	);
+	}, 90_000);
 });