diff --git a/software/wget/native/c/overlay/src/wasi_ssl.c b/software/wget/native/c/overlay/src/wasi_ssl.c new file mode 100644 index 0000000000..222ad09429 --- /dev/null +++ b/software/wget/native/c/overlay/src/wasi_ssl.c @@ -0,0 +1,535 @@ +/* In-guest TLS backend for GNU Wget on wasm32-wasip1, built on mbedTLS. + + GNU Wget has no upstream mbedTLS backend; its TLS abstraction is the four + functions declared in src/ssl.h (ssl_init, ssl_cleanup, ssl_connect_wget, + ssl_check_certificate). This file implements those four against mbedTLS, + performing a real TLS handshake and X.509 chain + hostname verification + entirely inside the guest -- the sidecar is a dumb ciphertext pipe. + + The already-connected TCP file descriptor handed to ssl_connect_wget is a + real socket carried by the patched wasi-libc sysroot (host_net imports), so + the mbedTLS BIO callbacks simply read()/write() that fd. On success the fd is + registered with Wget's transport layer (fd_register_transport) so that all + subsequent fd_read/fd_write/fd_peek calls flow through the TLS session. + + Trust configuration mirrors Linux Wget: certificates are verified against + /etc/ssl/certs/ca-certificates.crt by default, --ca-certificate + (opt.ca_cert) and --ca-directory (opt.ca_directory) override the trust + anchors, --crl-file (opt.crl_file) adds a revocation list, and + --no-check-certificate (opt.check_cert != CHECK_CERT_ON) downgrades a + verification failure to a warning. A failed handshake or a rejected + certificate makes the relevant function return false, so http.c reports + CONSSLERR / VERIFCERTERR exactly as with the GnuTLS/OpenSSL backends. */ + +#include "wget.h" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "connect.h" +#include "log.h" +#include "ssl.h" +#include "url.h" +#include "utils.h" + +/* Debian-shaped default trust store, seeded into the VM by the native + bootstrap. Matches curl's compile-time CA bundle default and OpenSSL's + OPENSSLDIR resolution on Debian. */ +#ifndef WASI_TLS_DEFAULT_CA_BUNDLE +# define WASI_TLS_DEFAULT_CA_BUNDLE "/etc/ssl/certs/ca-certificates.crt" +#endif + +/* The TLS transport singleton, defined after the I/O callbacks below. */ +static struct transport_implementation wasi_tls_transport; + +struct wasi_ssl_context +{ + mbedtls_ssl_context ssl; + mbedtls_ssl_config conf; + mbedtls_x509_crt cacert; + mbedtls_x509_crl crl; + bool have_crl; + mbedtls_ctr_drbg_context ctr_drbg; + mbedtls_entropy_context entropy; + int fd; + int last_err; /* last mbedTLS error, for errstr */ + unsigned char *peekbuf; /* buffered-but-unconsumed plaintext */ + int peeklen; + int peekcap; +}; + +/* mbedTLS BIO send/recv over the raw, already-connected socket fd. Sockets are + blocking on wasip1 (Wget only flips them non-blocking on Windows), but we map + EAGAIN/EINTR anyway so the handshake and I/O paths stay correct if a socket + is ever non-blocking. */ + +static int +wasi_bio_send (void *arg, const unsigned char *buf, size_t len) +{ + struct wasi_ssl_context *ctx = arg; + ssize_t n; + + do + n = write (ctx->fd, buf, len); + while (n < 0 && errno == EINTR); + + if (n >= 0) + return (int) n; + if (errno == EAGAIN || errno == EWOULDBLOCK) + return MBEDTLS_ERR_SSL_WANT_WRITE; + return MBEDTLS_ERR_NET_SEND_FAILED; +} + +static int +wasi_bio_recv (void *arg, unsigned char *buf, size_t len) +{ + struct wasi_ssl_context *ctx = arg; + ssize_t n; + + do + n = read (ctx->fd, buf, len); + while (n < 0 && errno == EINTR); + + if (n >= 0) + return (int) n; + if (errno == EAGAIN || errno == EWOULDBLOCK) + return MBEDTLS_ERR_SSL_WANT_READ; + return MBEDTLS_ERR_NET_RECV_FAILED; +} + +/* Global init. mbedTLS keeps no process-wide state we must set up, so this is + a no-op that simply reports readiness, like the GnuTLS backend. */ +bool +ssl_init (void) +{ + return true; +} + +void +ssl_cleanup (void) +{ +} + +/* Map --secure-protocol onto the mbedTLS min/max TLS version knobs. mbedTLS + 3.6 only speaks TLS 1.2 and 1.3, so the legacy SSLv3/TLS1.0/1.1 selectors + pin the floor at TLS 1.2 (the lowest still supported), matching how a modern + Wget build behaves. */ +static void +wasi_apply_secure_protocol (mbedtls_ssl_config *conf) +{ + switch (opt.secure_protocol) + { + case secure_protocol_tlsv1_3: + mbedtls_ssl_conf_min_tls_version (conf, MBEDTLS_SSL_VERSION_TLS1_3); + mbedtls_ssl_conf_max_tls_version (conf, MBEDTLS_SSL_VERSION_TLS1_3); + break; + case secure_protocol_tlsv1_2: + mbedtls_ssl_conf_min_tls_version (conf, MBEDTLS_SSL_VERSION_TLS1_2); + mbedtls_ssl_conf_max_tls_version (conf, MBEDTLS_SSL_VERSION_TLS1_2); + break; + case secure_protocol_tlsv1: + case secure_protocol_tlsv1_1: + case secure_protocol_sslv2: + case secure_protocol_sslv3: + /* Not supported by mbedTLS 3.6; fall back to the lowest available. */ + mbedtls_ssl_conf_min_tls_version (conf, MBEDTLS_SSL_VERSION_TLS1_2); + break; + case secure_protocol_auto: + case secure_protocol_pfs: + default: + /* Library defaults: TLS 1.2 .. 1.3. */ + break; + } +} + +/* Load the trust anchors exactly like Linux Wget: --ca-certificate and + --ca-directory when given, otherwise the seeded Debian bundle. Returns the + number of certificates parsed (>= 0) or a negative mbedTLS error. */ +static int +wasi_load_trust (struct wasi_ssl_context *ctx) +{ + int loaded = 0; + int ret; + + if (opt.ca_cert) + { + ret = mbedtls_x509_crt_parse_file (&ctx->cacert, opt.ca_cert); + if (ret < 0) + return ret; + loaded += 1; + } + + if (opt.ca_directory && 0 != strcmp (opt.ca_directory, "")) + { + ret = mbedtls_x509_crt_parse_path (&ctx->cacert, opt.ca_directory); + /* parse_path returns the number of files that failed to parse as a + positive value; only a negative value is a hard error. */ + if (ret < 0) + return ret; + loaded += 1; + } + + if (loaded == 0) + { + ret = mbedtls_x509_crt_parse_file (&ctx->cacert, WASI_TLS_DEFAULT_CA_BUNDLE); + if (ret < 0) + return ret; + loaded += 1; + } + + return loaded; +} + +/* Perform the TLS handshake on FD and, on success, register the TLS transport + so Wget's fd_* helpers use it. CONTINUE_SESSION (session resumption) is not + supported here; http.c always passes NULL. Returns true on success. */ +bool +ssl_connect_wget (int fd, const char *hostname, int *continue_session) +{ + struct wasi_ssl_context *ctx; + int ret; + + (void) continue_session; + + DEBUGP (("Initiating SSL handshake (mbedTLS).\n")); + + ctx = xnew0 (struct wasi_ssl_context); + ctx->fd = fd; + + mbedtls_ssl_init (&ctx->ssl); + mbedtls_ssl_config_init (&ctx->conf); + mbedtls_x509_crt_init (&ctx->cacert); + mbedtls_x509_crl_init (&ctx->crl); + mbedtls_ctr_drbg_init (&ctx->ctr_drbg); + mbedtls_entropy_init (&ctx->entropy); + + ret = mbedtls_ctr_drbg_seed (&ctx->ctr_drbg, mbedtls_entropy_func, + &ctx->entropy, + (const unsigned char *) "agentos-wget-tls", 16); + if (ret != 0) + goto error; + + /* Load trust anchors. A failure to read the bundle is only fatal when the + user asked us to verify; with --no-check-certificate we proceed with an + empty trust store (verification is skipped in ssl_check_certificate). */ + ret = wasi_load_trust (ctx); + if (ret < 0 && opt.check_cert == CHECK_CERT_ON) + { + char errbuf[128]; + mbedtls_strerror (ret, errbuf, sizeof errbuf); + logprintf (LOG_NOTQUIET, + _("Could not load CA certificates: %s\n"), errbuf); + goto error; + } + + if (opt.crl_file) + { + ret = mbedtls_x509_crl_parse_file (&ctx->crl, opt.crl_file); + if (ret != 0 && opt.check_cert == CHECK_CERT_ON) + { + char errbuf[128]; + mbedtls_strerror (ret, errbuf, sizeof errbuf); + logprintf (LOG_NOTQUIET, + _("Could not load CRL from %s: %s\n"), + opt.crl_file, errbuf); + goto error; + } + if (ret == 0) + ctx->have_crl = true; + } + + ret = mbedtls_ssl_config_defaults (&ctx->conf, MBEDTLS_SSL_IS_CLIENT, + MBEDTLS_SSL_TRANSPORT_STREAM, + MBEDTLS_SSL_PRESET_DEFAULT); + if (ret != 0) + goto error; + + wasi_apply_secure_protocol (&ctx->conf); + + /* Verify optionally: the handshake always completes and records the chain + + hostname result, which ssl_check_certificate reads and turns into a + pass/fail decision honoring opt.check_cert -- the same split the OpenSSL + backend uses (SSL_VERIFY_NONE at handshake, manual check afterwards). */ + mbedtls_ssl_conf_authmode (&ctx->conf, MBEDTLS_SSL_VERIFY_OPTIONAL); + mbedtls_ssl_conf_ca_chain (&ctx->conf, &ctx->cacert, + ctx->have_crl ? &ctx->crl : NULL); + mbedtls_ssl_conf_rng (&ctx->conf, mbedtls_ctr_drbg_random, &ctx->ctr_drbg); + + ret = mbedtls_ssl_setup (&ctx->ssl, &ctx->conf); + if (ret != 0) + goto error; + + /* SNI + the name checked during verification. Covers DNS and IP-address + SANs (mbedTLS matches an IP literal against iPAddress SAN entries). */ + ret = mbedtls_ssl_set_hostname (&ctx->ssl, hostname); + if (ret != 0) + goto error; + + mbedtls_ssl_set_bio (&ctx->ssl, ctx, wasi_bio_send, wasi_bio_recv, NULL); + + do + { + ret = mbedtls_ssl_handshake (&ctx->ssl); + if (ret == MBEDTLS_ERR_SSL_WANT_READ) + select_fd (fd, opt.read_timeout, WAIT_FOR_READ); + else if (ret == MBEDTLS_ERR_SSL_WANT_WRITE) + select_fd (fd, opt.read_timeout, WAIT_FOR_WRITE); + } + while (ret == MBEDTLS_ERR_SSL_WANT_READ || ret == MBEDTLS_ERR_SSL_WANT_WRITE); + + if (ret != 0) + { + char errbuf[128]; + mbedtls_strerror (ret, errbuf, sizeof errbuf); + DEBUGP (("SSL handshake failed: %s\n", errbuf)); + logprintf (LOG_NOTQUIET, _("SSL handshake failed: %s\n"), errbuf); + goto error; + } + + /* Register FD with Wget's transport layer so fd_read/fd_write/fd_peek use + our TLS callbacks from here on. */ + fd_register_transport (fd, &wasi_tls_transport, ctx); + DEBUGP (("Handshake successful; TLS registered on socket %d\n", fd)); + + return true; + + error: + mbedtls_ssl_free (&ctx->ssl); + mbedtls_ssl_config_free (&ctx->conf); + mbedtls_x509_crt_free (&ctx->cacert); + mbedtls_x509_crl_free (&ctx->crl); + mbedtls_ctr_drbg_free (&ctx->ctr_drbg); + mbedtls_entropy_free (&ctx->entropy); + xfree (ctx); + return false; +} + +/* --- Wget transport implementation over the TLS session --- */ + +static int +wasi_tls_read (int fd, char *buf, int bufsize, void *arg, double timeout) +{ + struct wasi_ssl_context *ctx = arg; + int ret; + + /* Serve any peeked-but-unconsumed plaintext first. */ + if (ctx->peeklen > 0) + { + int n = ctx->peeklen < bufsize ? ctx->peeklen : bufsize; + memcpy (buf, ctx->peekbuf, n); + if (n < ctx->peeklen) + memmove (ctx->peekbuf, ctx->peekbuf + n, ctx->peeklen - n); + ctx->peeklen -= n; + return n; + } + + if (timeout == -1) + timeout = opt.read_timeout; + if (timeout && mbedtls_ssl_get_bytes_avail (&ctx->ssl) == 0) + { + int sel = select_fd (fd, timeout, WAIT_FOR_READ); + if (sel <= 0) + return sel; /* 0 = timeout, -1 = error; matches Wget's expectation */ + } + + do + ret = mbedtls_ssl_read (&ctx->ssl, (unsigned char *) buf, bufsize); + while (ret == MBEDTLS_ERR_SSL_WANT_READ || ret == MBEDTLS_ERR_SSL_WANT_WRITE); + + if (ret == MBEDTLS_ERR_SSL_PEER_CLOSE_NOTIFY) + return 0; + if (ret < 0) + { + ctx->last_err = ret; + return -1; + } + return ret; +} + +static int +wasi_tls_write (int fd _GL_UNUSED, char *buf, int bufsize, void *arg) +{ + struct wasi_ssl_context *ctx = arg; + int written = 0; + + while (written < bufsize) + { + int ret = mbedtls_ssl_write (&ctx->ssl, + (const unsigned char *) buf + written, + bufsize - written); + if (ret == MBEDTLS_ERR_SSL_WANT_READ || ret == MBEDTLS_ERR_SSL_WANT_WRITE) + continue; + if (ret < 0) + { + ctx->last_err = ret; + return -1; + } + written += ret; + } + return written; +} + +static int +wasi_tls_poll (int fd, double timeout, int wait_for, void *arg) +{ + struct wasi_ssl_context *ctx = arg; + + if ((wait_for & WAIT_FOR_READ) + && (ctx->peeklen > 0 || mbedtls_ssl_get_bytes_avail (&ctx->ssl) > 0)) + return 1; + if (timeout == -1) + timeout = opt.read_timeout; + return select_fd (fd, timeout, wait_for); +} + +static int +wasi_tls_peek (int fd, char *buf, int bufsize, void *arg, double timeout) +{ + struct wasi_ssl_context *ctx = arg; + int n; + + /* Mirror recv(MSG_PEEK)/SSL_peek semantics: preview data without consuming + it, returning as soon as *some* data is available -- one TLS record's + worth. Never try to fill BUFSIZE (that would block on a keep-alive + connection once the whole response fits in one record). Wget's + fd_read_hunk consumes each preview and re-peeks for the next record, so + returning one chunk at a time is sufficient and cannot deadlock. + + Data that mbedtls_ssl_read pulls off the wire here is retained in peekbuf + so wasi_tls_read drains it first -- i.e. the "peek" does not lose it. */ + if (ctx->peeklen == 0) + { + int ret; + + if (ctx->peekcap < bufsize) + { + ctx->peekbuf = xrealloc (ctx->peekbuf, bufsize); + ctx->peekcap = bufsize; + } + + if (timeout == -1) + timeout = opt.read_timeout; + if (timeout && mbedtls_ssl_get_bytes_avail (&ctx->ssl) == 0) + { + int sel = select_fd (fd, timeout, WAIT_FOR_READ); + if (sel <= 0) + return sel; /* 0 = timeout, -1 = error */ + } + + do + ret = mbedtls_ssl_read (&ctx->ssl, ctx->peekbuf, bufsize); + while (ret == MBEDTLS_ERR_SSL_WANT_READ + || ret == MBEDTLS_ERR_SSL_WANT_WRITE); + + if (ret == MBEDTLS_ERR_SSL_PEER_CLOSE_NOTIFY) + return 0; + if (ret < 0) + { + ctx->last_err = ret; + return -1; + } + ctx->peeklen = ret; + } + + n = ctx->peeklen < bufsize ? ctx->peeklen : bufsize; + if (n > 0) + memcpy (buf, ctx->peekbuf, n); + return n; +} + +static const char * +wasi_tls_errstr (int fd _GL_UNUSED, void *arg) +{ + struct wasi_ssl_context *ctx = arg; + static char errbuf[160]; + + if (ctx->last_err == 0) + return NULL; + mbedtls_strerror (ctx->last_err, errbuf, sizeof errbuf); + return errbuf; +} + +static void +wasi_tls_close (int fd, void *arg) +{ + struct wasi_ssl_context *ctx = arg; + + mbedtls_ssl_close_notify (&ctx->ssl); + mbedtls_ssl_free (&ctx->ssl); + mbedtls_ssl_config_free (&ctx->conf); + mbedtls_x509_crt_free (&ctx->cacert); + mbedtls_x509_crl_free (&ctx->crl); + mbedtls_ctr_drbg_free (&ctx->ctr_drbg); + mbedtls_entropy_free (&ctx->entropy); + xfree (ctx->peekbuf); + xfree (ctx); + + close (fd); + DEBUGP (("Closed %d/SSL (mbedTLS)\n", fd)); +} + +static struct transport_implementation wasi_tls_transport = { + wasi_tls_read, wasi_tls_write, wasi_tls_poll, + wasi_tls_peek, wasi_tls_errstr, wasi_tls_close +}; + +/* Verify the peer certificate against the configured trust anchors and check + that it matches HOST. Reads the verify result recorded during the handshake. + Returns false only when verification failed AND the user requested checking + (opt.check_cert == CHECK_CERT_ON); otherwise it warns and returns true, + matching Wget's --no-check-certificate semantics. */ +bool +ssl_check_certificate (int fd, const char *host) +{ + struct wasi_ssl_context *ctx = fd_transport_context (fd); + uint32_t flags; + bool success; + const char *severity = opt.check_cert ? _("ERROR") : _("WARNING"); + + if (!ctx) + return opt.check_cert != CHECK_CERT_ON; + + flags = mbedtls_ssl_get_verify_result (&ctx->ssl); + success = (flags == 0); + + if (!success) + { + char vbuf[512]; + int n = mbedtls_x509_crt_verify_info (vbuf, sizeof vbuf, " ", flags); + if (n < 0) + { + vbuf[0] = '\0'; + n = 0; + } + + logprintf (LOG_NOTQUIET, + _("%s: cannot verify %s's certificate:\n"), + severity, quote (host)); + if (n > 0) + logprintf (LOG_NOTQUIET, "%s", vbuf); + + if (opt.check_cert == CHECK_CERT_ON) + logprintf (LOG_NOTQUIET, + _("To connect to %s insecurely, use `--no-check-certificate'.\n"), + quote (host)); + } + else + DEBUGP (("X509 certificate successfully verified and matches host %s\n", + quote (host))); + + return opt.check_cert == CHECK_CERT_ON ? success : true; +} + +/* + * vim: tabstop=2 shiftwidth=2 softtabstop=2 + */ diff --git a/software/wget/test/wget.test.ts b/software/wget/test/wget.test.ts index c5c13ff20b..05d20d0e72 100644 --- a/software/wget/test/wget.test.ts +++ b/software/wget/test/wget.test.ts @@ -1,12 +1,26 @@ import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; -import { existsSync } from "node:fs"; +import { execSync } from "node:child_process"; +import { + existsSync, + mkdtempSync, + readFileSync, + rmSync, + unlinkSync, + writeFileSync, +} from "node:fs"; import { createServer, type IncomingMessage, type Server, type ServerResponse, } from "node:http"; -import { resolve } from "node:path"; +import { + createServer as createHttpsServer, + type Server as HttpsServer, +} from "node:https"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { gzipSync } from "node:zlib"; import { createWasmVmRuntime } from "@agentos/test-harness"; import { allowAll, @@ -15,6 +29,7 @@ import { createInMemoryFileSystem, createKernel, describeIf, + itIf, } from "@agentos/test-harness"; import type { Kernel } from "@agentos/test-harness"; @@ -26,10 +41,96 @@ const hasWgetBinary = WGET_COMMAND_DIRS.some((dir) => ); const WGET_EXEC_TIMEOUT_MS = 10_000; +let hasOpenssl = false; +try { + execSync("openssl version", { stdio: "pipe" }); + hasOpenssl = true; +} catch { + hasOpenssl = false; +} + +// A long, highly compressible payload so the gzipped body is clearly distinct +// from the plaintext (proving wget actually inflated it via zlib). +const COMPRESSION_PAYLOAD = + "agentos-wget-compression " + + "the quick brown fox jumps over the lazy dog. ".repeat(64); + +// Build a real CA and a leaf cert signed by it, with a SAN covering the +// 127.0.0.1 loopback endpoint the tests connect to. This lets wget's mbedTLS +// backend perform genuine chain + hostname verification, exactly like Linux +// wget against a private CA. +function makeCaSignedCert(caCommonName: string): { + caPem: string; + serverKey: string; + serverCert: string; +} { + const dir = mkdtempSync(join(tmpdir(), "wget-ca-")); + try { + execSync( + `openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "${dir}/ca.key" 2>/dev/null`, + ); + execSync( + `openssl req -x509 -new -key "${dir}/ca.key" -days 3650 -subj "/CN=${caCommonName}" -out "${dir}/ca.crt" 2>/dev/null`, + ); + execSync( + `openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "${dir}/srv.key" 2>/dev/null`, + ); + execSync( + `openssl req -new -key "${dir}/srv.key" -subj "/CN=localhost" -out "${dir}/srv.csr" 2>/dev/null`, + ); + writeFileSync(`${dir}/ext.cnf`, "subjectAltName=DNS:localhost,IP:127.0.0.1\n"); + execSync( + `openssl x509 -req -in "${dir}/srv.csr" -CA "${dir}/ca.crt" -CAkey "${dir}/ca.key" ` + + `-CAcreateserial -days 3650 -extfile "${dir}/ext.cnf" -out "${dir}/srv.crt" 2>/dev/null`, + ); + return { + caPem: readFileSync(`${dir}/ca.crt`, "utf8"), + serverKey: readFileSync(`${dir}/srv.key`, "utf8"), + serverCert: readFileSync(`${dir}/srv.crt`, "utf8"), + }; + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +function generateSelfSignedCert(): { key: string; cert: string } { + const keyPath = join(tmpdir(), `wget-test-key-${process.pid}-${Date.now()}.pem`); + try { + const key = execSync( + "openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 2>/dev/null", + { encoding: "utf8" }, + ); + writeFileSync(keyPath, key); + const cert = execSync( + `openssl req -new -x509 -key "${keyPath}" -days 1 -subj "/CN=localhost" -addext "subjectAltName=DNS:localhost,IP:127.0.0.1" 2>/dev/null`, + { encoding: "utf8" }, + ); + return { key, cert }; + } finally { + try { + unlinkSync(keyPath); + } catch { + // Best effort cleanup for test temp files. + } + } +} + describeIf(hasWgetBinary, "wget command", () => { let kernel: Kernel; let server: Server; + let selfSignedServer: HttpsServer; + let validHttpsServer: HttpsServer; + let caHttpsServer: HttpsServer; let port: number; + let selfSignedPort: number; + let validHttpsPort: number; + let caHttpsPort: number; + // CA (PEM) trusted by the seeded /etc/ssl/certs/ca-certificates.crt bundle; + // it signs validHttpsServer's leaf. caOnlyPem signs caHttpsServer's leaf and + // is deliberately NOT in the bundle, so it verifies only via + // --ca-certificate. + let seededCaPem = ""; + let caOnlyPem = ""; beforeAll(async () => { server = createServer((req: IncomingMessage, res: ServerResponse) => { @@ -47,6 +148,17 @@ describeIf(hasWgetBinary, "wget command", () => { return; } + if (url === "/gzip") { + const body = gzipSync(Buffer.from(COMPRESSION_PAYLOAD)); + res.writeHead(200, { + "Content-Type": "text/plain", + "Content-Encoding": "gzip", + "Content-Length": String(body.length), + }); + res.end(body); + return; + } + if (url === "/redirect") { res.writeHead(302, { Location: `http://127.0.0.1:${port}/redirected`, @@ -69,12 +181,72 @@ describeIf(hasWgetBinary, "wget command", () => { server.listen(0, "127.0.0.1", resolveListen), ); port = (server.address() as import("node:net").AddressInfo).port; + + if (hasOpenssl) { + // Self-signed leaf: no chain to any trusted CA -> must fail verify. + const selfSigned = generateSelfSignedCert(); + selfSignedServer = createHttpsServer( + { key: selfSigned.key, cert: selfSigned.cert }, + (req, res) => { + res.writeHead(200, { "Content-Type": "text/plain" }); + res.end("self-signed secure content"); + }, + ); + await new Promise((resolveListen) => + selfSignedServer.listen(0, "127.0.0.1", resolveListen), + ); + selfSignedPort = ( + selfSignedServer.address() as import("node:net").AddressInfo + ).port; + + // Leaf chaining to a CA seeded into the guest's bundle -> verifies + // with no --no-check-certificate / --ca-certificate. + const trusted = makeCaSignedCert("AgentOS Wget Test Root CA"); + seededCaPem = trusted.caPem; + validHttpsServer = createHttpsServer( + { key: trusted.serverKey, cert: trusted.serverCert }, + (req, res) => { + res.writeHead(200, { "Content-Type": "text/plain" }); + res.end("verified https content"); + }, + ); + await new Promise((resolveListen) => + validHttpsServer.listen(0, "127.0.0.1", resolveListen), + ); + validHttpsPort = ( + validHttpsServer.address() as import("node:net").AddressInfo + ).port; + + // Leaf whose CA is provided ONLY via --ca-certificate (not in bundle). + const caOnly = makeCaSignedCert("AgentOS Wget Cacert-Only CA"); + caOnlyPem = caOnly.caPem; + caHttpsServer = createHttpsServer( + { key: caOnly.serverKey, cert: caOnly.serverCert }, + (req, res) => { + res.writeHead(200, { "Content-Type": "text/plain" }); + res.end("cacert https content"); + }, + ); + await new Promise((resolveListen) => + caHttpsServer.listen(0, "127.0.0.1", resolveListen), + ); + caHttpsPort = ( + caHttpsServer.address() as import("node:net").AddressInfo + ).port; + } }); afterAll(async () => { - await new Promise((resolveClose) => - server.close(() => resolveClose()), - ); + for (const s of [ + server, + selfSignedServer, + validHttpsServer, + caHttpsServer, + ]) { + if (s) { + await new Promise((resolveClose) => s.close(() => resolveClose())); + } + } }); afterEach(async () => { @@ -86,9 +258,23 @@ describeIf(hasWgetBinary, "wget command", () => { kernel = createKernel({ filesystem, permissions: allowAll, - loopbackExemptPorts: [port], + loopbackExemptPorts: [ + port, + selfSignedPort, + validHttpsPort, + caHttpsPort, + ].filter((p) => typeof p === "number"), }); await kernel.mount(createWasmVmRuntime({ commandDirs: WGET_COMMAND_DIRS })); + + // Seed the Debian-shaped trust store the way the native VM bootstrap + // does, so wget's default CA bundle resolves in-guest. Only the + // "trusted" CA is placed here; the cacert-only CA is intentionally + // absent. + if (seededCaPem) { + await filesystem.mkdir("/etc/ssl/certs", { recursive: true }); + await kernel.writeFile("/etc/ssl/certs/ca-certificates.crt", seededCaPem); + } return filesystem; } @@ -159,4 +345,105 @@ describeIf(hasWgetBinary, "wget command", () => { "arrived after redirect", ); }, 15_000); + + it("--version reports the mbedTLS HTTPS backend", async () => { + await mountKernel(); + + const result = await kernel.exec("wget --version", { + timeout: WGET_EXEC_TIMEOUT_MS, + }); + + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(result.stdout).toContain("GNU Wget 1.24.5"); + // Real in-guest TLS: HTTPS is compiled in and the backend is mbedTLS. + expect(result.stdout).toMatch(/\+https/); + expect(result.stdout).toMatch(/ssl\/mbedtls/i); + }, 15_000); + + it("--compression=auto inflates a gzip response body", async () => { + const filesystem = await mountKernel(); + + const result = await kernel.exec( + `wget --compression=auto -O /gz.txt http://127.0.0.1:${port}/gzip`, + { timeout: WGET_EXEC_TIMEOUT_MS }, + ); + + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(await filesystem.readTextFile("/gz.txt")).toBe(COMPRESSION_PAYLOAD); + }, 15_000); + + itIf(hasOpenssl, "downloads over HTTPS verifying against the seeded CA bundle", async () => { + const filesystem = await mountKernel(); + + // No --no-check-certificate, no --ca-certificate: trust comes solely + // from the seeded /etc/ssl/certs/ca-certificates.crt, like Debian wget. + const result = await kernel.exec( + `wget -O /secure.txt https://127.0.0.1:${validHttpsPort}/file`, + { timeout: WGET_EXEC_TIMEOUT_MS }, + ); + + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(await filesystem.readTextFile("/secure.txt")).toBe( + "verified https content", + ); + }, 15_000); + + itIf(hasOpenssl, "fails with a real cert error on an untrusted (self-signed) server", async () => { + await mountKernel(); + + const result = await kernel.exec( + `wget -O /nope.txt https://127.0.0.1:${selfSignedPort}/file`, + { timeout: WGET_EXEC_TIMEOUT_MS }, + ); + + // VERIFCERTERR -> WGET_EXIT_SSL_AUTH_FAIL == 5, the native taxonomy. + expect(result.exitCode).toBe(5); + expect(result.stderr).toMatch(/cannot verify|certificate|not trusted/i); + }, 15_000); + + itIf(hasOpenssl, "--no-check-certificate accepts a self-signed server", async () => { + const filesystem = await mountKernel(); + + const result = await kernel.exec( + `wget --no-check-certificate -O /insecure.txt https://127.0.0.1:${selfSignedPort}/file`, + { timeout: WGET_EXEC_TIMEOUT_MS }, + ); + + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(await filesystem.readTextFile("/insecure.txt")).toBe( + "self-signed secure content", + ); + }, 15_000); + + itIf(hasOpenssl, "--ca-certificate trusts a server signed by that CA", async () => { + const filesystem = await mountKernel(); + + // caHttpsServer's CA is NOT in the seeded bundle, so this only passes if + // --ca-certificate is honored (real file read + chain build in-guest). + await kernel.writeFile("/tmp/cacert-only.pem", caOnlyPem); + const result = await kernel.exec( + `wget --ca-certificate=/tmp/cacert-only.pem -O /cacert.txt https://127.0.0.1:${caHttpsPort}/file`, + { timeout: WGET_EXEC_TIMEOUT_MS }, + ); + + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(await filesystem.readTextFile("/cacert.txt")).toBe( + "cacert https content", + ); + }, 15_000); + + itIf(hasOpenssl, "--ca-certificate with the wrong CA still fails verification", async () => { + await mountKernel(); + + // Point --ca-certificate at the seeded CA, which did NOT sign + // caHttpsServer's leaf. + await kernel.writeFile("/tmp/wrong-ca.pem", seededCaPem); + const result = await kernel.exec( + `wget --ca-certificate=/tmp/wrong-ca.pem -O /wrong.txt https://127.0.0.1:${caHttpsPort}/file`, + { timeout: WGET_EXEC_TIMEOUT_MS }, + ); + + expect(result.exitCode).toBe(5); + expect(result.stderr).toMatch(/cannot verify|certificate|not trusted/i); + }, 15_000); }); diff --git a/toolchain/c/Makefile b/toolchain/c/Makefile index ceccb616b3..7ea3d8899b 100644 --- a/toolchain/c/Makefile +++ b/toolchain/c/Makefile @@ -183,7 +183,11 @@ WGET_VERSION := 1.24.5 WGET_URL := https://ftp.gnu.org/gnu/wget/wget-$(WGET_VERSION).tar.gz WGET_UPSTREAM_BUILD_DIR := $(BUILD_DIR)/wget-upstream WGET_UPSTREAM_OVERLAY_INCLUDE_DIR := overlays/wget/include -WGET_UPSTREAM_OVERLAY_FILES := $(wildcard $(WGET_UPSTREAM_OVERLAY_INCLUDE_DIR)/*.h) +# mbedTLS backend overlay (mirrors the curl overlay layout): src/wasi_ssl.c +# is copied into the wget source tree at configure time. +WGET_UPSTREAM_OVERLAY_DIR := ../../software/wget/native/c/overlay +WGET_UPSTREAM_OVERLAY_FILES := $(wildcard $(WGET_UPSTREAM_OVERLAY_INCLUDE_DIR)/*.h) \ + $(shell find $(WGET_UPSTREAM_OVERLAY_DIR) -type f 2>/dev/null) GIT_VERSION := 2.55.0 GIT_URL := https://www.kernel.org/pub/software/scm/git/git-$(GIT_VERSION).tar.xz GIT_UPSTREAM_BUILD_DIR := $(BUILD_DIR)/git-upstream @@ -807,7 +811,7 @@ $(BUILD_DIR)/curl: scripts/build-curl-upstream.sh $(CURL_UPSTREAM_OVERLAY_FILES) $(CURL_LIBCURL_ARTIFACT): $(BUILD_DIR)/curl @test -f $@ || { echo "Error: libcurl artifact missing at $@ (curl build did not install it)"; exit 1; } -$(BUILD_DIR)/wget: scripts/build-wget-upstream.sh $(WGET_UPSTREAM_OVERLAY_FILES) $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a $(WASI_SDK_DIR)/bin/clang +$(BUILD_DIR)/wget: scripts/build-wget-upstream.sh $(WGET_UPSTREAM_OVERLAY_FILES) $(MBEDTLS_LIBS) $(ZLIB_BUILD_DIR)/libz.a $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a $(WASI_SDK_DIR)/bin/clang @mkdir -p $(BUILD_DIR) bash scripts/build-wget-upstream.sh \ --version "$(WGET_VERSION)" \ @@ -815,6 +819,11 @@ $(BUILD_DIR)/wget: scripts/build-wget-upstream.sh $(WGET_UPSTREAM_OVERLAY_FILES) --cache-dir "$(abspath $(LIBS_CACHE))" \ --build-dir "$(abspath $(WGET_UPSTREAM_BUILD_DIR))" \ --overlay-include-dir "$(abspath $(WGET_UPSTREAM_OVERLAY_INCLUDE_DIR))" \ + --overlay-dir "$(abspath $(WGET_UPSTREAM_OVERLAY_DIR))" \ + --mbedtls-include "$(abspath $(MBEDTLS_SRC_DIR)/include)" \ + --mbedtls-libdir "$(abspath $(MBEDTLS_BUILD_DIR))" \ + --zlib-include "$(abspath $(LIBS_CACHE)/zlib-$(ZLIB_VERSION))" \ + --zlib-libdir "$(abspath $(ZLIB_BUILD_DIR))" \ --cc "$(abspath $(CC)) --target=wasm32-wasip1 --sysroot=$(abspath $(SYSROOT))" \ --ar "$(abspath $(WASI_SDK_DIR)/bin/llvm-ar)" \ --ranlib "$(abspath $(WASI_SDK_DIR)/bin/llvm-ranlib)" \ diff --git a/toolchain/c/scripts/build-wget-upstream.sh b/toolchain/c/scripts/build-wget-upstream.sh index 312937df63..93fa629140 100644 --- a/toolchain/c/scripts/build-wget-upstream.sh +++ b/toolchain/c/scripts/build-wget-upstream.sh @@ -9,6 +9,11 @@ Usage: build-wget-upstream.sh \ --cache-dir \ --build-dir \ --overlay-include-dir \ + --overlay-dir \ + --mbedtls-include \ + --mbedtls-libdir \ + --zlib-include \ + --zlib-libdir \ --cc \ --ar \ --ranlib \ @@ -21,6 +26,11 @@ URL="" CACHE_DIR="" BUILD_DIR="" OVERLAY_INCLUDE_DIR="" +OVERLAY_DIR="" +MBEDTLS_INCLUDE="" +MBEDTLS_LIBDIR="" +ZLIB_INCLUDE="" +ZLIB_LIBDIR="" CC_CMD="" AR_CMD="" RANLIB_CMD="" @@ -28,42 +38,20 @@ OUTPUT="" while [[ $# -gt 0 ]]; do case "$1" in - --version) - VERSION="$2" - shift 2 - ;; - --url) - URL="$2" - shift 2 - ;; - --cache-dir) - CACHE_DIR="$2" - shift 2 - ;; - --build-dir) - BUILD_DIR="$2" - shift 2 - ;; - --overlay-include-dir) - OVERLAY_INCLUDE_DIR="$2" - shift 2 - ;; - --cc) - CC_CMD="$2" - shift 2 - ;; - --ar) - AR_CMD="$2" - shift 2 - ;; - --ranlib) - RANLIB_CMD="$2" - shift 2 - ;; - --output) - OUTPUT="$2" - shift 2 - ;; + --version) VERSION="$2"; shift 2 ;; + --url) URL="$2"; shift 2 ;; + --cache-dir) CACHE_DIR="$2"; shift 2 ;; + --build-dir) BUILD_DIR="$2"; shift 2 ;; + --overlay-include-dir) OVERLAY_INCLUDE_DIR="$2"; shift 2 ;; + --overlay-dir) OVERLAY_DIR="$2"; shift 2 ;; + --mbedtls-include) MBEDTLS_INCLUDE="$2"; shift 2 ;; + --mbedtls-libdir) MBEDTLS_LIBDIR="$2"; shift 2 ;; + --zlib-include) ZLIB_INCLUDE="$2"; shift 2 ;; + --zlib-libdir) ZLIB_LIBDIR="$2"; shift 2 ;; + --cc) CC_CMD="$2"; shift 2 ;; + --ar) AR_CMD="$2"; shift 2 ;; + --ranlib) RANLIB_CMD="$2"; shift 2 ;; + --output) OUTPUT="$2"; shift 2 ;; *) echo "Unknown argument: $1" >&2 usage >&2 @@ -72,7 +60,7 @@ while [[ $# -gt 0 ]]; do esac done -if [[ -z "$VERSION" || -z "$URL" || -z "$CACHE_DIR" || -z "$BUILD_DIR" || -z "$OVERLAY_INCLUDE_DIR" || -z "$CC_CMD" || -z "$AR_CMD" || -z "$RANLIB_CMD" || -z "$OUTPUT" ]]; then +if [[ -z "$VERSION" || -z "$URL" || -z "$CACHE_DIR" || -z "$BUILD_DIR" || -z "$OVERLAY_INCLUDE_DIR" || -z "$OVERLAY_DIR" || -z "$MBEDTLS_INCLUDE" || -z "$MBEDTLS_LIBDIR" || -z "$ZLIB_INCLUDE" || -z "$ZLIB_LIBDIR" || -z "$CC_CMD" || -z "$AR_CMD" || -z "$RANLIB_CMD" || -z "$OUTPUT" ]]; then usage >&2 exit 1 fi @@ -111,7 +99,15 @@ fi pushd "$SRC_DIR" >/dev/null -echo "Configuring upstream GNU Wget for wasm32-wasip1..." +# GNU Wget has no upstream mbedTLS backend. We keep ./configure --without-ssl so +# it never probes GnuTLS/OpenSSL, then light up its SSL abstraction manually: +# patch src/wget.h's HAVE_SSL derivation to trigger on HAVE_WASI_TLS, compile +# -DHAVE_WASI_TLS everywhere (so http.c/main.c/init.c enable the https scheme, +# --secure-protocol/--ca-certificate/--no-check-certificate and HSTS), overlay +# our mbedTLS backend (src/wasi_ssl.c), and append its object + the mbedTLS +# archives to the generated src/Makefile link. This mirrors the curl build's +# playbook. zlib is enabled for gzip Content-Encoding via ZLIB_CFLAGS/ZLIB_LIBS. +echo "Configuring upstream GNU Wget for wasm32-wasip1 (in-guest mbedTLS + zlib)..." CC="$CC_CMD" \ AR="$AR_CMD" \ RANLIB="$RANLIB_CMD" \ @@ -122,7 +118,9 @@ PCRE2_CFLAGS="" \ PCRE2_LIBS="" \ PCRE_CFLAGS="" \ PCRE_LIBS="" \ -CPPFLAGS="-I$OVERLAY_INCLUDE_DIR" \ +ZLIB_CFLAGS="-I$ZLIB_INCLUDE" \ +ZLIB_LIBS="-L$ZLIB_LIBDIR -lz" \ +CPPFLAGS="-I$OVERLAY_INCLUDE_DIR -I$MBEDTLS_INCLUDE" \ gl_cv_func_posix_spawn_works=yes \ gl_cv_func_posix_spawn_secure_exec=yes \ gl_cv_func_posix_spawnp_secure_exec=yes \ @@ -134,7 +132,7 @@ gl_cv_func_select_detects_ebadf=yes \ gl_cv_func_pselect_detects_ebadf=yes \ ac_cv_func_clock_getres=no \ ac_cv_func_clock_gettime=no \ -CFLAGS="-O2 -flto -D_WASI_EMULATED_SIGNAL -D_WASI_EMULATED_MMAN -D_WASI_EMULATED_PROCESS_CLOCKS -DFD_SETSIZE=8192" \ +CFLAGS="-O2 -flto -DHAVE_WASI_TLS -D_WASI_EMULATED_SIGNAL -D_WASI_EMULATED_MMAN -D_WASI_EMULATED_PROCESS_CLOCKS -DFD_SETSIZE=8192" \ LIBS="-lwasi-emulated-signal -lwasi-emulated-mman -lwasi-emulated-process-clocks" \ ./configure \ --host=wasm32-unknown-wasi \ @@ -148,10 +146,86 @@ LIBS="-lwasi-emulated-signal -lwasi-emulated-mman -lwasi-emulated-process-clocks --disable-pcre2 \ --without-ssl \ --without-libpsl \ - --without-zlib \ --without-libuuid \ --without-metalink +# Patch HAVE_SSL derivation: wget only defines HAVE_SSL when a probed backend +# (OpenSSL/GnuTLS) is present. Add our in-guest backend to the condition. +echo "Patching src/wget.h HAVE_SSL derivation for HAVE_WASI_TLS..." +python3 - <<'PY' +from pathlib import Path + +path = Path("src/wget.h") +text = path.read_text() +needle = "#if defined HAVE_LIBSSL || defined HAVE_LIBSSL32 || defined HAVE_LIBGNUTLS" +replacement = needle + " || defined HAVE_WASI_TLS" +if "defined HAVE_WASI_TLS" not in text: + if needle not in text: + raise SystemExit("Expected HAVE_SSL derivation not found in src/wget.h") + text = text.replace(needle, replacement, 1) + path.write_text(text) + print(" patched src/wget.h") +else: + print(" src/wget.h already patched") + +# Report the real backend in `wget --version` (otherwise it prints "-ssl", +# since neither HAVE_LIBSSL nor HAVE_LIBGNUTLS is defined). +bi = Path("src/build_info.c") +bitext = bi.read_text() +ssl_needle = ('#if defined HAVE_LIBSSL || defined HAVE_LIBSSL32\n' + ' "+ssl/openssl",\n' + '#elif defined HAVE_LIBGNUTLS\n' + ' "+ssl/gnutls",\n' + '#else\n' + ' "-ssl",\n' + '#endif') +ssl_repl = ('#if defined HAVE_LIBSSL || defined HAVE_LIBSSL32\n' + ' "+ssl/openssl",\n' + '#elif defined HAVE_LIBGNUTLS\n' + ' "+ssl/gnutls",\n' + '#elif defined HAVE_WASI_TLS\n' + ' "+ssl/mbedtls",\n' + '#else\n' + ' "-ssl",\n' + '#endif') +if '"+ssl/mbedtls"' not in bitext: + if ssl_needle not in bitext: + raise SystemExit("Expected ssl feature block not found in src/build_info.c") + bi.write_text(bitext.replace(ssl_needle, ssl_repl, 1)) + print(" patched src/build_info.c") +else: + print(" src/build_info.c already patched") +PY + +# Overlay the mbedTLS backend (src/wasi_ssl.c) and any other overlay files. +echo "Applying wget overlay from $OVERLAY_DIR ..." +while IFS= read -r -d '' file; do + rel="${file#"$OVERLAY_DIR"/}" + mkdir -p "$(dirname "$rel")" + cp "$file" "$rel" + echo " overlaid $rel" +done < <(find "$OVERLAY_DIR" -type f -print0) + +# Append the wasi_ssl.o object + mbedTLS archives to the generated src/Makefile. +# wget_OBJECTS/wget_LDADD are recursively-expanded, so += appends cleanly and +# the object is picked up by automake's built-in .c.o rule (which already +# carries our CPPFLAGS mbedTLS include and -DHAVE_WASI_TLS from CFLAGS). The +# archives are ordered tls -> x509 -> crypto for the static link. +echo "Appending wasi_ssl.o + mbedTLS link rules to src/Makefile..." +cat >> src/Makefile <