From ac90b867b64adda8e9cd98eee210caccf30957c6 Mon Sep 17 00:00:00 2001 From: Nathan Flurry Date: Wed, 8 Jul 2026 21:35:28 -0700 Subject: [PATCH] feat(curl): real in-guest TLS (mbedTLS) + gzip/brotli/zstd + CA bundle --- .../execution/assets/runners/wasm-runner.mjs | 7 +- .../curl/native/c/overlay/lib/curl_setup.h | 2 +- .../curl/native/c/overlay/lib/vtls/vtls.c | 6 - .../curl/native/c/overlay/lib/vtls/wasi_tls.c | 187 --------------- .../curl/native/c/overlay/lib/vtls/wasi_tls.h | 18 -- software/curl/test/curl.test.ts | 213 +++++++++++++++++- toolchain/c/Makefile | 95 +++++++- toolchain/c/scripts/build-curl-upstream.sh | 154 ++++++------- 8 files changed, 380 insertions(+), 302 deletions(-) delete mode 100644 software/curl/native/c/overlay/lib/vtls/wasi_tls.c delete mode 100644 software/curl/native/c/overlay/lib/vtls/wasi_tls.h diff --git a/crates/execution/assets/runners/wasm-runner.mjs b/crates/execution/assets/runners/wasm-runner.mjs index f385336bd8..d3240a42c0 100644 --- a/crates/execution/assets/runners/wasm-runner.mjs +++ b/crates/execution/assets/runners/wasm-runner.mjs @@ -1414,7 +1414,12 @@ function seekGuestFileHandle(handle, offset, whence) { } else if (numericWhence === WASI_WHENCE_CUR) { base = BigInt(handle.position ?? 0); } else if (numericWhence === WASI_WHENCE_END) { - base = BigInt(Number(fsModule.fstatSync(handle.targetFd).size ?? 0)); + // Passthrough (read-only delegate) handles keep the real host fd in ioFd; + // targetFd is only a synthetic guest fd number and fstat'ing it reports + // size 0. Prefer ioFd so SEEK_END returns the true file size (e.g. mbedTLS + // sizing a CA bundle via fseek(SEEK_END)+ftell before reading it). + const sizeFd = typeof handle.ioFd === 'number' ? handle.ioFd : handle.targetFd; + base = BigInt(Number(fsModule.fstatSync(sizeFd).size ?? 0)); } else { return null; } diff --git a/software/curl/native/c/overlay/lib/curl_setup.h b/software/curl/native/c/overlay/lib/curl_setup.h index 8edcf809eb..50b4479f23 100644 --- a/software/curl/native/c/overlay/lib/curl_setup.h +++ b/software/curl/native/c/overlay/lib/curl_setup.h @@ -742,7 +742,7 @@ #if defined(USE_GNUTLS) || defined(USE_OPENSSL) || defined(USE_MBEDTLS) || \ defined(USE_WOLFSSL) || defined(USE_SCHANNEL) || defined(USE_SECTRANSP) || \ - defined(USE_BEARSSL) || defined(USE_RUSTLS) || defined(USE_WASI_TLS) + defined(USE_BEARSSL) || defined(USE_RUSTLS) #define USE_SSL /* SSL support has been enabled */ #endif diff --git a/software/curl/native/c/overlay/lib/vtls/vtls.c b/software/curl/native/c/overlay/lib/vtls/vtls.c index 21b758f2e8..61b407ab22 100644 --- a/software/curl/native/c/overlay/lib/vtls/vtls.c +++ b/software/curl/native/c/overlay/lib/vtls/vtls.c @@ -64,7 +64,6 @@ #include "mbedtls.h" /* mbedTLS versions */ #include "bearssl.h" /* BearSSL versions */ #include "rustls.h" /* Rustls versions */ -#include "wasi_tls.h" /* WASI host TLS */ #include "slist.h" #include "sendf.h" @@ -1380,8 +1379,6 @@ static const struct Curl_ssl Curl_ssl_multi = { const struct Curl_ssl *Curl_ssl = #if defined(CURL_WITH_MULTI_SSL) &Curl_ssl_multi; -#elif defined(USE_WASI_TLS) - &Curl_ssl_wasi_tls; #elif defined(USE_WOLFSSL) &Curl_ssl_wolfssl; #elif defined(USE_GNUTLS) @@ -1426,9 +1423,6 @@ static const struct Curl_ssl *available_backends[] = { #endif #if defined(USE_RUSTLS) &Curl_ssl_rustls, -#endif -#if defined(USE_WASI_TLS) - &Curl_ssl_wasi_tls, #endif NULL }; diff --git a/software/curl/native/c/overlay/lib/vtls/wasi_tls.c b/software/curl/native/c/overlay/lib/vtls/wasi_tls.c deleted file mode 100644 index a6a59e1ee7..0000000000 --- a/software/curl/native/c/overlay/lib/vtls/wasi_tls.c +++ /dev/null @@ -1,187 +0,0 @@ -/* - * WASI TLS backend for libcurl - * - * Uses the host_net WASM import net_tls_connect() to upgrade a TCP - * socket to TLS. The host runtime (Node.js tls.connect()) handles all - * cryptographic operations transparently — after the upgrade, regular - * send()/recv() carry encrypted traffic. This backend therefore just - * calls the host import during connect and passes data through to the - * socket layer for send/recv. - */ - -#include "curl_setup.h" - -#ifdef USE_WASI_TLS - -#include - -#include "urldata.h" -#include "sendf.h" -#include "vtls.h" -#include "vtls_int.h" -#include "connect.h" -#include "select.h" -#include "curl_printf.h" - -/* The last #include files should be: */ -#include "curl_memory.h" -#include "memdebug.h" - -/* WASM import: upgrade a TCP socket FD to TLS. - * flags: 0 = verify peer certificate (default) - * 1 = skip peer certificate verification (-k / --insecure) - * Returns 0 on success, errno on failure. */ -__attribute__((import_module("host_net"), import_name("net_tls_connect"))) -extern int __wasi_net_tls_connect(int fd, const char *hostname_ptr, - int hostname_len, int flags); - -/* Per-connection backend data (minimal — host handles all TLS state) */ -struct wasi_tls_backend_data { - int dummy; /* struct must be non-empty */ -}; - -static size_t wasi_tls_version(char *buffer, size_t size) -{ - return msnprintf(buffer, size, "WASI-TLS/host"); -} - -static CURLcode wasi_tls_connect_blocking(struct Curl_cfilter *cf, - struct Curl_easy *data) -{ - struct ssl_connect_data *connssl = cf->ctx; - curl_socket_t sockfd = Curl_conn_cf_get_socket(cf, data); - const char *hostname = connssl->peer.hostname; - int flags = 0; - - /* Check if peer verification is disabled (curl -k / --insecure) */ - struct ssl_primary_config *conn_config = Curl_ssl_cf_get_primary_config(cf); - if(conn_config && !conn_config->verifypeer) { - flags = 1; - } - - int ret = __wasi_net_tls_connect((int)sockfd, hostname, - (int)strlen(hostname), flags); - if(ret != 0) { - failf(data, "WASI TLS: host TLS connect failed (errno %d)", ret); - return CURLE_SSL_CONNECT_ERROR; - } - - connssl->state = ssl_connection_complete; - return CURLE_OK; -} - -static CURLcode wasi_tls_connect_nonblocking(struct Curl_cfilter *cf, - struct Curl_easy *data, - bool *done) -{ - /* Our host TLS connect is synchronous — just call blocking version */ - CURLcode result = wasi_tls_connect_blocking(cf, data); - *done = (result == CURLE_OK); - return result; -} - -/* recv: pass through to the socket layer — host handles decryption */ -static ssize_t wasi_tls_recv(struct Curl_cfilter *cf, - struct Curl_easy *data, - char *buf, size_t len, CURLcode *err) -{ - return Curl_conn_cf_recv(cf->next, data, buf, len, err); -} - -/* send: pass through to the socket layer — host handles encryption */ -static ssize_t wasi_tls_send(struct Curl_cfilter *cf, - struct Curl_easy *data, - const void *buf, size_t len, CURLcode *err) -{ - return Curl_conn_cf_send(cf->next, data, buf, len, FALSE, err); -} - -static CURLcode wasi_tls_shutdown(struct Curl_cfilter *cf, - struct Curl_easy *data, - bool send_shutdown, bool *done) -{ - (void)cf; - (void)data; - (void)send_shutdown; - *done = TRUE; - return CURLE_OK; -} - -static bool wasi_tls_data_pending(struct Curl_cfilter *cf, - const struct Curl_easy *data) -{ - (void)cf; - (void)data; - return FALSE; -} - -/* data may be NULL */ -static CURLcode wasi_tls_random(struct Curl_easy *data, - unsigned char *entropy, size_t length) -{ - size_t offset = 0; - (void)data; - - while(offset < length) { - size_t chunk = length - offset; - if(chunk > 256) - chunk = 256; - - if(getentropy(entropy + offset, chunk) != 0) - return CURLE_FAILED_INIT; - - offset += chunk; - } - - return CURLE_OK; -} - -static void wasi_tls_close(struct Curl_cfilter *cf, struct Curl_easy *data) -{ - (void)cf; - (void)data; - /* Host-side TLS is cleaned up when the socket is closed */ -} - -static void *wasi_tls_get_internals(struct ssl_connect_data *connssl, - CURLINFO info) -{ - (void)connssl; - (void)info; - return NULL; -} - -const struct Curl_ssl Curl_ssl_wasi_tls = { - { CURLSSLBACKEND_NONE, "wasi-tls" }, /* info */ - - 0, /* supports — no special features */ - - sizeof(struct wasi_tls_backend_data), - - Curl_none_init, /* init */ - Curl_none_cleanup, /* cleanup */ - wasi_tls_version, /* version */ - Curl_none_check_cxn, /* check_cxn */ - wasi_tls_shutdown, /* shutdown */ - wasi_tls_data_pending, /* data_pending */ - wasi_tls_random, /* random */ - Curl_none_cert_status_request, /* cert_status_request */ - wasi_tls_connect_blocking, /* connect */ - wasi_tls_connect_nonblocking, /* connect_nonblocking */ - Curl_ssl_adjust_pollset, /* adjust_pollset */ - wasi_tls_get_internals, /* get_internals */ - wasi_tls_close, /* close */ - Curl_none_close_all, /* close_all */ - Curl_none_set_engine, /* set_engine */ - Curl_none_set_engine_default, /* set_engine_default */ - Curl_none_engines_list, /* engines_list */ - Curl_none_false_start, /* false_start */ - NULL, /* sha256sum */ - NULL, /* associate_connection */ - NULL, /* disassociate_connection */ - wasi_tls_recv, /* recv decrypted data */ - wasi_tls_send, /* send data to encrypt */ - NULL, /* get_channel_binding */ -}; - -#endif /* USE_WASI_TLS */ diff --git a/software/curl/native/c/overlay/lib/vtls/wasi_tls.h b/software/curl/native/c/overlay/lib/vtls/wasi_tls.h deleted file mode 100644 index d75fdf1d93..0000000000 --- a/software/curl/native/c/overlay/lib/vtls/wasi_tls.h +++ /dev/null @@ -1,18 +0,0 @@ -#ifndef HEADER_CURL_WASI_TLS_H -#define HEADER_CURL_WASI_TLS_H -/* - * WASI TLS backend for libcurl - * - * Delegates TLS to the host runtime via the host_net WASM import - * net_tls_connect(). After the host upgrades the socket, regular - * send()/recv() transparently carry encrypted traffic, so the - * backend's recv_plain/send_plain are simple pass-throughs. - */ - -#include "curl_setup.h" - -#ifdef USE_WASI_TLS -extern const struct Curl_ssl Curl_ssl_wasi_tls; -#endif - -#endif /* HEADER_CURL_WASI_TLS_H */ diff --git a/software/curl/test/curl.test.ts b/software/curl/test/curl.test.ts index 1097b12022..d79395baf1 100644 --- a/software/curl/test/curl.test.ts +++ b/software/curl/test/curl.test.ts @@ -38,9 +38,17 @@ import { type Server as TcpServer, } from 'node:net'; import { execSync } from 'node:child_process'; -import { existsSync, unlinkSync, writeFileSync } from 'node:fs'; +import { + existsSync, + mkdtempSync, + readFileSync, + rmSync, + unlinkSync, + writeFileSync, +} from 'node:fs'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; +import { brotliCompressSync, gzipSync, zstdCompressSync } from 'node:zlib'; // The upstream curl parity assertions below only hold for the C-built curl // artifact; the Rust fallback in COMMANDS_DIR intentionally supports a smaller @@ -150,6 +158,44 @@ const externalNetworkSkipReason = runExternalNetwork })() : 'set AGENTOS_E2E_NETWORK=1 to enable external-network coverage'; +// A known, highly compressible payload used by the --compressed round-trip +// tests. Long + repetitive so every codec produces a body distinct from the +// plaintext (proving curl actually inflated it, not just passed it through). +const COMPRESSION_PAYLOAD = + 'agentos-compression-parity ' + 'the quick brown fox jumps over the lazy dog. '.repeat(64); + +// Build a real CA and a leaf server certificate signed by it, with a SAN that +// covers the 127.0.0.1 loopback endpoint the tests connect to. This lets curl's +// mbedTLS backend perform genuine chain + hostname verification, exactly like +// Linux curl against a private CA. +function makeCaSignedCert(caCommonName: string): { + caPem: string; + serverKey: string; + serverCert: string; +} { + const dir = mkdtempSync(join(tmpdir(), 'curl-ca-')); + try { + execSync(`openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "${dir}/ca.key" 2>/dev/null`); + execSync( + `openssl req -x509 -new -key "${dir}/ca.key" -days 3650 -subj "/CN=${caCommonName}" -out "${dir}/ca.crt" 2>/dev/null`, + ); + execSync(`openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "${dir}/srv.key" 2>/dev/null`); + execSync(`openssl req -new -key "${dir}/srv.key" -subj "/CN=localhost" -out "${dir}/srv.csr" 2>/dev/null`); + writeFileSync(`${dir}/ext.cnf`, 'subjectAltName=DNS:localhost,IP:127.0.0.1\n'); + execSync( + `openssl x509 -req -in "${dir}/srv.csr" -CA "${dir}/ca.crt" -CAkey "${dir}/ca.key" ` + + `-CAcreateserial -days 3650 -extfile "${dir}/ext.cnf" -out "${dir}/srv.crt" 2>/dev/null`, + ); + return { + caPem: readFileSync(`${dir}/ca.crt`, 'utf8'), + serverKey: readFileSync(`${dir}/srv.key`, 'utf8'), + serverCert: readFileSync(`${dir}/srv.crt`, 'utf8'), + }; + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + function generateSelfSignedCert(): { key: string; cert: string } { const keyPath = join(tmpdir(), `curl-test-key-${process.pid}-${Date.now()}.pem`); try { @@ -176,10 +222,19 @@ describeIf(hasCurl || hasHttpGetTest, 'curl and socket layer', () => { let kernel: Kernel; let httpServer: HttpServer; let httpsServer: HttpsServer; + let validHttpsServer: HttpsServer; + let caHttpsServer: HttpsServer; let keepAliveServer: TcpServer; let httpPort: number; let httpsPort: number; + let validHttpsPort: number; + let caHttpsPort: number; let keepAlivePort: number; + // CA (PEM) trusted by the seeded /etc/ssl/certs/ca-certificates.crt bundle — + // it signs validHttpsServer's leaf. caOnlyPem signs caHttpsServer's leaf and + // is deliberately NOT in the bundle, so it verifies only via --cacert. + let seededCaPem = ''; + let caOnlyPem = ''; let flakyRequestCount = 0; beforeAll(async () => { @@ -333,6 +388,29 @@ describeIf(hasCurl || hasHttpGetTest, 'curl and socket layer', () => { return; } + if (url === '/gzip' || url === '/brotli' || url === '/zstd') { + const raw = Buffer.from(COMPRESSION_PAYLOAD); + let encoding: string; + let body: Buffer; + if (url === '/gzip') { + encoding = 'gzip'; + body = gzipSync(raw); + } else if (url === '/brotli') { + encoding = 'br'; + body = brotliCompressSync(raw); + } else { + encoding = 'zstd'; + body = zstdCompressSync(raw); + } + res.writeHead(200, { + 'Content-Type': 'text/plain', + 'Content-Encoding': encoding, + 'Content-Length': String(body.length), + }); + res.end(body); + return; + } + res.writeHead(404, { 'Content-Type': 'text/plain' }); res.end('not found'); }); @@ -372,6 +450,37 @@ describeIf(hasCurl || hasHttpGetTest, 'curl and socket layer', () => { httpsServer.listen(0, '127.0.0.1', resolveListen); }); httpsPort = (httpsServer.address() as import('node:net').AddressInfo).port; + + // HTTPS server whose leaf chains to a CA seeded into the guest's + // /etc/ssl/certs/ca-certificates.crt — verified with NO -k / --cacert. + const trusted = makeCaSignedCert('AgentOS Test Root CA'); + seededCaPem = trusted.caPem; + validHttpsServer = createHttpsServer( + { key: trusted.serverKey, cert: trusted.serverCert }, + (req, res) => { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ verified: true, path: req.url })); + }, + ); + await new Promise((resolveListen) => { + validHttpsServer.listen(0, '127.0.0.1', resolveListen); + }); + validHttpsPort = (validHttpsServer.address() as import('node:net').AddressInfo).port; + + // HTTPS server whose CA is provided ONLY via --cacert (not in the bundle). + const caOnly = makeCaSignedCert('AgentOS Cacert-Only CA'); + caOnlyPem = caOnly.caPem; + caHttpsServer = createHttpsServer( + { key: caOnly.serverKey, cert: caOnly.serverCert }, + (req, res) => { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ cacert: true, path: req.url })); + }, + ); + await new Promise((resolveListen) => { + caHttpsServer.listen(0, '127.0.0.1', resolveListen); + }); + caHttpsPort = (caHttpsServer.address() as import('node:net').AddressInfo).port; } keepAliveServer = createTcpServer((socket) => { @@ -403,6 +512,12 @@ describeIf(hasCurl || hasHttpGetTest, 'curl and socket layer', () => { if (httpsServer) { await new Promise((resolveClose) => httpsServer.close(() => resolveClose())); } + if (validHttpsServer) { + await new Promise((resolveClose) => validHttpsServer.close(() => resolveClose())); + } + if (caHttpsServer) { + await new Promise((resolveClose) => caHttpsServer.close(() => resolveClose())); + } if (keepAliveServer) { await new Promise((resolveClose) => keepAliveServer.close(() => resolveClose())); } @@ -418,9 +533,23 @@ describeIf(hasCurl || hasHttpGetTest, 'curl and socket layer', () => { kernel = createKernel({ filesystem, permissions: allowAll, - loopbackExemptPorts: [httpPort, httpsPort, keepAlivePort], + loopbackExemptPorts: [ + httpPort, + httpsPort, + validHttpsPort, + caHttpsPort, + keepAlivePort, + ], }); await kernel.mount(createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] })); + + // Seed the Debian-shaped trust store the way the native VM bootstrap does, + // so curl's compile-time default CA bundle resolves in-guest. Only the + // "trusted" CA is placed here; the cacert-only CA is intentionally absent. + if (seededCaPem) { + await filesystem.mkdir('/etc/ssl/certs', { recursive: true }); + await kernel.writeFile('/etc/ssl/certs/ca-certificates.crt', seededCaPem); + } return kernel; } @@ -669,18 +798,67 @@ describeIf(hasCurl || hasHttpGetTest, 'curl and socket layer', () => { expect(Date.now() - startedAt).toBeLessThan(8000); }, 15000); - itIf(hasCurl && hasOpenssl, 'curl -k performs an HTTPS request through the WASI TLS backend', async () => { + itIf(hasCurl, 'curl --version reports the mbedTLS backend', async () => { + await createKernelWithNet(); + const result = await kernel.exec('curl --version'); + expect(result.exitCode).toBe(0); + // Real in-guest TLS: the SSL version line must name mbedTLS, and the + // retired host shim string must be gone. + expect(result.stdout).toMatch(/mbedTLS/i); + expect(result.stdout).not.toMatch(/WASI-TLS|wasi-tls/i); + expect(result.stdout).toMatch(/^Features:.*\bSSL\b/m); + expect(result.stdout).toMatch(/^Features:.*\bbrotli\b/m); + expect(result.stdout).toMatch(/^Features:.*\bzstd\b/m); + }, 15000); + + itIf(hasCurl && hasOpenssl, 'curl verifies a CA-signed cert against the seeded CA bundle', async () => { + await createKernelWithNet(); + // No -k, no --cacert: trust comes solely from the seeded + // /etc/ssl/certs/ca-certificates.crt, exactly like Debian curl. + const result = await kernel.exec(`curl -sS https://127.0.0.1:${validHttpsPort}/json`); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain('"verified":true'); + }, 15000); + + itIf(hasCurl && hasOpenssl, 'curl fails with exit 60 and a real verify message on an untrusted cert', async () => { + await createKernelWithNet(); + const result = await kernel.exec(`curl -sS https://127.0.0.1:${httpsPort}/json`); + // CURLE_PEER_FAILED_VERIFICATION == 60, the native Linux taxonomy. NOT 35 + // (SSL connect error, the old host-shim behavior). + expect(result.exitCode).toBe(60); + expect(result.stderr).toMatch(/certificate|verify|self[- ]?signed|CA/i); + expect(result.stderr).not.toMatch(/WASI TLS|wasi-tls/i); + expect(result.stdout).toBe(''); + }, 15000); + + itIf(hasCurl && hasOpenssl, 'curl -k skips verification and succeeds on an untrusted cert', async () => { await createKernelWithNet(); const result = await kernel.exec(`curl -ks https://127.0.0.1:${httpsPort}/json`); expect(result.exitCode).toBe(0); expect(result.stdout).toContain('"secure":true'); }, 15000); - itIf(hasCurl && hasOpenssl, 'curl fails TLS verification without -k on a self-signed endpoint', async () => { + itIf(hasCurl && hasOpenssl, 'curl --cacert accepts a server signed by that CA', async () => { await createKernelWithNet(); - const result = await kernel.exec(`curl -sS https://127.0.0.1:${httpsPort}/json`); - expect(result.exitCode).not.toBe(0); - expect(result.stderr).toMatch(/certificate|tls|ssl|verify/i); + // caHttpsServer's CA is NOT in the seeded bundle, so this only passes if + // --cacert is honored (real file read + chain build in-guest). + await kernel.writeFile('/tmp/cacert-only.pem', caOnlyPem); + const result = await kernel.exec( + `curl -sS --cacert /tmp/cacert-only.pem https://127.0.0.1:${caHttpsPort}/json`, + ); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain('"cacert":true'); + }, 15000); + + itIf(hasCurl && hasOpenssl, 'curl --cacert with the wrong CA still fails verification (exit 60)', async () => { + await createKernelWithNet(); + // Point --cacert at the seeded CA, which did NOT sign caHttpsServer's leaf. + await kernel.writeFile('/tmp/wrong-ca.pem', seededCaPem); + const result = await kernel.exec( + `curl -sS --cacert /tmp/wrong-ca.pem https://127.0.0.1:${caHttpsPort}/json`, + ); + expect(result.exitCode).toBe(60); + expect(result.stderr).toMatch(/certificate|verify|CA/i); }, 15000); itIf(hasCurl && hasOpenssl, 'curl -k exits promptly after an HTTPS keep-alive response', async () => { @@ -692,6 +870,27 @@ describeIf(hasCurl || hasHttpGetTest, 'curl and socket layer', () => { expect(Date.now() - startedAt).toBeLessThan(8000); }, 15000); + itIf(hasCurl, 'curl --compressed round-trips a gzip response body', async () => { + await createKernelWithNet(); + const result = await kernel.exec(`curl -s --compressed http://127.0.0.1:${httpPort}/gzip`); + expect(result.exitCode).toBe(0); + expect(result.stdout).toBe(COMPRESSION_PAYLOAD); + }, 15000); + + itIf(hasCurl, 'curl --compressed round-trips a brotli response body', async () => { + await createKernelWithNet(); + const result = await kernel.exec(`curl -s --compressed http://127.0.0.1:${httpPort}/brotli`); + expect(result.exitCode).toBe(0); + expect(result.stdout).toBe(COMPRESSION_PAYLOAD); + }, 15000); + + itIf(hasCurl, 'curl --compressed round-trips a zstd response body', async () => { + await createKernelWithNet(); + const result = await kernel.exec(`curl -s --compressed http://127.0.0.1:${httpPort}/zstd`); + expect(result.exitCode).toBe(0); + expect(result.stdout).toBe(COMPRESSION_PAYLOAD); + }, 15000); + itIf(hasHttpGetTest && !externalNetworkSkipReason, 'http_get_test reaches an external host over real TCP', async () => { await createKernelWithNet(); const result = await execWithRetry(`http_get_test ${EXTERNAL_HOST} ${EXTERNAL_TCP_PORT} /`); diff --git a/toolchain/c/Makefile b/toolchain/c/Makefile index cab2fde607..64dcfea4ad 100644 --- a/toolchain/c/Makefile +++ b/toolchain/c/Makefile @@ -608,6 +608,90 @@ $(MBEDTLS_LIBS): $(MBEDTLS_SRC_DIR)/library/ssl_tls.c \ @echo "=== mbedTLS libs built ===" @ls -l $(MBEDTLS_LIBS) +# --- brotli 1.1.0 (decoder) for wasm32-wasip1 --- +# curl's --compressed brotli decode. Only the dependency-free portable C +# `common` + `dec` sources are needed (no encoder). Mirrors the zlib rule: +# cross-compile each .c against the wasi-sdk sysroot and archive into the two +# canonical libs (libbrotlicommon.a, libbrotlidec.a; dec depends on common). +BROTLI_VERSION := 1.1.0 +BROTLI_URL := https://github.com/google/brotli/archive/refs/tags/v$(BROTLI_VERSION).tar.gz +BROTLI_SRC_DIR = $(LIBS_CACHE)/brotli-$(BROTLI_VERSION) +BROTLI_BUILD_DIR := $(BUILD_DIR)/brotli +BROTLI_INCLUDE_DIR = $(BROTLI_SRC_DIR)/c/include +BROTLI_COMMON_SRCS := constants.c context.c dictionary.c platform.c shared_dictionary.c transform.c +BROTLI_DEC_SRCS := bit_reader.c decode.c huffman.c state.c +BROTLI_LIBS := $(BROTLI_BUILD_DIR)/libbrotlidec.a $(BROTLI_BUILD_DIR)/libbrotlicommon.a + +$(BROTLI_SRC_DIR)/c/dec/decode.c: + @echo "Fetching brotli $(BROTLI_VERSION)..." + @mkdir -p $(LIBS_CACHE) + @curl -fSL "$(BROTLI_URL)" -o "$(LIBS_CACHE)/brotli-$(BROTLI_VERSION).tar.gz" + @rm -rf "$(BROTLI_SRC_DIR)" + @tar -xzf "$(LIBS_CACHE)/brotli-$(BROTLI_VERSION).tar.gz" -C "$(LIBS_CACHE)" + +.PHONY: brotli +brotli: $(BROTLI_LIBS) + +$(BROTLI_LIBS): $(BROTLI_SRC_DIR)/c/dec/decode.c $(WASI_SDK_DIR)/bin/clang + @echo "=== Building brotli $(BROTLI_VERSION) (dec) for wasm32-wasip1 ===" + @mkdir -p $(BROTLI_BUILD_DIR)/common $(BROTLI_BUILD_DIR)/dec + @for src in $(BROTLI_COMMON_SRCS); do \ + $(CC) --target=wasm32-wasip1 --sysroot=$(SYSROOT) -O2 \ + -I$(BROTLI_INCLUDE_DIR) \ + -c "$(BROTLI_SRC_DIR)/c/common/$$src" \ + -o "$(BROTLI_BUILD_DIR)/common/$${src%.c}.o" || exit 1; \ + done + @for src in $(BROTLI_DEC_SRCS); do \ + $(CC) --target=wasm32-wasip1 --sysroot=$(SYSROOT) -O2 \ + -I$(BROTLI_INCLUDE_DIR) \ + -c "$(BROTLI_SRC_DIR)/c/dec/$$src" \ + -o "$(BROTLI_BUILD_DIR)/dec/$${src%.c}.o" || exit 1; \ + done + @rm -f $(BROTLI_LIBS) + $(WASI_SDK_DIR)/bin/llvm-ar rcs $(BROTLI_BUILD_DIR)/libbrotlicommon.a $(BROTLI_BUILD_DIR)/common/*.o + $(WASI_SDK_DIR)/bin/llvm-ar rcs $(BROTLI_BUILD_DIR)/libbrotlidec.a $(BROTLI_BUILD_DIR)/dec/*.o + @echo "=== brotli libs built ===" + @ls -l $(BROTLI_LIBS) + +# --- zstd 1.5.6 (decoder) for wasm32-wasip1 --- +# curl's --compressed zstd decode. Single-threaded (ZSTD_MULTITHREAD off), +# ZSTD_DISABLE_ASM=1 so the portable C huffman path is used instead of the +# amd64 .S. Only lib/common + lib/decompress are needed for ZSTD_*DStream. +ZSTD_VERSION := 1.5.6 +ZSTD_URL := https://github.com/facebook/zstd/releases/download/v$(ZSTD_VERSION)/zstd-$(ZSTD_VERSION).tar.gz +ZSTD_SRC_DIR = $(LIBS_CACHE)/zstd-$(ZSTD_VERSION) +ZSTD_BUILD_DIR := $(BUILD_DIR)/zstd +ZSTD_INCLUDE_DIR = $(ZSTD_SRC_DIR)/lib +ZSTD_LIB := $(ZSTD_BUILD_DIR)/libzstd.a +# NB: leave ZSTD_MULTITHREAD *undefined* (not =0). zstd's threading.h keys the +# pthread path off `#if defined(ZSTD_MULTITHREAD)`, so even =0 pulls in +# pthread_create/join, which wasi-libc static-asserts away without threads. +ZSTD_CFLAGS := -O2 -DZSTD_DISABLE_ASM=1 \ + -I$(ZSTD_SRC_DIR)/lib -I$(ZSTD_SRC_DIR)/lib/common + +$(ZSTD_SRC_DIR)/lib/zstd.h: + @echo "Fetching zstd $(ZSTD_VERSION)..." + @mkdir -p $(LIBS_CACHE) + @curl -fSL "$(ZSTD_URL)" -o "$(LIBS_CACHE)/zstd-$(ZSTD_VERSION).tar.gz" + @rm -rf "$(ZSTD_SRC_DIR)" + @tar -xzf "$(LIBS_CACHE)/zstd-$(ZSTD_VERSION).tar.gz" -C "$(LIBS_CACHE)" + +.PHONY: zstd +zstd: $(ZSTD_LIB) + +$(ZSTD_LIB): $(ZSTD_SRC_DIR)/lib/zstd.h $(WASI_SDK_DIR)/bin/clang + @echo "=== Building zstd $(ZSTD_VERSION) (decompress) for wasm32-wasip1 ===" + @mkdir -p $(ZSTD_BUILD_DIR)/obj + @for src in $(ZSTD_SRC_DIR)/lib/common/*.c $(ZSTD_SRC_DIR)/lib/decompress/*.c; do \ + name=$$(basename "$${src%.c}"); \ + $(CC) --target=wasm32-wasip1 --sysroot=$(SYSROOT) $(ZSTD_CFLAGS) \ + -c "$$src" -o "$(ZSTD_BUILD_DIR)/obj/$$name.o" || exit 1; \ + done + @rm -f $(ZSTD_LIB) + $(WASI_SDK_DIR)/bin/llvm-ar rcs $(ZSTD_LIB) $(ZSTD_BUILD_DIR)/obj/*.o + @echo "=== zstd lib built ===" + @ls -l $(ZSTD_LIB) + # --- CA bundle (Mozilla / Debian ca-certificates) --- # Fetch the pinned Mozilla CA bundle into the sidecar asset staged by build.rs. # The blob is gitignored; run this once to enable real HTTPS verification. @@ -681,7 +765,7 @@ CURL_SRCS := $(wildcard libs/curl/lib/*.c) $(wildcard libs/curl/lib/vauth/*.c) \ CURL_INCLUDES := -Ilibs/curl/include -Ilibs/curl/lib -include libs/curl/lib/curl_setup.h -include libs/curl/lib/curl_printf.h CURL_LIB_DEFS := -DHAVE_CONFIG_H -DBUILDING_LIBCURL -D_WASI_EMULATED_SIGNAL -DHAVE_BASENAME -DHAVE_LIBGEN_H -$(BUILD_DIR)/curl: scripts/build-curl-upstream.sh $(CURL_UPSTREAM_OVERLAY_FILES) $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a $(WASI_SDK_DIR)/bin/clang +$(BUILD_DIR)/curl: scripts/build-curl-upstream.sh $(CURL_UPSTREAM_OVERLAY_FILES) $(MBEDTLS_LIBS) $(ZLIB_BUILD_DIR)/libz.a $(BROTLI_LIBS) $(ZSTD_LIB) $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a $(WASI_SDK_DIR)/bin/clang @mkdir -p $(BUILD_DIR) bash scripts/build-curl-upstream.sh \ --version "$(CURL_RELEASE_VERSION)" \ @@ -693,6 +777,15 @@ $(BUILD_DIR)/curl: scripts/build-curl-upstream.sh $(CURL_UPSTREAM_OVERLAY_FILES) --cc "$(abspath $(CC)) --target=wasm32-wasip1 --sysroot=$(abspath $(SYSROOT))" \ --ar "$(abspath $(WASI_SDK_DIR)/bin/llvm-ar)" \ --ranlib "$(abspath $(WASI_SDK_DIR)/bin/llvm-ranlib)" \ + --mbedtls-include "$(abspath $(MBEDTLS_SRC_DIR)/include)" \ + --mbedtls-libdir "$(abspath $(MBEDTLS_BUILD_DIR))" \ + --zlib-include "$(abspath $(LIBS_CACHE)/zlib-$(ZLIB_VERSION))" \ + --zlib-libdir "$(abspath $(ZLIB_BUILD_DIR))" \ + --brotli-include "$(abspath $(BROTLI_INCLUDE_DIR))" \ + --brotli-libdir "$(abspath $(BROTLI_BUILD_DIR))" \ + --zstd-include "$(abspath $(ZSTD_INCLUDE_DIR))" \ + --zstd-libdir "$(abspath $(ZSTD_BUILD_DIR))" \ + --ca-bundle "/etc/ssl/certs/ca-certificates.crt" \ --output "$(abspath $@)" $(BUILD_DIR)/wget: scripts/build-wget-upstream.sh $(WGET_UPSTREAM_OVERLAY_FILES) $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a $(WASI_SDK_DIR)/bin/clang diff --git a/toolchain/c/scripts/build-curl-upstream.sh b/toolchain/c/scripts/build-curl-upstream.sh index d2ffe82c2e..5545d85c96 100644 --- a/toolchain/c/scripts/build-curl-upstream.sh +++ b/toolchain/c/scripts/build-curl-upstream.sh @@ -13,6 +13,15 @@ Usage: build-curl-upstream.sh \ --cc \ --ar \ --ranlib \ + --mbedtls-include \ + --mbedtls-libdir \ + --zlib-include \ + --zlib-libdir \ + --brotli-include \ + --brotli-libdir \ + --zstd-include \ + --zstd-libdir \ + --ca-bundle \ --output EOF } @@ -26,50 +35,38 @@ OVERLAY_DIR="" CC_CMD="" AR_CMD="" RANLIB_CMD="" +MBEDTLS_INCLUDE="" +MBEDTLS_LIBDIR="" +ZLIB_INCLUDE="" +ZLIB_LIBDIR="" +BROTLI_INCLUDE="" +BROTLI_LIBDIR="" +ZSTD_INCLUDE="" +ZSTD_LIBDIR="" +CA_BUNDLE="" OUTPUT="" while [[ $# -gt 0 ]]; do case "$1" in - --version) - VERSION="$2" - shift 2 - ;; - --tag) - TAG="$2" - shift 2 - ;; - --url) - URL="$2" - shift 2 - ;; - --cache-dir) - CACHE_DIR="$2" - shift 2 - ;; - --build-dir) - BUILD_DIR="$2" - shift 2 - ;; - --overlay-dir) - OVERLAY_DIR="$2" - shift 2 - ;; - --cc) - CC_CMD="$2" - shift 2 - ;; - --ar) - AR_CMD="$2" - shift 2 - ;; - --ranlib) - RANLIB_CMD="$2" - shift 2 - ;; - --output) - OUTPUT="$2" - shift 2 - ;; + --version) VERSION="$2"; shift 2 ;; + --tag) TAG="$2"; shift 2 ;; + --url) URL="$2"; shift 2 ;; + --cache-dir) CACHE_DIR="$2"; shift 2 ;; + --build-dir) BUILD_DIR="$2"; shift 2 ;; + --overlay-dir) OVERLAY_DIR="$2"; shift 2 ;; + --cc) CC_CMD="$2"; shift 2 ;; + --ar) AR_CMD="$2"; shift 2 ;; + --ranlib) RANLIB_CMD="$2"; shift 2 ;; + --mbedtls-include) MBEDTLS_INCLUDE="$2"; shift 2 ;; + --mbedtls-libdir) MBEDTLS_LIBDIR="$2"; shift 2 ;; + --zlib-include) ZLIB_INCLUDE="$2"; shift 2 ;; + --zlib-libdir) ZLIB_LIBDIR="$2"; shift 2 ;; + --brotli-include) BROTLI_INCLUDE="$2"; shift 2 ;; + --brotli-libdir) BROTLI_LIBDIR="$2"; shift 2 ;; + --zstd-include) ZSTD_INCLUDE="$2"; shift 2 ;; + --zstd-libdir) ZSTD_LIBDIR="$2"; shift 2 ;; + --ca-bundle) CA_BUNDLE="$2"; shift 2 ;; + --output) OUTPUT="$2"; shift 2 ;; *) echo "Unknown argument: $1" >&2 usage >&2 @@ -78,7 +75,7 @@ while [[ $# -gt 0 ]]; do esac done -if [[ -z "$VERSION" || -z "$TAG" || -z "$URL" || -z "$CACHE_DIR" || -z "$BUILD_DIR" || -z "$OVERLAY_DIR" || -z "$CC_CMD" || -z "$AR_CMD" || -z "$RANLIB_CMD" || -z "$OUTPUT" ]]; then +if [[ -z "$VERSION" || -z "$TAG" || -z "$URL" || -z "$CACHE_DIR" || -z "$BUILD_DIR" || -z "$OVERLAY_DIR" || -z "$CC_CMD" || -z "$AR_CMD" || -z "$RANLIB_CMD" || -z "$MBEDTLS_INCLUDE" || -z "$MBEDTLS_LIBDIR" || -z "$ZLIB_INCLUDE" || -z "$ZLIB_LIBDIR" || -z "$BROTLI_INCLUDE" || -z "$BROTLI_LIBDIR" || -z "$ZSTD_INCLUDE" || -z "$ZSTD_LIBDIR" || -z "$CA_BUNDLE" || -z "$OUTPUT" ]]; then usage >&2 exit 1 fi @@ -122,6 +119,27 @@ while IFS= read -r -d '' file; do cp "$file" "$SRC_DIR/$rel" done < <(find "$OVERLAY_DIR" -type f -print0) +# Stage a single dependency prefix (include/ + lib/) so curl's configure can +# find mbedTLS/zlib/brotli/zstd with plain --with-=. PKG_CONFIG is +# disabled (no .pc files for our cross-built statics), so configure derives +# -I/include -L/lib and the correct -l flags from the prefix. +DEPS="$BUILD_DIR/deps" +rm -rf "$DEPS" +mkdir -p "$DEPS/include" "$DEPS/lib" + +# Headers: mbedtls/ + psa/, zlib.h/zconf.h, brotli/, zstd.h — merged into one tree. +cp -a "$MBEDTLS_INCLUDE/." "$DEPS/include/" +cp -a "$ZLIB_INCLUDE/zlib.h" "$ZLIB_INCLUDE/zconf.h" "$DEPS/include/" +cp -a "$BROTLI_INCLUDE/." "$DEPS/include/" +cp -a "$ZSTD_INCLUDE/zstd.h" "$DEPS/include/" + +# Static archives. +cp -a "$MBEDTLS_LIBDIR/libmbedtls.a" "$MBEDTLS_LIBDIR/libmbedx509.a" \ + "$MBEDTLS_LIBDIR/libmbedcrypto.a" "$DEPS/lib/" +cp -a "$ZLIB_LIBDIR/libz.a" "$DEPS/lib/" +cp -a "$BROTLI_LIBDIR/libbrotlidec.a" "$BROTLI_LIBDIR/libbrotlicommon.a" "$DEPS/lib/" +cp -a "$ZSTD_LIBDIR/libzstd.a" "$DEPS/lib/" + pushd "$SRC_DIR" >/dev/null echo "Patching WASI-incompatible signal/setjmp includes..." @@ -168,56 +186,30 @@ for rel_path, edits in replacements.items(): path.write_text(updated) PY -echo "Configuring upstream curl for wasm32-wasip1..." +echo "Configuring upstream curl for wasm32-wasip1 (in-guest mbedTLS + zlib/brotli/zstd)..." +# mbedTLS 3.x removed the legacy havege RNG, so curl's configure runs its +# "mbedtls_ssl_init in -lmbedtls" link probe — which needs the three archives +# in the correct static order plus brotlicommon (brotlidec depends on it). +# LIBS carries brotlicommon (configure only appends -lbrotlidec on its own). CC="$CC_CMD" \ AR="$AR_CMD" \ RANLIB="$RANLIB_CMD" \ +PKG_CONFIG="false" \ CFLAGS="-O2 -flto" \ +CPPFLAGS="-I$DEPS/include" \ +LDFLAGS="-L$DEPS/lib" \ +LIBS="-lbrotlicommon" \ ./configure \ --host=wasm32-unknown-wasi \ --disable-shared \ --disable-threaded-resolver \ --disable-ldap \ - --without-zlib \ - --without-brotli \ - --without-zstd \ --without-libpsl \ - --without-ca-bundle \ - --without-ca-path \ - --without-ssl - -echo "Enabling the secure-exec WASI TLS backend in generated curl config..." -python3 - <<'PY' -from pathlib import Path - -config = Path("lib/curl_config.h") -text = config.read_text() - -updates = { - "#define CURL_DISABLE_HSTS 1": "/* #undef CURL_DISABLE_HSTS */", -} -for old, new in updates.items(): - text = text.replace(old, new) - -if "#define USE_WASI_TLS 1" not in text: - text += "\n#define USE_WASI_TLS 1\n" -if "#define CURL_DISABLE_OPENSSL_AUTO_LOAD_CONFIG 1" not in text: - text += "#define CURL_DISABLE_OPENSSL_AUTO_LOAD_CONFIG 1\n" - -config.write_text(text) -PY - -echo "Patching generated lib/Makefile to compile wasi_tls.c..." -cat >> lib/Makefile <<'EOF' - -am_libcurl_la_OBJECTS += vtls/libcurl_la-wasi_tls.lo -libcurl_la_LIBADD += vtls/libcurl_la-wasi_tls.lo -libcurl.la: vtls/libcurl_la-wasi_tls.lo - -vtls/libcurl_la-wasi_tls.lo: vtls/$(am__dirstamp) vtls/$(DEPDIR)/$(am__dirstamp) vtls/wasi_tls.c - $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libcurl_la_CPPFLAGS) $(CPPFLAGS) $(libcurl_la_CFLAGS) $(CFLAGS) -MT vtls/libcurl_la-wasi_tls.lo -MD -MP -MF vtls/$(DEPDIR)/libcurl_la-wasi_tls.Tpo -c -o vtls/libcurl_la-wasi_tls.lo `test -f 'vtls/wasi_tls.c' || echo '$(srcdir)/'`vtls/wasi_tls.c - $(AM_V_at)$(am__mv) vtls/$(DEPDIR)/libcurl_la-wasi_tls.Tpo vtls/$(DEPDIR)/libcurl_la-wasi_tls.Plo -EOF + --with-mbedtls="$DEPS" \ + --with-zlib="$DEPS" \ + --with-brotli="$DEPS" \ + --with-zstd="$DEPS" \ + --with-ca-bundle="$CA_BUNDLE" echo "Building upstream libcurl..." make -C lib libcurl.la