diff --git a/packages/runtime-core/commands/git b/packages/runtime-core/commands/git index 869f2fae78..3a1fa31f8a 100644 Binary files a/packages/runtime-core/commands/git and b/packages/runtime-core/commands/git differ diff --git a/packages/runtime-core/commands/git-receive-pack b/packages/runtime-core/commands/git-receive-pack index 869f2fae78..3a1fa31f8a 100644 Binary files a/packages/runtime-core/commands/git-receive-pack and b/packages/runtime-core/commands/git-receive-pack differ diff --git a/packages/runtime-core/commands/git-remote-http b/packages/runtime-core/commands/git-remote-http index 869f2fae78..e8f895370d 100644 Binary files a/packages/runtime-core/commands/git-remote-http and b/packages/runtime-core/commands/git-remote-http differ diff --git a/packages/runtime-core/commands/git-remote-https b/packages/runtime-core/commands/git-remote-https index 869f2fae78..e8f895370d 100644 Binary files a/packages/runtime-core/commands/git-remote-https and b/packages/runtime-core/commands/git-remote-https differ diff --git a/packages/runtime-core/commands/git-upload-archive b/packages/runtime-core/commands/git-upload-archive index 869f2fae78..3a1fa31f8a 100644 Binary files a/packages/runtime-core/commands/git-upload-archive and b/packages/runtime-core/commands/git-upload-archive differ diff --git a/packages/runtime-core/commands/git-upload-pack b/packages/runtime-core/commands/git-upload-pack index 869f2fae78..3a1fa31f8a 100644 Binary files a/packages/runtime-core/commands/git-upload-pack and b/packages/runtime-core/commands/git-upload-pack differ diff --git a/software/git/agentos-package.json b/software/git/agentos-package.json index 31c38e66e4..b1c4f9c536 100644 --- a/software/git/agentos-package.json +++ b/software/git/agentos-package.json @@ -1,10 +1,10 @@ { "commands": [ - "git" + "git", + "git-remote-http" ], "aliases": { - "git-remote-http": "git", - "git-remote-https": "git", + "git-remote-https": "git-remote-http", "git-upload-pack": "git", "git-receive-pack": "git", "git-upload-archive": "git" diff --git a/software/git/test/git.test.ts b/software/git/test/git.test.ts index 2516b7dc14..f159618173 100644 --- a/software/git/test/git.test.ts +++ b/software/git/test/git.test.ts @@ -6,11 +6,11 @@ */ import { describe, it, expect, afterEach, beforeAll, afterAll, vi } from 'vitest'; -import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { resolve, join } from 'node:path'; import { tmpdir } from 'node:os'; -import { createServer, type Server as HttpServer } from 'node:http'; -import { spawn, spawnSync } from 'node:child_process'; +import { createServer as createHttpsServer, type Server as HttpsServer } from 'node:https'; +import { spawn, spawnSync, execSync } from 'node:child_process'; import { createWasmVmRuntime } from '@agentos/test-harness'; import { allowAll, @@ -27,8 +27,11 @@ vi.setConfig({ testTimeout: 30_000 }); /** Check git binary exists in addition to base WASM binaries */ const hasGit = hasWasmBinaries && existsSync(resolve(COMMANDS_DIR, 'git')); const hasHostGit = spawnSync('git', ['--version'], { stdio: 'ignore' }).status === 0; -// Smart HTTP needs Git's libcurl-backed remote helpers; this WASM build is NO_CURL. -const hasGitHttpHelper = false; +// Smart HTTP needs Git's libcurl-backed remote helper. It is now a real second +// WASM binary (git-remote-http links the overlaid mbedTLS libcurl in-process); +// git-remote-https aliases to it. +const hasGitHttpHelper = + hasGit && existsSync(resolve(COMMANDS_DIR, 'git-remote-http')); const gitConfig = [ '-c safe.directory=*', @@ -53,7 +56,7 @@ async function createGitKernel() { return { kernel, vfs, dispose: () => kernel.dispose() }; } -async function createGitKernelWithNet(loopbackExemptPorts: number[]) { +async function createGitKernelWithNet(loopbackExemptPorts: number[], seededCaPem?: string) { const vfs = createInMemoryFileSystem(); await (vfs as any).chmod('/', 0o1777); await vfs.mkdir('/tmp', { recursive: true }); @@ -65,9 +68,48 @@ async function createGitKernelWithNet(loopbackExemptPorts: number[]) { syncFilesystemOnDispose: false, }); await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); + // Seed the Debian-shaped trust store the way the native VM bootstrap does, so + // libcurl's compile-time default CA bundle (/etc/ssl/certs/ca-certificates.crt) + // resolves in-guest for git-remote-http's mbedTLS backend. + if (seededCaPem) { + await vfs.mkdir('/etc/ssl/certs', { recursive: true }); + await kernel.writeFile('/etc/ssl/certs/ca-certificates.crt', seededCaPem); + } return { kernel, vfs, dispose: () => kernel.dispose() }; } +// Build a real CA and a leaf server certificate signed by it, with a SAN that +// covers the 127.0.0.1 loopback endpoint the VM connects to. This lets +// git-remote-http's mbedTLS backend perform genuine chain + hostname +// verification, exactly like Linux git against a private CA. +function makeCaSignedCert(caCommonName: string): { + caPem: string; + serverKey: string; + serverCert: string; +} { + const dir = mkdtempSync(join(tmpdir(), 'git-ca-')); + try { + execSync(`openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "${dir}/ca.key" 2>/dev/null`); + execSync( + `openssl req -x509 -new -key "${dir}/ca.key" -days 3650 -subj "/CN=${caCommonName}" -out "${dir}/ca.crt" 2>/dev/null`, + ); + execSync(`openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "${dir}/srv.key" 2>/dev/null`); + execSync(`openssl req -new -key "${dir}/srv.key" -subj "/CN=localhost" -out "${dir}/srv.csr" 2>/dev/null`); + writeFileSync(`${dir}/ext.cnf`, 'subjectAltName=DNS:localhost,IP:127.0.0.1\n'); + execSync( + `openssl x509 -req -in "${dir}/srv.csr" -CA "${dir}/ca.crt" -CAkey "${dir}/ca.key" ` + + `-CAcreateserial -days 3650 -extfile "${dir}/ext.cnf" -out "${dir}/srv.crt" 2>/dev/null`, + ); + return { + caPem: readFileSync(`${dir}/ca.crt`, 'utf8'), + serverKey: readFileSync(`${dir}/srv.key`, 'utf8'), + serverCert: readFileSync(`${dir}/srv.crt`, 'utf8'), + }; + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + function runHostGit(args: string[], cwd?: string) { const result = spawnSync('git', args, { cwd, @@ -395,64 +437,30 @@ describeIf(hasGit, 'git command', () => { expect(result.stderr).not.toContain('GitSubcommandUnsupported'); }); - it('clone rejects HTTPS remotes until Git is linked with libcurl remote helpers', async () => { - ({ kernel, dispose } = await createGitKernel()); - - const result = await kernel.exec( - git('clone https://private@example.com/owner/repo.git /tmp/clone'), - { env: { GIT_AUTH_TOKEN: 'test-token' } }, - ); - expect(result.exitCode).not.toBe(0); - expect(result.stderr).toMatch(/remote-https|remote helper|fatal|unable/i); - expect(result.stderr).not.toContain('GitSubcommandUnsupported'); - }); - - describeIf(hasHostGit && hasGitHttpHelper, 'remote clone over smart HTTP', () => { + // Real smart-HTTP over TLS: git-remote-http (libcurl + in-guest mbedTLS) + // clones/fetches/pushes against `git http-backend` behind a Node HTTPS + // endpoint. Certificate trust comes from a private CA seeded into the guest's + // /etc/ssl/certs bundle (the "trusted" server) — exactly the Debian trust + // path — while a second server signed by a CA absent from the bundle exercises + // verify-fail, http.sslVerify=false, GIT_SSL_NO_VERIFY, and http.sslCAInfo. + describeIf(hasHostGit && hasGitHttpHelper, 'smart-HTTP clone/fetch/push over TLS', () => { let repoRoot: string; - let httpServer: HttpServer; - let httpPort: number; - - beforeAll(async () => { - repoRoot = mkdtempSync(join(tmpdir(), 'agentos-git-http-')); - const worktree = join(repoRoot, 'worktree'); - const origin = join(repoRoot, 'origin.git'); - - runHostGit(['-c', 'init.defaultBranch=main', 'init', worktree]); - writeFileSync(join(worktree, 'README.md'), 'remote smart clone\n'); - runHostGit(['-C', worktree, 'add', 'README.md']); - runHostGit([ - '-C', worktree, - '-c', 'user.name=secure-exec', - '-c', 'user.email=agent@example.com', - 'commit', - '-m', - 'seed', - ]); - - runHostGit(['-C', worktree, 'checkout', '-b', 'feature/deep']); - writeFileSync(join(worktree, 'feature.txt'), 'remote branch payload\n'); - runHostGit(['-C', worktree, 'add', 'feature.txt']); - runHostGit([ - '-C', worktree, - '-c', 'user.name=secure-exec', - '-c', 'user.email=agent@example.com', - 'commit', - '-m', - 'feature branch', - ]); - - runHostGit(['-C', worktree, 'checkout', 'main']); - runHostGit(['clone', '--bare', worktree, origin]); - runHostGit(['-C', origin, 'repack', '-a', '-d', '-f', '--depth=50', '--window=50']); - - httpServer = createServer((req, res) => { - const url = new URL(req.url ?? '/', 'http://127.0.0.1'); + let trustedServer: HttpsServer; + let untrustedServer: HttpsServer; + let trustedPort: number; + let untrustedPort: number; + let trustedCaPem = ''; + let untrustedCaPem = ''; + + // A CGI bridge to `git http-backend`. receive-pack is enabled on the origin + // (below) so pushes are accepted; GIT_HTTP_EXPORT_ALL allows anonymous read. + function makeBackendHandler() { + return (req: import('node:http').IncomingMessage, res: import('node:http').ServerResponse) => { + const url = new URL(req.url ?? '/', 'https://127.0.0.1'); const bodyChunks: Buffer[] = []; - req.on('data', (chunk) => { bodyChunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); }); - req.on('end', () => { const requestBody = Buffer.concat(bodyChunks); const gitProtocol = req.headers['git-protocol']; @@ -473,7 +481,6 @@ describeIf(hasGit, 'git command', () => { const child = spawn('git', ['http-backend'], { env }); const stdout: Buffer[] = []; const stderr: Buffer[] = []; - child.stdout.on('data', (chunk) => { stdout.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); }); @@ -490,24 +497,20 @@ describeIf(hasGit, 'git command', () => { const altSep = output.indexOf(Buffer.from('\n\n')); const sepIndex = headerSep >= 0 ? headerSep : altSep; const sepLen = headerSep >= 0 ? 4 : altSep >= 0 ? 2 : 0; - if (code !== 0 && sepIndex === -1) { res.writeHead(500, { 'Content-Type': 'text/plain' }); res.end(Buffer.concat(stderr)); return; } - if (sepIndex === -1) { res.writeHead(500, { 'Content-Type': 'text/plain' }); res.end(output); return; } - const headerText = output.subarray(0, sepIndex).toString('utf8'); const responseBody = output.subarray(sepIndex + sepLen); let status = 200; const headers: Record = {}; - for (const line of headerText.split(/\r?\n/)) { if (!line) continue; const colon = line.indexOf(':'); @@ -520,34 +523,84 @@ describeIf(hasGit, 'git command', () => { headers[name] = value; } } - res.writeHead(status, headers); res.end(responseBody); }); - child.stdin.end(requestBody); }); - }); + }; + } - await new Promise((resolveListen) => { - httpServer.listen(0, '127.0.0.1', resolveListen); - }); - httpPort = (httpServer.address() as import('node:net').AddressInfo).port; + async function listen(server: HttpsServer): Promise { + await new Promise((r) => server.listen(0, '127.0.0.1', r)); + return (server.address() as import('node:net').AddressInfo).port; + } + + beforeAll(async () => { + repoRoot = mkdtempSync(join(tmpdir(), 'agentos-git-https-')); + const worktree = join(repoRoot, 'worktree'); + const origin = join(repoRoot, 'origin.git'); + + runHostGit(['-c', 'init.defaultBranch=main', 'init', worktree]); + writeFileSync(join(worktree, 'README.md'), 'remote smart clone\n'); + runHostGit(['-C', worktree, 'add', 'README.md']); + runHostGit([ + '-C', worktree, + '-c', 'user.name=secure-exec', '-c', 'user.email=agent@example.com', + 'commit', '-m', 'seed', + ]); + runHostGit(['-C', worktree, 'checkout', '-b', 'feature/deep']); + writeFileSync(join(worktree, 'feature.txt'), 'remote branch payload\n'); + runHostGit(['-C', worktree, 'add', 'feature.txt']); + runHostGit([ + '-C', worktree, + '-c', 'user.name=secure-exec', '-c', 'user.email=agent@example.com', + 'commit', '-m', 'feature branch', + ]); + runHostGit(['-C', worktree, 'checkout', 'main']); + runHostGit(['clone', '--bare', worktree, origin]); + runHostGit(['-C', origin, 'repack', '-a', '-d', '-f', '--depth=50', '--window=50']); + // Accept anonymous pushes over smart HTTP. + runHostGit(['-C', origin, 'config', 'http.receivepack', 'true']); + + const trusted = makeCaSignedCert('AgentOS Git Test Root CA'); + trustedCaPem = trusted.caPem; + trustedServer = createHttpsServer( + { key: trusted.serverKey, cert: trusted.serverCert }, + makeBackendHandler(), + ); + trustedPort = await listen(trustedServer); + + const untrusted = makeCaSignedCert('AgentOS Git Untrusted CA'); + untrustedCaPem = untrusted.caPem; + untrustedServer = createHttpsServer( + { key: untrusted.serverKey, cert: untrusted.serverCert }, + makeBackendHandler(), + ); + untrustedPort = await listen(untrustedServer); }); afterAll(async () => { - await new Promise((resolveClose) => httpServer.close(() => resolveClose())); + if (trustedServer) await new Promise((r) => trustedServer.close(() => r())); + if (untrustedServer) await new Promise((r) => untrustedServer.close(() => r())); rmSync(repoRoot, { recursive: true, force: true }); }); - it('clone fetches refs and worktree contents from a smart HTTP remote', async () => { - ({ kernel, vfs, dispose } = await createGitKernelWithNet([httpPort])); + const trustedUrl = () => `https://127.0.0.1:${trustedPort}/origin.git`; + const untrustedUrl = () => `https://127.0.0.1:${untrustedPort}/origin.git`; + + it('clone fetches refs and worktree contents over HTTPS with a trusted CA', async () => { + ({ kernel, vfs, dispose } = await createGitKernelWithNet([trustedPort], trustedCaPem)); - await run(kernel, git(`clone http://127.0.0.1:${httpPort}/origin.git /tmp/clone`)); + const res = await kernel.exec(git(`clone ${trustedUrl()} /tmp/clone`), { + env: { GIT_CURL_VERBOSE: '1' }, + }); + expect(res.exitCode).toBe(0); + // Proof a real TLS handshake happened in-guest (mbedTLS via libcurl). + expect(res.stderr).toMatch(/SSL connection|TLS|SSL certificate|CAfile/i); const head = new TextDecoder().decode(await kernel.readFile('/tmp/clone/.git/HEAD')); expect(head.trim()).toBe('ref: refs/heads/main'); - const readme = new TextDecoder().decode(await kernel.readFile('/tmp/clone/README.md')); expect(readme).toBe('remote smart clone\n'); await expectGitRef(kernel, '/tmp/clone', 'refs/remotes/origin/feature/deep'); @@ -556,5 +609,111 @@ describeIf(hasGit, 'git command', () => { const feature = new TextDecoder().decode(await kernel.readFile('/tmp/clone/feature.txt')); expect(feature).toBe('remote branch payload\n'); }); + + it('fetch picks up a new remote branch over HTTPS', async () => { + ({ kernel, vfs, dispose } = await createGitKernelWithNet([trustedPort], trustedCaPem)); + await run(kernel, git(`clone ${trustedUrl()} /tmp/clone`)); + + // Add a new branch on the origin (host side), then fetch it in-guest. + const bareBranch = 'fetched-branch'; + runHostGit(['-C', join(repoRoot, 'origin.git'), 'branch', bareBranch, 'main']); + + await run(kernel, git('-C /tmp/clone fetch origin')); + await expectGitRef(kernel, '/tmp/clone', `refs/remotes/origin/${bareBranch}`); + }); + + it('push sends a small commit over HTTPS smart-HTTP', async () => { + ({ kernel, vfs, dispose } = await createGitKernelWithNet([trustedPort], trustedCaPem)); + await run(kernel, git(`clone ${trustedUrl()} /tmp/clone`)); + + await kernel.writeFile('/tmp/clone/pushed.txt', 'pushed over https\n'); + await run(kernel, git('-C /tmp/clone add pushed.txt')); + await run(kernel, git("-C /tmp/clone commit -m 'push small'")); + const pushed = await kernel.exec(git('-C /tmp/clone push origin HEAD:refs/heads/small-push')); + expect(pushed.exitCode).toBe(0); + + // Verify the ref really landed in the origin bare repo (host side). + const originRef = spawnSync( + 'git', + ['-C', join(repoRoot, 'origin.git'), 'rev-parse', '--verify', 'refs/heads/small-push'], + { encoding: 'utf8' }, + ); + expect(originRef.status).toBe(0); + expect(originRef.stdout.trim()).toMatch(/^[0-9a-f]{40,64}$/); + }); + + it('push streams a >1 MiB commit over HTTPS (chunked POST)', async () => { + ({ kernel, vfs, dispose } = await createGitKernelWithNet([trustedPort], trustedCaPem)); + await run(kernel, git(`clone ${trustedUrl()} /tmp/clone`)); + + // Incompressible >1 MiB payload so the pack exceeds http.postBuffer (1 MiB) + // and libcurl must use chunked Transfer-Encoding + Expect: 100-continue. + const { randomBytes } = await import('node:crypto'); + const big = randomBytes(2 * 1024 * 1024); + await kernel.writeFile('/tmp/clone/big.bin', big); + await run(kernel, git('-C /tmp/clone add big.bin')); + await run(kernel, git("-C /tmp/clone commit -m 'push large'")); + const pushed = await kernel.exec(git('-C /tmp/clone push origin HEAD:refs/heads/large-push')); + expect(pushed.exitCode).toBe(0); + + const originRef = spawnSync( + 'git', + ['-C', join(repoRoot, 'origin.git'), 'rev-parse', '--verify', 'refs/heads/large-push'], + { encoding: 'utf8' }, + ); + expect(originRef.status).toBe(0); + // Confirm the large object is actually present in the origin object store. + const cat = spawnSync( + 'git', + ['-C', join(repoRoot, 'origin.git'), 'cat-file', '-s', `${originRef.stdout.trim()}^{tree}`], + { encoding: 'utf8' }, + ); + expect(cat.status).toBe(0); + }); + + it('clone fails with a real certificate-verification error on an untrusted CA', async () => { + // Only the trusted CA is seeded; the untrusted server's CA is absent. + ({ kernel, vfs, dispose } = await createGitKernelWithNet([untrustedPort], trustedCaPem)); + + const res = await kernel.exec(git(`clone ${untrustedUrl()} /tmp/clone`)); + expect(res.exitCode).not.toBe(0); + expect(res.stderr).toMatch(/certificate|SSL|TLS|verify|CAfile|unable to (access|get local)/i); + expect(res.stderr).not.toContain('GitSubcommandUnsupported'); + expect(await vfs.exists('/tmp/clone/.git')).toBe(false); + }); + + it('http.sslVerify=false bypasses verification for an untrusted CA', async () => { + ({ kernel, vfs, dispose } = await createGitKernelWithNet([untrustedPort], trustedCaPem)); + + const res = await kernel.exec(git(`-c http.sslVerify=false clone ${untrustedUrl()} /tmp/clone`)); + expect(res.exitCode).toBe(0); + const readme = new TextDecoder().decode(await kernel.readFile('/tmp/clone/README.md')); + expect(readme).toBe('remote smart clone\n'); + }); + + it('GIT_SSL_NO_VERIFY bypasses verification for an untrusted CA', async () => { + ({ kernel, vfs, dispose } = await createGitKernelWithNet([untrustedPort], trustedCaPem)); + + const res = await kernel.exec(git(`clone ${untrustedUrl()} /tmp/clone`), { + env: { GIT_SSL_NO_VERIFY: '1' }, + }); + expect(res.exitCode).toBe(0); + expect(await vfs.exists('/tmp/clone/.git/HEAD')).toBe(true); + }); + + it('http.sslCAInfo trusts an explicitly supplied CA bundle', async () => { + // Seed only the trusted CA in the default bundle; supply the untrusted + // server's CA via a VFS file referenced with http.sslCAInfo. + ({ kernel, vfs, dispose } = await createGitKernelWithNet([untrustedPort], trustedCaPem)); + await vfs.mkdir('/tmp/ca', { recursive: true }); + await kernel.writeFile('/tmp/ca/untrusted.pem', untrustedCaPem); + + const res = await kernel.exec( + git(`-c http.sslCAInfo=/tmp/ca/untrusted.pem clone ${untrustedUrl()} /tmp/clone`), + ); + expect(res.exitCode).toBe(0); + const readme = new TextDecoder().decode(await kernel.readFile('/tmp/clone/README.md')); + expect(readme).toBe('remote smart clone\n'); + }); }); }); diff --git a/toolchain/c/Makefile b/toolchain/c/Makefile index 64dcfea4ad..ceccb616b3 100644 --- a/toolchain/c/Makefile +++ b/toolchain/c/Makefile @@ -68,7 +68,7 @@ COMMANDS_DIR ?= ../target/wasm32-wasip1/release/commands # Fast real commands installed by the default software gate. Slow/heavy commands # (duckdb, vim) remain available through explicit parent `make cmd/`. -COMMANDS := zip unzip envsubst sqlite3 curl wget grep git tree +COMMANDS := zip unzip envsubst sqlite3 curl wget grep git git-remote-http tree # Programs requiring patched sysroot (Tier 2+ custom host imports) PATCHED_PROGRAMS := http_get_test isatty_test getpid_test getppid_test getppid_verify userinfo pipe_test dup_test spawn_child spawn_exit_code pipeline kill_child waitpid_return waitpid_edge syscall_coverage getpwuid_test signal_tests sigaction_self sigaction_behavior delayed_tcp_echo delayed_kill pipe_edge tcp_accept_spawn tcp_echo tcp_server http_server udp_echo unix_socket signal_handler dns_lookup sqlite3 curl wget grep tree zip unzip fs_probe @@ -148,6 +148,10 @@ CURL_RELEASE_VERSION := 8.11.1 CURL_RELEASE_TAG := 8_11_1 CURL_RELEASE_URL := https://github.com/curl/curl/releases/download/curl-$(CURL_RELEASE_TAG)/curl-$(CURL_RELEASE_VERSION).tar.xz CURL_UPSTREAM_BUILD_DIR := $(BUILD_DIR)/curl-upstream +# Reusable libcurl artifact (headers + libcurl.a) installed by the curl build so +# git's git-remote-http links the same overlaid, mbedTLS-linked libcurl. +CURL_LIBCURL_PREFIX := $(CURL_UPSTREAM_BUILD_DIR)/install +CURL_LIBCURL_ARTIFACT := $(CURL_LIBCURL_PREFIX)/lib/libcurl.a CURL_UPSTREAM_OVERLAY_DIR := ../../software/curl/native/c/overlay CURL_UPSTREAM_OVERLAY_FILES := $(wildcard $(CURL_UPSTREAM_OVERLAY_DIR)/lib/*.c $(CURL_UPSTREAM_OVERLAY_DIR)/lib/*.h $(CURL_UPSTREAM_OVERLAY_DIR)/lib/vtls/*.c $(CURL_UPSTREAM_OVERLAY_DIR)/lib/vtls/*.h) ZLIB_VERSION := 1.3.1 @@ -742,7 +746,7 @@ $(BUILD_DIR)/unzip: scripts/build-infozip-upstream.sh $(PATCHED_SYSROOT)/lib/was --cc "$(abspath $(CC)) --target=wasm32-wasip1 --sysroot=$(abspath $(SYSROOT))" \ --output "$(abspath $@)" -$(BUILD_DIR)/git: scripts/build-git-upstream.sh $(ZLIB_BUILD_DIR)/libz.a $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a $(WASI_SDK_DIR)/bin/clang +$(BUILD_DIR)/git: scripts/build-git-upstream.sh $(CURL_LIBCURL_ARTIFACT) $(MBEDTLS_LIBS) $(BROTLI_LIBS) $(ZSTD_LIB) $(ZLIB_BUILD_DIR)/libz.a $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a $(WASI_SDK_DIR)/bin/clang @mkdir -p $(BUILD_DIR) bash scripts/build-git-upstream.sh \ --version "$(GIT_VERSION)" \ @@ -752,12 +756,21 @@ $(BUILD_DIR)/git: scripts/build-git-upstream.sh $(ZLIB_BUILD_DIR)/libz.a $(PATCH --patch-dir "$(abspath $(GIT_PATCH_DIR))" \ --zlib-dir "$(abspath $(LIBS_CACHE)/zlib-$(ZLIB_VERSION))" \ --zlib-build-dir "$(abspath $(ZLIB_BUILD_DIR))" \ + --curl-prefix "$(abspath $(CURL_LIBCURL_PREFIX))" \ + --mbedtls-libdir "$(abspath $(MBEDTLS_BUILD_DIR))" \ + --brotli-libdir "$(abspath $(BROTLI_BUILD_DIR))" \ + --zstd-libdir "$(abspath $(ZSTD_BUILD_DIR))" \ --cc "$(abspath $(CC))" \ --ar "$(abspath $(WASI_SDK_DIR)/bin/llvm-ar)" \ --ranlib "$(abspath $(WASI_SDK_DIR)/bin/llvm-ranlib)" \ --sysroot "$(abspath $(SYSROOT))" \ + --remote-http-output "$(abspath $(BUILD_DIR)/git-remote-http)" \ --output "$(abspath $@)" +# git-remote-http is produced by the same git build invocation. +$(BUILD_DIR)/git-remote-http: $(BUILD_DIR)/git + @test -f $@ || { echo "Error: git-remote-http missing at $@ (git build did not produce it)"; exit 1; } + # curl_test: links libcurl (HTTP/HTTPS build for WASM via host_net + host_tls) CURL_SRCS := $(wildcard libs/curl/lib/*.c) $(wildcard libs/curl/lib/vauth/*.c) \ $(wildcard libs/curl/lib/vtls/*.c) $(wildcard libs/curl/lib/vquic/*.c) \ @@ -786,8 +799,14 @@ $(BUILD_DIR)/curl: scripts/build-curl-upstream.sh $(CURL_UPSTREAM_OVERLAY_FILES) --zstd-include "$(abspath $(ZSTD_INCLUDE_DIR))" \ --zstd-libdir "$(abspath $(ZSTD_BUILD_DIR))" \ --ca-bundle "/etc/ssl/certs/ca-certificates.crt" \ + --install-prefix "$(abspath $(CURL_LIBCURL_PREFIX))" \ --output "$(abspath $@)" +# The libcurl install artifact is produced as a side effect of the curl build. +# Declare it so git can depend on it; verify it exists after the curl target ran. +$(CURL_LIBCURL_ARTIFACT): $(BUILD_DIR)/curl + @test -f $@ || { echo "Error: libcurl artifact missing at $@ (curl build did not install it)"; exit 1; } + $(BUILD_DIR)/wget: scripts/build-wget-upstream.sh $(WGET_UPSTREAM_OVERLAY_FILES) $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a $(WASI_SDK_DIR)/bin/clang @mkdir -p $(BUILD_DIR) bash scripts/build-wget-upstream.sh \ @@ -863,12 +882,18 @@ install: fi; \ done; \ echo "Installed $$INSTALLED C command(s) to $(COMMANDS_DIR)" - @# Create symlinks for Git helper entry points used by upstream Git. + @# Git helper entry points. upload-pack/receive-pack/upload-archive ARE git + @# builtins, so they alias the git binary. The HTTPS transport lives in the + @# real git-remote-http binary (libcurl in-process); git-remote-https is an + @# alias to it (same binary, ftp/ftps schemes unsupported and unused). @if [ -f "$(COMMANDS_DIR)/git" ]; then \ - for alias in git-remote-http git-remote-https git-upload-pack git-receive-pack git-upload-archive; do \ + for alias in git-upload-pack git-receive-pack git-upload-archive; do \ ln -sf git "$(COMMANDS_DIR)/$$alias"; \ done; \ fi + @if [ -f "$(COMMANDS_DIR)/git-remote-http" ]; then \ + ln -sf git-remote-http "$(COMMANDS_DIR)/git-remote-https"; \ + fi # --- Clean --- diff --git a/toolchain/c/scripts/build-curl-upstream.sh b/toolchain/c/scripts/build-curl-upstream.sh index 5545d85c96..106393a959 100644 --- a/toolchain/c/scripts/build-curl-upstream.sh +++ b/toolchain/c/scripts/build-curl-upstream.sh @@ -22,7 +22,8 @@ Usage: build-curl-upstream.sh \ --zstd-include \ --zstd-libdir \ --ca-bundle \ - --output + --output \ + [--install-prefix ] EOF } @@ -45,9 +46,11 @@ ZSTD_INCLUDE="" ZSTD_LIBDIR="" CA_BUNDLE="" OUTPUT="" +INSTALL_PREFIX="" while [[ $# -gt 0 ]]; do case "$1" in + --install-prefix) INSTALL_PREFIX="$2"; shift 2 ;; --version) VERSION="$2"; shift 2 ;; --tag) TAG="$2"; shift 2 ;; --url) URL="$2"; shift 2 ;; @@ -217,6 +220,33 @@ make -C lib libcurl.la echo "Building upstream curl tool..." make -C src curl +# Reusable libcurl artifact: install the overlaid, mbedTLS-linked libcurl into a +# prefix (headers + libcurl.a) so git can link it in-process. Same overlay and +# configure as above, so git's git-remote-http gets the identical TLS backend. +if [[ -n "$INSTALL_PREFIX" ]]; then + echo "Installing libcurl to $INSTALL_PREFIX ..." + rm -rf "$INSTALL_PREFIX" + mkdir -p "$INSTALL_PREFIX/include/curl" "$INSTALL_PREFIX/lib/pkgconfig" + make -C lib install prefix="$INSTALL_PREFIX" >/dev/null 2>&1 || true + make -C include install prefix="$INSTALL_PREFIX" >/dev/null 2>&1 || true + # Guarantee headers + the static archive land regardless of libtool install + # quirks under a cross toolchain. + cp -a include/curl/*.h "$INSTALL_PREFIX/include/curl/" 2>/dev/null || true + if [[ -f lib/.libs/libcurl.a && ! -f "$INSTALL_PREFIX/lib/libcurl.a" ]]; then + cp -a lib/.libs/libcurl.a "$INSTALL_PREFIX/lib/libcurl.a" + fi + cp -a libcurl.pc "$INSTALL_PREFIX/lib/pkgconfig/" 2>/dev/null || true + if [[ ! -f "$INSTALL_PREFIX/lib/libcurl.a" ]]; then + echo "libcurl.a missing after install into $INSTALL_PREFIX" >&2 + exit 1 + fi + if [[ ! -f "$INSTALL_PREFIX/include/curl/curl.h" ]]; then + echo "curl/curl.h missing after install into $INSTALL_PREFIX" >&2 + exit 1 + fi + echo "Installed libcurl artifact ($(wc -c < "$INSTALL_PREFIX/lib/libcurl.a") bytes) at $INSTALL_PREFIX" +fi + BIN="" for candidate in "src/.libs/curl" "src/curl" "src/curl.wasm"; do if [[ -f "$candidate" ]]; then diff --git a/toolchain/c/scripts/build-git-upstream.sh b/toolchain/c/scripts/build-git-upstream.sh index f5268b6943..304e004647 100755 --- a/toolchain/c/scripts/build-git-upstream.sh +++ b/toolchain/c/scripts/build-git-upstream.sh @@ -11,11 +11,16 @@ Usage: build-git-upstream.sh \ --patch-dir \ --zlib-dir \ --zlib-build-dir \ + --curl-prefix \ + --mbedtls-libdir \ + --brotli-libdir \ + --zstd-libdir \ --cc \ --ar \ --ranlib \ --sysroot \ - --output + --output \ + --remote-http-output EOF } @@ -25,12 +30,17 @@ CACHE_DIR="" BUILD_DIR="" ZLIB_DIR="" ZLIB_BUILD_DIR="" +CURL_PREFIX="" +MBEDTLS_LIBDIR="" +BROTLI_LIBDIR="" +ZSTD_LIBDIR="" PATCH_DIR="" CC_CMD="" AR_CMD="" RANLIB_CMD="" SYSROOT="" OUTPUT="" +REMOTE_HTTP_OUTPUT="" while [[ $# -gt 0 ]]; do case "$1" in @@ -41,16 +51,21 @@ while [[ $# -gt 0 ]]; do --patch-dir) PATCH_DIR="$2"; shift 2 ;; --zlib-dir) ZLIB_DIR="$2"; shift 2 ;; --zlib-build-dir) ZLIB_BUILD_DIR="$2"; shift 2 ;; + --curl-prefix) CURL_PREFIX="$2"; shift 2 ;; + --mbedtls-libdir) MBEDTLS_LIBDIR="$2"; shift 2 ;; + --brotli-libdir) BROTLI_LIBDIR="$2"; shift 2 ;; + --zstd-libdir) ZSTD_LIBDIR="$2"; shift 2 ;; --cc) CC_CMD="$2"; shift 2 ;; --ar) AR_CMD="$2"; shift 2 ;; --ranlib) RANLIB_CMD="$2"; shift 2 ;; --sysroot) SYSROOT="$2"; shift 2 ;; --output) OUTPUT="$2"; shift 2 ;; + --remote-http-output) REMOTE_HTTP_OUTPUT="$2"; shift 2 ;; *) echo "Unknown argument: $1" >&2; usage >&2; exit 1 ;; esac done -if [[ -z "$VERSION" || -z "$URL" || -z "$CACHE_DIR" || -z "$BUILD_DIR" || -z "$PATCH_DIR" || -z "$ZLIB_DIR" || -z "$ZLIB_BUILD_DIR" || -z "$CC_CMD" || -z "$AR_CMD" || -z "$RANLIB_CMD" || -z "$SYSROOT" || -z "$OUTPUT" ]]; then +if [[ -z "$VERSION" || -z "$URL" || -z "$CACHE_DIR" || -z "$BUILD_DIR" || -z "$PATCH_DIR" || -z "$ZLIB_DIR" || -z "$ZLIB_BUILD_DIR" || -z "$CURL_PREFIX" || -z "$MBEDTLS_LIBDIR" || -z "$BROTLI_LIBDIR" || -z "$ZSTD_LIBDIR" || -z "$CC_CMD" || -z "$AR_CMD" || -z "$RANLIB_CMD" || -z "$SYSROOT" || -z "$OUTPUT" || -z "$REMOTE_HTTP_OUTPUT" ]]; then usage >&2 exit 1 fi @@ -97,7 +112,15 @@ if [[ -d "$PATCH_DIR" ]]; then done fi -echo "Building upstream Git ${VERSION} for wasm32-wasip1..." +# libcurl + its TLS/compression backends, linked in-process by git-remote-http. +# Defining CURL_CFLAGS/CURL_LDFLAGS makes Git's Makefile skip the curl-config +# probe entirely and use these flags verbatim. Static link order matters: +# libcurl first, then mbedTLS (tls -> x509 -> crypto), then zlib and the brotli / +# zstd decoders curl's --compressed path pulls in. +CURL_CFLAGS="-I$CURL_PREFIX/include" +CURL_LDFLAGS="-L$CURL_PREFIX/lib -lcurl -L$MBEDTLS_LIBDIR -lmbedtls -lmbedx509 -lmbedcrypto -L$ZLIB_BUILD_DIR -lz -L$BROTLI_LIBDIR -lbrotlidec -lbrotlicommon -L$ZSTD_LIBDIR -lzstd" + +echo "Building upstream Git ${VERSION} for wasm32-wasip1 (with in-process libcurl)..." make -j"${MAKE_JOBS:-2}" \ uname_S=WASI \ CC="$CC_CMD" \ @@ -106,6 +129,8 @@ make -j"${MAKE_JOBS:-2}" \ RANLIB="$RANLIB_CMD" \ CFLAGS="--target=wasm32-wasip1 --sysroot=$SYSROOT -I$ZLIB_DIR -O2 -D_WASI_EMULATED_PROCESS_CLOCKS -D_WASI_EMULATED_MMAN" \ LDFLAGS="--target=wasm32-wasip1 --sysroot=$SYSROOT -L$ZLIB_BUILD_DIR -lwasi-emulated-process-clocks -lwasi-emulated-mman" \ + CURL_CFLAGS="$CURL_CFLAGS" \ + CURL_LDFLAGS="$CURL_LDFLAGS" \ CSPRNG_METHOD=getentropy \ HAVE_PATHS_H=YesPlease \ HAVE_DEV_TTY=YesPlease \ @@ -114,7 +139,6 @@ make -j"${MAKE_JOBS:-2}" \ HAVE_GETDELIM=YesPlease \ NO_RUST=1 \ NO_OPENSSL=1 \ - NO_CURL=1 \ NO_EXPAT=1 \ NO_GETTEXT=1 \ NO_TCLTK=1 \ @@ -128,16 +152,29 @@ make -j"${MAKE_JOBS:-2}" \ NO_UNIX_SOCKETS=1 \ NO_SYS_POLL_H=1 \ NO_NSEC=1 \ - git - -mkdir -p "$(dirname "$OUTPUT")" -if command -v wasm-opt >/dev/null 2>&1; then - echo "Optimizing Git WASM binary..." - wasm-opt -O3 --strip-debug --all-features git -o "$OUTPUT" -else - cp git "$OUTPUT" + git git-remote-http + +if [[ ! -f git-remote-http ]]; then + echo "Expected git-remote-http binary was not produced" >&2 + exit 1 fi +emit() { + local src="$1" + local out="$2" + mkdir -p "$(dirname "$out")" + if command -v wasm-opt >/dev/null 2>&1; then + echo "Optimizing $src WASM binary -> $out..." + wasm-opt -O3 --strip-debug --all-features "$src" -o "$out" + else + cp "$src" "$out" + fi +} + +emit git "$OUTPUT" +emit git-remote-http "$REMOTE_HTTP_OUTPUT" + popd >/dev/null echo "Built upstream Git at $OUTPUT" +echo "Built git-remote-http at $REMOTE_HTTP_OUTPUT"