diff --git a/dstack/local-key-provider/build/Dockerfile.key-provider b/dstack/local-key-provider/build/Dockerfile.key-provider index 9cfca8424..5b6e25938 100644 --- a/dstack/local-key-provider/build/Dockerfile.key-provider +++ b/dstack/local-key-provider/build/Dockerfile.key-provider @@ -61,7 +61,16 @@ RUN make target/release/local-key-provider \ ! -name local-key-provider -exec rm -rf {} + \ && rm -rf /root/.cargo/registry /root/.cargo/git -RUN gramine-sgx-gen-private-key \ +# Do not use `gramine-sgx-gen-private-key` here. On some hosts the Python +# cryptography/OpenSSL stack in the Gramine image fails while generating the +# RSA key with a generic "Unknown OpenSSL error". Generate the exact same kind +# of Gramine signing key directly with OpenSSL instead (3072-bit RSA, public +# exponent 3, and traditional PEM accepted by gramine-sgx-sign). +RUN mkdir -p /root/.config/gramine \ + && chmod 0700 /root/.config /root/.config/gramine \ + && openssl genrsa -traditional -3 \ + -out /root/.config/gramine/enclave-key.pem 3072 \ + && chmod 0600 /root/.config/gramine/enclave-key.pem \ && make \ && rm -f /root/.config/gramine/enclave-key.pem RUN gramine-sgx-sigstruct-view --output-format json local-key-provider.sig diff --git a/dstack/verifier/src/verification.rs b/dstack/verifier/src/verification.rs index 2404ec163..6fd97e98a 100644 --- a/dstack/verifier/src/verification.rs +++ b/dstack/verifier/src/verification.rs @@ -421,6 +421,31 @@ impl CvmVerifier { .is_some_and(|digest| digest == expected)) } + fn prune_unlisted_image_files(extracted_dir: &Path, files_doc: &str) -> Result<()> { + let listed_files: Vec<&OsStr> = files_doc + .lines() + .flat_map(|line| line.split_whitespace().nth(1)) + .map(|s| s.as_ref()) + .collect(); + let files = fs_err::read_dir(extracted_dir).context("Failed to read directory")?; + for file in files { + let file = file.context("Failed to read directory entry")?; + let filename = file.file_name(); + // sha256sum.txt is the content-addressed OS identity and is needed + // again when a legacy TDX quote is verified from the cache. + if filename != OsStr::new("sha256sum.txt") + && !listed_files.contains(&filename.as_os_str()) + { + if file.path().is_dir() { + fs_err::remove_dir_all(file.path()).context("Failed to remove directory")?; + } else { + fs_err::remove_file(file.path()).context("Failed to remove file")?; + } + } + } + Ok(()) + } + fn tdx_acpi_hashes_from_event_log(event_log: &[TdxEvent]) -> Result { let rtmr0_events = event_log .iter() @@ -1180,23 +1205,7 @@ impl CvmVerifier { let sha256sum_path = extracted_dir.join("sha256sum.txt"); let files_doc = fs_err::read_to_string(&sha256sum_path).context("Failed to read sha256sum.txt")?; - let listed_files: Vec<&OsStr> = files_doc - .lines() - .flat_map(|line| line.split_whitespace().nth(1)) - .map(|s| s.as_ref()) - .collect(); - let files = fs_err::read_dir(&extracted_dir).context("Failed to read directory")?; - for file in files { - let file = file.context("Failed to read directory entry")?; - let filename = file.file_name(); - if !listed_files.contains(&filename.as_os_str()) { - if file.path().is_dir() { - fs_err::remove_dir_all(file.path()).context("Failed to remove directory")?; - } else { - fs_err::remove_file(file.path()).context("Failed to remove file")?; - } - } - } + Self::prune_unlisted_image_files(&extracted_dir, &files_doc)?; // All image modes are addressed by sha256(sha256sum.txt). Extra // measurement CBOR files are ordinary sha256sum.txt entries and do not @@ -1385,6 +1394,21 @@ mod tests { assert!(decode_key_provider_info(b"not json").is_none()); } + #[test] + fn image_cache_pruning_keeps_checksum_identity() { + let dir = tempfile::tempdir().expect("temp image directory"); + let files_doc = "00 metadata.json\n"; + fs_err::write(dir.path().join("sha256sum.txt"), files_doc).unwrap(); + fs_err::write(dir.path().join("metadata.json"), "{}").unwrap(); + fs_err::write(dir.path().join("unmeasured"), "remove me").unwrap(); + + CvmVerifier::prune_unlisted_image_files(dir.path(), files_doc).unwrap(); + + assert!(dir.path().join("sha256sum.txt").exists()); + assert!(dir.path().join("metadata.json").exists()); + assert!(!dir.path().join("unmeasured").exists()); + } + #[tokio::test] async fn verifies_sev_snp_attestation_fixture_without_image_download() { let request: VerificationRequest = diff --git a/dstack/vmm/src/tests/test_vmm_cli.py b/dstack/vmm/src/tests/test_vmm_cli.py new file mode 100644 index 000000000..f3c33c371 --- /dev/null +++ b/dstack/vmm/src/tests/test_vmm_cli.py @@ -0,0 +1,59 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 + +"""Focused regression tests for vmm-cli update request construction.""" + +from __future__ import annotations + +import contextlib +import importlib.util +import io +import sys +import unittest +from pathlib import Path + + +def load_vmm_cli(): + """Load the executable vmm-cli.py as a normal Python module.""" + path = Path(__file__).resolve().parents[1] / "vmm-cli.py" + spec = importlib.util.spec_from_file_location("dstack_vmm_cli", path) + if spec is None or spec.loader is None: + raise RuntimeError(f"cannot load {path}") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +VMM_CLI = load_vmm_cli() + + +class UpdateVmTests(unittest.TestCase): + """Verify update flags are not silently discarded by the CLI.""" + + def test_kms_urls_are_sent_to_upgrade_app(self) -> None: + """Send both the update flag and requested KMS URL list.""" + cli = VMM_CLI.VmmCLI("http://127.0.0.1:8080") + calls: list[tuple[str, dict]] = [] + cli.rpc_call = lambda method, params=None: calls.append((method, params)) or {} + + with contextlib.redirect_stdout(io.StringIO()): + cli.update_vm("vm-id", kms_urls=["https://kms.example:8000"]) + + self.assertEqual( + calls, + [ + ( + "UpgradeApp", + { + "id": "vm-id", + "update_kms_urls": True, + "kms_urls": ["https://kms.example:8000"], + }, + ) + ], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/dstack/vmm/src/vmm-cli.py b/dstack/vmm/src/vmm-cli.py index 290ee2440..e393e1323 100755 --- a/dstack/vmm/src/vmm-cli.py +++ b/dstack/vmm/src/vmm-cli.py @@ -1127,6 +1127,11 @@ def update_vm( upgrade_params["user_config"] = user_config updates.append("user config") + if kms_urls is not None: + upgrade_params["update_kms_urls"] = True + upgrade_params["kms_urls"] = kms_urls + updates.append(f"KMS URLs ({len(kms_urls)})") + # handle port updates - only update if --port or --no-ports is specified if no_ports or ports is not None: if no_ports: diff --git a/test-suites/full-stack-compose/.env.example b/test-suites/full-stack-compose/.env.example new file mode 100644 index 000000000..ad86f18ec --- /dev/null +++ b/test-suites/full-stack-compose/.env.example @@ -0,0 +1,63 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 + +# Directory containing unpacked dstack guest images: +# //{bzImage,initramfs...,rootfs...,digest.txt,sha256sum.txt} +DSTACK_E2E_IMAGE_STORE=../../../meta-dstack/build/images +DSTACK_E2E_IMAGE_NAME=dstack-0.6.0 +DSTACK_E2E_PLATFORM=tdx + +# Exact released upgrade sources. Both are digest-pinned on Docker Hub and +# checked by their in-image --version output before any CVM is launched. +DSTACK_E2E_OLD_KMS_IMAGE=dstacktee/dstack-kms:0.5.8@sha256:9650dcb47dad0065470f432f00e78e012912214ef1a5b1d7272918817e61a26d +DSTACK_E2E_OLD_GATEWAY_IMAGE=dstacktee/dstack-gateway:0.5.8@sha256:6eb1dc1a5000f37cc5b0322d3fdb71e7f2e31859b5e3a611634919278cee2411 +DSTACK_E2E_APP_IMAGE=nginx:alpine + +# Mock ACME/DNS + externally visible test domain for Gateway SNI. +DSTACK_E2E_BASE_DOMAIN=e2e.test +DSTACK_E2E_PEBBLE_IMAGE=kvin/pebble:latest + +# Host ports. Defaults are chosen to avoid common dstackup ports. +DSTACK_E2E_VMM_PORT=29080 +DSTACK_E2E_AUTH_PORT=28011 +DSTACK_E2E_ARTIFACT_PORT=38081 +DSTACK_E2E_KMS_OLD_HOST_PORT=28082 +DSTACK_E2E_KMS_LATEST_HOST_PORT=28083 +# KMS 0.5.8 encodes this value as a DNS SAN. It must resolve to the test host +# from BOTH the host-side deployment client and each CVM. If omitted, the +# renderer derives .nip.io (for example below). +# DSTACK_E2E_KMS_RPC_DOMAIN=192.168.1.90.nip.io +DSTACK_E2E_GATEWAY1_RPC_HOST_PORT=28000 +DSTACK_E2E_GATEWAY1_ADMIN_HOST_PORT=28001 +DSTACK_E2E_GATEWAY1_PROXY_HOST_PORT=28443 +DSTACK_E2E_GATEWAY1_WG_HOST_PORT=28120 +DSTACK_E2E_GATEWAY2_RPC_HOST_PORT=28100 +DSTACK_E2E_GATEWAY2_ADMIN_HOST_PORT=28101 +DSTACK_E2E_GATEWAY2_PROXY_HOST_PORT=28543 +DSTACK_E2E_GATEWAY2_WG_HOST_PORT=28121 +DSTACK_E2E_KEY_PROVIDER_PORT=13443 +DSTACK_E2E_HOST_API_PORT=20011 +DSTACK_E2E_MOCK_CF_HTTP_PORT=38080 +DSTACK_E2E_PEBBLE_HTTP_PORT=34000 +DSTACK_E2E_PEBBLE_MGMT_PORT=35000 + +# Optional exact 20-byte Gateway application ID. If omitted, the renderer +# deterministically derives one. "any" is intentionally rejected. +# DSTACK_E2E_GATEWAY_APP_ID=0123456789abcdef0123456789abcdef01234567 + +# VMM/QEMU knobs. +DSTACK_E2E_CID_START=15000 +DSTACK_E2E_QGS_PORT=4050 +DSTACK_E2E_QEMU_PATH=/usr/bin/qemu-system-x86_64 + +# SGX QCNL config used by the Local-Key-Provider that seals each KMS CVM disk. +DSTACK_E2E_QCNL_CONF=../../dstack/local-key-provider/build/sgx_default_qcnl.conf + +# Runner behaviour. +DSTACK_E2E_CLEAN_START=true +DSTACK_E2E_CLEANUP_AFTER=false +DSTACK_E2E_UPGRADE_CLEAN_STATE=true +DSTACK_E2E_KEEP_STACK=true + +# Set only when the current x86_64-musl KMS/Gateway binaries were already built. +DSTACK_E2E_SKIP_CURRENT_BUILD=false diff --git a/test-suites/full-stack-compose/.gitignore b/test-suites/full-stack-compose/.gitignore new file mode 100644 index 000000000..89e702f8d --- /dev/null +++ b/test-suites/full-stack-compose/.gitignore @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 + +.env +/state/* +!/state/.gitkeep +__pycache__/ +*.pyc diff --git a/test-suites/full-stack-compose/README.md b/test-suites/full-stack-compose/README.md new file mode 100644 index 000000000..6329cb042 --- /dev/null +++ b/test-suites/full-stack-compose/README.md @@ -0,0 +1,140 @@ + + +# Production-compatible KMS/Gateway upgrade E2E + +This suite runs the stateful services and applications in **real TDX CVMs**. +Docker Compose is used only for host infrastructure (VMM, authorization, +Local-Key-Provider, image server, and mock ACME/DNS): + +```text +host Docker Compose + ├─ dstack-vmm + dstack-auth + ├─ AESMD + SGX Local-Key-Provider + ├─ verified OS/image artifact server + └─ Pebble + mock Cloudflare DNS + +real TDX CVMs + ├─ KMS 0.5.8 (Local-Key-Provider, quote-enabled onboarding) + ├─ current KMS (new CVM, quote-attested replication from 0.5.8) + ├─ two-node Gateway 0.5.8 cluster, rolled one node at a time to current + ├─ forced-legacy nginx app CVM + └─ forced-lite nginx app CVM +``` + +The upgrade scenario intentionally rejects the development trust settings that +would make a passing result irrelevant to production: + +- KMS 0.5.8 uses `quote_enabled = true`. +- Both KMS manifests explicitly set `enforce_self_authorization = true`. +- KMS OS image verification is enabled and downloads a checksum-verified image + archive from an initially empty cache. +- KMS root keys and Gateway TLS identity are created inside TDX CVMs; the suite + never generates or injects them on the host. +- The authorization webhook pins one OS digest, exact KMS aggregate + measurements, the quote-derived physical TDX device ID, exact app compose + hashes, and a concrete 20-byte Gateway app ID. `allowAnyDevice = true` and + `gatewayAppId = "any"` are rejected. +- Container references in measured app manifests are content-addressed Docker + image IDs. Tags are used only as the Docker Hub/build inputs that are pulled, + inspected, saved, and imported before boot. + +## Prerequisites + +1. Build the host-side control binaries: + + ```bash + cargo build --release --manifest-path ../../dstack/Cargo.toml \ + -p dstack-cli -p dstack-auth -p dstack-vmm -p supervisor + ``` + + The driver builds current KMS and Gateway binaries for + `x86_64-unknown-linux-musl` and places them in test-only container images. + The static binaries are layered on the released 0.5.8 production runtime; + the KMS production builder still uses the same Debian/QEMU revision, and the + current Gateway entrypoint is copied from this checkout. The built binaries + and in-CVM version endpoints must report the current workspace version and + Git revision, so Docker cache or an old binary cannot silently turn the + upgrade into a no-op. + +2. Provide a TDX/SGX host with: + + - `/dev/kvm`, `/dev/vhost-vsock`, Intel TDX, and QGS (default port `4050`) + - `/dev/sgx_enclave` and `/dev/sgx_provision` + - a working SGX QCNL/PCCS configuration + - an IPv4/DNS name for the host that is reachable from both the host-side + deployment client and QEMU user-networked CVMs; the suite derives + `.nip.io`, or accepts `DSTACK_E2E_KMS_RPC_DOMAIN` + - Docker and WireGuard kernel support + +3. Provide an unpacked dstack image directory containing `digest.txt` and + `sha256sum.txt`. Copy `.env.example` to `.env` when paths or ports differ. + +## Run + +```bash +DOCKER_BUILDKIT=0 ./run-upgrade-e2e.sh +``` + +The defaults pull these released images from Docker Hub: + +- `dstacktee/dstack-kms:0.5.8@sha256:9650dcb47dad0065470f432f00e78e012912214ef1a5b1d7272918817e61a26d` +- `dstacktee/dstack-gateway:0.5.8@sha256:6eb1dc1a5000f37cc5b0322d3fdb71e7f2e31859b5e3a611634919278cee2411` + +The driver checks each released binary's `--version` output before deploying +anything. Set `DSTACK_E2E_SKIP_CURRENT_BUILD=true` only when the current musl +binaries were already built. + +## Upgrade sequence and assertions + +### KMS 0.5.8 to current + +1. Deploy KMS 0.5.8 as a Local-Key-Provider TDX CVM with no pre-created keys. +2. Call `Onboard.GetAttestationInfo`, authorize its exact `mrAggregated`, then + bootstrap it. The bootstrap response must contain non-empty attestation. +3. Boot the forced-legacy app through KMS 0.5.8 and record the KMS CA SPKI, + root k256 public key, and the app's derived environment-encryption public + key. (The expected onboarding path reissues the self-signed CA certificate, + so its DER bytes/expiry are not a stable identity; its private key/SPKI is.) +4. Deploy current KMS in a new Local-Key-Provider TDX CVM, authorize its exact + measurement, and call `Onboard.Onboard` against KMS 0.5.8. This exercises the + production RA-TLS root-key replication path; it is not a directory copy. +5. Require all recorded identities/derived keys to be byte-for-byte equal, + stop KMS 0.5.8, and force the legacy app to boot against only current KMS. +6. Boot a fresh forced-lite app against current KMS. Both modes must reach + `boot_progress=done`, which occurs after boot-time `GetAppKey` succeeds. + +The artifact server journal must contain two successful GETs for the exact +measured OS archive (one from each fresh KMS data disk). VM logs are also +rejected if they say image verification or self-authorization is disabled. + +### Gateway 0.5.8 to current + +1. Deploy two Gateway 0.5.8 TDX CVMs under one pinned Gateway app ID. Both get + app keys and RA-TLS certificates from KMS; their environment is encrypted by + VMM/KMS. +2. Form a WaveKV cluster, configure Pebble/Cloudflare mock DNS once, and require + the second node to receive the configuration and wildcard certificate. +3. Verify the legacy and lite app SNI routes through **both** Gateway nodes. +4. Snapshot each node's UUID, CVM registrations/IPs, certbot configuration, DNS + credential metadata, ZT domains, and wildcard-certificate fingerprint. +5. Run a rapid HA probe that alternates the preferred physical Gateway for + every legacy/lite request and falls back to the other node in the same cycle. +6. Stop, update the measured app compose, and restart node 1 on current KMS; + require it to carry both apps and retain all durable state. Repeat for node 2. +7. Require zero failed HA cycles and unchanged durable state/certificates. +8. Force certificate issuance through each upgraded node against a mock DNS API + that rejects anything except the original bearer token. This proves the DNS + credential value survived even though the current admin API redacts it. + +This is a production-style **rolling two-node** zero-downtime assertion. The +suite does not claim that rebooting a single Gateway CVM can preserve that +node's listener; availability is maintained by the peer, as it must be in a +production rollout. + +Artifacts, manifests, version headers, canonical state snapshots, probe +results, attestation information, and VM logs are written to `state/work/`. +The stack is retained by default for inspection; set +`DSTACK_E2E_KEEP_STACK=false` and/or `DSTACK_E2E_CLEANUP_AFTER=true` to clean it. diff --git a/test-suites/full-stack-compose/compose.yml b/test-suites/full-stack-compose/compose.yml new file mode 100644 index 000000000..f2fb877ad --- /dev/null +++ b/test-suites/full-stack-compose/compose.yml @@ -0,0 +1,256 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +name: dstack-full-e2e + +x-suite-env: &suite-env + DSTACK_E2E_STATE_DIR: /suite-state + DSTACK_E2E_CONFIG_DIR: /suite-state/config + DSTACK_E2E_RUN_DIR: /suite-state/run + DSTACK_E2E_VM_DIR: /suite-state/vm + DSTACK_E2E_VOLUMES_DIR: /suite-state/volumes + DSTACK_E2E_IMAGE_ROOT: /images + DSTACK_E2E_IMAGE_NAME: ${DSTACK_E2E_IMAGE_NAME:-dstack-0.6.0} + DSTACK_E2E_PLATFORM: ${DSTACK_E2E_PLATFORM:-tdx} + DSTACK_E2E_VMM_PORT: ${DSTACK_E2E_VMM_PORT:-29080} + DSTACK_E2E_AUTH_PORT: ${DSTACK_E2E_AUTH_PORT:-28011} + DSTACK_E2E_ARTIFACT_PORT: ${DSTACK_E2E_ARTIFACT_PORT:-38081} + DSTACK_E2E_KMS_OLD_HOST_PORT: ${DSTACK_E2E_KMS_OLD_HOST_PORT:-28082} + DSTACK_E2E_KMS_LATEST_HOST_PORT: ${DSTACK_E2E_KMS_LATEST_HOST_PORT:-28083} + DSTACK_E2E_KMS_RPC_DOMAIN: ${DSTACK_E2E_KMS_RPC_DOMAIN:-} + DSTACK_E2E_GATEWAY1_RPC_HOST_PORT: ${DSTACK_E2E_GATEWAY1_RPC_HOST_PORT:-28000} + DSTACK_E2E_GATEWAY1_ADMIN_HOST_PORT: ${DSTACK_E2E_GATEWAY1_ADMIN_HOST_PORT:-28001} + DSTACK_E2E_GATEWAY1_PROXY_HOST_PORT: ${DSTACK_E2E_GATEWAY1_PROXY_HOST_PORT:-28443} + DSTACK_E2E_GATEWAY1_WG_HOST_PORT: ${DSTACK_E2E_GATEWAY1_WG_HOST_PORT:-28120} + DSTACK_E2E_GATEWAY2_RPC_HOST_PORT: ${DSTACK_E2E_GATEWAY2_RPC_HOST_PORT:-28100} + DSTACK_E2E_GATEWAY2_ADMIN_HOST_PORT: ${DSTACK_E2E_GATEWAY2_ADMIN_HOST_PORT:-28101} + DSTACK_E2E_GATEWAY2_PROXY_HOST_PORT: ${DSTACK_E2E_GATEWAY2_PROXY_HOST_PORT:-28543} + DSTACK_E2E_GATEWAY2_WG_HOST_PORT: ${DSTACK_E2E_GATEWAY2_WG_HOST_PORT:-28121} + DSTACK_E2E_GATEWAY_APP_ID: ${DSTACK_E2E_GATEWAY_APP_ID:-} + DSTACK_E2E_KEY_PROVIDER_PORT: ${DSTACK_E2E_KEY_PROVIDER_PORT:-13443} + DSTACK_E2E_HOST_API_PORT: ${DSTACK_E2E_HOST_API_PORT:-20011} + DSTACK_E2E_CID_START: ${DSTACK_E2E_CID_START:-15000} + DSTACK_E2E_QGS_PORT: ${DSTACK_E2E_QGS_PORT:-4050} + DSTACK_E2E_QEMU_PATH: ${DSTACK_E2E_QEMU_PATH:-/usr/bin/qemu-system-x86_64} + DSTACK_E2E_BASE_DOMAIN: ${DSTACK_E2E_BASE_DOMAIN:-e2e.test} + DSTACK_E2E_MOCK_CF_HTTP_PORT: ${DSTACK_E2E_MOCK_CF_HTTP_PORT:-38080} + DSTACK_E2E_PEBBLE_HTTP_PORT: ${DSTACK_E2E_PEBBLE_HTTP_PORT:-34000} + DSTACK_E2E_APP_NAME: ${DSTACK_E2E_APP_NAME:-dstack-e2e-nginx} + DSTACK_E2E_NAME_PREFIX: ${DSTACK_E2E_NAME_PREFIX:-dstack-e2e} + DSTACK_E2E_PHASE: ${DSTACK_E2E_PHASE:-upgrade} + DSTACK_E2E_CURRENT_VERSION: ${DSTACK_E2E_CURRENT_VERSION:-0.6.0} + +services: + init-config: + build: + context: ./runtime + image: dstack-e2e-runtime:local + network_mode: host + volumes: + - ./state:/suite-state + - ./scripts:/suite/scripts:ro + - ${DSTACK_E2E_IMAGE_STORE:-../../../meta-dstack/build/images}:/images:ro + environment: + <<: *suite-env + command: ["/suite/scripts/render-config.sh"] + + mock-cf-dns-api: + build: + context: ./mock-cf-dns + image: dstack-e2e-mock-cf-dns:local + networks: + e2e-net: + ipv4_address: 172.31.0.10 + ports: + - "0.0.0.0:${DSTACK_E2E_MOCK_CF_HTTP_PORT:-38080}:8080" + environment: + PORT: 8080 + DEBUG: ${DSTACK_E2E_MOCK_DEBUG:-false} + MOCK_CF_ZONES: ${DSTACK_E2E_BASE_DOMAIN:-e2e.test},test,local + MOCK_CF_API_TOKEN: test-token + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health')"] + interval: 5s + timeout: 3s + retries: 10 + + pebble: + # Custom Pebble build used by the existing gateway E2E suite. It supports + # plain HTTP ACME URLs, which keeps the host-side Gateway bootstrap simple. + image: ${DSTACK_E2E_PEBBLE_IMAGE:-kvin/pebble:latest} + command: ["-http", "-dnsserver", "172.31.0.10:53"] + networks: + e2e-net: + ipv4_address: 172.31.0.11 + ports: + - "0.0.0.0:${DSTACK_E2E_PEBBLE_HTTP_PORT:-34000}:14000" + - "0.0.0.0:${DSTACK_E2E_PEBBLE_MGMT_PORT:-35000}:15000" + environment: + PEBBLE_VA_NOSLEEP: "1" + PEBBLE_VA_ALWAYS_VALID: "1" + depends_on: + mock-cf-dns-api: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "wget -q --spider http://127.0.0.1:14000/dir || curl -fsS http://127.0.0.1:14000/dir >/dev/null"] + interval: 5s + timeout: 3s + retries: 20 + + aesmd: + build: + context: ../../dstack/local-key-provider + dockerfile: build/Dockerfile.aesmd + image: dstack-e2e-aesmd:local + privileged: true + network_mode: host + devices: + - "/dev/sgx_enclave:/dev/sgx_enclave" + - "/dev/sgx_provision:/dev/sgx_provision" + volumes: + - ${DSTACK_E2E_QCNL_CONF:-../../dstack/local-key-provider/build/sgx_default_qcnl.conf}:/etc/sgx_default_qcnl.conf:ro + - aesmd-sock:/var/run/aesmd/ + restart: unless-stopped + + local-keyprovider: + build: + context: ../../dstack + dockerfile: local-key-provider/build/Dockerfile.key-provider + args: + APT_SNAPSHOT: ${APT_SNAPSHOT:-20260423T000000Z} + SOURCE_DATE_EPOCH: ${SOURCE_DATE_EPOCH:-1776902400} + RUSTUP_VERSION: ${RUSTUP_VERSION:-1.28.2} + RUSTUP_INIT_SHA256: ${RUSTUP_INIT_SHA256:-20a06e644b0d9bd2fbdbfd52d42540bdde820ea7df86e92e533c073da0cdd43c} + RUST_TOOLCHAIN: ${RUST_TOOLCHAIN:-1.92.0} + image: dstack-e2e-local-keyprovider:local + privileged: true + devices: + - "/dev/sgx_enclave:/dev/sgx_enclave" + - "/dev/sgx_provision:/dev/sgx_provision" + depends_on: + - aesmd + environment: + PCCS_URL: ${PCCS_URL:-https://pccs.phala.network} + LISTEN_ADDR: 0.0.0.0:3443 + volumes: + - aesmd-sock:/var/run/aesmd/ + ports: + - "127.0.0.1:${DSTACK_E2E_KEY_PROVIDER_PORT:-13443}:3443" + healthcheck: + # The provider is a raw TCP service without an HTTP health endpoint. + # Opening the socket proves that the production SGX enclave was loaded; + # the container cannot listen while AESMD/DCAP provisioning is broken. + test: ["CMD-SHELL", "timeout 2 bash -c '- + exec python3 -u -m http.server "${DSTACK_E2E_ARTIFACT_PORT:-38081}" + --bind 0.0.0.0 --directory /suite-state/artifacts + 2> >(tee -a /suite-state/work/artifacts-access.log >&2) + healthcheck: + test: ["CMD", "curl", "-fsS", "http://127.0.0.1:${DSTACK_E2E_ARTIFACT_PORT:-38081}/"] + interval: 5s + timeout: 3s + retries: 20 + restart: unless-stopped + + vmm: + image: dstack-e2e-runtime:local + privileged: true + pid: host + network_mode: host + devices: + - "/dev/kvm:/dev/kvm" + - "/dev/vhost-vsock:/dev/vhost-vsock" + depends_on: + init-config: + condition: service_completed_successfully + auth: + condition: service_started + artifacts: + condition: service_healthy + local-keyprovider: + condition: service_healthy + volumes: + - ./state:/suite-state + - ../../dstack:/workspace:ro + - ${DSTACK_E2E_IMAGE_STORE:-../../../meta-dstack/build/images}:/images:ro + - /dev:/dev + - /lib/modules:/lib/modules:ro + environment: + RUST_LOG: ${DSTACK_E2E_VMM_RUST_LOG:-info,dstack_vmm=info} + command: + - bash + - -lc + - exec /workspace/target/release/dstack-vmm -c /suite-state/config/vmm.toml + restart: unless-stopped + + runner: + image: dstack-e2e-runtime:local + network_mode: host + depends_on: + mock-cf-dns-api: + condition: service_healthy + pebble: + condition: service_healthy + auth: + condition: service_started + artifacts: + condition: service_healthy + vmm: + condition: service_started + volumes: + - ./state:/suite-state + - ./scripts:/suite/scripts:ro + - ../../dstack:/workspace:ro + environment: + <<: *suite-env + DSTACK_E2E_CLEAN_START: ${DSTACK_E2E_CLEAN_START:-true} + DSTACK_E2E_CLEANUP_AFTER: ${DSTACK_E2E_CLEANUP_AFTER:-false} + DSTACK_E2E_APP_VCPU: ${DSTACK_E2E_APP_VCPU:-2} + DSTACK_E2E_APP_MEMORY: ${DSTACK_E2E_APP_MEMORY:-2048} + command: ["/suite/scripts/runner.sh"] + +networks: + e2e-net: + driver: bridge + ipam: + config: + - subnet: 172.31.0.0/24 + +volumes: + aesmd-sock: diff --git a/test-suites/full-stack-compose/mock-cf-dns/Dockerfile b/test-suites/full-stack-compose/mock-cf-dns/Dockerfile new file mode 100644 index 000000000..8c68f2ec1 --- /dev/null +++ b/test-suites/full-stack-compose/mock-cf-dns/Dockerfile @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +FROM python:3.12-slim +WORKDIR /app +COPY server.py /app/server.py +EXPOSE 8080/tcp 53/udp +CMD ["python", "/app/server.py"] diff --git a/test-suites/full-stack-compose/mock-cf-dns/server.py b/test-suites/full-stack-compose/mock-cf-dns/server.py new file mode 100755 index 000000000..b3bc07d51 --- /dev/null +++ b/test-suites/full-stack-compose/mock-cf-dns/server.py @@ -0,0 +1,293 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 + +"""Tiny Cloudflare DNS API + UDP TXT DNS mock for dstack gateway E2E. + +It implements just the Cloudflare endpoints used by certbot's CloudflareClient: + GET /client/v4/zones + GET /client/v4/zones//dns_records?name= + POST /client/v4/zones//dns_records + DELETE /client/v4/zones//dns_records/ + +It also serves TXT answers on UDP/53 so Pebble can validate DNS-01 when the +ACME server is not configured with PEBBLE_VA_ALWAYS_VALID=1. +""" + +from __future__ import annotations + +import json +import os +import re +import socket +import struct +import threading +import time +import urllib.parse +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any + +STATE_LOCK = threading.RLock() +RECORDS: list[dict[str, Any]] = [] +NEXT_ID = 1 + + +def _zones() -> list[dict[str, str]]: + raw = os.environ.get("MOCK_CF_ZONES", "e2e.test test local") + names = [ + z.strip().strip(".").lower() for z in re.split(r"[,\s]+", raw) if z.strip() + ] + if not names: + names = ["e2e.test"] + out = [] + seen = set() + for name in names: + if name in seen: + continue + seen.add(name) + out.append({"id": zone_id_for(name), "name": name}) + return out + + +def zone_id_for(name: str) -> str: + """Return a deterministic mock zone ID for a DNS name.""" + safe = re.sub(r"[^a-z0-9]+", "-", name.lower()).strip("-") or "zone" + return f"zone-{safe}" + + +def _json(handler: BaseHTTPRequestHandler, code: int, body: Any) -> None: + data = json.dumps(body, sort_keys=True).encode() + handler.send_response(code) + handler.send_header("content-type", "application/json") + handler.send_header("content-length", str(len(data))) + handler.end_headers() + handler.wfile.write(data) + + +def _record_content(payload: dict[str, Any]) -> str: + if "content" in payload: + return str(payload.get("content") or "") + data = payload.get("data") or {} + if payload.get("type") == "CAA" and isinstance(data, dict): + return f'{data.get("flags", 0)} {data.get("tag", "issue")} "{data.get("value", "")}"' + return "" + + +def _authorized(handler: BaseHTTPRequestHandler) -> bool: + """Require the configured Cloudflare bearer token for API operations.""" + expected = os.environ.get("MOCK_CF_API_TOKEN", "test-token") + if handler.headers.get("Authorization", "") == f"Bearer {expected}": + return True + _json( + handler, + 403, + {"success": False, "errors": [{"message": "invalid API token"}]}, + ) + return False + + +class Handler(BaseHTTPRequestHandler): + """Handle the mock Cloudflare HTTP API.""" + + server_version = "dstack-mock-cf-dns/0.1" + + def log_message(self, fmt: str, *args: Any) -> None: + """Emit request logs when debug logging is enabled.""" + if os.environ.get("DEBUG", "").lower() in {"1", "true", "yes"}: + super().log_message(fmt, *args) + + def do_GET(self) -> None: # noqa: N802 + """Handle supported HTTP GET endpoints.""" + parsed = urllib.parse.urlparse(self.path) + path = parsed.path.rstrip("/") or "/" + query = urllib.parse.parse_qs(parsed.query) + if path == "/health": + _json(self, 200, {"ok": True}) + return + if path == "/api/records": + with STATE_LOCK: + _json(self, 200, {"records": RECORDS}) + return + if path.startswith("/client/v4/") and not _authorized(self): + return + if path == "/client/v4/zones": + zones = _zones() + _json( + self, + 200, + { + "success": True, + "result": zones, + "result_info": { + "page": 1, + "per_page": 50, + "total_pages": 1, + "count": len(zones), + "total_count": len(zones), + }, + }, + ) + return + m = re.fullmatch(r"/client/v4/zones/([^/]+)/dns_records", path) + if m: + name = (query.get("name") or [""])[0].strip().strip(".").lower() + with STATE_LOCK: + records = [ + r + for r in RECORDS + if not name or r["name"].strip(".").lower() == name + ] + _json( + self, + 200, + { + "success": True, + "result": records, + "result_info": {"page": 1, "per_page": 100, "total_pages": 1}, + }, + ) + return + _json( + self, 404, {"success": False, "errors": [{"message": f"not found: {path}"}]} + ) + + def do_POST(self) -> None: # noqa: N802 + """Create a mock DNS record.""" + global NEXT_ID + parsed = urllib.parse.urlparse(self.path) + path = parsed.path.rstrip("/") or "/" + m = re.fullmatch(r"/client/v4/zones/([^/]+)/dns_records", path) + if not m: + _json( + self, + 404, + {"success": False, "errors": [{"message": f"not found: {path}"}]}, + ) + return + if not _authorized(self): + return + length = int(self.headers.get("content-length", "0") or "0") + payload = json.loads(self.rfile.read(length) or b"{}") + with STATE_LOCK: + record_id = f"rec-{NEXT_ID}" + NEXT_ID += 1 + record = { + "id": record_id, + "zone_id": m.group(1), + "type": str(payload.get("type") or "TXT").upper(), + "name": str(payload.get("name") or "").strip().strip("."), + "content": _record_content(payload), + "ttl": int(payload.get("ttl") or 60), + "created_on": int(time.time()), + } + RECORDS.append(record) + _json(self, 200, {"success": True, "result": record}) + + def do_DELETE(self) -> None: # noqa: N802 + """Delete a mock DNS record.""" + parsed = urllib.parse.urlparse(self.path) + path = parsed.path.rstrip("/") or "/" + m = re.fullmatch(r"/client/v4/zones/([^/]+)/dns_records/([^/]+)", path) + if not m: + _json( + self, + 404, + {"success": False, "errors": [{"message": f"not found: {path}"}]}, + ) + return + if not _authorized(self): + return + record_id = urllib.parse.unquote(m.group(2)) + with STATE_LOCK: + before = len(RECORDS) + RECORDS[:] = [r for r in RECORDS if r["id"] != record_id] + removed = before != len(RECORDS) + _json( + self, + 200, + {"success": True, "result": {"id": record_id, "removed": removed}}, + ) + + +def parse_qname(packet: bytes, offset: int = 12) -> tuple[str, int]: + """Parse a DNS question name and return its value and end offset.""" + labels: list[str] = [] + while True: + if offset >= len(packet): + raise ValueError("bad qname") + length = packet[offset] + offset += 1 + if length == 0: + break + labels.append(packet[offset : offset + length].decode("ascii", "ignore")) + offset += length + return ".".join(labels).strip(".").lower(), offset + + +def txt_rdata(text: str) -> bytes: + """Encode text as DNS TXT record data.""" + raw = text.encode() + chunks = [raw[i : i + 255] for i in range(0, len(raw), 255)] or [b""] + return b"".join(bytes([len(c)]) + c for c in chunks) + + +def dns_response(packet: bytes) -> bytes: + """Build a DNS response containing matching TXT records.""" + if len(packet) < 12: + return b"" + txid = packet[:2] + qdcount = struct.unpack("!H", packet[4:6])[0] + if qdcount < 1: + return b"" + name, qend = parse_qname(packet) + question = packet[12 : qend + 4] + qtype = ( + struct.unpack("!H", packet[qend : qend + 2])[0] + if qend + 4 <= len(packet) + else 16 + ) + with STATE_LOCK: + answers = [ + r + for r in RECORDS + if r["type"] == "TXT" and r["name"].strip(".").lower() == name + ] + if qtype not in (16, 255): + answers = [] + header = txid + struct.pack("!HHHHH", 0x8180, 1, len(answers), 0, 0) + body = question + for record in answers: + rdata = txt_rdata(record["content"]) + body += ( + b"\xc0\x0c" + + struct.pack("!HHIH", 16, 1, int(record.get("ttl", 60)), len(rdata)) + + rdata + ) + return header + body + + +def dns_loop() -> None: + """Serve mock DNS responses over UDP.""" + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.bind(("0.0.0.0", 53)) + while True: + packet, addr = sock.recvfrom(4096) + try: + response = dns_response(packet) + if response: + sock.sendto(response, addr) + except Exception as exc: # pragma: no cover - diagnostic only + if os.environ.get("DEBUG", "").lower() in {"1", "true", "yes"}: + print(f"dns error from {addr}: {exc}", flush=True) + + +def main() -> None: + """Run the mock Cloudflare API and DNS servers.""" + threading.Thread(target=dns_loop, daemon=True).start() + port = int(os.environ.get("PORT", "8080")) + print(f"mock CF API on :{port}; zones={_zones()}", flush=True) + ThreadingHTTPServer(("0.0.0.0", port), Handler).serve_forever() + + +if __name__ == "__main__": + main() diff --git a/test-suites/full-stack-compose/run-upgrade-e2e.sh b/test-suites/full-stack-compose/run-upgrade-e2e.sh new file mode 100755 index 000000000..b0ee43b55 --- /dev/null +++ b/test-suites/full-stack-compose/run-upgrade-e2e.sh @@ -0,0 +1,244 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +REPO_DIR=$(cd -- "$SCRIPT_DIR/../.." && pwd) +STATE_DIR="$SCRIPT_DIR/state" +ENV_FILE=${DSTACK_E2E_ENV_FILE:-$SCRIPT_DIR/.env} + +setting() { + local name=$1 fallback=$2 line value + if [[ -v $name ]]; then + printf '%s' "${!name}" + return + fi + if [[ -f "$ENV_FILE" ]]; then + line=$(grep -E "^[[:space:]]*${name}=" "$ENV_FILE" | tail -n1 || true) + if [[ -n "$line" ]]; then + value=${line#*=} + value=${value%$'\r'} + if [[ "$value" == \"*\" && "$value" == *\" ]]; then + value=${value:1:${#value}-2} + elif [[ "$value" == \'*\' && "$value" == *\' ]]; then + value=${value:1:${#value}-2} + fi + printf '%s' "$value" + return + fi + fi + printf '%s' "$fallback" +} + +OLD_KMS_IMAGE=$(setting DSTACK_E2E_OLD_KMS_IMAGE \ + dstacktee/dstack-kms:0.5.8@sha256:9650dcb47dad0065470f432f00e78e012912214ef1a5b1d7272918817e61a26d) +OLD_GATEWAY_IMAGE=$(setting DSTACK_E2E_OLD_GATEWAY_IMAGE \ + dstacktee/dstack-gateway:0.5.8@sha256:6eb1dc1a5000f37cc5b0322d3fdb71e7f2e31859b5e3a611634919278cee2411) +APP_IMAGE=$(setting DSTACK_E2E_APP_IMAGE nginx:alpine) +KEEP_STACK=$(setting DSTACK_E2E_KEEP_STACK true) +CLEAN_STATE=$(setting DSTACK_E2E_UPGRADE_CLEAN_STATE true) +SKIP_BUILD=$(setting DSTACK_E2E_SKIP_CURRENT_BUILD false) +# dstack's build metadata deliberately embeds a 20-hex abbreviated revision. +CURRENT_REV=$(git -C "$REPO_DIR" rev-parse --short=20 HEAD) +CURRENT_VERSION=$(sed -n 's/^version = "\([^"]*\)"/\1/p' \ + "$REPO_DIR/dstack/Cargo.toml" | head -n1) +[[ -n "$CURRENT_VERSION" ]] || { + echo "ERROR: could not read current workspace version" >&2 + exit 1 +} + +COMPOSE=(docker compose -f "$SCRIPT_DIR/compose.yml") +if [[ -f "$ENV_FILE" ]]; then + COMPOSE=(docker compose --env-file "$ENV_FILE" -f "$SCRIPT_DIR/compose.yml") +fi + +log() { printf '[%(%H:%M:%S)T] %s\n' -1 "$*"; } +die() { log "ERROR: $*" >&2; exit 1; } +compose() { "${COMPOSE[@]}" "$@"; } + +need_bin() { + [[ -x "$1" ]] || die "missing executable $1" +} + +pull_released_image() { + local image=$1 component=$2 + log "pulling released $component image from Docker Hub: $image" + docker pull "$image" || die "cannot pull released $component image $image" +} + +reset_state() { + log "resetting Compose stack and E2E state" + compose down --remove-orphans >/dev/null 2>&1 || true + docker run --rm -v "$STATE_DIR:/state" alpine:3.22 sh -c \ + 'find /state -mindepth 1 ! -name .gitkeep -exec rm -rf {} +' +} + +build_current_binaries() { + if [[ "$SKIP_BUILD" == true ]]; then + log "using prebuilt current musl KMS/Gateway binaries" + else + log "building current KMS/Gateway as production-style static musl binaries" + cargo build --manifest-path "$REPO_DIR/dstack/Cargo.toml" \ + --release --target x86_64-unknown-linux-musl \ + -p dstack-kms -p dstack-gateway + fi + need_bin "$REPO_DIR/dstack/target/x86_64-unknown-linux-musl/release/dstack-kms" + need_bin "$REPO_DIR/dstack/target/x86_64-unknown-linux-musl/release/dstack-gateway" +} + +prepare_container_artifacts() { + local artifact_dir="$STATE_DIR/artifacts/images" + local context_dir="$STATE_DIR/image-build" + local rev current_kms_image current_gateway_image + local old_kms_id old_gateway_id current_kms_id current_gateway_id app_id + rev=$(git -C "$REPO_DIR" rev-parse --short=16 HEAD) + current_kms_image="dstack-e2e-kms-current:${rev}" + current_gateway_image="dstack-e2e-gateway-current:${rev}" + mkdir -p "$artifact_dir" "$context_dir/kms" "$context_dir/gateway" + + cp "$REPO_DIR/dstack/target/x86_64-unknown-linux-musl/release/dstack-kms" \ + "$context_dir/kms/dstack-kms" + cat > "$context_dir/kms/Dockerfile" < "$context_dir/gateway/Dockerfile" < "$STATE_DIR/artifacts/images.env" </dev/null \ + || die "$OLD_KMS_IMAGE is not KMS 0.5.8" + grep -E '^dstack-gateway v0\.5\.8 \(git:' "$STATE_DIR/work/gateway-old.version.txt" >/dev/null \ + || die "$OLD_GATEWAY_IMAGE is not Gateway 0.5.8" + grep -E "^dstack-kms v${CURRENT_VERSION//./\\.} \\(git:" \ + "$STATE_DIR/work/kms-current.version.txt" >/dev/null \ + || die "locally built KMS is not current v$CURRENT_VERSION" + grep -E "^dstack-gateway v${CURRENT_VERSION//./\\.} \\(git:" \ + "$STATE_DIR/work/gateway-current.version.txt" >/dev/null \ + || die "locally built Gateway is not current v$CURRENT_VERSION" + grep -F "$CURRENT_REV" "$STATE_DIR/work/kms-current.version.txt" >/dev/null \ + || die "KMS binary was not built from current revision $CURRENT_REV" + grep -F "$CURRENT_REV" "$STATE_DIR/work/gateway-current.version.txt" >/dev/null \ + || die "Gateway binary was not built from current revision $CURRENT_REV" +} + +wait_local_key_provider() { + local port deadline status + port=$(setting DSTACK_E2E_KEY_PROVIDER_PORT 13443) + deadline=$((SECONDS + 120)) + log "waiting for production SGX Local-Key-Provider on 127.0.0.1:${port}" + while (( SECONDS < deadline )); do + status=$(compose ps --format json local-keyprovider 2>/dev/null \ + | jq -rs 'map(select(.Service == "local-keyprovider"))[0].Health // ""' 2>/dev/null \ + || true) + if [[ "$status" == healthy ]]; then + log "Local-Key-Provider enclave is healthy" + return 0 + fi + sleep 2 + done + compose logs --tail=200 aesmd local-keyprovider >&2 || true + die "Local-Key-Provider did not become healthy; fix SGX/DCAP/PCCS provisioning rather than disabling attestation" +} + +on_exit() { + local rc=$? + if (( rc != 0 )); then + log "upgrade E2E failed; recent infrastructure logs follow" + compose logs --tail=250 auth artifacts vmm runner >&2 || true + fi + if [[ "$KEEP_STACK" != true ]]; then + compose down --remove-orphans >/dev/null 2>&1 || true + else + log "leaving stack running for inspection (DSTACK_E2E_KEEP_STACK=true)" + fi + exit "$rc" +} +trap on_exit EXIT + +main() { + need_bin "$REPO_DIR/dstack/target/release/dstack" + need_bin "$REPO_DIR/dstack/target/release/dstack-auth" + need_bin "$REPO_DIR/dstack/target/release/dstack-vmm" + need_bin "$REPO_DIR/dstack/target/release/supervisor" + + pull_released_image "$OLD_KMS_IMAGE" kms + pull_released_image "$OLD_GATEWAY_IMAGE" gateway + pull_released_image "$APP_IMAGE" application + build_current_binaries + [[ "$CLEAN_STATE" == true ]] && reset_state + + log "building E2E infrastructure" + compose build init-config mock-cf-dns-api aesmd local-keyprovider + compose run --rm init-config + prepare_container_artifacts + + log "starting authorization, artifact, attestation, ACME and VMM infrastructure" + compose up -d mock-cf-dns-api pebble aesmd local-keyprovider auth artifacts + wait_local_key_provider + compose up -d vmm + + log "running production-compatible KMS/Gateway rolling-upgrade E2E" + # Keep the complete runner transcript in state/work even though the runner is + # an ephemeral Compose container. This is especially important for failures + # during deployment, before a per-VM log file exists. + DSTACK_E2E_PHASE=upgrade compose run --rm --no-deps \ + -e DSTACK_E2E_PHASE=upgrade \ + -e DSTACK_E2E_CURRENT_VERSION="$CURRENT_VERSION" \ + -e DSTACK_E2E_CURRENT_REV="$CURRENT_REV" runner \ + 2>&1 | tee "$STATE_DIR/work/runner.log" + + log "upgrade E2E success" + log "artifacts: $STATE_DIR/work" +} + +main "$@" diff --git a/test-suites/full-stack-compose/runtime/Dockerfile b/test-suites/full-stack-compose/runtime/Dockerfile new file mode 100644 index 000000000..3d6fc1ae2 --- /dev/null +++ b/test-suites/full-stack-compose/runtime/Dockerfile @@ -0,0 +1,46 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + ca-certificates \ + software-properties-common \ + && add-apt-repository -y ppa:kobuk-team/tdx-release && \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + bash \ + coreutils \ + curl \ + iproute2 \ + iptables \ + iputils-ping \ + jq \ + netcat-openbsd \ + openssl \ + procps \ + python3 \ + python3-venv \ + qemu-system-x86 \ + qemu-utils \ + socat \ + tini \ + wireguard-tools \ + xz-utils \ + && rm -rf /var/lib/apt/lists/* + +# vmm-cli encrypts deployment environment variables with the KMS key and +# verifies the KMS signature before a production CVM is created. +COPY requirements.txt /tmp/vmm-requirements.txt +RUN python3 -m venv /opt/dstack-venv && \ + /opt/dstack-venv/bin/pip install --no-cache-dir \ + -r /tmp/vmm-requirements.txt && \ + rm /tmp/vmm-requirements.txt +ENV PATH="/opt/dstack-venv/bin:${PATH}" + +WORKDIR /suite +ENTRYPOINT ["/usr/bin/tini", "--"] +CMD ["bash"] diff --git a/test-suites/full-stack-compose/runtime/requirements.txt b/test-suites/full-stack-compose/runtime/requirements.txt new file mode 100644 index 000000000..4c016c51d --- /dev/null +++ b/test-suites/full-stack-compose/runtime/requirements.txt @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 + +# Keep the runtime subset synchronized with dstack/vmm/requirements.txt. +cryptography==46.0.7 +eth-hash[pycryptodome]==0.7.1 +eth-keys==0.7.0 +eth-utils==5.3.0 diff --git a/test-suites/full-stack-compose/scripts/allowlist.py b/test-suites/full-stack-compose/scripts/allowlist.py new file mode 100755 index 000000000..d67865cdf --- /dev/null +++ b/test-suites/full-stack-compose/scripts/allowlist.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 + +"""Manage the mutable KMS authorization allowlist for the compose E2E suite.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path + + +def norm_hex(value: str) -> str: + """Normalize a possibly prefixed hexadecimal string.""" + value = value.strip() + if value.lower().startswith("0x"): + value = value[2:] + return value.lower() + + +def checked_hex(value: str, length: int, label: str) -> str: + """Normalize and validate a fixed-width hexadecimal authorization value.""" + value = norm_hex(value) + if len(value) != length or any(ch not in "0123456789abcdef" for ch in value): + raise ValueError(f"invalid {label}: expected {length} hexadecimal characters") + return value + + +def load(path: Path) -> dict: + """Load an allowlist or return an empty default policy.""" + if path.exists(): + return json.loads(path.read_text()) + return { + "osImages": [], + "gatewayAppId": "", + "kms": {"mrAggregated": [], "devices": [], "allowAnyDevice": False}, + "apps": {}, + } + + +def save(path: Path, data: dict) -> None: + """Atomically write an allowlist to disk.""" + tmp = path.with_suffix(path.suffix + ".tmp") + tmp.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n") + os.replace(tmp, path) + + +def add_app(args: argparse.Namespace) -> None: + """Add an application and compose hash to the allowlist.""" + path = Path(args.path) + data = load(path) + apps = data.setdefault("apps", {}) + app_id = checked_hex(args.app_id, 40, "app ID") + compose_hash = checked_hex(args.compose_hash, 64, "compose hash") + device_id = checked_hex(args.device_id, 64, "device ID") + entry = apps.setdefault( + app_id, {"composeHashes": [], "devices": [], "allowAnyDevice": False} + ) + hashes = entry.setdefault("composeHashes", []) + if not any(norm_hex(h) == compose_hash for h in hashes): + hashes.append(compose_hash) + devices = entry.setdefault("devices", []) + if not any(norm_hex(item) == device_id for item in devices): + devices.append(device_id) + entry["allowAnyDevice"] = False + if args.gateway_app_id is not None: + data["gatewayAppId"] = checked_hex(args.gateway_app_id, 40, "Gateway app ID") + save(path, data) + print(json.dumps({"appId": app_id, "composeHash": compose_hash, "path": str(path)})) + + +def set_gateway(args: argparse.Namespace) -> None: + """Set the allowlisted Gateway application ID.""" + path = Path(args.path) + data = load(path) + data["gatewayAppId"] = checked_hex(args.gateway_app_id, 40, "Gateway app ID") + save(path, data) + print(json.dumps({"gatewayAppId": data["gatewayAppId"], "path": str(path)})) + + +def add_kms(args: argparse.Namespace) -> None: + """Authorize one exact KMS aggregate measurement.""" + path = Path(args.path) + data = load(path) + kms = data.setdefault( + "kms", {"mrAggregated": [], "devices": [], "allowAnyDevice": False} + ) + measurement = checked_hex(args.mr_aggregated, 64, "KMS mrAggregated") + device_id = checked_hex(args.device_id, 64, "device ID") + measurements = kms.setdefault("mrAggregated", []) + if not any(norm_hex(item) == measurement for item in measurements): + measurements.append(measurement) + devices = kms.setdefault("devices", []) + if not any(norm_hex(item) == device_id for item in devices): + devices.append(device_id) + kms["allowAnyDevice"] = False + save(path, data) + print(json.dumps({"mrAggregated": measurement, "path": str(path)})) + + +def main() -> None: + """Parse command-line arguments and update the allowlist.""" + parser = argparse.ArgumentParser() + sub = parser.add_subparsers(required=True) + p = sub.add_parser("add-app") + p.add_argument("--path", required=True) + p.add_argument("--app-id", required=True) + p.add_argument("--compose-hash", required=True) + p.add_argument("--device-id", required=True) + p.add_argument("--gateway-app-id") + p.set_defaults(func=add_app) + p = sub.add_parser("set-gateway") + p.add_argument("--path", required=True) + p.add_argument("--gateway-app-id", required=True) + p.set_defaults(func=set_gateway) + p = sub.add_parser("add-kms") + p.add_argument("--path", required=True) + p.add_argument("--mr-aggregated", required=True) + p.add_argument("--device-id", required=True) + p.set_defaults(func=add_kms) + args = parser.parse_args() + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/test-suites/full-stack-compose/scripts/app_compose.py b/test-suites/full-stack-compose/scripts/app_compose.py new file mode 100755 index 000000000..6a8cd579a --- /dev/null +++ b/test-suites/full-stack-compose/scripts/app_compose.py @@ -0,0 +1,252 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 + +"""Render measured app-compose manifests used by the full-stack E2E suite.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path + + +def image_loader(artifact_port: int, archive: str) -> str: + """Return a pre-launch script that imports one content-addressed image.""" + return f"""set -eu +archive=/tmp/{archive} +curl -fL --retry 5 --retry-delay 1 \\ + http://10.0.2.2:{artifact_port}/images/{archive} -o "$archive" +docker load -i "$archive" +rm -f "$archive" +""" + + +def write_manifest(path: Path, manifest: dict) -> None: + """Write a manifest and print its byte-exact derived identifiers.""" + body = json.dumps(manifest, indent=2, sort_keys=True) + "\n" + path.write_text(body) + digest = hashlib.sha256(body.encode()).hexdigest() + print(json.dumps({"path": str(path), "composeHash": digest, "appId": digest[:40]})) + + +def common_manifest(name: str, docker_compose: str, pre_launch_script: str) -> dict: + """Return the production CVM manifest baseline used by service apps.""" + return { + "docker_compose_file": docker_compose, + "gateway_enabled": False, + "kms_enabled": False, + "local_key_provider_enabled": False, + "manifest_version": 2, + "name": name, + "no_instance_id": True, + "pre_launch_script": pre_launch_script, + "public_logs": True, + "public_sysinfo": True, + "runner": "docker-compose", + "secure_time": True, + "storage_fs": "ext4", + } + + +def kms(args: argparse.Namespace) -> None: + """Render a KMS CVM using the production onboarding and verifier path.""" + config = f"""[rpc] +address = "0.0.0.0" +port = 8000 + +[rpc.tls] +key = "/kms/certs/rpc.key" +certs = "/kms/certs/rpc.crt" + +[rpc.tls.mutual] +ca_certs = "/kms/certs/tmp-ca.crt" +mandatory = false + +[core] +cert_dir = "/kms/certs" +admin_token_hash = "" +pccs_url = "" +enforce_self_authorization = true + +[core.image] +verify = true +cache_dir = "/kms/images" +download_url = "http://10.0.2.2:{args.artifact_port}/os/mr_{{OS_IMAGE_HASH}}.tar.gz" +download_timeout = "2m" + +[core.metrics] +enabled = true + +[core.auth_api] +type = "webhook" + +[core.auth_api.webhook] +url = "http://10.0.2.2:{args.auth_port}" + +[core.onboard] +enabled = true +auto_bootstrap_domain = "" +# Required by KMS 0.5.8. Current KMS ignores this retired field. +quote_enabled = true +address = "0.0.0.0" +port = 8000 +""" + docker_compose = f"""services: + kms: + image: {args.image_ref} + volumes: + - kms-volume:/kms + - /var/run/dstack.sock:/var/run/dstack.sock + ports: + - "8000:8000" + restart: unless-stopped + configs: + - source: kms_config + target: /kms/kms.toml + command: sh -c 'mkdir -p /kms/certs /kms/images && exec dstack-kms -c /kms/kms.toml' + +volumes: + kms-volume: + +configs: + kms_config: + content: | +""" + "".join(f" {line}\n" for line in config.splitlines()) + manifest = common_manifest( + args.name, + docker_compose, + image_loader(args.artifact_port, args.image_archive), + ) + manifest["local_key_provider_enabled"] = True + write_manifest(Path(args.output), manifest) + + +def gateway(args: argparse.Namespace) -> None: + """Render one production Gateway CVM node.""" + docker_compose = f"""services: + gateway: + image: {args.image_ref} + volumes: + - /var/run/dstack.sock:/var/run/dstack.sock + - /dstack:/dstack + - data:/data + network_mode: host + privileged: true + environment: + - WG_ENDPOINT=${{WG_ENDPOINT}} + - MY_URL=${{MY_URL}} + - BOOTNODE_URL=${{BOOTNODE_URL}} + - WG_IP=${{WG_IP}} + - WG_RESERVED_NET=${{WG_RESERVED_NET}} + - WG_CLIENT_RANGE=${{WG_CLIENT_RANGE}} + - NODE_ID=${{NODE_ID}} + - RUST_LOG=info,certbot=debug + - PCCS_URL=${{PCCS_URL}} + - RPC_DOMAIN=${{RPC_DOMAIN}} + - PROXY_LISTEN_PORT=443 + - PROXY_WORKERS=4 + - SYNC_INTERVAL=1s + - SYNC_TIMEOUT=10s + - SYNC_PERSIST_INTERVAL=1s + - SYNC_CONNECTIONS_ENABLED=true + - SYNC_CONNECTIONS_INTERVAL=1s + - ADMIN_LISTEN_ADDR=0.0.0.0 + - ADMIN_LISTEN_PORT=8001 + - ADMIN_API_TOKEN=${{ADMIN_API_TOKEN}} + restart: always + +volumes: + data: +""" + manifest = common_manifest( + args.name, + docker_compose, + image_loader(args.artifact_port, args.image_archive), + ) + manifest["kms_enabled"] = True + manifest["allowed_envs"] = [ + "ADMIN_API_TOKEN", + "BOOTNODE_URL", + "MY_URL", + "NODE_ID", + "PCCS_URL", + "RPC_DOMAIN", + "WG_CLIENT_RANGE", + "WG_ENDPOINT", + "WG_IP", + "WG_RESERVED_NET", + ] + write_manifest(Path(args.output), manifest) + + +def nginx(args: argparse.Namespace) -> None: + """Render a digest-pinned nginx application CVM.""" + docker_compose = f"""services: + web: + image: {args.image_ref} + ports: + - "80:80" +""" + manifest = common_manifest( + args.name, + docker_compose, + image_loader(args.artifact_port, args.image_archive), + ) + manifest.update( + { + "gateway_enabled": True, + "kms_enabled": True, + "no_instance_id": False, + "secure_time": False, + } + ) + if args.attestation_mode != "auto": + # A string v3 manifest makes old guests fail closed instead of silently + # ignoring the requested TDX quote mode. + manifest["manifest_version"] = "3" + manifest["requirements"] = { + "tdx_measure_acpi_tables": args.attestation_mode == "legacy" + } + write_manifest(Path(args.output), manifest) + + +def add_image_args(parser: argparse.ArgumentParser) -> None: + """Add the content-addressed container-image arguments shared by apps.""" + parser.add_argument("--image-ref", required=True) + parser.add_argument("--image-archive", required=True) + parser.add_argument("--artifact-port", required=True, type=int) + parser.add_argument("--name", required=True) + parser.add_argument("--output", required=True) + + +def main() -> None: + """Parse command-line arguments and render a manifest.""" + parser = argparse.ArgumentParser() + sub = parser.add_subparsers(required=True) + + p = sub.add_parser("kms") + add_image_args(p) + p.add_argument("--auth-port", required=True, type=int) + p.set_defaults(func=kms) + + p = sub.add_parser("gateway") + add_image_args(p) + p.set_defaults(func=gateway) + + p = sub.add_parser("nginx") + add_image_args(p) + p.add_argument( + "--attestation-mode", + choices=("auto", "legacy", "lite"), + default="auto", + ) + p.set_defaults(func=nginx) + + args = parser.parse_args() + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/test-suites/full-stack-compose/scripts/network-probe.sh b/test-suites/full-stack-compose/scripts/network-probe.sh new file mode 100755 index 000000000..9e0387bc7 --- /dev/null +++ b/test-suites/full-stack-compose/scripts/network-probe.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail + +STATE_DIR=${DSTACK_E2E_STATE_DIR:-/suite-state} +WORK_DIR=${DSTACK_E2E_WORK_DIR:-$STATE_DIR/work} +# shellcheck disable=SC1091 +source "$STATE_DIR/state.env" + +ready="$WORK_DIR/network-probe.ready" +stop="$WORK_DIR/network-probe.stop" +result="$WORK_DIR/network-probe.result.json" +attempts=0 +failures=0 +failovers=0 +cycle=0 +started_ns=$(date +%s%N) + +probe_endpoint() { + local label=$1 instance=$2 port=$3 sni + sni="${instance}-80.${BASE_DOMAIN}" + curl -kfsS --connect-timeout 1 --max-time 2 \ + --connect-to "${sni}:${port}:127.0.0.1:${port}" \ + "https://${sni}:${port}/" 2>/dev/null \ + | grep -qi 'welcome to nginx' +} + +probe_ha() { + local label=$1 instance=$2 first second + if (( cycle % 2 == 0 )); then + first=$GATEWAY1_PROXY_HOST_PORT + second=$GATEWAY2_PROXY_HOST_PORT + else + first=$GATEWAY2_PROXY_HOST_PORT + second=$GATEWAY1_PROXY_HOST_PORT + fi + attempts=$((attempts + 1)) + if probe_endpoint "$label" "$instance" "$first"; then + return 0 + fi + if probe_endpoint "$label" "$instance" "$second"; then + failovers=$((failovers + 1)) + return 0 + fi + failures=$((failures + 1)) + printf '%(%FT%T%z)T route unavailable label=%s preferred=%s fallback=%s\n' \ + -1 "$label" "$first" "$second" >&2 + return 1 +} + +declare -a labels=(legacy lite) +declare -A instances +for label in "${labels[@]}"; do + instances[$label]=$(jq -r '.instance_id // ""' "$WORK_DIR/${label}.info.json") + [[ -n "${instances[$label]}" ]] || { + echo "missing instance id for $label" >&2 + exit 1 + } +done + +# Warm up against both physical Gateway nodes before declaring the HA probe live. +for label in "${labels[@]}"; do + for port in "$GATEWAY1_PROXY_HOST_PORT" "$GATEWAY2_PROXY_HOST_PORT"; do + probe_endpoint "$label" "${instances[$label]}" "$port" || { + echo "warmup failed: $label via Gateway proxy port $port" >&2 + exit 1 + } + done +done +touch "$ready" + +while [[ ! -e "$stop" ]]; do + cycle=$((cycle + 1)) + for label in "${labels[@]}"; do + probe_ha "$label" "${instances[$label]}" || true + done + sleep 0.05 +done + +ended_ns=$(date +%s%N) +duration_ms=$(((ended_ns - started_ns) / 1000000)) +jq -n \ + --argjson duration_ms "$duration_ms" \ + --argjson attempts "$attempts" \ + --argjson failures "$failures" \ + --argjson failovers "$failovers" \ + '{duration_ms:$duration_ms,attempts:$attempts,failures:$failures,successful_failovers:$failovers}' \ + | tee "$result" + +(( failures == 0 )) diff --git a/test-suites/full-stack-compose/scripts/render-config.sh b/test-suites/full-stack-compose/scripts/render-config.sh new file mode 100755 index 000000000..500b61c4b --- /dev/null +++ b/test-suites/full-stack-compose/scripts/render-config.sh @@ -0,0 +1,275 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail + +STATE_DIR=${DSTACK_E2E_STATE_DIR:-/suite-state} +CONFIG_DIR=${DSTACK_E2E_CONFIG_DIR:-$STATE_DIR/config} +RUN_DIR=${DSTACK_E2E_RUN_DIR:-$STATE_DIR/run} +VM_DIR=${DSTACK_E2E_VM_DIR:-$STATE_DIR/vm} +VOLUMES_DIR=${DSTACK_E2E_VOLUMES_DIR:-$STATE_DIR/volumes} +ARTIFACT_DIR=${DSTACK_E2E_ARTIFACT_DIR:-$STATE_DIR/artifacts} +IMAGE_ROOT=${DSTACK_E2E_IMAGE_ROOT:-/images} +IMAGE_NAME=${DSTACK_E2E_IMAGE_NAME:-dstack-0.6.0} +PLATFORM=${DSTACK_E2E_PLATFORM:-tdx} + +VMM_PORT=${DSTACK_E2E_VMM_PORT:-29080} +AUTH_PORT=${DSTACK_E2E_AUTH_PORT:-28011} +ARTIFACT_PORT=${DSTACK_E2E_ARTIFACT_PORT:-38081} +KMS_OLD_HOST_PORT=${DSTACK_E2E_KMS_OLD_HOST_PORT:-28082} +KMS_LATEST_HOST_PORT=${DSTACK_E2E_KMS_LATEST_HOST_PORT:-28083} +KMS_RPC_DOMAIN=${DSTACK_E2E_KMS_RPC_DOMAIN:-} +GATEWAY1_RPC_HOST_PORT=${DSTACK_E2E_GATEWAY1_RPC_HOST_PORT:-28000} +GATEWAY1_ADMIN_HOST_PORT=${DSTACK_E2E_GATEWAY1_ADMIN_HOST_PORT:-28001} +GATEWAY1_PROXY_HOST_PORT=${DSTACK_E2E_GATEWAY1_PROXY_HOST_PORT:-28443} +GATEWAY1_WG_HOST_PORT=${DSTACK_E2E_GATEWAY1_WG_HOST_PORT:-28120} +GATEWAY2_RPC_HOST_PORT=${DSTACK_E2E_GATEWAY2_RPC_HOST_PORT:-28100} +GATEWAY2_ADMIN_HOST_PORT=${DSTACK_E2E_GATEWAY2_ADMIN_HOST_PORT:-28101} +GATEWAY2_PROXY_HOST_PORT=${DSTACK_E2E_GATEWAY2_PROXY_HOST_PORT:-28543} +GATEWAY2_WG_HOST_PORT=${DSTACK_E2E_GATEWAY2_WG_HOST_PORT:-28121} +GATEWAY_APP_ID=${DSTACK_E2E_GATEWAY_APP_ID:-$(printf dstack-e2e-gateway | sha256sum | cut -c1-40)} +KEY_PROVIDER_PORT=${DSTACK_E2E_KEY_PROVIDER_PORT:-13443} +HOST_API_PORT=${DSTACK_E2E_HOST_API_PORT:-20011} +CID_START=${DSTACK_E2E_CID_START:-15000} +QGS_PORT=${DSTACK_E2E_QGS_PORT:-4050} +QEMU_PATH=${DSTACK_E2E_QEMU_PATH:-/usr/bin/qemu-system-x86_64} +BASE_DOMAIN=${DSTACK_E2E_BASE_DOMAIN:-e2e.test} +MOCK_CF_HTTP_PORT=${DSTACK_E2E_MOCK_CF_HTTP_PORT:-38080} +PEBBLE_HTTP_PORT=${DSTACK_E2E_PEBBLE_HTTP_PORT:-34000} +APP_NAME=${DSTACK_E2E_APP_NAME:-dstack-e2e-nginx} +SUITE_PREFIX=${DSTACK_E2E_NAME_PREFIX:-dstack-e2e} + +mkdir -p \ + "$CONFIG_DIR" "$RUN_DIR" "$VM_DIR" "$VOLUMES_DIR" \ + "$ARTIFACT_DIR/images" "$ARTIFACT_DIR/os" "$ARTIFACT_DIR/control" \ + "$STATE_DIR/work" +chmod 0777 "$STATE_DIR/work" "$ARTIFACT_DIR" "$ARTIFACT_DIR/images" "$ARTIFACT_DIR/control" + +if [[ ! "$GATEWAY_APP_ID" =~ ^[0-9a-fA-F]{40}$ ]]; then + echo "ERROR: DSTACK_E2E_GATEWAY_APP_ID must be an exact 20-byte hex app id" >&2 + exit 1 +fi +GATEWAY_APP_ID=${GATEWAY_APP_ID,,} + +if [[ -z "$KMS_RPC_DOMAIN" ]]; then + host_ip=$(ip -4 route get 1.1.1.1 2>/dev/null \ + | sed -n 's/.* src \([^ ]*\).*/\1/p' | head -n1) + if [[ ! "$host_ip" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "ERROR: unable to derive a host IPv4 address; set DSTACK_E2E_KMS_RPC_DOMAIN" >&2 + exit 1 + fi + KMS_RPC_DOMAIN="${host_ip}.nip.io" +fi +if ! getent ahostsv4 "$KMS_RPC_DOMAIN" >/dev/null 2>&1; then + echo "ERROR: DSTACK_E2E_KMS_RPC_DOMAIN does not resolve: $KMS_RPC_DOMAIN" >&2 + exit 1 +fi + +image_dir="$IMAGE_ROOT/$IMAGE_NAME" +if [[ ! -d "$image_dir" ]]; then + echo "ERROR: image directory not found: $image_dir" >&2 + echo "Set DSTACK_E2E_IMAGE_STORE to a host directory containing $IMAGE_NAME/" >&2 + exit 1 +fi + +digest_file=digest.txt +if [[ "$PLATFORM" == "amd-sev-snp" || "$PLATFORM" == "sev-snp" || "$PLATFORM" == "snp" ]]; then + PLATFORM=amd-sev-snp + digest_file=digest.sev.txt +elif [[ "$PLATFORM" != tdx ]]; then + echo "ERROR: unsupported DSTACK_E2E_PLATFORM=$PLATFORM (expected tdx or amd-sev-snp)" >&2 + exit 1 +fi + +if [[ ! -s "$image_dir/$digest_file" ]]; then + echo "ERROR: missing $image_dir/$digest_file; production-compatible E2E never permits an unpinned OS image" >&2 + exit 1 +fi +OS_IMAGE_HASH=$(tr -d '[:space:]' < "$image_dir/$digest_file") +if [[ ! "$OS_IMAGE_HASH" =~ ^[0-9a-f]{64}$ ]]; then + echo "ERROR: invalid OS image digest: $OS_IMAGE_HASH" >&2 + exit 1 +fi + +# Build the exact flat archive consumed by the KMS verifier. Verify the source +# first and include only the measured files named by sha256sum.txt. +( + cd "$image_dir" + sha256sum -c sha256sum.txt + mapfile -t measured_files < <(awk '{print $2}' sha256sum.txt) + for file in "${measured_files[@]}"; do + [[ "$file" != */* && -f "$file" ]] || { + echo "ERROR: unsafe or missing measured image file: $file" >&2 + exit 1 + } + done + tar -czf "$ARTIFACT_DIR/os/mr_${OS_IMAGE_HASH}.tar.gz.tmp" \ + sha256sum.txt "${measured_files[@]}" +) +mv "$ARTIFACT_DIR/os/mr_${OS_IMAGE_HASH}.tar.gz.tmp" \ + "$ARTIFACT_DIR/os/mr_${OS_IMAGE_HASH}.tar.gz" + +token_file="$STATE_DIR/gateway-admin-token" +if [[ ! -s "$token_file" ]]; then + (umask 077; openssl rand -hex 24 > "$token_file") +fi +GATEWAY_ADMIN_TOKEN=$(tr -d '[:space:]' < "$token_file") + +cat > "$CONFIG_DIR/auth-allowlist.json" < "$CONFIG_DIR/vmm.toml" < "$STATE_DIR/state.env" < +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail + +STATE_DIR=${DSTACK_E2E_STATE_DIR:-/suite-state} +CONFIG_DIR=${DSTACK_E2E_CONFIG_DIR:-$STATE_DIR/config} +WORK_DIR=${DSTACK_E2E_WORK_DIR:-$STATE_DIR/work} +VM_DIR=${DSTACK_E2E_VM_DIR:-$STATE_DIR/vm} +PHASE=${DSTACK_E2E_PHASE:-upgrade} +CURRENT_VERSION=${DSTACK_E2E_CURRENT_VERSION:?missing current workspace version} +CURRENT_REV=${DSTACK_E2E_CURRENT_REV:?missing current workspace revision} +mkdir -p "$WORK_DIR" +# shellcheck disable=SC1091 +source "$STATE_DIR/state.env" +# shellcheck disable=SC1091 +source "$STATE_DIR/artifacts/images.env" + +VMM_URL="http://127.0.0.1:${VMM_PORT}" +VMM_CLI=(python3 /workspace/vmm/src/vmm-cli.py --url "$VMM_URL") +DSTACK_CLI=(/workspace/target/release/dstack --host "$VMM_URL") +ALLOWLIST="$CONFIG_DIR/auth-allowlist.json" +KMS_OLD_URL="https://${KMS_RPC_DOMAIN}:${KMS_OLD_HOST_PORT}" +KMS_LATEST_URL="https://${KMS_RPC_DOMAIN}:${KMS_LATEST_HOST_PORT}" +GATEWAY1_URL="https://${KMS_RPC_DOMAIN}:${GATEWAY1_RPC_HOST_PORT}" +GATEWAY2_URL="https://${KMS_RPC_DOMAIN}:${GATEWAY2_RPC_HOST_PORT}" +probe_pid="" +ATTESTED_DEVICE_ID="" + +log() { printf '[%(%H:%M:%S)T] %s\n' -1 "$*"; } +die() { log "ERROR: $*" >&2; exit 1; } + +need_bin() { + [[ -x "$1" ]] || die "missing executable $1" +} + +wait_vmm() { + log "waiting for VMM at $VMM_URL" + for _ in $(seq 1 120); do + if "${DSTACK_CLI[@]}" apps -j >/dev/null 2>&1; then + log "VMM ready" + return + fi + sleep 2 + done + die "VMM not ready" +} + +vm_ids_by_prefix() { + local prefix=$1 + "${VMM_CLI[@]}" lsvm --json 2>/dev/null \ + | jq -r --arg p "$prefix" '.[] | select(.name | startswith($p)) | .id' +} + +remove_vm() { + local id=$1 + [[ -n "$id" ]] || return 0 + log "removing VM $id" + "${VMM_CLI[@]}" remove "$id" >/dev/null 2>&1 || true +} + +clean_start() { + [[ "${DSTACK_E2E_CLEAN_START:-true}" == true ]] || return 0 + while read -r id; do + remove_vm "$id" + done < <(vm_ids_by_prefix "$SUITE_PREFIX") + for _ in $(seq 1 90); do + [[ -z "$(vm_ids_by_prefix "$SUITE_PREFIX")" ]] && return 0 + sleep 2 + done + die "stale VMs were not removed" +} + +print_vm_info_safe() { + jq '{id,name,status,uptime,app_id,instance_id,boot_progress,boot_error,image_version,events}' <<<"$1" +} + +wait_boot_done() { + local id=$1 label=$2 timeout=${3:-720} + local deadline=$((SECONDS + timeout)) last="" + while (( SECONDS < deadline )); do + local info status progress error line + info=$("${VMM_CLI[@]}" info "$id" --json 2>/dev/null || true) + if [[ -n "$info" ]]; then + status=$(jq -r '.status // ""' <<<"$info") + progress=$(jq -r '.boot_progress // ""' <<<"$info") + error=$(jq -r '.boot_error // ""' <<<"$info") + line="status=$status progress=${progress:-none} error=${error:-none}" + if [[ "$line" != "$last" ]]; then + log "$label: $line" + last=$line + fi + if [[ -n "$error" && "$error" != null ]]; then + print_vm_info_safe "$info" >&2 || true + "${VMM_CLI[@]}" logs "$id" -n 240 >&2 || true + die "$label boot failed: $error" + fi + if [[ "$status" == running && "$progress" == "done" ]]; then + printf '%s' "$info" > "$WORK_DIR/${label}.info.json" + return + fi + if [[ "$status" == exited || "$status" == stopped ]]; then + print_vm_info_safe "$info" >&2 || true + "${VMM_CLI[@]}" logs "$id" -n 240 >&2 || true + die "$label exited before boot finished" + fi + fi + sleep 5 + done + "${VMM_CLI[@]}" logs "$id" -n 300 >&2 || true + die "timed out waiting for $label" +} + +wait_vm_stopped() { + local id=$1 deadline=$((SECONDS + 180)) status + while (( SECONDS < deadline )); do + status=$("${VMM_CLI[@]}" info "$id" --json 2>/dev/null | jq -r '.status // ""' || true) + [[ "$status" != running && "$status" != starting ]] && return + sleep 2 + done + die "VM $id did not stop" +} + +stop_vm() { + local id=$1 label=$2 + log "stopping $label ($id)" + "${VMM_CLI[@]}" stop "$id" --force >/dev/null + wait_vm_stopped "$id" +} + +start_vm() { + local id=$1 label=$2 + log "starting $label ($id)" + "${VMM_CLI[@]}" start "$id" >/dev/null + wait_boot_done "$id" "$label" +} + +register_app() { + local app_id=$1 hash=$2 + [[ -n "$ATTESTED_DEVICE_ID" ]] || die "cannot authorize app before pinning the attested TDX device" + /suite/scripts/allowlist.py add-app --path "$ALLOWLIST" \ + --app-id "$app_id" --compose-hash "$hash" \ + --device-id "$ATTESTED_DEVICE_ID" >/dev/null +} + +register_kms_measurement() { + local measurement=$1 device_id=$2 + /suite/scripts/allowlist.py add-kms --path "$ALLOWLIST" \ + --mr-aggregated "$measurement" --device-id "$device_id" >/dev/null +} + +render_kms() { + local stage=$1 image_ref=$2 archive=$3 meta + meta=$(/suite/scripts/app_compose.py kms \ + --name "dstack-e2e-kms-${stage}" \ + --image-ref "$image_ref" \ + --image-archive "$archive" \ + --artifact-port "$ARTIFACT_PORT" \ + --auth-port "$AUTH_PORT" \ + --output "$WORK_DIR/kms-${stage}.app-compose.json") + printf '%s\n' "$meta" > "$WORK_DIR/kms-${stage}.manifest-meta.json" +} + +deploy_kms_onboard() { + local stage=$1 port=$2 out vm_id + log "deploying $stage KMS CVM in Local-Key-Provider mode" + if ! out=$("${VMM_CLI[@]}" deploy \ + --name "${SUITE_PREFIX}-kms-${stage}" \ + --image "$IMAGE_NAME" \ + --compose "$WORK_DIR/kms-${stage}.app-compose.json" \ + --port "tcp:0.0.0.0:${port}:8000" \ + --vcpu "${DSTACK_E2E_KMS_VCPU:-4}" \ + --memory "${DSTACK_E2E_KMS_MEMORY:-4096}" \ + --disk "${DSTACK_E2E_KMS_DISK:-30}" 2>&1); then + die "failed to deploy $stage KMS CVM: $out" + fi + printf '%s\n' "$out" | tee "$WORK_DIR/kms-${stage}.deploy.log" + vm_id=$(sed -n 's/^Created VM with ID: //p' <<<"$out" | tail -n1) + [[ -n "$vm_id" ]] || die "failed to parse $stage KMS VM id" + printf '%s' "$vm_id" > "$WORK_DIR/kms-${stage}.vm_id" + wait_boot_done "$vm_id" "kms-${stage}" +} + +wait_onboard() { + local stage=$1 port=$2 deadline=$((SECONDS + 300)) url + url="http://127.0.0.1:${port}/prpc/Onboard.GetAttestationInfo?json" + log "waiting for $stage KMS quote-enabled onboarding endpoint" + while (( SECONDS < deadline )); do + if curl -fsS "$url" -o "$WORK_DIR/kms-${stage}.attestation-info.json"; then + jq -e '((.mr_aggregated // .mrAggregated // "") | length) > 0 and ((.device_id // .deviceId // "") | length) > 0' \ + "$WORK_DIR/kms-${stage}.attestation-info.json" >/dev/null && return + fi + sleep 3 + done + "${VMM_CLI[@]}" logs "$(cat "$WORK_DIR/kms-${stage}.vm_id")" -n 300 >&2 || true + die "$stage KMS onboarding endpoint not ready" +} + +authorize_kms_from_attestation() { + local stage=$1 measurement device_id + measurement=$(jq -r '.mr_aggregated // .mrAggregated' "$WORK_DIR/kms-${stage}.attestation-info.json") + device_id=$(jq -r '.device_id // .deviceId' "$WORK_DIR/kms-${stage}.attestation-info.json") + [[ "$measurement" =~ ^[0-9a-fA-F]+$ ]] || die "invalid $stage KMS mrAggregated" + [[ "$device_id" =~ ^[0-9a-fA-F]{64}$ ]] || die "invalid $stage KMS device ID" + if [[ -n "$ATTESTED_DEVICE_ID" && "$device_id" != "$ATTESTED_DEVICE_ID" ]]; then + die "$stage KMS quote came from unexpected TDX device $device_id" + fi + ATTESTED_DEVICE_ID=$device_id + register_kms_measurement "$measurement" "$device_id" + log "authorized exact $stage KMS mrAggregated=${measurement} on device=${device_id}" +} + +onboard_rpc() { + local port=$1 method=$2 data=$3 out=$4 + curl -fsS -X POST -H 'Content-Type: application/json' \ + "http://127.0.0.1:${port}/prpc/Onboard.${method}?json" \ + --data-raw "$data" -o "$out" + jq -e 'has("error") | not' "$out" >/dev/null \ + || die "Onboard.${method} returned an error: $(cat "$out")" + log "Onboard.${method} completed" +} + +finish_onboarding() { + local stage=$1 port=$2 + curl -fsS "http://127.0.0.1:${port}/finish" > "$WORK_DIR/kms-${stage}.finish.txt" + wait_kms_tls "$stage" "$port" +} + +wait_kms_tls() { + local stage=$1 port=$2 deadline=$((SECONDS + 300)) + log "waiting for $stage KMS TLS service" + while (( SECONDS < deadline )); do + if curl -kfsS "https://127.0.0.1:${port}/prpc/GetMeta?json" \ + -o "$WORK_DIR/kms-${stage}.ready-meta.json"; then + log "$stage KMS TLS service ready" + return + fi + sleep 3 + done + "${VMM_CLI[@]}" logs "$(cat "$WORK_DIR/kms-${stage}.vm_id")" -n 300 >&2 || true + die "$stage KMS TLS service not ready" +} + +kms_rpc_get() { + local port=$1 method=$2 + curl -kfsS "https://127.0.0.1:${port}/prpc/${method}?json" +} + +kms_rpc_post() { + local port=$1 method=$2 data=$3 + curl -kfsS -X POST -H 'Content-Type: application/json' \ + "https://127.0.0.1:${port}/prpc/${method}?json" --data-raw "$data" +} + +capture_kms_identity() { + local stage=$1 port=$2 app_id=$3 ca_pem ca_spki k256_pubkey + ca_pem="$WORK_DIR/kms-${stage}.ca.pem" + kms_rpc_get "$port" GetMeta | tee "$WORK_DIR/kms-${stage}.meta.raw.json" \ + | jq -S '{ca_cert, k256_pubkey}' > "$WORK_DIR/kms-${stage}.meta.json" + jq -er '.ca_cert' "$WORK_DIR/kms-${stage}.meta.json" > "$ca_pem" + openssl x509 -in "$ca_pem" -noout -checkend 0 >/dev/null + ca_spki=$(openssl x509 -in "$ca_pem" -pubkey -noout \ + | openssl pkey -pubin -outform DER \ + | openssl dgst -sha256 \ + | awk '{print $NF}') + k256_pubkey=$(jq -er '.k256_pubkey' "$WORK_DIR/kms-${stage}.meta.json") + jq -nS --arg ca_spki_sha256 "$ca_spki" --arg k256_pubkey "$k256_pubkey" \ + '{ca_spki_sha256:$ca_spki_sha256,k256_pubkey:$k256_pubkey}' \ + > "$WORK_DIR/kms-${stage}.identity.json" + kms_rpc_post "$port" GetAppEnvEncryptPubKey \ + "$(jq -cn --arg id "$app_id" '{app_id:$id}')" \ + | tee "$WORK_DIR/kms-${stage}.app-key.raw.json" \ + | jq -S '{public_key}' > "$WORK_DIR/kms-${stage}.app-key.json" + jq -e '.ca_spki_sha256 != "" and .k256_pubkey != ""' \ + "$WORK_DIR/kms-${stage}.identity.json" >/dev/null + jq -e '.public_key != ""' "$WORK_DIR/kms-${stage}.app-key.json" >/dev/null +} + +assert_kms_identity_unchanged() { + # Onboarding deliberately reissues the self-signed CA certificate with a new + # validity end time. The transferable identity is its private key/SPKI, not + # the DER bytes of that freshly issued certificate. + diff -u "$WORK_DIR/kms-old.identity.json" "$WORK_DIR/kms-latest.identity.json" \ + || die "KMS CA key or root k256 key changed during 0.5.8 -> current onboarding" + diff -u "$WORK_DIR/kms-old.app-key.json" "$WORK_DIR/kms-latest.app-key.json" \ + || die "per-app environment encryption key changed during KMS upgrade" + log "KMS CA SPKI, root k256 key and per-app environment key are unchanged" +} + +render_gateway_manifests() { + local stage=$1 image_ref=$2 archive=$3 meta + meta=$(/suite/scripts/app_compose.py gateway \ + --name dstack-e2e-gateway \ + --image-ref "$image_ref" \ + --image-archive "$archive" \ + --artifact-port "$ARTIFACT_PORT" \ + --output "$WORK_DIR/gateway-${stage}.app-compose.json") + printf '%s\n' "$meta" > "$WORK_DIR/gateway-${stage}.manifest-meta.json" + register_app "$GATEWAY_APP_ID" "$(jq -r .composeHash <<<"$meta")" +} + +write_gateway_env() { + local node=$1 rpc_port wg_port third_octet bootnode + if [[ "$node" == 1 ]]; then + rpc_port=$GATEWAY1_RPC_HOST_PORT + wg_port=$GATEWAY1_WG_HOST_PORT + third_octet=0 + bootnode="" + else + rpc_port=$GATEWAY2_RPC_HOST_PORT + wg_port=$GATEWAY2_WG_HOST_PORT + third_octet=64 + bootnode="$GATEWAY1_URL" + fi + cat > "$WORK_DIR/gateway${node}.env" <&1); then + die "failed to deploy Gateway node $node ($stage): $out" + fi + printf '%s\n' "$out" | tee "$WORK_DIR/gateway${node}.${stage}.deploy.log" + vm_id=$(sed -n 's/^Created VM with ID: //p' <<<"$out" | tail -n1) + [[ -n "$vm_id" ]] || die "failed to parse Gateway node $node VM id" + printf '%s' "$vm_id" > "$WORK_DIR/gateway${node}.vm_id" + wait_boot_done "$vm_id" "gateway${node}-${stage}" + wait_gateway "$node" +} + +gateway_version() { + local node=$1 stage=$2 rpc _ + read -r rpc _ < <(gateway_ports "$node") + curl -kfsS -D "$WORK_DIR/gateway${node}-${stage}.headers" -o /dev/null \ + "https://127.0.0.1:${rpc}/prpc/Info?json" + tr -d '\r' < "$WORK_DIR/gateway${node}-${stage}.headers" \ + | awk 'tolower($1)=="x-app-version:" {$1=""; sub(/^ /,""); print}' \ + > "$WORK_DIR/gateway${node}-${stage}.version.txt" + [[ -s "$WORK_DIR/gateway${node}-${stage}.version.txt" ]] \ + || die "Gateway node $node did not return X-App-Version" + if [[ "$stage" == old ]]; then + grep -E '^v0\.5\.8 \(git:' "$WORK_DIR/gateway${node}-${stage}.version.txt" >/dev/null \ + || die "Gateway node $node did not run released v0.5.8" + else + grep -E "^v${CURRENT_VERSION//./\\.} \\(git:" \ + "$WORK_DIR/gateway${node}-${stage}.version.txt" >/dev/null \ + || die "Gateway node $node did not run current v$CURRENT_VERSION" + grep -F "$CURRENT_REV" "$WORK_DIR/gateway${node}-${stage}.version.txt" >/dev/null \ + || die "Gateway node $node did not run current revision $CURRENT_REV" + fi + log "Gateway node $node $stage: $(cat "$WORK_DIR/gateway${node}-${stage}.version.txt")" +} + +admin_curl() { + local node=$1 method=$2 data=${3:-'{}'} admin out code + read -r _ admin _ < <(gateway_ports "$node") + out=$(mktemp) + code=$(curl -sS -o "$out" -w '%{http_code}' -X POST \ + -H "Authorization: Bearer ${GATEWAY_ADMIN_TOKEN}" \ + -H 'Content-Type: application/json' \ + "http://127.0.0.1:${admin}/prpc/Admin.${method}?json" \ + --data-raw "$data" || true) + if [[ "$code" =~ ^2 ]]; then + cat "$out" + rm -f "$out" + return + fi + log "Gateway $node Admin.${method} failed HTTP $code: $(cat "$out")" >&2 + rm -f "$out" + return 1 +} + +wait_gateway() { + local node=$1 rpc deadline=$((SECONDS + 300)) + read -r rpc _ < <(gateway_ports "$node") + while (( SECONDS < deadline )); do + if curl -kfsS "https://127.0.0.1:${rpc}/prpc/Info?json" >/dev/null 2>&1 \ + && admin_curl "$node" Status >/dev/null 2>&1; then + log "Gateway node $node ready" + return + fi + sleep 3 + done + "${VMM_CLI[@]}" logs "$(cat "$WORK_DIR/gateway${node}.vm_id")" -n 300 >&2 || true + die "Gateway node $node not ready" +} + +bootstrap_gateway() { + log "configuring Gateway cluster through node 1" + admin_curl 1 SetCertbotConfig \ + "$(jq -cn --arg u "http://10.0.2.2:${PEBBLE_HTTP_PORT}/dir" \ + '{acme_url:$u, renew_before_expiration_secs:3600}')" >/dev/null + admin_curl 1 CreateDnsCredential \ + "$(jq -cn --arg u "http://10.0.2.2:${MOCK_CF_HTTP_PORT}/client/v4" \ + '{name:"mock-cloudflare",provider_type:"cloudflare",cf_api_token:"test-token",cf_api_url:$u,set_as_default:true,dns_txt_ttl:1,max_dns_wait:0}')" \ + >/dev/null + admin_curl 1 AddZtDomain \ + "$(jq -cn --arg d "$BASE_DOMAIN" '{domain:$d,port:443,priority:100}')" \ + >/dev/null + admin_curl 1 RenewZtDomainCert \ + "$(jq -cn --arg d "$BASE_DOMAIN" '{domain:$d,force:true}')" \ + | tee "$WORK_DIR/gateway-renew-cert.json" + wait_gateway_cert 1 + wait_gateway_cluster_sync + wait_gateway_cert 2 +} + +wait_gateway_cluster_sync() { + local deadline=$((SECONDS + 240)) domains certbot + log "waiting for Gateway WaveKV config/certificate sync to node 2" + while (( SECONDS < deadline )); do + domains=$(admin_curl 2 ListZtDomains 2>/dev/null || true) + certbot=$(admin_curl 2 GetCertbotConfig 2>/dev/null || true) + if jq -e --arg d "$BASE_DOMAIN" '.domains[]? | (.domain // .config.domain) == $d' \ + <<<"$domains" >/dev/null 2>&1 \ + && jq -e '.acme_url != ""' <<<"$certbot" >/dev/null 2>&1; then + log "Gateway cluster state synced" + return + fi + sleep 3 + done + die "Gateway node 2 did not receive synced cluster state" +} + +wait_gateway_cert() { + local node=$1 proxy deadline=$((SECONDS + 300)) sni="gateway.${BASE_DOMAIN}" + read -r _ _ proxy _ < <(gateway_ports "$node") + while (( SECONDS < deadline )); do + if echo | timeout 8 openssl s_client -connect "127.0.0.1:${proxy}" -servername "$sni" 2>/dev/null \ + | openssl x509 -noout -ext subjectAltName 2>/dev/null \ + | grep -Fq "*.${BASE_DOMAIN}"; then + log "Gateway node $node wildcard certificate active" + return + fi + sleep 3 + done + die "Gateway node $node wildcard certificate not active" +} + +gateway_cert_fingerprint() { + local node=$1 proxy sni="gateway.${BASE_DOMAIN}" + read -r _ _ proxy _ < <(gateway_ports "$node") + echo | timeout 8 openssl s_client -connect "127.0.0.1:${proxy}" -servername "$sni" 2>/dev/null \ + | openssl x509 -noout -fingerprint -sha256 2>/dev/null +} + +wait_gateway_persisted() { + local node=$1 deadline=$((SECONDS + 120)) status + while (( SECONDS < deadline )); do + status=$(admin_curl "$node" WaveKvStatus 2>/dev/null || true) + if jq -e '.persistent.dirty == false and .persistent.n_keys > 0' <<<"$status" >/dev/null 2>&1; then + return + fi + sleep 2 + done + die "Gateway node $node persistent store did not flush" +} + +capture_gateway_state() { + local node=$1 stage=$2 + wait_gateway_persisted "$node" + admin_curl "$node" Status | tee "$WORK_DIR/gateway${node}-${stage}.status.raw.json" \ + | jq -S '. as $status | { + id: $status.id, + uuid: ([$status.nodes[] | select(.id == $status.id)][0].uuid), + hosts: ([$status.hosts[] | {instance_id,ip,app_id,base_domain}] | sort_by(.instance_id)) + }' \ + > "$WORK_DIR/gateway${node}-${stage}.status.json" + admin_curl "$node" GetCertbotConfig | jq -S \ + '{acme_url,renew_interval_secs,renew_before_expiration_secs,renew_timeout_secs}' \ + > "$WORK_DIR/gateway${node}-${stage}.certbot.json" + admin_curl "$node" ListDnsCredentials | jq -S \ + '{default_id,credentials:([.credentials[]|{ + id,name,provider_type, + cf_api_token_set: ((.cf_api_token // "") | length > 0), + cf_zone_id,cf_api_url,dns_txt_ttl,max_dns_wait + }]|sort_by(.id))}' \ + > "$WORK_DIR/gateway${node}-${stage}.dns.json" + admin_curl "$node" ListZtDomains | jq -S \ + '{domains:([.domains[]|(.config // .)]|sort_by(.domain))}' \ + > "$WORK_DIR/gateway${node}-${stage}.domains.json" + gateway_cert_fingerprint "$node" > "$WORK_DIR/gateway${node}-${stage}.cert.txt" +} + +assert_gateway_state_unchanged() { + local node=$1 file + capture_gateway_state "$node" after + for file in status certbot dns domains; do + diff -u "$WORK_DIR/gateway${node}-before.${file}.json" \ + "$WORK_DIR/gateway${node}-after.${file}.json" \ + || die "Gateway node $node $file state changed across upgrade" + done + diff -u "$WORK_DIR/gateway${node}-before.cert.txt" "$WORK_DIR/gateway${node}-after.cert.txt" \ + || die "Gateway node $node wildcard certificate changed across upgrade" + log "Gateway node $node durable state is unchanged" +} + +assert_gateway_dns_credential_usable() { + local node=$1 result deadline=$((SECONDS + 90)) + log "proving Gateway node $node retained the Cloudflare credential" + while (( SECONDS < deadline )); do + result=$(admin_curl "$node" RenewZtDomainCert \ + "$(jq -cn --arg d "$BASE_DOMAIN" '{domain:$d,force:true}')") + printf '%s\n' "$result" > "$WORK_DIR/gateway${node}-post-upgrade-renew.json" + if jq -e '.renewed == true and .not_after > 0' <<<"$result" >/dev/null; then + wait_gateway_cert "$node" + return + fi + # WaveKV's distributed certificate lock may still be held by the peer's + # immediately preceding issuance. + sleep 3 + done + die "Gateway node $node could not issue with its persisted DNS credential" +} + +render_app() { + local label=$1 mode=$2 meta app_id hash + meta=$(/suite/scripts/app_compose.py nginx \ + --name "${APP_NAME}-${label}" \ + --image-ref "$APP_IMAGE_ID" \ + --image-archive app.tar \ + --artifact-port "$ARTIFACT_PORT" \ + --attestation-mode "$mode" \ + --output "$WORK_DIR/${label}.app-compose.json") + printf '%s\n' "$meta" > "$WORK_DIR/${label}.manifest-meta.json" + app_id=$(jq -r .appId <<<"$meta") + hash=$(jq -r .composeHash <<<"$meta") + printf '%s' "$app_id" > "$WORK_DIR/${label}.app_id" + register_app "$app_id" "$hash" +} + +deploy_app() { + local label=$1 mode=$2 kms_url=$3 out vm_id + render_app "$label" "$mode" + log "deploying $label app CVM (forced TDX $mode)" + if ! out=$("${VMM_CLI[@]}" deploy \ + --name "${SUITE_PREFIX}-${label}" \ + --image "$IMAGE_NAME" \ + --compose "$WORK_DIR/${label}.app-compose.json" \ + --kms-url "$kms_url" \ + --gateway-url "$GATEWAY1_URL" \ + --gateway-url "$GATEWAY2_URL" \ + --vcpu "${DSTACK_E2E_APP_VCPU:-2}" \ + --memory "${DSTACK_E2E_APP_MEMORY:-2048}" \ + --disk "${DSTACK_E2E_APP_DISK:-20}" 2>&1); then + die "failed to deploy $label app CVM: $out" + fi + printf '%s\n' "$out" | tee "$WORK_DIR/${label}.deploy.log" + vm_id=$(sed -n 's/^Created VM with ID: //p' <<<"$out" | tail -n1) + [[ -n "$vm_id" ]] || die "failed to parse $label VM id" + printf '%s' "$vm_id" > "$WORK_DIR/${label}.vm_id" + wait_boot_done "$vm_id" "$label" + assert_attestation_mode "$label" "$mode" +} + +assert_attestation_mode() { + local label=$1 expected=$2 id sys_config actual + id=$(cat "$WORK_DIR/${label}.vm_id") + sys_config="$VM_DIR/$id/shared/.sys-config.json" + [[ -s "$sys_config" ]] || die "missing $label sys config" + actual=$(jq -r '.vm_config | fromjson | .tdx_attestation_variant // "legacy"' "$sys_config") + [[ "$actual" == "$expected" ]] || die "$label mode=$actual, expected $expected" + jq -e '(.kms_urls | length) > 0 and (.gateway_urls | length) == 2' "$sys_config" >/dev/null + log "$label completed boot-time KMS key provisioning in TDX $actual mode" +} + +verify_app_via_gateway() { + local label=$1 node=$2 instance sni proxy url deadline + instance=$(jq -r '.instance_id' "$WORK_DIR/${label}.info.json") + read -r _ _ proxy _ < <(gateway_ports "$node") + sni="${instance}-80.${BASE_DOMAIN}" + url="https://${sni}:${proxy}/" + deadline=$((SECONDS + 300)) + while (( SECONDS < deadline )); do + if curl -kfsS --max-time 5 \ + --connect-to "${sni}:${proxy}:127.0.0.1:${proxy}" "$url" \ + | grep -qi 'welcome to nginx'; then + log "$label reachable through Gateway node $node" + return + fi + sleep 3 + done + die "$label not reachable through Gateway node $node" +} + +verify_all_routes() { + local label node + for label in legacy lite; do + for node in 1 2; do + verify_app_via_gateway "$label" "$node" + done + done +} + +restart_legacy_on_latest_kms() { + local id + id=$(cat "$WORK_DIR/legacy.vm_id") + stop_vm "$id" legacy + "${VMM_CLI[@]}" update "$id" --kms-url "$KMS_LATEST_URL" >/dev/null + start_vm "$id" legacy + assert_attestation_mode legacy legacy +} + +upgrade_gateway_node() { + local node=$1 id + id=$(cat "$WORK_DIR/gateway${node}.vm_id") + stop_vm "$id" "Gateway node $node" + "${VMM_CLI[@]}" update-app-compose "$id" "$WORK_DIR/gateway-latest.app-compose.json" >/dev/null + "${VMM_CLI[@]}" update "$id" --kms-url "$KMS_LATEST_URL" >/dev/null + start_vm "$id" "gateway${node}-latest" + wait_gateway "$node" + wait_gateway_cert "$node" + gateway_version "$node" latest + # The newly restarted node must carry traffic before its peer is touched. + verify_app_via_gateway legacy "$node" + verify_app_via_gateway lite "$node" + assert_gateway_state_unchanged "$node" +} + +start_network_probe() { + rm -f "$WORK_DIR/network-probe."{ready,stop,result.json,log} + /suite/scripts/network-probe.sh > "$WORK_DIR/network-probe.log" 2>&1 & + probe_pid=$! + for _ in $(seq 1 120); do + [[ -e "$WORK_DIR/network-probe.ready" ]] && return + kill -0 "$probe_pid" 2>/dev/null || { + cat "$WORK_DIR/network-probe.log" >&2 + die "HA network probe exited during warmup" + } + sleep 1 + done + die "HA network probe did not become ready" +} + +stop_network_probe() { + touch "$WORK_DIR/network-probe.stop" + if ! wait "$probe_pid"; then + cat "$WORK_DIR/network-probe.log" >&2 + die "CVM traffic had downtime during rolling Gateway upgrade" + fi + probe_pid="" + cat "$WORK_DIR/network-probe.log" + jq -e '.attempts > 0 and .failures == 0 and .successful_failovers > 0' \ + "$WORK_DIR/network-probe.result.json" >/dev/null \ + || die "HA probe reported downtime" +} + +assert_no_insecure_shortcuts() { + log "auditing rendered manifests for production trust settings" + local manifest + local forbidden='quote_enabled[[:space:]]*=[[:space:]]*false|enforce_self_authorization[[:space:]]*=[[:space:]]*false|verify[[:space:]]*=[[:space:]]*false' + if grep -R -E "$forbidden" "$WORK_DIR"/*.app-compose.json; then + die "rendered app manifest contains a forbidden development trust setting" + fi + + for manifest in "$WORK_DIR/kms-old.app-compose.json" \ + "$WORK_DIR/kms-latest.app-compose.json"; do + if ! python3 - "$manifest" <<'PY' +import json +import sys +import tomllib + +manifest = json.load(open(sys.argv[1], encoding="utf-8")) +assert manifest["local_key_provider_enabled"] is True +assert manifest["kms_enabled"] is False + +# Parse the actual kms.toml embedded in the rendered Compose YAML. Checking +# TOML values instead of matching adjacent strings keeps this guard strict +# while allowing comments and blank lines in the production-style config. +marker = "configs:\n kms_config:\n content: |\n" +compose = manifest["docker_compose_file"] +assert compose.count(marker) == 1 +indented_config = compose.split(marker, 1)[1] +config_lines = [] +for line in indented_config.splitlines(): + if line and not line.startswith(" "): + break + config_lines.append(line[6:] if line else "") +config = tomllib.loads("\n".join(config_lines)) + +core = config["core"] +assert core["enforce_self_authorization"] is True +assert core["image"]["verify"] is True +assert core["auth_api"]["type"] == "webhook" +assert core["onboard"]["enabled"] is True +assert core["onboard"]["quote_enabled"] is True +PY + then + die "$manifest does not use quote-attested production KMS settings" + fi + done + + for manifest in "$WORK_DIR/gateway-old.app-compose.json" \ + "$WORK_DIR/gateway-latest.app-compose.json"; do + jq -e '.kms_enabled == true and .local_key_provider_enabled == false' \ + "$manifest" >/dev/null \ + || die "$manifest does not obtain its keys from KMS" + done + + jq -e ' + .kms_enabled == true and .gateway_enabled == true + and .manifest_version == "3" + and .requirements.tdx_measure_acpi_tables == true + ' "$WORK_DIR/legacy.app-compose.json" >/dev/null \ + || die "legacy app manifest did not force the legacy TDX verifier" + jq -e ' + .kms_enabled == true and .gateway_enabled == true + and .manifest_version == "3" + and .requirements.tdx_measure_acpi_tables == false + ' "$WORK_DIR/lite.app-compose.json" >/dev/null \ + || die "lite app manifest did not force the lite TDX verifier" + + jq -e --arg id "$GATEWAY_APP_ID" \ + --arg device "$ATTESTED_DEVICE_ID" \ + '.gatewayAppId == $id and .gatewayAppId != "any" + and (.osImages | length) == 1 + and (.kms.mrAggregated | length) == 2 + and (.kms.allowAnyDevice == false) + and (.kms.devices == [$device]) + and (.apps | length) == 3 + and ([.apps[].allowAnyDevice] | all(. == false)) + and ([.apps[].devices] | all(. == [$device]))' \ + "$ALLOWLIST" >/dev/null +} + +save_vm_logs() { + local name file id + for file in "$WORK_DIR"/*.vm_id; do + [[ -s "$file" ]] || continue + name=$(basename "$file" .vm_id) + id=$(cat "$file") + "${VMM_CLI[@]}" logs "$id" -n 2000 > "$WORK_DIR/${name}.vm.log" 2>&1 || true + done +} + +cleanup_after() { + [[ "${DSTACK_E2E_CLEANUP_AFTER:-false}" == true ]] || return 0 + local file + for file in "$WORK_DIR"/*.vm_id; do + [[ -s "$file" ]] && remove_vm "$(cat "$file")" + done +} + +phase_upgrade() { + [[ "$PLATFORM" == tdx ]] || die "upgrade phase requires TDX" + wait_vmm + clean_start + + render_kms old "$OLD_KMS_IMAGE_ID" kms-0.5.8.tar + deploy_kms_onboard old "$KMS_OLD_HOST_PORT" + wait_onboard old "$KMS_OLD_HOST_PORT" + authorize_kms_from_attestation old + onboard_rpc "$KMS_OLD_HOST_PORT" Bootstrap \ + "$(jq -cn --arg d "$KMS_RPC_DOMAIN" '{domain:$d}')" \ + "$WORK_DIR/kms-old.bootstrap.json" + jq -e '(.attestation // "") | length > 0' "$WORK_DIR/kms-old.bootstrap.json" >/dev/null \ + || die "KMS 0.5.8 bootstrap did not return quote-bound attestation" + log "KMS 0.5.8 bootstrap returned quote-bound attestation" + finish_onboarding old "$KMS_OLD_HOST_PORT" + + render_gateway_manifests old "$OLD_GATEWAY_IMAGE_ID" gateway-0.5.8.tar + render_gateway_manifests latest "$CURRENT_GATEWAY_IMAGE_ID" gateway-current.tar + write_gateway_env 1 + write_gateway_env 2 + deploy_gateway 1 old "$KMS_OLD_URL" + gateway_version 1 old + deploy_gateway 2 old "$KMS_OLD_URL" + gateway_version 2 old + bootstrap_gateway + + deploy_app legacy legacy "$KMS_OLD_URL" + verify_app_via_gateway legacy 1 + verify_app_via_gateway legacy 2 + capture_kms_identity old "$KMS_OLD_HOST_PORT" "$(cat "$WORK_DIR/legacy.app_id")" + + # Production KMS upgrades are rolling, quote-attested replication into a new + # Local-Key-Provider CVM, not an unmeasured binary swap in the old CVM. + render_kms latest "$CURRENT_KMS_IMAGE_ID" kms-current.tar + deploy_kms_onboard latest "$KMS_LATEST_HOST_PORT" + wait_onboard latest "$KMS_LATEST_HOST_PORT" + authorize_kms_from_attestation latest + onboard_rpc "$KMS_LATEST_HOST_PORT" Onboard \ + "$(jq -cn --arg s "$KMS_OLD_URL" --arg d "$KMS_RPC_DOMAIN" '{source_url:$s,domain:$d}')" \ + "$WORK_DIR/kms-latest.onboard.json" + # Current Onboard returns only the replicated public key. Quote evidence is + # carried by the mutual RA-TLS connection and authorized above through the + # target's exact quote-derived measurement and physical TDX device ID. + finish_onboarding latest "$KMS_LATEST_HOST_PORT" + capture_kms_identity latest "$KMS_LATEST_HOST_PORT" "$(cat "$WORK_DIR/legacy.app_id")" + assert_kms_identity_unchanged + + stop_vm "$(cat "$WORK_DIR/kms-old.vm_id")" "KMS 0.5.8 after cutover" + restart_legacy_on_latest_kms + deploy_app lite lite "$KMS_LATEST_URL" + verify_all_routes + + capture_gateway_state 1 before + capture_gateway_state 2 before + start_network_probe + upgrade_gateway_node 1 + upgrade_gateway_node 2 + stop_network_probe + verify_all_routes + # The current API redacts stored tokens, so metadata equality alone cannot + # prove the secret survived. The mock rejects any token other than the exact + # original value; a forced issuance through each upgraded node proves it. + assert_gateway_dns_credential_usable 1 + assert_gateway_dns_credential_usable 2 + + assert_no_insecure_shortcuts + save_vm_logs + # dstack-kms runs in an inner container, whose stdout is not part of the CVM + # serial log returned by VMM. Each KMS has a fresh data disk, so two 200 GETs + # for the exact measured archive prove that both verifiers downloaded it. + local os_archive_gets + os_archive_gets=$(grep -Fc \ + "GET /os/mr_${OS_IMAGE_HASH}.tar.gz HTTP/1.1\" 200" \ + "$WORK_DIR/artifacts-access.log" || true) + (( os_archive_gets >= 2 )) \ + || die "expected verified OS image downloads by both KMS versions, saw ${os_archive_gets}" + if grep -E 'Image verification is disabled|self-authorization is disabled' \ + "$WORK_DIR/kms-"*.vm.log; then + die "KMS logs contain a forbidden disabled verification path" + fi + + log "production-compatible KMS/Gateway rolling-upgrade E2E success" + cleanup_after +} + +on_exit() { + local rc=$? + if [[ -n "$probe_pid" ]] && kill -0 "$probe_pid" 2>/dev/null; then + touch "$WORK_DIR/network-probe.stop" 2>/dev/null || true + wait "$probe_pid" 2>/dev/null || true + fi + if (( rc != 0 )); then + save_vm_logs || true + fi + exit "$rc" +} +trap on_exit EXIT + +main() { + need_bin /workspace/target/release/dstack + [[ -s "$STATE_DIR/artifacts/images.env" ]] || die "missing prepared image metadata" + case "$PHASE" in + upgrade) phase_upgrade ;; + *) die "unknown DSTACK_E2E_PHASE=$PHASE" ;; + esac + log "Work artifacts: $WORK_DIR" +} + +main "$@" diff --git a/test-suites/full-stack-compose/state/.gitkeep b/test-suites/full-stack-compose/state/.gitkeep new file mode 100644 index 000000000..e69de29bb