From a65abb01c5284d711c8c5dff2b1b4168e0f90854 Mon Sep 17 00:00:00 2001 From: Hang Yin Date: Fri, 10 Jul 2026 21:56:35 +0000 Subject: [PATCH 1/2] guest: support development-only no-TEE VMs --- CONTRIBUTING.md | 19 +++ docs/security/cvm-boundaries.md | 3 +- dstack/Cargo.lock | 1 + dstack/crates/dstack-cli-core/src/compose.rs | 33 ++++- dstack/crates/dstack-cli/src/main.rs | 24 +++- dstack/dstack-types/src/lib.rs | 3 + dstack/dstack-util/src/main.rs | 44 +++++- dstack/dstack-util/src/system_setup.rs | 133 +++++++++++------- dstack/guest-agent/Cargo.toml | 1 + dstack/guest-agent/src/config.rs | 2 + dstack/guest-agent/src/rpc_service.rs | 79 +++++++++++ dstack/vmm/rpc/proto/vmm_rpc.proto | 4 +- dstack/vmm/src/app.rs | 56 +++++++- dstack/vmm/src/main_service.rs | 89 ++++++++++-- dstack/vmm/src/one_shot.rs | 20 ++- dstack/vmm/src/vmm-cli.py | 59 ++++---- .../vmm/ui/src/components/CreateVmDialog.ts | 1 - dstack/vmm/ui/src/composables/useVmManager.ts | 9 -- dstack/vmm/ui/src/templates/app.html | 2 +- os/common/rootfs/dstack-prepare.sh | 5 +- 20 files changed, 456 insertions(+), 131 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8c5df8bd5..3c48e5273 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -32,6 +32,25 @@ Use the narrowest owning directory. Fixture explanations and component READMEs should stay with their fixtures/components; general guides should be linked from the root README and live under `docs/`. +## Test without TEE + +Maintainers can run a development image without TDX to test VMM and guest changes. + +> [!WARNING] +> A no-TEE VM has no hardware isolation or attestation. Its disk is unencrypted, and its temporary app keys are not stable across boots. Do not use this mode for production workloads or secrets. + +Deploy with KMS and TEE disabled: + +```bash +dstack deploy ./docker-compose.yml \ + --name no-tee-dev \ + --image "YOUR_IMAGE" \ + --no-kms \ + --no-tee +``` + +Attestation and certificate APIs return errors in this mode. TEE mode cannot change after VM creation; recreate the VM to switch modes. + ## Commit Convention This project uses [Conventional Commits](https://www.conventionalcommits.org/). Please format your commit messages as: diff --git a/docs/security/cvm-boundaries.md b/docs/security/cvm-boundaries.md index 8e04108ce..d538997d6 100644 --- a/docs/security/cvm-boundaries.md +++ b/docs/security/cvm-boundaries.md @@ -39,6 +39,7 @@ This is the main configuration file for the application in JSON format: | public_tcbinfo | 0.5.1 | boolean | Whether TCB info is public | | allowed_envs | 0.4.2 | array of string | List of allowed environment variable names | | no_instance_id | 0.4.2 | boolean | Disable instance ID generation | +| no_tee | 0.6.0 | boolean | Development only. Disable TEE and storage encryption. | | secure_time | 0.5.0 | boolean | Whether secure time is enabled | | pre_launch_script | 0.4.0 | string | Prelaunch bash script that runs before execute `docker compose up` | | init_script | 0.5.5 | string | Bash script that executed prior to dockerd startup | @@ -47,7 +48,7 @@ This is the main configuration file for the application in JSON format: | key_provider | 0.5.6 | string | Key provider type. Supported values: "none", "kms", "local", "tpm". | -The hash of this file content is extended to RTMR3 as event name `compose-hash`. Remote verifier can extract the compose-hash during remote attestation. +In TEE mode, the hash of this file is extended to RTMR3 as `compose-hash` for remote verification. ### .instance-info diff --git a/dstack/Cargo.lock b/dstack/Cargo.lock index 65edd0421..22f5c1b8f 100644 --- a/dstack/Cargo.lock +++ b/dstack/Cargo.lock @@ -2546,6 +2546,7 @@ dependencies = [ "sysinfo", "tdx-attest", "tempfile", + "thiserror 2.0.18", "tokio", "tpm-attest", "tracing", diff --git a/dstack/crates/dstack-cli-core/src/compose.rs b/dstack/crates/dstack-cli-core/src/compose.rs index 5c395213d..dbf9e9501 100644 --- a/dstack/crates/dstack-cli-core/src/compose.rs +++ b/dstack/crates/dstack-cli-core/src/compose.rs @@ -10,10 +10,15 @@ use serde_json::json; /// build a minimal app-compose manifest from a docker-compose YAML body /// (single-node, no gateway). /// -/// `kms_enabled` selects KMS mode (deterministic, upgradeable per-app keys); -/// gateway and local-key-provider are off for the direct-port single-node flow. -pub fn build_app_compose(name: &str, docker_compose_yaml: &str, kms_enabled: bool) -> String { - let manifest = json!({ +/// `kms_enabled` selects KMS mode (deterministic, upgradeable per-app keys). +/// `no_tee` is development-only and requires KMS to be disabled by the caller. +pub fn build_app_compose( + name: &str, + docker_compose_yaml: &str, + kms_enabled: bool, + no_tee: bool, +) -> String { + let mut manifest = json!({ "manifest_version": 2, "name": name, "runner": "docker-compose", @@ -31,7 +36,27 @@ pub fn build_app_compose(name: &str, docker_compose_yaml: &str, kms_enabled: boo // (NTS is also currently broken in guest images — see dstack#745.) "secure_time": false, }); + if no_tee { + manifest["no_tee"] = true.into(); + } // pretty-print via Value's Display (`{:#}`) — infallible, and byte-identical // to serde_json::to_string_pretty (avoids an expect on an unfailable Result). format!("{manifest:#}") } + +#[cfg(test)] +mod tests { + use super::build_app_compose; + + #[test] + fn no_tee_is_emitted_only_when_enabled() { + let tee: serde_json::Value = + serde_json::from_str(&build_app_compose("app", "services: {}", true, false)).unwrap(); + assert!(tee.get("no_tee").is_none()); + + let no_tee: serde_json::Value = + serde_json::from_str(&build_app_compose("app", "services: {}", false, true)).unwrap(); + assert_eq!(no_tee["no_tee"], true); + assert_eq!(no_tee["kms_enabled"], false); + } +} diff --git a/dstack/crates/dstack-cli/src/main.rs b/dstack/crates/dstack-cli/src/main.rs index 1d31897d3..fe76ba5d9 100644 --- a/dstack/crates/dstack-cli/src/main.rs +++ b/dstack/crates/dstack-cli/src/main.rs @@ -75,6 +75,9 @@ enum Command { /// deploy in non-KMS mode (ephemeral keys; no KMS required). #[arg(long)] no_kms: bool, + /// disable TEE and storage encryption for development. + #[arg(long, requires = "no_kms")] + no_tee: bool, /// register the app's compose hash in this auth-allowlist.json. Defaults /// to the local allowlist from `dstackup install`. #[arg(long, value_name = "PATH")] @@ -129,6 +132,7 @@ async fn main() -> Result<()> { disk, ports, no_kms, + no_tee, allowlist, dry_run, } => { @@ -157,6 +161,7 @@ async fn main() -> Result<()> { disk, &ports, no_kms, + no_tee, allowlist.as_deref(), dry_run, json, @@ -302,6 +307,21 @@ mod tests { _ => panic!("expected deploy command"), } } + + #[test] + fn no_tee_requires_no_kms() { + assert!(Cli::try_parse_from(["dstack", "deploy", "compose.yaml", "--no-tee"]).is_err()); + + let cli = Cli::parse_from(["dstack", "deploy", "compose.yaml", "--no-kms", "--no-tee"]); + assert!(matches!( + cli.command, + Command::Deploy { + no_kms: true, + no_tee: true, + .. + } + )); + } } #[allow(clippy::too_many_arguments)] @@ -315,13 +335,14 @@ async fn cmd_deploy( disk: u32, port_specs: &[String], no_kms: bool, + no_tee: bool, allowlist: Option<&str>, dry_run: bool, json: bool, ) -> Result<()> { let yaml = std::fs::read_to_string(compose_path) .with_context(|| format!("reading compose file '{compose_path}'"))?; - let app_compose = compose::build_app_compose(name, &yaml, !no_kms); + let app_compose = compose::build_app_compose(name, &yaml, !no_kms, no_tee); let mut port_maps = Vec::new(); for spec in port_specs { @@ -336,6 +357,7 @@ async fn cmd_deploy( memory, disk_size: disk, ports: port_maps.clone(), + no_tee, ..Default::default() }; diff --git a/dstack/dstack-types/src/lib.rs b/dstack/dstack-types/src/lib.rs index 31ec16ba9..7a34654b2 100644 --- a/dstack/dstack-types/src/lib.rs +++ b/dstack/dstack-types/src/lib.rs @@ -101,6 +101,9 @@ pub struct AppCompose { pub allowed_envs: Vec, #[serde(default)] pub no_instance_id: bool, + /// Development only. Disables TEE and storage encryption. + #[serde(default)] + pub no_tee: bool, #[serde(default = "default_true")] pub secure_time: bool, #[serde(default)] diff --git a/dstack/dstack-util/src/main.rs b/dstack/dstack-util/src/main.rs index c9ec7ef70..39347c436 100644 --- a/dstack/dstack-util/src/main.rs +++ b/dstack/dstack-util/src/main.rs @@ -829,7 +829,14 @@ fn cmd_gen_app_keys(args: GenAppKeysArgs) -> Result<()> { let key_provider = KeyProvider::None { key: key.serialize_pem(), }; - let app_keys = make_app_keys(&key, &disk_key, &k256_key, args.ca_level, key_provider)?; + let app_keys = make_app_keys( + &key, + &disk_key, + &k256_key, + args.ca_level, + key_provider, + true, + )?; let app_keys = serde_json::to_string(&app_keys).context("Failed to serialize app keys")?; fs::write(&args.output, app_keys).context("Failed to write app keys")?; Ok(()) @@ -839,6 +846,7 @@ fn gen_app_keys_from_seed( seed: &[u8], provider: KeyProviderKind, mr: Option>, + with_attestation: bool, ) -> Result { let key = derive_p256_key_pair_from_bytes(seed, &["app-key".as_bytes()])?; let disk_key = derive_p256_key_pair_from_bytes(seed, &["app-disk-key".as_bytes()])?; @@ -860,7 +868,14 @@ fn gen_app_keys_from_seed( anyhow::bail!("KMS keys must be fetched from the KMS server") } }; - make_app_keys(&key, &disk_key, &k256_key, 1, key_provider) + make_app_keys( + &key, + &disk_key, + &k256_key, + 1, + key_provider, + with_attestation, + ) } fn make_app_keys( @@ -869,16 +884,23 @@ fn make_app_keys( k256_key: &SigningKey, ca_level: u8, key_provider: KeyProvider, + with_attestation: bool, ) -> Result { use ra_tls::cert::CertRequest; let pubkey = app_key.public_key_der(); let report_data = QuoteContentType::RaTlsCert.to_report_data(&pubkey); - let attestation = Attestation::quote(&report_data) - .context("Failed to get attestation")? - .into_versioned(); + let attestation = if with_attestation { + Some( + Attestation::quote(&report_data) + .context("Failed to get attestation")? + .into_versioned(), + ) + } else { + None + }; let req = CertRequest::builder() .subject("App Root Cert") - .attestation(&attestation) + .maybe_attestation(attestation.as_ref()) .key(app_key) .ca_level(ca_level) .build(); @@ -891,12 +913,20 @@ fn make_app_keys( env_crypt_key: vec![], k256_key: k256_key.to_bytes().to_vec(), k256_signature: vec![], - gateway_app_id: "".to_string(), + gateway_app_id: String::new(), ca_cert: cert.pem(), key_provider, }) } +#[cfg(test)] +#[test] +fn generates_unattested_app_keys_without_tee() { + let keys = gen_app_keys_from_seed(&[0x42; 32], KeyProviderKind::None, None, false).unwrap(); + assert!(!keys.k256_key.is_empty()); + assert!(!keys.ca_cert.is_empty()); +} + async fn cmd_notify_host(args: HostNotifyArgs) -> Result<()> { let client = HostApi::load_or_default(args.url)?; client.notify(&args.event, &args.payload).await?; diff --git a/dstack/dstack-util/src/system_setup.rs b/dstack/dstack-util/src/system_setup.rs index 702f8a76b..6d1f5fee0 100644 --- a/dstack/dstack-util/src/system_setup.rs +++ b/dstack/dstack-util/src/system_setup.rs @@ -159,6 +159,7 @@ impl FromStr for FsType { struct DstackOptions { storage_encrypted: bool, storage_fs: FsType, + no_tee: bool, } fn parse_dstack_options(shared: &HostShared) -> Result { @@ -167,6 +168,7 @@ fn parse_dstack_options(shared: &HostShared) -> Result { let mut options = DstackOptions { storage_encrypted: true, // Default to encryption enabled storage_fs: FsType::Zfs, // Default to ZFS + no_tee: shared.app_compose.no_tee, }; for param in cmdline.split_whitespace() { @@ -186,9 +188,19 @@ fn parse_dstack_options(shared: &HostShared) -> Result { if let Some(fs) = &shared.app_compose.storage_fs { options.storage_fs = fs.parse().context("Failed to parse storage_fs")?; } + if options.no_tee { + options.storage_encrypted = false; + } Ok(options) } +fn emit_runtime_event_for_setup(opts: &DstackOptions, event: &str, payload: &[u8]) -> Result<()> { + if opts.no_tee { + return Ok(()); + } + emit_runtime_event(event, payload) +} + #[derive(Clone)] pub struct HostShareDir { base_dir: PathBuf, @@ -759,10 +771,10 @@ fn platform_instance_binding() -> Result>> { } } -fn emit_key_provider_info(provider_info: &KeyProviderInfo) -> Result<()> { +fn emit_key_provider_info(provider_info: &KeyProviderInfo, opts: &DstackOptions) -> Result<()> { info!("Key provider info: {provider_info:?}"); let provider_info_json = serde_json::to_vec(&provider_info)?; - emit_runtime_event("key-provider", &provider_info_json)?; + emit_runtime_event_for_setup(opts, "key-provider", &provider_info_json)?; Ok(()) } @@ -1037,6 +1049,10 @@ pub async fn cmd_sys_setup(args: SetupArgs) -> Result<()> { } async fn do_sys_setup(stage0: Stage0<'_>) -> Result<()> { + let opts = parse_dstack_options(&stage0.shared).context("Failed to parse system options")?; + if !opts.no_tee { + AttestationMode::detect().context("TEE guest interface is unavailable")?; + } verify_app_compose_policy(&stage0.shared).context("Failed to verify app-compose policy")?; if stage0.shared.app_compose.secure_time { info!("Waiting for the system time to be synchronized"); @@ -1047,11 +1063,13 @@ async fn do_sys_setup(stage0: Stage0<'_>) -> Result<()> { } else { info!("System time will be synchronized by chronyd in background"); } - stage0 - .setup_gpu() - .await - .context("Failed to verify GPU TEE attestation")?; - let stage1 = stage0.setup_fs().await?; + if !opts.no_tee { + stage0 + .setup_gpu() + .await + .context("Failed to verify GPU TEE attestation")?; + } + let stage1 = stage0.setup_fs(opts).await?; stage1.setup().await } @@ -1278,6 +1296,7 @@ struct Stage1<'a> { vmm: HostApi, shared: HostShared, keys: AppKeys, + no_tee: bool, } impl<'a> Stage0<'a> { @@ -1445,6 +1464,7 @@ impl<'a> Stage0<'a> { &provision.sk, KeyProviderKind::Local, Some(provision.mr.to_vec()), + true, ) .context("Failed to generate app keys")?; Ok(app_keys) @@ -1465,7 +1485,7 @@ impl<'a> Stage0<'a> { "unsealed root key seed from TPM (PCR policy: {})", pcr_policy.to_arg() ); - return gen_app_keys_from_seed(&seed, KeyProviderKind::Tpm, None) + return gen_app_keys_from_seed(&seed, KeyProviderKind::Tpm, None, true) .context("failed to generate TPM app keys"); } @@ -1481,19 +1501,22 @@ impl<'a> Stage0<'a> { ) .context("failed to seal seed to TPM")?; - gen_app_keys_from_seed(&seed, KeyProviderKind::Tpm, None) + gen_app_keys_from_seed(&seed, KeyProviderKind::Tpm, None, true) .context("failed to generate TPM app keys") } - async fn request_app_keys(&self) -> Result { + async fn request_app_keys(&self, opts: &DstackOptions) -> Result { let key_provider = self.shared.app_compose.key_provider(); + if opts.no_tee && !key_provider.is_none() { + bail!("no-TEE mode requires key_provider=none"); + } match key_provider { KeyProviderKind::Kms => self.request_app_keys_from_kms().await, KeyProviderKind::Local => self.get_keys_from_local_key_provider().await, KeyProviderKind::None => { info!("No key provider is enabled, generating temporary app keys"); let seed: [u8; 32] = rand::thread_rng().gen(); - gen_app_keys_from_seed(&seed, KeyProviderKind::None, None) + gen_app_keys_from_seed(&seed, KeyProviderKind::None, None, !opts.no_tee) .context("Failed to generate app keys") } KeyProviderKind::Tpm => { @@ -1826,14 +1849,18 @@ impl<'a> Stage0<'a> { Ok(()) } - fn measure_app_info(&self) -> Result { + fn measure_app_info(&self, opts: &DstackOptions) -> Result { let compose_hash = sha256_file(self.shared.dir.app_compose_file())?; let truncated_compose_hash = truncate(&compose_hash, 20); let key_provider = self.shared.app_compose.key_provider(); let mut instance_info = self.shared.instance_info.clone(); - let is_snp = AttestationMode::detect() - .map(|mode| mode == AttestationMode::DstackAmdSevSnp) - .unwrap_or(false); + let is_snp = if opts.no_tee { + false + } else { + AttestationMode::detect() + .map(|mode| mode == AttestationMode::DstackAmdSevSnp) + .unwrap_or(false) + }; if instance_info.app_id.is_empty() { instance_info.app_id = truncated_compose_hash.to_vec(); @@ -1858,7 +1885,7 @@ impl<'a> Stage0<'a> { } else { let mut id_path = instance_info.instance_id_seed.clone(); id_path.extend_from_slice(&instance_info.app_id); - if !is_snp { + if !is_snp && !opts.no_tee { if let Some(binding) = platform_instance_binding()? { info!("mixing platform per-instance binding into instance_id"); id_path.extend_from_slice(&binding); @@ -1875,31 +1902,33 @@ impl<'a> Stage0<'a> { // no KMS to bind it, the relying party MUST gate the compose_hash // (which launcher build) separately from the app_id (which app). - emit_runtime_event("system-preparing", &[])?; - emit_runtime_event("app-id", &instance_info.app_id)?; - emit_runtime_event("compose-hash", &compose_hash)?; - emit_runtime_event("instance-id", &instance_id)?; - emit_runtime_event("boot-mr-done", &[])?; + emit_runtime_event_for_setup(opts, "system-preparing", &[])?; + emit_runtime_event_for_setup(opts, "app-id", &instance_info.app_id)?; + emit_runtime_event_for_setup(opts, "compose-hash", &compose_hash)?; + emit_runtime_event_for_setup(opts, "instance-id", &instance_id)?; + emit_runtime_event_for_setup(opts, "boot-mr-done", &[])?; Ok(AppInfo { instance_info, compose_hash, }) } - fn verify_app(&self, app_info: &AppInfo, keys: &AppKeys) -> Result<()> { - config_id_verifier::verify_mr_config_id( - &app_info.compose_hash, - &app_info - .instance_info - .app_id - .as_slice() - .try_into() - .ok() - .context("Invalid app id")?, - &app_info.instance_info.instance_id, - keys.key_provider.kind(), - keys.key_provider.id(), - )?; + fn verify_app(&self, app_info: &AppInfo, keys: &AppKeys, opts: &DstackOptions) -> Result<()> { + if !opts.no_tee { + config_id_verifier::verify_mr_config_id( + &app_info.compose_hash, + &app_info + .instance_info + .app_id + .as_slice() + .try_into() + .ok() + .context("Invalid app id")?, + &app_info.instance_info.instance_id, + keys.key_provider.kind(), + keys.key_provider.id(), + )?; + } self.verify_key_provider_id(keys.key_provider.id())?; // TPM uses an empty id: the instance app-root pubkey is not a stable // provider identity and must not enter the launch measurement chain. @@ -1913,38 +1942,42 @@ impl<'a> Stage0<'a> { KeyProviderInfo::new("kms".into(), hex::encode(keys.key_provider.id())) } }; - emit_key_provider_info(&kp_info)?; + emit_key_provider_info(&kp_info, opts)?; Ok(()) } - async fn setup_fs(self) -> Result> { + async fn setup_fs(self, opts: DstackOptions) -> Result> { + if opts.no_tee { + warn!("Development-only no-TEE mode enabled; storage is unencrypted"); + } let app_info = self - .measure_app_info() + .measure_app_info(&opts) .context("Failed to measure app info")?; - if self.shared.app_compose.key_provider().is_kms() { + if self.shared.app_compose.key_provider().is_kms() && !opts.no_tee { cmd_show_mrs()?; } - self.vmm - .notify_q("boot.progress", "requesting app keys") - .await; + let key_progress = if opts.no_tee { + "generating temporary app keys" + } else { + "requesting app keys" + }; + self.vmm.notify_q("boot.progress", key_progress).await; let app_keys = self - .request_app_keys() + .request_app_keys(&opts) .await .context("Failed to request app keys")?; if app_keys.disk_crypt_key.is_empty() { bail!("Failed to get valid key phrase from KMS"); } - self.verify_app(&app_info, &app_keys) + self.verify_app(&app_info, &app_keys, &opts) .context("Failed to verify app")?; // Save app keys let keys_json = serde_json::to_string(&app_keys).context("Failed to serialize app keys")?; fs::write(self.app_keys_file(), keys_json).context("Failed to write app keys")?; - // Parse kernel command line options - let opts = parse_dstack_options(&self.shared).context("Failed to parse kernel cmdline")?; - emit_runtime_event("storage-fs", opts.storage_fs.to_string().as_bytes())?; + emit_runtime_event_for_setup(&opts, "storage-fs", opts.storage_fs.to_string().as_bytes())?; info!( "Filesystem options: encryption={}, filesystem={:?}", opts.storage_encrypted, opts.storage_fs @@ -1960,10 +1993,10 @@ impl<'a> Stage0<'a> { &serde_json::to_string(&app_info.instance_info)?, ) .await; - emit_runtime_event("system-ready", &[])?; + emit_runtime_event_for_setup(&opts, "system-ready", &[])?; self.vmm.notify_q("boot.progress", "data disk ready").await; - if !self.shared.app_compose.key_provider().is_kms() { + if !self.shared.app_compose.key_provider().is_kms() && !opts.no_tee { cmd_show_mrs()?; } Ok(Stage1 { @@ -1971,6 +2004,7 @@ impl<'a> Stage0<'a> { shared: self.shared, vmm: self.vmm, keys: app_keys, + no_tee: opts.no_tee, }) } } @@ -2072,6 +2106,7 @@ impl Stage1<'_> { "core": { "pccs_url": self.shared.sys_config.pccs_url, "data_disks": data_disks, + "no_tee": self.no_tee, } } }); diff --git a/dstack/guest-agent/Cargo.toml b/dstack/guest-agent/Cargo.toml index 8f2235f2e..9334f5ee9 100644 --- a/dstack/guest-agent/Cargo.toml +++ b/dstack/guest-agent/Cargo.toml @@ -14,6 +14,7 @@ rocket.workspace = true tracing.workspace = true tracing-subscriber.workspace = true anyhow.workspace = true +thiserror.workspace = true serde.workspace = true fs-err.workspace = true rcgen.workspace = true diff --git a/dstack/guest-agent/src/config.rs b/dstack/guest-agent/src/config.rs index 7f94185d7..d771a46ab 100644 --- a/dstack/guest-agent/src/config.rs +++ b/dstack/guest-agent/src/config.rs @@ -51,6 +51,8 @@ pub struct Config { #[serde(default)] pub pccs_url: Option, pub data_disks: HashSet, + #[serde(default)] + pub no_tee: bool, } fn deserialize_app_compose<'de, D>(deserializer: D) -> Result diff --git a/dstack/guest-agent/src/rpc_service.rs b/dstack/guest-agent/src/rpc_service.rs index cb9680353..270882459 100644 --- a/dstack/guest-agent/src/rpc_service.rs +++ b/dstack/guest-agent/src/rpc_service.rs @@ -43,6 +43,12 @@ use crate::{ config::Config, }; +#[derive(Debug, thiserror::Error)] +enum RpcError { + #[error("Attestation is unavailable in no-TEE mode")] + AttestationUnavailable, +} + fn read_dmi_file(name: &str) -> String { fs::read_to_string(format!("/sys/class/dmi/id/{name}")) .map(|s| s.trim().to_string()) @@ -64,11 +70,20 @@ struct AppStateInner { } impl AppStateInner { + fn ensure_attestation_available(&self) -> Result<()> { + if self.config.no_tee { + return Err(RpcError::AttestationUnavailable.into()); + } + Ok(()) + } + fn info_attestation(&self) -> Result { + self.ensure_attestation_available()?; self.platform.attestation_for_info() } async fn issue_cert(&self, key: &KeyPair, config: CertConfigV2) -> Result> { + self.ensure_attestation_available()?; let pubkey = key.public_key_der(); let attestation = self .platform @@ -113,6 +128,9 @@ impl AppStateInner { impl AppState { fn maybe_request_demo_cert(&self) { + if self.config().no_tee { + return; + } let state = self.inner.clone(); if !state .demo_cert @@ -171,12 +189,14 @@ impl AppState { } fn quote_response(&self, report_data: [u8; 64]) -> Result { + self.inner.ensure_attestation_available()?; self.inner .platform .quote_response(report_data, &self.inner.vm_config) } fn attest_response(&self, report_data: [u8; 64]) -> Result { + self.inner.ensure_attestation_available()?; self.inner.platform.attest_response(report_data) } } @@ -685,6 +705,10 @@ mod tests { } async fn setup_test_state() -> (AppState, tempfile::NamedTempFile) { + setup_test_state_with_mode(false).await + } + + async fn setup_test_state_with_mode(no_tee: bool) -> (AppState, tempfile::NamedTempFile) { let mut temp_attestation_file = tempfile::NamedTempFile::new().unwrap(); let attestation = include_bytes!("../fixtures/attestation.bin"); @@ -707,6 +731,7 @@ mod tests { key_provider_id: Vec::new(), allowed_envs: Vec::new(), no_instance_id: false, + no_tee: false, secure_time: false, storage_fs: None, swap_size: 0, @@ -725,6 +750,7 @@ mod tests { sys_config_file: String::new().into(), pccs_url: None, data_disks: HashSet::new(), + no_tee, }; const DUMMY_PEM_KEY: &str = r#"-----BEGIN PRIVATE KEY----- @@ -802,6 +828,7 @@ pNs85uhOZE8z2jr8Pg== struct TestSimulatorPlatform { attestation: VersionedAttestation, + deny_attestation: bool, } fn patch_report_data( @@ -813,10 +840,12 @@ pNs85uhOZE8z2jr8Pg== impl PlatformBackend for TestSimulatorPlatform { fn attestation_for_info(&self) -> Result { + assert!(!self.deny_attestation, "unexpected attestation request"); Ok(self.attestation.clone()) } fn certificate_attestation(&self, pubkey: &[u8]) -> Result { + assert!(!self.deny_attestation, "unexpected attestation request"); let report_data = ra_tls::attestation::QuoteContentType::RaTlsCert.to_report_data(pubkey); let attestation = patch_report_data(&self.attestation, report_data); @@ -828,6 +857,7 @@ pNs85uhOZE8z2jr8Pg== report_data: [u8; 64], vm_config: &str, ) -> Result { + assert!(!self.deny_attestation, "unexpected attestation request"); let attestation = patch_report_data(&self.attestation, report_data); let Some(quote) = attestation.platform.tdx_quote().map(ToOwned::to_owned) else { return Err(anyhow::anyhow!("Quote not found")); @@ -845,6 +875,7 @@ pNs85uhOZE8z2jr8Pg== } fn attest_response(&self, report_data: [u8; 64]) -> Result { + assert!(!self.deny_attestation, "unexpected attestation request"); let attestation = patch_report_data(&self.attestation, report_data); Ok(AttestResponse { attestation: VersionedAttestation::V1 { attestation }.to_bytes()?, @@ -863,6 +894,7 @@ pNs85uhOZE8z2jr8Pg== &std::fs::read(temp_attestation_file.path()).unwrap(), ) .unwrap(), + deny_attestation: no_tee, }), }; @@ -874,6 +906,53 @@ pNs85uhOZE8z2jr8Pg== ) } + #[tokio::test] + async fn no_tee_supports_raw_keys_and_rejects_attestation() { + let (state, _guard) = setup_test_state_with_mode(true).await; + let key = InternalRpcHandler { + state: state.clone(), + } + .get_key(GetKeyArgs { + path: "development".to_string(), + purpose: "signing".to_string(), + algorithm: "ed25519".to_string(), + }) + .await + .unwrap(); + assert!(!key.key.is_empty()); + + for err in [ + get_info(&state, false).await.unwrap_err(), + state.quote_response([0; 64]).unwrap_err(), + state.attest_response([0; 64]).unwrap_err(), + ] { + assert_eq!(err.to_string(), "Attestation is unavailable in no-TEE mode"); + } + for request in [ + GetTlsKeyArgs { + subject: "development".to_string(), + usage_server_auth: true, + ..Default::default() + }, + GetTlsKeyArgs { + usage_ra_tls: true, + ..Default::default() + }, + GetTlsKeyArgs { + with_app_info: true, + ..Default::default() + }, + ] { + let err = InternalRpcHandler { + state: state.clone(), + } + .get_tls_key(request) + .await + .unwrap_err(); + assert!(format!("{err:#}").contains("Attestation is unavailable in no-TEE mode")); + } + } + #[tokio::test] async fn test_verify_ed25519_success() { let (state, _guard) = setup_test_state().await; diff --git a/dstack/vmm/rpc/proto/vmm_rpc.proto b/dstack/vmm/rpc/proto/vmm_rpc.proto index 8d79fea66..ccd6ed1a8 100644 --- a/dstack/vmm/rpc/proto/vmm_rpc.proto +++ b/dstack/vmm/rpc/proto/vmm_rpc.proto @@ -97,7 +97,7 @@ message VmConfiguration { repeated string gateway_urls = 15; // The VM is stopped bool stopped = 16; - // Disable confidential computing (fallback to non-TEE VM). + // Deprecated. Use the development-only app-compose no_tee field. bool no_tee = 17; // Per-VM networking mode override (if unset, uses global cvm.networking). optional NetworkingConfig networking = 18; @@ -161,7 +161,7 @@ message UpdateVmRequest { optional uint32 memory = 15; optional uint32 disk_size = 16; optional string image = 17; - // Disable or re-enable TEE for an existing VM. + // Deprecated. Use the app-compose no_tee field. optional bool no_tee = 18; } diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 0ce99b437..0ceebf171 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -8,10 +8,10 @@ use dstack_port_forward::{ForwardRule, ForwardService, Protocol as FwdProtocol}; use anyhow::{bail, Context, Result}; use bon::Builder; use dstack_kms_rpc::kms_client::KmsClient; -use dstack_types::mr_config::MrConfigV3; use dstack_types::shared_filenames::{ APP_COMPOSE, ENCRYPTED_ENV, INSTANCE_INFO, SYS_CONFIG, USER_CONFIG, }; +use dstack_types::{mr_config::MrConfigV3, AppCompose}; use dstack_vmm_rpc::{ self as pb, GpuInfo, ReloadVmsResponse, StatusRequest, StatusResponse, VmConfiguration, }; @@ -130,6 +130,31 @@ pub(crate) fn round_up(value: u32, multiple: u32) -> u32 { value + (multiple - remainder) } +pub(crate) fn validate_no_tee_compose(no_tee: bool, app_compose: &AppCompose) -> Result<()> { + if no_tee != app_compose.no_tee { + bail!("TEE mode in manifest and app compose does not match"); + } + if !no_tee { + return Ok(()); + } + if !app_compose.key_provider().is_none() { + bail!("no-TEE mode requires key_provider=none"); + } + if app_compose.gateway_enabled() { + bail!("no-TEE mode does not support the gateway"); + } + if app_compose + .requirements + .as_ref() + .is_some_and(|requirements| { + requirements.platforms.is_some() || requirements.tdx_measure_acpi_tables.is_some() + }) + { + bail!("no-TEE mode does not support TEE requirements"); + } + Ok(()) +} + /// Get the NUMA node associated with a PCI device. pub(crate) fn pci_numa_node(device: &str) -> Result { // Ensure the device string only contains valid hexadecimal characters and colons. @@ -279,6 +304,7 @@ impl App { let app_compose = vm_work_dir .app_compose() .context("Failed to read compose file")?; + validate_no_tee_compose(manifest.no_tee, &app_compose)?; { let mut states = self.lock(); let cid = states @@ -859,6 +885,7 @@ impl App { let app_compose = vm_work_dir .app_compose() .context("Failed to read compose file")?; + validate_no_tee_compose(manifest.no_tee, &app_compose)?; let mut is_new = false; { @@ -1630,6 +1657,33 @@ mod tests { assert_eq!(effective_vcpu_count(3, None), 3); } + #[test] + fn no_tee_compose_rejects_attestation_dependencies() { + fn app_compose(extra: serde_json::Value) -> AppCompose { + let mut value = serde_json::json!({ + "manifest_version": "3", + "name": "test", + "runner": "docker-compose", + "key_provider": "none", + "no_tee": true + }); + value + .as_object_mut() + .unwrap() + .extend(extra.as_object().unwrap().clone()); + serde_json::from_value(value).unwrap() + } + + validate_no_tee_compose(true, &app_compose(serde_json::json!({}))).unwrap(); + for extra in [ + serde_json::json!({"key_provider": "kms"}), + serde_json::json!({"gateway_enabled": true}), + serde_json::json!({"requirements": {"platforms": ["tdx"]}}), + ] { + assert!(validate_no_tee_compose(true, &app_compose(extra)).is_err()); + } + } + #[test] fn tdx_auto_variant_uses_legacy_for_low_non_2g_memory() -> Result<()> { let config = test_tdx_config()?; diff --git a/dstack/vmm/src/main_service.rs b/dstack/vmm/src/main_service.rs index 88aa884c1..cdece2e38 100644 --- a/dstack/vmm/src/main_service.rs +++ b/dstack/vmm/src/main_service.rs @@ -22,7 +22,9 @@ use or_panic::ResultOrPanic; use ra_rpc::{CallContext, RpcCall}; use tracing::{info, warn}; -use crate::app::{App, AttachMode, GpuConfig, GpuSpec, Manifest, PortMapping, VmWorkDir}; +use crate::app::{ + validate_no_tee_compose, App, AttachMode, GpuConfig, GpuSpec, Manifest, PortMapping, VmWorkDir, +}; fn hex_sha256(data: &str) -> String { use sha2::Digest; @@ -54,6 +56,32 @@ fn app_id_of(compose_file: &str) -> String { truncate40(&hex_sha256(compose_file)).to_string() } +/// Normalize legacy RPC input into the app compose, which is the source of truth. +pub(crate) fn normalize_app_compose( + compose_file: &str, + default_no_tee: bool, +) -> Result<(String, AppCompose)> { + let mut value: serde_json::Value = + serde_json::from_str(compose_file).context("Invalid compose file")?; + let object = value + .as_object_mut() + .context("App compose must be a JSON object")?; + let had_no_tee = object.contains_key("no_tee"); + if !had_no_tee && default_no_tee { + object.insert("no_tee".to_string(), true.into()); + } + let normalized = if had_no_tee || !default_no_tee { + compose_file.to_string() + } else { + serde_json::to_string_pretty(&value).context("Failed to serialize app compose")? + }; + let app_compose: AppCompose = serde_json::from_value(value).context("Invalid compose file")?; + if had_no_tee && default_no_tee && !app_compose.no_tee { + bail!("no_tee in the RPC request and app compose does not match"); + } + Ok((normalized, app_compose)) +} + /// Validate the VM label, restricting it to a safe character set to prevent injection vectors. fn validate_label(label: &str) -> Result<()> { fn is_valid_label_char(c: char) -> bool { @@ -288,8 +316,13 @@ impl RpcHandler { } impl VmmRpc for RpcHandler { - async fn create_vm(self, request: VmConfiguration) -> Result { + async fn create_vm(self, mut request: VmConfiguration) -> Result { + let (compose_file, app_compose) = + normalize_app_compose(&request.compose_file, request.no_tee)?; + request.compose_file = compose_file; + request.no_tee = app_compose.no_tee; let manifest = create_manifest_from_vm_config(request.clone(), &self.app.config.cvm)?; + validate_no_tee_compose(manifest.no_tee, &app_compose)?; let id = manifest.id.clone(); let app_id = manifest.app_id.clone(); let vm_work_dir = self.app.work_dir(&id); @@ -375,18 +408,26 @@ impl VmmRpc for RpcHandler { } async fn update_vm(self, request: UpdateVmRequest) -> Result { + let vm_work_dir = self.app.work_dir(&request.id); + let mut manifest = vm_work_dir.manifest().context("Failed to read manifest")?; + if request + .no_tee + .is_some_and(|no_tee| no_tee != manifest.no_tee) + { + bail!("TEE mode cannot be changed after VM creation"); + } + let new_id = if !request.compose_file.is_empty() { - // check the compose file is valid - let _app_compose: AppCompose = - serde_json::from_str(&request.compose_file).context("Invalid compose file")?; + let (compose_file, app_compose) = + normalize_app_compose(&request.compose_file, manifest.no_tee)?; + validate_no_tee_compose(manifest.no_tee, &app_compose)?; let compose_file_path = self.compose_file_path(&request.id); if !compose_file_path.exists() { bail!("The instance {} not found", request.id); } - fs::write(compose_file_path, &request.compose_file) - .context("Failed to write compose file")?; + fs::write(compose_file_path, &compose_file).context("Failed to write compose file")?; - app_id_of(&request.compose_file) + app_id_of(&compose_file) } else { Default::default() }; @@ -400,8 +441,6 @@ impl VmmRpc for RpcHandler { fs::write(user_config_path, &request.user_config) .context("Failed to write user config")?; } - let vm_work_dir = self.app.work_dir(&request.id); - let mut manifest = vm_work_dir.manifest().context("Failed to read manifest")?; self.apply_resource_updates( &request.id, &mut manifest, @@ -415,9 +454,6 @@ impl VmmRpc for RpcHandler { if let Some(gpus) = request.gpus { manifest.gpus = Some(self.resolve_gpus(&gpus)?); } - if let Some(no_tee) = request.no_tee { - manifest.no_tee = no_tee; - } if request.update_ports { manifest.port_map = request .ports @@ -749,3 +785,30 @@ impl RpcCall for RpcHandler { }) } } + +#[cfg(test)] +mod tests { + use super::normalize_app_compose; + + const COMPOSE: &str = r#"{"manifest_version":"2","name":"test","runner":"docker-compose"}"#; + + #[test] + fn legacy_no_tee_request_is_written_to_compose() { + let (normalized, app_compose) = normalize_app_compose(COMPOSE, true).unwrap(); + assert!(app_compose.no_tee); + assert!(normalized.contains(r#""no_tee": true"#)); + } + + #[test] + fn compose_no_tee_is_authoritative() { + let source = + r#"{"manifest_version":"2","name":"test","runner":"docker-compose","no_tee":true}"#; + let (normalized, app_compose) = normalize_app_compose(source, false).unwrap(); + assert!(app_compose.no_tee); + assert_eq!(normalized, source); + + let source = + r#"{"manifest_version":"2","name":"test","runner":"docker-compose","no_tee":false}"#; + assert!(normalize_app_compose(source, true).is_err()); + } +} diff --git a/dstack/vmm/src/one_shot.rs b/dstack/vmm/src/one_shot.rs index 3773fbcb2..f5739fa64 100644 --- a/dstack/vmm/src/one_shot.rs +++ b/dstack/vmm/src/one_shot.rs @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 -use crate::app::{make_sys_config, Image, VmConfig, VmWorkDir}; +use crate::app::{make_sys_config, validate_no_tee_compose, Image, VmConfig, VmWorkDir}; use crate::config::Config; use crate::main_service; use anyhow::{Context, Result}; @@ -15,7 +15,7 @@ pub async fn run_one_shot( ) -> Result<()> { use dstack_types::AppCompose; use dstack_vmm_rpc::VmConfiguration; - use main_service::create_manifest_from_vm_config; + use main_service::{create_manifest_from_vm_config, normalize_app_compose}; // Dynamically allocate CID by scanning running QEMU processes (ps aux method) let mut existing_cids = Vec::new(); @@ -64,8 +64,14 @@ pub async fn run_one_shot( .with_context(|| format!("Failed to read VM configuration file: {}", vm_config_path))?; // Parse VM configuration - let vm_config: VmConfiguration = serde_json::from_str(&vm_config_json) + let mut vm_config: VmConfiguration = serde_json::from_str(&vm_config_json) .with_context(|| format!("Failed to parse VM configuration from: {}", vm_config_path))?; + if !vm_config.compose_file.is_empty() { + let (compose_file, app_compose) = + normalize_app_compose(&vm_config.compose_file, vm_config.no_tee)?; + vm_config.compose_file = compose_file; + vm_config.no_tee = app_compose.no_tee; + } // Calculate compose_hash using the same logic as main_service let compose_hash = { @@ -82,7 +88,6 @@ pub async fn run_one_shot( let image_path = config.image.path.join(&manifest.image); let image = Image::load(&image_path) .with_context(|| format!("Failed to load image: {}", image_path.display()))?; - // Create or use specified workdir and setup files let workdir_path = match workdir_option { Some(workdir_str) => { @@ -117,7 +122,7 @@ pub async fn run_one_shot( fs_err::create_dir_all(&shared_dir).context("Failed to create shared directory")?; // Create app compose file content and parse AppCompose instance - let (app_compose_content, _app_compose) = if vm_config.compose_file.is_empty() { + let (app_compose_content, app_compose) = if vm_config.compose_file.is_empty() { // Create default compose JSON directly as string let gateway_enabled = !vm_config.gateway_urls.is_empty(); let kms_enabled = !vm_config.kms_urls.is_empty(); @@ -135,11 +140,12 @@ pub async fn run_one_shot( "public_tcbinfo": true, "local_key_provider_enabled": false, "no_instance_id": false, +"no_tee": {}, "secure_time": true, "features": [], "allowed_envs": [] }}"#, - vm_config.name, gateway_enabled, kms_enabled + vm_config.name, gateway_enabled, kms_enabled, vm_config.no_tee ); // Parse the default compose to get AppCompose instance for gateway_enabled() call @@ -204,6 +210,8 @@ Compose file content (first 200 chars): } }; + validate_no_tee_compose(manifest.no_tee, &app_compose)?; + // Write the JSON string directly (no serialization needed) fs_err::write(vm_work_dir.app_compose_path(), app_compose_content) .context("Failed to write app compose file")?; diff --git a/dstack/vmm/src/vmm-cli.py b/dstack/vmm/src/vmm-cli.py index 290ee2440..22a0f1f83 100755 --- a/dstack/vmm/src/vmm-cli.py +++ b/dstack/vmm/src/vmm-cli.py @@ -808,6 +808,8 @@ def create_app_compose(self, args) -> None: } if args.key_provider: app_compose["key_provider"] = args.key_provider + if args.no_tee: + app_compose["no_tee"] = True if args.prelaunch_script: app_compose["pre_launch_script"] = ( open(args.prelaunch_script, "rb").read().decode("utf-8") @@ -835,6 +837,18 @@ def create_vm(self, args) -> None: raise Exception(f"Compose file not found: {args.compose}") compose_content = read_utf8(args.compose) + try: + compose_json = json.loads(compose_content) + except json.JSONDecodeError as err: + raise Exception(f"Invalid app compose file: {err}") from err + if not isinstance(compose_json, dict): + raise Exception("App compose must be a JSON object") + + if args.no_tee is not None: + compose_json["no_tee"] = args.no_tee + compose_content = json.dumps(compose_json, indent=4, ensure_ascii=False) + + no_tee = bool(compose_json.get("no_tee", False)) envs = parse_env_file(args.env_file) @@ -844,14 +858,10 @@ def create_vm(self, args) -> None: raise Exception( "--env-file requires --kms-url to encrypt environment variables" ) - try: - compose_json = json.loads(compose_content) - if not compose_json.get("kms_enabled", False): - raise Exception( - "--env-file requires kms_enabled=true in the compose file (use --kms when creating compose)" - ) - except json.JSONDecodeError: - pass # Let the server handle invalid JSON + if not compose_json.get("kms_enabled", False): + raise Exception( + "--env-file requires kms_enabled=true in the compose file (use --kms when creating compose)" + ) # Read user config file if provided user_config = "" @@ -872,7 +882,7 @@ def create_vm(self, args) -> None: "hugepages": args.hugepages, "pin_numa": args.pin_numa, "stopped": args.stopped, - "no_tee": args.no_tee, + "no_tee": no_tee, } if args.swap is not None: swap_bytes = max(0, int(round(args.swap)) * 1024 * 1024) @@ -999,7 +1009,6 @@ def update_vm( attach_all: bool = False, no_gpus: bool = False, kms_urls: Optional[List[str]] = None, - no_tee: Optional[bool] = None, ) -> None: """Update multiple aspects of a VM in one command.""" # Validate: --env-file requires --kms-url @@ -1163,10 +1172,6 @@ def update_vm( updates.append("GPUs (none)") upgrade_params["gpus"] = gpu_config - if no_tee is not None: - upgrade_params["no_tee"] = no_tee - updates.append("TEE disabled" if no_tee else "TEE enabled") - if len(upgrade_params) > 1: # more than just the id self.rpc_call("UpgradeApp", upgrade_params) @@ -1675,6 +1680,11 @@ def _patched_format_help(): compose_parser.add_argument( "--no-instance-id", action="store_true", help="Disable instance ID" ) + compose_parser.add_argument( + "--no-tee", + action="store_true", + help="Disable TEE and disk encryption for development", + ) compose_parser.add_argument( "--secure-time", action="store_true", help="Enable secure time" ) @@ -1754,7 +1764,7 @@ def _patched_format_help(): "--no-tee", dest="no_tee", action="store_true", - help="Disable Intel TDX / run without TEE", + help="Disable TEE and disk encryption for development", ) deploy_parser.add_argument( "--tee", @@ -1762,7 +1772,7 @@ def _patched_format_help(): action="store_false", help="Force-enable Intel TDX (default)", ) - deploy_parser.set_defaults(no_tee=False) + deploy_parser.set_defaults(no_tee=None) deploy_parser.add_argument( "--net", choices=["bridge", "user"], @@ -1906,22 +1916,6 @@ def _patched_format_help(): help="Detach all GPUs from the VM", ) - # TDX toggle - tee_group = update_parser.add_mutually_exclusive_group() - tee_group.add_argument( - "--no-tee", - dest="no_tee", - action="store_true", - help="Disable Intel TDX / run without TEE", - ) - tee_group.add_argument( - "--tee", - dest="no_tee", - action="store_false", - help="Enable Intel TDX for the VM", - ) - update_parser.set_defaults(no_tee=None) - # KMS URL for environment encryption update_parser.add_argument("--kms-url", action="append", type=str, help="KMS URL") @@ -2006,7 +2000,6 @@ def _patched_format_help(): attach_all=args.ppcie, no_gpus=args.no_gpus if hasattr(args, "no_gpus") else False, kms_urls=args.kms_url, - no_tee=args.no_tee, ) elif args.command == "kms": if not args.kms_action: diff --git a/dstack/vmm/ui/src/components/CreateVmDialog.ts b/dstack/vmm/ui/src/components/CreateVmDialog.ts index 22f473435..0e378bea5 100644 --- a/dstack/vmm/ui/src/components/CreateVmDialog.ts +++ b/dstack/vmm/ui/src/components/CreateVmDialog.ts @@ -164,7 +164,6 @@ const CreateVmDialogComponent = { - diff --git a/dstack/vmm/ui/src/composables/useVmManager.ts b/dstack/vmm/ui/src/composables/useVmManager.ts index a994ee1d8..6eed60763 100644 --- a/dstack/vmm/ui/src/composables/useVmManager.ts +++ b/dstack/vmm/ui/src/composables/useVmManager.ts @@ -115,7 +115,6 @@ type VmFormState = { public_logs: boolean; public_sysinfo: boolean; public_tcbinfo: boolean; - no_tee: boolean; pin_numa: boolean; hugepages: boolean; net_mode: string; @@ -165,7 +164,6 @@ type CloneConfigDialogState = { gateway_urls?: string[]; hugepages: boolean; pin_numa: boolean; - no_tee: boolean; encrypted_env?: Uint8Array; app_id?: string; stopped: boolean; @@ -198,7 +196,6 @@ function createVmFormState(preLaunchScript: string): VmFormState { public_logs: true, public_sysinfo: true, public_tcbinfo: true, - no_tee: false, pin_numa: false, hugepages: false, net_mode: '', @@ -252,7 +249,6 @@ function createCloneConfigDialogState(): CloneConfigDialogState { gateway_urls: undefined, hugepages: false, pin_numa: false, - no_tee: false, encrypted_env: undefined, app_id: undefined, stopped: false, @@ -443,7 +439,6 @@ type CreateVmPayloadSource = { user_config?: string; hugepages?: boolean; pin_numa?: boolean; - no_tee?: boolean; net_mode?: string; gpus?: VmmTypes.IGpuConfig; kms_urls?: string[]; @@ -466,7 +461,6 @@ type CreateVmPayloadSource = { user_config: source.user_config || '', hugepages: !!source.hugepages, pin_numa: !!source.pin_numa, - no_tee: source.no_tee ?? false, networking: source.net_mode ? { mode: source.net_mode } : undefined, gpus: source.gpus, kms_urls: source.kms_urls?.filter((url) => url && url.trim().length) ?? [], @@ -1003,7 +997,6 @@ type CreateVmPayloadSource = { user_config: vmForm.value.user_config, hugepages: vmForm.value.hugepages, pin_numa: vmForm.value.pin_numa, - no_tee: vmForm.value.no_tee, net_mode: vmForm.value.net_mode, gpus: configGpu(vmForm.value) || undefined, kms_urls: vmForm.value.kms_urls, @@ -1154,7 +1147,6 @@ type CreateVmPayloadSource = { public_tcbinfo: !!theVm.appCompose?.public_tcbinfo, pin_numa: !!config.pin_numa, hugepages: !!config.hugepages, - no_tee: !!config.no_tee, net_mode: config.networking?.mode || '', user_config: config.user_config || '', stopped: !!config.stopped, @@ -1184,7 +1176,6 @@ type CreateVmPayloadSource = { user_config: source.user_config, hugepages: source.hugepages, pin_numa: source.pin_numa, - no_tee: source.no_tee, gpus: source.gpus, kms_urls: source.kms_urls, gateway_urls: source.gateway_urls, diff --git a/dstack/vmm/ui/src/templates/app.html b/dstack/vmm/ui/src/templates/app.html index 7b6a63bdc..b0cee85de 100644 --- a/dstack/vmm/ui/src/templates/app.html +++ b/dstack/vmm/ui/src/templates/app.html @@ -307,7 +307,7 @@

dstack-vmm

TEE - {{ vm.configuration?.no_tee ? 'Disabled' : 'Enabled' }} + {{ vm.configuration?.no_tee ? 'Disabled (development)' : 'Enabled' }}
GPUs diff --git a/os/common/rootfs/dstack-prepare.sh b/os/common/rootfs/dstack-prepare.sh index 81e23363a..e36b923ea 100755 --- a/os/common/rootfs/dstack-prepare.sh +++ b/os/common/rootfs/dstack-prepare.sh @@ -108,8 +108,7 @@ elif modprobe sev-guest 2>/dev/null; then elif modprobe tdx-guest 2>/dev/null; then log "Loaded tdx-guest module" else - log "Error: neither sev-guest nor tdx-guest module is available" - exit 1 + log "No TEE guest module is available" fi # Setup configfs and TSM for TDX attestation @@ -269,7 +268,7 @@ if [ -f "/sys/class/block/${device_name}/partition" ]; then fi fi -dstack-util setup --work-dir $WORK_DIR --device "$DATA_DEVICE" --mount-point $DATA_MNT +dstack-util setup --work-dir "$WORK_DIR" --device "$DATA_DEVICE" --mount-point "$DATA_MNT" log "Mounting container runtime dirs to persistent storage" mkdir -p $DATA_MNT/var/lib/docker From 29f1fd467780da95eea4f95f00043dd9c896433d Mon Sep 17 00:00:00 2001 From: Hang Yin Date: Tue, 14 Jul 2026 11:03:16 +0000 Subject: [PATCH 2/2] fix no-tee compose normalization --- dstack/dstack-util/src/main.rs | 2 +- dstack/dstack-util/src/system_setup.rs | 10 +- dstack/guest-agent/src/rpc_service.rs | 6 +- dstack/vmm/src/app.rs | 2 +- dstack/vmm/src/main_service.rs | 35 ++++-- dstack/vmm/src/one_shot.rs | 155 +++++++++---------------- dstack/vmm/src/vmm-cli.py | 10 +- os/common/rootfs/dstack-prepare.sh | 2 +- 8 files changed, 95 insertions(+), 127 deletions(-) diff --git a/dstack/dstack-util/src/main.rs b/dstack/dstack-util/src/main.rs index 39347c436..eb73d7a4a 100644 --- a/dstack/dstack-util/src/main.rs +++ b/dstack/dstack-util/src/main.rs @@ -892,7 +892,7 @@ fn make_app_keys( let attestation = if with_attestation { Some( Attestation::quote(&report_data) - .context("Failed to get attestation")? + .context("failed to get attestation")? .into_versioned(), ) } else { diff --git a/dstack/dstack-util/src/system_setup.rs b/dstack/dstack-util/src/system_setup.rs index 6d1f5fee0..de8190e8c 100644 --- a/dstack/dstack-util/src/system_setup.rs +++ b/dstack/dstack-util/src/system_setup.rs @@ -1049,9 +1049,9 @@ pub async fn cmd_sys_setup(args: SetupArgs) -> Result<()> { } async fn do_sys_setup(stage0: Stage0<'_>) -> Result<()> { - let opts = parse_dstack_options(&stage0.shared).context("Failed to parse system options")?; + let opts = parse_dstack_options(&stage0.shared).context("failed to parse system options")?; if !opts.no_tee { - AttestationMode::detect().context("TEE guest interface is unavailable")?; + AttestationMode::detect().context("missing TEE guest interface")?; } verify_app_compose_policy(&stage0.shared).context("Failed to verify app-compose policy")?; if stage0.shared.app_compose.secure_time { @@ -1067,7 +1067,7 @@ async fn do_sys_setup(stage0: Stage0<'_>) -> Result<()> { stage0 .setup_gpu() .await - .context("Failed to verify GPU TEE attestation")?; + .context("failed to verify GPU TEE attestation")?; } let stage1 = stage0.setup_fs(opts).await?; stage1.setup().await @@ -1923,7 +1923,7 @@ impl<'a> Stage0<'a> { .as_slice() .try_into() .ok() - .context("Invalid app id")?, + .context("invalid app id")?, &app_info.instance_info.instance_id, keys.key_provider.kind(), keys.key_provider.id(), @@ -1948,7 +1948,7 @@ impl<'a> Stage0<'a> { async fn setup_fs(self, opts: DstackOptions) -> Result> { if opts.no_tee { - warn!("Development-only no-TEE mode enabled; storage is unencrypted"); + warn!("development-only no-TEE mode enabled; storage is unencrypted"); } let app_info = self .measure_app_info(&opts) diff --git a/dstack/guest-agent/src/rpc_service.rs b/dstack/guest-agent/src/rpc_service.rs index 270882459..1e8bd7dd4 100644 --- a/dstack/guest-agent/src/rpc_service.rs +++ b/dstack/guest-agent/src/rpc_service.rs @@ -45,7 +45,7 @@ use crate::{ #[derive(Debug, thiserror::Error)] enum RpcError { - #[error("Attestation is unavailable in no-TEE mode")] + #[error("attestation is unavailable in no-TEE mode")] AttestationUnavailable, } @@ -926,7 +926,7 @@ pNs85uhOZE8z2jr8Pg== state.quote_response([0; 64]).unwrap_err(), state.attest_response([0; 64]).unwrap_err(), ] { - assert_eq!(err.to_string(), "Attestation is unavailable in no-TEE mode"); + assert_eq!(err.to_string(), "attestation is unavailable in no-TEE mode"); } for request in [ GetTlsKeyArgs { @@ -949,7 +949,7 @@ pNs85uhOZE8z2jr8Pg== .get_tls_key(request) .await .unwrap_err(); - assert!(format!("{err:#}").contains("Attestation is unavailable in no-TEE mode")); + assert!(format!("{err:#}").contains("attestation is unavailable in no-TEE mode")); } } diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 0ceebf171..ee7920d0b 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -132,7 +132,7 @@ pub(crate) fn round_up(value: u32, multiple: u32) -> u32 { pub(crate) fn validate_no_tee_compose(no_tee: bool, app_compose: &AppCompose) -> Result<()> { if no_tee != app_compose.no_tee { - bail!("TEE mode in manifest and app compose does not match"); + bail!("manifest and app compose TEE modes do not match"); } if !no_tee { return Ok(()); diff --git a/dstack/vmm/src/main_service.rs b/dstack/vmm/src/main_service.rs index cdece2e38..05cfb26a6 100644 --- a/dstack/vmm/src/main_service.rs +++ b/dstack/vmm/src/main_service.rs @@ -62,10 +62,10 @@ pub(crate) fn normalize_app_compose( default_no_tee: bool, ) -> Result<(String, AppCompose)> { let mut value: serde_json::Value = - serde_json::from_str(compose_file).context("Invalid compose file")?; + serde_json::from_str(compose_file).context("invalid compose file")?; let object = value .as_object_mut() - .context("App compose must be a JSON object")?; + .context("app compose must be a JSON object")?; let had_no_tee = object.contains_key("no_tee"); if !had_no_tee && default_no_tee { object.insert("no_tee".to_string(), true.into()); @@ -73,15 +73,20 @@ pub(crate) fn normalize_app_compose( let normalized = if had_no_tee || !default_no_tee { compose_file.to_string() } else { - serde_json::to_string_pretty(&value).context("Failed to serialize app compose")? + serde_json::to_string_pretty(&value).context("failed to serialize app compose")? }; - let app_compose: AppCompose = serde_json::from_value(value).context("Invalid compose file")?; + let app_compose: AppCompose = serde_json::from_value(value).context("invalid compose file")?; if had_no_tee && default_no_tee && !app_compose.no_tee { bail!("no_tee in the RPC request and app compose does not match"); } Ok((normalized, app_compose)) } +fn normalized_compose_hash(compose_file: &str, default_no_tee: bool) -> Result { + let (compose_file, _) = normalize_app_compose(compose_file, default_no_tee)?; + Ok(hex_sha256(&compose_file)) +} + /// Validate the VM label, restricting it to a safe character set to prevent injection vectors. fn validate_label(label: &str) -> Result<()> { fn is_valid_label_char(c: char) -> bool { @@ -409,12 +414,12 @@ impl VmmRpc for RpcHandler { async fn update_vm(self, request: UpdateVmRequest) -> Result { let vm_work_dir = self.app.work_dir(&request.id); - let mut manifest = vm_work_dir.manifest().context("Failed to read manifest")?; + let mut manifest = vm_work_dir.manifest().context("failed to read manifest")?; if request .no_tee .is_some_and(|no_tee| no_tee != manifest.no_tee) { - bail!("TEE mode cannot be changed after VM creation"); + bail!("cannot change TEE mode after VM creation"); } let new_id = if !request.compose_file.is_empty() { @@ -425,7 +430,7 @@ impl VmmRpc for RpcHandler { if !compose_file_path.exists() { bail!("The instance {} not found", request.id); } - fs::write(compose_file_path, &compose_file).context("Failed to write compose file")?; + fs::write(compose_file_path, &compose_file).context("failed to write compose file")?; app_id_of(&compose_file) } else { @@ -600,10 +605,7 @@ impl VmmRpc for RpcHandler { async fn get_compose_hash(self, request: VmConfiguration) -> Result { validate_label(&request.name)?; - // check the compose file is valid - let _app_compose: AppCompose = - serde_json::from_str(&request.compose_file).context("Invalid compose file")?; - let hash = hex_sha256(&request.compose_file); + let hash = normalized_compose_hash(&request.compose_file, request.no_tee)?; Ok(RpcComposeHash { hash }) } @@ -788,7 +790,7 @@ impl RpcCall for RpcHandler { #[cfg(test)] mod tests { - use super::normalize_app_compose; + use super::{hex_sha256, normalize_app_compose, normalized_compose_hash}; const COMPOSE: &str = r#"{"manifest_version":"2","name":"test","runner":"docker-compose"}"#; @@ -811,4 +813,13 @@ mod tests { r#"{"manifest_version":"2","name":"test","runner":"docker-compose","no_tee":false}"#; assert!(normalize_app_compose(source, true).is_err()); } + + #[test] + fn compose_hash_uses_legacy_no_tee_normalization() { + let (normalized, _) = normalize_app_compose(COMPOSE, true).unwrap(); + assert_eq!( + normalized_compose_hash(COMPOSE, true).unwrap(), + hex_sha256(&normalized) + ); + } } diff --git a/dstack/vmm/src/one_shot.rs b/dstack/vmm/src/one_shot.rs index f5739fa64..6378c9259 100644 --- a/dstack/vmm/src/one_shot.rs +++ b/dstack/vmm/src/one_shot.rs @@ -6,6 +6,37 @@ use crate::app::{make_sys_config, validate_no_tee_compose, Image, VmConfig, VmWo use crate::config::Config; use crate::main_service; use anyhow::{Context, Result}; +use dstack_types::AppCompose; +use dstack_vmm_rpc::VmConfiguration; + +fn resolve_app_compose(vm_config: &mut VmConfiguration) -> Result { + if vm_config.compose_file.is_empty() { + vm_config.compose_file = serde_json::to_string_pretty(&serde_json::json!({ + "manifest_version": 1, + "name": vm_config.name, + "runner": "none", + "gateway_enabled": !vm_config.gateway_urls.is_empty(), + "tproxy_enabled": false, + "kms_enabled": !vm_config.kms_urls.is_empty(), + "public_logs": false, + "public_sysinfo": false, + "public_tcbinfo": true, + "local_key_provider_enabled": false, + "no_instance_id": false, + "no_tee": vm_config.no_tee, + "secure_time": true, + "features": [], + "allowed_envs": [] + })) + .context("failed to serialize default app compose")?; + } + + let (compose_file, app_compose) = + main_service::normalize_app_compose(&vm_config.compose_file, vm_config.no_tee)?; + vm_config.compose_file = compose_file; + vm_config.no_tee = app_compose.no_tee; + Ok(app_compose) +} pub async fn run_one_shot( vm_config_path: &str, @@ -13,9 +44,7 @@ pub async fn run_one_shot( workdir_option: Option, dry_run: bool, ) -> Result<()> { - use dstack_types::AppCompose; - use dstack_vmm_rpc::VmConfiguration; - use main_service::{create_manifest_from_vm_config, normalize_app_compose}; + use main_service::create_manifest_from_vm_config; // Dynamically allocate CID by scanning running QEMU processes (ps aux method) let mut existing_cids = Vec::new(); @@ -66,12 +95,7 @@ pub async fn run_one_shot( // Parse VM configuration let mut vm_config: VmConfiguration = serde_json::from_str(&vm_config_json) .with_context(|| format!("Failed to parse VM configuration from: {}", vm_config_path))?; - if !vm_config.compose_file.is_empty() { - let (compose_file, app_compose) = - normalize_app_compose(&vm_config.compose_file, vm_config.no_tee)?; - vm_config.compose_file = compose_file; - vm_config.no_tee = app_compose.no_tee; - } + let app_compose = resolve_app_compose(&mut vm_config)?; // Calculate compose_hash using the same logic as main_service let compose_hash = { @@ -121,99 +145,9 @@ pub async fn run_one_shot( let shared_dir = vm_work_dir.shared_dir(); fs_err::create_dir_all(&shared_dir).context("Failed to create shared directory")?; - // Create app compose file content and parse AppCompose instance - let (app_compose_content, app_compose) = if vm_config.compose_file.is_empty() { - // Create default compose JSON directly as string - let gateway_enabled = !vm_config.gateway_urls.is_empty(); - let kms_enabled = !vm_config.kms_urls.is_empty(); - - let default_compose = format!( - r#"{{ -"manifest_version": 1, -"name": "{}", -"runner": "none", -"gateway_enabled": {}, -"tproxy_enabled": false, -"kms_enabled": {}, -"public_logs": false, -"public_sysinfo": false, -"public_tcbinfo": true, -"local_key_provider_enabled": false, -"no_instance_id": false, -"no_tee": {}, -"secure_time": true, -"features": [], -"allowed_envs": [] -}}"#, - vm_config.name, gateway_enabled, kms_enabled, vm_config.no_tee - ); - - // Parse the default compose to get AppCompose instance for gateway_enabled() call - let app_compose: AppCompose = - serde_json::from_str(&default_compose).context("Failed to parse default AppCompose")?; - - (default_compose, app_compose) - } else { - // Parse AppCompose with enhanced error handling for flatten issues - match serde_json::from_str::(&vm_config.compose_file) { - Ok(compose) => (vm_config.compose_file.clone(), compose), - Err(e) => { - let error_msg = e.to_string(); - if error_msg.contains("can only flatten structs and maps") { - anyhow::bail!( - "AppCompose flatten error when parsing compose_file: {} - -This error occurs because the AppCompose struct has a flattened field for gateway settings. -The issue is likely in the compose_file content: - -Common causes: -1. 'gateway_enabled' or 'tproxy_enabled' fields have wrong type (should be boolean) -2. Boolean fields are provided as strings (\"true\" instead of true) -3. Missing quotes around boolean values in JSON - -Example of correct compose_file structure: -{{ -\"manifest_version\": 1, -\"name\": \"my-app\", -\"runner\": \"none\", -\"gateway_enabled\": true, -\"tproxy_enabled\": false, -\"kms_enabled\": false -}} - -Debug: Compose file content (first 200 chars): -{}", - error_msg, - if vm_config.compose_file.len() > 200 { - format!("{}...", &vm_config.compose_file[..200]) - } else { - vm_config.compose_file.clone() - } - ); - } - - return Err(e).with_context(|| { - format!( - "Failed to parse compose_file as AppCompose: {} - -Compose file content (first 200 chars): -{}", - error_msg, - if vm_config.compose_file.len() > 200 { - format!("{}...", &vm_config.compose_file[..200]) - } else { - vm_config.compose_file.clone() - } - ) - }); - } - } - }; - validate_no_tee_compose(manifest.no_tee, &app_compose)?; - // Write the JSON string directly (no serialization needed) - fs_err::write(vm_work_dir.app_compose_path(), app_compose_content) + fs_err::write(vm_work_dir.app_compose_path(), &vm_config.compose_file) .context("Failed to write app compose file")?; // Write other files if present @@ -379,3 +313,24 @@ Compose file content (first 200 chars): Ok(()) } + +#[cfg(test)] +mod tests { + use super::resolve_app_compose; + use dstack_vmm_rpc::VmConfiguration; + + #[test] + fn materializes_default_compose() { + let mut vm_config = VmConfiguration { + name: "test".into(), + no_tee: true, + ..Default::default() + }; + + let app_compose = resolve_app_compose(&mut vm_config).unwrap(); + + assert!(!vm_config.compose_file.is_empty()); + assert!(app_compose.no_tee); + assert!(vm_config.no_tee); + } +} diff --git a/dstack/vmm/src/vmm-cli.py b/dstack/vmm/src/vmm-cli.py index 22a0f1f83..dc8837630 100755 --- a/dstack/vmm/src/vmm-cli.py +++ b/dstack/vmm/src/vmm-cli.py @@ -840,13 +840,15 @@ def create_vm(self, args) -> None: try: compose_json = json.loads(compose_content) except json.JSONDecodeError as err: - raise Exception(f"Invalid app compose file: {err}") from err + raise Exception(f"invalid app compose file: {err}") from err if not isinstance(compose_json, dict): - raise Exception("App compose must be a JSON object") + raise Exception("app compose must be a JSON object") - if args.no_tee is not None: - compose_json["no_tee"] = args.no_tee + if args.no_tee is True and compose_json.get("no_tee") is not True: + compose_json["no_tee"] = True compose_content = json.dumps(compose_json, indent=4, ensure_ascii=False) + elif args.no_tee is False and compose_json.get("no_tee") is True: + raise Exception("--tee conflicts with no_tee=true in app compose") no_tee = bool(compose_json.get("no_tee", False)) diff --git a/os/common/rootfs/dstack-prepare.sh b/os/common/rootfs/dstack-prepare.sh index e36b923ea..e3dba4902 100755 --- a/os/common/rootfs/dstack-prepare.sh +++ b/os/common/rootfs/dstack-prepare.sh @@ -108,7 +108,7 @@ elif modprobe sev-guest 2>/dev/null; then elif modprobe tdx-guest 2>/dev/null; then log "Loaded tdx-guest module" else - log "No TEE guest module is available" + log "no TEE guest module is available" fi # Setup configfs and TSM for TDX attestation