diff --git a/docs/verity-volumes.md b/docs/verity-volumes.md new file mode 100644 index 000000000..6c7dac295 --- /dev/null +++ b/docs/verity-volumes.md @@ -0,0 +1,89 @@ +# Verity data volumes + +A verity volume is a read-only filesystem image protected by dm-verity. The host +attaches untrusted bytes; the guest mounts them only when they match the root +hash measured in `app-compose.json`. + +## Disk format + +A built volume is a raw GPT image: + +```text +p1: DSTACK_VOLUME metadata envelope +p2: filesystem data +p3: dm-verity superblock and hash tree +``` + +The envelope identifies a candidate disk. It is not trusted. The guest passes +the root from the measured app compose to `veritysetup`, so a forged envelope or +partition table cannot substitute different contents. + +## Build + +Pack a directory as a reproducible squashfs volume: + +```bash +dstack verity --dir ./models -o models.img +``` + +Or wrap an existing filesystem image: + +```bash +dstack verity --fs-image ./models.ext4 -o models.img +``` + +The command prints the root and a deploy argument. Place the image in the VMM's +configured `cvm.volumes_dir`, then deploy it by bare file name, root, and guest +mount point: + +```bash +dstack deploy -c docker-compose.yaml \ + --volume models.img:a1b2c3d4...:/run/models +``` + +This adds the following measured entry to `app-compose.json`: + +```json +{ + "verity_volumes": [ + { + "source": "models.img", + "verity_root": "a1b2c3d4...", + "target": "/run/models" + } + ] +} +``` + +The target must be an absolute path on a writable guest filesystem, such as +`/run` or the app data disk. The guest root filesystem itself is read-only. + +## Guest activation + +Before the application starts, `dstack-volume mount-all app-compose.json`: + +1. Scans `/sys/class/block` for disks whose first partition, or whole disk when + unpartitioned, starts with the `DSTACK_VOLUME` magic. +2. Matches the full root advertised by the envelope against the measured root. +3. Opens p2 and p3 with `veritysetup` using the measured root. +4. Reads the first mapped block to force an initial integrity check. +5. Mounts the mapped filesystem read-only at the measured target. + +A required volume that is missing, malformed, or fails verification stops guest +preparation. dm-verity continues verifying blocks lazily as the application +reads them. + +The same `verity_root` may be declared more than once with different targets. +The VMM attaches one disk for that root, and the guest mounts the verified +content at each requested target. + +For diagnostics, `dstack-volume scan` lists recognized disks and +`dstack-volume status app-compose.json` compares requested roots with attached +and active devices. + +## Trust and limitations + +The root hash authenticates bytes, not availability. A malicious host can omit a +volume or cause I/O failures, but cannot silently replace its contents. Volumes +are unencrypted and are intended for public, shareable data; confidential data +requires a separate encryption design. diff --git a/dstack/Cargo.lock b/dstack/Cargo.lock index c51708f84..7e940d40e 100644 --- a/dstack/Cargo.lock +++ b/dstack/Cargo.lock @@ -1157,6 +1157,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + [[package]] name = "crc32fast" version = "1.5.0" @@ -1812,8 +1827,14 @@ dependencies = [ "anyhow", "clap", "dstack-cli-core", + "dstack-types", + "dstack-volume", + "fs-err", + "hex", "serde_json", "tokio", + "tracing", + "tracing-subscriber", ] [[package]] @@ -1821,6 +1842,7 @@ name = "dstack-cli-core" version = "0.6.0" dependencies = [ "anyhow", + "dstack-types", "dstack-vmm-rpc", "http-client", "rustix 0.38.44", @@ -2324,6 +2346,27 @@ dependencies = [ "serde_json", ] +[[package]] +name = "dstack-volume" +version = "0.6.0" +dependencies = [ + "anyhow", + "binrw", + "clap", + "cmd_lib", + "dstack-types", + "fs-err", + "gpt", + "hex", + "serde_json", + "sha2 0.10.9", + "tempfile", + "tokio", + "tracing", + "tracing-subscriber", + "uuid", +] + [[package]] name = "dstackup" version = "0.6.0" @@ -2965,6 +3008,18 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "gpt" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3696fafb1ecdcc2ae3ce337de73e9202806068594b77d22fdf2f3573c5ec2219" +dependencies = [ + "bitflags 2.11.1", + "crc", + "simple-bytes", + "uuid", +] + [[package]] name = "group" version = "0.13.0" @@ -7040,6 +7095,12 @@ version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" +[[package]] +name = "simple-bytes" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c11532d9d241904f095185f35dcdaf930b1427a94d5b01d7002d74ba19b44cc4" + [[package]] name = "siphasher" version = "1.0.3" diff --git a/dstack/Cargo.toml b/dstack/Cargo.toml index 68da4a425..05011b843 100644 --- a/dstack/Cargo.toml +++ b/dstack/Cargo.toml @@ -64,6 +64,7 @@ members = [ "size-parser", "crates/dstack-cli-core", "crates/dstack-cli", + "crates/dstack-volume", "crates/dstackup", "crates/dstack-auth", "crates/api-auth", @@ -81,6 +82,7 @@ dstack-kms-rpc = { path = "kms/rpc" } dstack-guest-agent-rpc = { path = "guest-agent/rpc" } dstack-vmm-rpc = { path = "vmm/rpc" } dstack-cli-core = { path = "crates/dstack-cli-core" } +dstack-volume = { path = "crates/dstack-volume" } dstack-api-auth = { path = "crates/api-auth" } dstack-build-info = { path = "crates/build-info" } cc-eventlog = { path = "cc-eventlog" } diff --git a/dstack/crates/dstack-cli-core/Cargo.toml b/dstack/crates/dstack-cli-core/Cargo.toml index ea74358bc..720946063 100644 --- a/dstack/crates/dstack-cli-core/Cargo.toml +++ b/dstack/crates/dstack-cli-core/Cargo.toml @@ -12,6 +12,7 @@ license.workspace = true anyhow.workspace = true http-client = { workspace = true, features = ["prpc"] } dstack-vmm-rpc.workspace = true +dstack-types.workspace = true serde_json.workspace = true # advisory file locking (flock) for the allowlist/state read-modify-write; # already in the dependency tree transitively, so no extra compile cost. diff --git a/dstack/crates/dstack-cli-core/src/compose.rs b/dstack/crates/dstack-cli-core/src/compose.rs index 4ae252390..4db4f53a6 100644 --- a/dstack/crates/dstack-cli-core/src/compose.rs +++ b/dstack/crates/dstack-cli-core/src/compose.rs @@ -30,6 +30,25 @@ pub fn build_app_compose_with_runtime( kms_enabled: bool, runner: &str, snapshotter: Option<&str>, +) -> String { + build_app_compose_with_runtime_and_volumes( + name, + docker_compose_yaml, + kms_enabled, + runner, + snapshotter, + &[], + ) +} + +/// Build an app-compose manifest with measured verity volume declarations. +pub fn build_app_compose_with_runtime_and_volumes( + name: &str, + docker_compose_yaml: &str, + kms_enabled: bool, + runner: &str, + snapshotter: Option<&str>, + verity_volumes: &[dstack_types::VerityVolume], ) -> String { let mut manifest = json!({ "manifest_version": if runner == "nerdctl-compose" { json!("3") } else { json!(2) }, @@ -52,27 +71,10 @@ pub fn build_app_compose_with_runtime( if let Some(snapshotter) = snapshotter { manifest["snapshotter"] = json!(snapshotter); } + if !verity_volumes.is_empty() { + manifest["verity_volumes"] = json!(verity_volumes); + } // pretty-print via Value's Display (`{:#}`) — infallible, and byte-identical // to serde_json::to_string_pretty (avoids an expect on an unfailable Result). format!("{manifest:#}") } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn nerdctl_manifest_uses_v3_and_records_snapshotter() { - let body = build_app_compose_with_runtime( - "test", - "services: {}", - false, - "nerdctl-compose", - Some("stargz"), - ); - let value: serde_json::Value = serde_json::from_str(&body).unwrap(); - assert_eq!(value["manifest_version"], "3"); - assert_eq!(value["runner"], "nerdctl-compose"); - assert_eq!(value["snapshotter"], "stargz"); - } -} diff --git a/dstack/crates/dstack-cli/Cargo.toml b/dstack/crates/dstack-cli/Cargo.toml index 1ff3c41ba..d1ef71268 100644 --- a/dstack/crates/dstack-cli/Cargo.toml +++ b/dstack/crates/dstack-cli/Cargo.toml @@ -18,5 +18,11 @@ path = "src/main.rs" anyhow.workspace = true clap.workspace = true dstack-cli-core.workspace = true +dstack-volume.workspace = true +dstack-types.workspace = true +fs-err.workspace = true +hex.workspace = true serde_json.workspace = true tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } +tracing.workspace = true +tracing-subscriber = { workspace = true } diff --git a/dstack/crates/dstack-cli/src/main.rs b/dstack/crates/dstack-cli/src/main.rs index 5f047c15a..9624c3a27 100644 --- a/dstack/crates/dstack-cli/src/main.rs +++ b/dstack/crates/dstack-cli/src/main.rs @@ -15,6 +15,7 @@ use clap::{Parser, Subcommand, ValueEnum}; use dstack_cli_core::layout::InstallLayout; use dstack_cli_core::vmm::{Vmm, DEFAULT_HOST}; use dstack_cli_core::{compose, ports, rpc}; +use fs_err as fs; #[derive(Parser)] #[command( @@ -105,6 +106,11 @@ enum Command { /// (host omitted/`auto`/`0` ⇒ a free host port is picked). Repeatable. #[arg(long = "port", value_name = "SPEC")] ports: Vec, + /// attach a verity volume, as printed by `dstack verity`: the file name + /// in the vmm's volumes_dir, its verity_root, and the target + /// (an absolute mount path). Repeatable. + #[arg(long = "volume", value_name = "NAME:VERITY_ROOT:TARGET")] + volumes: Vec, /// deploy in non-KMS mode (ephemeral keys; no KMS required). #[arg(long)] no_kms: bool, @@ -139,10 +145,59 @@ enum Command { }, /// Scaffold a new app project in the current directory. Init, + /// Build a read-only verity data volume from a directory or filesystem image. + /// + /// The build needs no daemon or TEE. It prints a verity_root to paste into + /// the deploy command. See docs/verity-volumes.md. + Verity { + /// Pack this directory into a read-only data volume. + #[arg(long, value_name = "PATH")] + dir: Option, + /// wrap an existing filesystem image instead of building squashfs. The + /// guest mounts it read-only after dm-verity verification. + #[arg(long = "fs-image", value_name = "PATH", conflicts_with = "dir")] + fs_image: Option, + /// where to write the volume. + #[arg(long, short = 'o', default_value = "verity.img")] + output: String, + /// squashfs compression: `none` (the default), `zstd`, or `gzip`. + #[arg(long, value_enum, default_value_t, conflicts_with = "fs_image")] + compress: CompressionArg, + }, +} + +#[derive(Clone, Copy, Default, ValueEnum)] +enum CompressionArg { + #[default] + None, + Zstd, + Gzip, +} + +impl From for dstack_volume::Compression { + fn from(value: CompressionArg) -> Self { + match value { + CompressionArg::None => Self::None, + CompressionArg::Zstd => Self::Zstd, + CompressionArg::Gzip => Self::Gzip, + } + } } #[tokio::main] async fn main() -> Result<()> { + // progress (e.g. `verity` pulling layers) goes to stderr so it never mixes + // with `--json` on stdout. RUST_LOG overrides. + use tracing_subscriber::EnvFilter; + tracing_subscriber::fmt() + .with_writer(std::io::stderr) + .with_target(false) + .without_time() + .with_env_filter( + EnvFilter::try_from_default_env() + .unwrap_or_else(|_| EnvFilter::new("warn,dstack_volume=info")), + ) + .init(); let cli = Cli::parse(); let defaults = LocalDefaults::read(cli.prefix.as_deref()); let use_local_defaults = cli.host.is_none(); @@ -178,6 +233,7 @@ async fn main() -> Result<()> { memory, disk, ports, + volumes, no_kms, allowlist, dry_run, @@ -209,6 +265,7 @@ async fn main() -> Result<()> { memory, disk, &ports, + &volumes, no_kms, allowlist.as_deref(), dry_run, @@ -220,7 +277,99 @@ async fn main() -> Result<()> { } Command::Info { .. } => stub("info"), Command::Init => stub("init"), + Command::Verity { + dir, + fs_image, + output, + compress, + } => cmd_verity(dir.as_deref(), fs_image.as_deref(), &output, compress, json).await, + } +} + +async fn cmd_verity( + dir: Option<&str>, + fs_image: Option<&str>, + output: &str, + compress: CompressionArg, + json: bool, +) -> Result<()> { + let result = dstack_volume::verity(dstack_volume::VerityOptions { + dir: dir.map(std::path::PathBuf::from), + fs_image: fs_image.map(std::path::PathBuf::from), + output: output.into(), + compress: compress.into(), + }) + .await?; + + let volume_size = fs::metadata(&result.output) + .with_context(|| format!("stat {}", result.output.display()))? + .len(); + + if json { + print_json(&serde_json::json!({ + "verityRoot": result.verity_root, + "output": result.output.display().to_string(), + "dataSize": result.data_size, + "volumeSize": volume_size, + })); + return Ok(()); } + + let mib = volume_size as f64 / 1_048_576.0; + println!("wrote {} ({mib:.1} MiB)", result.output.display()); + let file = result + .output + .file_name() + .map(|s| s.to_string_lossy().into_owned()) + .unwrap_or_else(|| result.output.display().to_string()); + // a data volume mounts at a path you choose; it must be writable (the guest + // rootfs is read-only), e.g. under /run. + let target = "/run/models"; + println!("\ncopy {file} into the vmm's volumes_dir, then deploy with:"); + println!( + " dstack deploy -c docker-compose.yaml --volume {file}:{}:{target}", + result.verity_root + ); + println!(" (change {target} to your mount path)"); + Ok(()) +} + +/// Parse a `--volume` spec `NAME:VERITY_ROOT:TARGET`. +/// +/// `NAME` is the volume file in the vmm's volumes_dir. `VERITY_ROOT` and `TARGET` +/// become a measured `verity_volumes` entry in the app-compose, so the guest only +/// seeds content matching the attested root. `dstack verity` prints the exact +/// spec to paste. +/// +/// `TARGET` is an absolute read-only mount path in the guest. +fn parse_volume(spec: &str) -> Result { + let mut parts = spec.splitn(3, ':'); + let name = parts.next().unwrap_or_default(); + let (root, target) = match (parts.next(), parts.next()) { + (Some(root), Some(target)) if !root.is_empty() && !target.is_empty() => (root, target), + _ => bail!("--volume must be NAME:VERITY_ROOT:TARGET (as printed by `dstack verity`), got '{spec}'"), + }; + if name.is_empty() + || name.contains('/') + || name.contains("..") + || name.contains(',') + || name.contains('=') + { + bail!("volume name '{name}' must be a bare file name (no '/', '..', ',', '=')"); + } + if root.len() != 64 || !root.bytes().all(|b| b.is_ascii_hexdigit()) { + bail!("verity_root '{root}' must be 64 hex chars (copy it from `dstack verity`)"); + } + if !target.starts_with('/') { + bail!("target '{target}' must be an absolute path"); + } + let mut verity_root = [0; 32]; + hex::decode_to_slice(root, &mut verity_root).context("decoding verity_root")?; + Ok(dstack_types::VerityVolume { + source: name.to_string(), + verity_root, + target: target.into(), + }) } fn resolve_compose_arg(positional: Option, flagged: Option) -> Result { @@ -241,7 +390,7 @@ struct LocalDefaults { impl LocalDefaults { fn read(prefix: Option<&str>) -> Option { let path = InstallLayout::state_path_for_prefix(prefix); - let body = std::fs::read_to_string(path).ok()?; + let body = fs::read_to_string(path).ok()?; let v: serde_json::Value = serde_json::from_str(&body).ok()?; Some(Self::from_value(&v)) } @@ -284,95 +433,6 @@ impl LocalDefaults { } } -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn parses_local_install_defaults() { - let value = serde_json::json!({ - "client_url": "http://127.0.0.1:19080", - "image": "dstack-0.5.11", - "allowlist_path": "/tmp/dstack/etc/dstack/auth-allowlist.json" - }); - let defaults = LocalDefaults::from_value(&value); - assert_eq!( - defaults.client_url.as_deref(), - Some("http://127.0.0.1:19080") - ); - assert_eq!(defaults.image.as_deref(), Some("dstack-0.5.11")); - assert_eq!( - defaults.allowlist_path().as_deref(), - Some("/tmp/dstack/etc/dstack/auth-allowlist.json") - ); - } - - #[test] - fn reads_local_install_defaults_from_prefix() { - let install_root = - std::env::temp_dir().join(format!("dstack-cli-state-test-{}", std::process::id())); - let state_dir = install_root.join("var/lib/dstack"); - std::fs::create_dir_all(&state_dir).unwrap(); - std::fs::write( - state_dir.join(dstack_cli_core::layout::STATE_FILE), - r#"{ - "client_url": "http://127.0.0.1:29080", - "image": "dstack-0.5.12", - "allowlist_path": "/tmp/custom-dstack/etc/dstack/auth-allowlist.json" - }"#, - ) - .unwrap(); - - let prefix = dstack_cli_core::layout::path_string(&install_root); - let defaults = LocalDefaults::read(Some(&prefix)).unwrap(); - assert_eq!( - defaults.client_url.as_deref(), - Some("http://127.0.0.1:29080") - ); - assert_eq!(defaults.image.as_deref(), Some("dstack-0.5.12")); - assert_eq!( - defaults.allowlist_path().as_deref(), - Some("/tmp/custom-dstack/etc/dstack/auth-allowlist.json") - ); - - let _ = std::fs::remove_dir_all(install_root); - } - - #[test] - fn parses_phala_style_deploy_flags() { - let cli = Cli::parse_from([ - "dstack", - "deploy", - "-n", - "hello", - "-c", - "examples/hello-nginx/docker-compose.yaml", - "--port", - "8080:80", - ]); - match cli.command { - Command::Deploy { - compose, - compose_file, - name, - memory, - ports, - .. - } => { - assert_eq!(compose, None); - assert_eq!( - compose_file.as_deref(), - Some("examples/hello-nginx/docker-compose.yaml") - ); - assert_eq!(name, "hello"); - assert_eq!(memory, 2048); - assert_eq!(ports, vec!["8080:80"]); - } - _ => panic!("expected deploy command"), - } - } -} - #[allow(clippy::too_many_arguments)] async fn cmd_deploy( host: &str, @@ -384,6 +444,7 @@ async fn cmd_deploy( memory: u32, disk: u32, port_specs: &[String], + volume_specs: &[String], no_kms: bool, allowlist: Option<&str>, dry_run: bool, @@ -394,21 +455,30 @@ async fn cmd_deploy( if matches!(runner, ComposeRunner::DockerCompose) && snapshotter.is_some() { bail!("--snapshotter is only supported with --runner nerdctl-compose"); } - let yaml = std::fs::read_to_string(compose_path) + let yaml = fs::read_to_string(compose_path) .with_context(|| format!("reading compose file '{compose_path}'"))?; - let app_compose = compose::build_app_compose_with_runtime( + + let port_maps = port_specs + .iter() + .map(|s| ports::parse_port(s)) + .collect::>>()?; + let parsed_volumes = volume_specs + .iter() + .map(|s| parse_volume(s)) + .collect::>>()?; + dstack_types::validate_verity_volumes(&parsed_volumes).map_err(anyhow::Error::msg)?; + + // each --volume declares a measured verity_volumes entry, so the built + // app-compose (and thus app_id) binds the attested roots. + let app_compose = compose::build_app_compose_with_runtime_and_volumes( name, &yaml, !no_kms, runner.as_str(), snapshotter.map(Snapshotter::as_str), + &parsed_volumes, ); - let mut port_maps = Vec::new(); - for spec in port_specs { - port_maps.push(ports::parse_port(spec)?); - } - let mut cfg = rpc::VmConfiguration { name: name.to_string(), image: image.unwrap_or_default().to_string(), @@ -587,3 +657,148 @@ fn trunc(s: &str, n: usize) -> String { out } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_local_install_defaults() { + let value = serde_json::json!({ + "client_url": "http://127.0.0.1:19080", + "image": "dstack-0.5.11", + "allowlist_path": "/tmp/dstack/etc/dstack/auth-allowlist.json" + }); + let defaults = LocalDefaults::from_value(&value); + assert_eq!( + defaults.client_url.as_deref(), + Some("http://127.0.0.1:19080") + ); + assert_eq!(defaults.image.as_deref(), Some("dstack-0.5.11")); + assert_eq!( + defaults.allowlist_path().as_deref(), + Some("/tmp/dstack/etc/dstack/auth-allowlist.json") + ); + } + + #[test] + fn reads_local_install_defaults_from_prefix() { + let install_root = + std::env::temp_dir().join(format!("dstack-cli-state-test-{}", std::process::id())); + let state_dir = install_root.join("var/lib/dstack"); + fs::create_dir_all(&state_dir).unwrap(); + fs::write( + state_dir.join(dstack_cli_core::layout::STATE_FILE), + r#"{ + "client_url": "http://127.0.0.1:29080", + "image": "dstack-0.5.12", + "allowlist_path": "/tmp/custom-dstack/etc/dstack/auth-allowlist.json" + }"#, + ) + .unwrap(); + + let prefix = dstack_cli_core::layout::path_string(&install_root); + let defaults = LocalDefaults::read(Some(&prefix)).unwrap(); + assert_eq!( + defaults.client_url.as_deref(), + Some("http://127.0.0.1:29080") + ); + assert_eq!(defaults.image.as_deref(), Some("dstack-0.5.12")); + assert_eq!( + defaults.allowlist_path().as_deref(), + Some("/tmp/custom-dstack/etc/dstack/auth-allowlist.json") + ); + + let _ = fs::remove_dir_all(install_root); + } + + #[test] + fn parses_volume_specs() { + let root = "a".repeat(64); + let data = parse_volume(&format!("weights.img:{root}:/models/llama")).unwrap(); + assert_eq!(data.source, "weights.img"); + assert_eq!(data.verity_root, [0xaa; 32]); + assert_eq!(data.target, std::path::Path::new("/models/llama")); + + assert!(parse_volume("weights.img").is_err()); // missing verity_root:target + assert!(parse_volume(&format!("weights.img:{root}")).is_err()); // missing target + assert!(parse_volume(&format!("../escape.img:{root}:/models")).is_err()); // path separator + assert!(parse_volume("x.img:nothex:/models").is_err()); // verity_root not hex + assert!(parse_volume(&format!("x.img:{root}:docker")).is_err()); // docker seed removed + assert!(parse_volume(&format!("x.img:{root}:relative/path")).is_err()); // bad target + } + + #[test] + fn parses_phala_style_deploy_flags() { + let cli = Cli::parse_from([ + "dstack", + "deploy", + "-n", + "hello", + "-c", + "examples/hello-nginx/docker-compose.yaml", + "--port", + "8080:80", + ]); + match cli.command { + Command::Deploy { + compose, + compose_file, + name, + memory, + ports, + .. + } => { + assert_eq!(compose, None); + assert_eq!( + compose_file.as_deref(), + Some("examples/hello-nginx/docker-compose.yaml") + ); + assert_eq!(name, "hello"); + assert_eq!(memory, 2048); + assert_eq!(ports, vec!["8080:80"]); + } + _ => panic!("expected deploy command"), + } + } + + #[test] + fn parses_verity_fs_image_flag() { + let cli = Cli::parse_from(["dstack", "verity", "--fs-image", "rootfs.ext4"]); + match cli.command { + Command::Verity { dir, fs_image, .. } => { + assert_eq!(dir, None); + assert_eq!(fs_image.as_deref(), Some("rootfs.ext4")); + } + _ => panic!("expected verity command"), + } + + assert!(Cli::try_parse_from([ + "dstack", + "verity", + "--dir", + "data", + "--fs-image", + "rootfs.ext4" + ]) + .is_err()); + assert!(Cli::try_parse_from([ + "dstack", + "verity", + "--fs-image", + "rootfs.ext4", + "--compress", + "zstd" + ]) + .is_err()); + assert!(Cli::try_parse_from([ + "dstack", + "verity", + "--dir", + "data", + "--compress", + "invalid" + ]) + .is_err()); + } +} diff --git a/dstack/crates/dstack-volume/Cargo.toml b/dstack/crates/dstack-volume/Cargo.toml new file mode 100644 index 000000000..76262b6fe --- /dev/null +++ b/dstack/crates/dstack-volume/Cargo.toml @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "dstack-volume" +version.workspace = true +edition.workspace = true +license.workspace = true + +[dependencies] +anyhow = { workspace = true, features = ["std"] } +binrw.workspace = true +cmd_lib.workspace = true +clap.workspace = true +dstack-types.workspace = true +tokio = { workspace = true, features = ["rt"] } +serde_json = { workspace = true, features = ["std"] } +sha2 = { workspace = true, features = ["std"] } +hex = { workspace = true, features = ["std"] } +fs-err.workspace = true +gpt = "4.1.0" +tempfile.workspace = true +tracing.workspace = true +tracing-subscriber.workspace = true +uuid.workspace = true + +[dev-dependencies] diff --git a/dstack/crates/dstack-volume/src/bin/dstack-volume.rs b/dstack/crates/dstack-volume/src/bin/dstack-volume.rs new file mode 100644 index 000000000..367fc33fc --- /dev/null +++ b/dstack/crates/dstack-volume/src/bin/dstack-volume.rs @@ -0,0 +1,523 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! Discover and activate volumes supplied to a dstack guest. +//! +//! A volume is recognized by a `DSTACK_VOLUME` envelope at the start of its +//! first partition, or at the start of the whole disk when it has no partition +//! table. Everything read from a disk is untrusted: kind handlers use the +//! measured app compose as their source of policy and cryptographic identity. + +use std::collections::HashMap; +use std::io::Read; +use std::os::unix::ffi::OsStrExt; +use std::os::unix::fs::MetadataExt; +use std::path::{Path, PathBuf}; + +use anyhow::{bail, Context, Result}; +use clap::{Parser, Subcommand}; +use cmd_lib::{run_cmd, run_fun}; +use dstack_types::{AppCompose, VerityVolume as RequestedVolume}; +use dstack_volume::volume_format::{ + DstackVolumeHeader, DSTACK_VOLUME_HEADER_SIZE, DSTACK_VOLUME_KIND_VERITY, DSTACK_VOLUME_MAGIC, +}; +use fs_err::{self as fs, File}; +use tracing::{info, warn}; + +const MAX_DISKS: usize = 64; + +#[derive(Clone, Debug, PartialEq, Eq)] +struct BlockDisk { + path: PathBuf, + partitions: Vec<(u32, PathBuf)>, +} + +#[derive(Debug)] +struct VerityVolume { + data: PathBuf, + hash: PathBuf, + root_hash: [u8; 32], +} + +#[derive(Parser)] +#[command(about = "Discover and activate dstack verity volumes")] +struct Cli { + #[command(subcommand)] + command: VolumeCommand, +} + +#[derive(Subcommand)] +enum VolumeCommand { + /// Activate every verity volume required by the measured app compose. + MountAll { + #[arg(default_value = "app-compose.json")] + compose: PathBuf, + }, + /// List recognized dstack volume devices. + Scan, + /// Compare required volumes with attached and active devices. + Status { + #[arg(default_value = "app-compose.json")] + compose: PathBuf, + }, +} + +fn main() -> Result<()> { + tracing_subscriber::fmt().init(); + match Cli::parse().command { + VolumeCommand::MountAll { compose } => mount_all(compose), + VolumeCommand::Scan => scan(), + VolumeCommand::Status { compose } => status(compose), + } +} + +fn read_compose(compose_path: &Path) -> Result { + // Deserialize the complete shared type before probing any untrusted disk. + // This validates roots and targets as part of parsing the measured compose. + serde_json::from_slice(&fs::read(compose_path)?) + .with_context(|| format!("parsing {}", compose_path.display())) +} + +fn prepare_volumes() -> Result> { + let _ = run_cmd!(modprobe dm-verity); + let _ = run_cmd!(udevadm settle --timeout=5); + discover_volumes() +} + +fn mount_all(compose_path: PathBuf) -> Result<()> { + let compose = read_compose(&compose_path)?; + dstack_types::validate_verity_volumes(&compose.verity_volumes).map_err(anyhow::Error::msg)?; + if compose.verity_volumes.is_empty() { + return Ok(()); + } + let volumes = prepare_volumes()?; + info!( + found = volumes.len(), + requested = compose.verity_volumes.len(), + "discovered dstack volumes" + ); + let mut active_roots: HashMap<[u8; 32], PathBuf> = HashMap::new(); + for (index, requested) in compose.verity_volumes.iter().enumerate() { + if let Some(mapped) = active_roots.get(&requested.verity_root) { + mount_volume(requested, mapped).with_context(|| { + format!( + "failed to mount required volume {index} at {}", + requested.target.display() + ) + })?; + continue; + } + let mapped = activate_requested(index, requested, &volumes).with_context(|| { + format!( + "failed to activate required volume {index} at {}", + requested.target.display() + ) + })?; + active_roots.insert(requested.verity_root, mapped); + } + Ok(()) +} + +fn scan() -> Result<()> { + for volume in prepare_volumes()? { + println!( + "{}\tdata={}\thash={}", + hex::encode(volume.root_hash), + volume.data.display(), + volume.hash.display() + ); + } + Ok(()) +} + +fn status(compose_path: PathBuf) -> Result<()> { + let compose = read_compose(&compose_path)?; + let volumes = prepare_volumes()?; + for (index, requested) in compose.verity_volumes.iter().enumerate() { + let attached = volumes + .iter() + .any(|volume| volume.root_hash == requested.verity_root); + let expected_root = hex::encode(requested.verity_root); + let mut active = false; + for mapper_index in 0..compose.verity_volumes.len() { + let mapper_name = format!("dstack-verity{mapper_index}"); + if Path::new("/dev/mapper").join(&mapper_name).exists() + && mapping_root(&mapper_name)?.eq_ignore_ascii_case(&expected_root) + { + active = true; + break; + } + } + println!( + "{index}\troot={}\ttarget={}\tattached={attached}\tactive={active}", + hex::encode(requested.verity_root), + requested.target.display() + ); + } + Ok(()) +} + +fn discover_volumes() -> Result> { + let mut found = Vec::new(); + let disks = list_disks()?; + if disks.len() > MAX_DISKS { + warn!( + found = disks.len(), + limit = MAX_DISKS, + "block-device scan truncated" + ); + } + for disk in disks.into_iter().take(MAX_DISKS) { + // A partitioned disk has exactly one envelope location: partition 1. + // Only a disk with no kernel-recognized partitions is probed at offset 0. + let probe = if disk.partitions.is_empty() { + &disk.path + } else if let Some((_, path)) = disk.partitions.iter().find(|(number, _)| *number == 1) { + path + } else { + continue; + }; + let Some(header) = read_header(probe)? else { + continue; + }; + match header.kind { + DSTACK_VOLUME_KIND_VERITY => match resolve_verity(&disk, header) { + Ok(volume) => found.push(volume), + Err(err) => { + warn!(disk = %disk.path.display(), error = %format_args!("{err:#}"), "ignoring malformed verity volume") + } + }, + kind => warn!(kind, disk = %disk.path.display(), "ignoring unsupported volume kind"), + } + } + Ok(found) +} + +fn list_disks() -> Result> { + list_disks_at(Path::new("/sys/class/block"), Path::new("/dev")) +} + +fn list_disks_at(sysfs: &Path, devfs: &Path) -> Result> { + struct Entry { + name: std::ffi::OsString, + sysfs_path: PathBuf, + partition: Option, + } + + let mut entries = Vec::new(); + for entry in fs::read_dir(sysfs)? { + let entry = entry?; + let class_path = entry.path(); + let partition = if class_path.join("partition").exists() { + Some( + fs::read_to_string(class_path.join("partition"))? + .trim() + .parse()?, + ) + } else { + None + }; + entries.push(Entry { + name: entry.file_name(), + sysfs_path: fs::canonicalize(class_path)?, + partition, + }); + } + + let mut disks = entries + .iter() + .filter(|entry| entry.partition.is_none() && entry.sysfs_path.join("device").exists()) + .map(|entry| BlockDisk { + path: devfs.join(&entry.name), + partitions: entries + .iter() + .filter_map(|partition| { + let number = partition.partition?; + (partition.sysfs_path.parent() == Some(entry.sysfs_path.as_path())) + .then(|| (number, devfs.join(&partition.name))) + }) + .collect(), + }) + .collect::>(); + for disk in &mut disks { + disk.partitions.sort_by_key(|(number, _)| *number); + } + disks.sort_by(|left, right| left.path.cmp(&right.path)); + Ok(disks) +} + +fn read_header(path: &Path) -> Result> { + let mut file = match File::open(path) { + Ok(file) => file, + Err(err) => { + warn!(path = %path.display(), %err, "cannot open block device"); + return Ok(None); + } + }; + let mut bytes = [0u8; DSTACK_VOLUME_HEADER_SIZE]; + if let Err(err) = file.read_exact(&mut bytes) { + if err.kind() == std::io::ErrorKind::UnexpectedEof { + return Ok(None); + } + return Err(err).with_context(|| format!("reading {}", path.display())); + } + parse_header(&bytes).with_context(|| format!("parsing envelope on {}", path.display())) +} + +fn parse_header(bytes: &[u8]) -> Result> { + if bytes.len() < DSTACK_VOLUME_HEADER_SIZE || &bytes[..16] != DSTACK_VOLUME_MAGIC { + return Ok(None); + } + Ok(Some(DstackVolumeHeader::decode(bytes)?)) +} + +fn resolve_verity(disk: &BlockDisk, header: DstackVolumeHeader) -> Result { + if disk.partitions.is_empty() { + bail!("raw verity layout is not defined by kind version 1"); + } + let partition = |number| { + disk.partitions + .iter() + .find(|(n, _)| *n == number) + .map(|(_, path)| path.clone()) + .with_context(|| format!("missing partition {number}")) + }; + Ok(VerityVolume { + data: partition(2)?, + hash: partition(3)?, + root_hash: header.root_hash, + }) +} + +fn activate_requested( + index: usize, + requested: &RequestedVolume, + volumes: &[VerityVolume], +) -> Result { + let candidate = volumes + .iter() + .find(|volume| volume.root_hash == requested.verity_root) + .context("no attached volume advertises the measured root")?; + + let mapper_name = format!("dstack-verity{index}"); + let mapped = PathBuf::from(format!("/dev/mapper/{mapper_name}")); + let expected_root = hex::encode(requested.verity_root); + if mapped.exists() { + if mapping_root(&mapper_name)?.eq_ignore_ascii_case(&expected_root) { + if let Err(err) = + verify_first_block(&mapped).and_then(|_| mount_volume(requested, &mapped)) + { + let _ = run_cmd!(veritysetup close $mapper_name); + return Err(err); + } + info!(mapper = mapper_name, "reused active verity mapping"); + return Ok(mapped); + } + run_cmd!(veritysetup close $mapper_name).context("closing stale verity mapping")?; + } + + // The on-disk root only selected a candidate. Pass the root from the + // measured compose to veritysetup, which is the actual trust decision. + let data = &candidate.data; + let hash = &candidate.hash; + run_cmd!(veritysetup open $data $mapper_name $hash $expected_root) + .context("opening dm-verity volume")?; + if let Err(err) = verify_first_block(&mapped).and_then(|_| mount_volume(requested, &mapped)) { + let _ = run_cmd!(veritysetup close $mapper_name); + return Err(err); + } + Ok(mapped) +} + +fn verify_first_block(path: &Path) -> Result<()> { + let mut file = File::open(path).with_context(|| format!("opening {}", path.display()))?; + let mut block = [0u8; 4096]; + file.read_exact(&mut block) + .with_context(|| format!("verifying first block of {}", path.display())) +} + +fn mount_volume(requested: &RequestedVolume, mapped: &Path) -> Result<()> { + let fs_type = run_fun!(blkid -o value -s TYPE $mapped).unwrap_or_default(); + let target = &requested.target; + fs::create_dir_all(target)?; + if is_mountpoint(target)? { + ensure_mounted_from(target, mapped)?; + } else { + mount_read_only(mapped, target, fs_type.trim())?; + } + info!(root = %hex::encode(requested.verity_root), target = %target.display(), "mounted verity volume"); + Ok(()) +} + +fn mount_read_only(device: &Path, target: &Path, fs_type: &str) -> Result<()> { + let options = if matches!(fs_type, "ext3" | "ext4") { + "ro,noload" + } else { + "ro" + }; + if fs_type.is_empty() { + run_cmd!(mount -o $options $device $target).context("mounting verity volume")?; + } else { + run_cmd!(mount -t $fs_type -o $options $device $target) + .context("mounting verity volume")?; + } + Ok(()) +} + +fn is_mountpoint(path: &Path) -> Result { + Ok(mountpoint_device(path)?.is_some()) +} + +fn ensure_mounted_from(target: &Path, device: &Path) -> Result<()> { + let mounted = mountpoint_device(target)?.context("mount point disappeared")?; + let expected = device_number(fs::metadata(device)?.rdev()); + if mounted != expected { + bail!( + "{} is mounted from device {}:{}, expected {}:{}", + target.display(), + mounted.0, + mounted.1, + expected.0, + expected.1 + ); + } + Ok(()) +} + +fn mountpoint_device(path: &Path) -> Result> { + let target = path.as_os_str().as_bytes(); + for line in fs::read_to_string("/proc/self/mountinfo")?.lines() { + let mut fields = line.split_ascii_whitespace(); + let Some(device) = fields.nth(2) else { + continue; + }; + let Some(mountpoint) = fields.nth(1) else { + continue; + }; + if unescape_mountinfo(mountpoint.as_bytes()) == target { + let (major, minor) = device + .split_once(':') + .context("invalid mountinfo device number")?; + return Ok(Some((major.parse()?, minor.parse()?))); + } + } + Ok(None) +} + +fn device_number(device: u64) -> (u64, u64) { + let major = ((device >> 8) & 0xfff) | ((device >> 32) & 0xffff_f000); + let minor = (device & 0xff) | ((device >> 12) & 0xffff_ff00); + (major, minor) +} + +fn unescape_mountinfo(value: &[u8]) -> Vec { + let mut decoded = Vec::with_capacity(value.len()); + let mut index = 0; + while index < value.len() { + if value[index] == b'\\' && index + 3 < value.len() { + let octal = &value[index + 1..index + 4]; + if octal.iter().all(|byte| matches!(byte, b'0'..=b'7')) { + decoded.push((octal[0] - b'0') * 64 + (octal[1] - b'0') * 8 + (octal[2] - b'0')); + index += 4; + continue; + } + } + decoded.push(value[index]); + index += 1; + } + decoded +} + +fn mapping_root(mapper_name: &str) -> Result { + let status = run_fun!(veritysetup status $mapper_name)?; + status + .lines() + .find_map(|line| { + let line = line.trim(); + line.strip_prefix("root hash:") + .or_else(|| line.strip_prefix("Root hash:")) + }) + .map(|root| root.trim().to_string()) + .context("verity mapping status has no root hash") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cli_uses_default_compose_for_mount_all() { + let cli = Cli::try_parse_from(["dstack-volume", "mount-all"]).unwrap(); + let VolumeCommand::MountAll { compose } = cli.command else { + panic!("unexpected command"); + }; + assert_eq!(compose, Path::new("app-compose.json")); + } + + fn header() -> DstackVolumeHeader { + DstackVolumeHeader::new_verity([0x5a; 32]) + } + + #[test] + fn ignores_non_volume_data() { + assert_eq!( + parse_header(&[0u8; DSTACK_VOLUME_HEADER_SIZE]).unwrap(), + None + ); + } + + #[test] + fn verity_kind_uses_second_and_third_partitions() { + let disk = BlockDisk { + path: "/dev/test".into(), + partitions: vec![ + (1, "/dev/test1".into()), + (2, "/dev/test2".into()), + (3, "/dev/test3".into()), + ], + }; + let volume = resolve_verity(&disk, header()).unwrap(); + assert_eq!(volume.data, Path::new("/dev/test2")); + assert_eq!(volume.hash, Path::new("/dev/test3")); + } + + #[test] + fn verity_kind_rejects_undefined_raw_layout() { + let disk = BlockDisk { + path: "/dev/test".into(), + partitions: vec![], + }; + assert!(resolve_verity(&disk, header()).is_err()); + } + + #[test] + fn decodes_mountinfo_escapes() { + assert_eq!(unescape_mountinfo(b"/run/my\\040volume"), b"/run/my volume"); + } + + #[test] + fn discovers_partitions_from_sysfs_parentage() -> Result<()> { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir()?; + let sysfs = temp.path().join("sys/class/block"); + let devices = temp.path().join("sys/devices/block/vda"); + let partition = devices.join("vda1"); + fs::create_dir_all(&sysfs)?; + fs::create_dir_all(devices.join("device"))?; + fs::create_dir_all(&partition)?; + fs::write(partition.join("partition"), "1\n")?; + symlink(&devices, sysfs.join("vda"))?; + symlink(&partition, sysfs.join("vda1"))?; + + assert_eq!( + list_disks_at(&sysfs, Path::new("/dev"))?, + vec![BlockDisk { + path: "/dev/vda".into(), + partitions: vec![(1, "/dev/vda1".into())], + }] + ); + Ok(()) + } +} diff --git a/dstack/crates/dstack-volume/src/lib.rs b/dstack/crates/dstack-volume/src/lib.rs new file mode 100644 index 000000000..5a8aa5f28 --- /dev/null +++ b/dstack/crates/dstack-volume/src/lib.rs @@ -0,0 +1,95 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! Build, describe, and activate dstack volumes. +//! +//! The output is a reproducible, dm-verity-protected raw disk image containing +//! a filesystem that the guest mounts read-only at a measured path. +//! +//! The build needs no docker daemon and no TEE, and it's reproducible: the same +//! inputs always give the same `verity_root`. The first partition contains a +//! generic `DSTACK_VOLUME` envelope, followed by data and verity partitions. +//! So anyone can recompute the root +//! and check it against `app-compose.json`, without trusting the builder. See +//! docs/verity-volumes.md. + +use std::path::PathBuf; + +use anyhow::{bail, Context, Result}; + +mod volume; +pub mod volume_format; + +pub use volume::Compression; + +/// A fixed dm-verity salt. +/// +/// The root is a function of the squashfs bytes and this salt, so keeping the +/// salt constant is what lets anyone recompute the root. It isn't a secret: +/// veritysetup writes it into the on-disk verity superblock anyway. +const VERITY_SALT: &str = "0000000000000000000000000000000000000000000000000000000000000000"; + +pub struct VerityOptions { + /// Build a volume from this directory. + pub dir: Option, + /// wrap an existing filesystem image as the verity data partition. This is + /// for hand-built ext4/xfs/etc. images; `dstack verity --dir` still produces + /// squashfs by default. + pub fs_image: Option, + pub output: PathBuf, + /// squashfs compression (default: none — zero decompression at read time). + pub compress: Compression, +} + +pub struct VerityResult { + pub verity_root: String, + pub data_size: u64, + pub output: PathBuf, +} + +pub async fn verity(opts: VerityOptions) -> Result { + match (opts.dir, opts.fs_image) { + (Some(dir), None) => verity_dir(dir, opts.output, opts.compress).await, + (None, Some(fs_image)) => verity_fs_image(fs_image, opts.output).await, + _ => bail!("give exactly one source: --dir or --fs-image "), + } +} + +/// Bake a directory tree into a reproducible squashfs data volume. +async fn verity_dir(dir: PathBuf, output: PathBuf, compress: Compression) -> Result { + if !dir.is_dir() { + bail!("--dir '{}' is not a directory", dir.display()); + } + let out = output.clone(); + let built = tokio::task::spawn_blocking(move || { + volume::build_volume(&dir, &out, VERITY_SALT, compress) + }) + .await + .context("the build task failed")??; + + Ok(VerityResult { + verity_root: built.verity_root, + data_size: built.data_size, + output, + }) +} + +/// Wrap an already-built filesystem image. The guest discovers and mounts the +/// filesystem only after dm-verity is active. +async fn verity_fs_image(fs_image: PathBuf, output: PathBuf) -> Result { + if !fs_image.is_file() { + bail!("--fs-image '{}' is not a file", fs_image.display()); + } + let out = output.clone(); + let built = + tokio::task::spawn_blocking(move || volume::build_fs_image(&fs_image, &out, VERITY_SALT)) + .await + .context("the build task failed")??; + + Ok(VerityResult { + verity_root: built.verity_root, + data_size: built.data_size, + output, + }) +} diff --git a/dstack/crates/dstack-volume/src/volume.rs b/dstack/crates/dstack-volume/src/volume.rs new file mode 100644 index 000000000..6c8175788 --- /dev/null +++ b/dstack/crates/dstack-volume/src/volume.rs @@ -0,0 +1,504 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! Pack a directory into a reproducible, verity-protected disk image. +//! +//! The output is a raw disk with three deterministic GPT partitions: +//! +//! 1. a generic `DSTACK_VOLUME` metadata block +//! 2. the filesystem image +//! 3. the dm-verity superblock and hash tree +//! +//! Keeping the verity data and hash devices in separate partitions means the +//! guest never has to inspect filesystem metadata to find the hash tree. The +//! partition table is only a locator hint; the measured verity root is still the +//! content identity. + +use std::collections::BTreeMap; +use std::path::Path; + +use crate::volume_format::{DstackVolumeHeader, DSTACK_VOLUME_HEADER_SIZE}; +use anyhow::{bail, Context, Result}; +use cmd_lib::{run_cmd, run_fun}; +use fs_err as fs; +use sha2::{Digest, Sha256}; +use uuid::Uuid; + +/// Fixed build timestamp (2024-01-01), so the image is the same no matter when +/// it's built. It never shows up at runtime: `mksquashfs -all-time` normalizes +/// the store's own timestamps away. +const EPOCH: &str = "1704067200"; +const BLOCK: u64 = 4096; +const SECTOR: u64 = 512; +// 1 MiB alignment. +const PARTITION_ALIGNMENT_SECTORS: u64 = 2048; +// The `gpt` crate writes the primary/backup headers and partition arrays. We +// still reserve enough trailing sectors in the raw image for the backup array +// (128 entries * 128 bytes) plus the backup header. +const GPT_ENTRY_SECTORS: u64 = 32; +const VOLUME_HEADER_SIZE: usize = DSTACK_VOLUME_HEADER_SIZE; + +#[derive(Clone, Copy)] +pub enum Compression { + /// No compression, so nothing is decompressed at read time (the default). + None, + Zstd, + Gzip, +} + +impl Compression { + fn args(self) -> Vec<&'static str> { + match self { + // disable every compressible section: inodes, data, fragments, + // xattrs, id table. + Compression::None => vec!["-noI", "-noD", "-noF", "-noX", "-noId"], + Compression::Zstd => vec!["-comp", "zstd"], + Compression::Gzip => vec![], + } + } +} + +pub struct BuiltVolume { + pub verity_root: String, + /// Size of the filesystem data partition. + pub data_size: u64, +} + +/// Build `output`: a GPT disk image with the squashfs data partition and a +/// separate dm-verity hash partition. +/// +/// `store` is the directory to pack; `salt` fixes the verity root. +/// +/// The verity UUID is derived from the squashfs bytes, so two different volumes +/// get two different UUIDs. That matters because a VM can mount several volumes +/// at once, and a fixed UUID would be shared by all of them. +pub fn build_volume( + store: &Path, + output: &Path, + salt_hex: &str, + compress: Compression, +) -> Result { + require_tool("mksquashfs")?; + require_tool("veritysetup")?; + + let tmp = tempfile::tempdir().context("creating volume scratch dir")?; + let data_path = tmp.path().join("data.fs"); + + // 1. reproducible squashfs. + let compression_args = compress.args(); + run_cmd!( + mksquashfs $store $data_path $[compression_args] + -all-time $EPOCH -mkfs-time $EPOCH -noappend -no-progress -xattrs + >/dev/null + ) + .context("running mksquashfs")?; + + // 2. the verity data region must be block-aligned; pad the squashfs up. + let bytes_used = squashfs_bytes_used(&data_path)?; + seal_data_image(&data_path, bytes_used, output, salt_hex) +} + +/// Wrap an existing filesystem image in the same partitioned verity disk format. +/// +/// The input image is copied to a scratch file and padded to a 4096-byte verity +/// block boundary. Its filesystem type is otherwise opaque to the verity layer. +pub fn build_fs_image(fs_image: &Path, output: &Path, salt_hex: &str) -> Result { + require_tool("veritysetup")?; + let len = fs::metadata(fs_image) + .with_context(|| format!("stat {}", fs_image.display()))? + .len(); + if len == 0 { + bail!("filesystem image '{}' is empty", fs_image.display()); + } + + let tmp = tempfile::tempdir().context("creating volume scratch dir")?; + let data_path = tmp.path().join("data.fs"); + fs::copy(fs_image, &data_path) + .with_context(|| format!("copying {} into scratch", fs_image.display()))?; + seal_data_image(&data_path, len, output, salt_hex) +} + +fn seal_data_image( + data_path: &Path, + data_len: u64, + output: &Path, + salt_hex: &str, +) -> Result { + let hash_tmp = tempfile::tempdir().context("creating verity hash scratch dir")?; + let hash_path = hash_tmp.path().join("verity.hash"); + let data_size = data_len.div_ceil(BLOCK) * BLOCK; + { + let f = fs::OpenOptions::new().write(true).open(data_path)?; + f.set_len(data_size)?; + } + + // Build the verity hash device as a separate image. At runtime this is + // partition 2, so no --hash-offset is needed and the guest does not parse + // filesystem metadata before verity is active. + // Pin the superblock UUID too: veritysetup randomizes it otherwise, and we + // want the whole image reproducible, not just the root. The UUID sits in the + // hash tree, not the hashed data, so it never changes the root. + let uuid = uuid_from_data(data_path, data_size)?; + let out = run_fun!( + veritysetup format --salt $salt_hex --uuid $uuid + --hash sha256 --format 1 --data-block-size 4096 --hash-block-size 4096 + $data_path $hash_path + ) + .context("running veritysetup format")?; + let verity_root = + parse_root_hash(&out).context("could not find the root hash in veritysetup output")?; + + // Wrap the two blobs in a deterministic GPT disk image. Partition 1 is the + // generic volume envelope, partition 2 is data, and partition 3 is verity. + build_partitioned_image(data_path, data_size, &hash_path, output, &verity_root)?; + + Ok(BuiltVolume { + verity_root, + data_size, + }) +} + +/// squashfs superblock: `bytes_used` is a little-endian u64 at offset 40. +fn squashfs_bytes_used(path: &Path) -> Result { + use std::io::{Read, Seek, SeekFrom}; + let mut f = fs::File::open(path)?; + let mut magic = [0u8; 4]; + f.read_exact(&mut magic)?; + if &magic != b"hsqs" { + bail!("not a squashfs image (bad magic)"); + } + f.seek(SeekFrom::Start(40))?; + let mut buf = [0u8; 8]; + f.read_exact(&mut buf)?; + Ok(u64::from_le_bytes(buf)) +} + +/// Derive a UUID from the first `len` bytes of `path`. +fn uuid_from_data(path: &Path, len: u64) -> Result { + Ok(uuid_from_file(path, len)?.to_string()) +} + +/// Deterministic, version-4-shaped UUID from arbitrary domain-separated +/// material. +fn uuid_from_material(parts: &[&[u8]]) -> Uuid { + let mut h = Sha256::new(); + for part in parts { + h.update((part.len() as u64).to_le_bytes()); + h.update(part); + } + uuid_from_digest(&h.finalize()) +} + +fn uuid_from_file(path: &Path, len: u64) -> Result { + use std::io::Read; + let mut f = fs::File::open(path)?; + let mut h = Sha256::new(); + let mut remaining = len; + let mut buf = vec![0u8; 1 << 20]; + while remaining > 0 { + let n = remaining.min(buf.len() as u64) as usize; + f.read_exact(&mut buf[..n])?; + h.update(&buf[..n]); + remaining -= n as u64; + } + Ok(uuid_from_digest(&h.finalize())) +} + +fn uuid_from_digest(d: &[u8]) -> Uuid { + let mut u = [0u8; 16]; + u.copy_from_slice(&d[..16]); + u[6] = (u[6] & 0x0f) | 0x40; // version 4 + u[8] = (u[8] & 0x3f) | 0x80; // RFC 4122 variant + Uuid::from_bytes(u) +} + +#[derive(Clone, Copy)] +struct Partition { + first_lba: u64, + last_lba: u64, +} + +struct GptLayout { + total_lbas: u64, + metadata: Partition, + data: Partition, + hash: Partition, +} + +impl GptLayout { + fn new(data_size: u64, hash_size: u64) -> Result { + if data_size == 0 || hash_size == 0 { + bail!("data and hash images must be non-empty"); + } + let data_sectors = data_size.div_ceil(SECTOR); + let hash_sectors = hash_size.div_ceil(SECTOR); + + let metadata_first = PARTITION_ALIGNMENT_SECTORS; + let metadata_last = metadata_first + VOLUME_HEADER_SIZE as u64 / SECTOR - 1; + let data_first = align_up(metadata_last + 1, PARTITION_ALIGNMENT_SECTORS); + let data_last = data_first + data_sectors - 1; + let hash_first = align_up(data_last + 1, PARTITION_ALIGNMENT_SECTORS); + let hash_last = hash_first + hash_sectors - 1; + + // Leave room for the backup GPT entry array and header at the end. + let min_lbas = hash_last + 1 + GPT_ENTRY_SECTORS + 1; + let total_lbas = align_up(min_lbas, PARTITION_ALIGNMENT_SECTORS); + Ok(Self { + total_lbas, + metadata: Partition { + first_lba: metadata_first, + last_lba: metadata_last, + }, + data: Partition { + first_lba: data_first, + last_lba: data_last, + }, + hash: Partition { + first_lba: hash_first, + last_lba: hash_last, + }, + }) + } + + fn total_bytes(&self) -> u64 { + self.total_lbas * SECTOR + } +} + +fn align_up(value: u64, alignment: u64) -> u64 { + value.div_ceil(alignment) * alignment +} + +fn build_partitioned_image( + data_path: &Path, + data_size: u64, + hash_path: &Path, + output: &Path, + root_hash: &str, +) -> Result<()> { + let hash_size = fs::metadata(hash_path) + .with_context(|| format!("stat {}", hash_path.display()))? + .len(); + let layout = GptLayout::new(data_size, hash_size)?; + + let mut out = fs::OpenOptions::new() + .create(true) + .truncate(true) + .read(true) + .write(true) + .open(output) + .with_context(|| format!("creating {}", output.display()))?; + out.set_len(layout.total_bytes())?; + + out = write_gpt(out, &layout, root_hash)?; + + write_volume_header(&mut out, layout.metadata.first_lba * SECTOR, root_hash)?; + copy_into(data_path, &mut out, layout.data.first_lba * SECTOR)?; + copy_into(hash_path, &mut out, layout.hash.first_lba * SECTOR)?; + Ok(()) +} + +fn write_gpt(mut out: fs::File, layout: &GptLayout, root_hash: &str) -> Result { + let protective_size = (layout.total_lbas - 1).min(u32::MAX as u64) as u32; + gpt::mbr::ProtectiveMBR::with_lb_size(protective_size) + .overwrite_lba0(&mut out) + .context("writing protective MBR")?; + + let disk_uuid = uuid_from_material(&[b"dstack-verity-disk", root_hash.as_bytes()]); + let mut disk = gpt::GptConfig::new() + .writable(true) + .logical_block_size(gpt::disk::LogicalBlockSize::Lb512) + .create_from_device(out, Some(disk_uuid)) + .context("initializing GPT")?; + + let mut parts = BTreeMap::new(); + parts.insert( + 1, + gpt::partition::Partition { + part_type_guid: gpt::partition_types::LINUX_FS, + part_guid: uuid_from_material(&[b"dstack-volume-metadata", root_hash.as_bytes()]), + first_lba: layout.metadata.first_lba, + last_lba: layout.metadata.last_lba, + flags: 0, + name: "dstack-volume".to_string(), + }, + ); + parts.insert( + 2, + gpt::partition::Partition { + part_type_guid: gpt::partition_types::LINUX_FS, + part_guid: uuid_from_material(&[b"dstack-verity-data", root_hash.as_bytes()]), + first_lba: layout.data.first_lba, + last_lba: layout.data.last_lba, + flags: 0, + name: "dstack-data".to_string(), + }, + ); + parts.insert( + 3, + gpt::partition::Partition { + part_type_guid: gpt::partition_types::LINUX_FS, + part_guid: uuid_from_material(&[b"dstack-verity-hash", root_hash.as_bytes()]), + first_lba: layout.hash.first_lba, + last_lba: layout.hash.last_lba, + flags: 0, + name: "dstack-verity".to_string(), + }, + ); + disk.update_partitions(parts) + .context("installing GPT partitions")?; + disk.write().context("writing GPT") +} + +fn write_volume_header(out: &mut fs::File, offset: u64, root_hash: &str) -> Result<()> { + use std::io::{Seek, SeekFrom, Write}; + + let root = hex::decode(root_hash).context("decoding verity root hash")?; + let root: [u8; 32] = root + .as_slice() + .try_into() + .context("verity root must be a 32-byte SHA-256 digest")?; + let header = DstackVolumeHeader::new_verity(root) + .encode() + .context("encoding volume header")?; + + out.seek(SeekFrom::Start(offset))?; + out.write_all(&header)?; + Ok(()) +} + +fn copy_into(src: &Path, out: &mut fs::File, offset: u64) -> Result<()> { + use std::io::{Read, Seek, SeekFrom, Write}; + let mut input = fs::File::open(src)?; + out.seek(SeekFrom::Start(offset))?; + let mut buf = vec![0u8; 1 << 20]; + loop { + let n = input.read(&mut buf)?; + if n == 0 { + break; + } + out.write_all(&buf[..n])?; + } + Ok(()) +} + +fn parse_root_hash(output: &str) -> Option { + output + .lines() + .find_map(|l| l.strip_prefix("Root hash:")) + .map(|v| v.trim().to_string()) +} + +fn require_tool(name: &str) -> Result<()> { + let present = run_cmd!(which $name >/dev/null 2>&1).is_ok(); + if !present { + bail!("`{name}` not found on PATH (install squashfs-tools / cryptsetup)"); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_veritysetup_root() { + let sample = "VERITY header information for x\nUUID: \nHash type: 1\n\ + Data blocks: 10\nRoot hash: abc123def\n"; + assert_eq!(parse_root_hash(sample).as_deref(), Some("abc123def")); + } + + #[test] + fn uuid_is_deterministic_and_content_specific() { + use std::io::Write; + let mut a = tempfile::NamedTempFile::new().unwrap(); + a.write_all(b"hello world data").unwrap(); + let mut b = tempfile::NamedTempFile::new().unwrap(); + b.write_all(b"different data..").unwrap(); + let ua = uuid_from_data(a.path(), 16).unwrap(); + assert_eq!(ua, uuid_from_data(a.path(), 16).unwrap()); // deterministic + assert_ne!(ua, uuid_from_data(b.path(), 16).unwrap()); // content-specific + assert_eq!(ua.len(), 36); + assert_eq!(&ua[14..15], "4"); // version nibble + } + + #[test] + fn gpt_layout_uses_three_aligned_partitions() { + let layout = GptLayout::new(4096, 8192).unwrap(); + assert_eq!(layout.metadata.first_lba, PARTITION_ALIGNMENT_SECTORS); + assert_eq!(layout.metadata.last_lba, PARTITION_ALIGNMENT_SECTORS + 7); + assert_eq!(layout.data.first_lba, PARTITION_ALIGNMENT_SECTORS * 2); + assert_eq!(layout.data.last_lba, PARTITION_ALIGNMENT_SECTORS * 2 + 7); + assert_eq!(layout.hash.first_lba, PARTITION_ALIGNMENT_SECTORS * 3); + assert_eq!(layout.hash.last_lba, PARTITION_ALIGNMENT_SECTORS * 3 + 15); + assert!(layout.total_lbas - 1 - GPT_ENTRY_SECTORS > layout.hash.last_lba); + } + + #[test] + fn partitioned_image_is_valid_gpt() -> Result<()> { + use std::io::{Read, Seek, SeekFrom, Write}; + + let tmp = tempfile::tempdir()?; + let data_path = tmp.path().join("data.fs"); + let hash_path = tmp.path().join("verity.hash"); + let image_path = tmp.path().join("volume.img"); + let data = vec![0x11; 4096]; + let hash = vec![0x22; 8192]; + fs::File::create(&data_path)?.write_all(&data)?; + fs::File::create(&hash_path)?.write_all(&hash)?; + + let root_hash = "abababababababababababababababababababababababababababababababab"; + build_partitioned_image( + &data_path, + data.len() as u64, + &hash_path, + &image_path, + root_hash, + )?; + + let disk = gpt::GptConfig::new() + .logical_block_size(gpt::disk::LogicalBlockSize::Lb512) + .open(&image_path)?; + assert_eq!( + *disk.guid(), + uuid_from_material(&[b"dstack-verity-disk", root_hash.as_bytes()]) + ); + + let metadata = disk.partitions().get(&1).unwrap(); + let data_partition = disk.partitions().get(&2).unwrap(); + let hash_partition = disk.partitions().get(&3).unwrap(); + assert_eq!(metadata.name, "dstack-volume"); + assert_eq!(data_partition.name, "dstack-data"); + assert_eq!(hash_partition.name, "dstack-verity"); + assert_eq!( + data_partition.part_guid, + uuid_from_material(&[b"dstack-verity-data", root_hash.as_bytes()]) + ); + assert_eq!( + hash_partition.part_guid, + uuid_from_material(&[b"dstack-verity-hash", root_hash.as_bytes()]) + ); + assert_eq!(metadata.first_lba, PARTITION_ALIGNMENT_SECTORS); + assert_eq!(data_partition.first_lba, PARTITION_ALIGNMENT_SECTORS * 2); + assert_eq!(hash_partition.first_lba, PARTITION_ALIGNMENT_SECTORS * 3); + + let mut img = fs::File::open(&image_path)?; + let mut header = [0u8; VOLUME_HEADER_SIZE]; + img.seek(SeekFrom::Start(metadata.first_lba * SECTOR))?; + img.read_exact(&mut header)?; + assert_eq!(&header[..16], b"DSTACK_VOLUME\0\0\0"); + let decoded = DstackVolumeHeader::decode(&header)?; + assert_eq!(decoded.root_hash.as_slice(), hex::decode(root_hash)?); + let mut buf = vec![0; data.len()]; + img.seek(SeekFrom::Start(data_partition.first_lba * SECTOR))?; + img.read_exact(&mut buf)?; + assert_eq!(buf, data); + let mut buf = vec![0; hash.len()]; + img.seek(SeekFrom::Start(hash_partition.first_lba * SECTOR))?; + img.read_exact(&mut buf)?; + assert_eq!(buf, hash); + + Ok(()) + } +} diff --git a/dstack/crates/dstack-volume/src/volume_format.rs b/dstack/crates/dstack-volume/src/volume_format.rs new file mode 100644 index 000000000..df9fd4328 --- /dev/null +++ b/dstack/crates/dstack-volume/src/volume_format.rs @@ -0,0 +1,93 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! On-disk dstack volume envelope shared by builders and guests. + +use std::io::Cursor; + +use binrw::{binrw, BinRead, BinWrite}; + +pub const DSTACK_VOLUME_MAGIC: &[u8; 16] = b"DSTACK_VOLUME\0\0\0"; +pub const DSTACK_VOLUME_HEADER_SIZE: usize = 4096; +pub const DSTACK_VOLUME_KIND_VERITY: u32 = 1; + +#[binrw] +#[brw(little, magic = b"DSTACK_VOLUME\0\0\0")] +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DstackVolumeHeader { + pub kind: u32, + pub root_hash: [u8; 32], +} + +impl DstackVolumeHeader { + pub fn new_verity(root_hash: [u8; 32]) -> Self { + Self { + kind: DSTACK_VOLUME_KIND_VERITY, + root_hash, + } + } + + pub fn encode(&self) -> binrw::BinResult<[u8; DSTACK_VOLUME_HEADER_SIZE]> { + let mut block = [0u8; DSTACK_VOLUME_HEADER_SIZE]; + self.write(&mut Cursor::new(&mut block[..]))?; + Ok(block) + } + + pub fn decode(block: &[u8]) -> binrw::BinResult { + if block.len() < DSTACK_VOLUME_HEADER_SIZE { + return Err(binrw::Error::AssertFail { + pos: 0, + message: format!( + "volume header is truncated: {} < {DSTACK_VOLUME_HEADER_SIZE}", + block.len() + ), + }); + } + Self::read(&mut Cursor::new(block)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use dstack_types::VerityVolume; + + #[test] + fn volume_header_round_trip() { + let expected = DstackVolumeHeader::new_verity([0x5a; 32]); + let encoded = expected.encode().unwrap(); + assert_eq!(&encoded[..16], b"DSTACK_VOLUME\0\0\0"); + assert_eq!(DstackVolumeHeader::decode(&encoded).unwrap(), expected); + } + + #[test] + fn verity_volume_validates_root_and_target_during_deserialization( + ) -> Result<(), serde_json::Error> { + let volume: VerityVolume = serde_json::from_value(serde_json::json!({ + "source": "models.img", + "verity_root": "5a".repeat(32), + "target": "/run/models" + }))?; + assert_eq!(volume.verity_root, [0x5a; 32]); + assert_eq!(volume.target, std::path::PathBuf::from("/run/models")); + assert_eq!( + serde_json::to_value(&volume)?["verity_root"], + "5a".repeat(32) + ); + + assert!(serde_json::from_value::(serde_json::json!({ + "source": "models.img", + "verity_root": "abcd", + "target": "/run/models" + })) + .is_err()); + assert!(serde_json::from_value::(serde_json::json!({ + "source": "models.img", + "verity_root": "5a".repeat(32), + "target": "relative/path" + })) + .is_err()); + Ok(()) + } +} diff --git a/dstack/dstack-mr/cli/src/main.rs b/dstack/dstack-mr/cli/src/main.rs index 146112b3d..258d99f5d 100644 --- a/dstack/dstack-mr/cli/src/main.rs +++ b/dstack/dstack-mr/cli/src/main.rs @@ -70,6 +70,10 @@ struct MachineConfig { #[arg(long, default_value = "1")] num_nics: u32, + /// Number of virtio-blk verity volumes + #[arg(long, default_value = "0")] + num_verity_volumes: u32, + /// Disable hotplug #[arg(long, default_value = "false")] hotplug_off: Bool, @@ -138,6 +142,7 @@ fn main() -> Result<()> { .num_gpus(config.num_gpus) .num_nvswitches(config.num_nvswitches) .num_nics(config.num_nics) + .num_verity_volumes(config.num_verity_volumes) .hotplug_off(config.hotplug_off) .root_verity(config.root_verity) .maybe_qemu_version(config.qemu_version.clone()) diff --git a/dstack/dstack-mr/src/acpi.rs b/dstack/dstack-mr/src/acpi.rs index db8f52845..f9df12a2f 100644 --- a/dstack/dstack-mr/src/acpi.rs +++ b/dstack/dstack-mr/src/acpi.rs @@ -57,6 +57,16 @@ impl Machine<'_> { "virtio-blk-pci,drive=hd1", ]); + // Verity volumes are emitted after hd1 and before networking, matching + // dstack-vmm's QEMU device order. + for i in 0..self.num_verity_volumes { + let id = format!("vol{i}"); + cmd.arg("-drive").arg(format!( + "file={dummy_disk},if=none,id={id},format=raw,readonly=on" + )); + cmd.arg("-device").arg(format!("virtio-blk-pci,drive={id}")); + } + // One virtio-net-pci per NIC. Emitted at the same position and, for the // default single-NIC case, with the exact same args as the previous // hardcoded layout so RTMR0 stays byte-for-byte unchanged. diff --git a/dstack/dstack-mr/src/machine.rs b/dstack/dstack-mr/src/machine.rs index 59d061084..59b91c088 100644 --- a/dstack/dstack-mr/src/machine.rs +++ b/dstack/dstack-mr/src/machine.rs @@ -33,6 +33,9 @@ pub struct Machine<'a> { /// predate this field keep the historical single-NIC layout. #[builder(default = 1)] pub num_nics: u32, + /// Number of virtio-blk verity volumes attached before the NICs. + #[builder(default)] + pub num_verity_volumes: u32, pub hotplug_off: bool, pub root_verity: bool, #[builder(default)] diff --git a/dstack/dstack-mr/src/tdx.rs b/dstack/dstack-mr/src/tdx.rs index abd6c7f99..e2243a584 100644 --- a/dstack/dstack-mr/src/tdx.rs +++ b/dstack/dstack-mr/src/tdx.rs @@ -83,6 +83,7 @@ fn select_mrtd(measurement: &TdxOsImageMeasurement, vm_config: &VmConfig) -> Res .hugepages(vm_config.hugepages) .num_gpus(vm_config.num_gpus) .num_nics(vm_config.num_nics) + .num_verity_volumes(vm_config.num_verity_volumes) .num_nvswitches(vm_config.num_nvswitches) .host_share_mode(vm_config.host_share_mode.clone()) .ovmf_variant(measurement.tdvf.ovmf_variant) @@ -474,6 +475,7 @@ pub fn tdx_measurements_for_image_dir_without_rtmr0( .hugepages(vm_config.hugepages) .num_gpus(vm_config.num_gpus) .num_nics(vm_config.num_nics) + .num_verity_volumes(vm_config.num_verity_volumes) .num_nvswitches(vm_config.num_nvswitches) .host_share_mode(vm_config.host_share_mode.clone()) .ovmf_variant(ovmf_variant) @@ -570,6 +572,7 @@ pub fn tdx_measurements_for_image_dir_with_acpi_hashes( .hugepages(vm_config.hugepages) .num_gpus(vm_config.num_gpus) .num_nics(vm_config.num_nics) + .num_verity_volumes(vm_config.num_verity_volumes) .num_nvswitches(vm_config.num_nvswitches) .host_share_mode(vm_config.host_share_mode.clone()) .ovmf_variant(ovmf_variant) diff --git a/dstack/dstack-types/src/lib.rs b/dstack/dstack-types/src/lib.rs index 183a3501a..69870a5d9 100644 --- a/dstack/dstack-types/src/lib.rs +++ b/dstack/dstack-types/src/lib.rs @@ -123,6 +123,78 @@ pub struct AppCompose { /// of silently ignoring the requirements. #[serde(default, skip_serializing_if = "Option::is_none")] pub requirements: Option, + /// Read-only, dm-verity-protected volumes pre-seeded into the CVM. Each + /// `verity_root` is measured (it is part of these compose bytes), so the + /// guest only mounts content matching the attested app. See + /// docs/verity-volumes.md. + #[serde(default)] + pub verity_volumes: Vec, +} + +/// A pre-baked, read-only dm-verity volume attached to the CVM. +#[derive(Deserialize, Serialize, Debug, Clone)] +pub struct VerityVolume { + /// Bare image file name resolved by the VMM under `cvm.volumes_dir`. + pub source: String, + /// dm-verity root hash (hex): the volume's content identity and integrity + /// check. The guest matches attached devices against it. + #[serde(with = "hex_bytes")] + pub verity_root: [u8; 32], + /// Absolute path where the volume's filesystem is mounted. + #[serde(deserialize_with = "deserialize_absolute_path")] + pub target: std::path::PathBuf, +} + +/// Reject ambiguous mount declarations before any disk is attached or +/// activated. The same root may intentionally be mounted at multiple targets, +/// but a target can only be owned by one volume. +pub fn validate_verity_volumes(volumes: &[VerityVolume]) -> Result<(), String> { + let mut targets = std::collections::HashSet::new(); + for volume in volumes { + if !targets.insert(&volume.target) { + return Err(format!( + "duplicate verity volume target {}", + volume.target.display() + )); + } + } + Ok(()) +} + +fn deserialize_absolute_path<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + let value = String::deserialize(deserializer)?; + let path = std::path::PathBuf::from(&value); + if !path.is_absolute() { + return Err(serde::de::Error::custom(format!( + "volume target must be an absolute path, got '{value}'" + ))); + } + Ok(path) +} + +#[cfg(test)] +mod verity_volume_tests { + use super::{validate_verity_volumes, VerityVolume}; + + fn volume(root: u8, target: &str) -> VerityVolume { + VerityVolume { + source: format!("{root}.img"), + verity_root: [root; 32], + target: target.into(), + } + } + + #[test] + fn allows_duplicate_roots_but_rejects_duplicate_targets() { + validate_verity_volumes(&[volume(1, "/a"), volume(1, "/b")]).unwrap(); + assert!(validate_verity_volumes(&[volume(1, "/a"), volume(2, "/a")]) + .unwrap_err() + .contains("duplicate verity volume target")); + validate_verity_volumes(&[volume(1, "/a"), volume(2, "/b")]).unwrap(); + } } #[derive(Deserialize, Serialize, Debug, Clone, Copy, PartialEq, Eq)] @@ -893,6 +965,10 @@ fn is_default_num_nics(n: &u32) -> bool { *n == default_num_nics() } +fn is_zero(n: &u32) -> bool { + *n == 0 +} + #[derive(Deserialize, Serialize, Debug, Clone)] pub struct VmConfig { #[serde(with = "hex_bytes", default)] @@ -926,6 +1002,11 @@ pub struct VmConfig { skip_serializing_if = "is_default_num_nics" )] pub num_nics: u32, + /// Number of read-only verity volume devices attached to the guest. Each + /// volume adds a virtio-blk PCI device before the NICs and therefore + /// changes the measured ACPI/DSDT layout. + #[serde(default, skip_serializing_if = "is_zero")] + pub num_verity_volumes: u32, #[serde(default)] pub hotplug_off: bool, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -1921,7 +2002,7 @@ mod platform_tests { } #[cfg(test)] -mod vm_config_num_nics_tests { +mod vm_config_device_count_tests { use super::VmConfig; fn legacy_json() -> serde_json::Value { @@ -1937,6 +2018,7 @@ mod vm_config_num_nics_tests { fn legacy_config_without_num_nics_defaults_to_one() { let cfg: VmConfig = serde_json::from_value(legacy_json()).unwrap(); assert_eq!(cfg.num_nics, 1); + assert_eq!(cfg.num_verity_volumes, 0); } #[test] @@ -1959,4 +2041,20 @@ mod vm_config_num_nics_tests { let serialized = serde_json::to_value(&cfg).unwrap(); assert_eq!(serialized.get("num_nics").and_then(|v| v.as_u64()), Some(2)); } + + #[test] + fn verity_volume_count_is_serialized_only_when_nonzero() { + let mut cfg: VmConfig = serde_json::from_value(legacy_json()).unwrap(); + let serialized = serde_json::to_value(&cfg).unwrap(); + assert!(serialized.get("num_verity_volumes").is_none()); + + cfg.num_verity_volumes = 2; + let serialized = serde_json::to_value(&cfg).unwrap(); + assert_eq!( + serialized + .get("num_verity_volumes") + .and_then(|v| v.as_u64()), + Some(2) + ); + } } diff --git a/dstack/guest-agent/src/rpc_service.rs b/dstack/guest-agent/src/rpc_service.rs index 396be82e7..d9a8faa33 100644 --- a/dstack/guest-agent/src/rpc_service.rs +++ b/dstack/guest-agent/src/rpc_service.rs @@ -754,6 +754,7 @@ mod tests { swap_size: 0, port_policy: Default::default(), requirements: None, + verity_volumes: Vec::new(), }; let dummy_appcompose_wrapper = AppComposeWrapper { diff --git a/dstack/verifier/src/verification.rs b/dstack/verifier/src/verification.rs index 3b1e8fad7..34c7a835e 100644 --- a/dstack/verifier/src/verification.rs +++ b/dstack/verifier/src/verification.rs @@ -333,6 +333,7 @@ impl CvmVerifier { .hugepages(vm_config.hugepages) .num_gpus(vm_config.num_gpus) .num_nics(vm_config.num_nics) + .num_verity_volumes(vm_config.num_verity_volumes) .num_nvswitches(vm_config.num_nvswitches) .host_share_mode(vm_config.host_share_mode.clone()) .ovmf_variant(ovmf_variant) diff --git a/dstack/vmm/rpc/proto/vmm_rpc.proto b/dstack/vmm/rpc/proto/vmm_rpc.proto index 17c6cba00..ff4b8ac25 100644 --- a/dstack/vmm/rpc/proto/vmm_rpc.proto +++ b/dstack/vmm/rpc/proto/vmm_rpc.proto @@ -118,6 +118,7 @@ message VmConfiguration { optional NetworkingConfig networking = 18; // Per-VM networking overrides. If set, wins over singular networking. repeated NetworkingConfig networks = 19; + reserved 20; } // Per-VM networking configuration. diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 797185622..b7b58e50a 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -79,6 +79,14 @@ pub struct PortMapping { pub to: u16, } +/// An extra disk attached to the VM (e.g. a pre-baked verity volume). `source` +/// is an absolute host path already resolved and validated by the VMM against +/// `cvm.volumes_dir`. +#[derive(Deserialize, Serialize, Debug, Clone)] +pub struct VmVolume { + pub source: String, +} + #[derive(Deserialize, Serialize, Clone, Builder, Debug)] pub struct Manifest { pub id: String, @@ -104,6 +112,8 @@ pub struct Manifest { pub no_tee: bool, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub networks: Vec, + #[serde(default)] + pub volumes: Vec, } impl Manifest { @@ -1181,7 +1191,7 @@ fn append_boot_separator(path: &std::path::Path) { if !path.exists() { return; } - let Ok(mut file) = std::fs::OpenOptions::new().append(true).open(path) else { + let Ok(mut file) = fs::OpenOptions::new().append(true).open(path) else { return; }; let timestamp = humantime::format_rfc3339_seconds(std::time::SystemTime::now()); @@ -1204,7 +1214,7 @@ fn rotate_serial_log(work_dir: &VmWorkDir, max_bytes: u64) { return; } let history = work_dir.serial_history_file(); - let Ok(mut file) = std::fs::OpenOptions::new() + let Ok(mut file) = fs::OpenOptions::new() .create(true) .append(true) .open(&history) @@ -1390,6 +1400,7 @@ fn make_vm_config( // ACPI/DSDT layout and therefore RTMR0. Measure the interface count so the // verifier reconstructs the exact device layout. let num_nics = resolved_networks(manifest, &cfg.cvm).len() as u32; + let num_verity_volumes = manifest.volumes.len() as u32; let mut config = serde_json::to_value(dstack_types::VmConfig { os_image_hash, cpu_count: effective_vcpus, @@ -1402,6 +1413,7 @@ fn make_vm_config( num_gpus: gpus.gpus.len() as u32, num_nvswitches: gpus.bridges.len() as u32, num_nics, + num_verity_volumes, host_share_mode: cfg.cvm.host_share_mode.clone(), hotplug_off: cfg.cvm.qemu_hotplug_off, image: Some(manifest.image.clone()), @@ -1616,6 +1628,7 @@ mod tests { gateway_urls: vec![], no_tee: false, networks: vec![], + volumes: vec![], } } @@ -1721,6 +1734,31 @@ mod tests { Ok(()) } + #[test] + fn vm_measurement_config_includes_verity_volume_count() -> Result<()> { + let config = test_tdx_config()?; + let mut manifest = test_manifest(2048); + manifest.volumes = vec![ + VmVolume { + source: "/volumes/a.img".into(), + }, + VmVolume { + source: "/volumes/b.img".into(), + }, + ]; + let vm_config = make_vm_config( + &config, + &manifest, + &test_tdx_image(true), + &hex_of(0x22, 32), + None, + None, + )?; + + assert_eq!(vm_config["num_verity_volumes"], 2); + Ok(()) + } + #[test] fn tdx_auto_variant_uses_legacy_for_low_non_2g_memory() -> Result<()> { let config = test_tdx_config()?; @@ -1877,6 +1915,7 @@ mod tests { gateway_urls: vec![], no_tee: false, networks: vec![], + volumes: vec![], }; let mr_config = MrConfigV3::new( diff --git a/dstack/vmm/src/app/qemu.rs b/dstack/vmm/src/app/qemu.rs index dd45cbf7f..a84cc4d64 100644 --- a/dstack/vmm/src/app/qemu.rs +++ b/dstack/vmm/src/app/qemu.rs @@ -175,6 +175,10 @@ fn virtio_pci_device(device: &str, snp: bool) -> String { } } +struct PreparedVolume { + source: String, +} + fn needs_qemu_swtpm(key_provider: KeyProviderKind, simulator: Option<&TeeSimulatorConfig>) -> bool { if !matches!(key_provider, KeyProviderKind::Tpm) { return false; @@ -184,11 +188,11 @@ fn needs_qemu_swtpm(key_provider: KeyProviderKind, simulator: Option<&TeeSimulat Some(TeeVariant::DstackGcpTdx | TeeVariant::DstackAwsNitroTpm) ) } - struct PreparedQemuLaunch { workdir: VmWorkDir, platform: CvmPlatform, networks: Vec, + volumes: Vec, hugepage_numa_nodes: Option>, gpu_numa_nodes: HashMap, numa_cpus: Option, @@ -213,6 +217,14 @@ impl PreparedQemuLaunch { let platform = cfg.resolved_platform(); let networks = resolved_networks(&vm.manifest, cfg); validate_resolved_networks(&networks)?; + let volumes = vm + .manifest + .volumes + .iter() + .map(|volume| PreparedVolume { + source: volume.source.clone(), + }) + .collect(); let hugepage_numa_nodes = if vm.manifest.hugepages { Some(hugepage_numa_nodes(gpus)?) @@ -277,6 +289,7 @@ impl PreparedQemuLaunch { workdir, platform, networks, + volumes, hugepage_numa_nodes, gpu_numa_nodes, numa_cpus, @@ -420,6 +433,7 @@ impl QemuCommandBuilder<'_> { let mut command = self.base_command(); self.configure_rootfs(&mut command)?; self.configure_data_disk(&mut command); + self.configure_volumes(&mut command); self.configure_networking(&mut command)?; self.vm.configure_smbios(&mut command, self.cfg); self.configure_tpm_and_vsock(&mut command); @@ -538,6 +552,26 @@ impl QemuCommandBuilder<'_> { )); } + fn configure_volumes(&self, command: &mut Command) { + // Sources are host paths already validated by the VMM. Attach extra + // volumes after the data disk and before networking, matching the + // established device order. + for (index, volume) in self.prepared.volumes.iter().enumerate() { + let id = format!("vol{index}"); + let drive = format!( + "file={},if=none,id={id},format=raw,readonly=on", + volume.source + ); + + let device = format!("virtio-blk-pci,drive={id}"); + command + .arg("-drive") + .arg(drive) + .arg("-device") + .arg(virtio_pci_device(&device, self.is_amd_sev_snp())); + } + } + fn configure_networking(&self, command: &mut Command) -> Result<()> { let hostfwd_index = self .prepared @@ -950,10 +984,10 @@ mod tests { use super::{ amd_sev_snp_memory_backend_arg, needs_qemu_swtpm, parse_amd_sev_snp_qmp_capabilities, - virtio_pci_device, PreparedQemuLaunch, QemuCommandBuilder, VmConfig, + virtio_pci_device, PreparedQemuLaunch, PreparedVolume, QemuCommandBuilder, VmConfig, }; use crate::app::image::{Image, ImageInfo}; - use crate::app::{GpuConfig, Manifest, PortMapping, VmWorkDir}; + use crate::app::{GpuConfig, Manifest, PortMapping, VmVolume, VmWorkDir}; use crate::config::{Config, CvmPlatform, Protocol, DEFAULT_CONFIG}; use dstack_types::{KeyProviderKind, TeeSimulatorConfig, TeeVariant}; @@ -1037,6 +1071,9 @@ mod tests { gateway_urls: vec![], no_tee: true, networks: vec![], + volumes: vec![VmVolume { + source: "/does-not-exist/volume.img".into(), + }], }, image: Image { info: ImageInfo { @@ -1071,6 +1108,9 @@ mod tests { workdir: VmWorkDir::new("/does-not-exist/vm-1"), platform: CvmPlatform::Tdx, networks: vec![config.cvm.networking.clone(), config.cvm.networking.clone()], + volumes: vec![PreparedVolume { + source: "/does-not-exist/volume.img".into(), + }], hugepage_numa_nodes: None, gpu_numa_nodes: HashMap::new(), numa_cpus: None, @@ -1103,6 +1143,27 @@ mod tests { .args .windows(2) .any(|args| args == ["-append", "console=hvc0"])); + assert!(process.args.windows(2).any(|args| { + args == [ + "-drive", + "file=/does-not-exist/volume.img,if=none,id=vol0,format=raw,readonly=on", + ] + })); + assert!(process + .args + .iter() + .any(|arg| { arg == "virtio-blk-pci,drive=vol0" })); + let volume_position = process + .args + .iter() + .position(|arg| arg.contains("id=vol0")) + .unwrap(); + let network_position = process + .args + .iter() + .position(|arg| arg == "-netdev") + .unwrap(); + assert!(volume_position < network_position); let netdevs = process .args .windows(2) diff --git a/dstack/vmm/src/config.rs b/dstack/vmm/src/config.rs index 658074648..5c81f77c7 100644 --- a/dstack/vmm/src/config.rs +++ b/dstack/vmm/src/config.rs @@ -332,6 +332,12 @@ pub struct CvmConfig { #[serde(default = "default_serial_history_max_bytes")] #[serde(with = "size_parser::human_size")] pub serial_history_max_bytes: u64, + + /// Directory holding attachable volume images (e.g. pre-baked verity + /// volumes). A deploy may only attach files under this directory, referenced + /// by bare file name. Empty (the default) disables volume attachment. + #[serde(default)] + pub volumes_dir: String, } /// SMBIOS product information configuration. diff --git a/dstack/vmm/src/discovery.rs b/dstack/vmm/src/discovery.rs index 8a2440f39..181e7da52 100644 --- a/dstack/vmm/src/discovery.rs +++ b/dstack/vmm/src/discovery.rs @@ -8,6 +8,7 @@ //! directory so that CLI tools can discover all running instances on the host. use anyhow::{Context, Result}; +use fs_err as fs; use serde::{Deserialize, Serialize}; use std::path::{Path, PathBuf}; use tracing::{info, warn}; @@ -99,7 +100,7 @@ impl DiscoveryRegistration { impl Drop for DiscoveryRegistration { fn drop(&mut self) { - match std::fs::remove_file(&self.path) { + match fs::remove_file(&self.path) { Ok(()) => info!("unregistered VMM instance at {}", self.path.display()), Err(e) => warn!( "failed to remove discovery file {}: {e}", @@ -112,7 +113,7 @@ impl Drop for DiscoveryRegistration { /// Clean up stale discovery files from dead processes. pub fn cleanup_stale_registrations() { let dir = discovery_dir(); - let entries = match std::fs::read_dir(dir) { + let entries = match fs::read_dir(dir) { Ok(e) => e, Err(_) => return, }; @@ -122,7 +123,7 @@ pub fn cleanup_stale_registrations() { if path.extension().and_then(|e| e.to_str()) != Some("json") { continue; } - let content = match std::fs::read_to_string(&path) { + let content = match fs::read_to_string(&path) { Ok(c) => c, Err(_) => continue, }; @@ -138,7 +139,7 @@ pub fn cleanup_stale_registrations() { info.pid, path.display() ); - let _ = std::fs::remove_file(&path); + let _ = fs::remove_file(&path); } } } diff --git a/dstack/vmm/src/main_service.rs b/dstack/vmm/src/main_service.rs index dbc35b383..42a9cc781 100644 --- a/dstack/vmm/src/main_service.rs +++ b/dstack/vmm/src/main_service.rs @@ -2,7 +2,9 @@ // // SPDX-License-Identifier: Apache-2.0 +use std::collections::HashSet; use std::ops::Deref; +use std::path::{Component, Path, PathBuf}; use std::time::{SystemTime, UNIX_EPOCH}; use anyhow::{bail, Context, Result}; @@ -18,6 +20,7 @@ use dstack_vmm_rpc::{ }; use fs_err as fs; use or_panic::ResultOrPanic; +use path_absolutize::Absolutize; use ra_rpc::{CallContext, RpcCall}; use tracing::{info, warn}; @@ -182,6 +185,9 @@ pub fn create_manifest_from_vm_config( Some(gpus) => resolve_gpus_with_config(gpus, cvm_config)?, None => GpuConfig::default(), }; + let verity_volumes = extract_verity_volumes(&request.compose_file)?; + dstack_types::validate_verity_volumes(&verity_volumes).map_err(anyhow::Error::msg)?; + let volumes = resolve_volumes(&verity_volumes, cvm_config)?; Ok(Manifest { id, @@ -200,9 +206,86 @@ pub fn create_manifest_from_vm_config( gateway_urls: request.gateway_urls.clone(), no_tee: request.no_tee, networks: networks_from_vm_config(&request, cvm_config)?, + volumes, }) } +/// Extract only the field understood by this VMM. Keep every other app-compose +/// field opaque so newer guest schemas and legacy third-party clients remain +/// compatible with older VMMs. +fn extract_verity_volumes(compose: &str) -> Result> { + let Ok(compose) = serde_json::from_str::(compose) else { + return Ok(vec![]); + }; + let Some(volumes) = compose.get("verity_volumes") else { + return Ok(vec![]); + }; + serde_json::from_value(volumes.clone()).context("invalid verity_volumes in app-compose") +} + +/// Resolve requested volumes against `cvm.volumes_dir`. Each `source` must be a +/// bare file name under that directory; the host attaches the bytes, and the +/// guest verifies content against the measured `verity_root`. +fn resolve_volumes( + reqs: &[dstack_types::VerityVolume], + cvm_config: &crate::config::CvmConfig, +) -> Result> { + if reqs.is_empty() { + return Ok(vec![]); + } + let dir = cvm_config.volumes_dir.trim(); + if dir.is_empty() { + bail!("volumes requested but cvm.volumes_dir is not configured"); + } + let base = fs::canonicalize(dir)?; + let mut roots = HashSet::new(); + reqs.iter() + .filter(|volume| { + let first = roots.insert(volume.verity_root); + if !first { + warn!( + root = %hex::encode(volume.verity_root), + source = volume.source, + "not attaching duplicate verity root" + ); + } + first + }) + .map(|v| { + let real = resolve_volume_source(&base, &v.source)?; + Ok(crate::app::VmVolume { + source: real.to_string_lossy().into_owned(), + }) + }) + .collect() +} + +fn resolve_volume_source(base: &Path, source: &str) -> Result { + if source.is_empty() { + bail!("invalid volume source: empty path"); + } + + let source_path = Path::new(source); + let mut components = source_path.components(); + if !matches!(components.next(), Some(Component::Normal(_))) || components.next().is_some() { + bail!("invalid volume source '{source}': must be a bare file name"); + } + + let real = fs::canonicalize(base.join(source_path))?; + real.absolutize_virtually(base) + .with_context(|| format!("volume '{source}' escapes volumes_dir"))?; + + // QEMU's -drive parser treats ',' as an option separator and '=' as an + // option key/value delimiter. Keep this guard while volumes are attached + // through `-drive file=...`. + let real_str = real.to_string_lossy(); + if real_str.contains(',') || real_str.contains('=') { + bail!("volume '{source}' resolves to a path with ',' or '='"); + } + + Ok(real) +} + fn networking_from_proto(proto: &rpc::NetworkingConfig) -> Result> { let bridge = proto.bridge_name.trim().to_string(); let mode = match proto.mode.as_str() { @@ -871,6 +954,25 @@ mod tests { assert!(manifest.networks.is_empty()); } + #[test] + fn volume_extraction_keeps_other_compose_fields_opaque() -> Result<()> { + assert!(extract_verity_volumes("not json")?.is_empty()); + assert!(extract_verity_volumes(r#"{"future_manifest":true}"#)?.is_empty()); + + let compose = serde_json::json!({ + "unknown_future_field": { "anything": true }, + "verity_volumes": [{ + "source": "volume.img", + "verity_root": "11".repeat(32), + "target": "/run/volume" + }] + }); + let volumes = extract_verity_volumes(&compose.to_string())?; + assert_eq!(volumes.len(), 1); + assert_eq!(volumes[0].verity_root, [0x11; 32]); + Ok(()) + } + #[test] fn explicit_user_networking_is_resolved_before_persist() { let mut request = test_vm_configuration(); @@ -918,4 +1020,88 @@ mod tests { assert!(err.to_string().contains("custom networking mode")); } + + #[test] + fn resolve_volume_source_rejects_escape_symlink_and_qemu_metachars() -> Result<()> { + let tmp = tempfile::tempdir()?; + let volumes = tmp.path().join("volumes"); + fs::create_dir_all(&volumes)?; + fs::write(volumes.join("ok.img"), b"ok")?; + let base = fs::canonicalize(&volumes)?; + + let ok = resolve_volume_source(&base, "ok.img")?; + assert_eq!(ok, base.join("ok.img")); + + let err = resolve_volume_source(&base, "../ok.img").unwrap_err(); + assert!(format!("{err:#}").contains("must be a bare file name")); + + fs::write(tmp.path().join("outside.img"), b"outside")?; + std::os::unix::fs::symlink(tmp.path().join("outside.img"), volumes.join("link.img"))?; + let err = resolve_volume_source(&base, "link.img").unwrap_err(); + assert!(format!("{err:#}").contains("escapes volumes_dir")); + + fs::write(volumes.join("bad,readonly=off"), b"bad")?; + let err = resolve_volume_source(&base, "bad,readonly=off").unwrap_err(); + assert!(format!("{err:#}").contains("',' or '='")); + + Ok(()) + } + + #[test] + fn resolve_volumes_resolves_measured_source() -> Result<()> { + let tmp = tempfile::tempdir()?; + fs::write(tmp.path().join("volume.img"), b"volume")?; + let mut cvm_config = test_cvm_config(); + cvm_config.volumes_dir = tmp.path().to_string_lossy().into_owned(); + + let volumes = resolve_volumes( + &[dstack_types::VerityVolume { + source: "volume.img".into(), + verity_root: [0; 32], + target: "/run/volume".into(), + }], + &cvm_config, + )?; + + assert_eq!(volumes.len(), 1); + assert_eq!( + volumes[0].source, + tmp.path().join("volume.img").display().to_string() + ); + Ok(()) + } + + #[test] + fn resolve_volumes_attaches_duplicate_root_once() -> Result<()> { + let tmp = tempfile::tempdir()?; + fs::write(tmp.path().join("first.img"), b"volume")?; + let mut cvm_config = test_cvm_config(); + cvm_config.volumes_dir = tmp.path().to_string_lossy().into_owned(); + let root = [7; 32]; + + let volumes = resolve_volumes( + &[ + dstack_types::VerityVolume { + source: "first.img".into(), + verity_root: root, + target: "/run/first".into(), + }, + dstack_types::VerityVolume { + // This source deliberately does not exist: the first entry + // owns the single attachment for this content root. + source: "duplicate.img".into(), + verity_root: root, + target: "/run/second".into(), + }, + ], + &cvm_config, + )?; + + assert_eq!(volumes.len(), 1); + assert_eq!( + volumes[0].source, + tmp.path().join("first.img").display().to_string() + ); + Ok(()) + } } diff --git a/dstack/vmm/src/one_shot.rs b/dstack/vmm/src/one_shot.rs index c02b2ce44..5fbcdb498 100644 --- a/dstack/vmm/src/one_shot.rs +++ b/dstack/vmm/src/one_shot.rs @@ -6,6 +6,7 @@ use crate::app::{make_sys_config, Image, VmConfig, VmWorkDir}; use crate::config::Config; use crate::main_service; use anyhow::{Context, Result}; +use fs_err as fs; pub async fn run_one_shot( vm_config_path: &str, @@ -328,14 +329,14 @@ Compose file content (first 200 chars): // Configure stdio to match supervisor behavior if !process_config.stdout.is_empty() { - let stdout_file = std::fs::File::create(&process_config.stdout) - .context("Failed to create stdout file")?; - cmd.stdout(stdout_file); + let stdout_file = + fs::File::create(&process_config.stdout).context("Failed to create stdout file")?; + cmd.stdout(stdout_file.into_file()); } if !process_config.stderr.is_empty() { - let stderr_file = std::fs::File::create(&process_config.stderr) - .context("Failed to create stderr file")?; - cmd.stderr(stderr_file); + let stderr_file = + fs::File::create(&process_config.stderr).context("Failed to create stderr file")?; + cmd.stderr(stderr_file.into_file()); } cmd.current_dir(&workdir_path); diff --git a/dstack/vmm/vmm.toml b/dstack/vmm/vmm.toml index bd91f8380..b405e5e9f 100644 --- a/dstack/vmm/vmm.toml +++ b/dstack/vmm/vmm.toml @@ -32,6 +32,10 @@ pccs_url = "" # persistent NVIDIA OCSP and RIM caches during local GPU verification. # nvidia_attestation_proxy_url = "http://10.0.2.1:8090" docker_registry = "" +# Directory of attachable volume images (e.g. pre-baked verity volumes). A +# deploy may only attach files under this directory, by bare file name. Empty +# disables volume attachment. +volumes_dir = "" cid_start = 1000 cid_pool_size = 1000 max_allocable_vcpu = 20 diff --git a/os/common/rootfs/dstack-prepare.sh b/os/common/rootfs/dstack-prepare.sh index 18c634e2d..0885bd506 100755 --- a/os/common/rootfs/dstack-prepare.sh +++ b/os/common/rootfs/dstack-prepare.sh @@ -306,6 +306,9 @@ echo "============================" cd /dstack +# Verify and mount required read-only data volumes before the application starts. +/bin/dstack-volume mount-all app-compose.json + if [ "$(jq 'has("init_script")' app-compose.json)" == true ]; then log "Running init script" dstack-util notify-host -e "boot.progress" -d "init-script" || true diff --git a/os/yocto/layers/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/os/yocto/layers/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index daa00f9ed..8dfd86e7e 100644 --- a/os/yocto/layers/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/os/yocto/layers/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -15,7 +15,7 @@ DSTACK_ROOTFS_SRC ?= "${DSTACK_MONOREPO_ROOT}/os/common/rootfs" S = "${UNPACKDIR}/repo/dstack" DSTACK_ROOTFS_FILES = "${UNPACKDIR}/repo/os/common/rootfs" -RDEPENDS:${PN} += "bash" +RDEPENDS:${PN} += "bash cryptsetup util-linux-blkid util-linux-mount" DEPENDS += "rsync-native tpm2-tss" DEPENDS += "cmake-native" @@ -32,7 +32,7 @@ DSTACK_SERVICES = "dstack-guest-agent.service dstack-guest-agent.socket dstack-p SYSTEMD_PACKAGES = "${@bb.utils.contains('DISTRO_FEATURES','systemd','${PN}','',d)}" SYSTEMD_SERVICE:${PN} = "${@bb.utils.contains('DISTRO_FEATURES','systemd','${DSTACK_SERVICES}','',d)}" SYSTEMD_AUTO_ENABLE:${PN} = "enable" -EXTRA_CARGO_FLAGS = "-p dstack-guest-agent -p dstack-util" +EXTRA_CARGO_FLAGS = "-p dstack-guest-agent -p dstack-util -p dstack-volume" inherit cargo_bin @@ -66,6 +66,7 @@ do_install() { install -d ${D}${sysconfdir}/systemd/journald.conf.d install -m 0755 ${CARGO_BINDIR}/dstack-util ${D}${bindir} install -m 0755 ${CARGO_BINDIR}/dstack-guest-agent ${D}${bindir} + install -m 0755 ${CARGO_BINDIR}/dstack-volume ${D}${bindir} install -m 0755 ${DSTACK_ROOTFS_FILES}/dstack-prepare.sh ${D}${bindir} install -m 0755 ${DSTACK_ROOTFS_FILES}/ephemeral-docker.sh ${D}${bindir} install -m 0755 ${DSTACK_ROOTFS_FILES}/wg-checker.sh ${D}${bindir}