verity volumes: attach verified read-only data disks to CVMs

docs/verity-volumes.md

@@ -0,0 +1,85 @@
+# Verity data volumes
+
+A verity volume is a read-only filesystem image protected by dm-verity. The host
+attaches untrusted bytes; the guest mounts them only when they match the root
+hash measured in `app-compose.json`.
+
+## Disk format
+
+A built volume is a raw GPT image:
+
+```text
+p1: DSTACK_VOLUME metadata envelope
+p2: filesystem data
+p3: dm-verity superblock and hash tree
+```
+
+The envelope identifies a candidate disk. It is not trusted. The guest passes
+the root from the measured app compose to `veritysetup`, so a forged envelope or
+partition table cannot substitute different contents.
+
+## Build
+
+Pack a directory as a reproducible squashfs volume:
+
+```bash
+dstack verity --dir ./models -o models.img
+```
+
+Or wrap an existing filesystem image:
+
+```bash
+dstack verity --fs-image ./models.ext4 -o models.img
+```
+
+The command prints the root and a deploy argument. Place the image in the VMM's
+configured `cvm.volumes_dir`, then deploy it by bare file name, root, and guest
+mount point:
+
+```bash
+dstack deploy -c docker-compose.yaml \
+  --volume models.img:a1b2c3d4...:/run/models
+```
+
+This adds the following measured entry to `app-compose.json`:
+
+```json
+{
+  "verity_volumes": [
+    {
+      "source": "models.img",
+      "verity_root": "a1b2c3d4...",
+      "target": "/run/models"
+    }
+  ]
+}
+```
+
+The target must be an absolute path on a writable guest filesystem, such as
+`/run` or the app data disk. The guest root filesystem itself is read-only.
+
+## Guest activation
+
+Before the application starts, `dstack-volume mount-all app-compose.json`:
+
+1. Scans `/sys/class/block` for disks whose first partition, or whole disk when
+   unpartitioned, starts with the `DSTACK_VOLUME` magic.
+2. Matches the full root advertised by the envelope against the measured root.
+3. Opens p2 and p3 with `veritysetup` using the measured root.
+4. Reads the first mapped block to force an initial integrity check.
+5. Mounts the mapped filesystem read-only at the measured target.
+
+A required volume that is missing, malformed, or fails verification stops guest
+preparation. dm-verity continues verifying blocks lazily as the application
+reads them.
+
+For diagnostics, `dstack-volume scan` lists recognized disks and
+`dstack-volume status app-compose.json` compares requested roots with attached
+and active devices.
+
+## Trust and limitations
+
+The root hash authenticates bytes, not availability. A malicious host can omit a
+volume or cause I/O failures, but cannot silently replace its contents. Volumes
+are unencrypted and are intended for public, shareable data; confidential data
+requires a separate encryption design.

dstack/Cargo.toml

@@ -64,6 +64,7 @@ members = [
     "size-parser",
     "crates/dstack-cli-core",
     "crates/dstack-cli",
+    "crates/dstack-volume",
     "crates/dstackup",
     "crates/dstack-auth",
     "crates/api-auth",
@@ -80,6 +81,7 @@ dstack-kms-rpc = { path = "kms/rpc" }
 dstack-guest-agent-rpc = { path = "guest-agent/rpc" }
 dstack-vmm-rpc = { path = "vmm/rpc" }
 dstack-cli-core = { path = "crates/dstack-cli-core" }
+dstack-volume = { path = "crates/dstack-volume" }
 dstack-api-auth = { path = "crates/api-auth" }
 dstack-build-info = { path = "crates/build-info" }
 cc-eventlog = { path = "cc-eventlog" }

dstack/crates/dstack-cli-core/Cargo.toml

@@ -12,6 +12,7 @@ license.workspace = true
 anyhow.workspace = true
 http-client = { workspace = true, features = ["prpc"] }
 dstack-vmm-rpc.workspace = true
+dstack-types.workspace = true
 serde_json.workspace = true
 # advisory file locking (flock) for the allowlist/state read-modify-write;
 # already in the dependency tree transitively, so no extra compile cost.

dstack/crates/dstack-cli-core/src/compose.rs

@@ -12,8 +12,16 @@ use serde_json::json;
 ///
 /// `kms_enabled` selects KMS mode (deterministic, upgradeable per-app keys);
 /// gateway and local-key-provider are off for the direct-port single-node flow.
-pub fn build_app_compose(name: &str, docker_compose_yaml: &str, kms_enabled: bool) -> String {
-    let manifest = json!({
+///
+/// Each `verity_volumes` entry is measured, so the CVM only mounts content
+/// matching the attested root. Empty for a normal deploy.
+pub fn build_app_compose(
+    name: &str,
+    docker_compose_yaml: &str,
+    kms_enabled: bool,
+    verity_volumes: &[dstack_types::VerityVolume],
+) -> String {
+    let mut manifest = json!({
         "manifest_version": 2,
         "name": name,
         "runner": "docker-compose",
@@ -31,6 +39,9 @@ pub fn build_app_compose(name: &str, docker_compose_yaml: &str, kms_enabled: boo
         // (NTS is also currently broken in guest images — see dstack#745.)
         "secure_time": false,
     });
+    if !verity_volumes.is_empty() {
+        manifest["verity_volumes"] = json!(verity_volumes);
+    }
     // pretty-print via Value's Display (`{:#}`) — infallible, and byte-identical
     // to serde_json::to_string_pretty (avoids an expect on an unfailable Result).
     format!("{manifest:#}")

dstack/crates/dstack-cli/Cargo.toml

@@ -18,5 +18,11 @@ path = "src/main.rs"
 anyhow.workspace = true
 clap.workspace = true
 dstack-cli-core.workspace = true
+dstack-volume.workspace = true
+dstack-types.workspace = true
+fs-err.workspace = true
+hex.workspace = true
 serde_json.workspace = true
 tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
+tracing.workspace = true
+tracing-subscriber = { workspace = true }