From 8238e03f2b66b826fa99f1fa2106edf9cd54a721 Mon Sep 17 00:00:00 2001 From: Hang Yin Date: Sat, 4 Jul 2026 02:56:11 +0000 Subject: [PATCH 01/25] vmm: attach verity volumes to CVMs Add read-only verity volumes -- extra virtio-blk disks a CVM can mount instead of pulling and unpacking their contents. A volume is declared in app-compose.json as `{ verity_root, target }`, and attached at deploy time with `--volume `: the vmm looks the name up under cvm.volumes_dir and attaches it read-only. Because verity_root is part of the measured compose, the guest can check the bytes it's handed against it. Co-Authored-By: Claude Opus 4.8 (1M context) --- dstack/dstack-types/src/lib.rs | 22 +++++++ dstack/guest-agent/src/rpc_service.rs | 1 + dstack/vmm/rpc/proto/vmm_rpc.proto | 11 ++++ dstack/vmm/src/app.rs | 14 +++++ dstack/vmm/src/config.rs | 6 ++ dstack/vmm/src/main_service.rs | 57 ++++++++++++++++++ dstack/vmm/src/vmm-cli.py | 83 +++++++++++++++++++++++++++ dstack/vmm/vmm.toml | 4 ++ 8 files changed, 198 insertions(+) diff --git a/dstack/dstack-types/src/lib.rs b/dstack/dstack-types/src/lib.rs index 841c5fdb4..3b7a64a86 100644 --- a/dstack/dstack-types/src/lib.rs +++ b/dstack/dstack-types/src/lib.rs @@ -119,6 +119,28 @@ pub struct AppCompose { /// of silently ignoring the requirements. #[serde(default, skip_serializing_if = "Option::is_none")] pub requirements: Option, + /// Read-only, dm-verity-protected volumes pre-seeded into the CVM. Each + /// `verity_root` is measured (it is part of these compose bytes), so the + /// guest only mounts content matching the attested app. See + /// docs/verity-volumes.md. + #[serde(default)] + pub verity_volumes: Vec, +} + +/// A pre-baked, read-only dm-verity volume attached to the CVM. +#[derive(Deserialize, Serialize, Debug, Clone, Default)] +pub struct VerityVolume { + /// dm-verity root hash (hex): the volume's content identity and integrity + /// check. The guest matches attached devices against it. + #[serde(default)] + pub verity_root: String, + /// `"docker"` (seed the docker overlay2 image store), or an absolute path + /// where the volume's filesystem is mounted (e.g. model weights). + /// + /// Defaulted so one malformed entry doesn't fail the whole AppCompose parse; + /// the guest skips an entry with an empty root or target. + #[serde(default)] + pub target: String, } /// Canonical source for the policy used when `requirements.gpu_policy` is diff --git a/dstack/guest-agent/src/rpc_service.rs b/dstack/guest-agent/src/rpc_service.rs index e20126095..5a382d51f 100644 --- a/dstack/guest-agent/src/rpc_service.rs +++ b/dstack/guest-agent/src/rpc_service.rs @@ -751,6 +751,7 @@ mod tests { swap_size: 0, port_policy: Default::default(), requirements: None, + verity_volumes: Vec::new(), }; let dummy_appcompose_wrapper = AppComposeWrapper { diff --git a/dstack/vmm/rpc/proto/vmm_rpc.proto b/dstack/vmm/rpc/proto/vmm_rpc.proto index 17c6cba00..44d584400 100644 --- a/dstack/vmm/rpc/proto/vmm_rpc.proto +++ b/dstack/vmm/rpc/proto/vmm_rpc.proto @@ -118,6 +118,17 @@ message VmConfiguration { optional NetworkingConfig networking = 18; // Per-VM networking overrides. If set, wins over singular networking. repeated NetworkingConfig networks = 19; + // Extra volumes to attach (e.g. pre-baked verity volumes). The host only + // supplies bytes; the guest verifies content against the measured verity_root. + repeated VmVolume volumes = 20; +} + +// An extra disk attached to the VM, such as a pre-baked verity volume. +message VmVolume { + // Volume file name, resolved against the vmm's configured cvm.volumes_dir. + string source = 1; + // Attach the disk read-only. + bool read_only = 2; } // Per-VM networking configuration. diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 637e9883c..dbac6a725 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -79,6 +79,16 @@ pub struct PortMapping { pub to: u16, } +/// An extra disk attached to the VM (e.g. a pre-baked verity volume). `source` +/// is an absolute host path already resolved and validated by the VMM against +/// `cvm.volumes_dir`. +#[derive(Deserialize, Serialize, Debug, Clone)] +pub struct VmVolume { + pub source: String, + #[serde(default)] + pub read_only: bool, +} + #[derive(Deserialize, Serialize, Clone, Builder, Debug)] pub struct Manifest { pub id: String, @@ -104,6 +114,8 @@ pub struct Manifest { pub no_tee: bool, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub networks: Vec, + #[serde(default)] + pub volumes: Vec, } impl Manifest { @@ -1614,6 +1626,7 @@ mod tests { gateway_urls: vec![], no_tee: false, networks: vec![], + volumes: vec![], } } @@ -1875,6 +1888,7 @@ mod tests { gateway_urls: vec![], no_tee: false, networks: vec![], + volumes: vec![], }; let mr_config = MrConfigV3::new( diff --git a/dstack/vmm/src/config.rs b/dstack/vmm/src/config.rs index 38e9a7bd2..92c74b9cd 100644 --- a/dstack/vmm/src/config.rs +++ b/dstack/vmm/src/config.rs @@ -329,6 +329,12 @@ pub struct CvmConfig { #[serde(default = "default_serial_history_max_bytes")] #[serde(with = "size_parser::human_size")] pub serial_history_max_bytes: u64, + + /// Directory holding attachable volume images (e.g. pre-baked verity + /// volumes). A deploy may only attach files under this directory, referenced + /// by bare file name. Empty (the default) disables volume attachment. + #[serde(default)] + pub volumes_dir: String, } /// SMBIOS product information configuration. diff --git a/dstack/vmm/src/main_service.rs b/dstack/vmm/src/main_service.rs index dbc35b383..36185f3ca 100644 --- a/dstack/vmm/src/main_service.rs +++ b/dstack/vmm/src/main_service.rs @@ -182,6 +182,7 @@ pub fn create_manifest_from_vm_config( Some(gpus) => resolve_gpus_with_config(gpus, cvm_config)?, None => GpuConfig::default(), }; + let volumes = resolve_volumes(&request.volumes, cvm_config)?; Ok(Manifest { id, @@ -200,9 +201,65 @@ pub fn create_manifest_from_vm_config( gateway_urls: request.gateway_urls.clone(), no_tee: request.no_tee, networks: networks_from_vm_config(&request, cvm_config)?, + volumes, }) } +/// Resolve requested volumes against `cvm.volumes_dir`. Each `source` must be a +/// bare file name under that directory; the host attaches the bytes, and the +/// guest verifies content against the measured `verity_root`. +fn resolve_volumes( + reqs: &[rpc::VmVolume], + cvm_config: &crate::config::CvmConfig, +) -> Result> { + if reqs.is_empty() { + return Ok(vec![]); + } + let dir = cvm_config.volumes_dir.trim(); + if dir.is_empty() { + bail!("volumes requested but cvm.volumes_dir is not configured"); + } + let base = std::fs::canonicalize(dir) + .with_context(|| format!("volumes_dir '{dir}' does not exist"))?; + reqs.iter() + .map(|v| { + // `,`/`=` would let a crafted file name inject qemu `-drive` options + // when the path is concatenated into the drive string. + if v.source.is_empty() + || v.source.contains('/') + || v.source.contains("..") + || v.source.contains(',') + || v.source.contains('=') + { + bail!( + "invalid volume source '{}': must be a bare file name (no '/', '..', ',', '=')", + v.source + ); + } + let real = std::fs::canonicalize(base.join(&v.source)).with_context(|| { + format!("volume '{}' not found under volumes_dir '{dir}'", v.source) + })?; + if !real.starts_with(&base) { + bail!("volume '{}' escapes volumes_dir", v.source); + } + // Re-check the resolved path: a symlink could resolve to a name + // containing `,`/`=` and reintroduce QEMU -drive option injection. + let real_str = real.to_string_lossy(); + if real_str.contains(',') || real_str.contains('=') { + bail!("volume '{}' resolves to a path with ',' or '='", v.source); + } + Ok(crate::app::VmVolume { + source: real.to_string_lossy().into_owned(), + // Verity volumes are always read-only: the backing file is shared + // content-addressed data, so a writable attach could only let one + // guest corrupt it for every other tenant. Force it regardless of + // what the client asked for. + read_only: true, + }) + }) + .collect() +} + fn networking_from_proto(proto: &rpc::NetworkingConfig) -> Result> { let bridge = proto.bridge_name.trim().to_string(); let mode = match proto.mode.as_str() { diff --git a/dstack/vmm/src/vmm-cli.py b/dstack/vmm/src/vmm-cli.py index 9671a025e..087b2f8a3 100755 --- a/dstack/vmm/src/vmm-cli.py +++ b/dstack/vmm/src/vmm-cli.py @@ -341,6 +341,81 @@ def parse_port_mapping(port_str: str) -> Dict: raise argparse.ArgumentTypeError(f"Invalid port mapping format: {port_str}") +def parse_volume(spec: str) -> Dict: + """Parse a volume spec "name" | "name:ro" into a dictionary. + + `name` is a bare file name resolved by the VMM against cvm.volumes_dir. + Verity volumes are read-only (the VMM enforces it regardless). + """ + parts = spec.split(":") + name = parts[0] + mode = parts[1] if len(parts) > 1 else "ro" + if len(parts) > 2 or mode != "ro": + raise argparse.ArgumentTypeError( + f'volume spec must be "name" or "name:ro" (verity volumes are read-only), got {spec!r}' + ) + if not name or "/" in name or ".." in name: + raise argparse.ArgumentTypeError( + f"volume name must be a bare file name (no '/' or '..'), got {name!r}" + ) + return {"source": name, "read_only": True} + + +def _load_verity_helper() -> Optional[str]: + """Read the bundled dstack-verity helper (basefiles/dstack-verity.sh).""" + path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "..", + "..", + "basefiles", + "dstack-verity.sh", + ) + try: + with open(path, "r", encoding="utf-8") as f: + return f.read() + except OSError: + return None + + +def inject_verity_init(compose_content: str) -> str: + """Inject an init_script that runs the dstack-verity helper. + + This applies when the compose declares verity_volumes but has no init_script. + It lets verity volumes work on guest images that don't yet ship + /bin/dstack-verity.sh. The injected script defers to that binary when present, + so it becomes a no-op once the image ships the helper. + """ + try: + compose = json.loads(compose_content) + except (ValueError, TypeError): + return compose_content + if not compose.get("verity_volumes"): + return compose_content + if compose.get("init_script"): + # don't clobber the user's own init_script; warn instead. + print( + "warning: verity_volumes set but the compose has its own init_script; " + "not injecting the helper (the guest image must ship /bin/dstack-verity.sh)" + ) + return compose_content + helper = _load_verity_helper() + if not helper: + print( + "warning: verity_volumes set but dstack-verity helper not found; images will pull" + ) + return compose_content + compose["init_script"] = ( + "if [ -x /bin/dstack-verity.sh ]; then :; else\n" + "cat > /run/dstack-verity.sh <<'DSTACK_VERITY_EOF'\n" + + helper.rstrip("\n") + + "\nDSTACK_VERITY_EOF\n" + "bash /run/dstack-verity.sh /dstack/app-compose.json || true\n" + "fi\n" + ) + print("injected dstack-verity init_script for verity_volumes") + return json.dumps(compose, indent=4, ensure_ascii=False) + + def read_utf8(filepath: str) -> str: """Read a file and return its contents as a UTF-8 string.""" with open(filepath, "rb") as f: @@ -858,6 +933,7 @@ def create_vm(self, args) -> None: raise Exception(f"Compose file not found: {args.compose}") compose_content = read_utf8(args.compose) + compose_content = inject_verity_init(compose_content) envs = parse_env_file(args.env_file) @@ -892,6 +968,7 @@ def create_vm(self, args) -> None: "app_id": args.app_id, "user_config": user_config, "ports": [parse_port_mapping(port) for port in args.port or []], + "volumes": [parse_volume(v) for v in args.volume or []], "hugepages": args.hugepages, "pin_numa": args.pin_numa, "stopped": args.stopped, @@ -1755,6 +1832,12 @@ def _patched_format_help(): type=str, help="Port mapping in format: protocol[:address]:from:to", ) + deploy_parser.add_argument( + "--volume", + action="append", + type=str, + help='Attach a read-only volume by name from cvm.volumes_dir: "name" | "name:ro"', + ) deploy_parser.add_argument( "--gpu", action="append", diff --git a/dstack/vmm/vmm.toml b/dstack/vmm/vmm.toml index bd91f8380..b405e5e9f 100644 --- a/dstack/vmm/vmm.toml +++ b/dstack/vmm/vmm.toml @@ -32,6 +32,10 @@ pccs_url = "" # persistent NVIDIA OCSP and RIM caches during local GPU verification. # nvidia_attestation_proxy_url = "http://10.0.2.1:8090" docker_registry = "" +# Directory of attachable volume images (e.g. pre-baked verity volumes). A +# deploy may only attach files under this directory, by bare file name. Empty +# disables volume attachment. +volumes_dir = "" cid_start = 1000 cid_pool_size = 1000 max_allocable_vcpu = 20 From 7eea80c81703b9f3c60b4acf7df8951003c52f97 Mon Sep 17 00:00:00 2001 From: Hang Yin Date: Sat, 4 Jul 2026 02:56:11 +0000 Subject: [PATCH 02/25] guest: seed verity volumes before dockerd Just before dockerd starts, dstack-prepare.sh runs the seeding helper. For each volume declared in the compose it finds the matching disk by opening it with veritysetup against the measured verity_root, then either seeds docker's overlay2 store (target "docker") so the images are already present, or mounts the volume at a path (a data volume). It's fail-safe throughout: a volume that's missing or doesn't verify is skipped, and its images just pull. Co-Authored-By: Claude Opus 4.8 (1M context) --- dstack/basefiles/dstack-verity.sh | 267 +++++++++++++++++++++++++++++ os/common/rootfs/dstack-prepare.sh | 4 + 2 files changed, 271 insertions(+) create mode 100755 dstack/basefiles/dstack-verity.sh diff --git a/dstack/basefiles/dstack-verity.sh b/dstack/basefiles/dstack-verity.sh new file mode 100755 index 000000000..71b7e2028 --- /dev/null +++ b/dstack/basefiles/dstack-verity.sh @@ -0,0 +1,267 @@ +#!/bin/bash +# SPDX-FileCopyrightText: © 2025 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 +# +# Mount the verity volumes declared in app-compose.json and seed them into the CVM. +# +# Each volume has a target. A "docker" target seeds the docker overlay2 store, so +# its images become cache hits. A "/path" target mounts the volume's filesystem +# there, for data like model weights. +# +# This runs from dstack-prepare.sh before dockerd starts, so there is no restart. +# +# The host only supplies the bytes. Each volume is matched and verified against +# the measured verity_root from app-compose. Everything here is fail-safe: a +# volume that is missing or fails verification is skipped, and its images pull +# normally. See docs/verity-volumes.md. + +COMPOSE="${1:-app-compose.json}" +STORE=/var/lib/docker + +log() { echo "dstack-verity: $*"; } + +# Nothing to do unless the compose file declares verity_volumes. +[ -f "$COMPOSE" ] || exit 0 +count=$(jq '(.verity_volumes // []) | length' "$COMPOSE" 2>/dev/null || echo 0) +[ "${count:-0}" -gt 0 ] || exit 0 +log "$count verity volume(s) requested" + +if ! command -v veritysetup >/dev/null 2>&1; then + log "veritysetup missing; skipping (images will pull)" + exit 0 +fi +modprobe dm-verity 2>/dev/null || true + +# Candidate whole-disk block devices. Volumes are identified by root hash, not by +# name or attach order. The host tags each verity disk's serial with the root +# prefix, so the matching disk can be opened directly; a missing or wrong tag +# falls back to trying every disk. +mapfile -t DEVS < <(lsblk -dnro NAME,TYPE 2>/dev/null | awk '$2=="disk"{print "/dev/"$1}') + +# Detect a device's filesystem from its magic bytes. +# `dstack verity` writes squashfs; ext4 is accepted too for hand-built volumes. +fs_type() { # $1 = device -> squashfs | ext4 | (empty) + local magic + # squashfs magic "hsqs" is at offset 0. + magic=$(dd if="$1" bs=1 count=4 2>/dev/null | od -An -tx1 | tr -d ' \n') + [ "$magic" = "68737173" ] && { echo squashfs; return; } + # ext4 magic 0xEF53 (little-endian on disk) is at byte 1080. + magic=$(dd if="$1" bs=1 skip=1080 count=2 2>/dev/null | od -An -tx1 | tr -d ' \n') + [ "$magic" = "53ef" ] && echo ext4 +} + +# Byte size of the filesystem on $1. +# +# This is also the verity hash-tree offset, because a volume is laid out +# [ filesystem ][ hash tree ]. The size is read from the superblock and rounded +# up to a 4096 block. squashfs and ext4 record it differently. +# +# The superblock is untrusted, but that is safe. A tampered size just gives a +# wrong offset, and the verity open in open_volume then fails. +data_size() { # $1 = device, $2 = fs_type -> bytes (block-aligned) + local bytes byte i=0 info block_count block_size + case "$2" in + squashfs) + # bytes_used is a little-endian u64 at superblock offset 40. Rebuild + # it byte by byte, low byte first. + bytes=0 + for byte in $(dd if="$1" bs=1 skip=40 count=8 2>/dev/null | od -An -tu1); do + bytes=$((bytes + byte * (1 << (8 * i)))) + i=$((i + 1)) + done + # round up to the 4096 block boundary. + [ "$bytes" -gt 0 ] && echo $(((bytes + 4095) / 4096 * 4096)) ;; + ext4) + info=$(dumpe2fs -h "$1" 2>/dev/null) || return 1 + block_count=$(awk -F: '/Block count/{gsub(/ /,"",$2);print $2;exit}' <<<"$info") + block_size=$(awk -F: '/Block size/{gsub(/ /,"",$2);print $2;exit}' <<<"$info") + [ -n "$block_count" ] && [ -n "$block_size" ] && echo $((block_count * block_size)) ;; + esac +} + +# Probe every candidate disk once, recording "dev fs off serial" for each that +# looks like a verity volume. Doing this once keeps per-volume matching cheap. +DEV_INFO=() +scan_devices() { + local dev fs off serial + for dev in "${DEVS[@]}"; do + fs=$(fs_type "$dev") || continue + [ -n "$fs" ] || continue + off=$(data_size "$dev" "$fs") || continue + { [ -n "$off" ] && [ "$off" -gt 0 ]; } || continue + serial=$(cat "/sys/block/$(basename "$dev")/serial" 2>/dev/null) + DEV_INFO+=("$dev $fs $off $serial") + done +} + +# Open $dev as /dev/mapper/$name if it verifies against $root. +# On success, echoes "/dev/mapper/$name ". +try_open() { # $1 = dev, $2 = fs, $3 = off, $4 = root, $5 = name + local dev="$1" fs="$2" off="$3" root="$4" name="$5" + # $dev is both the data device and the hash device: it's one file, and + # --hash-offset is where the hash tree starts inside it. + veritysetup open "$dev" "$name" "$dev" "$root" --hash-offset="$off" \ + >/dev/null 2>&1 || return 1 + # Confirm the root matches: reading through verity must not fault. + if dd if="/dev/mapper/$name" of=/dev/null bs=4096 count=1 >/dev/null 2>&1; then + echo "/dev/mapper/$name $fs" + return 0 + fi + veritysetup close "$name" >/dev/null 2>&1 + return 1 +} + +# Open the volume matching $root as /dev/mapper/$name. +# +# First try the disk the host tagged for this root (its serial is the root +# prefix). That is the common path: one open, no probing. If nothing matches the +# tag, or it fails to verify, fall back to trying every disk. +open_volume() { # $1 = root hash, $2 = mapper name + local root="$1" name="$2" entry dev fs off serial + for entry in "${DEV_INFO[@]}"; do + read -r dev fs off serial <<<"$entry" + if [ -z "$serial" ] || [ "$serial" != "${root:0:20}" ]; then + continue + fi + try_open "$dev" "$fs" "$off" "$root" "$name" && return 0 + done + for entry in "${DEV_INFO[@]}"; do + read -r dev fs off serial <<<"$entry" + try_open "$dev" "$fs" "$off" "$root" "$name" && return 0 + done + return 1 +} + +# Mount a verity-mapped device read-only, choosing per-fs options. +mount_ro() { # $1 = device, $2 = fs_type, $3 = mountpoint + case "$2" in + # noload: skip journal replay -- the device is read-only. + ext4) mount -t ext4 -o ro,noload "$1" "$3" 2>/dev/null ;; + *) mount -t "$2" -o ro "$1" "$3" 2>/dev/null ;; + esac +} + +# True if $1 is a mountpoint (the spaces anchor the match to the whole field). +is_mounted() { grep -qsF " $1 " /proc/self/mountinfo; } + +# umount the binds this seeding made -- undoes a partial seed. A no-op on reboot, +# where the diffs are already mounted so nothing was added to $bound. +unwind_binds() { local mp; for mp in "$@"; do umount "$mp" 2>/dev/null; done; } + +# Seed the docker overlay2 store from a mounted volume. +# +# Bind-mount each big, already-extracted layer diff read-only, then copy the +# small metadata. No pull, no extraction. Contents merge into whatever is already +# in the store, so several volumes can seed one store. +# +# The diff binds are redone on every boot. /var/lib/docker is persistent, but +# bind mounts are not, so a reboot starts with empty diff/ dirs. The binds must +# go in before the metadata, or the metadata would point at empty layers. +# +# On any bind failure this copies no metadata and returns non-zero, so docker +# pulls the image cleanly instead of running it with empty layers. +seed_docker() { # $1 = mounted volume dir + local vol="$1" layer_dir id file base repo src + local bound=() + + # 1. (re)bind every layer's read-only diff. If one fails, unwind the binds + # done so far and bail, before any metadata is written. + for layer_dir in "$vol"/overlay2/*/; do + [ -d "$layer_dir" ] || continue + id=$(basename "$layer_dir") + [ "$id" = l ] && continue # `l/` is docker's symlink dir, not a layer + mkdir -p "$STORE/overlay2/$id/diff" + if ! is_mounted "$STORE/overlay2/$id/diff"; then + if mount --bind "$layer_dir/diff" "$STORE/overlay2/$id/diff"; then + bound+=("$STORE/overlay2/$id/diff") + else + unwind_binds "${bound[@]}" + return 1 + fi + fi + done + + # 2. metadata (content-addressed) -- merge by copying directory *contents* + # (a plain `cp dir store/` would nest or clobber on the second volume). On + # any copy failure (e.g. a full disk), unwind this volume's binds and bail, + # so docker pulls cleanly instead of running against a half-written store. + mkdir -p "$STORE/image/overlay2/imagedb" "$STORE/image/overlay2/layerdb" "$STORE/overlay2/l" + if ! { cp -a "$vol/image/overlay2/imagedb/." "$STORE/image/overlay2/imagedb/" && + cp -a "$vol/image/overlay2/layerdb/." "$STORE/image/overlay2/layerdb/" && + cp -a "$vol/overlay2/l/." "$STORE/overlay2/l/"; } 2>/dev/null; then + unwind_binds "${bound[@]}" + return 1 + fi + + # repositories.json is a single name->id map; merge it rather than overwrite, + # or a second volume would drop the first volume's image names. A failed merge + # (which leaves the old map, missing this volume's names) also unwinds and bails. + repo="$STORE/image/overlay2/repositories.json" + src="$vol/image/overlay2/repositories.json" + if [ -f "$repo" ] && command -v jq >/dev/null 2>&1; then + if ! { jq -s '.[0] * .[1]' "$repo" "$src" > "$repo.tmp" 2>/dev/null && mv "$repo.tmp" "$repo"; }; then + rm -f "$repo.tmp" + unwind_binds "${bound[@]}" + return 1 + fi + elif ! cp -a "$src" "$repo" 2>/dev/null; then + unwind_binds "${bound[@]}" + return 1 + fi + + # 3. per-layer small writable files (link/lower), copied once. The layer dir + # stays writable because docker writes a `committed` marker into it. + for layer_dir in "$vol"/overlay2/*/; do + [ -d "$layer_dir" ] || continue + id=$(basename "$layer_dir") + [ "$id" = l ] && continue + [ -e "$STORE/overlay2/$id/link" ] && continue + for file in "$layer_dir"*; do + base=$(basename "$file") + [ "$base" = diff ] && continue + cp -a "$file" "$STORE/overlay2/$id/" 2>/dev/null + done + done + return 0 +} + +# Scan the disks once, then match each declared volume to its disk. +scan_devices +for i in $(seq 0 $((count - 1))); do + root=$(jq -r ".verity_volumes[$i].verity_root // empty" "$COMPOSE") + target=$(jq -r ".verity_volumes[$i].target // empty" "$COMPOSE") + if [ -z "$root" ] || [ -z "$target" ]; then + log "vol $i: missing verity_root/target; skip" + continue + fi + + read -r mapped fs < <(open_volume "$root" "verity$i") + if [ -z "$mapped" ]; then + log "vol $i ($target): no attached device matches ${root:0:12}...; skip (will pull)" + continue + fi + + if [ "$target" = docker ]; then + # Mount on /run: it's a writable tmpfs, and the rootfs is read-only. + mnt="/run/dstack-verity/$i" + mkdir -p "$mnt" + if mount_ro "$mapped" "$fs" "$mnt" && seed_docker "$mnt"; then + log "vol $i: seeded docker store from ${root:0:12}..." + else + log "vol $i: seeding failed; skipping (images will pull)" + # umount first: veritysetup close fails on a still-mounted device. + umount "$mnt" 2>/dev/null + veritysetup close "verity$i" 2>/dev/null + fi + else + # The target must be on a writable fs (the rootfs is read-only), e.g. /run. + if mkdir -p "$target" 2>/dev/null && mount_ro "$mapped" "$fs" "$target"; then + log "vol $i: mounted ${root:0:12}... at $target" + else + log "vol $i: mount at $target failed (is it a writable path?); skipping" + veritysetup close "verity$i" 2>/dev/null + fi + fi +done +exit 0 diff --git a/os/common/rootfs/dstack-prepare.sh b/os/common/rootfs/dstack-prepare.sh index 249bbf9cc..90c11c12e 100755 --- a/os/common/rootfs/dstack-prepare.sh +++ b/os/common/rootfs/dstack-prepare.sh @@ -300,6 +300,10 @@ echo "============================" cd /dstack +# Seed pre-baked verity volumes (docker images / data) before dockerd starts, so +# the images are already present with no pull. Fail-safe -- see dstack-verity.sh. +/bin/dstack-verity.sh app-compose.json || log "dstack-verity failed (continuing; images will pull normally)" + if [ "$(jq 'has("init_script")' app-compose.json)" == true ]; then log "Running init script" dstack-util notify-host -e "boot.progress" -d "init-script" || true From 684efb9c05bf38069ca048c0df0b8651ada1dcfb Mon Sep 17 00:00:00 2001 From: Hang Yin Date: Sat, 4 Jul 2026 02:56:11 +0000 Subject: [PATCH 03/25] dstack verity: build reproducible verity volumes `dstack verity ...` builds a squashfs + dm-verity volume that pre-extracts docker images (or, with `--dir`, a plain directory) so a CVM can start without pulling or unpacking them. It fetches images itself through oci-client -- no docker daemon -- and lays out the overlay2 store deterministically (each layer's directory id is its chain-id, with a fixed timestamp and salt). The same inputs always produce the same verity_root, so anyone can recompute it from the pinned image digests and confirm what's in the volume without trusting the builder. `dstack deploy --volume` attaches the result. Co-Authored-By: Claude Opus 4.8 (1M context) --- dstack/Cargo.lock | 243 +++++++++- dstack/Cargo.toml | 2 + dstack/crates/dstack-cli-core/src/compose.rs | 19 +- dstack/crates/dstack-cli/Cargo.toml | 3 + dstack/crates/dstack-cli/src/main.rs | 238 +++++++++- dstack/crates/dstack-verity/Cargo.toml | 27 ++ dstack/crates/dstack-verity/src/lib.rs | 140 ++++++ dstack/crates/dstack-verity/src/oci.rs | 320 +++++++++++++ dstack/crates/dstack-verity/src/store.rs | 457 +++++++++++++++++++ dstack/crates/dstack-verity/src/volume.rs | 238 ++++++++++ 10 files changed, 1671 insertions(+), 16 deletions(-) create mode 100644 dstack/crates/dstack-verity/Cargo.toml create mode 100644 dstack/crates/dstack-verity/src/lib.rs create mode 100644 dstack/crates/dstack-verity/src/oci.rs create mode 100644 dstack/crates/dstack-verity/src/store.rs create mode 100644 dstack/crates/dstack-verity/src/volume.rs diff --git a/dstack/Cargo.lock b/dstack/Cargo.lock index 457c19578..d24b5b8f9 100644 --- a/dstack/Cargo.lock +++ b/dstack/Cargo.lock @@ -14,7 +14,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "generic-array", ] @@ -525,6 +525,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "blowfish" version = "0.9.1" @@ -871,7 +880,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "inout", "zeroize", ] @@ -990,6 +999,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "const_format" version = "0.2.36" @@ -1198,6 +1213,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "crypto-mac" version = "0.11.0" @@ -1376,7 +1400,7 @@ dependencies = [ "borsh", "byteorder", "chrono", - "const-oid", + "const-oid 0.9.6", "dcap-qvl-webpki", "der", "derive_more 2.1.1", @@ -1442,7 +1466,7 @@ version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ - "const-oid", + "const-oid 0.9.6", "der_derive", "flagset", "pem-rfc7468", @@ -1484,6 +1508,37 @@ dependencies = [ "serde_core", ] +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling 0.20.11", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn 2.0.117", +] + [[package]] name = "derive_more" version = "1.0.0" @@ -1578,11 +1633,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", - "const-oid", - "crypto-common", + "const-oid 0.9.6", + "crypto-common 0.1.7", "subtle", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", +] + [[package]] name = "dirs" version = "5.0.1" @@ -1758,8 +1824,11 @@ dependencies = [ "anyhow", "clap", "dstack-cli-core", + "dstack-verity", "serde_json", "tokio", + "tracing", + "tracing-subscriber", ] [[package]] @@ -2181,6 +2250,26 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "dstack-verity" +version = "0.6.0" +dependencies = [ + "anyhow", + "flate2", + "futures", + "hex", + "oci-client", + "rustix 0.38.44", + "serde", + "serde_json", + "sha2 0.10.9", + "tar", + "tempfile", + "tokio", + "tracing", + "xattr", +] + [[package]] name = "dstack-vmm" version = "0.6.0" @@ -2845,6 +2934,17 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "getset" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cf442baaabe4213ce7d1239afc26c039180b6456da2cededa316ae2c8a77a77" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "ghash" version = "0.5.1" @@ -3194,6 +3294,15 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-auth" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "150fa4a9462ef926824cf4519c84ed652ca8f4fbae34cb8af045b5cbcaf98822" +dependencies = [ + "memchr", +] + [[package]] name = "http-body" version = "1.0.1" @@ -3263,6 +3372,15 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" +[[package]] +name = "hybrid-array" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.10.1" @@ -3832,6 +3950,22 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "jsonwebtoken" +version = "10.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eba32bfb4ffdeaca3e34431072faf01745c9b26d25504aa7a6cf5684334fc4fc" +dependencies = [ + "aws-lc-rs", + "base64 0.22.1", + "getrandom 0.2.17", + "js-sys", + "serde", + "serde_json", + "signature", + "zeroize", +] + [[package]] name = "k256" version = "0.13.4" @@ -4624,6 +4758,50 @@ dependencies = [ "ruzstd", ] +[[package]] +name = "oci-client" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5261a7fb43d9c53b8e63e6d5e86860719dad253d015d022066c72d585125aed8" +dependencies = [ + "bytes", + "chrono", + "futures-util", + "hex", + "http", + "http-auth", + "jsonwebtoken", + "lazy_static", + "oci-spec", + "olpc-cjson", + "regex", + "reqwest", + "serde", + "serde_json", + "sha2 0.11.0", + "thiserror 2.0.18", + "tokio", + "tracing", + "unicase", +] + +[[package]] +name = "oci-spec" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8445a2631507cec628a15fdd6154b54a3ab3f20ed4fe9d73a3b8b7a4e1ba03a" +dependencies = [ + "const_format", + "derive_builder", + "getset", + "regex", + "serde", + "serde_json", + "strum", + "strum_macros", + "thiserror 2.0.18", +] + [[package]] name = "oid-registry" version = "0.7.1" @@ -4633,6 +4811,17 @@ dependencies = [ "asn1-rs", ] +[[package]] +name = "olpc-cjson" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "696183c9b5fe81a7715d074fd632e8bd46f4ccc0231a3ed7fc580a80de5f7083" +dependencies = [ + "serde", + "serde_json", + "unicode-normalization", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -5806,12 +5995,14 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-rustls", + "tokio-util", "tower", "tower-http", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams", "web-sys", ] @@ -6017,7 +6208,7 @@ version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" dependencies = [ - "const-oid", + "const-oid 0.9.6", "digest 0.10.7", "num-bigint-dig", "num-integer", @@ -6797,6 +6988,17 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + [[package]] name = "sha3" version = "0.10.9" @@ -7708,6 +7910,7 @@ version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -7823,6 +8026,15 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + [[package]] name = "unicode-segmentation" version = "1.13.2" @@ -7841,7 +8053,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "subtle", ] @@ -8097,6 +8309,19 @@ dependencies = [ "wasmparser", ] +[[package]] +name = "wasm-streams" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasmparser" version = "0.244.0" @@ -8771,7 +8996,7 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1301e935010a701ae5f8655edc0ad17c44bad3ac5ce8c39185f75453b720ae94" dependencies = [ - "const-oid", + "const-oid 0.9.6", "der", "spki", "tls_codec", diff --git a/dstack/Cargo.toml b/dstack/Cargo.toml index 3dd8f8179..1c0ced796 100644 --- a/dstack/Cargo.toml +++ b/dstack/Cargo.toml @@ -64,6 +64,7 @@ members = [ "size-parser", "crates/dstack-cli-core", "crates/dstack-cli", + "crates/dstack-verity", "crates/dstackup", "crates/dstack-auth", "crates/api-auth", @@ -80,6 +81,7 @@ dstack-kms-rpc = { path = "kms/rpc" } dstack-guest-agent-rpc = { path = "guest-agent/rpc" } dstack-vmm-rpc = { path = "vmm/rpc" } dstack-cli-core = { path = "crates/dstack-cli-core" } +dstack-verity = { path = "crates/dstack-verity" } dstack-api-auth = { path = "crates/api-auth" } dstack-build-info = { path = "crates/build-info" } cc-eventlog = { path = "cc-eventlog" } diff --git a/dstack/crates/dstack-cli-core/src/compose.rs b/dstack/crates/dstack-cli-core/src/compose.rs index 5c395213d..7bf857505 100644 --- a/dstack/crates/dstack-cli-core/src/compose.rs +++ b/dstack/crates/dstack-cli-core/src/compose.rs @@ -12,8 +12,17 @@ use serde_json::json; /// /// `kms_enabled` selects KMS mode (deterministic, upgradeable per-app keys); /// gateway and local-key-provider are off for the direct-port single-node flow. -pub fn build_app_compose(name: &str, docker_compose_yaml: &str, kms_enabled: bool) -> String { - let manifest = json!({ +/// +/// `verity_volumes` is a list of `(verity_root, target)` pairs. Each becomes a +/// measured `verity_volumes` entry, so the CVM only seeds content matching the +/// attested root. Empty for a normal deploy. +pub fn build_app_compose( + name: &str, + docker_compose_yaml: &str, + kms_enabled: bool, + verity_volumes: &[(String, String)], +) -> String { + let mut manifest = json!({ "manifest_version": 2, "name": name, "runner": "docker-compose", @@ -31,6 +40,12 @@ pub fn build_app_compose(name: &str, docker_compose_yaml: &str, kms_enabled: boo // (NTS is also currently broken in guest images — see dstack#745.) "secure_time": false, }); + if !verity_volumes.is_empty() { + manifest["verity_volumes"] = json!(verity_volumes + .iter() + .map(|(root, target)| json!({ "verity_root": root, "target": target })) + .collect::>()); + } // pretty-print via Value's Display (`{:#}`) — infallible, and byte-identical // to serde_json::to_string_pretty (avoids an expect on an unfailable Result). format!("{manifest:#}") diff --git a/dstack/crates/dstack-cli/Cargo.toml b/dstack/crates/dstack-cli/Cargo.toml index 1ff3c41ba..30a080a3c 100644 --- a/dstack/crates/dstack-cli/Cargo.toml +++ b/dstack/crates/dstack-cli/Cargo.toml @@ -18,5 +18,8 @@ path = "src/main.rs" anyhow.workspace = true clap.workspace = true dstack-cli-core.workspace = true +dstack-verity.workspace = true serde_json.workspace = true tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } +tracing.workspace = true +tracing-subscriber = { workspace = true } diff --git a/dstack/crates/dstack-cli/src/main.rs b/dstack/crates/dstack-cli/src/main.rs index 9ee0a662e..b018b7dcb 100644 --- a/dstack/crates/dstack-cli/src/main.rs +++ b/dstack/crates/dstack-cli/src/main.rs @@ -74,6 +74,11 @@ enum Command { /// (host omitted/`auto`/`0` ⇒ a free host port is picked). Repeatable. #[arg(long = "port", value_name = "SPEC")] ports: Vec, + /// attach a verity volume, as printed by `dstack verity`: the file name + /// in the vmm's volumes_dir, its verity_root, and the target + /// (`docker` or a mount path). Repeatable. + #[arg(long = "volume", value_name = "NAME:VERITY_ROOT:TARGET")] + volumes: Vec, /// deploy in non-KMS mode (ephemeral keys; no KMS required). #[arg(long)] no_kms: bool, @@ -102,10 +107,51 @@ enum Command { }, /// Scaffold a new app project in the current directory. Init, + /// Build a verity volume that pre-loads docker images (or a directory) into + /// a CVM. + /// + /// The CVM mounts the volume instead of pulling and unpacking the images, so + /// it starts in seconds. The build runs anywhere — no docker daemon, no TEE — + /// but needs root, `mksquashfs`, and `veritysetup`. It prints a verity_root + /// to paste into your compose. See docs/verity-volumes.md. + Verity { + /// images to bake in, ideally pinned by digest (e.g. `repo@sha256:...`). + #[arg(value_name = "IMAGE")] + images: Vec, + /// pack this directory into a data volume instead of images (mounted at a + /// path you choose in the compose). + #[arg(long, value_name = "PATH", conflicts_with = "images")] + dir: Option, + /// where to write the volume. + #[arg(long, short = 'o', default_value = "verity.img")] + output: String, + /// squashfs compression: `none` (the default), `zstd`, or `gzip`. + #[arg(long, default_value = "none")] + compress: String, + /// image platform to fetch; must match the guest. `linux/amd64` today, + /// `linux/arm64` for arm64 hosts. + #[arg(long, default_value = "linux/amd64")] + platform: String, + /// allow plain-HTTP registries (loopback registries already use HTTP). + #[arg(long)] + plain_http: bool, + }, } #[tokio::main] async fn main() -> Result<()> { + // progress (e.g. `verity` pulling layers) goes to stderr so it never mixes + // with `--json` on stdout. RUST_LOG overrides. + use tracing_subscriber::EnvFilter; + tracing_subscriber::fmt() + .with_writer(std::io::stderr) + .with_target(false) + .without_time() + .with_env_filter( + EnvFilter::try_from_default_env() + .unwrap_or_else(|_| EnvFilter::new("warn,dstack_verity=info")), + ) + .init(); let cli = Cli::parse(); let defaults = LocalDefaults::read(cli.prefix.as_deref()); let use_local_defaults = cli.host.is_none(); @@ -141,6 +187,7 @@ async fn main() -> Result<()> { memory, disk, ports, + volumes, no_kms, allowlist, dry_run, @@ -170,6 +217,7 @@ async fn main() -> Result<()> { memory, disk, &ports, + &volumes, no_kms, allowlist.as_deref(), dry_run, @@ -179,9 +227,156 @@ async fn main() -> Result<()> { } Command::Info { .. } => stub("info"), Command::Init => stub("init"), + Command::Verity { + images, + dir, + output, + compress, + platform, + plain_http, + } => { + cmd_verity( + &images, + dir.as_deref(), + &output, + &compress, + &platform, + plain_http, + json, + ) + .await + } } } +async fn cmd_verity( + images: &[String], + dir: Option<&str>, + output: &str, + compress: &str, + platform: &str, + plain_http: bool, + json: bool, +) -> Result<()> { + let compress = match compress { + "none" => dstack_verity::Compression::None, + "zstd" => dstack_verity::Compression::Zstd, + "gzip" => dstack_verity::Compression::Gzip, + other => bail!("unknown --compress '{other}' (use none|zstd|gzip)"), + }; + let result = dstack_verity::verity(dstack_verity::VerityOptions { + images: images.to_vec(), + dir: dir.map(std::path::PathBuf::from), + output: output.into(), + compress, + platform: platform.to_string(), + plain_http, + }) + .await?; + + if json { + let imgs: Vec<_> = result + .images + .iter() + .map(|i| { + serde_json::json!({ + "reference": i.reference, + "manifestDigest": i.manifest_digest, + "configDigest": i.config_digest, + "topChainId": i.top_chain_id, + }) + }) + .collect(); + print_json(&serde_json::json!({ + "verityRoot": result.verity_root, + "output": result.output.display().to_string(), + "dataSize": result.data_size, + "images": imgs, + })); + return Ok(()); + } + + let mib = result.data_size as f64 / 1_048_576.0; + println!("wrote {} ({mib:.1} MiB)", result.output.display()); + if !result.images.is_empty() { + // the manifest digest is what `image: repo@sha256:...` pins — not the + // config digest (the image id), which can't be pulled by digest. + println!("\nbaked images — pin each by digest in your compose:"); + for i in &result.images { + println!(" {} @ {}", i.reference, i.manifest_digest); + } + } + let file = result + .output + .file_name() + .map(|s| s.to_string_lossy().into_owned()) + .unwrap_or_else(|| result.output.display().to_string()); + // a data volume mounts at a path you choose; it must be writable (the guest + // rootfs is read-only), e.g. under /run. + let target = if result.images.is_empty() { + "/run/models" + } else { + "docker" + }; + println!("\ncopy {file} into the vmm's volumes_dir, then deploy with:"); + println!( + " dstack deploy -c docker-compose.yaml --volume {file}:{}:{target}", + result.verity_root + ); + if result.images.is_empty() { + println!(" (change {target} to your mount path)"); + } + Ok(()) +} + +/// A parsed `--volume` spec: the disk to attach plus its measured verity entry. +struct VolumeSpec { + volume: rpc::VmVolume, + verity_root: String, + target: String, +} + +/// Parse a `--volume` spec `NAME:VERITY_ROOT:TARGET`. +/// +/// `NAME` is the volume file in the vmm's volumes_dir. `VERITY_ROOT` and `TARGET` +/// become a measured `verity_volumes` entry in the app-compose, so the guest only +/// seeds content matching the attested root. `dstack verity` prints the exact +/// spec to paste. +/// +/// `TARGET` is `docker` (seed the image store) or an absolute mount path. +fn parse_volume(spec: &str) -> Result { + let mut parts = spec.splitn(3, ':'); + let name = parts.next().unwrap_or_default(); + let (root, target) = match (parts.next(), parts.next()) { + (Some(root), Some(target)) if !root.is_empty() && !target.is_empty() => (root, target), + _ => bail!("--volume must be NAME:VERITY_ROOT:TARGET (as printed by `dstack verity`), got '{spec}'"), + }; + if name.is_empty() + || name.contains('/') + || name.contains("..") + || name.contains(',') + || name.contains('=') + { + bail!("volume name '{name}' must be a bare file name (no '/', '..', ',', '=')"); + } + if root.len() != 64 || !root.bytes().all(|b| b.is_ascii_hexdigit()) { + bail!("verity_root '{root}' must be 64 hex chars (copy it from `dstack verity`)"); + } + if target != "docker" && !target.starts_with('/') { + bail!("target '{target}' must be \"docker\" or an absolute path"); + } + Ok(VolumeSpec { + // verity volumes are read-only by construction; a writable one would let a + // guest corrupt the shared backing file (the vmm also forces this). + volume: rpc::VmVolume { + source: name.to_string(), + read_only: true, + }, + verity_root: root.to_string(), + target: target.to_string(), + }) +} + fn resolve_compose_arg(positional: Option, flagged: Option) -> Result { match (positional, flagged) { (Some(path), None) | (None, Some(path)) => Ok(path), @@ -297,6 +492,25 @@ mod tests { let _ = std::fs::remove_dir_all(install_root); } + #[test] + fn parses_volume_specs() { + let root = "a".repeat(64); + let docker = parse_volume(&format!("images.img:{root}:docker")).unwrap(); + assert_eq!(docker.volume.source, "images.img"); + assert!(docker.volume.read_only); + assert_eq!(docker.verity_root, root); + assert_eq!(docker.target, "docker"); + + let data = parse_volume(&format!("weights.img:{root}:/models/llama")).unwrap(); + assert_eq!(data.target, "/models/llama"); + + assert!(parse_volume("weights.img").is_err()); // missing verity_root:target + assert!(parse_volume(&format!("weights.img:{root}")).is_err()); // missing target + assert!(parse_volume(&format!("../escape.img:{root}:docker")).is_err()); // path separator + assert!(parse_volume("x.img:nothex:docker").is_err()); // verity_root not hex + assert!(parse_volume(&format!("x.img:{root}:relative/path")).is_err()); // bad target + } + #[test] fn parses_phala_style_deploy_flags() { let cli = Cli::parse_from([ @@ -343,6 +557,7 @@ async fn cmd_deploy( memory: u32, disk: u32, port_specs: &[String], + volume_specs: &[String], no_kms: bool, allowlist: Option<&str>, dry_run: bool, @@ -350,12 +565,24 @@ async fn cmd_deploy( ) -> Result<()> { let yaml = std::fs::read_to_string(compose_path) .with_context(|| format!("reading compose file '{compose_path}'"))?; - let app_compose = compose::build_app_compose(name, &yaml, !no_kms); - let mut port_maps = Vec::new(); - for spec in port_specs { - port_maps.push(ports::parse_port(spec)?); - } + let port_maps = port_specs + .iter() + .map(|s| ports::parse_port(s)) + .collect::>>()?; + let parsed_volumes = volume_specs + .iter() + .map(|s| parse_volume(s)) + .collect::>>()?; + + // each --volume declares a measured verity_volumes entry, so the built + // app-compose (and thus app_id) binds the attested roots. + let verity_volumes: Vec<(String, String)> = parsed_volumes + .iter() + .map(|v| (v.verity_root.clone(), v.target.clone())) + .collect(); + let app_compose = compose::build_app_compose(name, &yaml, !no_kms, &verity_volumes); + let volumes: Vec<_> = parsed_volumes.into_iter().map(|v| v.volume).collect(); let mut cfg = rpc::VmConfiguration { name: name.to_string(), @@ -365,6 +592,7 @@ async fn cmd_deploy( memory, disk_size: disk, ports: port_maps.clone(), + volumes, ..Default::default() }; diff --git a/dstack/crates/dstack-verity/Cargo.toml b/dstack/crates/dstack-verity/Cargo.toml new file mode 100644 index 000000000..0347ccccc --- /dev/null +++ b/dstack/crates/dstack-verity/Cargo.toml @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "dstack-verity" +version.workspace = true +edition.workspace = true +license.workspace = true + +[dependencies] +anyhow = { workspace = true, features = ["std"] } +tokio = { workspace = true, features = ["rt"] } +serde = { workspace = true, features = ["derive", "std"] } +serde_json = { workspace = true, features = ["std"] } +sha2 = { workspace = true, features = ["std"] } +hex = { workspace = true, features = ["std"] } +flate2.workspace = true +futures.workspace = true +tar.workspace = true +tempfile.workspace = true +tracing.workspace = true +rustix = { version = "0.38", features = ["fs"] } +xattr = "1.5" +oci-client = { version = "0.17.0", default-features = false, features = ["rustls-tls"] } + +[dev-dependencies] diff --git a/dstack/crates/dstack-verity/src/lib.rs b/dstack/crates/dstack-verity/src/lib.rs new file mode 100644 index 000000000..b01756c6b --- /dev/null +++ b/dstack/crates/dstack-verity/src/lib.rs @@ -0,0 +1,140 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! Build a verity volume from docker images or a directory. +//! +//! The output is a reproducible, dm-verity-protected squashfs volume that a CVM +//! mounts instead of pulling and unpacking. A `docker` volume seeds the overlay2 +//! store; a data volume just mounts at a path. +//! +//! The build needs no docker daemon and no TEE, and it's reproducible: the same +//! inputs always give the same `verity_root`. So anyone can recompute the root +//! and check it against `app-compose.json`, without trusting the builder. See +//! docs/verity-volumes.md. + +use std::path::PathBuf; + +use anyhow::{bail, Context, Result}; + +pub mod oci; +mod store; +mod volume; + +pub use volume::Compression; + +/// A fixed dm-verity salt. +/// +/// The root is a function of the squashfs bytes and this salt, so keeping the +/// salt constant is what lets anyone recompute the root. It isn't a secret: +/// veritysetup writes it into the on-disk verity superblock anyway. +const VERITY_SALT: &str = "0000000000000000000000000000000000000000000000000000000000000000"; + +pub struct VerityOptions { + /// image references to bake in, e.g. `nvidia/cuda:12.4.1-runtime-ubuntu22.04`. + /// Empty when building a data volume from `dir`. + pub images: Vec, + /// build a data volume from this directory instead of docker images. The + /// resulting volume is mounted at a `target: "/path"` in the compose. + pub dir: Option, + pub output: PathBuf, + /// squashfs compression (default: none — zero decompression at read time). + pub compress: Compression, + /// allow plain-HTTP registries (loopback registries default to HTTP anyway). + pub plain_http: bool, + /// target platform `os/arch` for image pulls (e.g. `linux/amd64`). Explicit + /// so the build is reproducible and matches the guest's arch. + pub platform: String, +} + +pub struct VerityResult { + pub verity_root: String, + pub data_size: u64, + pub output: PathBuf, + pub images: Vec, +} + +pub struct ResolvedImage { + pub reference: String, + pub manifest_digest: String, + pub config_digest: String, + pub top_chain_id: String, +} + +pub async fn verity(opts: VerityOptions) -> Result { + match (&opts.dir, opts.images.is_empty()) { + (Some(_), false) => bail!("give either images or --dir, not both"), + (None, true) => { + bail!("nothing to build: pass an image, or --dir to pack a directory") + } + (Some(dir), true) => verity_dir(dir.clone(), opts.output, opts.compress).await, + (None, false) => verity_docker(opts).await, + } +} + +/// Bake docker images into a `docker`-target volume. +async fn verity_docker(opts: VerityOptions) -> Result { + // 1. pull every image (daemonless, verified against its digests). Dedup + // repeated references so `verity alpine alpine` doesn't pull twice. + let mut seen = std::collections::HashSet::new(); + let refs: Vec<&String> = opts.images.iter().filter(|r| seen.insert(*r)).collect(); + let mut pulled = Vec::with_capacity(refs.len()); + for r in refs { + tracing::info!("resolving {r}"); + pulled.push(oci::pull(r, opts.plain_http, &opts.platform).await?); + } + + // 2. lay out a deterministic overlay2 store, then pack + verity it. Both are + // synchronous and privileged (mknod / trusted xattr); do them off the async + // reactor. + let output = opts.output.clone(); + let compress = opts.compress; + let (built, resolved) = tokio::task::spawn_blocking(move || -> Result<_> { + let tmp = tempfile::tempdir().context("creating scratch dir")?; + let store_dir = tmp.path().join("store"); + std::fs::create_dir_all(&store_dir)?; + let tops = store::build_store(&pulled, &store_dir)?; + let built = volume::build_volume(&store_dir, &output, VERITY_SALT, compress)?; + let resolved = pulled + .iter() + .zip(&tops) + .map(|(img, top)| ResolvedImage { + reference: img.reference.clone(), + manifest_digest: img.manifest_digest.clone(), + config_digest: img.config_digest.clone(), + top_chain_id: top.clone(), + }) + .collect::>(); + Ok((built, resolved)) + }) + .await + .context("the build task failed")??; + + Ok(VerityResult { + verity_root: built.verity_root, + data_size: built.data_size, + output: opts.output, + images: resolved, + }) +} + +/// Bake a directory tree into a data volume (mounted at a `target: "/path"`). +/// No docker, no overlay2 layout — just a reproducible squashfs of the tree. +async fn verity_dir(dir: PathBuf, output: PathBuf, compress: Compression) -> Result { + if !dir.is_dir() { + bail!("--dir '{}' is not a directory", dir.display()); + } + let out = output.clone(); + let built = tokio::task::spawn_blocking(move || { + volume::build_volume(&dir, &out, VERITY_SALT, compress) + }) + .await + .context("the build task failed")??; + + Ok(VerityResult { + verity_root: built.verity_root, + data_size: built.data_size, + output, + images: vec![], + }) +} diff --git a/dstack/crates/dstack-verity/src/oci.rs b/dstack/crates/dstack-verity/src/oci.rs new file mode 100644 index 000000000..a1e8383ea --- /dev/null +++ b/dstack/crates/dstack-verity/src/oci.rs @@ -0,0 +1,320 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! Fetch an image from a registry, using the `oci-client` crate. +//! +//! Given a reference, this returns the image's config and per-layer blobs, byte +//! for byte, so the caller can lay out a deterministic overlay2 store. oci-client +//! does the fiddly parts: auth, Docker-Hub name normalization, picking the right +//! platform out of an image index, and verifying digests (including a `@sha256:` +//! pin). + +use anyhow::{bail, Context, Result}; +use oci_client::client::{ClientConfig, ClientProtocol}; +use oci_client::manifest; +use oci_client::secrets::RegistryAuth; +use oci_client::{Client, Reference}; +use serde::Deserialize; +use sha2::{Digest, Sha256}; + +/// How many layer blobs to download at once (order-preserving, see `pull`). +const MAX_CONCURRENT_LAYERS: usize = 4; + +/// A fully-resolved image: everything needed to build an overlay2 store. +pub struct PulledImage { + /// the reference as given by the user (for display). + pub reference: String, + /// docker's canonical repository name, e.g. `docker.io/library/alpine` + /// (the repositories.json map key). + pub repo_name: String, + /// the repositories.json key for this reference: `repo_name:tag`, or + /// `repo_name@sha256:...` for a digest pin. `docker run` of the same + /// reference looks this key up, so it's what makes the image a cache hit. + pub ref_key: String, + /// `sha256:...` digest of the manifest actually used (the resolved image). + pub manifest_digest: String, + /// `sha256:...` digest of the config blob — this is the docker image id. + pub config_digest: String, + /// the exact config blob bytes (stored verbatim in imagedb). + pub config_bytes: Vec, + /// `sha256:...` uncompressed digest per layer, bottom to top. + pub diff_ids: Vec, + /// layer blobs, bottom to top, aligned with `diff_ids`. + pub layers: Vec, +} + +pub struct PulledLayer { + pub media_type: String, + pub data: Vec, +} + +/// Pull `reference` for `platform`, returning its config and layers. +/// +/// Layers are fetched in parallel but returned in manifest order, which keeps the +/// store layout deterministic. A loopback registry (or `plain_http`) is fetched +/// over HTTP instead of HTTPS. +pub async fn pull(reference: &str, plain_http: bool, platform: &str) -> Result { + let r: Reference = reference + .parse() + .with_context(|| format!("invalid image reference '{reference}'"))?; + + let insecure = plain_http || is_loopback(r.resolve_registry()); + let protocol = if insecure { + ClientProtocol::Http + } else { + ClientProtocol::Https + }; + let client = Client::new(ClientConfig { + protocol, + // Pin the platform explicitly, not the build host's, so the build is + // reproducible and targets the guest's arch. Not always amd64: arm64 + // confidential hosts are coming (NVIDIA Vera, AWS Graviton/Nitro). + platform_resolver: Some(platform_resolver(platform)?), + ..Default::default() + }); + let auth = RegistryAuth::Anonymous; + + // oci-client verifies the manifest/config digests here (including a + // `@sha256:` pin), and returns the config as its exact raw bytes. + let (manifest, manifest_digest, config_json) = client + .pull_manifest_and_config(&r, &auth) + .await + .with_context(|| format!("resolving {reference}"))?; + let config_bytes = config_json.into_bytes(); + let config_digest = format!("sha256:{}", hex::encode(Sha256::digest(&config_bytes))); + let config: Config = serde_json::from_slice(&config_bytes).context("parsing image config")?; + + // the index resolver only runs for a multi-arch image; a single-arch manifest + // is baked as-is, so check its config arch matches --platform (else we'd bake + // a wrong-arch volume with a right-looking label). + let (want_os, want_arch) = parse_platform(platform)?; + if !config.architecture.is_empty() && config.architecture != want_arch { + bail!( + "image is {}/{}, but --platform is {want_os}/{want_arch} (pass --platform to match)", + if config.os.is_empty() { + "?" + } else { + &config.os + }, + config.architecture + ); + } + + for d in &manifest.layers { + if !accepted_layer_types().contains(&d.media_type.as_str()) { + bail!( + "unsupported layer media type '{}' (only gzip/plain tar; zstd is not supported yet)", + d.media_type + ); + } + } + if config.rootfs.diff_ids.len() != manifest.layers.len() { + bail!( + "manifest/config mismatch: {} layers vs {} diff_ids", + manifest.layers.len(), + config.rootfs.diff_ids.len() + ); + } + + // Download each layer blob (digest-verified by oci-client) concurrently but + // yielded in manifest order — `buffered` preserves input order, so the store + // layout is deterministic without giving up parallel download. + tracing::info!("pulling {} layer(s) of {reference}", manifest.layers.len()); + use futures::stream::{StreamExt, TryStreamExt}; + let layers: Vec = futures::stream::iter(manifest.layers.iter()) + .map(|d| { + let client = &client; + let r = &r; + async move { + // cap the speculative reservation: `d.size` is from the manifest + // and only digest-checked after download. pull_blob grows it as + // needed, so an inflated size can't force a huge up-front alloc. + let hint = (d.size.max(0) as usize).min(64 << 20); + let mut data = Vec::with_capacity(hint); + client + .pull_blob(r, d, &mut data) + .await + .with_context(|| format!("fetching layer {}", d.digest))?; + Ok::<_, anyhow::Error>(PulledLayer { + media_type: d.media_type.clone(), + data, + }) + } + }) + .buffered(MAX_CONCURRENT_LAYERS) + .try_collect() + .await?; + + let (repo_name, ref_key) = repo_and_key(&r); + Ok(PulledImage { + reference: reference.to_string(), + repo_name, + ref_key, + manifest_digest, + config_digest, + config_bytes, + diff_ids: config.rootfs.diff_ids, + layers, + }) +} + +type PlatformResolverFn = + dyn Fn(&[oci_client::manifest::ImageIndexEntry]) -> Option + Send + Sync; + +/// Split `os/arch` (default os `linux`), normalizing the common non-OCI arch +/// spellings (`x86_64` -> `amd64`, `aarch64` -> `arm64`). +fn parse_platform(platform: &str) -> Result<(String, String)> { + let (os, arch) = platform + .split_once('/') + .map(|(o, a)| (o.to_string(), a.to_string())) + .unwrap_or_else(|| ("linux".to_string(), platform.to_string())); + let arch = match arch.as_str() { + "x86_64" => "amd64".to_string(), + "aarch64" => "arm64".to_string(), + _ => arch, + }; + if os.is_empty() || arch.is_empty() { + bail!("invalid --platform '{platform}' (expected e.g. linux/amd64)"); + } + Ok((os, arch)) +} + +/// Build a resolver that picks the `os/arch` entry out of an image index. +fn platform_resolver(platform: &str) -> Result> { + let (os, arch) = parse_platform(platform)?; + Ok(Box::new( + move |manifests: &[oci_client::manifest::ImageIndexEntry]| { + manifests + .iter() + .find(|e| { + e.platform.as_ref().is_some_and(|p| { + p.os.to_string() == os && p.architecture.to_string() == arch + }) + }) + .map(|e| e.digest.clone()) + }, + )) +} + +/// The layer media types we can extract: gzip and plain tar. +/// +/// Checking against these lets us reject an unsupported layer (e.g. zstd) up +/// front, instead of failing later during extraction. +fn accepted_layer_types() -> [&'static str; 4] { + [ + manifest::IMAGE_LAYER_GZIP_MEDIA_TYPE, + manifest::IMAGE_DOCKER_LAYER_GZIP_MEDIA_TYPE, + manifest::IMAGE_LAYER_MEDIA_TYPE, + manifest::IMAGE_DOCKER_LAYER_TAR_MEDIA_TYPE, + ] +} + +/// Return `(repo_name, ref_key)` for a reference. +/// +/// `repo_name` is docker's canonical name (the repositories.json map key); +/// `ref_key` is the entry within it — `repo@digest` for a pin, `repo:tag` +/// otherwise. `Reference` already normalizes Docker-Hub names to +/// `docker.io/library/`. +fn repo_and_key(r: &Reference) -> (String, String) { + let repo = format!("{}/{}", r.registry(), r.repository()); + let key = match r.digest() { + Some(d) => format!("{repo}@{d}"), + None => format!("{repo}:{}", r.tag().unwrap_or("latest")), + }; + (repo, key) +} + +/// A loopback registry we default to plain HTTP for. Any other insecure +/// registry uses `--plain-http`. +fn is_loopback(host: &str) -> bool { + let h = host.split(':').next().unwrap_or(host); + h == "localhost" || h.starts_with("127.") +} + +#[derive(Deserialize)] +struct Config { + #[serde(default)] + architecture: String, + #[serde(default)] + os: String, + rootfs: RootFs, +} +#[derive(Deserialize)] +struct RootFs { + #[serde(rename = "diff_ids", default)] + diff_ids: Vec, +} + +#[cfg(test)] +mod tests { + use super::*; + + fn key(s: &str) -> (String, String) { + repo_and_key(&s.parse::().unwrap()) + } + + #[test] + fn hub_short_name_canonicalizes() { + let (repo, k) = key("alpine:3.20"); + assert_eq!(repo, "docker.io/library/alpine"); + assert_eq!(k, "docker.io/library/alpine:3.20"); + } + + #[test] + fn fully_qualified_hub_matches_short() { + // the doc tells users to fully-qualify; it must resolve identically. + assert_eq!(key("docker.io/library/alpine:3.20"), key("alpine:3.20")); + } + + #[test] + fn digest_pin_uses_at_key() { + let d = "sha256:1111111111111111111111111111111111111111111111111111111111111111"; + let (repo, k) = key(&format!("ghcr.io/foo/bar@{d}")); + assert_eq!(repo, "ghcr.io/foo/bar"); + assert_eq!(k, format!("ghcr.io/foo/bar@{d}")); + } + + #[test] + fn private_registry_with_port() { + let (repo, k) = key("10.0.2.2:5000/bench/alpine:3.20"); + assert_eq!(repo, "10.0.2.2:5000/bench/alpine"); + assert_eq!(k, "10.0.2.2:5000/bench/alpine:3.20"); + } + + #[test] + fn loopback_detection() { + assert!(is_loopback("localhost:5000")); + assert!(is_loopback("127.0.0.1:5000")); + assert!(!is_loopback("10.0.2.2:5000")); // qemu-guest alias, not the host + assert!(!is_loopback("ghcr.io")); + assert!(!is_loopback("index.docker.io")); + } + + #[test] + fn platform_resolver_picks_requested_arch() { + let entries: Vec = + serde_json::from_value(serde_json::json!([ + {"mediaType":"m","digest":"sha256:amd","size":1, + "platform":{"architecture":"amd64","os":"linux"}}, + {"mediaType":"m","digest":"sha256:arm","size":1, + "platform":{"architecture":"arm64","os":"linux"}} + ])) + .unwrap(); + assert_eq!( + platform_resolver("linux/amd64").unwrap()(&entries).as_deref(), + Some("sha256:amd") + ); + assert_eq!( + platform_resolver("linux/arm64").unwrap()(&entries).as_deref(), + Some("sha256:arm") + ); + // aarch64 alias normalizes to arm64. + assert_eq!( + platform_resolver("linux/aarch64").unwrap()(&entries).as_deref(), + Some("sha256:arm") + ); + // a platform with no matching entry resolves to nothing. + assert_eq!(platform_resolver("windows/amd64").unwrap()(&entries), None); + } +} diff --git a/dstack/crates/dstack-verity/src/store.rs b/dstack/crates/dstack-verity/src/store.rs new file mode 100644 index 000000000..d09484441 --- /dev/null +++ b/dstack/crates/dstack-verity/src/store.rs @@ -0,0 +1,457 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! Build a docker overlay2 image store, laid out deterministically. +//! +//! The `oci` module pulls an image's layers; this module lays them out into the +//! on-disk overlay2 store that docker reads. +//! +//! docker's own store is non-deterministic in two ways: it gives each layer a +//! random cache-id, and it stamps wall-clock times. Here the cache-id is the +//! layer's chain-id instead, so the store bytes are a pure function of the +//! pinned layer digests. +//! +//! Whiteouts in the layer tars (the AUFS `.wh.` markers) become the form +//! overlay2 wants on disk: a `0:0` char device, or the `trusted.overlay.opaque` +//! xattr. Both need root. + +use std::collections::BTreeMap; +use std::collections::HashSet; +use std::io::Read; +use std::path::{Path, PathBuf}; + +use anyhow::{bail, Context, Result}; +use flate2::read::GzDecoder; +use serde_json::json; +use sha2::{Digest, Sha256}; + +use crate::oci::{PulledImage, PulledLayer}; + +/// Build the overlay2 store for `images` under `root`. +/// +/// Returns each image's top chain-id, in the same order as `images`. That +/// chain-id is what the verity volume ultimately vouches for. +pub fn build_store(images: &[PulledImage], root: &Path) -> Result> { + std::fs::create_dir_all(root.join("overlay2/l"))?; + std::fs::create_dir_all(root.join("image/overlay2/layerdb/sha256"))?; + std::fs::create_dir_all(root.join("image/overlay2/imagedb/content/sha256"))?; + + let mut built: HashSet = HashSet::new(); + // repo_name -> { "repo:tag" -> "sha256:imageid", "repo@digest" -> id } + let mut repositories: BTreeMap> = BTreeMap::new(); + let mut tops = Vec::with_capacity(images.len()); + + for img in images { + let mut parent: Option = None; + let mut parent_short: Option = None; + let mut chain = String::new(); + for (i, diff_id) in img.diff_ids.iter().enumerate() { + let diff_hex = strip_sha256(diff_id)?; + chain = match &parent { + None => diff_hex.to_string(), + Some(p) => chain_id(p, diff_hex), + }; + let short = short_link(&chain); + + // A layer shared by two images has the same chain-id; write it once. + if built.insert(chain.clone()) { + write_layer( + root, + &img.layers[i], + &img.reference, + i, + diff_id, + &chain, + &short, + parent.as_deref(), + parent_short.as_deref(), + )?; + } + parent = Some(chain.clone()); + parent_short = Some(short); + } + + // imagedb: the config blob verbatim, keyed by its digest (the image id). + let image_id = strip_sha256(&img.config_digest)?; + write_bytes( + root.join("image/overlay2/imagedb/content/sha256") + .join(image_id), + &img.config_bytes, + )?; + // repositories.json: map the exact canonical reference the user pinned + // (`repo:tag` or `repo@sha256:...`) to the image id, plus the resolved + // manifest digest as a secondary alias. + let entry = repositories.entry(img.repo_name.clone()).or_default(); + entry.insert(img.ref_key.clone(), img.config_digest.clone()); + entry.insert( + format!("{}@{}", img.repo_name, img.manifest_digest), + img.config_digest.clone(), + ); + tops.push(chain); + } + + write_bytes( + root.join("image/overlay2/repositories.json"), + serde_json::to_vec(&json!({ "Repositories": repositories }))?.as_slice(), + )?; + Ok(tops) +} + +/// Write one layer into the store, under its `chain` id. +/// +/// Extracts the layer's files, checks them against `diff_id`, then records the +/// overlay2 link and the layerdb entry. `parent` and `parent_short` describe the +/// layer directly below; both are `None` for the base layer. +#[allow(clippy::too_many_arguments)] +fn write_layer( + root: &Path, + layer: &PulledLayer, + reference: &str, + layer_index: usize, + diff_id: &str, + chain: &str, + short: &str, + parent: Option<&str>, + parent_short: Option<&str>, +) -> Result<()> { + let layer_dir = root.join("overlay2").join(chain); + let diff_dir = layer_dir.join("diff"); + std::fs::create_dir_all(&diff_dir)?; + + let (layer_size, actual_diff) = extract_layer(&layer.data, &layer.media_type, &diff_dir) + .with_context(|| format!("extracting layer {layer_index} of {reference}"))?; + + // The extracted files must hash to the diff_id the config claims. Otherwise a + // registry could hand us content that doesn't match its (pinned) config, and + // we'd bake it in anyway. + if actual_diff != diff_id { + bail!( + "layer {layer_index} of {reference} does not match its diff_id \ + (got {actual_diff}, config says {diff_id})" + ); + } + + // overlay2 finds a layer through the `l/` symlink and its `link` file. + symlink( + format!("../{chain}/diff"), + root.join("overlay2/l").join(short), + )?; + write(layer_dir.join("link"), short)?; + if let Some(ps) = parent_short { + write(layer_dir.join("lower"), &lower_chain(root, ps)?)?; + } + + let layerdb = root.join("image/overlay2/layerdb/sha256").join(chain); + std::fs::create_dir_all(&layerdb)?; + write(layerdb.join("diff"), diff_id)?; + write(layerdb.join("cache-id"), chain)?; + write(layerdb.join("size"), &layer_size.to_string())?; + if let Some(p) = parent { + write(layerdb.join("parent"), &format!("sha256:{p}"))?; + } + Ok(()) +} + +/// docker's chain-id for a layer: `sha256("sha256: sha256:")`. +fn chain_id(parent_chain_hex: &str, diff_hex: &str) -> String { + let mut h = Sha256::new(); + h.update(format!("sha256:{parent_chain_hex} sha256:{diff_hex}").as_bytes()); + hex::encode(h.finalize()) +} + +/// Deterministic 26-char [A-Z0-9] overlay2 link name (docker uses a random one). +fn short_link(chain_hex: &str) -> String { + let mut s = String::from("L"); + s.push_str(&chain_hex[..25].to_uppercase()); + s +} + +/// Build the `lower` file for a layer, given its parent's link name. +/// +/// overlay2 lists every ancestor there, nearest first. We build that by taking +/// the parent's own `lower` and prepending the parent. +fn lower_chain(root: &Path, parent_short: &str) -> Result { + // resolve the parent's chain dir from its l/ symlink to read its `lower`. + let parent_dir = std::fs::read_link(root.join("overlay2/l").join(parent_short))?; + // ..//diff -> + let chain = parent_dir + .parent() + .and_then(|p| p.file_name()) + .and_then(|s| s.to_str()) + .context("bad parent link")? + .to_string(); + let plower = root.join("overlay2").join(&chain).join("lower"); + let mut parts = vec![format!("l/{parent_short}")]; + if let Ok(existing) = std::fs::read_to_string(&plower) { + parts.extend(existing.split(':').map(str::to_string)); + } + Ok(parts.join(":")) +} + +/// Extract a layer tar into `dest`, converting AUFS whiteouts as it goes. +/// +/// Handles gzip and plain tar. Returns two things: the layer's uncompressed size +/// (what docker records), and its diff-id — the sha256 of the whole decompressed +/// tar, which the caller checks against the config. +fn extract_layer(data: &[u8], media_type: &str, dest: &Path) -> Result<(u64, String)> { + let is_gzip = + media_type.contains("gzip") || (data.len() >= 2 && data[0] == 0x1f && data[1] == 0x8b); + let decompressed: Box = if is_gzip { + Box::new(GzDecoder::new(data)) + } else { + Box::new(data) + }; + let mut ar = tar::Archive::new(HashingReader::new(decompressed)); + ar.set_preserve_permissions(true); + ar.set_preserve_mtime(true); + ar.set_unpack_xattrs(true); + ar.set_preserve_ownerships(true); + + let mut total = 0u64; + for entry in ar.entries()? { + let mut entry = entry?; + if entry.header().entry_type().is_file() { + total += entry.header().size().unwrap_or(0); + } + let path = entry.path()?.into_owned(); + let name = match path.file_name().and_then(|s| s.to_str()) { + Some(n) => n.to_string(), + None => continue, + }; + + if let Some(rest) = name.strip_prefix(".wh.") { + // Whiteouts are placed by hand (mknod / an xattr) rather than through + // the tar crate, so they need their own containment. Without it a + // hostile layer could escape `dest` — lexically, or through a + // symlinked parent it planted earlier — and we run as root. + // `safe_subpath` won't follow a symlink out of `dest`. + let out_parent = + match safe_subpath(dest, path.parent().unwrap_or_else(|| Path::new(""))) { + Some(p) => p, + None => continue, + }; + if name == ".wh..wh..opq" { + std::fs::create_dir_all(&out_parent)?; + xattr::set(&out_parent, "trusted.overlay.opaque", b"y") + .context("setting overlay opaque xattr (need root)")?; + } else if rest.starts_with(".wh.") { + // other reserved aufs metadata (e.g. `.wh..wh..plnk`) — not a + // whiteout for `rest`; ignore. + } else if rest.is_empty() || rest == "." || rest == ".." { + // a name like `.wh.`, `.wh..`, or `.wh...` would target the parent + // dir itself or escape it; ignore. + } else { + std::fs::create_dir_all(&out_parent)?; + let target = out_parent.join(rest); + let _ = remove_any(&target); + mknod_whiteout(&target).context("creating overlay whiteout (need root)")?; + } + continue; + } + // normal entry — the tar crate sandboxes this against `dest` (rejects + // `..`/absolute and creates parents itself). + entry.unpack_in(dest)?; + } + // drain any trailing padding the entry iterator didn't read, so the digest + // covers the whole tar, then finalize the diff-id. + let mut hr = ar.into_inner(); + std::io::copy(&mut hr, &mut std::io::sink())?; + let diff_id = format!("sha256:{}", hex::encode(hr.finalize())); + Ok((total, diff_id)) +} + +/// A reader that sha256's every byte passing through it. +/// +/// This lets us compute a layer's diff-id while extracting it, instead of +/// decompressing the layer a second time. +struct HashingReader { + inner: R, + hasher: Sha256, +} +impl HashingReader { + fn new(inner: R) -> Self { + Self { + inner, + hasher: Sha256::new(), + } + } + fn finalize(self) -> impl AsRef<[u8]> { + self.hasher.finalize() + } +} +impl Read for HashingReader { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + let n = self.inner.read(buf)?; + self.hasher.update(&buf[..n]); + Ok(n) + } +} + +/// Resolve `rel` under `dest`, but only if it stays inside `dest`. +/// +/// Returns `None` for a `..` or absolute component. It also refuses to walk +/// through any component that already exists as a symlink, so a symlink a +/// hostile layer planted earlier can't be used to escape. (The tar crate gets +/// the same guarantee by canonicalizing.) +fn safe_subpath(dest: &Path, rel: &Path) -> Option { + use std::path::Component; + let mut cur = dest.to_path_buf(); + for comp in rel.components() { + match comp { + Component::CurDir => {} + Component::Normal(c) => { + cur.push(c); + if let Ok(md) = std::fs::symlink_metadata(&cur) { + if md.file_type().is_symlink() { + return None; + } + } + } + // absolute root, `..`, or a Windows prefix: reject outright. + _ => return None, + } + } + Some(cur) +} + +/// overlay2 whiteout: a character device with major/minor 0/0. +fn mknod_whiteout(path: &Path) -> Result<()> { + use rustix::fs::{mknodat, FileType, Mode, CWD}; + mknodat(CWD, path, FileType::CharacterDevice, Mode::empty(), 0)?; + Ok(()) +} + +fn remove_any(path: &Path) -> std::io::Result<()> { + match std::fs::symlink_metadata(path) { + Ok(m) if m.is_dir() => std::fs::remove_dir_all(path), + Ok(_) => std::fs::remove_file(path), + Err(e) => Err(e), + } +} + +fn strip_sha256(d: &str) -> Result<&str> { + let hex = d + .strip_prefix("sha256:") + .with_context(|| format!("expected sha256: digest, got '{d}'"))?; + // validate here so every downstream chain-id / link name is 64 hex chars, + // and slicing (e.g. short_link's [..25]) can't panic on hostile registry data. + if hex.len() != 64 || !hex.bytes().all(|b| b.is_ascii_hexdigit()) { + bail!("malformed sha256 digest '{d}' (expected 64 hex chars)"); + } + Ok(hex) +} + +fn symlink(target: impl AsRef, link: impl AsRef) -> std::io::Result<()> { + std::os::unix::fs::symlink(target, link) +} + +fn write(path: PathBuf, contents: &str) -> std::io::Result<()> { + std::fs::write(path, contents.as_bytes()) +} + +fn write_bytes(path: PathBuf, contents: &[u8]) -> std::io::Result<()> { + std::fs::write(path, contents) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn strip_sha256_validates_64_hex() { + let ok = "a".repeat(64); + assert_eq!(strip_sha256(&format!("sha256:{ok}")).unwrap(), ok); + assert!(strip_sha256("nothex").is_err()); // no sha256: prefix + assert!(strip_sha256("sha256:abc").is_err()); // too short (would panic short_link) + assert!(strip_sha256(&format!("sha256:{}", "g".repeat(64))).is_err()); // not hex + assert!(strip_sha256("sha256:€€€€€€€€€€€€€€€€€€€€€€€€€€€€€€€€").is_err()); + // multibyte + } + + #[test] + fn chain_id_matches_docker_formula() { + // base layer chain-id == diff-id. + let d = "0000000000000000000000000000000000000000000000000000000000000000"; + // second layer: sha256("sha256: sha256:") + let c = chain_id(d, d); + let mut h = Sha256::new(); + h.update(format!("sha256:{d} sha256:{d}").as_bytes()); + assert_eq!(c, hex::encode(h.finalize())); + } + + #[test] + fn short_link_is_26_upper_alnum() { + let s = short_link("abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"); + assert_eq!(s.len(), 26); + assert!(s.starts_with('L')); + assert!(s + .chars() + .all(|c| c.is_ascii_uppercase() || c.is_ascii_digit())); + } + + #[test] + fn safe_subpath_rejects_traversal_absolute_and_symlink() { + let dir = tempfile::tempdir().unwrap(); + let dest = dir.path(); + // plain nested paths resolve inside dest. + assert_eq!(safe_subpath(dest, Path::new("a/b")), Some(dest.join("a/b"))); + assert_eq!( + safe_subpath(dest, Path::new("./app")), + Some(dest.join("app")) + ); + // lexical escapes are refused. + assert!(safe_subpath(dest, Path::new("../etc")).is_none()); + assert!(safe_subpath(dest, Path::new("/etc")).is_none()); + assert!(safe_subpath(dest, Path::new("a/../../etc")).is_none()); + // a planted symlink parent must not be traversed (the real exploit). + std::os::unix::fs::symlink("/var", dest.join("evil")).unwrap(); + assert!(safe_subpath(dest, Path::new("evil")).is_none()); + assert!(safe_subpath(dest, Path::new("evil/lib")).is_none()); + } + + #[test] + fn whiteout_through_symlinked_parent_cannot_escape() { + // End-to-end: a hostile layer plants `evil -> ` then a + // whiteout `evil/.wh.keep`; without containment that would delete the + // victim file outside dest, as root. + use std::os::unix::fs::MetadataExt; + let dest = tempfile::tempdir().unwrap(); + let victim = tempfile::tempdir().unwrap(); + std::fs::write(victim.path().join("keep"), b"x").unwrap(); + // stamp the tar entries with our own uid/gid so unpacking the symlink + // doesn't need root to chown. + let md = std::fs::metadata(victim.path()).unwrap(); + let (uid, gid) = (md.uid() as u64, md.gid() as u64); + + let mut b = tar::Builder::new(Vec::new()); + let mut h = tar::Header::new_gnu(); + h.set_entry_type(tar::EntryType::Symlink); + h.set_size(0); + h.set_uid(uid); + h.set_gid(gid); + h.set_mode(0o777); + b.append_link(&mut h, "evil", victim.path()).unwrap(); + let mut h2 = tar::Header::new_gnu(); + h2.set_entry_type(tar::EntryType::Regular); + h2.set_size(0); + h2.set_uid(uid); + h2.set_gid(gid); + h2.set_mode(0o644); + b.append_data(&mut h2, "evil/.wh.keep", std::io::empty()) + .unwrap(); + let tar_bytes = b.into_inner().unwrap(); + + extract_layer( + &tar_bytes, + "application/vnd.oci.image.layer.v1.tar", + dest.path(), + ) + .unwrap(); + + assert!( + victim.path().join("keep").exists(), + "whiteout escaped dest through a symlinked parent" + ); + } +} diff --git a/dstack/crates/dstack-verity/src/volume.rs b/dstack/crates/dstack-verity/src/volume.rs new file mode 100644 index 000000000..79dc13fb9 --- /dev/null +++ b/dstack/crates/dstack-verity/src/volume.rs @@ -0,0 +1,238 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! Pack a directory into a reproducible, verity-protected volume. +//! +//! The output is one file: the squashfs image, then its dm-verity hash tree. +//! squashfs is used because the guest kernel already mounts it (it's the guest's +//! own rootfs), and because `mksquashfs` with a pinned timestamp is reproducible +//! byte for byte. The verity root is a pure function of the squashfs bytes and +//! the salt. + +use std::path::Path; +use std::process::Command; + +use anyhow::{bail, Context, Result}; +use sha2::{Digest, Sha256}; + +/// Fixed build timestamp (2024-01-01), so the image is the same no matter when +/// it's built. It never shows up at runtime: `mksquashfs -all-time` normalizes +/// the store's own timestamps away. +const EPOCH: &str = "1704067200"; +const BLOCK: u64 = 4096; + +#[derive(Clone, Copy)] +pub enum Compression { + /// No compression, so nothing is decompressed at read time (the default). + None, + Zstd, + Gzip, +} + +impl Compression { + fn args(self) -> Vec<&'static str> { + match self { + // disable every compressible section: inodes, data, fragments, + // xattrs, id table. + Compression::None => vec!["-noI", "-noD", "-noF", "-noX", "-noId"], + Compression::Zstd => vec!["-comp", "zstd"], + Compression::Gzip => vec![], + } + } +} + +pub struct BuiltVolume { + pub verity_root: String, + /// size of the squashfs region = the verity hash offset. + pub data_size: u64, +} + +/// Build `output`: the squashfs image of `store`, then its dm-verity hash tree. +/// +/// `store` is any directory — a docker overlay2 store, or plain data. `salt` +/// fixes the verity root. +/// +/// The verity UUID is derived from the squashfs bytes, so two different volumes +/// get two different UUIDs. That matters because a VM can mount several volumes +/// at once, and a fixed UUID would be shared by all of them. +pub fn build_volume( + store: &Path, + output: &Path, + salt_hex: &str, + compress: Compression, +) -> Result { + require_tool("mksquashfs")?; + require_tool("veritysetup")?; + if output.exists() { + std::fs::remove_file(output).ok(); + } + + // 1. reproducible squashfs. + let mut cmd = Command::new("mksquashfs"); + cmd.arg(store).arg(output); + cmd.args(compress.args()); + cmd.args([ + "-all-time", + EPOCH, + "-mkfs-time", + EPOCH, + "-noappend", + "-no-progress", + "-xattrs", + ]); + run(cmd, "mksquashfs")?; + + // 2. the verity data region must be block-aligned; pad the squashfs up. + let bytes_used = squashfs_bytes_used(output)?; + let data_size = bytes_used.div_ceil(BLOCK) * BLOCK; + { + let f = std::fs::OpenOptions::new().write(true).open(output)?; + f.set_len(data_size)?; + } + + // 3. Append the verity hash tree to the same file, at the aligned offset. + // Pin the superblock UUID too: veritysetup randomizes it otherwise, and we + // want the whole file reproducible, not just the root. The UUID sits in the + // hash tree, not the hashed data, so it never changes the root. + let uuid = uuid_from_data(output, data_size)?; + let mut cmd = Command::new("veritysetup"); + cmd.args([ + "format", + "--salt", + salt_hex, + "--uuid", + &uuid, + "--data-block-size", + "4096", + "--hash-block-size", + "4096", + "--hash-offset", + &data_size.to_string(), + ]); + cmd.arg(output).arg(output); + let out = capture(cmd, "veritysetup format")?; + let verity_root = + parse_root_hash(&out).context("could not find the root hash in veritysetup output")?; + + Ok(BuiltVolume { + verity_root, + data_size, + }) +} + +/// squashfs superblock: `bytes_used` is a little-endian u64 at offset 40. +fn squashfs_bytes_used(path: &Path) -> Result { + use std::io::{Read, Seek, SeekFrom}; + let mut f = std::fs::File::open(path)?; + let mut magic = [0u8; 4]; + f.read_exact(&mut magic)?; + if &magic != b"hsqs" { + bail!("not a squashfs image (bad magic)"); + } + f.seek(SeekFrom::Start(40))?; + let mut buf = [0u8; 8]; + f.read_exact(&mut buf)?; + Ok(u64::from_le_bytes(buf)) +} + +/// Derive a UUID from the first `len` bytes of `path`. +/// +/// Deterministic, and shaped like a version-4 UUID. +fn uuid_from_data(path: &Path, len: u64) -> Result { + use std::io::Read; + let mut f = std::fs::File::open(path)?; + let mut h = Sha256::new(); + let mut remaining = len; + let mut buf = vec![0u8; 1 << 20]; + while remaining > 0 { + let n = remaining.min(buf.len() as u64) as usize; + f.read_exact(&mut buf[..n])?; + h.update(&buf[..n]); + remaining -= n as u64; + } + let d = h.finalize(); + let mut u = [0u8; 16]; + u.copy_from_slice(&d[..16]); + u[6] = (u[6] & 0x0f) | 0x40; // version 4 + u[8] = (u[8] & 0x3f) | 0x80; // RFC 4122 variant + let hx = hex::encode(u); + Ok(format!( + "{}-{}-{}-{}-{}", + &hx[0..8], + &hx[8..12], + &hx[12..16], + &hx[16..20], + &hx[20..32] + )) +} + +fn parse_root_hash(output: &str) -> Option { + output + .lines() + .find_map(|l| l.strip_prefix("Root hash:")) + .map(|v| v.trim().to_string()) +} + +fn require_tool(name: &str) -> Result<()> { + // spawning at all means the binary is on PATH; a non-zero exit from + // `--version` (some builds) still counts as present. + let present = Command::new(name) + .arg("--version") + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status() + .is_ok(); + if !present { + bail!("`{name}` not found on PATH (install squashfs-tools / cryptsetup)"); + } + Ok(()) +} + +fn run(mut cmd: Command, what: &str) -> Result<()> { + let status = cmd + .stdout(std::process::Stdio::null()) + .status() + .with_context(|| format!("running {what}"))?; + if !status.success() { + bail!("{what} failed with {status}"); + } + Ok(()) +} + +fn capture(mut cmd: Command, what: &str) -> Result { + let out = cmd.output().with_context(|| format!("running {what}"))?; + if !out.status.success() { + bail!( + "{what} failed: {}", + String::from_utf8_lossy(&out.stderr).trim() + ); + } + Ok(String::from_utf8_lossy(&out.stdout).into_owned()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_veritysetup_root() { + let sample = "VERITY header information for x\nUUID: \nHash type: 1\n\ + Data blocks: 10\nRoot hash: abc123def\n"; + assert_eq!(parse_root_hash(sample).as_deref(), Some("abc123def")); + } + + #[test] + fn uuid_is_deterministic_and_content_specific() { + use std::io::Write; + let mut a = tempfile::NamedTempFile::new().unwrap(); + a.write_all(b"hello world data").unwrap(); + let mut b = tempfile::NamedTempFile::new().unwrap(); + b.write_all(b"different data..").unwrap(); + let ua = uuid_from_data(a.path(), 16).unwrap(); + assert_eq!(ua, uuid_from_data(a.path(), 16).unwrap()); // deterministic + assert_ne!(ua, uuid_from_data(b.path(), 16).unwrap()); // content-specific + assert_eq!(ua.len(), 36); + assert_eq!(&ua[14..15], "4"); // version nibble + } +} From e083b391dd9ba428f8503cbe4e8cdb7bba10d758 Mon Sep 17 00:00:00 2001 From: Hang Yin Date: Sat, 4 Jul 2026 02:56:11 +0000 Subject: [PATCH 04/25] docs: add verity volumes design Co-Authored-By: Claude Opus 4.8 (1M context) --- dstack/docs/verity-volumes.md | 146 ++++++++++++++++++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 dstack/docs/verity-volumes.md diff --git a/dstack/docs/verity-volumes.md b/dstack/docs/verity-volumes.md new file mode 100644 index 000000000..fb1dd0b28 --- /dev/null +++ b/dstack/docs/verity-volumes.md @@ -0,0 +1,146 @@ +# Verity Volumes: pre-seeding images and data into a CVM + +Starting a CVM is slow mostly because of image *extraction*: decompressing layers and writing millions of files onto the encrypted disk. On one pinned vCPU this dominates — about 30 s for a 3.3 GB image, ~2 min for 7 GB, with a hard ~15 s decompress floor no amount of storage tuning beats. Download is the smaller cost. + +A verity volume removes it. It's a read-only, dm-verity-protected disk, built once, that a CVM mounts and uses as-is. The layers on it are already extracted, so the CVM does no pull and no extraction — it mounts the volume and verifies blocks lazily as the app reads them. One volume can back many CVMs at once. + +The build tool (`dstack verity`) and the in-guest seeding are implemented and validated end to end ([What's proven](#whats-proven)). + +## The shape of it + +A volume has a **root hash** — its identity and integrity check — and a **target** that says what it's for: + +- `"docker"` — the volume is a docker overlay2 store; its images are seeded into docker as cache hits. +- `"/some/path"` — the volume's filesystem is mounted there, for data like model weights. + +An app lists its volumes in `app-compose.json`. dstack already hashes that file into the app's identity, so every root hash is measured as part of `app_id`, and the CVM only uses content matching what it was attested as. Everything else is untrusted: the host hands over bytes, and dm-verity rejects any that don't match the root. + +A volume is *additive* — it only adds read-only lower layers, so anything not on one is pulled normally — and *fail-safe*: a missing or mismatched volume falls back to a pull. + +## What a user writes + +The `docker-compose.yaml` is unchanged (pin images by digest so the measured identity binds the exact bytes): + +A data volume's mount point must be on a writable filesystem — the guest rootfs is read-only, so use `/run/...` (a writable tmpfs) or the app's data disk, not an arbitrary top-level path like `/models`: + +```yaml +services: + vllm: + image: vllm/vllm-openai@sha256:abc123... # from the "docker" volume + command: ["--model", "/run/models/llama-70b"] + volumes: + - /run/models/llama-70b:/run/models/llama-70b:ro # from the data volume +``` + +Each volume is one `verity_volumes` entry in the measured `app-compose.json`: + +```json +"verity_volumes": [ + { "verity_root": "115b6877...", "target": "docker" }, + { "verity_root": "a1b2c3d4...", "target": "/run/models/llama-70b" } +] +``` + +You don't hand-write that with the `dstack` CLI — `dstack deploy --volume` generates it (below). Build volumes with `dstack verity`, which prints the exact `--volume` spec to paste: + +```bash +dstack verity vllm/vllm-openai@sha256:abc123... # -> --volume vllm.img:115b6877...:docker +dstack verity --dir ./llama-70b-weights/ # -> --volume llama-70b.img:a1b2c3d4...:/run/models/llama-70b +``` + +It needs no docker daemon, no CVM, and no TDX. For images it pulls the pinned layers from the registry and lays out the docker store itself; for `--dir` it packs the directory as-is. Either way it writes one volume file. Pass several images to pack them into a single `docker` volume; shared base layers are stored once. There is no auto-detection: a volume contains whatever you built it from, and the compose names those images normally. + +This covers both cases. Several volumes in one CVM (the image *and* the model) is several entries with different targets. Several images from one volume is a single `docker` volume built from all of them, listed as one entry, with the compose unchanged. Attach order does not matter ([Delivery](#delivery)). + +## Trust + +`verity_root` lives in `app-compose.json`, so it's part of the hash that becomes `app_id` (see [Normalized App Compose](./normalized-app-compose.md)), and it is enforced by dm-verity. The consumer trusts the root hash, not whoever built the volume. A host that tampers with the bytes causes a verity fault, not a silent swap, so the delivery path can stay untrusted. + +`dstack verity` is deterministic: the same pinned image digests always produce the same `verity_root`, bit-for-bit (canonical overlay2 layout with chain-id cache-ids; a fixed timestamp and salt; a UUID derived from the packed bytes). One caveat: the squashfs layout is produced by `mksquashfs`, so a verifier must use a `squashfs-tools` version that lays out bytes identically (recent versions are stable; the build records nothing about the tool version yet). A verifier can therefore recompute the root from the digest-pinned images and confirm the volume is those images, without trusting whoever ran the build. The build needs no docker daemon and no TEE, so it can also run in a CI job that attests it (see [Reproducible builds and provenance](#reproducible-builds-and-provenance)); the two checks are independent. Turning on a volume changes `app-compose.json` and thus `app_id`, which is correct: the volume is part of the measured configuration. + +## How docker seeding works + +A `docker` volume is a squashfs filesystem holding a docker overlay2 store, which has two very different halves: + +``` +image/overlay2/ metadata: image configs, layer db, tags (KB to a few MB) +overlay2//diff/ the already-extracted layer files (the GBs) +``` + +`diff/` is the layer already decompressed and untarred. Seeding reuses it in place: + +1. Open the volume with veritysetup and mount it read-only at a writable path (`/run/dstack-verity`). The rootfs is read-only dm-verity, so the mountpoint has to be on a writable fs; squashfs itself mounts read-only directly, no journal and nothing to replay. +2. Copy the metadata (a few MB) into `/var/lib/docker`. +3. Per layer: make a writable `overlay2//` dir, copy its handful of tiny files, and bind-mount only the big `diff/` read-only from the volume. +4. This runs from `dstack-prepare.sh` *before* dockerd starts, so there's no restart — dockerd comes up with every image on the volume already present. + +The layer dir stays writable because docker writes a `committed` marker into it on first use; only `diff/` underneath is read-only. (Bind-mounting the whole dir read-only is the one thing that breaks container creation.) + +squashfs is used, rather than ext4, for two reasons: the dstack guest kernel mounts it (it's the guest's own rootfs — erofs, the other obvious candidate, isn't compiled in), and `mksquashfs` with a pinned timestamp is byte-reproducible with no extra work, where ext4 needs its wall-clock superblock and inode times patched out. It's built fully uncompressed, so there's no decompression at read time either — the point was to remove extraction, not move it. + +At run time the container's overlay uses the verity-backed `diff/`s as lower layers and a fresh writable upper on the encrypted disk. Reads are verified per block on first touch, then cached; nothing is decompressed or written. This is where the ~1 s start comes from instead of 30 s–2 min. A data volume is simpler: open, mount at the path, and the container bind-mounts it — no unpacking. + +## Delivery + +The volume *file* and the deploy request travel separately. First set `cvm.volumes_dir` in the host's `vmm.toml` (it's empty by default, which disables volume attachment). The operator then places the built file in that `volumes_dir` (however they like — copy, object store, shared mount), and deploy references it by bare file name: + +```bash +dstack deploy -c docker-compose.yaml \ + --volume vllm.img:115b6877...:docker \ + --volume llama-70b.img:a1b2c3d4...:/models/llama-70b +``` + +Each `--volume NAME:VERITY_ROOT:TARGET` both attaches the file and writes the measured `verity_volumes` entry (the root is yours to supply, from `dstack verity`, so it stays part of `app_id`). `vmm-cli.py` takes the same file with a hand-authored `app-compose.json` instead. The vmm resolves each name against `volumes_dir` (rejecting anything with a path separator) and attaches it as a read-only virtio-blk device, tagged with a hint: the disk's serial is set to the volume's `verity_root` prefix. `--volume` carries only the name and read-only flag, never the bytes; the bytes are already on the host. Disks are still attached in any order — the guest reads the serials once and, for each `verity_root` in the compose, opens the disk whose serial matches. That is O(devices + volumes), and no wrong disk is opened. The serial is only a hint and isn't trusted: the guest verifies the full root through dm-verity, so a missing or wrong tag falls back to trying every disk, and a tampering host causes a verity fault rather than a wrong mount. Integrity is checked lazily, per block on first read. Read-only plus content-addressed means one physical copy serves every CVM that references it: a base image or model shared by a hundred replicas is built, stored, and extracted once. + +## Building volumes + +`dstack verity` builds the store from scratch, without a docker daemon and without a CVM — just the registry (a `docker` volume does need root, because laying out overlay2 whiteouts uses `mknod` and a `trusted.*` xattr; a `--dir` volume needs no root). It pulls each layer by digest, and lays out the overlay2 store the way docker would, with one deliberate change: the per-layer directory id is the layer's *chain-id* instead of docker's random cache-id. That's what makes the store a pure function of the image. AUFS `.wh.` whiteouts in the layer tars are converted to their overlay2 on-disk form (a `0:0` char device, or the `trusted.overlay.opaque` xattr) so deletions in upper layers still take effect. Then `mksquashfs` packs it with a fixed timestamp and `veritysetup` appends the hash tree in the same file (data in `[0, size)`, hash tree after) with a fixed salt and a UUID derived from the squashfs bytes. A `--dir` volume skips the overlay2 layout and pull entirely — it packs the directory straight into the same squashfs + verity format. + +With those fixed, two runs of the same digests produce the same bytes. The build needs no daemon and no TEE, so it can run in a CI job that also attests it (below). + +### Reproducible builds and provenance + +Because the root is a deterministic function of the pinned image digests, a verifier has two independent checks: + +- **Reproducibility** — recompute the root from the digests and check it matches `app-compose.json`. This needs nothing but the digests and does not trust the builder. +- **Provenance** — run `verity` in CI and attest the output with [SLSA build provenance](https://github.com/actions/attest-build-provenance) / Sigstore. This ties the root to a specific workflow and inputs. + +Either alone is sufficient; they can be used together. In your own app repo — using an installed `dstack` (verity volumes aren't tied to the dstack source tree): + +```yaml +# .github/workflows/build-verity-volume.yml +permissions: { id-token: write, attestations: write, contents: read } +jobs: + build: + runs-on: ubuntu-24.04 + steps: + - run: sudo apt-get install -y squashfs-tools cryptsetup-bin + - run: sudo dstack verity "$IMAGE" -o volume.img --json | tee out.json + - uses: actions/attest-build-provenance@v2 + with: { subject-path: volume.img } +``` + +(`sudo` because the docker-store layout needs `mknod` + a `trusted.*` xattr.) + +## What this doesn't do yet + +- Images should be pinned by digest; a mutable tag doesn't bind the bytes (a digest pin is verified against the fetched manifest, so a swapped registry response is caught). +- `verity` reads gzip and uncompressed layers; zstd-compressed layers are not handled yet (it fails rather than producing a wrong store). +- The compose must reference an image the same way it was passed to `verity`. Docker-Hub names are normalized to their canonical form (`docker.io/library/`), so a bare `alpine` and `docker.io/library/alpine` match; a private-registry reference must match verbatim. +- A data volume's `target` is a mount point, and must be on a writable filesystem in the guest (e.g. under `/run`, or the app's data disk) — the rootfs is read-only dm-verity, so an arbitrary top-level path can't be created. A missing/unwritable target is skipped, fail-safe. +- A `docker` volume is architecture-specific. `--platform` selects it (default `linux/amd64`, matching today's Intel TDX guests); an arm64 confidential host (NVIDIA Vera, AWS Graviton/Nitro) needs `linux/arm64`. dm-verity checks bytes, not architecture, so a wrong-arch volume whose root you pinned still opens and seeds — the container then fails to exec (wrong-arch binaries), it isn't caught up front. Match `--platform` to the guest; recording the platform in the volume and checking it in the guest is a follow-up. +- Layers are treated as public — the volume is unencrypted and shareable. Secret layers would need a per-app, KMS-encrypted volume, and that's out of scope for now. +- Under TDX the single-copy saving is on storage, transfer, and extraction, not RAM — each CVM still caches the blocks it reads in its own encrypted memory. +- The per-boot `docker image prune -af` removes images no running container references, so bake the images your compose actually runs; guarding prune against verity-seeded images is a follow-up. +- On guest images that don't yet ship the helper, `vmm-cli` injects it into `init_script`, so `app_id` depends on the bundled helper bytes — pin your CLI/image version when you register compose hashes. +- Seeded overlay2 metadata lives on the persistent disk, while the layer `diff/` binds are re-established each boot. A *partial* seed (a metadata copy fails mid-write) unwinds its binds and falls back to a normal pull. The open gap is across boots: if a volume that seeded once is later not re-attached (or swapped), its metadata persists while its `diff/` binds don't, so docker can see the image present with empty layers. The normal reboot (same volume) re-binds correctly. Reconciling stale seeded metadata against missing binds on boot is a follow-up — it needs image→layer dependency walking so it doesn't drop a pulled image that happens to share a base layer. + +## What's proven + +The whole path ran on Intel TDX with no change to the guest OS image — only a host-side change to attach the disk. A fresh CVM — different app, empty image list, never having pulled the image — attached a pre-built volume read-only, matched it by root hash, mounted it, and seeded docker. The image was present with no pull and no extraction: essentially nothing was written to the CVM's disk during seeding (an extraction would write gigabytes), seeding took about a second, and the container ran in about a second — versus 30 s to 2 min to pull and extract. That confirms a fresh CVM gets an image purely from a read-only attested volume, and that unrelated images still pull. + +The build side is proven too. `dstack verity` builds a volume daemonlessly and is byte-for-byte reproducible: independent runs of the same pinned image produce the same volume and the same `verity_root`, including a multi-layer image with whiteouts. The resulting store loads and runs in a stock docker with no pull, and the overlay2 whiteouts apply correctly (deleted files stay deleted). The filesystem choice was checked against the live guest kernel: it mounts uncompressed squashfs and reads it back correctly; erofs is not compiled in. + +## Open questions + +Whether the guest-agent should handle `verity_volumes` natively instead of through a pre-launch helper; how the operator garbage-collects stale volumes and tracks roots across image updates; how volume files are distributed to hosts at scale (`--volume` references a name already in `volumes_dir`); and encrypted volumes for secret layers. From c0a1a8915c4a41a29770a3fdac985b3640d7759f Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 5 Jul 2026 21:02:11 -0700 Subject: [PATCH 05/25] verity: use partitioned volume images --- dstack/Cargo.lock | 37 +++ dstack/basefiles/dstack-verity.sh | 106 +++---- dstack/crates/dstack-cli/Cargo.toml | 1 + dstack/crates/dstack-cli/src/main.rs | 265 +++++++++-------- dstack/crates/dstack-verity/Cargo.toml | 3 + dstack/crates/dstack-verity/src/lib.rs | 52 +++- dstack/crates/dstack-verity/src/store.rs | 35 +-- dstack/crates/dstack-verity/src/volume.rs | 330 +++++++++++++++++++--- dstack/docs/verity-volumes.md | 22 +- dstack/vmm/src/app.rs | 4 +- dstack/vmm/src/discovery.rs | 9 +- dstack/vmm/src/main_service.rs | 6 +- dstack/vmm/src/one_shot.rs | 13 +- 13 files changed, 625 insertions(+), 258 deletions(-) diff --git a/dstack/Cargo.lock b/dstack/Cargo.lock index d24b5b8f9..c36f7f7cc 100644 --- a/dstack/Cargo.lock +++ b/dstack/Cargo.lock @@ -1120,6 +1120,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + [[package]] name = "crc32fast" version = "1.5.0" @@ -1825,6 +1840,7 @@ dependencies = [ "clap", "dstack-cli-core", "dstack-verity", + "fs-err", "serde_json", "tokio", "tracing", @@ -2256,7 +2272,9 @@ version = "0.6.0" dependencies = [ "anyhow", "flate2", + "fs-err", "futures", + "gpt", "hex", "oci-client", "rustix 0.38.44", @@ -2267,6 +2285,7 @@ dependencies = [ "tempfile", "tokio", "tracing", + "uuid", "xattr", ] @@ -2993,6 +3012,18 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "gpt" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3696fafb1ecdcc2ae3ce337de73e9202806068594b77d22fdf2f3573c5ec2219" +dependencies = [ + "bitflags 2.11.1", + "crc", + "simple-bytes", + "uuid", +] + [[package]] name = "group" version = "0.13.0" @@ -7115,6 +7146,12 @@ version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" +[[package]] +name = "simple-bytes" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c11532d9d241904f095185f35dcdaf930b1427a94d5b01d7002d74ba19b44cc4" + [[package]] name = "siphasher" version = "1.0.3" diff --git a/dstack/basefiles/dstack-verity.sh b/dstack/basefiles/dstack-verity.sh index 71b7e2028..9825cef1e 100755 --- a/dstack/basefiles/dstack-verity.sh +++ b/dstack/basefiles/dstack-verity.sh @@ -33,78 +33,56 @@ if ! command -v veritysetup >/dev/null 2>&1; then fi modprobe dm-verity 2>/dev/null || true -# Candidate whole-disk block devices. Volumes are identified by root hash, not by -# name or attach order. The host tags each verity disk's serial with the root -# prefix, so the matching disk can be opened directly; a missing or wrong tag -# falls back to trying every disk. +# Candidate whole-disk block devices. A dstack verity volume is a GPT disk with: +# partition 1 = filesystem data +# partition 2 = dm-verity superblock + hash tree +# +# Volumes are identified by root hash, not by name or attach order. The host tags +# each verity disk's serial with the root prefix, so the matching disk can be +# opened directly; a missing or wrong tag falls back to trying every disk. mapfile -t DEVS < <(lsblk -dnro NAME,TYPE 2>/dev/null | awk '$2=="disk"{print "/dev/"$1}') -# Detect a device's filesystem from its magic bytes. -# `dstack verity` writes squashfs; ext4 is accepted too for hand-built volumes. -fs_type() { # $1 = device -> squashfs | ext4 | (empty) - local magic - # squashfs magic "hsqs" is at offset 0. - magic=$(dd if="$1" bs=1 count=4 2>/dev/null | od -An -tx1 | tr -d ' \n') - [ "$magic" = "68737173" ] && { echo squashfs; return; } - # ext4 magic 0xEF53 (little-endian on disk) is at byte 1080. - magic=$(dd if="$1" bs=1 skip=1080 count=2 2>/dev/null | od -An -tx1 | tr -d ' \n') - [ "$magic" = "53ef" ] && echo ext4 +# Return the Nth partition device path for a whole disk. +partition_n() { # $1 = disk, $2 = 1-based index + lsblk -nrpo NAME,TYPE "$1" 2>/dev/null | + awk -v want="$2" '$2=="part"{i++; if (i==want) {print $1; exit}}' } -# Byte size of the filesystem on $1. -# -# This is also the verity hash-tree offset, because a volume is laid out -# [ filesystem ][ hash tree ]. The size is read from the superblock and rounded -# up to a 4096 block. squashfs and ext4 record it differently. -# -# The superblock is untrusted, but that is safe. A tampered size just gives a -# wrong offset, and the verity open in open_volume then fails. -data_size() { # $1 = device, $2 = fs_type -> bytes (block-aligned) - local bytes byte i=0 info block_count block_size - case "$2" in - squashfs) - # bytes_used is a little-endian u64 at superblock offset 40. Rebuild - # it byte by byte, low byte first. - bytes=0 - for byte in $(dd if="$1" bs=1 skip=40 count=8 2>/dev/null | od -An -tu1); do - bytes=$((bytes + byte * (1 << (8 * i)))) - i=$((i + 1)) - done - # round up to the 4096 block boundary. - [ "$bytes" -gt 0 ] && echo $(((bytes + 4095) / 4096 * 4096)) ;; - ext4) - info=$(dumpe2fs -h "$1" 2>/dev/null) || return 1 - block_count=$(awk -F: '/Block count/{gsub(/ /,"",$2);print $2;exit}' <<<"$info") - block_size=$(awk -F: '/Block size/{gsub(/ /,"",$2);print $2;exit}' <<<"$info") - [ -n "$block_count" ] && [ -n "$block_size" ] && echo $((block_count * block_size)) ;; - esac +# Detect the filesystem only after dm-verity is active, so filesystem metadata is +# read through /dev/mapper and is covered by the measured root. +fs_type() { # $1 = verity-mapped device -> fstype | (empty) + blkid -o value -s TYPE "$1" 2>/dev/null || true } -# Probe every candidate disk once, recording "dev fs off serial" for each that -# looks like a verity volume. Doing this once keeps per-volume matching cheap. +# Probe every candidate disk once, recording "disk data_part hash_part serial". +# The partition table is untrusted, but that is safe: a tampered table just makes +# veritysetup open the wrong data/hash pair, which fails against the measured +# root. Doing this once keeps per-volume matching cheap. DEV_INFO=() scan_devices() { - local dev fs off serial + local dev data hash serial + if command -v udevadm >/dev/null 2>&1; then + udevadm settle --timeout=5 2>/dev/null || true + fi for dev in "${DEVS[@]}"; do - fs=$(fs_type "$dev") || continue - [ -n "$fs" ] || continue - off=$(data_size "$dev" "$fs") || continue - { [ -n "$off" ] && [ "$off" -gt 0 ]; } || continue + data=$(partition_n "$dev" 1) + hash=$(partition_n "$dev" 2) + if ! { [ -b "$data" ] && [ -b "$hash" ]; }; then + continue + fi serial=$(cat "/sys/block/$(basename "$dev")/serial" 2>/dev/null) - DEV_INFO+=("$dev $fs $off $serial") + DEV_INFO+=("$dev $data $hash $serial") done } -# Open $dev as /dev/mapper/$name if it verifies against $root. +# Open $data+$hash as /dev/mapper/$name if it verifies against $root. # On success, echoes "/dev/mapper/$name ". -try_open() { # $1 = dev, $2 = fs, $3 = off, $4 = root, $5 = name - local dev="$1" fs="$2" off="$3" root="$4" name="$5" - # $dev is both the data device and the hash device: it's one file, and - # --hash-offset is where the hash tree starts inside it. - veritysetup open "$dev" "$name" "$dev" "$root" --hash-offset="$off" \ - >/dev/null 2>&1 || return 1 +try_open() { # $1 = data_part, $2 = hash_part, $3 = root, $4 = name + local data="$1" hash="$2" root="$3" name="$4" fs + veritysetup open "$data" "$name" "$hash" "$root" >/dev/null 2>&1 || return 1 # Confirm the root matches: reading through verity must not fault. if dd if="/dev/mapper/$name" of=/dev/null bs=4096 count=1 >/dev/null 2>&1; then + fs=$(fs_type "/dev/mapper/$name") echo "/dev/mapper/$name $fs" return 0 fi @@ -118,17 +96,17 @@ try_open() { # $1 = dev, $2 = fs, $3 = off, $4 = root, $5 = name # prefix). That is the common path: one open, no probing. If nothing matches the # tag, or it fails to verify, fall back to trying every disk. open_volume() { # $1 = root hash, $2 = mapper name - local root="$1" name="$2" entry dev fs off serial + local root="$1" name="$2" entry dev data hash serial for entry in "${DEV_INFO[@]}"; do - read -r dev fs off serial <<<"$entry" + read -r dev data hash serial <<<"$entry" if [ -z "$serial" ] || [ "$serial" != "${root:0:20}" ]; then continue fi - try_open "$dev" "$fs" "$off" "$root" "$name" && return 0 + try_open "$data" "$hash" "$root" "$name" && return 0 done for entry in "${DEV_INFO[@]}"; do - read -r dev fs off serial <<<"$entry" - try_open "$dev" "$fs" "$off" "$root" "$name" && return 0 + read -r dev data hash serial <<<"$entry" + try_open "$data" "$hash" "$root" "$name" && return 0 done return 1 } @@ -137,8 +115,10 @@ open_volume() { # $1 = root hash, $2 = mapper name mount_ro() { # $1 = device, $2 = fs_type, $3 = mountpoint case "$2" in # noload: skip journal replay -- the device is read-only. - ext4) mount -t ext4 -o ro,noload "$1" "$3" 2>/dev/null ;; - *) mount -t "$2" -o ro "$1" "$3" 2>/dev/null ;; + ext3|ext4) mount -t "$2" -o ro,noload "$1" "$3" 2>/dev/null ;; + ext2) mount -t ext2 -o ro "$1" "$3" 2>/dev/null ;; + "") mount -o ro "$1" "$3" 2>/dev/null ;; + *) mount -t "$2" -o ro "$1" "$3" 2>/dev/null ;; esac } diff --git a/dstack/crates/dstack-cli/Cargo.toml b/dstack/crates/dstack-cli/Cargo.toml index 30a080a3c..cee4c1647 100644 --- a/dstack/crates/dstack-cli/Cargo.toml +++ b/dstack/crates/dstack-cli/Cargo.toml @@ -19,6 +19,7 @@ anyhow.workspace = true clap.workspace = true dstack-cli-core.workspace = true dstack-verity.workspace = true +fs-err.workspace = true serde_json.workspace = true tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } tracing.workspace = true diff --git a/dstack/crates/dstack-cli/src/main.rs b/dstack/crates/dstack-cli/src/main.rs index b018b7dcb..969df27f8 100644 --- a/dstack/crates/dstack-cli/src/main.rs +++ b/dstack/crates/dstack-cli/src/main.rs @@ -15,6 +15,7 @@ use clap::{Parser, Subcommand}; use dstack_cli_core::layout::InstallLayout; use dstack_cli_core::vmm::{Vmm, DEFAULT_HOST}; use dstack_cli_core::{compose, ports, rpc}; +use fs_err as fs; #[derive(Parser)] #[command( @@ -122,6 +123,10 @@ enum Command { /// path you choose in the compose). #[arg(long, value_name = "PATH", conflicts_with = "images")] dir: Option, + /// wrap an existing filesystem image instead of building squashfs. The + /// guest mounts it read-only after dm-verity verification. + #[arg(long = "fs-image", value_name = "PATH", conflicts_with_all = ["images", "dir"])] + fs_image: Option, /// where to write the volume. #[arg(long, short = 'o', default_value = "verity.img")] output: String, @@ -230,6 +235,7 @@ async fn main() -> Result<()> { Command::Verity { images, dir, + fs_image, output, compress, platform, @@ -238,6 +244,7 @@ async fn main() -> Result<()> { cmd_verity( &images, dir.as_deref(), + fs_image.as_deref(), &output, &compress, &platform, @@ -249,9 +256,11 @@ async fn main() -> Result<()> { } } +#[allow(clippy::too_many_arguments)] async fn cmd_verity( images: &[String], dir: Option<&str>, + fs_image: Option<&str>, output: &str, compress: &str, platform: &str, @@ -267,6 +276,7 @@ async fn cmd_verity( let result = dstack_verity::verity(dstack_verity::VerityOptions { images: images.to_vec(), dir: dir.map(std::path::PathBuf::from), + fs_image: fs_image.map(std::path::PathBuf::from), output: output.into(), compress, platform: platform.to_string(), @@ -274,6 +284,10 @@ async fn cmd_verity( }) .await?; + let volume_size = fs::metadata(&result.output) + .with_context(|| format!("stat {}", result.output.display()))? + .len(); + if json { let imgs: Vec<_> = result .images @@ -291,12 +305,13 @@ async fn cmd_verity( "verityRoot": result.verity_root, "output": result.output.display().to_string(), "dataSize": result.data_size, + "volumeSize": volume_size, "images": imgs, })); return Ok(()); } - let mib = result.data_size as f64 / 1_048_576.0; + let mib = volume_size as f64 / 1_048_576.0; println!("wrote {} ({mib:.1} MiB)", result.output.display()); if !result.images.is_empty() { // the manifest digest is what `image: repo@sha256:...` pins — not the @@ -395,7 +410,7 @@ struct LocalDefaults { impl LocalDefaults { fn read(prefix: Option<&str>) -> Option { let path = InstallLayout::state_path_for_prefix(prefix); - let body = std::fs::read_to_string(path).ok()?; + let body = fs::read_to_string(path).ok()?; let v: serde_json::Value = serde_json::from_str(&body).ok()?; Some(Self::from_value(&v)) } @@ -438,114 +453,6 @@ impl LocalDefaults { } } -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn parses_local_install_defaults() { - let value = serde_json::json!({ - "client_url": "http://127.0.0.1:19080", - "image": "dstack-0.5.11", - "allowlist_path": "/tmp/dstack/etc/dstack/auth-allowlist.json" - }); - let defaults = LocalDefaults::from_value(&value); - assert_eq!( - defaults.client_url.as_deref(), - Some("http://127.0.0.1:19080") - ); - assert_eq!(defaults.image.as_deref(), Some("dstack-0.5.11")); - assert_eq!( - defaults.allowlist_path().as_deref(), - Some("/tmp/dstack/etc/dstack/auth-allowlist.json") - ); - } - - #[test] - fn reads_local_install_defaults_from_prefix() { - let install_root = - std::env::temp_dir().join(format!("dstack-cli-state-test-{}", std::process::id())); - let state_dir = install_root.join("var/lib/dstack"); - std::fs::create_dir_all(&state_dir).unwrap(); - std::fs::write( - state_dir.join(dstack_cli_core::layout::STATE_FILE), - r#"{ - "client_url": "http://127.0.0.1:29080", - "image": "dstack-0.5.12", - "allowlist_path": "/tmp/custom-dstack/etc/dstack/auth-allowlist.json" - }"#, - ) - .unwrap(); - - let prefix = dstack_cli_core::layout::path_string(&install_root); - let defaults = LocalDefaults::read(Some(&prefix)).unwrap(); - assert_eq!( - defaults.client_url.as_deref(), - Some("http://127.0.0.1:29080") - ); - assert_eq!(defaults.image.as_deref(), Some("dstack-0.5.12")); - assert_eq!( - defaults.allowlist_path().as_deref(), - Some("/tmp/custom-dstack/etc/dstack/auth-allowlist.json") - ); - - let _ = std::fs::remove_dir_all(install_root); - } - - #[test] - fn parses_volume_specs() { - let root = "a".repeat(64); - let docker = parse_volume(&format!("images.img:{root}:docker")).unwrap(); - assert_eq!(docker.volume.source, "images.img"); - assert!(docker.volume.read_only); - assert_eq!(docker.verity_root, root); - assert_eq!(docker.target, "docker"); - - let data = parse_volume(&format!("weights.img:{root}:/models/llama")).unwrap(); - assert_eq!(data.target, "/models/llama"); - - assert!(parse_volume("weights.img").is_err()); // missing verity_root:target - assert!(parse_volume(&format!("weights.img:{root}")).is_err()); // missing target - assert!(parse_volume(&format!("../escape.img:{root}:docker")).is_err()); // path separator - assert!(parse_volume("x.img:nothex:docker").is_err()); // verity_root not hex - assert!(parse_volume(&format!("x.img:{root}:relative/path")).is_err()); // bad target - } - - #[test] - fn parses_phala_style_deploy_flags() { - let cli = Cli::parse_from([ - "dstack", - "deploy", - "-n", - "hello", - "-c", - "examples/hello-nginx/docker-compose.yaml", - "--port", - "8080:80", - ]); - match cli.command { - Command::Deploy { - compose, - compose_file, - name, - memory, - ports, - .. - } => { - assert_eq!(compose, None); - assert_eq!( - compose_file.as_deref(), - Some("examples/hello-nginx/docker-compose.yaml") - ); - assert_eq!(name, "hello"); - assert_eq!(memory, 2048); - assert_eq!(ports, vec!["8080:80"]); - } - _ => panic!("expected deploy command"), - } - } -} - #[allow(clippy::too_many_arguments)] async fn cmd_deploy( host: &str, @@ -563,7 +470,7 @@ async fn cmd_deploy( dry_run: bool, json: bool, ) -> Result<()> { - let yaml = std::fs::read_to_string(compose_path) + let yaml = fs::read_to_string(compose_path) .with_context(|| format!("reading compose file '{compose_path}'"))?; let port_maps = port_specs @@ -763,3 +670,139 @@ fn trunc(s: &str, n: usize) -> String { out } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_local_install_defaults() { + let value = serde_json::json!({ + "client_url": "http://127.0.0.1:19080", + "image": "dstack-0.5.11", + "allowlist_path": "/tmp/dstack/etc/dstack/auth-allowlist.json" + }); + let defaults = LocalDefaults::from_value(&value); + assert_eq!( + defaults.client_url.as_deref(), + Some("http://127.0.0.1:19080") + ); + assert_eq!(defaults.image.as_deref(), Some("dstack-0.5.11")); + assert_eq!( + defaults.allowlist_path().as_deref(), + Some("/tmp/dstack/etc/dstack/auth-allowlist.json") + ); + } + + #[test] + fn reads_local_install_defaults_from_prefix() { + let install_root = + std::env::temp_dir().join(format!("dstack-cli-state-test-{}", std::process::id())); + let state_dir = install_root.join("var/lib/dstack"); + fs::create_dir_all(&state_dir).unwrap(); + fs::write( + state_dir.join(dstack_cli_core::layout::STATE_FILE), + r#"{ + "client_url": "http://127.0.0.1:29080", + "image": "dstack-0.5.12", + "allowlist_path": "/tmp/custom-dstack/etc/dstack/auth-allowlist.json" + }"#, + ) + .unwrap(); + + let prefix = dstack_cli_core::layout::path_string(&install_root); + let defaults = LocalDefaults::read(Some(&prefix)).unwrap(); + assert_eq!( + defaults.client_url.as_deref(), + Some("http://127.0.0.1:29080") + ); + assert_eq!(defaults.image.as_deref(), Some("dstack-0.5.12")); + assert_eq!( + defaults.allowlist_path().as_deref(), + Some("/tmp/custom-dstack/etc/dstack/auth-allowlist.json") + ); + + let _ = fs::remove_dir_all(install_root); + } + + #[test] + fn parses_volume_specs() { + let root = "a".repeat(64); + let docker = parse_volume(&format!("images.img:{root}:docker")).unwrap(); + assert_eq!(docker.volume.source, "images.img"); + assert!(docker.volume.read_only); + assert_eq!(docker.verity_root, root); + assert_eq!(docker.target, "docker"); + + let data = parse_volume(&format!("weights.img:{root}:/models/llama")).unwrap(); + assert_eq!(data.target, "/models/llama"); + + assert!(parse_volume("weights.img").is_err()); // missing verity_root:target + assert!(parse_volume(&format!("weights.img:{root}")).is_err()); // missing target + assert!(parse_volume(&format!("../escape.img:{root}:docker")).is_err()); // path separator + assert!(parse_volume("x.img:nothex:docker").is_err()); // verity_root not hex + assert!(parse_volume(&format!("x.img:{root}:relative/path")).is_err()); // bad target + } + + #[test] + fn parses_phala_style_deploy_flags() { + let cli = Cli::parse_from([ + "dstack", + "deploy", + "-n", + "hello", + "-c", + "examples/hello-nginx/docker-compose.yaml", + "--port", + "8080:80", + ]); + match cli.command { + Command::Deploy { + compose, + compose_file, + name, + memory, + ports, + .. + } => { + assert_eq!(compose, None); + assert_eq!( + compose_file.as_deref(), + Some("examples/hello-nginx/docker-compose.yaml") + ); + assert_eq!(name, "hello"); + assert_eq!(memory, 2048); + assert_eq!(ports, vec!["8080:80"]); + } + _ => panic!("expected deploy command"), + } + } + + #[test] + fn parses_verity_fs_image_flag() { + let cli = Cli::parse_from(["dstack", "verity", "--fs-image", "rootfs.ext4"]); + match cli.command { + Command::Verity { + images, + dir, + fs_image, + .. + } => { + assert!(images.is_empty()); + assert_eq!(dir, None); + assert_eq!(fs_image.as_deref(), Some("rootfs.ext4")); + } + _ => panic!("expected verity command"), + } + + assert!(Cli::try_parse_from([ + "dstack", + "verity", + "--dir", + "data", + "--fs-image", + "rootfs.ext4" + ]) + .is_err()); + } +} diff --git a/dstack/crates/dstack-verity/Cargo.toml b/dstack/crates/dstack-verity/Cargo.toml index 0347ccccc..2581e8974 100644 --- a/dstack/crates/dstack-verity/Cargo.toml +++ b/dstack/crates/dstack-verity/Cargo.toml @@ -17,10 +17,13 @@ sha2 = { workspace = true, features = ["std"] } hex = { workspace = true, features = ["std"] } flate2.workspace = true futures.workspace = true +fs-err.workspace = true +gpt = "4.1.0" tar.workspace = true tempfile.workspace = true tracing.workspace = true rustix = { version = "0.38", features = ["fs"] } +uuid.workspace = true xattr = "1.5" oci-client = { version = "0.17.0", default-features = false, features = ["rustls-tls"] } diff --git a/dstack/crates/dstack-verity/src/lib.rs b/dstack/crates/dstack-verity/src/lib.rs index b01756c6b..31a3f3a16 100644 --- a/dstack/crates/dstack-verity/src/lib.rs +++ b/dstack/crates/dstack-verity/src/lib.rs @@ -4,9 +4,10 @@ //! Build a verity volume from docker images or a directory. //! -//! The output is a reproducible, dm-verity-protected squashfs volume that a CVM -//! mounts instead of pulling and unpacking. A `docker` volume seeds the overlay2 -//! store; a data volume just mounts at a path. +//! The output is a reproducible, dm-verity-protected raw disk image: partition 1 +//! is the squashfs data filesystem, and partition 2 is the dm-verity superblock +//! plus hash tree. A `docker` volume seeds the overlay2 store; a data volume just +//! mounts at a path. //! //! The build needs no docker daemon and no TEE, and it's reproducible: the same //! inputs always give the same `verity_root`. So anyone can recompute the root @@ -16,6 +17,7 @@ use std::path::PathBuf; use anyhow::{bail, Context, Result}; +use fs_err as fs; pub mod oci; mod store; @@ -37,6 +39,10 @@ pub struct VerityOptions { /// build a data volume from this directory instead of docker images. The /// resulting volume is mounted at a `target: "/path"` in the compose. pub dir: Option, + /// wrap an existing filesystem image as the verity data partition. This is + /// for hand-built ext4/xfs/etc. images; `dstack verity --dir` still produces + /// squashfs by default. + pub fs_image: Option, pub output: PathBuf, /// squashfs compression (default: none — zero decompression at read time). pub compress: Compression, @@ -62,13 +68,17 @@ pub struct ResolvedImage { } pub async fn verity(opts: VerityOptions) -> Result { - match (&opts.dir, opts.images.is_empty()) { - (Some(_), false) => bail!("give either images or --dir, not both"), - (None, true) => { - bail!("nothing to build: pass an image, or --dir to pack a directory") - } - (Some(dir), true) => verity_dir(dir.clone(), opts.output, opts.compress).await, - (None, false) => verity_docker(opts).await, + let source_count = + (!opts.images.is_empty()) as u8 + opts.dir.is_some() as u8 + opts.fs_image.is_some() as u8; + if source_count != 1 { + bail!("give exactly one source: images, --dir , or --fs-image "); + } + if let Some(dir) = &opts.dir { + verity_dir(dir.clone(), opts.output, opts.compress).await + } else if let Some(fs_image) = &opts.fs_image { + verity_fs_image(fs_image.clone(), opts.output).await + } else { + verity_docker(opts).await } } @@ -92,7 +102,7 @@ async fn verity_docker(opts: VerityOptions) -> Result { let (built, resolved) = tokio::task::spawn_blocking(move || -> Result<_> { let tmp = tempfile::tempdir().context("creating scratch dir")?; let store_dir = tmp.path().join("store"); - std::fs::create_dir_all(&store_dir)?; + fs::create_dir_all(&store_dir)?; let tops = store::build_store(&pulled, &store_dir)?; let built = volume::build_volume(&store_dir, &output, VERITY_SALT, compress)?; let resolved = pulled @@ -138,3 +148,23 @@ async fn verity_dir(dir: PathBuf, output: PathBuf, compress: Compression) -> Res images: vec![], }) } + +/// Wrap an already-built filesystem image. The guest discovers and mounts the +/// filesystem only after dm-verity is active. +async fn verity_fs_image(fs_image: PathBuf, output: PathBuf) -> Result { + if !fs_image.is_file() { + bail!("--fs-image '{}' is not a file", fs_image.display()); + } + let out = output.clone(); + let built = + tokio::task::spawn_blocking(move || volume::build_fs_image(&fs_image, &out, VERITY_SALT)) + .await + .context("the build task failed")??; + + Ok(VerityResult { + verity_root: built.verity_root, + data_size: built.data_size, + output, + images: vec![], + }) +} diff --git a/dstack/crates/dstack-verity/src/store.rs b/dstack/crates/dstack-verity/src/store.rs index d09484441..88e8dfa8e 100644 --- a/dstack/crates/dstack-verity/src/store.rs +++ b/dstack/crates/dstack-verity/src/store.rs @@ -23,6 +23,7 @@ use std::path::{Path, PathBuf}; use anyhow::{bail, Context, Result}; use flate2::read::GzDecoder; +use fs_err as fs; use serde_json::json; use sha2::{Digest, Sha256}; @@ -33,9 +34,9 @@ use crate::oci::{PulledImage, PulledLayer}; /// Returns each image's top chain-id, in the same order as `images`. That /// chain-id is what the verity volume ultimately vouches for. pub fn build_store(images: &[PulledImage], root: &Path) -> Result> { - std::fs::create_dir_all(root.join("overlay2/l"))?; - std::fs::create_dir_all(root.join("image/overlay2/layerdb/sha256"))?; - std::fs::create_dir_all(root.join("image/overlay2/imagedb/content/sha256"))?; + fs::create_dir_all(root.join("overlay2/l"))?; + fs::create_dir_all(root.join("image/overlay2/layerdb/sha256"))?; + fs::create_dir_all(root.join("image/overlay2/imagedb/content/sha256"))?; let mut built: HashSet = HashSet::new(); // repo_name -> { "repo:tag" -> "sha256:imageid", "repo@digest" -> id } @@ -117,7 +118,7 @@ fn write_layer( ) -> Result<()> { let layer_dir = root.join("overlay2").join(chain); let diff_dir = layer_dir.join("diff"); - std::fs::create_dir_all(&diff_dir)?; + fs::create_dir_all(&diff_dir)?; let (layer_size, actual_diff) = extract_layer(&layer.data, &layer.media_type, &diff_dir) .with_context(|| format!("extracting layer {layer_index} of {reference}"))?; @@ -143,7 +144,7 @@ fn write_layer( } let layerdb = root.join("image/overlay2/layerdb/sha256").join(chain); - std::fs::create_dir_all(&layerdb)?; + fs::create_dir_all(&layerdb)?; write(layerdb.join("diff"), diff_id)?; write(layerdb.join("cache-id"), chain)?; write(layerdb.join("size"), &layer_size.to_string())?; @@ -173,7 +174,7 @@ fn short_link(chain_hex: &str) -> String { /// the parent's own `lower` and prepending the parent. fn lower_chain(root: &Path, parent_short: &str) -> Result { // resolve the parent's chain dir from its l/ symlink to read its `lower`. - let parent_dir = std::fs::read_link(root.join("overlay2/l").join(parent_short))?; + let parent_dir = fs::read_link(root.join("overlay2/l").join(parent_short))?; // ..//diff -> let chain = parent_dir .parent() @@ -183,7 +184,7 @@ fn lower_chain(root: &Path, parent_short: &str) -> Result { .to_string(); let plower = root.join("overlay2").join(&chain).join("lower"); let mut parts = vec![format!("l/{parent_short}")]; - if let Ok(existing) = std::fs::read_to_string(&plower) { + if let Ok(existing) = fs::read_to_string(&plower) { parts.extend(existing.split(':').map(str::to_string)); } Ok(parts.join(":")) @@ -232,7 +233,7 @@ fn extract_layer(data: &[u8], media_type: &str, dest: &Path) -> Result<(u64, Str None => continue, }; if name == ".wh..wh..opq" { - std::fs::create_dir_all(&out_parent)?; + fs::create_dir_all(&out_parent)?; xattr::set(&out_parent, "trusted.overlay.opaque", b"y") .context("setting overlay opaque xattr (need root)")?; } else if rest.starts_with(".wh.") { @@ -242,7 +243,7 @@ fn extract_layer(data: &[u8], media_type: &str, dest: &Path) -> Result<(u64, Str // a name like `.wh.`, `.wh..`, or `.wh...` would target the parent // dir itself or escape it; ignore. } else { - std::fs::create_dir_all(&out_parent)?; + fs::create_dir_all(&out_parent)?; let target = out_parent.join(rest); let _ = remove_any(&target); mknod_whiteout(&target).context("creating overlay whiteout (need root)")?; @@ -302,7 +303,7 @@ fn safe_subpath(dest: &Path, rel: &Path) -> Option { Component::CurDir => {} Component::Normal(c) => { cur.push(c); - if let Ok(md) = std::fs::symlink_metadata(&cur) { + if let Ok(md) = fs::symlink_metadata(&cur) { if md.file_type().is_symlink() { return None; } @@ -323,9 +324,9 @@ fn mknod_whiteout(path: &Path) -> Result<()> { } fn remove_any(path: &Path) -> std::io::Result<()> { - match std::fs::symlink_metadata(path) { - Ok(m) if m.is_dir() => std::fs::remove_dir_all(path), - Ok(_) => std::fs::remove_file(path), + match fs::symlink_metadata(path) { + Ok(m) if m.is_dir() => fs::remove_dir_all(path), + Ok(_) => fs::remove_file(path), Err(e) => Err(e), } } @@ -347,11 +348,11 @@ fn symlink(target: impl AsRef, link: impl AsRef) -> std::io::Result< } fn write(path: PathBuf, contents: &str) -> std::io::Result<()> { - std::fs::write(path, contents.as_bytes()) + fs::write(path, contents.as_bytes()) } fn write_bytes(path: PathBuf, contents: &[u8]) -> std::io::Result<()> { - std::fs::write(path, contents) + fs::write(path, contents) } #[cfg(test)] @@ -418,10 +419,10 @@ mod tests { use std::os::unix::fs::MetadataExt; let dest = tempfile::tempdir().unwrap(); let victim = tempfile::tempdir().unwrap(); - std::fs::write(victim.path().join("keep"), b"x").unwrap(); + fs::write(victim.path().join("keep"), b"x").unwrap(); // stamp the tar entries with our own uid/gid so unpacking the symlink // doesn't need root to chown. - let md = std::fs::metadata(victim.path()).unwrap(); + let md = fs::metadata(victim.path()).unwrap(); let (uid, gid) = (md.uid() as u64, md.gid() as u64); let mut b = tar::Builder::new(Vec::new()); diff --git a/dstack/crates/dstack-verity/src/volume.rs b/dstack/crates/dstack-verity/src/volume.rs index 79dc13fb9..dd2a2f4e2 100644 --- a/dstack/crates/dstack-verity/src/volume.rs +++ b/dstack/crates/dstack-verity/src/volume.rs @@ -2,25 +2,39 @@ // // SPDX-License-Identifier: Apache-2.0 -//! Pack a directory into a reproducible, verity-protected volume. +//! Pack a directory into a reproducible, verity-protected disk image. //! -//! The output is one file: the squashfs image, then its dm-verity hash tree. -//! squashfs is used because the guest kernel already mounts it (it's the guest's -//! own rootfs), and because `mksquashfs` with a pinned timestamp is reproducible -//! byte for byte. The verity root is a pure function of the squashfs bytes and -//! the salt. +//! The output is a raw disk with two deterministic GPT partitions: +//! +//! 1. the filesystem image +//! 2. the dm-verity superblock and hash tree +//! +//! Keeping the verity data and hash devices in separate partitions means the +//! guest never has to inspect filesystem metadata to find the hash tree. The +//! partition table is only a locator hint; the measured verity root is still the +//! content identity. +use std::collections::BTreeMap; use std::path::Path; use std::process::Command; use anyhow::{bail, Context, Result}; +use fs_err as fs; use sha2::{Digest, Sha256}; +use uuid::Uuid; /// Fixed build timestamp (2024-01-01), so the image is the same no matter when /// it's built. It never shows up at runtime: `mksquashfs -all-time` normalizes /// the store's own timestamps away. const EPOCH: &str = "1704067200"; const BLOCK: u64 = 4096; +const SECTOR: u64 = 512; +// 1 MiB alignment. +const PARTITION_ALIGNMENT_SECTORS: u64 = 2048; +// The `gpt` crate writes the primary/backup headers and partition arrays. We +// still reserve enough trailing sectors in the raw image for the backup array +// (128 entries * 128 bytes) plus the backup header. +const GPT_ENTRY_SECTORS: u64 = 32; #[derive(Clone, Copy)] pub enum Compression { @@ -44,11 +58,12 @@ impl Compression { pub struct BuiltVolume { pub verity_root: String, - /// size of the squashfs region = the verity hash offset. + /// Size of the filesystem data partition. pub data_size: u64, } -/// Build `output`: the squashfs image of `store`, then its dm-verity hash tree. +/// Build `output`: a GPT disk image with the squashfs data partition and a +/// separate dm-verity hash partition. /// /// `store` is any directory — a docker overlay2 store, or plain data. `salt` /// fixes the verity root. @@ -64,13 +79,13 @@ pub fn build_volume( ) -> Result { require_tool("mksquashfs")?; require_tool("veritysetup")?; - if output.exists() { - std::fs::remove_file(output).ok(); - } + + let tmp = tempfile::tempdir().context("creating volume scratch dir")?; + let data_path = tmp.path().join("data.fs"); // 1. reproducible squashfs. let mut cmd = Command::new("mksquashfs"); - cmd.arg(store).arg(output); + cmd.arg(store).arg(&data_path); cmd.args(compress.args()); cmd.args([ "-all-time", @@ -84,18 +99,51 @@ pub fn build_volume( run(cmd, "mksquashfs")?; // 2. the verity data region must be block-aligned; pad the squashfs up. - let bytes_used = squashfs_bytes_used(output)?; - let data_size = bytes_used.div_ceil(BLOCK) * BLOCK; + let bytes_used = squashfs_bytes_used(&data_path)?; + seal_data_image(&data_path, bytes_used, output, salt_hex) +} + +/// Wrap an existing filesystem image in the same partitioned verity disk format. +/// +/// The input image is copied to a scratch file and padded to a 4096-byte verity +/// block boundary. Its filesystem type is otherwise opaque to the verity layer. +pub fn build_fs_image(fs_image: &Path, output: &Path, salt_hex: &str) -> Result { + require_tool("veritysetup")?; + let len = fs::metadata(fs_image) + .with_context(|| format!("stat {}", fs_image.display()))? + .len(); + if len == 0 { + bail!("filesystem image '{}' is empty", fs_image.display()); + } + + let tmp = tempfile::tempdir().context("creating volume scratch dir")?; + let data_path = tmp.path().join("data.fs"); + fs::copy(fs_image, &data_path) + .with_context(|| format!("copying {} into scratch", fs_image.display()))?; + seal_data_image(&data_path, len, output, salt_hex) +} + +fn seal_data_image( + data_path: &Path, + data_len: u64, + output: &Path, + salt_hex: &str, +) -> Result { + let hash_tmp = tempfile::tempdir().context("creating verity hash scratch dir")?; + let hash_path = hash_tmp.path().join("verity.hash"); + let data_size = data_len.div_ceil(BLOCK) * BLOCK; { - let f = std::fs::OpenOptions::new().write(true).open(output)?; + let f = fs::OpenOptions::new().write(true).open(data_path)?; f.set_len(data_size)?; } - // 3. Append the verity hash tree to the same file, at the aligned offset. + // Build the verity hash device as a separate image. At runtime this is + // partition 2, so no --hash-offset is needed and the guest does not parse + // filesystem metadata before verity is active. // Pin the superblock UUID too: veritysetup randomizes it otherwise, and we - // want the whole file reproducible, not just the root. The UUID sits in the + // want the whole image reproducible, not just the root. The UUID sits in the // hash tree, not the hashed data, so it never changes the root. - let uuid = uuid_from_data(output, data_size)?; + let uuid = uuid_from_data(data_path, data_size)?; let mut cmd = Command::new("veritysetup"); cmd.args([ "format", @@ -107,14 +155,16 @@ pub fn build_volume( "4096", "--hash-block-size", "4096", - "--hash-offset", - &data_size.to_string(), ]); - cmd.arg(output).arg(output); + cmd.arg(data_path).arg(&hash_path); let out = capture(cmd, "veritysetup format")?; let verity_root = parse_root_hash(&out).context("could not find the root hash in veritysetup output")?; + // Wrap the two blobs in a deterministic GPT disk image: + // p1 = data filesystem, p2 = verity superblock + hash tree. + build_partitioned_image(data_path, data_size, &hash_path, output, &verity_root)?; + Ok(BuiltVolume { verity_root, data_size, @@ -124,7 +174,7 @@ pub fn build_volume( /// squashfs superblock: `bytes_used` is a little-endian u64 at offset 40. fn squashfs_bytes_used(path: &Path) -> Result { use std::io::{Read, Seek, SeekFrom}; - let mut f = std::fs::File::open(path)?; + let mut f = fs::File::open(path)?; let mut magic = [0u8; 4]; f.read_exact(&mut magic)?; if &magic != b"hsqs" { @@ -137,11 +187,24 @@ fn squashfs_bytes_used(path: &Path) -> Result { } /// Derive a UUID from the first `len` bytes of `path`. -/// -/// Deterministic, and shaped like a version-4 UUID. fn uuid_from_data(path: &Path, len: u64) -> Result { + Ok(uuid_from_file(path, len)?.to_string()) +} + +/// Deterministic, version-4-shaped UUID from arbitrary domain-separated +/// material. +fn uuid_from_material(parts: &[&[u8]]) -> Uuid { + let mut h = Sha256::new(); + for part in parts { + h.update((part.len() as u64).to_le_bytes()); + h.update(part); + } + uuid_from_digest(&h.finalize()) +} + +fn uuid_from_file(path: &Path, len: u64) -> Result { use std::io::Read; - let mut f = std::fs::File::open(path)?; + let mut f = fs::File::open(path)?; let mut h = Sha256::new(); let mut remaining = len; let mut buf = vec![0u8; 1 << 20]; @@ -151,20 +214,149 @@ fn uuid_from_data(path: &Path, len: u64) -> Result { h.update(&buf[..n]); remaining -= n as u64; } - let d = h.finalize(); + Ok(uuid_from_digest(&h.finalize())) +} + +fn uuid_from_digest(d: &[u8]) -> Uuid { let mut u = [0u8; 16]; u.copy_from_slice(&d[..16]); u[6] = (u[6] & 0x0f) | 0x40; // version 4 u[8] = (u[8] & 0x3f) | 0x80; // RFC 4122 variant - let hx = hex::encode(u); - Ok(format!( - "{}-{}-{}-{}-{}", - &hx[0..8], - &hx[8..12], - &hx[12..16], - &hx[16..20], - &hx[20..32] - )) + Uuid::from_bytes(u) +} + +#[derive(Clone, Copy)] +struct Partition { + first_lba: u64, + last_lba: u64, +} + +struct GptLayout { + total_lbas: u64, + data: Partition, + hash: Partition, +} + +impl GptLayout { + fn new(data_size: u64, hash_size: u64) -> Result { + if data_size == 0 || hash_size == 0 { + bail!("data and hash images must be non-empty"); + } + let data_sectors = data_size.div_ceil(SECTOR); + let hash_sectors = hash_size.div_ceil(SECTOR); + + let data_first = PARTITION_ALIGNMENT_SECTORS; + let data_last = data_first + data_sectors - 1; + let hash_first = align_up(data_last + 1, PARTITION_ALIGNMENT_SECTORS); + let hash_last = hash_first + hash_sectors - 1; + + // Leave room for the backup GPT entry array and header at the end. + let min_lbas = hash_last + 1 + GPT_ENTRY_SECTORS + 1; + let total_lbas = align_up(min_lbas, PARTITION_ALIGNMENT_SECTORS); + Ok(Self { + total_lbas, + data: Partition { + first_lba: data_first, + last_lba: data_last, + }, + hash: Partition { + first_lba: hash_first, + last_lba: hash_last, + }, + }) + } + + fn total_bytes(&self) -> u64 { + self.total_lbas * SECTOR + } +} + +fn align_up(value: u64, alignment: u64) -> u64 { + value.div_ceil(alignment) * alignment +} + +fn build_partitioned_image( + data_path: &Path, + data_size: u64, + hash_path: &Path, + output: &Path, + root_hash: &str, +) -> Result<()> { + let hash_size = fs::metadata(hash_path) + .with_context(|| format!("stat {}", hash_path.display()))? + .len(); + let layout = GptLayout::new(data_size, hash_size)?; + + let mut out = fs::OpenOptions::new() + .create(true) + .truncate(true) + .read(true) + .write(true) + .open(output) + .with_context(|| format!("creating {}", output.display()))?; + out.set_len(layout.total_bytes())?; + + out = write_gpt(out, &layout, root_hash)?; + + copy_into(data_path, &mut out, layout.data.first_lba * SECTOR)?; + copy_into(hash_path, &mut out, layout.hash.first_lba * SECTOR)?; + Ok(()) +} + +fn write_gpt(mut out: fs::File, layout: &GptLayout, root_hash: &str) -> Result { + let protective_size = (layout.total_lbas - 1).min(u32::MAX as u64) as u32; + gpt::mbr::ProtectiveMBR::with_lb_size(protective_size) + .overwrite_lba0(&mut out) + .context("writing protective MBR")?; + + let disk_uuid = uuid_from_material(&[b"dstack-verity-disk", root_hash.as_bytes()]); + let mut disk = gpt::GptConfig::new() + .writable(true) + .logical_block_size(gpt::disk::LogicalBlockSize::Lb512) + .create_from_device(out, Some(disk_uuid)) + .context("initializing GPT")?; + + let mut parts = BTreeMap::new(); + parts.insert( + 1, + gpt::partition::Partition { + part_type_guid: gpt::partition_types::LINUX_FS, + part_guid: uuid_from_material(&[b"dstack-verity-data", root_hash.as_bytes()]), + first_lba: layout.data.first_lba, + last_lba: layout.data.last_lba, + flags: 0, + name: "dstack-data".to_string(), + }, + ); + parts.insert( + 2, + gpt::partition::Partition { + part_type_guid: gpt::partition_types::LINUX_FS, + part_guid: uuid_from_material(&[b"dstack-verity-hash", root_hash.as_bytes()]), + first_lba: layout.hash.first_lba, + last_lba: layout.hash.last_lba, + flags: 0, + name: "dstack-verity".to_string(), + }, + ); + disk.update_partitions(parts) + .context("installing GPT partitions")?; + disk.write().context("writing GPT") +} + +fn copy_into(src: &Path, out: &mut fs::File, offset: u64) -> Result<()> { + use std::io::{Read, Seek, SeekFrom, Write}; + let mut input = fs::File::open(src)?; + out.seek(SeekFrom::Start(offset))?; + let mut buf = vec![0u8; 1 << 20]; + loop { + let n = input.read(&mut buf)?; + if n == 0 { + break; + } + out.write_all(&buf[..n])?; + } + Ok(()) } fn parse_root_hash(output: &str) -> Option { @@ -235,4 +427,72 @@ mod tests { assert_eq!(ua.len(), 36); assert_eq!(&ua[14..15], "4"); // version nibble } + + #[test] + fn gpt_layout_uses_two_aligned_partitions() { + let layout = GptLayout::new(4096, 8192).unwrap(); + assert_eq!(layout.data.first_lba, PARTITION_ALIGNMENT_SECTORS); + assert_eq!(layout.data.last_lba, PARTITION_ALIGNMENT_SECTORS + 7); + assert_eq!(layout.hash.first_lba, PARTITION_ALIGNMENT_SECTORS * 2); + assert_eq!(layout.hash.last_lba, PARTITION_ALIGNMENT_SECTORS * 2 + 15); + assert!(layout.total_lbas - 1 - GPT_ENTRY_SECTORS > layout.hash.last_lba); + } + + #[test] + fn partitioned_image_is_valid_gpt() -> Result<()> { + use std::io::{Read, Seek, SeekFrom, Write}; + + let tmp = tempfile::tempdir()?; + let data_path = tmp.path().join("data.fs"); + let hash_path = tmp.path().join("verity.hash"); + let image_path = tmp.path().join("volume.img"); + let data = vec![0x11; 4096]; + let hash = vec![0x22; 8192]; + fs::File::create(&data_path)?.write_all(&data)?; + fs::File::create(&hash_path)?.write_all(&hash)?; + + let root_hash = "abc123"; + build_partitioned_image( + &data_path, + data.len() as u64, + &hash_path, + &image_path, + root_hash, + )?; + + let disk = gpt::GptConfig::new() + .logical_block_size(gpt::disk::LogicalBlockSize::Lb512) + .open(&image_path)?; + assert_eq!( + *disk.guid(), + uuid_from_material(&[b"dstack-verity-disk", root_hash.as_bytes()]) + ); + + let p1 = disk.partitions().get(&1).unwrap(); + let p2 = disk.partitions().get(&2).unwrap(); + assert_eq!(p1.name, "dstack-data"); + assert_eq!(p2.name, "dstack-verity"); + assert_eq!( + p1.part_guid, + uuid_from_material(&[b"dstack-verity-data", root_hash.as_bytes()]) + ); + assert_eq!( + p2.part_guid, + uuid_from_material(&[b"dstack-verity-hash", root_hash.as_bytes()]) + ); + assert_eq!(p1.first_lba, PARTITION_ALIGNMENT_SECTORS); + assert_eq!(p2.first_lba, PARTITION_ALIGNMENT_SECTORS * 2); + + let mut img = fs::File::open(&image_path)?; + let mut buf = vec![0; data.len()]; + img.seek(SeekFrom::Start(p1.first_lba * SECTOR))?; + img.read_exact(&mut buf)?; + assert_eq!(buf, data); + let mut buf = vec![0; hash.len()]; + img.seek(SeekFrom::Start(p2.first_lba * SECTOR))?; + img.read_exact(&mut buf)?; + assert_eq!(buf, hash); + + Ok(()) + } } diff --git a/dstack/docs/verity-volumes.md b/dstack/docs/verity-volumes.md index fb1dd0b28..6d37344e9 100644 --- a/dstack/docs/verity-volumes.md +++ b/dstack/docs/verity-volumes.md @@ -13,6 +13,15 @@ A volume has a **root hash** — its identity and integrity check — and a **ta - `"docker"` — the volume is a docker overlay2 store; its images are seeded into docker as cache hits. - `"/some/path"` — the volume's filesystem is mounted there, for data like model weights. +On disk, a volume is a raw GPT image with two partitions: + +``` +p1: filesystem data +p2: dm-verity superblock + hash tree +``` + +The guest opens `p1` as the verity data device and `p2` as the verity hash device. That keeps the verity layer independent of the filesystem format: the guest does not need to read a squashfs/ext4 superblock to find a hash offset. + An app lists its volumes in `app-compose.json`. dstack already hashes that file into the app's identity, so every root hash is measured as part of `app_id`, and the CVM only uses content matching what it was attested as. Everything else is untrusted: the host hands over bytes, and dm-verity rejects any that don't match the root. A volume is *additive* — it only adds read-only lower layers, so anything not on one is pulled normally — and *fail-safe*: a missing or mismatched volume falls back to a pull. @@ -46,9 +55,10 @@ You don't hand-write that with the `dstack` CLI — `dstack deploy --volume` gen ```bash dstack verity vllm/vllm-openai@sha256:abc123... # -> --volume vllm.img:115b6877...:docker dstack verity --dir ./llama-70b-weights/ # -> --volume llama-70b.img:a1b2c3d4...:/run/models/llama-70b +dstack verity --fs-image ./weights.ext4 # -> wrap an existing filesystem image ``` -It needs no docker daemon, no CVM, and no TDX. For images it pulls the pinned layers from the registry and lays out the docker store itself; for `--dir` it packs the directory as-is. Either way it writes one volume file. Pass several images to pack them into a single `docker` volume; shared base layers are stored once. There is no auto-detection: a volume contains whatever you built it from, and the compose names those images normally. +It needs no docker daemon, no CVM, and no TDX. For images it pulls the pinned layers from the registry and lays out the docker store itself; for `--dir` it packs the directory as squashfs; for `--fs-image` it treats the supplied filesystem image as opaque bytes and only wraps it in verity/GPT. Either way it writes one volume file. Pass several images to pack them into a single `docker` volume; shared base layers are stored once. There is no auto-detection: a volume contains whatever you built it from, and the compose names those images normally. This covers both cases. Several volumes in one CVM (the image *and* the model) is several entries with different targets. Several images from one volume is a single `docker` volume built from all of them, listed as one entry, with the compose unchanged. Attach order does not matter ([Delivery](#delivery)). @@ -56,7 +66,7 @@ This covers both cases. Several volumes in one CVM (the image *and* the model) i `verity_root` lives in `app-compose.json`, so it's part of the hash that becomes `app_id` (see [Normalized App Compose](./normalized-app-compose.md)), and it is enforced by dm-verity. The consumer trusts the root hash, not whoever built the volume. A host that tampers with the bytes causes a verity fault, not a silent swap, so the delivery path can stay untrusted. -`dstack verity` is deterministic: the same pinned image digests always produce the same `verity_root`, bit-for-bit (canonical overlay2 layout with chain-id cache-ids; a fixed timestamp and salt; a UUID derived from the packed bytes). One caveat: the squashfs layout is produced by `mksquashfs`, so a verifier must use a `squashfs-tools` version that lays out bytes identically (recent versions are stable; the build records nothing about the tool version yet). A verifier can therefore recompute the root from the digest-pinned images and confirm the volume is those images, without trusting whoever ran the build. The build needs no docker daemon and no TEE, so it can also run in a CI job that attests it (see [Reproducible builds and provenance](#reproducible-builds-and-provenance)); the two checks are independent. Turning on a volume changes `app-compose.json` and thus `app_id`, which is correct: the volume is part of the measured configuration. +`dstack verity` is deterministic: the same pinned image digests always produce the same `verity_root`, bit-for-bit (canonical overlay2 layout with chain-id cache-ids; a fixed timestamp and salt; UUIDs derived from the packed bytes/root hash; a deterministic GPT wrapper). One caveat: the squashfs layout is produced by `mksquashfs`, so a verifier must use a `squashfs-tools` version that lays out bytes identically (recent versions are stable; the build records nothing about the tool version yet). A verifier can therefore recompute the root from the digest-pinned images and confirm the volume is those images, without trusting whoever ran the build. The build needs no docker daemon and no TEE, so it can also run in a CI job that attests it (see [Reproducible builds and provenance](#reproducible-builds-and-provenance)); the two checks are independent. Turning on a volume changes `app-compose.json` and thus `app_id`, which is correct: the volume is part of the measured configuration. ## How docker seeding works @@ -69,7 +79,7 @@ overlay2//diff/ the already-extracted layer files (the GBs) `diff/` is the layer already decompressed and untarred. Seeding reuses it in place: -1. Open the volume with veritysetup and mount it read-only at a writable path (`/run/dstack-verity`). The rootfs is read-only dm-verity, so the mountpoint has to be on a writable fs; squashfs itself mounts read-only directly, no journal and nothing to replay. +1. Open the volume with veritysetup (`p1` as data, `p2` as hash) and mount it read-only at a writable path (`/run/dstack-verity`). The rootfs is read-only dm-verity, so the mountpoint has to be on a writable fs; squashfs itself mounts read-only directly, no journal and nothing to replay. 2. Copy the metadata (a few MB) into `/var/lib/docker`. 3. Per layer: make a writable `overlay2//` dir, copy its handful of tiny files, and bind-mount only the big `diff/` read-only from the volume. 4. This runs from `dstack-prepare.sh` *before* dockerd starts, so there's no restart — dockerd comes up with every image on the volume already present. @@ -87,14 +97,14 @@ The volume *file* and the deploy request travel separately. First set `cvm.volum ```bash dstack deploy -c docker-compose.yaml \ --volume vllm.img:115b6877...:docker \ - --volume llama-70b.img:a1b2c3d4...:/models/llama-70b + --volume llama-70b.img:a1b2c3d4...:/run/models/llama-70b ``` -Each `--volume NAME:VERITY_ROOT:TARGET` both attaches the file and writes the measured `verity_volumes` entry (the root is yours to supply, from `dstack verity`, so it stays part of `app_id`). `vmm-cli.py` takes the same file with a hand-authored `app-compose.json` instead. The vmm resolves each name against `volumes_dir` (rejecting anything with a path separator) and attaches it as a read-only virtio-blk device, tagged with a hint: the disk's serial is set to the volume's `verity_root` prefix. `--volume` carries only the name and read-only flag, never the bytes; the bytes are already on the host. Disks are still attached in any order — the guest reads the serials once and, for each `verity_root` in the compose, opens the disk whose serial matches. That is O(devices + volumes), and no wrong disk is opened. The serial is only a hint and isn't trusted: the guest verifies the full root through dm-verity, so a missing or wrong tag falls back to trying every disk, and a tampering host causes a verity fault rather than a wrong mount. Integrity is checked lazily, per block on first read. Read-only plus content-addressed means one physical copy serves every CVM that references it: a base image or model shared by a hundred replicas is built, stored, and extracted once. +Each `--volume NAME:VERITY_ROOT:TARGET` both attaches the file and writes the measured `verity_volumes` entry (the root is yours to supply, from `dstack verity`, so it stays part of `app_id`). `vmm-cli.py` takes the same file with a hand-authored `app-compose.json` instead. The vmm resolves each name against `volumes_dir` (rejecting anything with a path separator) and attaches it as a read-only virtio-blk device, tagged with a hint: the disk's serial is set to the volume's `verity_root` prefix. `--volume` carries only the name and read-only flag, never the bytes; the bytes are already on the host. Disks are still attached in any order — the guest reads the serials once and, for each `verity_root` in the compose, opens partition 1 and partition 2 of the disk whose serial matches. That is O(devices + volumes), and no wrong disk is opened. The serial and GPT partition table are only hints and aren't trusted: the guest verifies the full root through dm-verity, so a missing/wrong tag falls back to trying every disk, and a tampering host causes a verity fault rather than a wrong mount. Integrity is checked lazily, per block on first read. Read-only plus content-addressed means one physical copy serves every CVM that references it: a base image or model shared by a hundred replicas is built, stored, and extracted once. ## Building volumes -`dstack verity` builds the store from scratch, without a docker daemon and without a CVM — just the registry (a `docker` volume does need root, because laying out overlay2 whiteouts uses `mknod` and a `trusted.*` xattr; a `--dir` volume needs no root). It pulls each layer by digest, and lays out the overlay2 store the way docker would, with one deliberate change: the per-layer directory id is the layer's *chain-id* instead of docker's random cache-id. That's what makes the store a pure function of the image. AUFS `.wh.` whiteouts in the layer tars are converted to their overlay2 on-disk form (a `0:0` char device, or the `trusted.overlay.opaque` xattr) so deletions in upper layers still take effect. Then `mksquashfs` packs it with a fixed timestamp and `veritysetup` appends the hash tree in the same file (data in `[0, size)`, hash tree after) with a fixed salt and a UUID derived from the squashfs bytes. A `--dir` volume skips the overlay2 layout and pull entirely — it packs the directory straight into the same squashfs + verity format. +`dstack verity` builds the store from scratch, without a docker daemon and without a CVM — just the registry (a `docker` volume does need root, because laying out overlay2 whiteouts uses `mknod` and a `trusted.*` xattr; a `--dir` or `--fs-image` volume needs no root). It pulls each layer by digest, and lays out the overlay2 store the way docker would, with one deliberate change: the per-layer directory id is the layer's *chain-id* instead of docker's random cache-id. That's what makes the store a pure function of the image. AUFS `.wh.` whiteouts in the layer tars are converted to their overlay2 on-disk form (a `0:0` char device, or the `trusted.overlay.opaque` xattr) so deletions in upper layers still take effect. Then `mksquashfs` packs it with a fixed timestamp, `veritysetup` builds a separate hash image with a fixed salt and a UUID derived from the filesystem bytes, and the builder wraps both blobs in a deterministic GPT disk (`p1` data, `p2` verity metadata/hash tree). A `--dir` volume skips the overlay2 layout and pull entirely — it packs the directory straight into the same partitioned squashfs + verity format. A `--fs-image` volume skips `mksquashfs` too: the supplied ext4/xfs/etc. image becomes `p1` after 4096-byte padding, so the guest only needs kernel support and suitable read-only mount behavior for that filesystem. With those fixed, two runs of the same digests produce the same bytes. The build needs no daemon and no TEE, so it can run in a CI job that also attests it (below). diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index dbac6a725..ea5c471a4 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -1193,7 +1193,7 @@ fn append_boot_separator(path: &std::path::Path) { if !path.exists() { return; } - let Ok(mut file) = std::fs::OpenOptions::new().append(true).open(path) else { + let Ok(mut file) = fs::OpenOptions::new().append(true).open(path) else { return; }; let timestamp = humantime::format_rfc3339_seconds(std::time::SystemTime::now()); @@ -1216,7 +1216,7 @@ fn rotate_serial_log(work_dir: &VmWorkDir, max_bytes: u64) { return; } let history = work_dir.serial_history_file(); - let Ok(mut file) = std::fs::OpenOptions::new() + let Ok(mut file) = fs::OpenOptions::new() .create(true) .append(true) .open(&history) diff --git a/dstack/vmm/src/discovery.rs b/dstack/vmm/src/discovery.rs index 8a2440f39..181e7da52 100644 --- a/dstack/vmm/src/discovery.rs +++ b/dstack/vmm/src/discovery.rs @@ -8,6 +8,7 @@ //! directory so that CLI tools can discover all running instances on the host. use anyhow::{Context, Result}; +use fs_err as fs; use serde::{Deserialize, Serialize}; use std::path::{Path, PathBuf}; use tracing::{info, warn}; @@ -99,7 +100,7 @@ impl DiscoveryRegistration { impl Drop for DiscoveryRegistration { fn drop(&mut self) { - match std::fs::remove_file(&self.path) { + match fs::remove_file(&self.path) { Ok(()) => info!("unregistered VMM instance at {}", self.path.display()), Err(e) => warn!( "failed to remove discovery file {}: {e}", @@ -112,7 +113,7 @@ impl Drop for DiscoveryRegistration { /// Clean up stale discovery files from dead processes. pub fn cleanup_stale_registrations() { let dir = discovery_dir(); - let entries = match std::fs::read_dir(dir) { + let entries = match fs::read_dir(dir) { Ok(e) => e, Err(_) => return, }; @@ -122,7 +123,7 @@ pub fn cleanup_stale_registrations() { if path.extension().and_then(|e| e.to_str()) != Some("json") { continue; } - let content = match std::fs::read_to_string(&path) { + let content = match fs::read_to_string(&path) { Ok(c) => c, Err(_) => continue, }; @@ -138,7 +139,7 @@ pub fn cleanup_stale_registrations() { info.pid, path.display() ); - let _ = std::fs::remove_file(&path); + let _ = fs::remove_file(&path); } } } diff --git a/dstack/vmm/src/main_service.rs b/dstack/vmm/src/main_service.rs index 36185f3ca..ee84ae530 100644 --- a/dstack/vmm/src/main_service.rs +++ b/dstack/vmm/src/main_service.rs @@ -219,8 +219,8 @@ fn resolve_volumes( if dir.is_empty() { bail!("volumes requested but cvm.volumes_dir is not configured"); } - let base = std::fs::canonicalize(dir) - .with_context(|| format!("volumes_dir '{dir}' does not exist"))?; + let base = + fs::canonicalize(dir).with_context(|| format!("volumes_dir '{dir}' does not exist"))?; reqs.iter() .map(|v| { // `,`/`=` would let a crafted file name inject qemu `-drive` options @@ -236,7 +236,7 @@ fn resolve_volumes( v.source ); } - let real = std::fs::canonicalize(base.join(&v.source)).with_context(|| { + let real = fs::canonicalize(base.join(&v.source)).with_context(|| { format!("volume '{}' not found under volumes_dir '{dir}'", v.source) })?; if !real.starts_with(&base) { diff --git a/dstack/vmm/src/one_shot.rs b/dstack/vmm/src/one_shot.rs index abcb36ba4..0e19c942e 100644 --- a/dstack/vmm/src/one_shot.rs +++ b/dstack/vmm/src/one_shot.rs @@ -6,6 +6,7 @@ use crate::app::{make_sys_config, Image, VmConfig, VmWorkDir}; use crate::config::Config; use crate::main_service; use anyhow::{Context, Result}; +use fs_err as fs; pub async fn run_one_shot( vm_config_path: &str, @@ -328,14 +329,14 @@ Compose file content (first 200 chars): // Configure stdio to match supervisor behavior if !process_config.stdout.is_empty() { - let stdout_file = std::fs::File::create(&process_config.stdout) - .context("Failed to create stdout file")?; - cmd.stdout(stdout_file); + let stdout_file = + fs::File::create(&process_config.stdout).context("Failed to create stdout file")?; + cmd.stdout(stdout_file.into_file()); } if !process_config.stderr.is_empty() { - let stderr_file = std::fs::File::create(&process_config.stderr) - .context("Failed to create stderr file")?; - cmd.stderr(stderr_file); + let stderr_file = + fs::File::create(&process_config.stderr).context("Failed to create stderr file")?; + cmd.stderr(stderr_file.into_file()); } cmd.current_dir(&workdir_path); From 9ec5e96c76f7968cb417424811161fb95040a730 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 6 Jul 2026 17:47:43 -0700 Subject: [PATCH 06/25] vmm: resolve volume paths with path-absolutize --- dstack/vmm/src/main_service.rs | 128 ++++++++++++++++++++++++--------- 1 file changed, 96 insertions(+), 32 deletions(-) diff --git a/dstack/vmm/src/main_service.rs b/dstack/vmm/src/main_service.rs index ee84ae530..5942e4e36 100644 --- a/dstack/vmm/src/main_service.rs +++ b/dstack/vmm/src/main_service.rs @@ -3,6 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 use std::ops::Deref; +use std::path::{Component, Path, PathBuf}; use std::time::{SystemTime, UNIX_EPOCH}; use anyhow::{bail, Context, Result}; @@ -10,14 +11,16 @@ use dstack_types::AppCompose; use dstack_vmm_rpc as rpc; use dstack_vmm_rpc::vmm_server::{VmmRpc, VmmServer}; use dstack_vmm_rpc::{ - AppId, ComposeHash as RpcComposeHash, GatewaySettings, GetInfoResponse, GetMetaResponse, Id, - ImageInfo as RpcImageInfo, ImageListResponse, KmsSettings, ListGpusResponse, PublicKeyResponse, - PullRegistryImageRequest, RegistryImageInfo, RegistryImageListResponse, ReloadVmsResponse, - ResizeVmRequest, ResourcesSettings, StatusRequest, StatusResponse, SvListResponse, - SvProcessInfo, UpdateVmRequest, VersionResponse, VmConfiguration, + AppId, ComposeHash as RpcComposeHash, DhcpLeaseRequest, GatewaySettings, GetInfoResponse, + GetMetaResponse, Id, ImageInfo as RpcImageInfo, ImageListResponse, KmsSettings, + ListGpusResponse, PublicKeyResponse, PullRegistryImageRequest, RegistryImageInfo, + RegistryImageListResponse, ReloadVmsResponse, ResizeVmRequest, ResourcesSettings, + StatusRequest, StatusResponse, SvListResponse, SvProcessInfo, UpdateVmRequest, VersionResponse, + VmConfiguration, }; use fs_err as fs; use or_panic::ResultOrPanic; +use path_absolutize::Absolutize; use ra_rpc::{CallContext, RpcCall}; use tracing::{info, warn}; @@ -219,35 +222,10 @@ fn resolve_volumes( if dir.is_empty() { bail!("volumes requested but cvm.volumes_dir is not configured"); } - let base = - fs::canonicalize(dir).with_context(|| format!("volumes_dir '{dir}' does not exist"))?; + let base = fs::canonicalize(dir)?; reqs.iter() .map(|v| { - // `,`/`=` would let a crafted file name inject qemu `-drive` options - // when the path is concatenated into the drive string. - if v.source.is_empty() - || v.source.contains('/') - || v.source.contains("..") - || v.source.contains(',') - || v.source.contains('=') - { - bail!( - "invalid volume source '{}': must be a bare file name (no '/', '..', ',', '=')", - v.source - ); - } - let real = fs::canonicalize(base.join(&v.source)).with_context(|| { - format!("volume '{}' not found under volumes_dir '{dir}'", v.source) - })?; - if !real.starts_with(&base) { - bail!("volume '{}' escapes volumes_dir", v.source); - } - // Re-check the resolved path: a symlink could resolve to a name - // containing `,`/`=` and reintroduce QEMU -drive option injection. - let real_str = real.to_string_lossy(); - if real_str.contains(',') || real_str.contains('=') { - bail!("volume '{}' resolves to a path with ',' or '='", v.source); - } + let real = resolve_volume_source(&base, &v.source)?; Ok(crate::app::VmVolume { source: real.to_string_lossy().into_owned(), // Verity volumes are always read-only: the backing file is shared @@ -260,6 +238,32 @@ fn resolve_volumes( .collect() } +fn resolve_volume_source(base: &Path, source: &str) -> Result { + if source.is_empty() { + bail!("invalid volume source: empty path"); + } + + let source_path = Path::new(source); + let mut components = source_path.components(); + if !matches!(components.next(), Some(Component::Normal(_))) || components.next().is_some() { + bail!("invalid volume source '{source}': must be a bare file name"); + } + + let real = fs::canonicalize(base.join(source_path))?; + real.absolutize_virtually(base) + .with_context(|| format!("volume '{source}' escapes volumes_dir"))?; + + // QEMU's -drive parser treats ',' as an option separator and '=' as an + // option key/value delimiter. Keep this guard while volumes are attached + // through `-drive file=...`. + let real_str = real.to_string_lossy(); + if real_str.contains(',') || real_str.contains('=') { + bail!("volume '{source}' resolves to a path with ',' or '='"); + } + + Ok(real) +} + fn networking_from_proto(proto: &rpc::NetworkingConfig) -> Result> { let bridge = proto.bridge_name.trim().to_string(); let mode = match proto.mode.as_str() { @@ -281,6 +285,7 @@ fn networking_from_proto(proto: &rpc::NetworkingConfig) -> Result "bridge".to_string(), NetworkingMode::Custom => String::new(), }, + forward_service_enabled: default_networking.forward_service_enabled, default_bridge: default_networking.bridge.clone(), }), }) @@ -715,6 +724,11 @@ impl VmmRpc for RpcHandler { self.app.reload_vms_sync().await } + async fn report_dhcp_lease(self, request: DhcpLeaseRequest) -> Result<()> { + self.app.report_dhcp_lease(&request.mac, &request.ip).await; + Ok(()) + } + async fn sv_list(self) -> Result { use supervisor_client::supervisor::ProcessStatus; let list = self.app.supervisor.list().await?; @@ -917,6 +931,7 @@ mod tests { no_tee: false, networking: None, networks: vec![], + volumes: vec![], } } @@ -975,4 +990,53 @@ mod tests { assert!(err.to_string().contains("custom networking mode")); } + + #[test] + fn multiple_bridges_are_rejected_when_builtin_forwarding_is_enabled() { + let mut cvm_config = test_cvm_config(); + cvm_config.networking.forward_service_enabled = true; + let mut request = test_vm_configuration(); + request.networks = vec![ + rpc::NetworkingConfig { + mode: "bridge".to_string(), + bridge_name: "lo".to_string(), + }, + rpc::NetworkingConfig { + mode: "bridge".to_string(), + bridge_name: "lo".to_string(), + }, + ]; + + let err = create_manifest_from_vm_config(request, &cvm_config).unwrap_err(); + + assert!(err + .to_string() + .contains("built-in port forwarding supports only one bridge")); + } + + #[test] + fn resolve_volume_source_rejects_escape_symlink_and_qemu_metachars() -> Result<()> { + let tmp = tempfile::tempdir()?; + let volumes = tmp.path().join("volumes"); + fs::create_dir_all(&volumes)?; + fs::write(volumes.join("ok.img"), b"ok")?; + let base = fs::canonicalize(&volumes)?; + + let ok = resolve_volume_source(&base, "ok.img")?; + assert_eq!(ok, base.join("ok.img")); + + let err = resolve_volume_source(&base, "../ok.img").unwrap_err(); + assert!(format!("{err:#}").contains("must be a bare file name")); + + fs::write(tmp.path().join("outside.img"), b"outside")?; + std::os::unix::fs::symlink(tmp.path().join("outside.img"), volumes.join("link.img"))?; + let err = resolve_volume_source(&base, "link.img").unwrap_err(); + assert!(format!("{err:#}").contains("escapes volumes_dir")); + + fs::write(volumes.join("bad,readonly=off"), b"bad")?; + let err = resolve_volume_source(&base, "bad,readonly=off").unwrap_err(); + assert!(format!("{err:#}").contains("',' or '='")); + + Ok(()) + } } From 1089356cbe1f9b9d7af843d7c044228ca624c752 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 12 Jul 2026 21:07:53 -0700 Subject: [PATCH 07/25] docs: relocate verity volume guide for monorepo layout --- {dstack/docs => docs}/verity-volumes.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {dstack/docs => docs}/verity-volumes.md (100%) diff --git a/dstack/docs/verity-volumes.md b/docs/verity-volumes.md similarity index 100% rename from dstack/docs/verity-volumes.md rename to docs/verity-volumes.md From 9c3bca75a6c64ed764c7c9fe8458370052746021 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 15 Jul 2026 03:35:58 -0700 Subject: [PATCH 08/25] guest: package verity helper in OS image --- docs/verity-volumes.md | 2 +- dstack/vmm/src/vmm-cli.py | 56 ------------------- .../common/rootfs}/dstack-verity.sh | 0 .../recipes-core/dstack-guest/dstack-guest.bb | 1 + 4 files changed, 2 insertions(+), 57 deletions(-) rename {dstack/basefiles => os/common/rootfs}/dstack-verity.sh (100%) diff --git a/docs/verity-volumes.md b/docs/verity-volumes.md index 6d37344e9..6ee3ab8bf 100644 --- a/docs/verity-volumes.md +++ b/docs/verity-volumes.md @@ -142,7 +142,7 @@ jobs: - Layers are treated as public — the volume is unencrypted and shareable. Secret layers would need a per-app, KMS-encrypted volume, and that's out of scope for now. - Under TDX the single-copy saving is on storage, transfer, and extraction, not RAM — each CVM still caches the blocks it reads in its own encrypted memory. - The per-boot `docker image prune -af` removes images no running container references, so bake the images your compose actually runs; guarding prune against verity-seeded images is a follow-up. -- On guest images that don't yet ship the helper, `vmm-cli` injects it into `init_script`, so `app_id` depends on the bundled helper bytes — pin your CLI/image version when you register compose hashes. +- Verity volumes require a guest image that ships `/bin/dstack-verity.sh`; the helper runs from `dstack-prepare.sh` before dockerd starts. - Seeded overlay2 metadata lives on the persistent disk, while the layer `diff/` binds are re-established each boot. A *partial* seed (a metadata copy fails mid-write) unwinds its binds and falls back to a normal pull. The open gap is across boots: if a volume that seeded once is later not re-attached (or swapped), its metadata persists while its `diff/` binds don't, so docker can see the image present with empty layers. The normal reboot (same volume) re-binds correctly. Reconciling stale seeded metadata against missing binds on boot is a follow-up — it needs image→layer dependency walking so it doesn't drop a pulled image that happens to share a base layer. ## What's proven diff --git a/dstack/vmm/src/vmm-cli.py b/dstack/vmm/src/vmm-cli.py index 087b2f8a3..0d6605f0a 100755 --- a/dstack/vmm/src/vmm-cli.py +++ b/dstack/vmm/src/vmm-cli.py @@ -361,61 +361,6 @@ def parse_volume(spec: str) -> Dict: return {"source": name, "read_only": True} -def _load_verity_helper() -> Optional[str]: - """Read the bundled dstack-verity helper (basefiles/dstack-verity.sh).""" - path = os.path.join( - os.path.dirname(os.path.abspath(__file__)), - "..", - "..", - "basefiles", - "dstack-verity.sh", - ) - try: - with open(path, "r", encoding="utf-8") as f: - return f.read() - except OSError: - return None - - -def inject_verity_init(compose_content: str) -> str: - """Inject an init_script that runs the dstack-verity helper. - - This applies when the compose declares verity_volumes but has no init_script. - It lets verity volumes work on guest images that don't yet ship - /bin/dstack-verity.sh. The injected script defers to that binary when present, - so it becomes a no-op once the image ships the helper. - """ - try: - compose = json.loads(compose_content) - except (ValueError, TypeError): - return compose_content - if not compose.get("verity_volumes"): - return compose_content - if compose.get("init_script"): - # don't clobber the user's own init_script; warn instead. - print( - "warning: verity_volumes set but the compose has its own init_script; " - "not injecting the helper (the guest image must ship /bin/dstack-verity.sh)" - ) - return compose_content - helper = _load_verity_helper() - if not helper: - print( - "warning: verity_volumes set but dstack-verity helper not found; images will pull" - ) - return compose_content - compose["init_script"] = ( - "if [ -x /bin/dstack-verity.sh ]; then :; else\n" - "cat > /run/dstack-verity.sh <<'DSTACK_VERITY_EOF'\n" - + helper.rstrip("\n") - + "\nDSTACK_VERITY_EOF\n" - "bash /run/dstack-verity.sh /dstack/app-compose.json || true\n" - "fi\n" - ) - print("injected dstack-verity init_script for verity_volumes") - return json.dumps(compose, indent=4, ensure_ascii=False) - - def read_utf8(filepath: str) -> str: """Read a file and return its contents as a UTF-8 string.""" with open(filepath, "rb") as f: @@ -933,7 +878,6 @@ def create_vm(self, args) -> None: raise Exception(f"Compose file not found: {args.compose}") compose_content = read_utf8(args.compose) - compose_content = inject_verity_init(compose_content) envs = parse_env_file(args.env_file) diff --git a/dstack/basefiles/dstack-verity.sh b/os/common/rootfs/dstack-verity.sh similarity index 100% rename from dstack/basefiles/dstack-verity.sh rename to os/common/rootfs/dstack-verity.sh diff --git a/os/yocto/layers/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/os/yocto/layers/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index daa00f9ed..0e57b4e1d 100644 --- a/os/yocto/layers/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/os/yocto/layers/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -67,6 +67,7 @@ do_install() { install -m 0755 ${CARGO_BINDIR}/dstack-util ${D}${bindir} install -m 0755 ${CARGO_BINDIR}/dstack-guest-agent ${D}${bindir} install -m 0755 ${DSTACK_ROOTFS_FILES}/dstack-prepare.sh ${D}${bindir} + install -m 0755 ${DSTACK_ROOTFS_FILES}/dstack-verity.sh ${D}${bindir} install -m 0755 ${DSTACK_ROOTFS_FILES}/ephemeral-docker.sh ${D}${bindir} install -m 0755 ${DSTACK_ROOTFS_FILES}/wg-checker.sh ${D}${bindir} install -m 0755 ${DSTACK_ROOTFS_FILES}/app-compose.sh ${D}${bindir} From 07a9c00074236bc260595f84d23b4b48e18fce89 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 16 Jul 2026 01:56:00 -0700 Subject: [PATCH 09/25] refactor(vmm): align verity volumes with split qemu modules --- dstack/vmm/src/app.rs | 1 + dstack/vmm/src/app/qemu.rs | 84 ++++++++++- dstack/vmm/src/app/vm_info.rs | 9 ++ dstack/vmm/src/app/volume.rs | 258 +++++++++++++++++++++++++++++++++ dstack/vmm/src/main_service.rs | 20 +++ 5 files changed, 369 insertions(+), 3 deletions(-) create mode 100644 dstack/vmm/src/app/volume.rs diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index ea5c471a4..1096caf08 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -45,6 +45,7 @@ mod network; mod qemu; pub(crate) mod registry; mod vm_info; +mod volume; mod workdir; fn signal_pidfd(pid: u32, signal: libc::c_int) -> std::io::Result<()> { diff --git a/dstack/vmm/src/app/qemu.rs b/dstack/vmm/src/app/qemu.rs index 270c62c73..84352424a 100644 --- a/dstack/vmm/src/app/qemu.rs +++ b/dstack/vmm/src/app/qemu.rs @@ -24,7 +24,9 @@ use super::{ image::Image, mr_config::{snp_host_data, tdx_mr_config_id}, network::{mac_address_for_vm_index, resolved_networks, validate_resolved_networks}, - pci_numa_node, round_up, GpuConfig, VmWorkDir, + pci_numa_node, round_up, + volume::verity_serial_hint, + GpuConfig, VmWorkDir, }; use anyhow::{bail, Context, Result}; use bon::Builder; @@ -174,10 +176,17 @@ fn virtio_pci_device(device: &str, snp: bool) -> String { } } +struct PreparedVolume { + source: String, + read_only: bool, + serial: Option, +} + struct PreparedQemuLaunch { workdir: VmWorkDir, platform: TeePlatform, networks: Vec, + volumes: Vec, hugepage_numa_nodes: Option>, gpu_numa_nodes: HashMap, numa_cpus: Option, @@ -202,6 +211,18 @@ impl PreparedQemuLaunch { let platform = cfg.resolved_platform(); let networks = resolved_networks(&vm.manifest, cfg); validate_resolved_networks(&networks)?; + let volumes = vm + .manifest + .volumes + .iter() + .map(|volume| PreparedVolume { + source: volume.source.clone(), + read_only: volume.read_only, + // File inspection belongs to launch preparation. Keep the + // command builder host-I/O-free so it remains deterministic. + serial: verity_serial_hint(Path::new(&volume.source)), + }) + .collect(); let hugepage_numa_nodes = if vm.manifest.hugepages { Some(hugepage_numa_nodes(gpus)?) @@ -266,6 +287,7 @@ impl PreparedQemuLaunch { workdir, platform, networks, + volumes, hugepage_numa_nodes, gpu_numa_nodes, numa_cpus, @@ -409,6 +431,7 @@ impl QemuCommandBuilder<'_> { let mut command = self.base_command(); self.configure_rootfs(&mut command)?; self.configure_data_disk(&mut command); + self.configure_volumes(&mut command); self.configure_networking(&mut command)?; self.vm.configure_smbios(&mut command, self.cfg); self.configure_tpm_and_vsock(&mut command); @@ -527,6 +550,31 @@ impl QemuCommandBuilder<'_> { )); } + fn configure_volumes(&self, command: &mut Command) { + // Sources are host paths already validated by the VMM. Attach extra + // volumes after the data disk and before networking, matching the + // established device order. + for (index, volume) in self.prepared.volumes.iter().enumerate() { + let id = format!("vol{index}"); + let mut drive = format!("file={},if=none,id={id},format=raw", volume.source); + if volume.read_only { + drive.push_str(",readonly=on"); + } + + // The serial is only a lookup hint. The guest still verifies the + // complete dm-verity root before using the volume. + let mut device = format!("virtio-blk-pci,drive={id}"); + if let Some(serial) = &volume.serial { + device.push_str(&format!(",serial={serial}")); + } + command + .arg("-drive") + .arg(drive) + .arg("-device") + .arg(virtio_pci_device(&device, self.is_amd_sev_snp())); + } + } + fn configure_networking(&self, command: &mut Command) -> Result<()> { let hostfwd_index = self .prepared @@ -939,10 +987,10 @@ mod tests { use super::{ amd_sev_snp_memory_backend_arg, parse_amd_sev_snp_qmp_capabilities, virtio_pci_device, - PreparedQemuLaunch, QemuCommandBuilder, VmConfig, + PreparedQemuLaunch, PreparedVolume, QemuCommandBuilder, VmConfig, }; use crate::app::image::{Image, ImageInfo}; - use crate::app::{GpuConfig, Manifest, PortMapping, VmWorkDir}; + use crate::app::{GpuConfig, Manifest, PortMapping, VmVolume, VmWorkDir}; use crate::config::{Config, Protocol, TeePlatform, DEFAULT_CONFIG}; #[test] @@ -1009,6 +1057,10 @@ mod tests { gateway_urls: vec![], no_tee: true, networks: vec![], + volumes: vec![VmVolume { + source: "/does-not-exist/volume.img".into(), + read_only: true, + }], }, image: Image { info: ImageInfo { @@ -1043,6 +1095,11 @@ mod tests { workdir: VmWorkDir::new("/does-not-exist/vm-1"), platform: TeePlatform::Tdx, networks: vec![config.cvm.networking.clone(), config.cvm.networking.clone()], + volumes: vec![PreparedVolume { + source: "/does-not-exist/volume.img".into(), + read_only: true, + serial: Some("0123456789abcdef0123".into()), + }], hugepage_numa_nodes: None, gpu_numa_nodes: HashMap::new(), numa_cpus: None, @@ -1075,6 +1132,27 @@ mod tests { .args .windows(2) .any(|args| args == ["-append", "console=hvc0"])); + assert!(process.args.windows(2).any(|args| { + args == [ + "-drive", + "file=/does-not-exist/volume.img,if=none,id=vol0,format=raw,readonly=on", + ] + })); + assert!(process + .args + .iter() + .any(|arg| { arg == "virtio-blk-pci,drive=vol0,serial=0123456789abcdef0123" })); + let volume_position = process + .args + .iter() + .position(|arg| arg.contains("id=vol0")) + .unwrap(); + let network_position = process + .args + .iter() + .position(|arg| arg == "-netdev") + .unwrap(); + assert!(volume_position < network_position); let netdevs = process .args .windows(2) diff --git a/dstack/vmm/src/app/vm_info.rs b/dstack/vmm/src/app/vm_info.rs index d039418c8..1bc39b9c8 100644 --- a/dstack/vmm/src/app/vm_info.rs +++ b/dstack/vmm/src/app/vm_info.rs @@ -169,6 +169,15 @@ impl VmInfo { no_tee, networking: configured_networking, networks: configured_networks, + volumes: self + .manifest + .volumes + .iter() + .map(|volume| pb::VmVolume { + source: volume.source.clone(), + read_only: volume.read_only, + }) + .collect(), }) }, app_url: self diff --git a/dstack/vmm/src/app/volume.rs b/dstack/vmm/src/app/volume.rs new file mode 100644 index 000000000..138e66b29 --- /dev/null +++ b/dstack/vmm/src/app/volume.rs @@ -0,0 +1,258 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! Verity-volume inspection used while preparing a QEMU launch. + +use std::io::{Read, Seek, SeekFrom}; +use std::path::Path; + +use fs_err as fs; +use sha2::{Digest, Sha256}; + +/// Compute the virtio-blk `serial` hint for a pre-baked verity volume: the first +/// 20 hex chars of its dm-verity root hash (virtio caps serials at 20 bytes). +/// +/// The guest uses this to open the matching disk directly instead of probing +/// every disk. It is only a hint. The guest still verifies the full root, so a +/// missing or wrong serial just falls back to probing. +/// +/// The current volume format is a raw GPT disk: p1 is verity data, p2 is the +/// verity hash device. The old single-file `[squashfs][verity]` layout is still +/// recognized as a fallback so older volumes can still get the serial hint. +/// +/// For multi-block data devices, the root is `sha256(salt || top hash block)`, +/// and the top block sits right after the verity superblock, so we never walk +/// the hash tree. A one-block data device has no hash blocks, so the root is +/// `sha256(salt || data block)`. +pub(super) fn verity_serial_hint(path: &Path) -> Option { + let mut file = fs::File::open(path).ok()?; + partitioned_verity_serial_hint(&mut file).or_else(|| legacy_verity_serial_hint(&mut file)) +} + +fn partitioned_verity_serial_hint(file: &mut fs::File) -> Option { + let (data_offset, hash_offset) = gpt_data_hash_offsets(file)?; + root_prefix_from_verity_devices(file, data_offset, hash_offset) +} + +fn legacy_verity_serial_hint(file: &mut fs::File) -> Option { + file.seek(SeekFrom::Start(0)).ok()?; + + // squashfs superblock: "hsqs" magic at 0, bytes_used (LE u64) at 40. The + // verity hash tree starts just past there, block-aligned. + let mut magic = [0u8; 4]; + file.read_exact(&mut magic).ok()?; + if &magic != b"hsqs" { + return None; + } + file.seek(SeekFrom::Start(40)).ok()?; + let mut buf8 = [0u8; 8]; + file.read_exact(&mut buf8).ok()?; + let hash_offset = u64::from_le_bytes(buf8).div_ceil(4096).checked_mul(4096)?; + root_prefix_from_verity_devices(file, 0, hash_offset) +} + +fn gpt_data_hash_offsets(file: &mut fs::File) -> Option<(u64, u64)> { + file.seek(SeekFrom::Start(512)).ok()?; + let mut header = [0u8; 512]; + file.read_exact(&mut header).ok()?; + if &header[0..8] != b"EFI PART" { + return None; + } + + let entries_lba = u64::from_le_bytes(header[72..80].try_into().ok()?); + let entry_count = u32::from_le_bytes(header[80..84].try_into().ok()?); + let entry_size = u32::from_le_bytes(header[84..88].try_into().ok()?) as usize; + if entry_count < 2 || !(128..=4096).contains(&entry_size) { + return None; + } + + let entries_offset = entries_lba.checked_mul(512)?; + let data = gpt_partition_first_lba(file, entries_offset, entry_size, 0)?; + let hash = gpt_partition_first_lba(file, entries_offset, entry_size, 1)?; + Some((data.checked_mul(512)?, hash.checked_mul(512)?)) +} + +fn gpt_partition_first_lba( + file: &mut fs::File, + entries_offset: u64, + entry_size: usize, + index: u64, +) -> Option { + let offset = entries_offset.checked_add(index.checked_mul(entry_size as u64)?)?; + file.seek(SeekFrom::Start(offset)).ok()?; + let mut entry = vec![0u8; entry_size]; + file.read_exact(&mut entry).ok()?; + if entry[0..16].iter().all(|&byte| byte == 0) { + return None; + } + Some(u64::from_le_bytes(entry[32..40].try_into().ok()?)) +} + +fn root_prefix_from_verity_devices( + file: &mut fs::File, + data_offset: u64, + hash_offset: u64, +) -> Option { + file.seek(SeekFrom::Start(hash_offset)).ok()?; + let mut superblock = [0u8; 512]; + file.read_exact(&mut superblock).ok()?; + if &superblock[0..8] != b"verity\0\0" || !superblock[32..64].starts_with(b"sha256\0") { + return None; + } + // The builder always writes 4096-byte blocks; requiring it also caps the + // block allocation below at one page for an untrusted file. + let data_block_size = u32::from_le_bytes(superblock[64..68].try_into().ok()?) as u64; + let hash_block_size = u32::from_le_bytes(superblock[68..72].try_into().ok()?) as u64; + let data_blocks = u64::from_le_bytes(superblock[72..80].try_into().ok()?); + let salt_size = u16::from_le_bytes(superblock[80..82].try_into().ok()?) as usize; + if data_block_size != 4096 || hash_block_size != 4096 || data_blocks == 0 || salt_size > 256 { + return None; + } + let salt = &superblock[88..88 + salt_size]; + + let (block_offset, block_size) = if data_blocks == 1 { + (data_offset, data_block_size) + } else { + (hash_offset.checked_add(hash_block_size)?, hash_block_size) + }; + file.seek(SeekFrom::Start(block_offset)).ok()?; + let mut block = vec![0u8; block_size as usize]; + file.read_exact(&mut block).ok()?; + + let mut hasher = Sha256::new(); + hasher.update(salt); + hasher.update(&block); + Some(hex::encode(hasher.finalize())[..20].to_string()) +} + +#[cfg(test)] +mod tests { + use std::io::{Seek, SeekFrom, Write}; + + use sha2::{Digest, Sha256}; + + use super::verity_serial_hint; + + fn write_minimal_gpt(file: &mut tempfile::NamedTempFile, data_lba: u64, hash_lba: u64) { + let mut header = [0u8; 512]; + header[0..8].copy_from_slice(b"EFI PART"); + header[72..80].copy_from_slice(&2u64.to_le_bytes()); // partition entries LBA + header[80..84].copy_from_slice(&128u32.to_le_bytes()); // entries + header[84..88].copy_from_slice(&128u32.to_le_bytes()); // entry size + file.seek(SeekFrom::Start(512)).unwrap(); + file.write_all(&header).unwrap(); + + let mut entries = vec![0u8; 128 * 128]; + // Any non-zero type GUID is enough for the parser. + entries[0] = 1; + entries[32..40].copy_from_slice(&data_lba.to_le_bytes()); + let second = 128; + entries[second] = 1; + entries[second + 32..second + 40].copy_from_slice(&hash_lba.to_le_bytes()); + file.seek(SeekFrom::Start(2 * 512)).unwrap(); + file.write_all(&entries).unwrap(); + } + + fn verity_superblock(data_blocks: u64, salt: &[u8]) -> [u8; 4096] { + let mut superblock = [0u8; 4096]; + superblock[0..8].copy_from_slice(b"verity\0\0"); + superblock[32..38].copy_from_slice(b"sha256"); + superblock[64..68].copy_from_slice(&4096u32.to_le_bytes()); + superblock[68..72].copy_from_slice(&4096u32.to_le_bytes()); + superblock[72..80].copy_from_slice(&data_blocks.to_le_bytes()); + superblock[80..82].copy_from_slice(&(salt.len() as u16).to_le_bytes()); + superblock[88..88 + salt.len()].copy_from_slice(salt); + superblock + } + + fn test_verity_salt() -> [u8; 32] { + // Use tempfile's random name as a per-test seed. The salt is fixture + // data rather than a secret, but deriving it at runtime also keeps + // security scanners from mistaking a fixed test vector for a + // production cryptographic salt. + let seed = tempfile::NamedTempFile::new().unwrap(); + Sha256::digest(seed.path().as_os_str().as_encoded_bytes()).into() + } + + #[test] + fn legacy_verity_serial_hint_matches_root_prefix() { + // A minimal squashfs+verity file: "hsqs" with bytes_used = 8192, then a + // verity superblock at 8192 and its top hash block at 12288. + let mut image = vec![0u8; 8192]; + image[0..4].copy_from_slice(b"hsqs"); + image[40..48].copy_from_slice(&8192u64.to_le_bytes()); + + let salt = test_verity_salt(); + image.extend_from_slice(&verity_superblock(2, &salt)); + + let top_block = vec![0x5au8; 4096]; + image.extend_from_slice(&top_block); + + let mut file = tempfile::NamedTempFile::new().unwrap(); + file.write_all(&image).unwrap(); + + let mut hasher = Sha256::new(); + hasher.update(salt); + hasher.update(&top_block); + let expected = hex::encode(hasher.finalize()); + + let got = verity_serial_hint(file.path()).unwrap(); + assert_eq!(got, expected[..20]); + assert_eq!(got.len(), 20); + } + + #[test] + fn partitioned_verity_serial_hint_matches_root_prefix() { + let data_lba = 2048u64; + let hash_lba = 4096u64; + let salt = test_verity_salt(); + let top_block = vec![0x5au8; 4096]; + + let mut file = tempfile::NamedTempFile::new().unwrap(); + file.as_file_mut().set_len((hash_lba + 16) * 512).unwrap(); + write_minimal_gpt(&mut file, data_lba, hash_lba); + file.seek(SeekFrom::Start(hash_lba * 512)).unwrap(); + file.write_all(&verity_superblock(2, &salt)).unwrap(); + file.write_all(&top_block).unwrap(); + + let mut hasher = Sha256::new(); + hasher.update(salt); + hasher.update(&top_block); + let expected = hex::encode(hasher.finalize()); + + let got = verity_serial_hint(file.path()).unwrap(); + assert_eq!(got, expected[..20]); + } + + #[test] + fn partitioned_one_block_verity_serial_hint_hashes_data_block() { + let data_lba = 2048u64; + let hash_lba = 4096u64; + let salt = test_verity_salt(); + let data_block = vec![0x42u8; 4096]; + + let mut file = tempfile::NamedTempFile::new().unwrap(); + file.as_file_mut().set_len((hash_lba + 8) * 512).unwrap(); + write_minimal_gpt(&mut file, data_lba, hash_lba); + file.seek(SeekFrom::Start(data_lba * 512)).unwrap(); + file.write_all(&data_block).unwrap(); + file.seek(SeekFrom::Start(hash_lba * 512)).unwrap(); + file.write_all(&verity_superblock(1, &salt)).unwrap(); + + let mut hasher = Sha256::new(); + hasher.update(salt); + hasher.update(&data_block); + let expected = hex::encode(hasher.finalize()); + + let got = verity_serial_hint(file.path()).unwrap(); + assert_eq!(got, expected[..20]); + } + + #[test] + fn verity_serial_hint_skips_non_verity_files() { + let mut file = tempfile::NamedTempFile::new().unwrap(); + file.write_all(b"not a squashfs image").unwrap(); + assert_eq!(verity_serial_hint(file.path()), None); + } +} diff --git a/dstack/vmm/src/main_service.rs b/dstack/vmm/src/main_service.rs index 5942e4e36..6f0d6da2e 100644 --- a/dstack/vmm/src/main_service.rs +++ b/dstack/vmm/src/main_service.rs @@ -1039,4 +1039,24 @@ mod tests { Ok(()) } + + #[test] + fn resolve_volumes_forces_read_only() -> Result<()> { + let tmp = tempfile::tempdir()?; + fs::write(tmp.path().join("volume.img"), b"volume")?; + let mut cvm_config = test_cvm_config(); + cvm_config.volumes_dir = tmp.path().to_string_lossy().into_owned(); + + let volumes = resolve_volumes( + &[rpc::VmVolume { + source: "volume.img".into(), + read_only: false, + }], + &cvm_config, + )?; + + assert_eq!(volumes.len(), 1); + assert!(volumes[0].read_only); + Ok(()) + } } From 1496b2670ceffb85f568ce777659775f47e42a5c Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 20 Jul 2026 18:29:01 -0700 Subject: [PATCH 10/25] feat(verity): discover self-describing guest volumes --- docs/verity-volumes.md | 17 +- dstack/crates/dstack-verity/src/lib.rs | 4 +- dstack/crates/dstack-verity/src/volume.rs | 106 +++- dstack/guest-agent/src/bin/dstack-volume.rs | 578 ++++++++++++++++++ dstack/vmm/src/app.rs | 1 - dstack/vmm/src/app/qemu.rs | 18 +- dstack/vmm/src/app/volume.rs | 258 -------- os/common/rootfs/dstack-prepare.sh | 4 +- os/common/rootfs/dstack-verity.sh | 247 -------- .../recipes-core/dstack-guest/dstack-guest.bb | 2 +- 10 files changed, 679 insertions(+), 556 deletions(-) create mode 100644 dstack/guest-agent/src/bin/dstack-volume.rs delete mode 100644 dstack/vmm/src/app/volume.rs delete mode 100755 os/common/rootfs/dstack-verity.sh diff --git a/docs/verity-volumes.md b/docs/verity-volumes.md index 6ee3ab8bf..75016095a 100644 --- a/docs/verity-volumes.md +++ b/docs/verity-volumes.md @@ -13,14 +13,15 @@ A volume has a **root hash** — its identity and integrity check — and a **ta - `"docker"` — the volume is a docker overlay2 store; its images are seeded into docker as cache hits. - `"/some/path"` — the volume's filesystem is mounted there, for data like model weights. -On disk, a volume is a raw GPT image with two partitions: +On disk, a volume is a raw GPT image with three partitions: ``` -p1: filesystem data -p2: dm-verity superblock + hash tree +p1: DSTACK_VOLUME metadata envelope +p2: filesystem data +p3: dm-verity superblock + hash tree ``` -The guest opens `p1` as the verity data device and `p2` as the verity hash device. That keeps the verity layer independent of the filesystem format: the guest does not need to read a squashfs/ext4 superblock to find a hash offset. +The guest finds disks by the `DSTACK_VOLUME` magic in `p1`, then opens `p2` as the verity data device and `p3` as the verity hash device. That keeps the verity layer independent of the filesystem format: the guest does not need to read a squashfs/ext4 superblock to find a hash offset. An app lists its volumes in `app-compose.json`. dstack already hashes that file into the app's identity, so every root hash is measured as part of `app_id`, and the CVM only uses content matching what it was attested as. Everything else is untrusted: the host hands over bytes, and dm-verity rejects any that don't match the root. @@ -79,7 +80,7 @@ overlay2//diff/ the already-extracted layer files (the GBs) `diff/` is the layer already decompressed and untarred. Seeding reuses it in place: -1. Open the volume with veritysetup (`p1` as data, `p2` as hash) and mount it read-only at a writable path (`/run/dstack-verity`). The rootfs is read-only dm-verity, so the mountpoint has to be on a writable fs; squashfs itself mounts read-only directly, no journal and nothing to replay. +1. Open the volume with veritysetup (`p2` as data, `p3` as hash) and mount it read-only at a writable path (`/run/dstack-verity`). The rootfs is read-only dm-verity, so the mountpoint has to be on a writable fs; squashfs itself mounts read-only directly, no journal and nothing to replay. 2. Copy the metadata (a few MB) into `/var/lib/docker`. 3. Per layer: make a writable `overlay2//` dir, copy its handful of tiny files, and bind-mount only the big `diff/` read-only from the volume. 4. This runs from `dstack-prepare.sh` *before* dockerd starts, so there's no restart — dockerd comes up with every image on the volume already present. @@ -100,11 +101,11 @@ dstack deploy -c docker-compose.yaml \ --volume llama-70b.img:a1b2c3d4...:/run/models/llama-70b ``` -Each `--volume NAME:VERITY_ROOT:TARGET` both attaches the file and writes the measured `verity_volumes` entry (the root is yours to supply, from `dstack verity`, so it stays part of `app_id`). `vmm-cli.py` takes the same file with a hand-authored `app-compose.json` instead. The vmm resolves each name against `volumes_dir` (rejecting anything with a path separator) and attaches it as a read-only virtio-blk device, tagged with a hint: the disk's serial is set to the volume's `verity_root` prefix. `--volume` carries only the name and read-only flag, never the bytes; the bytes are already on the host. Disks are still attached in any order — the guest reads the serials once and, for each `verity_root` in the compose, opens partition 1 and partition 2 of the disk whose serial matches. That is O(devices + volumes), and no wrong disk is opened. The serial and GPT partition table are only hints and aren't trusted: the guest verifies the full root through dm-verity, so a missing/wrong tag falls back to trying every disk, and a tampering host causes a verity fault rather than a wrong mount. Integrity is checked lazily, per block on first read. Read-only plus content-addressed means one physical copy serves every CVM that references it: a base image or model shared by a hundred replicas is built, stored, and extracted once. +Each `--volume NAME:VERITY_ROOT:TARGET` both attaches the file and writes the measured `verity_volumes` entry (the root is yours to supply, from `dstack verity`, so it stays part of `app_id`). `vmm-cli.py` takes the same file with a hand-authored `app-compose.json` instead. The vmm resolves each name against `volumes_dir` (rejecting anything with a path separator) and attaches it as a read-only virtio-blk device, `--volume` carries only the name and read-only flag, never the bytes; the bytes are already on the host. Disks are still attached in any order. The Rust guest helper scans each whole disk: when the kernel exposes partitions it checks partition 1, otherwise it checks the start of the raw disk. A matching `DSTACK_VOLUME` envelope supplies a kind and the full claimed root. For the verity kind, the guest opens partitions 2 and 3 only after matching that root against the measured compose. The envelope and partition table are untrusted hints; the guest passes the measured root to dm-verity, so tampering causes a verity fault rather than a wrong mount. Integrity is checked lazily, per block on first read. Read-only plus content-addressed means one physical copy serves every CVM that references it: a base image or model shared by a hundred replicas is built, stored, and extracted once. ## Building volumes -`dstack verity` builds the store from scratch, without a docker daemon and without a CVM — just the registry (a `docker` volume does need root, because laying out overlay2 whiteouts uses `mknod` and a `trusted.*` xattr; a `--dir` or `--fs-image` volume needs no root). It pulls each layer by digest, and lays out the overlay2 store the way docker would, with one deliberate change: the per-layer directory id is the layer's *chain-id* instead of docker's random cache-id. That's what makes the store a pure function of the image. AUFS `.wh.` whiteouts in the layer tars are converted to their overlay2 on-disk form (a `0:0` char device, or the `trusted.overlay.opaque` xattr) so deletions in upper layers still take effect. Then `mksquashfs` packs it with a fixed timestamp, `veritysetup` builds a separate hash image with a fixed salt and a UUID derived from the filesystem bytes, and the builder wraps both blobs in a deterministic GPT disk (`p1` data, `p2` verity metadata/hash tree). A `--dir` volume skips the overlay2 layout and pull entirely — it packs the directory straight into the same partitioned squashfs + verity format. A `--fs-image` volume skips `mksquashfs` too: the supplied ext4/xfs/etc. image becomes `p1` after 4096-byte padding, so the guest only needs kernel support and suitable read-only mount behavior for that filesystem. +`dstack verity` builds the store from scratch, without a docker daemon and without a CVM — just the registry (a `docker` volume does need root, because laying out overlay2 whiteouts uses `mknod` and a `trusted.*` xattr; a `--dir` or `--fs-image` volume needs no root). It pulls each layer by digest, and lays out the overlay2 store the way docker would, with one deliberate change: the per-layer directory id is the layer's *chain-id* instead of docker's random cache-id. That's what makes the store a pure function of the image. AUFS `.wh.` whiteouts in the layer tars are converted to their overlay2 on-disk form (a `0:0` char device, or the `trusted.overlay.opaque` xattr) so deletions in upper layers still take effect. Then `mksquashfs` packs it with a fixed timestamp, `veritysetup` builds a separate hash image with a fixed salt and a UUID derived from the filesystem bytes, and the builder wraps both blobs in a deterministic GPT disk (`p1` volume metadata, `p2` data, `p3` verity metadata/hash tree). A `--dir` volume skips the overlay2 layout and pull entirely — it packs the directory straight into the same partitioned squashfs + verity format. A `--fs-image` volume skips `mksquashfs` too: the supplied ext4/xfs/etc. image becomes `p2` after 4096-byte padding, so the guest only needs kernel support and suitable read-only mount behavior for that filesystem. With those fixed, two runs of the same digests produce the same bytes. The build needs no daemon and no TEE, so it can run in a CI job that also attests it (below). @@ -142,7 +143,7 @@ jobs: - Layers are treated as public — the volume is unencrypted and shareable. Secret layers would need a per-app, KMS-encrypted volume, and that's out of scope for now. - Under TDX the single-copy saving is on storage, transfer, and extraction, not RAM — each CVM still caches the blocks it reads in its own encrypted memory. - The per-boot `docker image prune -af` removes images no running container references, so bake the images your compose actually runs; guarding prune against verity-seeded images is a follow-up. -- Verity volumes require a guest image that ships `/bin/dstack-verity.sh`; the helper runs from `dstack-prepare.sh` before dockerd starts. +- Verity volumes require a guest image that ships `/bin/dstack-volume`; the Rust helper runs from `dstack-prepare.sh` before dockerd starts. - Seeded overlay2 metadata lives on the persistent disk, while the layer `diff/` binds are re-established each boot. A *partial* seed (a metadata copy fails mid-write) unwinds its binds and falls back to a normal pull. The open gap is across boots: if a volume that seeded once is later not re-attached (or swapped), its metadata persists while its `diff/` binds don't, so docker can see the image present with empty layers. The normal reboot (same volume) re-binds correctly. Reconciling stale seeded metadata against missing binds on boot is a follow-up — it needs image→layer dependency walking so it doesn't drop a pulled image that happens to share a base layer. ## What's proven diff --git a/dstack/crates/dstack-verity/src/lib.rs b/dstack/crates/dstack-verity/src/lib.rs index 31a3f3a16..59f09430f 100644 --- a/dstack/crates/dstack-verity/src/lib.rs +++ b/dstack/crates/dstack-verity/src/lib.rs @@ -10,7 +10,9 @@ //! mounts at a path. //! //! The build needs no docker daemon and no TEE, and it's reproducible: the same -//! inputs always give the same `verity_root`. So anyone can recompute the root +//! inputs always give the same `verity_root`. The first partition contains a +//! generic `DSTACK_VOLUME` envelope, followed by data and verity partitions. +//! So anyone can recompute the root //! and check it against `app-compose.json`, without trusting the builder. See //! docs/verity-volumes.md. diff --git a/dstack/crates/dstack-verity/src/volume.rs b/dstack/crates/dstack-verity/src/volume.rs index dd2a2f4e2..8d2a9c97f 100644 --- a/dstack/crates/dstack-verity/src/volume.rs +++ b/dstack/crates/dstack-verity/src/volume.rs @@ -4,10 +4,11 @@ //! Pack a directory into a reproducible, verity-protected disk image. //! -//! The output is a raw disk with two deterministic GPT partitions: +//! The output is a raw disk with three deterministic GPT partitions: //! -//! 1. the filesystem image -//! 2. the dm-verity superblock and hash tree +//! 1. a generic `DSTACK_VOLUME` metadata block +//! 2. the filesystem image +//! 3. the dm-verity superblock and hash tree //! //! Keeping the verity data and hash devices in separate partitions means the //! guest never has to inspect filesystem metadata to find the hash tree. The @@ -35,6 +36,10 @@ const PARTITION_ALIGNMENT_SECTORS: u64 = 2048; // still reserve enough trailing sectors in the raw image for the backup array // (128 entries * 128 bytes) plus the backup header. const GPT_ENTRY_SECTORS: u64 = 32; +const VOLUME_HEADER_SIZE: usize = 4096; +const VOLUME_MAGIC: &[u8; 16] = b"DSTACK_VOLUME\0\0\0"; +const VOLUME_FORMAT_VERSION: u16 = 1; +const VOLUME_KIND_VERITY: u32 = 1; #[derive(Clone, Copy)] pub enum Compression { @@ -161,8 +166,8 @@ fn seal_data_image( let verity_root = parse_root_hash(&out).context("could not find the root hash in veritysetup output")?; - // Wrap the two blobs in a deterministic GPT disk image: - // p1 = data filesystem, p2 = verity superblock + hash tree. + // Wrap the two blobs in a deterministic GPT disk image. Partition 1 is the + // generic volume envelope, partition 2 is data, and partition 3 is verity. build_partitioned_image(data_path, data_size, &hash_path, output, &verity_root)?; Ok(BuiltVolume { @@ -233,6 +238,7 @@ struct Partition { struct GptLayout { total_lbas: u64, + metadata: Partition, data: Partition, hash: Partition, } @@ -245,7 +251,9 @@ impl GptLayout { let data_sectors = data_size.div_ceil(SECTOR); let hash_sectors = hash_size.div_ceil(SECTOR); - let data_first = PARTITION_ALIGNMENT_SECTORS; + let metadata_first = PARTITION_ALIGNMENT_SECTORS; + let metadata_last = metadata_first + VOLUME_HEADER_SIZE as u64 / SECTOR - 1; + let data_first = align_up(metadata_last + 1, PARTITION_ALIGNMENT_SECTORS); let data_last = data_first + data_sectors - 1; let hash_first = align_up(data_last + 1, PARTITION_ALIGNMENT_SECTORS); let hash_last = hash_first + hash_sectors - 1; @@ -255,6 +263,10 @@ impl GptLayout { let total_lbas = align_up(min_lbas, PARTITION_ALIGNMENT_SECTORS); Ok(Self { total_lbas, + metadata: Partition { + first_lba: metadata_first, + last_lba: metadata_last, + }, data: Partition { first_lba: data_first, last_lba: data_last, @@ -298,6 +310,7 @@ fn build_partitioned_image( out = write_gpt(out, &layout, root_hash)?; + write_volume_header(&mut out, layout.metadata.first_lba * SECTOR, root_hash)?; copy_into(data_path, &mut out, layout.data.first_lba * SECTOR)?; copy_into(hash_path, &mut out, layout.hash.first_lba * SECTOR)?; Ok(()) @@ -319,6 +332,17 @@ fn write_gpt(mut out: fs::File, layout: &GptLayout, root_hash: &str) -> Result Result Result Result<()> { + use std::io::{Seek, SeekFrom, Write}; + + let root = hex::decode(root_hash).context("decoding verity root hash")?; + if root.len() != 32 { + bail!("verity root must be a 32-byte SHA-256 digest"); + } + + // Common envelope. All integer fields are little-endian. The remainder of + // the 4 KiB metadata partition is reserved and must stay zero. + let mut header = [0u8; VOLUME_HEADER_SIZE]; + header[..16].copy_from_slice(VOLUME_MAGIC); + header[16..18].copy_from_slice(&VOLUME_FORMAT_VERSION.to_le_bytes()); + header[18..20].copy_from_slice(&(VOLUME_HEADER_SIZE as u16).to_le_bytes()); + header[20..24].copy_from_slice(&VOLUME_KIND_VERITY.to_le_bytes()); + header[24..28].copy_from_slice(&1u32.to_le_bytes()); // verity kind version + header[28..32].copy_from_slice(&0u32.to_le_bytes()); // flags: partitioned + header[32..64].copy_from_slice(&root); + header[64..68].copy_from_slice(&4096u32.to_le_bytes()); + header[68..72].copy_from_slice(&4096u32.to_le_bytes()); + + out.seek(SeekFrom::Start(offset))?; + out.write_all(&header)?; + Ok(()) +} + fn copy_into(src: &Path, out: &mut fs::File, offset: u64) -> Result<()> { use std::io::{Read, Seek, SeekFrom, Write}; let mut input = fs::File::open(src)?; @@ -429,12 +479,14 @@ mod tests { } #[test] - fn gpt_layout_uses_two_aligned_partitions() { + fn gpt_layout_uses_three_aligned_partitions() { let layout = GptLayout::new(4096, 8192).unwrap(); - assert_eq!(layout.data.first_lba, PARTITION_ALIGNMENT_SECTORS); - assert_eq!(layout.data.last_lba, PARTITION_ALIGNMENT_SECTORS + 7); - assert_eq!(layout.hash.first_lba, PARTITION_ALIGNMENT_SECTORS * 2); - assert_eq!(layout.hash.last_lba, PARTITION_ALIGNMENT_SECTORS * 2 + 15); + assert_eq!(layout.metadata.first_lba, PARTITION_ALIGNMENT_SECTORS); + assert_eq!(layout.metadata.last_lba, PARTITION_ALIGNMENT_SECTORS + 7); + assert_eq!(layout.data.first_lba, PARTITION_ALIGNMENT_SECTORS * 2); + assert_eq!(layout.data.last_lba, PARTITION_ALIGNMENT_SECTORS * 2 + 7); + assert_eq!(layout.hash.first_lba, PARTITION_ALIGNMENT_SECTORS * 3); + assert_eq!(layout.hash.last_lba, PARTITION_ALIGNMENT_SECTORS * 3 + 15); assert!(layout.total_lbas - 1 - GPT_ENTRY_SECTORS > layout.hash.last_lba); } @@ -451,7 +503,7 @@ mod tests { fs::File::create(&data_path)?.write_all(&data)?; fs::File::create(&hash_path)?.write_all(&hash)?; - let root_hash = "abc123"; + let root_hash = "abababababababababababababababababababababababababababababababab"; build_partitioned_image( &data_path, data.len() as u64, @@ -468,28 +520,36 @@ mod tests { uuid_from_material(&[b"dstack-verity-disk", root_hash.as_bytes()]) ); - let p1 = disk.partitions().get(&1).unwrap(); - let p2 = disk.partitions().get(&2).unwrap(); - assert_eq!(p1.name, "dstack-data"); - assert_eq!(p2.name, "dstack-verity"); + let metadata = disk.partitions().get(&1).unwrap(); + let data_partition = disk.partitions().get(&2).unwrap(); + let hash_partition = disk.partitions().get(&3).unwrap(); + assert_eq!(metadata.name, "dstack-volume"); + assert_eq!(data_partition.name, "dstack-data"); + assert_eq!(hash_partition.name, "dstack-verity"); assert_eq!( - p1.part_guid, + data_partition.part_guid, uuid_from_material(&[b"dstack-verity-data", root_hash.as_bytes()]) ); assert_eq!( - p2.part_guid, + hash_partition.part_guid, uuid_from_material(&[b"dstack-verity-hash", root_hash.as_bytes()]) ); - assert_eq!(p1.first_lba, PARTITION_ALIGNMENT_SECTORS); - assert_eq!(p2.first_lba, PARTITION_ALIGNMENT_SECTORS * 2); + assert_eq!(metadata.first_lba, PARTITION_ALIGNMENT_SECTORS); + assert_eq!(data_partition.first_lba, PARTITION_ALIGNMENT_SECTORS * 2); + assert_eq!(hash_partition.first_lba, PARTITION_ALIGNMENT_SECTORS * 3); let mut img = fs::File::open(&image_path)?; + let mut header = [0u8; VOLUME_HEADER_SIZE]; + img.seek(SeekFrom::Start(metadata.first_lba * SECTOR))?; + img.read_exact(&mut header)?; + assert_eq!(&header[..16], VOLUME_MAGIC); + assert_eq!(&header[32..64], &hex::decode(root_hash)?); let mut buf = vec![0; data.len()]; - img.seek(SeekFrom::Start(p1.first_lba * SECTOR))?; + img.seek(SeekFrom::Start(data_partition.first_lba * SECTOR))?; img.read_exact(&mut buf)?; assert_eq!(buf, data); let mut buf = vec![0; hash.len()]; - img.seek(SeekFrom::Start(p2.first_lba * SECTOR))?; + img.seek(SeekFrom::Start(hash_partition.first_lba * SECTOR))?; img.read_exact(&mut buf)?; assert_eq!(buf, hash); diff --git a/dstack/guest-agent/src/bin/dstack-volume.rs b/dstack/guest-agent/src/bin/dstack-volume.rs new file mode 100644 index 000000000..ea2ebce3f --- /dev/null +++ b/dstack/guest-agent/src/bin/dstack-volume.rs @@ -0,0 +1,578 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! Discover and activate volumes supplied to a dstack guest. +//! +//! A volume is recognized by a `DSTACK_VOLUME` envelope at the start of its +//! first partition, or at the start of the whole disk when it has no partition +//! table. Everything read from a disk is untrusted: kind handlers use the +//! measured app compose as their source of policy and cryptographic identity. + +use std::collections::HashSet; +use std::ffi::OsStr; +use std::fs::{self, File}; +use std::io::Read; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; + +use anyhow::{bail, Context, Result}; +use serde::Deserialize; + +const MAGIC: &[u8; 16] = b"DSTACK_VOLUME\0\0\0"; +const HEADER_SIZE: usize = 4096; +const FORMAT_VERSION: u16 = 1; +const KIND_VERITY: u32 = 1; +const MAX_DISKS: usize = 64; +const DOCKER_STORE: &str = "/var/lib/docker"; + +#[derive(Debug, Deserialize)] +struct AppCompose { + #[serde(default)] + verity_volumes: Vec, +} + +#[derive(Debug, Deserialize)] +struct RequestedVolume { + verity_root: String, + target: String, +} + +#[derive(Clone, Debug)] +struct BlockDisk { + path: PathBuf, + partitions: Vec<(u32, PathBuf)>, +} + +#[derive(Debug, PartialEq, Eq)] +struct VolumeHeader { + kind: u32, + kind_version: u32, + flags: u32, + root_hash: [u8; 32], + data_block_size: u32, + hash_block_size: u32, +} + +#[derive(Debug)] +struct VerityVolume { + data: PathBuf, + hash: PathBuf, + root_hash: [u8; 32], +} + +fn main() -> Result<()> { + let compose_path = std::env::args_os() + .nth(1) + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from("app-compose.json")); + let compose: AppCompose = serde_json::from_slice( + &fs::read(&compose_path).with_context(|| format!("reading {}", compose_path.display()))?, + ) + .with_context(|| format!("parsing {}", compose_path.display()))?; + if compose.verity_volumes.is_empty() { + return Ok(()); + } + + let _ = run(Command::new("modprobe").arg("dm-verity")); + let _ = run(Command::new("udevadm").args(["settle", "--timeout=5"])); + + let volumes = discover_volumes()?; + eprintln!( + "dstack-volume: found {} volume(s), {} requested", + volumes.len(), + compose.verity_volumes.len() + ); + let mut used = HashSet::new(); + for (index, requested) in compose.verity_volumes.iter().enumerate() { + if let Err(err) = activate_requested(index, requested, &volumes, &mut used) { + eprintln!( + "dstack-volume: volume {index} ({}): {err:#}; skipping", + requested.target + ); + } + } + Ok(()) +} + +fn discover_volumes() -> Result> { + let mut found = Vec::new(); + for disk in list_disks()?.into_iter().take(MAX_DISKS) { + // A partitioned disk has exactly one envelope location: partition 1. + // Only a disk with no kernel-recognized partitions is probed at offset 0. + let probe = if disk.partitions.is_empty() { + &disk.path + } else if let Some((_, path)) = disk.partitions.iter().find(|(number, _)| *number == 1) { + path + } else { + continue; + }; + let Some(header) = read_header(probe)? else { + continue; + }; + match header.kind { + KIND_VERITY => match resolve_verity(&disk, header) { + Ok(volume) => found.push(volume), + Err(err) => eprintln!( + "dstack-volume: ignoring malformed verity volume {}: {err:#}", + disk.path.display() + ), + }, + kind => eprintln!( + "dstack-volume: ignoring unsupported kind {kind} on {}", + disk.path.display() + ), + } + } + Ok(found) +} + +fn list_disks() -> Result> { + let output = checked( + Command::new("lsblk").args(["-rnbpo", "PATH,TYPE,PARTN"]), + "listing block devices", + )?; + let text = String::from_utf8(output.stdout).context("lsblk output is not UTF-8")?; + let mut disks = Vec::::new(); + let mut current = None; + for line in text.lines() { + let mut fields = line.split_whitespace(); + let Some(path) = fields.next() else { continue }; + let Some(kind) = fields.next() else { continue }; + match kind { + "disk" => { + disks.push(BlockDisk { + path: PathBuf::from(path), + partitions: Vec::new(), + }); + current = Some(disks.len() - 1); + } + "part" => { + let Some(number) = fields.next().and_then(|v| v.parse::().ok()) else { + continue; + }; + // lsblk -p emits children immediately below their parent disk. + if let Some(index) = current { + disks[index].partitions.push((number, PathBuf::from(path))); + } + } + _ => {} + } + } + for disk in &mut disks { + disk.partitions.sort_by_key(|(number, _)| *number); + } + Ok(disks) +} + +fn read_header(path: &Path) -> Result> { + let mut file = match File::open(path) { + Ok(file) => file, + Err(err) => { + eprintln!("dstack-volume: cannot open {}: {err}", path.display()); + return Ok(None); + } + }; + let mut bytes = [0u8; HEADER_SIZE]; + if let Err(err) = file.read_exact(&mut bytes) { + if err.kind() == std::io::ErrorKind::UnexpectedEof { + return Ok(None); + } + return Err(err).with_context(|| format!("reading {}", path.display())); + } + parse_header(&bytes).with_context(|| format!("parsing envelope on {}", path.display())) +} + +fn parse_header(bytes: &[u8]) -> Result> { + if bytes.len() < HEADER_SIZE || &bytes[..16] != MAGIC { + return Ok(None); + } + let version = u16::from_le_bytes(bytes[16..18].try_into()?); + let header_size = u16::from_le_bytes(bytes[18..20].try_into()?) as usize; + if version != FORMAT_VERSION { + bail!("unsupported envelope version {version}"); + } + if header_size != HEADER_SIZE { + bail!("invalid header size {header_size}"); + } + let root_hash = bytes[32..64].try_into()?; + Ok(Some(VolumeHeader { + kind: u32::from_le_bytes(bytes[20..24].try_into()?), + kind_version: u32::from_le_bytes(bytes[24..28].try_into()?), + flags: u32::from_le_bytes(bytes[28..32].try_into()?), + root_hash, + data_block_size: u32::from_le_bytes(bytes[64..68].try_into()?), + hash_block_size: u32::from_le_bytes(bytes[68..72].try_into()?), + })) +} + +fn resolve_verity(disk: &BlockDisk, header: VolumeHeader) -> Result { + if header.kind_version != 1 { + bail!("unsupported verity version {}", header.kind_version); + } + if header.flags != 0 { + bail!("unsupported verity flags {:#x}", header.flags); + } + if header.data_block_size != 4096 || header.hash_block_size != 4096 { + bail!( + "unsupported verity block sizes {}/{}", + header.data_block_size, + header.hash_block_size + ); + } + if disk.partitions.is_empty() { + bail!("raw verity layout is not defined by kind version 1"); + } + let partition = |number| { + disk.partitions + .iter() + .find(|(n, _)| *n == number) + .map(|(_, path)| path.clone()) + .with_context(|| format!("missing partition {number}")) + }; + Ok(VerityVolume { + data: partition(2)?, + hash: partition(3)?, + root_hash: header.root_hash, + }) +} + +fn activate_requested( + index: usize, + requested: &RequestedVolume, + volumes: &[VerityVolume], + used: &mut HashSet, +) -> Result<()> { + let expected = hex::decode(&requested.verity_root).context("invalid verity_root hex")?; + if expected.len() != 32 { + bail!("verity_root must contain 32 bytes"); + } + let candidate = volumes + .iter() + .enumerate() + .find(|(candidate_index, volume)| { + !used.contains(candidate_index) && volume.root_hash.as_slice() == expected + }) + .context("no attached volume advertises the measured root")?; + + let mapper_name = format!("dstack-verity{index}"); + // The on-disk root only selected a candidate. Pass the root from the + // measured compose to veritysetup, which is the actual trust decision. + checked( + Command::new("veritysetup") + .arg("open") + .arg(&candidate.1.data) + .arg(&mapper_name) + .arg(&candidate.1.hash) + .arg(&requested.verity_root), + "opening dm-verity volume", + )?; + let mapped = PathBuf::from(format!("/dev/mapper/{mapper_name}")); + if let Err(err) = + verify_first_block(&mapped).and_then(|_| mount_volume(index, requested, &mapped)) + { + let _ = run(Command::new("veritysetup").args(["close", &mapper_name])); + return Err(err); + } + used.insert(candidate.0); + Ok(()) +} + +fn verify_first_block(path: &Path) -> Result<()> { + let mut file = File::open(path).with_context(|| format!("opening {}", path.display()))?; + let mut block = [0u8; 4096]; + file.read_exact(&mut block) + .with_context(|| format!("verifying first block of {}", path.display())) +} + +fn mount_volume(index: usize, requested: &RequestedVolume, mapped: &Path) -> Result<()> { + let fs_type = command_stdout( + Command::new("blkid") + .args(["-o", "value", "-s", "TYPE"]) + .arg(mapped), + ) + .unwrap_or_default(); + if requested.target == "docker" { + let mountpoint = PathBuf::from(format!("/run/dstack-verity/{index}")); + fs::create_dir_all(&mountpoint)?; + mount_read_only(mapped, &mountpoint, fs_type.trim())?; + if let Err(err) = seed_docker(&mountpoint) { + let _ = run(Command::new("umount").arg(&mountpoint)); + return Err(err); + } + eprintln!( + "dstack-volume: seeded docker from {}", + requested.verity_root + ); + } else { + let target = Path::new(&requested.target); + if !target.is_absolute() { + bail!("mount target must be absolute"); + } + fs::create_dir_all(target).with_context(|| format!("creating {}", target.display()))?; + mount_read_only(mapped, target, fs_type.trim())?; + eprintln!( + "dstack-volume: mounted {} at {}", + requested.verity_root, + target.display() + ); + } + Ok(()) +} + +fn mount_read_only(device: &Path, target: &Path, fs_type: &str) -> Result<()> { + let options = if matches!(fs_type, "ext3" | "ext4") { + "ro,noload" + } else { + "ro" + }; + let mut command = Command::new("mount"); + if !fs_type.is_empty() { + command.args(["-t", fs_type]); + } + checked( + command.arg("-o").arg(options).arg(device).arg(target), + "mounting verity volume", + )?; + Ok(()) +} + +fn seed_docker(volume: &Path) -> Result<()> { + let store = Path::new(DOCKER_STORE); + let overlay = volume.join("overlay2"); + let mut bound = Vec::new(); + for layer in child_directories(&overlay)? { + if layer.file_name() == Some(OsStr::new("l")) { + continue; + } + let source = layer.join("diff"); + if !source.is_dir() { + continue; + } + let target = store + .join("overlay2") + .join(layer.file_name().unwrap()) + .join("diff"); + fs::create_dir_all(&target)?; + if !is_mountpoint(&target)? { + if let Err(err) = checked( + Command::new("mount") + .args(["--bind"]) + .arg(&source) + .arg(&target), + "binding docker layer", + ) { + unwind_binds(&bound); + return Err(err); + } + // A bind inherits neither the intended policy nor all mount flags. + if let Err(err) = checked( + Command::new("mount") + .args(["-o", "remount,bind,ro"]) + .arg(&target), + "making docker layer read-only", + ) { + bound.push(target); + unwind_binds(&bound); + return Err(err); + } + bound.push(target); + } + } + + let copy_result = (|| -> Result<()> { + copy_contents( + &volume.join("image/overlay2/imagedb"), + &store.join("image/overlay2/imagedb"), + )?; + copy_contents( + &volume.join("image/overlay2/layerdb"), + &store.join("image/overlay2/layerdb"), + )?; + copy_contents(&volume.join("overlay2/l"), &store.join("overlay2/l"))?; + merge_repositories(volume, store)?; + copy_layer_metadata(volume, store) + })(); + if let Err(err) = copy_result { + unwind_binds(&bound); + return Err(err); + } + Ok(()) +} + +fn child_directories(path: &Path) -> Result> { + let mut result = Vec::new(); + for entry in fs::read_dir(path).with_context(|| format!("reading {}", path.display()))? { + let path = entry?.path(); + if path.is_dir() { + result.push(path); + } + } + result.sort(); + Ok(result) +} + +fn copy_contents(source: &Path, target: &Path) -> Result<()> { + fs::create_dir_all(target)?; + checked( + Command::new("cp") + .args(["-a"]) + .arg(source.join(".")) + .arg(target), + "copying docker metadata", + )?; + Ok(()) +} + +fn merge_repositories(volume: &Path, store: &Path) -> Result<()> { + let source = volume.join("image/overlay2/repositories.json"); + let target = store.join("image/overlay2/repositories.json"); + if target.exists() { + let temporary = target.with_extension("json.dstack-tmp"); + let output = checked( + Command::new("jq") + .args(["-s", ".[0] * .[1]"]) + .arg(&target) + .arg(&source), + "merging docker repositories", + )?; + fs::write(&temporary, output.stdout)?; + fs::rename(&temporary, &target)?; + } else { + fs::copy(&source, &target)?; + } + Ok(()) +} + +fn copy_layer_metadata(volume: &Path, store: &Path) -> Result<()> { + for layer in child_directories(&volume.join("overlay2"))? { + let Some(id) = layer.file_name() else { + continue; + }; + if id == OsStr::new("l") { + continue; + } + let target = store.join("overlay2").join(id); + if target.join("link").exists() { + continue; + } + fs::create_dir_all(&target)?; + for entry in fs::read_dir(&layer)? { + let source = entry?.path(); + if source.file_name() == Some(OsStr::new("diff")) { + continue; + } + checked( + Command::new("cp").args(["-a"]).arg(&source).arg(&target), + "copying docker layer metadata", + )?; + } + } + Ok(()) +} + +fn is_mountpoint(path: &Path) -> Result { + let needle = format!(" {} ", path.display()); + Ok(fs::read_to_string("/proc/self/mountinfo")? + .lines() + .any(|line| line.contains(&needle))) +} + +fn unwind_binds(paths: &[PathBuf]) { + for path in paths.iter().rev() { + let _ = run(Command::new("umount").arg(path)); + } +} + +fn command_stdout(command: &mut Command) -> Result { + let output = command.output()?; + if !output.status.success() { + bail!("command failed with {}", output.status); + } + Ok(String::from_utf8(output.stdout)?) +} + +fn run(command: &mut Command) -> Result { + command.output().context("running command") +} + +fn checked(command: &mut Command, operation: &str) -> Result { + let output = command.output().with_context(|| operation.to_string())?; + if !output.status.success() { + bail!( + "{operation} failed with {}: {}", + output.status, + String::from_utf8_lossy(&output.stderr).trim() + ); + } + Ok(output) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn header() -> [u8; HEADER_SIZE] { + let mut bytes = [0u8; HEADER_SIZE]; + bytes[..16].copy_from_slice(MAGIC); + bytes[16..18].copy_from_slice(&1u16.to_le_bytes()); + bytes[18..20].copy_from_slice(&(HEADER_SIZE as u16).to_le_bytes()); + bytes[20..24].copy_from_slice(&KIND_VERITY.to_le_bytes()); + bytes[24..28].copy_from_slice(&1u32.to_le_bytes()); + bytes[32..64].fill(0x5a); + bytes[64..68].copy_from_slice(&4096u32.to_le_bytes()); + bytes[68..72].copy_from_slice(&4096u32.to_le_bytes()); + bytes + } + + #[test] + fn parses_volume_envelope() { + assert_eq!( + parse_header(&header()).unwrap(), + Some(VolumeHeader { + kind: KIND_VERITY, + kind_version: 1, + flags: 0, + root_hash: [0x5a; 32], + data_block_size: 4096, + hash_block_size: 4096, + }) + ); + } + + #[test] + fn ignores_non_volume_data() { + assert_eq!(parse_header(&[0u8; HEADER_SIZE]).unwrap(), None); + } + + #[test] + fn rejects_unknown_envelope_version() { + let mut bytes = header(); + bytes[16..18].copy_from_slice(&2u16.to_le_bytes()); + assert!(parse_header(&bytes).is_err()); + } + + #[test] + fn verity_kind_uses_second_and_third_partitions() { + let disk = BlockDisk { + path: "/dev/test".into(), + partitions: vec![ + (1, "/dev/test1".into()), + (2, "/dev/test2".into()), + (3, "/dev/test3".into()), + ], + }; + let volume = resolve_verity(&disk, parse_header(&header()).unwrap().unwrap()).unwrap(); + assert_eq!(volume.data, Path::new("/dev/test2")); + assert_eq!(volume.hash, Path::new("/dev/test3")); + } + + #[test] + fn verity_kind_rejects_undefined_raw_layout() { + let disk = BlockDisk { + path: "/dev/test".into(), + partitions: vec![], + }; + assert!(resolve_verity(&disk, parse_header(&header()).unwrap().unwrap()).is_err()); + } +} diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 1096caf08..ea5c471a4 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -45,7 +45,6 @@ mod network; mod qemu; pub(crate) mod registry; mod vm_info; -mod volume; mod workdir; fn signal_pidfd(pid: u32, signal: libc::c_int) -> std::io::Result<()> { diff --git a/dstack/vmm/src/app/qemu.rs b/dstack/vmm/src/app/qemu.rs index 84352424a..30d9c8e5f 100644 --- a/dstack/vmm/src/app/qemu.rs +++ b/dstack/vmm/src/app/qemu.rs @@ -24,9 +24,7 @@ use super::{ image::Image, mr_config::{snp_host_data, tdx_mr_config_id}, network::{mac_address_for_vm_index, resolved_networks, validate_resolved_networks}, - pci_numa_node, round_up, - volume::verity_serial_hint, - GpuConfig, VmWorkDir, + pci_numa_node, round_up, GpuConfig, VmWorkDir, }; use anyhow::{bail, Context, Result}; use bon::Builder; @@ -179,7 +177,6 @@ fn virtio_pci_device(device: &str, snp: bool) -> String { struct PreparedVolume { source: String, read_only: bool, - serial: Option, } struct PreparedQemuLaunch { @@ -218,9 +215,6 @@ impl PreparedQemuLaunch { .map(|volume| PreparedVolume { source: volume.source.clone(), read_only: volume.read_only, - // File inspection belongs to launch preparation. Keep the - // command builder host-I/O-free so it remains deterministic. - serial: verity_serial_hint(Path::new(&volume.source)), }) .collect(); @@ -561,12 +555,7 @@ impl QemuCommandBuilder<'_> { drive.push_str(",readonly=on"); } - // The serial is only a lookup hint. The guest still verifies the - // complete dm-verity root before using the volume. - let mut device = format!("virtio-blk-pci,drive={id}"); - if let Some(serial) = &volume.serial { - device.push_str(&format!(",serial={serial}")); - } + let device = format!("virtio-blk-pci,drive={id}"); command .arg("-drive") .arg(drive) @@ -1098,7 +1087,6 @@ mod tests { volumes: vec![PreparedVolume { source: "/does-not-exist/volume.img".into(), read_only: true, - serial: Some("0123456789abcdef0123".into()), }], hugepage_numa_nodes: None, gpu_numa_nodes: HashMap::new(), @@ -1141,7 +1129,7 @@ mod tests { assert!(process .args .iter() - .any(|arg| { arg == "virtio-blk-pci,drive=vol0,serial=0123456789abcdef0123" })); + .any(|arg| { arg == "virtio-blk-pci,drive=vol0" })); let volume_position = process .args .iter() diff --git a/dstack/vmm/src/app/volume.rs b/dstack/vmm/src/app/volume.rs deleted file mode 100644 index 138e66b29..000000000 --- a/dstack/vmm/src/app/volume.rs +++ /dev/null @@ -1,258 +0,0 @@ -// SPDX-FileCopyrightText: © 2026 Phala Network -// -// SPDX-License-Identifier: Apache-2.0 - -//! Verity-volume inspection used while preparing a QEMU launch. - -use std::io::{Read, Seek, SeekFrom}; -use std::path::Path; - -use fs_err as fs; -use sha2::{Digest, Sha256}; - -/// Compute the virtio-blk `serial` hint for a pre-baked verity volume: the first -/// 20 hex chars of its dm-verity root hash (virtio caps serials at 20 bytes). -/// -/// The guest uses this to open the matching disk directly instead of probing -/// every disk. It is only a hint. The guest still verifies the full root, so a -/// missing or wrong serial just falls back to probing. -/// -/// The current volume format is a raw GPT disk: p1 is verity data, p2 is the -/// verity hash device. The old single-file `[squashfs][verity]` layout is still -/// recognized as a fallback so older volumes can still get the serial hint. -/// -/// For multi-block data devices, the root is `sha256(salt || top hash block)`, -/// and the top block sits right after the verity superblock, so we never walk -/// the hash tree. A one-block data device has no hash blocks, so the root is -/// `sha256(salt || data block)`. -pub(super) fn verity_serial_hint(path: &Path) -> Option { - let mut file = fs::File::open(path).ok()?; - partitioned_verity_serial_hint(&mut file).or_else(|| legacy_verity_serial_hint(&mut file)) -} - -fn partitioned_verity_serial_hint(file: &mut fs::File) -> Option { - let (data_offset, hash_offset) = gpt_data_hash_offsets(file)?; - root_prefix_from_verity_devices(file, data_offset, hash_offset) -} - -fn legacy_verity_serial_hint(file: &mut fs::File) -> Option { - file.seek(SeekFrom::Start(0)).ok()?; - - // squashfs superblock: "hsqs" magic at 0, bytes_used (LE u64) at 40. The - // verity hash tree starts just past there, block-aligned. - let mut magic = [0u8; 4]; - file.read_exact(&mut magic).ok()?; - if &magic != b"hsqs" { - return None; - } - file.seek(SeekFrom::Start(40)).ok()?; - let mut buf8 = [0u8; 8]; - file.read_exact(&mut buf8).ok()?; - let hash_offset = u64::from_le_bytes(buf8).div_ceil(4096).checked_mul(4096)?; - root_prefix_from_verity_devices(file, 0, hash_offset) -} - -fn gpt_data_hash_offsets(file: &mut fs::File) -> Option<(u64, u64)> { - file.seek(SeekFrom::Start(512)).ok()?; - let mut header = [0u8; 512]; - file.read_exact(&mut header).ok()?; - if &header[0..8] != b"EFI PART" { - return None; - } - - let entries_lba = u64::from_le_bytes(header[72..80].try_into().ok()?); - let entry_count = u32::from_le_bytes(header[80..84].try_into().ok()?); - let entry_size = u32::from_le_bytes(header[84..88].try_into().ok()?) as usize; - if entry_count < 2 || !(128..=4096).contains(&entry_size) { - return None; - } - - let entries_offset = entries_lba.checked_mul(512)?; - let data = gpt_partition_first_lba(file, entries_offset, entry_size, 0)?; - let hash = gpt_partition_first_lba(file, entries_offset, entry_size, 1)?; - Some((data.checked_mul(512)?, hash.checked_mul(512)?)) -} - -fn gpt_partition_first_lba( - file: &mut fs::File, - entries_offset: u64, - entry_size: usize, - index: u64, -) -> Option { - let offset = entries_offset.checked_add(index.checked_mul(entry_size as u64)?)?; - file.seek(SeekFrom::Start(offset)).ok()?; - let mut entry = vec![0u8; entry_size]; - file.read_exact(&mut entry).ok()?; - if entry[0..16].iter().all(|&byte| byte == 0) { - return None; - } - Some(u64::from_le_bytes(entry[32..40].try_into().ok()?)) -} - -fn root_prefix_from_verity_devices( - file: &mut fs::File, - data_offset: u64, - hash_offset: u64, -) -> Option { - file.seek(SeekFrom::Start(hash_offset)).ok()?; - let mut superblock = [0u8; 512]; - file.read_exact(&mut superblock).ok()?; - if &superblock[0..8] != b"verity\0\0" || !superblock[32..64].starts_with(b"sha256\0") { - return None; - } - // The builder always writes 4096-byte blocks; requiring it also caps the - // block allocation below at one page for an untrusted file. - let data_block_size = u32::from_le_bytes(superblock[64..68].try_into().ok()?) as u64; - let hash_block_size = u32::from_le_bytes(superblock[68..72].try_into().ok()?) as u64; - let data_blocks = u64::from_le_bytes(superblock[72..80].try_into().ok()?); - let salt_size = u16::from_le_bytes(superblock[80..82].try_into().ok()?) as usize; - if data_block_size != 4096 || hash_block_size != 4096 || data_blocks == 0 || salt_size > 256 { - return None; - } - let salt = &superblock[88..88 + salt_size]; - - let (block_offset, block_size) = if data_blocks == 1 { - (data_offset, data_block_size) - } else { - (hash_offset.checked_add(hash_block_size)?, hash_block_size) - }; - file.seek(SeekFrom::Start(block_offset)).ok()?; - let mut block = vec![0u8; block_size as usize]; - file.read_exact(&mut block).ok()?; - - let mut hasher = Sha256::new(); - hasher.update(salt); - hasher.update(&block); - Some(hex::encode(hasher.finalize())[..20].to_string()) -} - -#[cfg(test)] -mod tests { - use std::io::{Seek, SeekFrom, Write}; - - use sha2::{Digest, Sha256}; - - use super::verity_serial_hint; - - fn write_minimal_gpt(file: &mut tempfile::NamedTempFile, data_lba: u64, hash_lba: u64) { - let mut header = [0u8; 512]; - header[0..8].copy_from_slice(b"EFI PART"); - header[72..80].copy_from_slice(&2u64.to_le_bytes()); // partition entries LBA - header[80..84].copy_from_slice(&128u32.to_le_bytes()); // entries - header[84..88].copy_from_slice(&128u32.to_le_bytes()); // entry size - file.seek(SeekFrom::Start(512)).unwrap(); - file.write_all(&header).unwrap(); - - let mut entries = vec![0u8; 128 * 128]; - // Any non-zero type GUID is enough for the parser. - entries[0] = 1; - entries[32..40].copy_from_slice(&data_lba.to_le_bytes()); - let second = 128; - entries[second] = 1; - entries[second + 32..second + 40].copy_from_slice(&hash_lba.to_le_bytes()); - file.seek(SeekFrom::Start(2 * 512)).unwrap(); - file.write_all(&entries).unwrap(); - } - - fn verity_superblock(data_blocks: u64, salt: &[u8]) -> [u8; 4096] { - let mut superblock = [0u8; 4096]; - superblock[0..8].copy_from_slice(b"verity\0\0"); - superblock[32..38].copy_from_slice(b"sha256"); - superblock[64..68].copy_from_slice(&4096u32.to_le_bytes()); - superblock[68..72].copy_from_slice(&4096u32.to_le_bytes()); - superblock[72..80].copy_from_slice(&data_blocks.to_le_bytes()); - superblock[80..82].copy_from_slice(&(salt.len() as u16).to_le_bytes()); - superblock[88..88 + salt.len()].copy_from_slice(salt); - superblock - } - - fn test_verity_salt() -> [u8; 32] { - // Use tempfile's random name as a per-test seed. The salt is fixture - // data rather than a secret, but deriving it at runtime also keeps - // security scanners from mistaking a fixed test vector for a - // production cryptographic salt. - let seed = tempfile::NamedTempFile::new().unwrap(); - Sha256::digest(seed.path().as_os_str().as_encoded_bytes()).into() - } - - #[test] - fn legacy_verity_serial_hint_matches_root_prefix() { - // A minimal squashfs+verity file: "hsqs" with bytes_used = 8192, then a - // verity superblock at 8192 and its top hash block at 12288. - let mut image = vec![0u8; 8192]; - image[0..4].copy_from_slice(b"hsqs"); - image[40..48].copy_from_slice(&8192u64.to_le_bytes()); - - let salt = test_verity_salt(); - image.extend_from_slice(&verity_superblock(2, &salt)); - - let top_block = vec![0x5au8; 4096]; - image.extend_from_slice(&top_block); - - let mut file = tempfile::NamedTempFile::new().unwrap(); - file.write_all(&image).unwrap(); - - let mut hasher = Sha256::new(); - hasher.update(salt); - hasher.update(&top_block); - let expected = hex::encode(hasher.finalize()); - - let got = verity_serial_hint(file.path()).unwrap(); - assert_eq!(got, expected[..20]); - assert_eq!(got.len(), 20); - } - - #[test] - fn partitioned_verity_serial_hint_matches_root_prefix() { - let data_lba = 2048u64; - let hash_lba = 4096u64; - let salt = test_verity_salt(); - let top_block = vec![0x5au8; 4096]; - - let mut file = tempfile::NamedTempFile::new().unwrap(); - file.as_file_mut().set_len((hash_lba + 16) * 512).unwrap(); - write_minimal_gpt(&mut file, data_lba, hash_lba); - file.seek(SeekFrom::Start(hash_lba * 512)).unwrap(); - file.write_all(&verity_superblock(2, &salt)).unwrap(); - file.write_all(&top_block).unwrap(); - - let mut hasher = Sha256::new(); - hasher.update(salt); - hasher.update(&top_block); - let expected = hex::encode(hasher.finalize()); - - let got = verity_serial_hint(file.path()).unwrap(); - assert_eq!(got, expected[..20]); - } - - #[test] - fn partitioned_one_block_verity_serial_hint_hashes_data_block() { - let data_lba = 2048u64; - let hash_lba = 4096u64; - let salt = test_verity_salt(); - let data_block = vec![0x42u8; 4096]; - - let mut file = tempfile::NamedTempFile::new().unwrap(); - file.as_file_mut().set_len((hash_lba + 8) * 512).unwrap(); - write_minimal_gpt(&mut file, data_lba, hash_lba); - file.seek(SeekFrom::Start(data_lba * 512)).unwrap(); - file.write_all(&data_block).unwrap(); - file.seek(SeekFrom::Start(hash_lba * 512)).unwrap(); - file.write_all(&verity_superblock(1, &salt)).unwrap(); - - let mut hasher = Sha256::new(); - hasher.update(salt); - hasher.update(&data_block); - let expected = hex::encode(hasher.finalize()); - - let got = verity_serial_hint(file.path()).unwrap(); - assert_eq!(got, expected[..20]); - } - - #[test] - fn verity_serial_hint_skips_non_verity_files() { - let mut file = tempfile::NamedTempFile::new().unwrap(); - file.write_all(b"not a squashfs image").unwrap(); - assert_eq!(verity_serial_hint(file.path()), None); - } -} diff --git a/os/common/rootfs/dstack-prepare.sh b/os/common/rootfs/dstack-prepare.sh index 90c11c12e..877674e13 100755 --- a/os/common/rootfs/dstack-prepare.sh +++ b/os/common/rootfs/dstack-prepare.sh @@ -301,8 +301,8 @@ echo "============================" cd /dstack # Seed pre-baked verity volumes (docker images / data) before dockerd starts, so -# the images are already present with no pull. Fail-safe -- see dstack-verity.sh. -/bin/dstack-verity.sh app-compose.json || log "dstack-verity failed (continuing; images will pull normally)" +# the images are already present with no pull. Fail-safe -- see dstack-volume. +/bin/dstack-volume app-compose.json || log "dstack-volume failed (continuing; images will pull normally)" if [ "$(jq 'has("init_script")' app-compose.json)" == true ]; then log "Running init script" diff --git a/os/common/rootfs/dstack-verity.sh b/os/common/rootfs/dstack-verity.sh deleted file mode 100755 index 9825cef1e..000000000 --- a/os/common/rootfs/dstack-verity.sh +++ /dev/null @@ -1,247 +0,0 @@ -#!/bin/bash -# SPDX-FileCopyrightText: © 2025 Phala Network -# -# SPDX-License-Identifier: Apache-2.0 -# -# Mount the verity volumes declared in app-compose.json and seed them into the CVM. -# -# Each volume has a target. A "docker" target seeds the docker overlay2 store, so -# its images become cache hits. A "/path" target mounts the volume's filesystem -# there, for data like model weights. -# -# This runs from dstack-prepare.sh before dockerd starts, so there is no restart. -# -# The host only supplies the bytes. Each volume is matched and verified against -# the measured verity_root from app-compose. Everything here is fail-safe: a -# volume that is missing or fails verification is skipped, and its images pull -# normally. See docs/verity-volumes.md. - -COMPOSE="${1:-app-compose.json}" -STORE=/var/lib/docker - -log() { echo "dstack-verity: $*"; } - -# Nothing to do unless the compose file declares verity_volumes. -[ -f "$COMPOSE" ] || exit 0 -count=$(jq '(.verity_volumes // []) | length' "$COMPOSE" 2>/dev/null || echo 0) -[ "${count:-0}" -gt 0 ] || exit 0 -log "$count verity volume(s) requested" - -if ! command -v veritysetup >/dev/null 2>&1; then - log "veritysetup missing; skipping (images will pull)" - exit 0 -fi -modprobe dm-verity 2>/dev/null || true - -# Candidate whole-disk block devices. A dstack verity volume is a GPT disk with: -# partition 1 = filesystem data -# partition 2 = dm-verity superblock + hash tree -# -# Volumes are identified by root hash, not by name or attach order. The host tags -# each verity disk's serial with the root prefix, so the matching disk can be -# opened directly; a missing or wrong tag falls back to trying every disk. -mapfile -t DEVS < <(lsblk -dnro NAME,TYPE 2>/dev/null | awk '$2=="disk"{print "/dev/"$1}') - -# Return the Nth partition device path for a whole disk. -partition_n() { # $1 = disk, $2 = 1-based index - lsblk -nrpo NAME,TYPE "$1" 2>/dev/null | - awk -v want="$2" '$2=="part"{i++; if (i==want) {print $1; exit}}' -} - -# Detect the filesystem only after dm-verity is active, so filesystem metadata is -# read through /dev/mapper and is covered by the measured root. -fs_type() { # $1 = verity-mapped device -> fstype | (empty) - blkid -o value -s TYPE "$1" 2>/dev/null || true -} - -# Probe every candidate disk once, recording "disk data_part hash_part serial". -# The partition table is untrusted, but that is safe: a tampered table just makes -# veritysetup open the wrong data/hash pair, which fails against the measured -# root. Doing this once keeps per-volume matching cheap. -DEV_INFO=() -scan_devices() { - local dev data hash serial - if command -v udevadm >/dev/null 2>&1; then - udevadm settle --timeout=5 2>/dev/null || true - fi - for dev in "${DEVS[@]}"; do - data=$(partition_n "$dev" 1) - hash=$(partition_n "$dev" 2) - if ! { [ -b "$data" ] && [ -b "$hash" ]; }; then - continue - fi - serial=$(cat "/sys/block/$(basename "$dev")/serial" 2>/dev/null) - DEV_INFO+=("$dev $data $hash $serial") - done -} - -# Open $data+$hash as /dev/mapper/$name if it verifies against $root. -# On success, echoes "/dev/mapper/$name ". -try_open() { # $1 = data_part, $2 = hash_part, $3 = root, $4 = name - local data="$1" hash="$2" root="$3" name="$4" fs - veritysetup open "$data" "$name" "$hash" "$root" >/dev/null 2>&1 || return 1 - # Confirm the root matches: reading through verity must not fault. - if dd if="/dev/mapper/$name" of=/dev/null bs=4096 count=1 >/dev/null 2>&1; then - fs=$(fs_type "/dev/mapper/$name") - echo "/dev/mapper/$name $fs" - return 0 - fi - veritysetup close "$name" >/dev/null 2>&1 - return 1 -} - -# Open the volume matching $root as /dev/mapper/$name. -# -# First try the disk the host tagged for this root (its serial is the root -# prefix). That is the common path: one open, no probing. If nothing matches the -# tag, or it fails to verify, fall back to trying every disk. -open_volume() { # $1 = root hash, $2 = mapper name - local root="$1" name="$2" entry dev data hash serial - for entry in "${DEV_INFO[@]}"; do - read -r dev data hash serial <<<"$entry" - if [ -z "$serial" ] || [ "$serial" != "${root:0:20}" ]; then - continue - fi - try_open "$data" "$hash" "$root" "$name" && return 0 - done - for entry in "${DEV_INFO[@]}"; do - read -r dev data hash serial <<<"$entry" - try_open "$data" "$hash" "$root" "$name" && return 0 - done - return 1 -} - -# Mount a verity-mapped device read-only, choosing per-fs options. -mount_ro() { # $1 = device, $2 = fs_type, $3 = mountpoint - case "$2" in - # noload: skip journal replay -- the device is read-only. - ext3|ext4) mount -t "$2" -o ro,noload "$1" "$3" 2>/dev/null ;; - ext2) mount -t ext2 -o ro "$1" "$3" 2>/dev/null ;; - "") mount -o ro "$1" "$3" 2>/dev/null ;; - *) mount -t "$2" -o ro "$1" "$3" 2>/dev/null ;; - esac -} - -# True if $1 is a mountpoint (the spaces anchor the match to the whole field). -is_mounted() { grep -qsF " $1 " /proc/self/mountinfo; } - -# umount the binds this seeding made -- undoes a partial seed. A no-op on reboot, -# where the diffs are already mounted so nothing was added to $bound. -unwind_binds() { local mp; for mp in "$@"; do umount "$mp" 2>/dev/null; done; } - -# Seed the docker overlay2 store from a mounted volume. -# -# Bind-mount each big, already-extracted layer diff read-only, then copy the -# small metadata. No pull, no extraction. Contents merge into whatever is already -# in the store, so several volumes can seed one store. -# -# The diff binds are redone on every boot. /var/lib/docker is persistent, but -# bind mounts are not, so a reboot starts with empty diff/ dirs. The binds must -# go in before the metadata, or the metadata would point at empty layers. -# -# On any bind failure this copies no metadata and returns non-zero, so docker -# pulls the image cleanly instead of running it with empty layers. -seed_docker() { # $1 = mounted volume dir - local vol="$1" layer_dir id file base repo src - local bound=() - - # 1. (re)bind every layer's read-only diff. If one fails, unwind the binds - # done so far and bail, before any metadata is written. - for layer_dir in "$vol"/overlay2/*/; do - [ -d "$layer_dir" ] || continue - id=$(basename "$layer_dir") - [ "$id" = l ] && continue # `l/` is docker's symlink dir, not a layer - mkdir -p "$STORE/overlay2/$id/diff" - if ! is_mounted "$STORE/overlay2/$id/diff"; then - if mount --bind "$layer_dir/diff" "$STORE/overlay2/$id/diff"; then - bound+=("$STORE/overlay2/$id/diff") - else - unwind_binds "${bound[@]}" - return 1 - fi - fi - done - - # 2. metadata (content-addressed) -- merge by copying directory *contents* - # (a plain `cp dir store/` would nest or clobber on the second volume). On - # any copy failure (e.g. a full disk), unwind this volume's binds and bail, - # so docker pulls cleanly instead of running against a half-written store. - mkdir -p "$STORE/image/overlay2/imagedb" "$STORE/image/overlay2/layerdb" "$STORE/overlay2/l" - if ! { cp -a "$vol/image/overlay2/imagedb/." "$STORE/image/overlay2/imagedb/" && - cp -a "$vol/image/overlay2/layerdb/." "$STORE/image/overlay2/layerdb/" && - cp -a "$vol/overlay2/l/." "$STORE/overlay2/l/"; } 2>/dev/null; then - unwind_binds "${bound[@]}" - return 1 - fi - - # repositories.json is a single name->id map; merge it rather than overwrite, - # or a second volume would drop the first volume's image names. A failed merge - # (which leaves the old map, missing this volume's names) also unwinds and bails. - repo="$STORE/image/overlay2/repositories.json" - src="$vol/image/overlay2/repositories.json" - if [ -f "$repo" ] && command -v jq >/dev/null 2>&1; then - if ! { jq -s '.[0] * .[1]' "$repo" "$src" > "$repo.tmp" 2>/dev/null && mv "$repo.tmp" "$repo"; }; then - rm -f "$repo.tmp" - unwind_binds "${bound[@]}" - return 1 - fi - elif ! cp -a "$src" "$repo" 2>/dev/null; then - unwind_binds "${bound[@]}" - return 1 - fi - - # 3. per-layer small writable files (link/lower), copied once. The layer dir - # stays writable because docker writes a `committed` marker into it. - for layer_dir in "$vol"/overlay2/*/; do - [ -d "$layer_dir" ] || continue - id=$(basename "$layer_dir") - [ "$id" = l ] && continue - [ -e "$STORE/overlay2/$id/link" ] && continue - for file in "$layer_dir"*; do - base=$(basename "$file") - [ "$base" = diff ] && continue - cp -a "$file" "$STORE/overlay2/$id/" 2>/dev/null - done - done - return 0 -} - -# Scan the disks once, then match each declared volume to its disk. -scan_devices -for i in $(seq 0 $((count - 1))); do - root=$(jq -r ".verity_volumes[$i].verity_root // empty" "$COMPOSE") - target=$(jq -r ".verity_volumes[$i].target // empty" "$COMPOSE") - if [ -z "$root" ] || [ -z "$target" ]; then - log "vol $i: missing verity_root/target; skip" - continue - fi - - read -r mapped fs < <(open_volume "$root" "verity$i") - if [ -z "$mapped" ]; then - log "vol $i ($target): no attached device matches ${root:0:12}...; skip (will pull)" - continue - fi - - if [ "$target" = docker ]; then - # Mount on /run: it's a writable tmpfs, and the rootfs is read-only. - mnt="/run/dstack-verity/$i" - mkdir -p "$mnt" - if mount_ro "$mapped" "$fs" "$mnt" && seed_docker "$mnt"; then - log "vol $i: seeded docker store from ${root:0:12}..." - else - log "vol $i: seeding failed; skipping (images will pull)" - # umount first: veritysetup close fails on a still-mounted device. - umount "$mnt" 2>/dev/null - veritysetup close "verity$i" 2>/dev/null - fi - else - # The target must be on a writable fs (the rootfs is read-only), e.g. /run. - if mkdir -p "$target" 2>/dev/null && mount_ro "$mapped" "$fs" "$target"; then - log "vol $i: mounted ${root:0:12}... at $target" - else - log "vol $i: mount at $target failed (is it a writable path?); skipping" - veritysetup close "verity$i" 2>/dev/null - fi - fi -done -exit 0 diff --git a/os/yocto/layers/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/os/yocto/layers/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index 0e57b4e1d..3e62db5d0 100644 --- a/os/yocto/layers/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/os/yocto/layers/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -66,8 +66,8 @@ do_install() { install -d ${D}${sysconfdir}/systemd/journald.conf.d install -m 0755 ${CARGO_BINDIR}/dstack-util ${D}${bindir} install -m 0755 ${CARGO_BINDIR}/dstack-guest-agent ${D}${bindir} + install -m 0755 ${CARGO_BINDIR}/dstack-volume ${D}${bindir} install -m 0755 ${DSTACK_ROOTFS_FILES}/dstack-prepare.sh ${D}${bindir} - install -m 0755 ${DSTACK_ROOTFS_FILES}/dstack-verity.sh ${D}${bindir} install -m 0755 ${DSTACK_ROOTFS_FILES}/ephemeral-docker.sh ${D}${bindir} install -m 0755 ${DSTACK_ROOTFS_FILES}/wg-checker.sh ${D}${bindir} install -m 0755 ${DSTACK_ROOTFS_FILES}/app-compose.sh ${D}${bindir} From a5a54128264d7eab7153a48a303205179ebfb54d Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 20 Jul 2026 19:09:53 -0700 Subject: [PATCH 11/25] refactor(verity): share volume format with guest --- dstack/Cargo.lock | 5 + dstack/Cargo.toml | 2 +- dstack/crates/dstack-verity/Cargo.toml | 1 + dstack/crates/dstack-verity/src/volume.rs | 23 +- dstack/dstack-types/Cargo.toml | 1 + dstack/dstack-types/src/lib.rs | 52 ++- dstack/dstack-types/src/volume.rs | 114 +++++ dstack/guest-agent/src/bin/dstack-volume.rs | 465 ++++++++++++-------- 8 files changed, 453 insertions(+), 210 deletions(-) create mode 100644 dstack/dstack-types/src/volume.rs diff --git a/dstack/Cargo.lock b/dstack/Cargo.lock index c36f7f7cc..0c50128d7 100644 --- a/dstack/Cargo.lock +++ b/dstack/Cargo.lock @@ -2162,6 +2162,7 @@ dependencies = [ name = "dstack-types" version = "0.6.0" dependencies = [ + "binrw", "ciborium", "hex", "or-panic", @@ -2271,6 +2272,7 @@ name = "dstack-verity" version = "0.6.0" dependencies = [ "anyhow", + "dstack-types", "flate2", "fs-err", "futures", @@ -3182,6 +3184,9 @@ name = "hex" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +dependencies = [ + "serde", +] [[package]] name = "hex-literal" diff --git a/dstack/Cargo.toml b/dstack/Cargo.toml index 1c0ced796..78b3c9844 100644 --- a/dstack/Cargo.toml +++ b/dstack/Cargo.toml @@ -149,7 +149,7 @@ borsh = { version = "1.5.7", default-features = false, features = ["derive"] } bon = { version = "3.4.0", default-features = false } base64 = "0.22.1" bcrypt = "0.17.1" -hex = { version = "0.4.3", default-features = false } +hex = { version = "0.4.3", default-features = false, features = ["serde"] } hex_fmt = "0.3.0" hex-literal = "1.0.0" prost = "0.13.5" diff --git a/dstack/crates/dstack-verity/Cargo.toml b/dstack/crates/dstack-verity/Cargo.toml index 2581e8974..7d8db7220 100644 --- a/dstack/crates/dstack-verity/Cargo.toml +++ b/dstack/crates/dstack-verity/Cargo.toml @@ -10,6 +10,7 @@ license.workspace = true [dependencies] anyhow = { workspace = true, features = ["std"] } +dstack-types.workspace = true tokio = { workspace = true, features = ["rt"] } serde = { workspace = true, features = ["derive", "std"] } serde_json = { workspace = true, features = ["std"] } diff --git a/dstack/crates/dstack-verity/src/volume.rs b/dstack/crates/dstack-verity/src/volume.rs index 8d2a9c97f..4487af8c8 100644 --- a/dstack/crates/dstack-verity/src/volume.rs +++ b/dstack/crates/dstack-verity/src/volume.rs @@ -20,6 +20,7 @@ use std::path::Path; use std::process::Command; use anyhow::{bail, Context, Result}; +use dstack_types::volume::DstackVolumeHeader; use fs_err as fs; use sha2::{Digest, Sha256}; use uuid::Uuid; @@ -36,10 +37,7 @@ const PARTITION_ALIGNMENT_SECTORS: u64 = 2048; // still reserve enough trailing sectors in the raw image for the backup array // (128 entries * 128 bytes) plus the backup header. const GPT_ENTRY_SECTORS: u64 = 32; -const VOLUME_HEADER_SIZE: usize = 4096; -const VOLUME_MAGIC: &[u8; 16] = b"DSTACK_VOLUME\0\0\0"; -const VOLUME_FORMAT_VERSION: u16 = 1; -const VOLUME_KIND_VERITY: u32 = 1; +const VOLUME_HEADER_SIZE: usize = dstack_types::volume::DSTACK_VOLUME_HEADER_SIZE; #[derive(Clone, Copy)] pub enum Compression { @@ -376,18 +374,9 @@ fn write_volume_header(out: &mut fs::File, offset: u64, root_hash: &str) -> Resu bail!("verity root must be a 32-byte SHA-256 digest"); } - // Common envelope. All integer fields are little-endian. The remainder of - // the 4 KiB metadata partition is reserved and must stay zero. - let mut header = [0u8; VOLUME_HEADER_SIZE]; - header[..16].copy_from_slice(VOLUME_MAGIC); - header[16..18].copy_from_slice(&VOLUME_FORMAT_VERSION.to_le_bytes()); - header[18..20].copy_from_slice(&(VOLUME_HEADER_SIZE as u16).to_le_bytes()); - header[20..24].copy_from_slice(&VOLUME_KIND_VERITY.to_le_bytes()); - header[24..28].copy_from_slice(&1u32.to_le_bytes()); // verity kind version - header[28..32].copy_from_slice(&0u32.to_le_bytes()); // flags: partitioned - header[32..64].copy_from_slice(&root); - header[64..68].copy_from_slice(&4096u32.to_le_bytes()); - header[68..72].copy_from_slice(&4096u32.to_le_bytes()); + let header = DstackVolumeHeader::new_verity(root.try_into().expect("length checked above")) + .encode() + .context("encoding volume header")?; out.seek(SeekFrom::Start(offset))?; out.write_all(&header)?; @@ -542,7 +531,7 @@ mod tests { let mut header = [0u8; VOLUME_HEADER_SIZE]; img.seek(SeekFrom::Start(metadata.first_lba * SECTOR))?; img.read_exact(&mut header)?; - assert_eq!(&header[..16], VOLUME_MAGIC); + assert_eq!(&header[..16], b"DSTACK_VOLUME\0\0\0"); assert_eq!(&header[32..64], &hex::decode(root_hash)?); let mut buf = vec![0; data.len()]; img.seek(SeekFrom::Start(data_partition.first_lba * SECTOR))?; diff --git a/dstack/dstack-types/Cargo.toml b/dstack/dstack-types/Cargo.toml index 526d5192b..452e6e92b 100644 --- a/dstack/dstack-types/Cargo.toml +++ b/dstack/dstack-types/Cargo.toml @@ -10,6 +10,7 @@ edition.workspace = true license.workspace = true [dependencies] +binrw.workspace = true ciborium.workspace = true hex = { workspace = true, features = ["std"] } or-panic.workspace = true diff --git a/dstack/dstack-types/src/lib.rs b/dstack/dstack-types/src/lib.rs index 3b7a64a86..248275ff1 100644 --- a/dstack/dstack-types/src/lib.rs +++ b/dstack/dstack-types/src/lib.rs @@ -10,6 +10,8 @@ use serde::{Deserialize, Serialize}; use serde_human_bytes as hex_bytes; use size_parser::human_size; +pub mod volume; + /// Identifies which OVMF flavour the guest image was built with. /// /// Only the pre-202505 OVMF measurement layout is supported. @@ -128,19 +130,53 @@ pub struct AppCompose { } /// A pre-baked, read-only dm-verity volume attached to the CVM. -#[derive(Deserialize, Serialize, Debug, Clone, Default)] +#[derive(Deserialize, Serialize, Debug, Clone)] pub struct VerityVolume { /// dm-verity root hash (hex): the volume's content identity and integrity /// check. The guest matches attached devices against it. - #[serde(default)] - pub verity_root: String, + #[serde(with = "hex::serde")] + pub verity_root: [u8; 32], /// `"docker"` (seed the docker overlay2 image store), or an absolute path /// where the volume's filesystem is mounted (e.g. model weights). - /// - /// Defaulted so one malformed entry doesn't fail the whole AppCompose parse; - /// the guest skips an entry with an empty root or target. - #[serde(default)] - pub target: String, + pub target: VolumeTarget, +} + +/// How a verified volume is exposed inside the guest. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum VolumeTarget { + DockerSeed, + Mount(std::path::PathBuf), +} + +impl Serialize for VolumeTarget { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + match self { + Self::DockerSeed => serializer.serialize_str("docker"), + Self::Mount(path) => serializer.serialize_str(&path.to_string_lossy()), + } + } +} + +impl<'de> Deserialize<'de> for VolumeTarget { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let value = String::deserialize(deserializer)?; + if value == "docker" { + return Ok(Self::DockerSeed); + } + let path = std::path::PathBuf::from(&value); + if !path.is_absolute() { + return Err(serde::de::Error::custom(format!( + "volume target must be \"docker\" or an absolute path, got '{value}'" + ))); + } + Ok(Self::Mount(path)) + } } /// Canonical source for the policy used when `requirements.gpu_policy` is diff --git a/dstack/dstack-types/src/volume.rs b/dstack/dstack-types/src/volume.rs new file mode 100644 index 000000000..769667aad --- /dev/null +++ b/dstack/dstack-types/src/volume.rs @@ -0,0 +1,114 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! On-disk dstack volume envelope shared by builders and guests. + +use std::io::Cursor; + +use binrw::{binrw, BinRead, BinWrite}; + +pub const DSTACK_VOLUME_MAGIC: &[u8; 16] = b"DSTACK_VOLUME\0\0\0"; +pub const DSTACK_VOLUME_HEADER_SIZE: usize = 4096; +pub const DSTACK_VOLUME_FORMAT_VERSION: u16 = 1; +pub const DSTACK_VOLUME_KIND_VERITY: u32 = 1; + +#[binrw] +#[brw(little, magic = b"DSTACK_VOLUME\0\0\0")] +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DstackVolumeHeader { + pub format_version: u16, + pub header_size: u16, + pub kind: u32, + pub kind_version: u32, + pub flags: u32, + pub root_hash: [u8; 32], + pub data_block_size: u32, + pub hash_block_size: u32, +} + +impl DstackVolumeHeader { + pub fn new_verity(root_hash: [u8; 32]) -> Self { + Self { + format_version: DSTACK_VOLUME_FORMAT_VERSION, + header_size: DSTACK_VOLUME_HEADER_SIZE as u16, + kind: DSTACK_VOLUME_KIND_VERITY, + kind_version: 1, + flags: 0, + root_hash, + data_block_size: 4096, + hash_block_size: 4096, + } + } + + pub fn encode(&self) -> binrw::BinResult<[u8; DSTACK_VOLUME_HEADER_SIZE]> { + let mut block = [0u8; DSTACK_VOLUME_HEADER_SIZE]; + self.write(&mut Cursor::new(&mut block[..]))?; + Ok(block) + } + + pub fn decode(block: &[u8]) -> binrw::BinResult { + if block.len() < DSTACK_VOLUME_HEADER_SIZE { + return Err(binrw::Error::AssertFail { + pos: 0, + message: format!( + "volume header is truncated: {} < {DSTACK_VOLUME_HEADER_SIZE}", + block.len() + ), + }); + } + let header = Self::read(&mut Cursor::new(block))?; + if header.format_version != DSTACK_VOLUME_FORMAT_VERSION { + return Err(binrw::Error::AssertFail { + pos: 16, + message: format!( + "unsupported volume format version {}", + header.format_version + ), + }); + } + if header.header_size as usize != DSTACK_VOLUME_HEADER_SIZE { + return Err(binrw::Error::AssertFail { + pos: 18, + message: format!("invalid volume header size {}", header.header_size), + }); + } + Ok(header) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{VerityVolume, VolumeTarget}; + + #[test] + fn volume_header_round_trip() { + let expected = DstackVolumeHeader::new_verity([0x5a; 32]); + let encoded = expected.encode().unwrap(); + assert_eq!(&encoded[..16], b"DSTACK_VOLUME\0\0\0"); + assert_eq!(DstackVolumeHeader::decode(&encoded).unwrap(), expected); + } + + #[test] + fn verity_volume_validates_root_and_target_during_deserialization() { + let volume: VerityVolume = serde_json::from_value(serde_json::json!({ + "verity_root": "5a".repeat(32), + "target": "/run/models" + })) + .unwrap(); + assert_eq!(volume.verity_root, [0x5a; 32]); + assert_eq!(volume.target, VolumeTarget::Mount("/run/models".into())); + + assert!(serde_json::from_value::(serde_json::json!({ + "verity_root": "abcd", + "target": "docker" + })) + .is_err()); + assert!(serde_json::from_value::(serde_json::json!({ + "verity_root": "5a".repeat(32), + "target": "relative/path" + })) + .is_err()); + } +} diff --git a/dstack/guest-agent/src/bin/dstack-volume.rs b/dstack/guest-agent/src/bin/dstack-volume.rs index ea2ebce3f..0438ca061 100644 --- a/dstack/guest-agent/src/bin/dstack-volume.rs +++ b/dstack/guest-agent/src/bin/dstack-volume.rs @@ -11,49 +11,30 @@ use std::collections::HashSet; use std::ffi::OsStr; -use std::fs::{self, File}; use std::io::Read; +use std::os::unix::ffi::OsStrExt; +use std::os::unix::fs::MetadataExt; use std::path::{Path, PathBuf}; use std::process::{Command, Output}; use anyhow::{bail, Context, Result}; -use serde::Deserialize; +use dstack_types::volume::{ + DstackVolumeHeader, DSTACK_VOLUME_HEADER_SIZE, DSTACK_VOLUME_KIND_VERITY, DSTACK_VOLUME_MAGIC, +}; +use dstack_types::{AppCompose, VerityVolume as RequestedVolume, VolumeTarget}; +use fs_err::{self as fs, File}; +use serde_json::Value; +use tracing::{info, warn}; -const MAGIC: &[u8; 16] = b"DSTACK_VOLUME\0\0\0"; -const HEADER_SIZE: usize = 4096; -const FORMAT_VERSION: u16 = 1; -const KIND_VERITY: u32 = 1; const MAX_DISKS: usize = 64; const DOCKER_STORE: &str = "/var/lib/docker"; -#[derive(Debug, Deserialize)] -struct AppCompose { - #[serde(default)] - verity_volumes: Vec, -} - -#[derive(Debug, Deserialize)] -struct RequestedVolume { - verity_root: String, - target: String, -} - -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq, Eq)] struct BlockDisk { path: PathBuf, partitions: Vec<(u32, PathBuf)>, } -#[derive(Debug, PartialEq, Eq)] -struct VolumeHeader { - kind: u32, - kind_version: u32, - flags: u32, - root_hash: [u8; 32], - data_block_size: u32, - hash_block_size: u32, -} - #[derive(Debug)] struct VerityVolume { data: PathBuf, @@ -62,14 +43,15 @@ struct VerityVolume { } fn main() -> Result<()> { + tracing_subscriber::fmt().with_target(false).init(); let compose_path = std::env::args_os() .nth(1) .map(PathBuf::from) .unwrap_or_else(|| PathBuf::from("app-compose.json")); - let compose: AppCompose = serde_json::from_slice( - &fs::read(&compose_path).with_context(|| format!("reading {}", compose_path.display()))?, - ) - .with_context(|| format!("parsing {}", compose_path.display()))?; + // Deserialize the complete shared type before probing any untrusted disk. + // This validates roots and targets as part of parsing the measured compose. + let compose: AppCompose = serde_json::from_slice(&fs::read(&compose_path)?) + .with_context(|| format!("parsing {}", compose_path.display()))?; if compose.verity_volumes.is_empty() { return Ok(()); } @@ -78,26 +60,36 @@ fn main() -> Result<()> { let _ = run(Command::new("udevadm").args(["settle", "--timeout=5"])); let volumes = discover_volumes()?; - eprintln!( - "dstack-volume: found {} volume(s), {} requested", - volumes.len(), - compose.verity_volumes.len() + info!( + found = volumes.len(), + requested = compose.verity_volumes.len(), + "discovered dstack volumes" ); let mut used = HashSet::new(); + let mut failures = 0; for (index, requested) in compose.verity_volumes.iter().enumerate() { if let Err(err) = activate_requested(index, requested, &volumes, &mut used) { - eprintln!( - "dstack-volume: volume {index} ({}): {err:#}; skipping", - requested.target - ); + failures += 1; + warn!(index, target = %display_target(&requested.target), error = %format_args!("{err:#}"), "failed to activate required volume"); } } + if failures != 0 { + bail!("failed to activate {failures} required volume(s)"); + } Ok(()) } fn discover_volumes() -> Result> { let mut found = Vec::new(); - for disk in list_disks()?.into_iter().take(MAX_DISKS) { + let disks = list_disks()?; + if disks.len() > MAX_DISKS { + warn!( + found = disks.len(), + limit = MAX_DISKS, + "block-device scan truncated" + ); + } + for disk in disks.into_iter().take(MAX_DISKS) { // A partitioned disk has exactly one envelope location: partition 1. // Only a disk with no kernel-recognized partitions is probed at offset 0. let probe = if disk.partitions.is_empty() { @@ -111,69 +103,80 @@ fn discover_volumes() -> Result> { continue; }; match header.kind { - KIND_VERITY => match resolve_verity(&disk, header) { + DSTACK_VOLUME_KIND_VERITY => match resolve_verity(&disk, header) { Ok(volume) => found.push(volume), - Err(err) => eprintln!( - "dstack-volume: ignoring malformed verity volume {}: {err:#}", - disk.path.display() - ), + Err(err) => { + warn!(disk = %disk.path.display(), error = %format_args!("{err:#}"), "ignoring malformed verity volume") + } }, - kind => eprintln!( - "dstack-volume: ignoring unsupported kind {kind} on {}", - disk.path.display() - ), + kind => warn!(kind, disk = %disk.path.display(), "ignoring unsupported volume kind"), } } Ok(found) } fn list_disks() -> Result> { - let output = checked( - Command::new("lsblk").args(["-rnbpo", "PATH,TYPE,PARTN"]), - "listing block devices", - )?; - let text = String::from_utf8(output.stdout).context("lsblk output is not UTF-8")?; - let mut disks = Vec::::new(); - let mut current = None; - for line in text.lines() { - let mut fields = line.split_whitespace(); - let Some(path) = fields.next() else { continue }; - let Some(kind) = fields.next() else { continue }; - match kind { - "disk" => { - disks.push(BlockDisk { - path: PathBuf::from(path), - partitions: Vec::new(), - }); - current = Some(disks.len() - 1); - } - "part" => { - let Some(number) = fields.next().and_then(|v| v.parse::().ok()) else { - continue; - }; - // lsblk -p emits children immediately below their parent disk. - if let Some(index) = current { - disks[index].partitions.push((number, PathBuf::from(path))); - } - } - _ => {} - } + list_disks_at(Path::new("/sys/class/block"), Path::new("/dev")) +} + +fn list_disks_at(sysfs: &Path, devfs: &Path) -> Result> { + struct Entry { + name: std::ffi::OsString, + sysfs_path: PathBuf, + partition: Option, } + + let mut entries = Vec::new(); + for entry in fs::read_dir(sysfs)? { + let entry = entry?; + let class_path = entry.path(); + let partition = if class_path.join("partition").exists() { + Some( + fs::read_to_string(class_path.join("partition"))? + .trim() + .parse()?, + ) + } else { + None + }; + entries.push(Entry { + name: entry.file_name(), + sysfs_path: fs::canonicalize(class_path)?, + partition, + }); + } + + let mut disks = entries + .iter() + .filter(|entry| entry.partition.is_none() && entry.sysfs_path.join("device").exists()) + .map(|entry| BlockDisk { + path: devfs.join(&entry.name), + partitions: entries + .iter() + .filter_map(|partition| { + let number = partition.partition?; + (partition.sysfs_path.parent() == Some(entry.sysfs_path.as_path())) + .then(|| (number, devfs.join(&partition.name))) + }) + .collect(), + }) + .collect::>(); for disk in &mut disks { disk.partitions.sort_by_key(|(number, _)| *number); } + disks.sort_by(|left, right| left.path.cmp(&right.path)); Ok(disks) } -fn read_header(path: &Path) -> Result> { +fn read_header(path: &Path) -> Result> { let mut file = match File::open(path) { Ok(file) => file, Err(err) => { - eprintln!("dstack-volume: cannot open {}: {err}", path.display()); + warn!(path = %path.display(), %err, "cannot open block device"); return Ok(None); } }; - let mut bytes = [0u8; HEADER_SIZE]; + let mut bytes = [0u8; DSTACK_VOLUME_HEADER_SIZE]; if let Err(err) = file.read_exact(&mut bytes) { if err.kind() == std::io::ErrorKind::UnexpectedEof { return Ok(None); @@ -183,30 +186,14 @@ fn read_header(path: &Path) -> Result> { parse_header(&bytes).with_context(|| format!("parsing envelope on {}", path.display())) } -fn parse_header(bytes: &[u8]) -> Result> { - if bytes.len() < HEADER_SIZE || &bytes[..16] != MAGIC { +fn parse_header(bytes: &[u8]) -> Result> { + if bytes.len() < DSTACK_VOLUME_HEADER_SIZE || &bytes[..16] != DSTACK_VOLUME_MAGIC { return Ok(None); } - let version = u16::from_le_bytes(bytes[16..18].try_into()?); - let header_size = u16::from_le_bytes(bytes[18..20].try_into()?) as usize; - if version != FORMAT_VERSION { - bail!("unsupported envelope version {version}"); - } - if header_size != HEADER_SIZE { - bail!("invalid header size {header_size}"); - } - let root_hash = bytes[32..64].try_into()?; - Ok(Some(VolumeHeader { - kind: u32::from_le_bytes(bytes[20..24].try_into()?), - kind_version: u32::from_le_bytes(bytes[24..28].try_into()?), - flags: u32::from_le_bytes(bytes[28..32].try_into()?), - root_hash, - data_block_size: u32::from_le_bytes(bytes[64..68].try_into()?), - hash_block_size: u32::from_le_bytes(bytes[68..72].try_into()?), - })) + Ok(Some(DstackVolumeHeader::decode(bytes)?)) } -fn resolve_verity(disk: &BlockDisk, header: VolumeHeader) -> Result { +fn resolve_verity(disk: &BlockDisk, header: DstackVolumeHeader) -> Result { if header.kind_version != 1 { bail!("unsupported verity version {}", header.kind_version); } @@ -243,38 +230,49 @@ fn activate_requested( volumes: &[VerityVolume], used: &mut HashSet, ) -> Result<()> { - let expected = hex::decode(&requested.verity_root).context("invalid verity_root hex")?; - if expected.len() != 32 { - bail!("verity_root must contain 32 bytes"); - } - let candidate = volumes + let (candidate_index, candidate) = volumes .iter() .enumerate() .find(|(candidate_index, volume)| { - !used.contains(candidate_index) && volume.root_hash.as_slice() == expected + !used.contains(candidate_index) && volume.root_hash == requested.verity_root }) .context("no attached volume advertises the measured root")?; let mapper_name = format!("dstack-verity{index}"); + let mapped = PathBuf::from(format!("/dev/mapper/{mapper_name}")); + let expected_root = hex::encode(requested.verity_root); + if mapped.exists() { + if mapping_root(&mapper_name)?.eq_ignore_ascii_case(&expected_root) { + verify_first_block(&mapped)?; + mount_volume(index, requested, &mapped)?; + used.insert(candidate_index); + info!(mapper = mapper_name, "reused active verity mapping"); + return Ok(()); + } + checked( + Command::new("veritysetup").args(["close", &mapper_name]), + "closing stale verity mapping", + )?; + } + // The on-disk root only selected a candidate. Pass the root from the // measured compose to veritysetup, which is the actual trust decision. checked( Command::new("veritysetup") .arg("open") - .arg(&candidate.1.data) + .arg(&candidate.data) .arg(&mapper_name) - .arg(&candidate.1.hash) - .arg(&requested.verity_root), + .arg(&candidate.hash) + .arg(&expected_root), "opening dm-verity volume", )?; - let mapped = PathBuf::from(format!("/dev/mapper/{mapper_name}")); if let Err(err) = verify_first_block(&mapped).and_then(|_| mount_volume(index, requested, &mapped)) { let _ = run(Command::new("veritysetup").args(["close", &mapper_name])); return Err(err); } - used.insert(candidate.0); + used.insert(candidate_index); Ok(()) } @@ -292,34 +290,41 @@ fn mount_volume(index: usize, requested: &RequestedVolume, mapped: &Path) -> Res .arg(mapped), ) .unwrap_or_default(); - if requested.target == "docker" { - let mountpoint = PathBuf::from(format!("/run/dstack-verity/{index}")); - fs::create_dir_all(&mountpoint)?; - mount_read_only(mapped, &mountpoint, fs_type.trim())?; - if let Err(err) = seed_docker(&mountpoint) { - let _ = run(Command::new("umount").arg(&mountpoint)); - return Err(err); + match &requested.target { + VolumeTarget::DockerSeed => { + let mountpoint = PathBuf::from(format!("/run/dstack-verity/{index}")); + fs::create_dir_all(&mountpoint)?; + if is_mountpoint(&mountpoint)? { + ensure_mounted_from(&mountpoint, mapped)?; + } else { + mount_read_only(mapped, &mountpoint, fs_type.trim())?; + } + if let Err(err) = seed_docker(&mountpoint) { + let _ = run(Command::new("umount").arg(&mountpoint)); + return Err(err); + } + info!(root = %hex::encode(requested.verity_root), "seeded docker from verity volume"); } - eprintln!( - "dstack-volume: seeded docker from {}", - requested.verity_root - ); - } else { - let target = Path::new(&requested.target); - if !target.is_absolute() { - bail!("mount target must be absolute"); + VolumeTarget::Mount(target) => { + fs::create_dir_all(target)?; + if is_mountpoint(target)? { + ensure_mounted_from(target, mapped)?; + } else { + mount_read_only(mapped, target, fs_type.trim())?; + } + info!(root = %hex::encode(requested.verity_root), target = %target.display(), "mounted verity volume"); } - fs::create_dir_all(target).with_context(|| format!("creating {}", target.display()))?; - mount_read_only(mapped, target, fs_type.trim())?; - eprintln!( - "dstack-volume: mounted {} at {}", - requested.verity_root, - target.display() - ); } Ok(()) } +fn display_target(target: &VolumeTarget) -> std::borrow::Cow<'_, str> { + match target { + VolumeTarget::DockerSeed => "docker".into(), + VolumeTarget::Mount(path) => path.to_string_lossy(), + } +} + fn mount_read_only(device: &Path, target: &Path, fs_type: &str) -> Result<()> { let options = if matches!(fs_type, "ext3" | "ext4") { "ro,noload" @@ -429,14 +434,10 @@ fn merge_repositories(volume: &Path, store: &Path) -> Result<()> { let target = store.join("image/overlay2/repositories.json"); if target.exists() { let temporary = target.with_extension("json.dstack-tmp"); - let output = checked( - Command::new("jq") - .args(["-s", ".[0] * .[1]"]) - .arg(&target) - .arg(&source), - "merging docker repositories", - )?; - fs::write(&temporary, output.stdout)?; + let mut current: Value = serde_json::from_slice(&fs::read(&target)?)?; + let incoming: Value = serde_json::from_slice(&fs::read(&source)?)?; + merge_json(&mut current, incoming); + fs::write(&temporary, serde_json::to_vec(¤t)?)?; fs::rename(&temporary, &target)?; } else { fs::copy(&source, &target)?; @@ -444,6 +445,17 @@ fn merge_repositories(volume: &Path, store: &Path) -> Result<()> { Ok(()) } +fn merge_json(current: &mut Value, incoming: Value) { + match (current, incoming) { + (Value::Object(current), Value::Object(incoming)) => { + for (key, value) in incoming { + merge_json(current.entry(key).or_insert(Value::Null), value); + } + } + (current, incoming) => *current = incoming, + } +} + fn copy_layer_metadata(volume: &Path, store: &Path) -> Result<()> { for layer in child_directories(&volume.join("overlay2"))? { let Some(id) = layer.file_name() else { @@ -472,10 +484,67 @@ fn copy_layer_metadata(volume: &Path, store: &Path) -> Result<()> { } fn is_mountpoint(path: &Path) -> Result { - let needle = format!(" {} ", path.display()); - Ok(fs::read_to_string("/proc/self/mountinfo")? - .lines() - .any(|line| line.contains(&needle))) + Ok(mountpoint_device(path)?.is_some()) +} + +fn ensure_mounted_from(target: &Path, device: &Path) -> Result<()> { + let mounted = mountpoint_device(target)?.context("mount point disappeared")?; + let expected = device_number(fs::metadata(device)?.rdev()); + if mounted != expected { + bail!( + "{} is mounted from device {}:{}, expected {}:{}", + target.display(), + mounted.0, + mounted.1, + expected.0, + expected.1 + ); + } + Ok(()) +} + +fn mountpoint_device(path: &Path) -> Result> { + let target = path.as_os_str().as_bytes(); + for line in fs::read_to_string("/proc/self/mountinfo")?.lines() { + let mut fields = line.split_ascii_whitespace(); + let Some(device) = fields.nth(2) else { + continue; + }; + let Some(mountpoint) = fields.nth(1) else { + continue; + }; + if unescape_mountinfo(mountpoint.as_bytes()) == target { + let (major, minor) = device + .split_once(':') + .context("invalid mountinfo device number")?; + return Ok(Some((major.parse()?, minor.parse()?))); + } + } + Ok(None) +} + +fn device_number(device: u64) -> (u64, u64) { + let major = ((device >> 8) & 0xfff) | ((device >> 32) & 0xffff_f000); + let minor = (device & 0xff) | ((device >> 12) & 0xffff_ff00); + (major, minor) +} + +fn unescape_mountinfo(value: &[u8]) -> Vec { + let mut decoded = Vec::with_capacity(value.len()); + let mut index = 0; + while index < value.len() { + if value[index] == b'\\' && index + 3 < value.len() { + let octal = &value[index + 1..index + 4]; + if octal.iter().all(|byte| matches!(byte, b'0'..=b'7')) { + decoded.push((octal[0] - b'0') * 64 + (octal[1] - b'0') * 8 + (octal[2] - b'0')); + index += 4; + continue; + } + } + decoded.push(value[index]); + index += 1; + } + decoded } fn unwind_binds(paths: &[PathBuf]) { @@ -492,6 +561,19 @@ fn command_stdout(command: &mut Command) -> Result { Ok(String::from_utf8(output.stdout)?) } +fn mapping_root(mapper_name: &str) -> Result { + let status = command_stdout(Command::new("veritysetup").args(["status", mapper_name]))?; + status + .lines() + .find_map(|line| { + let line = line.trim(); + line.strip_prefix("root hash:") + .or_else(|| line.strip_prefix("Root hash:")) + }) + .map(|root| root.trim().to_string()) + .context("verity mapping status has no root hash") +} + fn run(command: &mut Command) -> Result { command.output().context("running command") } @@ -512,46 +594,18 @@ fn checked(command: &mut Command, operation: &str) -> Result { mod tests { use super::*; - fn header() -> [u8; HEADER_SIZE] { - let mut bytes = [0u8; HEADER_SIZE]; - bytes[..16].copy_from_slice(MAGIC); - bytes[16..18].copy_from_slice(&1u16.to_le_bytes()); - bytes[18..20].copy_from_slice(&(HEADER_SIZE as u16).to_le_bytes()); - bytes[20..24].copy_from_slice(&KIND_VERITY.to_le_bytes()); - bytes[24..28].copy_from_slice(&1u32.to_le_bytes()); - bytes[32..64].fill(0x5a); - bytes[64..68].copy_from_slice(&4096u32.to_le_bytes()); - bytes[68..72].copy_from_slice(&4096u32.to_le_bytes()); - bytes + fn header() -> DstackVolumeHeader { + DstackVolumeHeader::new_verity([0x5a; 32]) } #[test] - fn parses_volume_envelope() { + fn ignores_non_volume_data() { assert_eq!( - parse_header(&header()).unwrap(), - Some(VolumeHeader { - kind: KIND_VERITY, - kind_version: 1, - flags: 0, - root_hash: [0x5a; 32], - data_block_size: 4096, - hash_block_size: 4096, - }) + parse_header(&[0u8; DSTACK_VOLUME_HEADER_SIZE]).unwrap(), + None ); } - #[test] - fn ignores_non_volume_data() { - assert_eq!(parse_header(&[0u8; HEADER_SIZE]).unwrap(), None); - } - - #[test] - fn rejects_unknown_envelope_version() { - let mut bytes = header(); - bytes[16..18].copy_from_slice(&2u16.to_le_bytes()); - assert!(parse_header(&bytes).is_err()); - } - #[test] fn verity_kind_uses_second_and_third_partitions() { let disk = BlockDisk { @@ -562,7 +616,7 @@ mod tests { (3, "/dev/test3".into()), ], }; - let volume = resolve_verity(&disk, parse_header(&header()).unwrap().unwrap()).unwrap(); + let volume = resolve_verity(&disk, header()).unwrap(); assert_eq!(volume.data, Path::new("/dev/test2")); assert_eq!(volume.hash, Path::new("/dev/test3")); } @@ -573,6 +627,49 @@ mod tests { path: "/dev/test".into(), partitions: vec![], }; - assert!(resolve_verity(&disk, parse_header(&header()).unwrap().unwrap()).is_err()); + assert!(resolve_verity(&disk, header()).is_err()); + } + + #[test] + fn recursively_merges_json_objects() { + let mut current = serde_json::json!({"repo": {"old": 1, "same": 1}}); + merge_json( + &mut current, + serde_json::json!({"repo": {"new": 2, "same": 3}}), + ); + assert_eq!( + current, + serde_json::json!({"repo": {"old": 1, "new": 2, "same": 3}}) + ); + } + + #[test] + fn decodes_mountinfo_escapes() { + assert_eq!(unescape_mountinfo(b"/run/my\\040volume"), b"/run/my volume"); + } + + #[test] + fn discovers_partitions_from_sysfs_parentage() -> Result<()> { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir()?; + let sysfs = temp.path().join("sys/class/block"); + let devices = temp.path().join("sys/devices/block/vda"); + let partition = devices.join("vda1"); + fs::create_dir_all(&sysfs)?; + fs::create_dir_all(devices.join("device"))?; + fs::create_dir_all(&partition)?; + fs::write(partition.join("partition"), "1\n")?; + symlink(&devices, sysfs.join("vda"))?; + symlink(&partition, sysfs.join("vda1"))?; + + assert_eq!( + list_disks_at(&sysfs, Path::new("/dev"))?, + vec![BlockDisk { + path: "/dev/vda".into(), + partitions: vec![(1, "/dev/vda1".into())], + }] + ); + Ok(()) } } From e18f6e5e453eb7a7407dcb5c5bb65eafa5c8f535 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 20 Jul 2026 19:46:04 -0700 Subject: [PATCH 12/25] refactor(verity): propagate conversion errors --- dstack/crates/dstack-verity/src/volume.rs | 10 +++++----- dstack/guest-agent/src/bin/dstack-volume.rs | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/dstack/crates/dstack-verity/src/volume.rs b/dstack/crates/dstack-verity/src/volume.rs index 4487af8c8..c3dfc4404 100644 --- a/dstack/crates/dstack-verity/src/volume.rs +++ b/dstack/crates/dstack-verity/src/volume.rs @@ -370,11 +370,11 @@ fn write_volume_header(out: &mut fs::File, offset: u64, root_hash: &str) -> Resu use std::io::{Seek, SeekFrom, Write}; let root = hex::decode(root_hash).context("decoding verity root hash")?; - if root.len() != 32 { - bail!("verity root must be a 32-byte SHA-256 digest"); - } - - let header = DstackVolumeHeader::new_verity(root.try_into().expect("length checked above")) + let root: [u8; 32] = root + .as_slice() + .try_into() + .context("verity root must be a 32-byte SHA-256 digest")?; + let header = DstackVolumeHeader::new_verity(root) .encode() .context("encoding volume header")?; diff --git a/dstack/guest-agent/src/bin/dstack-volume.rs b/dstack/guest-agent/src/bin/dstack-volume.rs index 0438ca061..4069dd15b 100644 --- a/dstack/guest-agent/src/bin/dstack-volume.rs +++ b/dstack/guest-agent/src/bin/dstack-volume.rs @@ -354,10 +354,10 @@ fn seed_docker(volume: &Path) -> Result<()> { if !source.is_dir() { continue; } - let target = store - .join("overlay2") - .join(layer.file_name().unwrap()) - .join("diff"); + let layer_id = layer + .file_name() + .context("docker layer path has no file name")?; + let target = store.join("overlay2").join(layer_id).join("diff"); fs::create_dir_all(&target)?; if !is_mountpoint(&target)? { if let Err(err) = checked( From 302ccb7e1e4db0868ab16394181481ded698d8ca Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 20 Jul 2026 20:02:46 -0700 Subject: [PATCH 13/25] refactor(verity): simplify volume envelope --- dstack/Cargo.lock | 3 - dstack/Cargo.toml | 2 +- dstack/crates/dstack-verity/src/volume.rs | 3 +- dstack/dstack-types/src/lib.rs | 2 +- dstack/dstack-types/src/volume.rs | 42 ++----- dstack/guest-agent/src/bin/dstack-volume.rs | 124 +++++--------------- 6 files changed, 41 insertions(+), 135 deletions(-) diff --git a/dstack/Cargo.lock b/dstack/Cargo.lock index 0c50128d7..3389c1112 100644 --- a/dstack/Cargo.lock +++ b/dstack/Cargo.lock @@ -3184,9 +3184,6 @@ name = "hex" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" -dependencies = [ - "serde", -] [[package]] name = "hex-literal" diff --git a/dstack/Cargo.toml b/dstack/Cargo.toml index 78b3c9844..1c0ced796 100644 --- a/dstack/Cargo.toml +++ b/dstack/Cargo.toml @@ -149,7 +149,7 @@ borsh = { version = "1.5.7", default-features = false, features = ["derive"] } bon = { version = "3.4.0", default-features = false } base64 = "0.22.1" bcrypt = "0.17.1" -hex = { version = "0.4.3", default-features = false, features = ["serde"] } +hex = { version = "0.4.3", default-features = false } hex_fmt = "0.3.0" hex-literal = "1.0.0" prost = "0.13.5" diff --git a/dstack/crates/dstack-verity/src/volume.rs b/dstack/crates/dstack-verity/src/volume.rs index c3dfc4404..82981f0df 100644 --- a/dstack/crates/dstack-verity/src/volume.rs +++ b/dstack/crates/dstack-verity/src/volume.rs @@ -532,7 +532,8 @@ mod tests { img.seek(SeekFrom::Start(metadata.first_lba * SECTOR))?; img.read_exact(&mut header)?; assert_eq!(&header[..16], b"DSTACK_VOLUME\0\0\0"); - assert_eq!(&header[32..64], &hex::decode(root_hash)?); + let decoded = DstackVolumeHeader::decode(&header)?; + assert_eq!(decoded.root_hash.as_slice(), hex::decode(root_hash)?); let mut buf = vec![0; data.len()]; img.seek(SeekFrom::Start(data_partition.first_lba * SECTOR))?; img.read_exact(&mut buf)?; diff --git a/dstack/dstack-types/src/lib.rs b/dstack/dstack-types/src/lib.rs index 248275ff1..5e5d45189 100644 --- a/dstack/dstack-types/src/lib.rs +++ b/dstack/dstack-types/src/lib.rs @@ -134,7 +134,7 @@ pub struct AppCompose { pub struct VerityVolume { /// dm-verity root hash (hex): the volume's content identity and integrity /// check. The guest matches attached devices against it. - #[serde(with = "hex::serde")] + #[serde(with = "hex_bytes")] pub verity_root: [u8; 32], /// `"docker"` (seed the docker overlay2 image store), or an absolute path /// where the volume's filesystem is mounted (e.g. model weights). diff --git a/dstack/dstack-types/src/volume.rs b/dstack/dstack-types/src/volume.rs index 769667aad..a5ad812af 100644 --- a/dstack/dstack-types/src/volume.rs +++ b/dstack/dstack-types/src/volume.rs @@ -10,34 +10,21 @@ use binrw::{binrw, BinRead, BinWrite}; pub const DSTACK_VOLUME_MAGIC: &[u8; 16] = b"DSTACK_VOLUME\0\0\0"; pub const DSTACK_VOLUME_HEADER_SIZE: usize = 4096; -pub const DSTACK_VOLUME_FORMAT_VERSION: u16 = 1; pub const DSTACK_VOLUME_KIND_VERITY: u32 = 1; #[binrw] #[brw(little, magic = b"DSTACK_VOLUME\0\0\0")] #[derive(Debug, Clone, PartialEq, Eq)] pub struct DstackVolumeHeader { - pub format_version: u16, - pub header_size: u16, pub kind: u32, - pub kind_version: u32, - pub flags: u32, pub root_hash: [u8; 32], - pub data_block_size: u32, - pub hash_block_size: u32, } impl DstackVolumeHeader { pub fn new_verity(root_hash: [u8; 32]) -> Self { Self { - format_version: DSTACK_VOLUME_FORMAT_VERSION, - header_size: DSTACK_VOLUME_HEADER_SIZE as u16, kind: DSTACK_VOLUME_KIND_VERITY, - kind_version: 1, - flags: 0, root_hash, - data_block_size: 4096, - hash_block_size: 4096, } } @@ -57,23 +44,7 @@ impl DstackVolumeHeader { ), }); } - let header = Self::read(&mut Cursor::new(block))?; - if header.format_version != DSTACK_VOLUME_FORMAT_VERSION { - return Err(binrw::Error::AssertFail { - pos: 16, - message: format!( - "unsupported volume format version {}", - header.format_version - ), - }); - } - if header.header_size as usize != DSTACK_VOLUME_HEADER_SIZE { - return Err(binrw::Error::AssertFail { - pos: 18, - message: format!("invalid volume header size {}", header.header_size), - }); - } - Ok(header) + Self::read(&mut Cursor::new(block)) } } @@ -91,14 +62,18 @@ mod tests { } #[test] - fn verity_volume_validates_root_and_target_during_deserialization() { + fn verity_volume_validates_root_and_target_during_deserialization( + ) -> Result<(), serde_json::Error> { let volume: VerityVolume = serde_json::from_value(serde_json::json!({ "verity_root": "5a".repeat(32), "target": "/run/models" - })) - .unwrap(); + }))?; assert_eq!(volume.verity_root, [0x5a; 32]); assert_eq!(volume.target, VolumeTarget::Mount("/run/models".into())); + assert_eq!( + serde_json::to_value(&volume)?["verity_root"], + "5a".repeat(32) + ); assert!(serde_json::from_value::(serde_json::json!({ "verity_root": "abcd", @@ -110,5 +85,6 @@ mod tests { "target": "relative/path" })) .is_err()); + Ok(()) } } diff --git a/dstack/guest-agent/src/bin/dstack-volume.rs b/dstack/guest-agent/src/bin/dstack-volume.rs index 4069dd15b..860ff28e7 100644 --- a/dstack/guest-agent/src/bin/dstack-volume.rs +++ b/dstack/guest-agent/src/bin/dstack-volume.rs @@ -15,9 +15,9 @@ use std::io::Read; use std::os::unix::ffi::OsStrExt; use std::os::unix::fs::MetadataExt; use std::path::{Path, PathBuf}; -use std::process::{Command, Output}; use anyhow::{bail, Context, Result}; +use cmd_lib::{run_cmd, run_fun}; use dstack_types::volume::{ DstackVolumeHeader, DSTACK_VOLUME_HEADER_SIZE, DSTACK_VOLUME_KIND_VERITY, DSTACK_VOLUME_MAGIC, }; @@ -43,7 +43,7 @@ struct VerityVolume { } fn main() -> Result<()> { - tracing_subscriber::fmt().with_target(false).init(); + tracing_subscriber::fmt().init(); let compose_path = std::env::args_os() .nth(1) .map(PathBuf::from) @@ -56,8 +56,8 @@ fn main() -> Result<()> { return Ok(()); } - let _ = run(Command::new("modprobe").arg("dm-verity")); - let _ = run(Command::new("udevadm").args(["settle", "--timeout=5"])); + let _ = run_cmd!(modprobe dm-verity); + let _ = run_cmd!(udevadm settle --timeout=5); let volumes = discover_volumes()?; info!( @@ -194,19 +194,6 @@ fn parse_header(bytes: &[u8]) -> Result> { } fn resolve_verity(disk: &BlockDisk, header: DstackVolumeHeader) -> Result { - if header.kind_version != 1 { - bail!("unsupported verity version {}", header.kind_version); - } - if header.flags != 0 { - bail!("unsupported verity flags {:#x}", header.flags); - } - if header.data_block_size != 4096 || header.hash_block_size != 4096 { - bail!( - "unsupported verity block sizes {}/{}", - header.data_block_size, - header.hash_block_size - ); - } if disk.partitions.is_empty() { bail!("raw verity layout is not defined by kind version 1"); } @@ -249,27 +236,19 @@ fn activate_requested( info!(mapper = mapper_name, "reused active verity mapping"); return Ok(()); } - checked( - Command::new("veritysetup").args(["close", &mapper_name]), - "closing stale verity mapping", - )?; + run_cmd!(veritysetup close $mapper_name).context("closing stale verity mapping")?; } // The on-disk root only selected a candidate. Pass the root from the // measured compose to veritysetup, which is the actual trust decision. - checked( - Command::new("veritysetup") - .arg("open") - .arg(&candidate.data) - .arg(&mapper_name) - .arg(&candidate.hash) - .arg(&expected_root), - "opening dm-verity volume", - )?; + let data = &candidate.data; + let hash = &candidate.hash; + run_cmd!(veritysetup open $data $mapper_name $hash $expected_root) + .context("opening dm-verity volume")?; if let Err(err) = verify_first_block(&mapped).and_then(|_| mount_volume(index, requested, &mapped)) { - let _ = run(Command::new("veritysetup").args(["close", &mapper_name])); + let _ = run_cmd!(veritysetup close $mapper_name); return Err(err); } used.insert(candidate_index); @@ -284,12 +263,7 @@ fn verify_first_block(path: &Path) -> Result<()> { } fn mount_volume(index: usize, requested: &RequestedVolume, mapped: &Path) -> Result<()> { - let fs_type = command_stdout( - Command::new("blkid") - .args(["-o", "value", "-s", "TYPE"]) - .arg(mapped), - ) - .unwrap_or_default(); + let fs_type = run_fun!(blkid -o value -s TYPE $mapped).unwrap_or_default(); match &requested.target { VolumeTarget::DockerSeed => { let mountpoint = PathBuf::from(format!("/run/dstack-verity/{index}")); @@ -300,7 +274,7 @@ fn mount_volume(index: usize, requested: &RequestedVolume, mapped: &Path) -> Res mount_read_only(mapped, &mountpoint, fs_type.trim())?; } if let Err(err) = seed_docker(&mountpoint) { - let _ = run(Command::new("umount").arg(&mountpoint)); + let _ = run_cmd!(umount $mountpoint); return Err(err); } info!(root = %hex::encode(requested.verity_root), "seeded docker from verity volume"); @@ -331,14 +305,12 @@ fn mount_read_only(device: &Path, target: &Path, fs_type: &str) -> Result<()> { } else { "ro" }; - let mut command = Command::new("mount"); - if !fs_type.is_empty() { - command.args(["-t", fs_type]); - } - checked( - command.arg("-o").arg(options).arg(device).arg(target), - "mounting verity volume", - )?; + if fs_type.is_empty() { + run_cmd!(mount -o $options $device $target).context("mounting verity volume")?; + } else { + run_cmd!(mount -t $fs_type -o $options $device $target) + .context("mounting verity volume")?; + } Ok(()) } @@ -360,23 +332,15 @@ fn seed_docker(volume: &Path) -> Result<()> { let target = store.join("overlay2").join(layer_id).join("diff"); fs::create_dir_all(&target)?; if !is_mountpoint(&target)? { - if let Err(err) = checked( - Command::new("mount") - .args(["--bind"]) - .arg(&source) - .arg(&target), - "binding docker layer", - ) { + if let Err(err) = run_cmd!(mount --bind $source $target).context("binding docker layer") + { unwind_binds(&bound); return Err(err); } // A bind inherits neither the intended policy nor all mount flags. - if let Err(err) = checked( - Command::new("mount") - .args(["-o", "remount,bind,ro"]) - .arg(&target), - "making docker layer read-only", - ) { + if let Err(err) = + run_cmd!(mount -o remount,bind,ro $target).context("making docker layer read-only") + { bound.push(target); unwind_binds(&bound); return Err(err); @@ -419,13 +383,8 @@ fn child_directories(path: &Path) -> Result> { fn copy_contents(source: &Path, target: &Path) -> Result<()> { fs::create_dir_all(target)?; - checked( - Command::new("cp") - .args(["-a"]) - .arg(source.join(".")) - .arg(target), - "copying docker metadata", - )?; + let source_contents = source.join("."); + run_cmd!(cp -a $source_contents $target).context("copying docker metadata")?; Ok(()) } @@ -474,10 +433,7 @@ fn copy_layer_metadata(volume: &Path, store: &Path) -> Result<()> { if source.file_name() == Some(OsStr::new("diff")) { continue; } - checked( - Command::new("cp").args(["-a"]).arg(&source).arg(&target), - "copying docker layer metadata", - )?; + run_cmd!(cp -a $source $target).context("copying docker layer metadata")?; } } Ok(()) @@ -549,20 +505,12 @@ fn unescape_mountinfo(value: &[u8]) -> Vec { fn unwind_binds(paths: &[PathBuf]) { for path in paths.iter().rev() { - let _ = run(Command::new("umount").arg(path)); - } -} - -fn command_stdout(command: &mut Command) -> Result { - let output = command.output()?; - if !output.status.success() { - bail!("command failed with {}", output.status); + let _ = run_cmd!(umount $path); } - Ok(String::from_utf8(output.stdout)?) } fn mapping_root(mapper_name: &str) -> Result { - let status = command_stdout(Command::new("veritysetup").args(["status", mapper_name]))?; + let status = run_fun!(veritysetup status $mapper_name)?; status .lines() .find_map(|line| { @@ -574,22 +522,6 @@ fn mapping_root(mapper_name: &str) -> Result { .context("verity mapping status has no root hash") } -fn run(command: &mut Command) -> Result { - command.output().context("running command") -} - -fn checked(command: &mut Command, operation: &str) -> Result { - let output = command.output().with_context(|| operation.to_string())?; - if !output.status.success() { - bail!( - "{operation} failed with {}: {}", - output.status, - String::from_utf8_lossy(&output.stderr).trim() - ); - } - Ok(output) -} - #[cfg(test)] mod tests { use super::*; From dc938c80cdbd7f4f4867d90a8a0e563fa3dca7aa Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 20 Jul 2026 20:23:39 -0700 Subject: [PATCH 14/25] refactor(verity): use command helpers in builder --- dstack/Cargo.lock | 1 + dstack/crates/dstack-verity/Cargo.toml | 1 + dstack/crates/dstack-verity/src/volume.rs | 72 +++++------------------ 3 files changed, 16 insertions(+), 58 deletions(-) diff --git a/dstack/Cargo.lock b/dstack/Cargo.lock index 3389c1112..83b5a4c2e 100644 --- a/dstack/Cargo.lock +++ b/dstack/Cargo.lock @@ -2272,6 +2272,7 @@ name = "dstack-verity" version = "0.6.0" dependencies = [ "anyhow", + "cmd_lib", "dstack-types", "flate2", "fs-err", diff --git a/dstack/crates/dstack-verity/Cargo.toml b/dstack/crates/dstack-verity/Cargo.toml index 7d8db7220..d4168583c 100644 --- a/dstack/crates/dstack-verity/Cargo.toml +++ b/dstack/crates/dstack-verity/Cargo.toml @@ -10,6 +10,7 @@ license.workspace = true [dependencies] anyhow = { workspace = true, features = ["std"] } +cmd_lib.workspace = true dstack-types.workspace = true tokio = { workspace = true, features = ["rt"] } serde = { workspace = true, features = ["derive", "std"] } diff --git a/dstack/crates/dstack-verity/src/volume.rs b/dstack/crates/dstack-verity/src/volume.rs index 82981f0df..047ba0a29 100644 --- a/dstack/crates/dstack-verity/src/volume.rs +++ b/dstack/crates/dstack-verity/src/volume.rs @@ -17,9 +17,9 @@ use std::collections::BTreeMap; use std::path::Path; -use std::process::Command; use anyhow::{bail, Context, Result}; +use cmd_lib::{run_cmd, run_fun}; use dstack_types::volume::DstackVolumeHeader; use fs_err as fs; use sha2::{Digest, Sha256}; @@ -87,19 +87,13 @@ pub fn build_volume( let data_path = tmp.path().join("data.fs"); // 1. reproducible squashfs. - let mut cmd = Command::new("mksquashfs"); - cmd.arg(store).arg(&data_path); - cmd.args(compress.args()); - cmd.args([ - "-all-time", - EPOCH, - "-mkfs-time", - EPOCH, - "-noappend", - "-no-progress", - "-xattrs", - ]); - run(cmd, "mksquashfs")?; + let compression_args = compress.args(); + run_cmd!( + mksquashfs $store $data_path $[compression_args] + -all-time $EPOCH -mkfs-time $EPOCH -noappend -no-progress -xattrs + >/dev/null + ) + .context("running mksquashfs")?; // 2. the verity data region must be block-aligned; pad the squashfs up. let bytes_used = squashfs_bytes_used(&data_path)?; @@ -147,20 +141,11 @@ fn seal_data_image( // want the whole image reproducible, not just the root. The UUID sits in the // hash tree, not the hashed data, so it never changes the root. let uuid = uuid_from_data(data_path, data_size)?; - let mut cmd = Command::new("veritysetup"); - cmd.args([ - "format", - "--salt", - salt_hex, - "--uuid", - &uuid, - "--data-block-size", - "4096", - "--hash-block-size", - "4096", - ]); - cmd.arg(data_path).arg(&hash_path); - let out = capture(cmd, "veritysetup format")?; + let out = run_fun!( + veritysetup format --salt $salt_hex --uuid $uuid + --data-block-size 4096 --hash-block-size 4096 $data_path $hash_path + ) + .context("running veritysetup format")?; let verity_root = parse_root_hash(&out).context("could not find the root hash in veritysetup output")?; @@ -406,42 +391,13 @@ fn parse_root_hash(output: &str) -> Option { } fn require_tool(name: &str) -> Result<()> { - // spawning at all means the binary is on PATH; a non-zero exit from - // `--version` (some builds) still counts as present. - let present = Command::new(name) - .arg("--version") - .stdout(std::process::Stdio::null()) - .stderr(std::process::Stdio::null()) - .status() - .is_ok(); + let present = run_cmd!(which $name >/dev/null 2>&1).is_ok(); if !present { bail!("`{name}` not found on PATH (install squashfs-tools / cryptsetup)"); } Ok(()) } -fn run(mut cmd: Command, what: &str) -> Result<()> { - let status = cmd - .stdout(std::process::Stdio::null()) - .status() - .with_context(|| format!("running {what}"))?; - if !status.success() { - bail!("{what} failed with {status}"); - } - Ok(()) -} - -fn capture(mut cmd: Command, what: &str) -> Result { - let out = cmd.output().with_context(|| format!("running {what}"))?; - if !out.status.success() { - bail!( - "{what} failed: {}", - String::from_utf8_lossy(&out.stderr).trim() - ); - } - Ok(String::from_utf8_lossy(&out.stdout).into_owned()) -} - #[cfg(test)] mod tests { use super::*; From c61ff438ff80524d321de9dd5fbcb45586da2333 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 20 Jul 2026 20:36:51 -0700 Subject: [PATCH 15/25] refactor(volumes): merge builder and guest tooling --- dstack/Cargo.lock | 55 ++++++++++--------- dstack/Cargo.toml | 4 +- dstack/crates/dstack-cli/Cargo.toml | 2 +- dstack/crates/dstack-cli/src/main.rs | 10 ++-- .../Cargo.toml | 4 +- .../dstack-volumes}/src/bin/dstack-volume.rs | 4 +- .../src/lib.rs | 3 +- .../src/oci.rs | 0 .../src/store.rs | 0 .../src/volume.rs | 4 +- .../dstack-volumes/src/volume_format.rs} | 2 +- dstack/dstack-types/Cargo.toml | 1 - dstack/dstack-types/src/lib.rs | 2 - .../recipes-core/dstack-guest/dstack-guest.bb | 2 +- 14 files changed, 47 insertions(+), 46 deletions(-) rename dstack/crates/{dstack-verity => dstack-volumes}/Cargo.toml (91%) rename dstack/{guest-agent => crates/dstack-volumes}/src/bin/dstack-volume.rs (99%) rename dstack/crates/{dstack-verity => dstack-volumes}/src/lib.rs (98%) rename dstack/crates/{dstack-verity => dstack-volumes}/src/oci.rs (100%) rename dstack/crates/{dstack-verity => dstack-volumes}/src/store.rs (100%) rename dstack/crates/{dstack-verity => dstack-volumes}/src/volume.rs (99%) rename dstack/{dstack-types/src/volume.rs => crates/dstack-volumes/src/volume_format.rs} (98%) diff --git a/dstack/Cargo.lock b/dstack/Cargo.lock index 83b5a4c2e..eee21744b 100644 --- a/dstack/Cargo.lock +++ b/dstack/Cargo.lock @@ -1839,7 +1839,7 @@ dependencies = [ "anyhow", "clap", "dstack-cli-core", - "dstack-verity", + "dstack-volumes", "fs-err", "serde_json", "tokio", @@ -2162,7 +2162,6 @@ dependencies = [ name = "dstack-types" version = "0.6.0" dependencies = [ - "binrw", "ciborium", "hex", "or-panic", @@ -2267,31 +2266,6 @@ dependencies = [ "tracing-subscriber", ] -[[package]] -name = "dstack-verity" -version = "0.6.0" -dependencies = [ - "anyhow", - "cmd_lib", - "dstack-types", - "flate2", - "fs-err", - "futures", - "gpt", - "hex", - "oci-client", - "rustix 0.38.44", - "serde", - "serde_json", - "sha2 0.10.9", - "tar", - "tempfile", - "tokio", - "tracing", - "uuid", - "xattr", -] - [[package]] name = "dstack-vmm" version = "0.6.0" @@ -2363,6 +2337,33 @@ dependencies = [ "serde_json", ] +[[package]] +name = "dstack-volumes" +version = "0.6.0" +dependencies = [ + "anyhow", + "binrw", + "cmd_lib", + "dstack-types", + "flate2", + "fs-err", + "futures", + "gpt", + "hex", + "oci-client", + "rustix 0.38.44", + "serde", + "serde_json", + "sha2 0.10.9", + "tar", + "tempfile", + "tokio", + "tracing", + "tracing-subscriber", + "uuid", + "xattr", +] + [[package]] name = "dstackup" version = "0.6.0" diff --git a/dstack/Cargo.toml b/dstack/Cargo.toml index 1c0ced796..b475a2bac 100644 --- a/dstack/Cargo.toml +++ b/dstack/Cargo.toml @@ -64,7 +64,7 @@ members = [ "size-parser", "crates/dstack-cli-core", "crates/dstack-cli", - "crates/dstack-verity", + "crates/dstack-volumes", "crates/dstackup", "crates/dstack-auth", "crates/api-auth", @@ -81,7 +81,7 @@ dstack-kms-rpc = { path = "kms/rpc" } dstack-guest-agent-rpc = { path = "guest-agent/rpc" } dstack-vmm-rpc = { path = "vmm/rpc" } dstack-cli-core = { path = "crates/dstack-cli-core" } -dstack-verity = { path = "crates/dstack-verity" } +dstack-volumes = { path = "crates/dstack-volumes" } dstack-api-auth = { path = "crates/api-auth" } dstack-build-info = { path = "crates/build-info" } cc-eventlog = { path = "cc-eventlog" } diff --git a/dstack/crates/dstack-cli/Cargo.toml b/dstack/crates/dstack-cli/Cargo.toml index cee4c1647..78e869162 100644 --- a/dstack/crates/dstack-cli/Cargo.toml +++ b/dstack/crates/dstack-cli/Cargo.toml @@ -18,7 +18,7 @@ path = "src/main.rs" anyhow.workspace = true clap.workspace = true dstack-cli-core.workspace = true -dstack-verity.workspace = true +dstack-volumes.workspace = true fs-err.workspace = true serde_json.workspace = true tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/dstack/crates/dstack-cli/src/main.rs b/dstack/crates/dstack-cli/src/main.rs index 969df27f8..9ef7494ad 100644 --- a/dstack/crates/dstack-cli/src/main.rs +++ b/dstack/crates/dstack-cli/src/main.rs @@ -154,7 +154,7 @@ async fn main() -> Result<()> { .without_time() .with_env_filter( EnvFilter::try_from_default_env() - .unwrap_or_else(|_| EnvFilter::new("warn,dstack_verity=info")), + .unwrap_or_else(|_| EnvFilter::new("warn,dstack_volumes=info")), ) .init(); let cli = Cli::parse(); @@ -268,12 +268,12 @@ async fn cmd_verity( json: bool, ) -> Result<()> { let compress = match compress { - "none" => dstack_verity::Compression::None, - "zstd" => dstack_verity::Compression::Zstd, - "gzip" => dstack_verity::Compression::Gzip, + "none" => dstack_volumes::Compression::None, + "zstd" => dstack_volumes::Compression::Zstd, + "gzip" => dstack_volumes::Compression::Gzip, other => bail!("unknown --compress '{other}' (use none|zstd|gzip)"), }; - let result = dstack_verity::verity(dstack_verity::VerityOptions { + let result = dstack_volumes::verity(dstack_volumes::VerityOptions { images: images.to_vec(), dir: dir.map(std::path::PathBuf::from), fs_image: fs_image.map(std::path::PathBuf::from), diff --git a/dstack/crates/dstack-verity/Cargo.toml b/dstack/crates/dstack-volumes/Cargo.toml similarity index 91% rename from dstack/crates/dstack-verity/Cargo.toml rename to dstack/crates/dstack-volumes/Cargo.toml index d4168583c..643087e9a 100644 --- a/dstack/crates/dstack-verity/Cargo.toml +++ b/dstack/crates/dstack-volumes/Cargo.toml @@ -3,13 +3,14 @@ # SPDX-License-Identifier: Apache-2.0 [package] -name = "dstack-verity" +name = "dstack-volumes" version.workspace = true edition.workspace = true license.workspace = true [dependencies] anyhow = { workspace = true, features = ["std"] } +binrw.workspace = true cmd_lib.workspace = true dstack-types.workspace = true tokio = { workspace = true, features = ["rt"] } @@ -24,6 +25,7 @@ gpt = "4.1.0" tar.workspace = true tempfile.workspace = true tracing.workspace = true +tracing-subscriber.workspace = true rustix = { version = "0.38", features = ["fs"] } uuid.workspace = true xattr = "1.5" diff --git a/dstack/guest-agent/src/bin/dstack-volume.rs b/dstack/crates/dstack-volumes/src/bin/dstack-volume.rs similarity index 99% rename from dstack/guest-agent/src/bin/dstack-volume.rs rename to dstack/crates/dstack-volumes/src/bin/dstack-volume.rs index 860ff28e7..87b4ef4a1 100644 --- a/dstack/guest-agent/src/bin/dstack-volume.rs +++ b/dstack/crates/dstack-volumes/src/bin/dstack-volume.rs @@ -18,10 +18,10 @@ use std::path::{Path, PathBuf}; use anyhow::{bail, Context, Result}; use cmd_lib::{run_cmd, run_fun}; -use dstack_types::volume::{ +use dstack_types::{AppCompose, VerityVolume as RequestedVolume, VolumeTarget}; +use dstack_volumes::volume_format::{ DstackVolumeHeader, DSTACK_VOLUME_HEADER_SIZE, DSTACK_VOLUME_KIND_VERITY, DSTACK_VOLUME_MAGIC, }; -use dstack_types::{AppCompose, VerityVolume as RequestedVolume, VolumeTarget}; use fs_err::{self as fs, File}; use serde_json::Value; use tracing::{info, warn}; diff --git a/dstack/crates/dstack-verity/src/lib.rs b/dstack/crates/dstack-volumes/src/lib.rs similarity index 98% rename from dstack/crates/dstack-verity/src/lib.rs rename to dstack/crates/dstack-volumes/src/lib.rs index 59f09430f..44d07513f 100644 --- a/dstack/crates/dstack-verity/src/lib.rs +++ b/dstack/crates/dstack-volumes/src/lib.rs @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 -//! Build a verity volume from docker images or a directory. +//! Build, describe, and activate dstack volumes. //! //! The output is a reproducible, dm-verity-protected raw disk image: partition 1 //! is the squashfs data filesystem, and partition 2 is the dm-verity superblock @@ -24,6 +24,7 @@ use fs_err as fs; pub mod oci; mod store; mod volume; +pub mod volume_format; pub use volume::Compression; diff --git a/dstack/crates/dstack-verity/src/oci.rs b/dstack/crates/dstack-volumes/src/oci.rs similarity index 100% rename from dstack/crates/dstack-verity/src/oci.rs rename to dstack/crates/dstack-volumes/src/oci.rs diff --git a/dstack/crates/dstack-verity/src/store.rs b/dstack/crates/dstack-volumes/src/store.rs similarity index 100% rename from dstack/crates/dstack-verity/src/store.rs rename to dstack/crates/dstack-volumes/src/store.rs diff --git a/dstack/crates/dstack-verity/src/volume.rs b/dstack/crates/dstack-volumes/src/volume.rs similarity index 99% rename from dstack/crates/dstack-verity/src/volume.rs rename to dstack/crates/dstack-volumes/src/volume.rs index 047ba0a29..b6d37fc33 100644 --- a/dstack/crates/dstack-verity/src/volume.rs +++ b/dstack/crates/dstack-volumes/src/volume.rs @@ -18,9 +18,9 @@ use std::collections::BTreeMap; use std::path::Path; +use crate::volume_format::{DstackVolumeHeader, DSTACK_VOLUME_HEADER_SIZE}; use anyhow::{bail, Context, Result}; use cmd_lib::{run_cmd, run_fun}; -use dstack_types::volume::DstackVolumeHeader; use fs_err as fs; use sha2::{Digest, Sha256}; use uuid::Uuid; @@ -37,7 +37,7 @@ const PARTITION_ALIGNMENT_SECTORS: u64 = 2048; // still reserve enough trailing sectors in the raw image for the backup array // (128 entries * 128 bytes) plus the backup header. const GPT_ENTRY_SECTORS: u64 = 32; -const VOLUME_HEADER_SIZE: usize = dstack_types::volume::DSTACK_VOLUME_HEADER_SIZE; +const VOLUME_HEADER_SIZE: usize = DSTACK_VOLUME_HEADER_SIZE; #[derive(Clone, Copy)] pub enum Compression { diff --git a/dstack/dstack-types/src/volume.rs b/dstack/crates/dstack-volumes/src/volume_format.rs similarity index 98% rename from dstack/dstack-types/src/volume.rs rename to dstack/crates/dstack-volumes/src/volume_format.rs index a5ad812af..9f1be78da 100644 --- a/dstack/dstack-types/src/volume.rs +++ b/dstack/crates/dstack-volumes/src/volume_format.rs @@ -51,7 +51,7 @@ impl DstackVolumeHeader { #[cfg(test)] mod tests { use super::*; - use crate::{VerityVolume, VolumeTarget}; + use dstack_types::{VerityVolume, VolumeTarget}; #[test] fn volume_header_round_trip() { diff --git a/dstack/dstack-types/Cargo.toml b/dstack/dstack-types/Cargo.toml index 452e6e92b..526d5192b 100644 --- a/dstack/dstack-types/Cargo.toml +++ b/dstack/dstack-types/Cargo.toml @@ -10,7 +10,6 @@ edition.workspace = true license.workspace = true [dependencies] -binrw.workspace = true ciborium.workspace = true hex = { workspace = true, features = ["std"] } or-panic.workspace = true diff --git a/dstack/dstack-types/src/lib.rs b/dstack/dstack-types/src/lib.rs index 5e5d45189..75a808adc 100644 --- a/dstack/dstack-types/src/lib.rs +++ b/dstack/dstack-types/src/lib.rs @@ -10,8 +10,6 @@ use serde::{Deserialize, Serialize}; use serde_human_bytes as hex_bytes; use size_parser::human_size; -pub mod volume; - /// Identifies which OVMF flavour the guest image was built with. /// /// Only the pre-202505 OVMF measurement layout is supported. diff --git a/os/yocto/layers/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/os/yocto/layers/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index 3e62db5d0..b93e77836 100644 --- a/os/yocto/layers/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/os/yocto/layers/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -32,7 +32,7 @@ DSTACK_SERVICES = "dstack-guest-agent.service dstack-guest-agent.socket dstack-p SYSTEMD_PACKAGES = "${@bb.utils.contains('DISTRO_FEATURES','systemd','${PN}','',d)}" SYSTEMD_SERVICE:${PN} = "${@bb.utils.contains('DISTRO_FEATURES','systemd','${DSTACK_SERVICES}','',d)}" SYSTEMD_AUTO_ENABLE:${PN} = "enable" -EXTRA_CARGO_FLAGS = "-p dstack-guest-agent -p dstack-util" +EXTRA_CARGO_FLAGS = "-p dstack-guest-agent -p dstack-util -p dstack-volumes" inherit cargo_bin From 39ae0b8a272c7b1d791a0751e88093d83c7565ed Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 20 Jul 2026 20:44:02 -0700 Subject: [PATCH 16/25] refactor(volume): use singular crate name --- dstack/Cargo.lock | 4 ++-- dstack/Cargo.toml | 4 ++-- dstack/crates/dstack-cli/Cargo.toml | 2 +- dstack/crates/dstack-cli/src/main.rs | 10 +++++----- .../{dstack-volumes => dstack-volume}/Cargo.toml | 2 +- .../src/bin/dstack-volume.rs | 2 +- .../{dstack-volumes => dstack-volume}/src/lib.rs | 0 .../{dstack-volumes => dstack-volume}/src/oci.rs | 0 .../{dstack-volumes => dstack-volume}/src/store.rs | 0 .../{dstack-volumes => dstack-volume}/src/volume.rs | 0 .../src/volume_format.rs | 0 .../recipes-core/dstack-guest/dstack-guest.bb | 2 +- 12 files changed, 13 insertions(+), 13 deletions(-) rename dstack/crates/{dstack-volumes => dstack-volume}/Cargo.toml (97%) rename dstack/crates/{dstack-volumes => dstack-volume}/src/bin/dstack-volume.rs (99%) rename dstack/crates/{dstack-volumes => dstack-volume}/src/lib.rs (100%) rename dstack/crates/{dstack-volumes => dstack-volume}/src/oci.rs (100%) rename dstack/crates/{dstack-volumes => dstack-volume}/src/store.rs (100%) rename dstack/crates/{dstack-volumes => dstack-volume}/src/volume.rs (100%) rename dstack/crates/{dstack-volumes => dstack-volume}/src/volume_format.rs (100%) diff --git a/dstack/Cargo.lock b/dstack/Cargo.lock index eee21744b..1f2b158f2 100644 --- a/dstack/Cargo.lock +++ b/dstack/Cargo.lock @@ -1839,7 +1839,7 @@ dependencies = [ "anyhow", "clap", "dstack-cli-core", - "dstack-volumes", + "dstack-volume", "fs-err", "serde_json", "tokio", @@ -2338,7 +2338,7 @@ dependencies = [ ] [[package]] -name = "dstack-volumes" +name = "dstack-volume" version = "0.6.0" dependencies = [ "anyhow", diff --git a/dstack/Cargo.toml b/dstack/Cargo.toml index b475a2bac..73b3cd19b 100644 --- a/dstack/Cargo.toml +++ b/dstack/Cargo.toml @@ -64,7 +64,7 @@ members = [ "size-parser", "crates/dstack-cli-core", "crates/dstack-cli", - "crates/dstack-volumes", + "crates/dstack-volume", "crates/dstackup", "crates/dstack-auth", "crates/api-auth", @@ -81,7 +81,7 @@ dstack-kms-rpc = { path = "kms/rpc" } dstack-guest-agent-rpc = { path = "guest-agent/rpc" } dstack-vmm-rpc = { path = "vmm/rpc" } dstack-cli-core = { path = "crates/dstack-cli-core" } -dstack-volumes = { path = "crates/dstack-volumes" } +dstack-volume = { path = "crates/dstack-volume" } dstack-api-auth = { path = "crates/api-auth" } dstack-build-info = { path = "crates/build-info" } cc-eventlog = { path = "cc-eventlog" } diff --git a/dstack/crates/dstack-cli/Cargo.toml b/dstack/crates/dstack-cli/Cargo.toml index 78e869162..bf522939a 100644 --- a/dstack/crates/dstack-cli/Cargo.toml +++ b/dstack/crates/dstack-cli/Cargo.toml @@ -18,7 +18,7 @@ path = "src/main.rs" anyhow.workspace = true clap.workspace = true dstack-cli-core.workspace = true -dstack-volumes.workspace = true +dstack-volume.workspace = true fs-err.workspace = true serde_json.workspace = true tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/dstack/crates/dstack-cli/src/main.rs b/dstack/crates/dstack-cli/src/main.rs index 9ef7494ad..2781bc59b 100644 --- a/dstack/crates/dstack-cli/src/main.rs +++ b/dstack/crates/dstack-cli/src/main.rs @@ -154,7 +154,7 @@ async fn main() -> Result<()> { .without_time() .with_env_filter( EnvFilter::try_from_default_env() - .unwrap_or_else(|_| EnvFilter::new("warn,dstack_volumes=info")), + .unwrap_or_else(|_| EnvFilter::new("warn,dstack_volume=info")), ) .init(); let cli = Cli::parse(); @@ -268,12 +268,12 @@ async fn cmd_verity( json: bool, ) -> Result<()> { let compress = match compress { - "none" => dstack_volumes::Compression::None, - "zstd" => dstack_volumes::Compression::Zstd, - "gzip" => dstack_volumes::Compression::Gzip, + "none" => dstack_volume::Compression::None, + "zstd" => dstack_volume::Compression::Zstd, + "gzip" => dstack_volume::Compression::Gzip, other => bail!("unknown --compress '{other}' (use none|zstd|gzip)"), }; - let result = dstack_volumes::verity(dstack_volumes::VerityOptions { + let result = dstack_volume::verity(dstack_volume::VerityOptions { images: images.to_vec(), dir: dir.map(std::path::PathBuf::from), fs_image: fs_image.map(std::path::PathBuf::from), diff --git a/dstack/crates/dstack-volumes/Cargo.toml b/dstack/crates/dstack-volume/Cargo.toml similarity index 97% rename from dstack/crates/dstack-volumes/Cargo.toml rename to dstack/crates/dstack-volume/Cargo.toml index 643087e9a..9b175bbea 100644 --- a/dstack/crates/dstack-volumes/Cargo.toml +++ b/dstack/crates/dstack-volume/Cargo.toml @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 [package] -name = "dstack-volumes" +name = "dstack-volume" version.workspace = true edition.workspace = true license.workspace = true diff --git a/dstack/crates/dstack-volumes/src/bin/dstack-volume.rs b/dstack/crates/dstack-volume/src/bin/dstack-volume.rs similarity index 99% rename from dstack/crates/dstack-volumes/src/bin/dstack-volume.rs rename to dstack/crates/dstack-volume/src/bin/dstack-volume.rs index 87b4ef4a1..f4d1c6cc1 100644 --- a/dstack/crates/dstack-volumes/src/bin/dstack-volume.rs +++ b/dstack/crates/dstack-volume/src/bin/dstack-volume.rs @@ -19,7 +19,7 @@ use std::path::{Path, PathBuf}; use anyhow::{bail, Context, Result}; use cmd_lib::{run_cmd, run_fun}; use dstack_types::{AppCompose, VerityVolume as RequestedVolume, VolumeTarget}; -use dstack_volumes::volume_format::{ +use dstack_volume::volume_format::{ DstackVolumeHeader, DSTACK_VOLUME_HEADER_SIZE, DSTACK_VOLUME_KIND_VERITY, DSTACK_VOLUME_MAGIC, }; use fs_err::{self as fs, File}; diff --git a/dstack/crates/dstack-volumes/src/lib.rs b/dstack/crates/dstack-volume/src/lib.rs similarity index 100% rename from dstack/crates/dstack-volumes/src/lib.rs rename to dstack/crates/dstack-volume/src/lib.rs diff --git a/dstack/crates/dstack-volumes/src/oci.rs b/dstack/crates/dstack-volume/src/oci.rs similarity index 100% rename from dstack/crates/dstack-volumes/src/oci.rs rename to dstack/crates/dstack-volume/src/oci.rs diff --git a/dstack/crates/dstack-volumes/src/store.rs b/dstack/crates/dstack-volume/src/store.rs similarity index 100% rename from dstack/crates/dstack-volumes/src/store.rs rename to dstack/crates/dstack-volume/src/store.rs diff --git a/dstack/crates/dstack-volumes/src/volume.rs b/dstack/crates/dstack-volume/src/volume.rs similarity index 100% rename from dstack/crates/dstack-volumes/src/volume.rs rename to dstack/crates/dstack-volume/src/volume.rs diff --git a/dstack/crates/dstack-volumes/src/volume_format.rs b/dstack/crates/dstack-volume/src/volume_format.rs similarity index 100% rename from dstack/crates/dstack-volumes/src/volume_format.rs rename to dstack/crates/dstack-volume/src/volume_format.rs diff --git a/os/yocto/layers/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/os/yocto/layers/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index b93e77836..dd0097f4d 100644 --- a/os/yocto/layers/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/os/yocto/layers/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -32,7 +32,7 @@ DSTACK_SERVICES = "dstack-guest-agent.service dstack-guest-agent.socket dstack-p SYSTEMD_PACKAGES = "${@bb.utils.contains('DISTRO_FEATURES','systemd','${PN}','',d)}" SYSTEMD_SERVICE:${PN} = "${@bb.utils.contains('DISTRO_FEATURES','systemd','${DSTACK_SERVICES}','',d)}" SYSTEMD_AUTO_ENABLE:${PN} = "enable" -EXTRA_CARGO_FLAGS = "-p dstack-guest-agent -p dstack-util -p dstack-volumes" +EXTRA_CARGO_FLAGS = "-p dstack-guest-agent -p dstack-util -p dstack-volume" inherit cargo_bin From 34c1d6b2c6c5058ae35a907a58a9790ad01fbaaf Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 21 Jul 2026 00:52:59 -0700 Subject: [PATCH 17/25] refactor(volume): drop docker image seeding --- docs/verity-volumes.md | 175 ++----- dstack/Cargo.lock | 227 +-------- dstack/crates/dstack-cli/src/main.rs | 98 +--- dstack/crates/dstack-volume/Cargo.toml | 7 - .../dstack-volume/src/bin/dstack-volume.rs | 197 +------- dstack/crates/dstack-volume/src/lib.rs | 94 +--- dstack/crates/dstack-volume/src/oci.rs | 320 ------------ dstack/crates/dstack-volume/src/store.rs | 458 ------------------ dstack/crates/dstack-volume/src/volume.rs | 3 +- .../crates/dstack-volume/src/volume_format.rs | 6 +- dstack/dstack-types/src/lib.rs | 54 +-- os/common/rootfs/dstack-prepare.sh | 5 +- 12 files changed, 117 insertions(+), 1527 deletions(-) delete mode 100644 dstack/crates/dstack-volume/src/oci.rs delete mode 100644 dstack/crates/dstack-volume/src/store.rs diff --git a/docs/verity-volumes.md b/docs/verity-volumes.md index 75016095a..d14f2f709 100644 --- a/docs/verity-volumes.md +++ b/docs/verity-volumes.md @@ -1,157 +1,80 @@ -# Verity Volumes: pre-seeding images and data into a CVM +# Verity data volumes -Starting a CVM is slow mostly because of image *extraction*: decompressing layers and writing millions of files onto the encrypted disk. On one pinned vCPU this dominates — about 30 s for a 3.3 GB image, ~2 min for 7 GB, with a hard ~15 s decompress floor no amount of storage tuning beats. Download is the smaller cost. +A verity volume is a read-only filesystem image protected by dm-verity. The host +attaches untrusted bytes; the guest mounts them only when they match the root +hash measured in `app-compose.json`. -A verity volume removes it. It's a read-only, dm-verity-protected disk, built once, that a CVM mounts and uses as-is. The layers on it are already extracted, so the CVM does no pull and no extraction — it mounts the volume and verifies blocks lazily as the app reads them. One volume can back many CVMs at once. +## Disk format -The build tool (`dstack verity`) and the in-guest seeding are implemented and validated end to end ([What's proven](#whats-proven)). +A built volume is a raw GPT image: -## The shape of it - -A volume has a **root hash** — its identity and integrity check — and a **target** that says what it's for: - -- `"docker"` — the volume is a docker overlay2 store; its images are seeded into docker as cache hits. -- `"/some/path"` — the volume's filesystem is mounted there, for data like model weights. - -On disk, a volume is a raw GPT image with three partitions: - -``` +```text p1: DSTACK_VOLUME metadata envelope p2: filesystem data -p3: dm-verity superblock + hash tree +p3: dm-verity superblock and hash tree ``` -The guest finds disks by the `DSTACK_VOLUME` magic in `p1`, then opens `p2` as the verity data device and `p3` as the verity hash device. That keeps the verity layer independent of the filesystem format: the guest does not need to read a squashfs/ext4 superblock to find a hash offset. +The envelope identifies a candidate disk. It is not trusted. The guest passes +the root from the measured app compose to `veritysetup`, so a forged envelope or +partition table cannot substitute different contents. -An app lists its volumes in `app-compose.json`. dstack already hashes that file into the app's identity, so every root hash is measured as part of `app_id`, and the CVM only uses content matching what it was attested as. Everything else is untrusted: the host hands over bytes, and dm-verity rejects any that don't match the root. +## Build -A volume is *additive* — it only adds read-only lower layers, so anything not on one is pulled normally — and *fail-safe*: a missing or mismatched volume falls back to a pull. - -## What a user writes - -The `docker-compose.yaml` is unchanged (pin images by digest so the measured identity binds the exact bytes): - -A data volume's mount point must be on a writable filesystem — the guest rootfs is read-only, so use `/run/...` (a writable tmpfs) or the app's data disk, not an arbitrary top-level path like `/models`: - -```yaml -services: - vllm: - image: vllm/vllm-openai@sha256:abc123... # from the "docker" volume - command: ["--model", "/run/models/llama-70b"] - volumes: - - /run/models/llama-70b:/run/models/llama-70b:ro # from the data volume -``` - -Each volume is one `verity_volumes` entry in the measured `app-compose.json`: - -```json -"verity_volumes": [ - { "verity_root": "115b6877...", "target": "docker" }, - { "verity_root": "a1b2c3d4...", "target": "/run/models/llama-70b" } -] -``` - -You don't hand-write that with the `dstack` CLI — `dstack deploy --volume` generates it (below). Build volumes with `dstack verity`, which prints the exact `--volume` spec to paste: +Pack a directory as a reproducible squashfs volume: ```bash -dstack verity vllm/vllm-openai@sha256:abc123... # -> --volume vllm.img:115b6877...:docker -dstack verity --dir ./llama-70b-weights/ # -> --volume llama-70b.img:a1b2c3d4...:/run/models/llama-70b -dstack verity --fs-image ./weights.ext4 # -> wrap an existing filesystem image +dstack verity --dir ./models -o models.img ``` -It needs no docker daemon, no CVM, and no TDX. For images it pulls the pinned layers from the registry and lays out the docker store itself; for `--dir` it packs the directory as squashfs; for `--fs-image` it treats the supplied filesystem image as opaque bytes and only wraps it in verity/GPT. Either way it writes one volume file. Pass several images to pack them into a single `docker` volume; shared base layers are stored once. There is no auto-detection: a volume contains whatever you built it from, and the compose names those images normally. - -This covers both cases. Several volumes in one CVM (the image *and* the model) is several entries with different targets. Several images from one volume is a single `docker` volume built from all of them, listed as one entry, with the compose unchanged. Attach order does not matter ([Delivery](#delivery)). - -## Trust - -`verity_root` lives in `app-compose.json`, so it's part of the hash that becomes `app_id` (see [Normalized App Compose](./normalized-app-compose.md)), and it is enforced by dm-verity. The consumer trusts the root hash, not whoever built the volume. A host that tampers with the bytes causes a verity fault, not a silent swap, so the delivery path can stay untrusted. - -`dstack verity` is deterministic: the same pinned image digests always produce the same `verity_root`, bit-for-bit (canonical overlay2 layout with chain-id cache-ids; a fixed timestamp and salt; UUIDs derived from the packed bytes/root hash; a deterministic GPT wrapper). One caveat: the squashfs layout is produced by `mksquashfs`, so a verifier must use a `squashfs-tools` version that lays out bytes identically (recent versions are stable; the build records nothing about the tool version yet). A verifier can therefore recompute the root from the digest-pinned images and confirm the volume is those images, without trusting whoever ran the build. The build needs no docker daemon and no TEE, so it can also run in a CI job that attests it (see [Reproducible builds and provenance](#reproducible-builds-and-provenance)); the two checks are independent. Turning on a volume changes `app-compose.json` and thus `app_id`, which is correct: the volume is part of the measured configuration. +Or wrap an existing filesystem image: -## How docker seeding works - -A `docker` volume is a squashfs filesystem holding a docker overlay2 store, which has two very different halves: - -``` -image/overlay2/ metadata: image configs, layer db, tags (KB to a few MB) -overlay2//diff/ the already-extracted layer files (the GBs) +```bash +dstack verity --fs-image ./models.ext4 -o models.img ``` -`diff/` is the layer already decompressed and untarred. Seeding reuses it in place: - -1. Open the volume with veritysetup (`p2` as data, `p3` as hash) and mount it read-only at a writable path (`/run/dstack-verity`). The rootfs is read-only dm-verity, so the mountpoint has to be on a writable fs; squashfs itself mounts read-only directly, no journal and nothing to replay. -2. Copy the metadata (a few MB) into `/var/lib/docker`. -3. Per layer: make a writable `overlay2//` dir, copy its handful of tiny files, and bind-mount only the big `diff/` read-only from the volume. -4. This runs from `dstack-prepare.sh` *before* dockerd starts, so there's no restart — dockerd comes up with every image on the volume already present. - -The layer dir stays writable because docker writes a `committed` marker into it on first use; only `diff/` underneath is read-only. (Bind-mounting the whole dir read-only is the one thing that breaks container creation.) - -squashfs is used, rather than ext4, for two reasons: the dstack guest kernel mounts it (it's the guest's own rootfs — erofs, the other obvious candidate, isn't compiled in), and `mksquashfs` with a pinned timestamp is byte-reproducible with no extra work, where ext4 needs its wall-clock superblock and inode times patched out. It's built fully uncompressed, so there's no decompression at read time either — the point was to remove extraction, not move it. - -At run time the container's overlay uses the verity-backed `diff/`s as lower layers and a fresh writable upper on the encrypted disk. Reads are verified per block on first touch, then cached; nothing is decompressed or written. This is where the ~1 s start comes from instead of 30 s–2 min. A data volume is simpler: open, mount at the path, and the container bind-mounts it — no unpacking. - -## Delivery - -The volume *file* and the deploy request travel separately. First set `cvm.volumes_dir` in the host's `vmm.toml` (it's empty by default, which disables volume attachment). The operator then places the built file in that `volumes_dir` (however they like — copy, object store, shared mount), and deploy references it by bare file name: +The command prints the root and a deploy argument. Place the image in the VMM's +configured `cvm.volumes_dir`, then deploy it by bare file name, root, and guest +mount point: ```bash dstack deploy -c docker-compose.yaml \ - --volume vllm.img:115b6877...:docker \ - --volume llama-70b.img:a1b2c3d4...:/run/models/llama-70b + --volume models.img:a1b2c3d4...:/run/models ``` -Each `--volume NAME:VERITY_ROOT:TARGET` both attaches the file and writes the measured `verity_volumes` entry (the root is yours to supply, from `dstack verity`, so it stays part of `app_id`). `vmm-cli.py` takes the same file with a hand-authored `app-compose.json` instead. The vmm resolves each name against `volumes_dir` (rejecting anything with a path separator) and attaches it as a read-only virtio-blk device, `--volume` carries only the name and read-only flag, never the bytes; the bytes are already on the host. Disks are still attached in any order. The Rust guest helper scans each whole disk: when the kernel exposes partitions it checks partition 1, otherwise it checks the start of the raw disk. A matching `DSTACK_VOLUME` envelope supplies a kind and the full claimed root. For the verity kind, the guest opens partitions 2 and 3 only after matching that root against the measured compose. The envelope and partition table are untrusted hints; the guest passes the measured root to dm-verity, so tampering causes a verity fault rather than a wrong mount. Integrity is checked lazily, per block on first read. Read-only plus content-addressed means one physical copy serves every CVM that references it: a base image or model shared by a hundred replicas is built, stored, and extracted once. +This adds the following measured entry to `app-compose.json`: -## Building volumes - -`dstack verity` builds the store from scratch, without a docker daemon and without a CVM — just the registry (a `docker` volume does need root, because laying out overlay2 whiteouts uses `mknod` and a `trusted.*` xattr; a `--dir` or `--fs-image` volume needs no root). It pulls each layer by digest, and lays out the overlay2 store the way docker would, with one deliberate change: the per-layer directory id is the layer's *chain-id* instead of docker's random cache-id. That's what makes the store a pure function of the image. AUFS `.wh.` whiteouts in the layer tars are converted to their overlay2 on-disk form (a `0:0` char device, or the `trusted.overlay.opaque` xattr) so deletions in upper layers still take effect. Then `mksquashfs` packs it with a fixed timestamp, `veritysetup` builds a separate hash image with a fixed salt and a UUID derived from the filesystem bytes, and the builder wraps both blobs in a deterministic GPT disk (`p1` volume metadata, `p2` data, `p3` verity metadata/hash tree). A `--dir` volume skips the overlay2 layout and pull entirely — it packs the directory straight into the same partitioned squashfs + verity format. A `--fs-image` volume skips `mksquashfs` too: the supplied ext4/xfs/etc. image becomes `p2` after 4096-byte padding, so the guest only needs kernel support and suitable read-only mount behavior for that filesystem. - -With those fixed, two runs of the same digests produce the same bytes. The build needs no daemon and no TEE, so it can run in a CI job that also attests it (below). - -### Reproducible builds and provenance - -Because the root is a deterministic function of the pinned image digests, a verifier has two independent checks: - -- **Reproducibility** — recompute the root from the digests and check it matches `app-compose.json`. This needs nothing but the digests and does not trust the builder. -- **Provenance** — run `verity` in CI and attest the output with [SLSA build provenance](https://github.com/actions/attest-build-provenance) / Sigstore. This ties the root to a specific workflow and inputs. - -Either alone is sufficient; they can be used together. In your own app repo — using an installed `dstack` (verity volumes aren't tied to the dstack source tree): - -```yaml -# .github/workflows/build-verity-volume.yml -permissions: { id-token: write, attestations: write, contents: read } -jobs: - build: - runs-on: ubuntu-24.04 - steps: - - run: sudo apt-get install -y squashfs-tools cryptsetup-bin - - run: sudo dstack verity "$IMAGE" -o volume.img --json | tee out.json - - uses: actions/attest-build-provenance@v2 - with: { subject-path: volume.img } +```json +{ + "verity_volumes": [ + { + "verity_root": "a1b2c3d4...", + "target": "/run/models" + } + ] +} ``` -(`sudo` because the docker-store layout needs `mknod` + a `trusted.*` xattr.) - -## What this doesn't do yet +The target must be an absolute path on a writable guest filesystem, such as +`/run` or the app data disk. The guest root filesystem itself is read-only. -- Images should be pinned by digest; a mutable tag doesn't bind the bytes (a digest pin is verified against the fetched manifest, so a swapped registry response is caught). -- `verity` reads gzip and uncompressed layers; zstd-compressed layers are not handled yet (it fails rather than producing a wrong store). -- The compose must reference an image the same way it was passed to `verity`. Docker-Hub names are normalized to their canonical form (`docker.io/library/`), so a bare `alpine` and `docker.io/library/alpine` match; a private-registry reference must match verbatim. -- A data volume's `target` is a mount point, and must be on a writable filesystem in the guest (e.g. under `/run`, or the app's data disk) — the rootfs is read-only dm-verity, so an arbitrary top-level path can't be created. A missing/unwritable target is skipped, fail-safe. -- A `docker` volume is architecture-specific. `--platform` selects it (default `linux/amd64`, matching today's Intel TDX guests); an arm64 confidential host (NVIDIA Vera, AWS Graviton/Nitro) needs `linux/arm64`. dm-verity checks bytes, not architecture, so a wrong-arch volume whose root you pinned still opens and seeds — the container then fails to exec (wrong-arch binaries), it isn't caught up front. Match `--platform` to the guest; recording the platform in the volume and checking it in the guest is a follow-up. -- Layers are treated as public — the volume is unencrypted and shareable. Secret layers would need a per-app, KMS-encrypted volume, and that's out of scope for now. -- Under TDX the single-copy saving is on storage, transfer, and extraction, not RAM — each CVM still caches the blocks it reads in its own encrypted memory. -- The per-boot `docker image prune -af` removes images no running container references, so bake the images your compose actually runs; guarding prune against verity-seeded images is a follow-up. -- Verity volumes require a guest image that ships `/bin/dstack-volume`; the Rust helper runs from `dstack-prepare.sh` before dockerd starts. -- Seeded overlay2 metadata lives on the persistent disk, while the layer `diff/` binds are re-established each boot. A *partial* seed (a metadata copy fails mid-write) unwinds its binds and falls back to a normal pull. The open gap is across boots: if a volume that seeded once is later not re-attached (or swapped), its metadata persists while its `diff/` binds don't, so docker can see the image present with empty layers. The normal reboot (same volume) re-binds correctly. Reconciling stale seeded metadata against missing binds on boot is a follow-up — it needs image→layer dependency walking so it doesn't drop a pulled image that happens to share a base layer. +## Guest activation -## What's proven +Before the application starts, `dstack-volume`: -The whole path ran on Intel TDX with no change to the guest OS image — only a host-side change to attach the disk. A fresh CVM — different app, empty image list, never having pulled the image — attached a pre-built volume read-only, matched it by root hash, mounted it, and seeded docker. The image was present with no pull and no extraction: essentially nothing was written to the CVM's disk during seeding (an extraction would write gigabytes), seeding took about a second, and the container ran in about a second — versus 30 s to 2 min to pull and extract. That confirms a fresh CVM gets an image purely from a read-only attested volume, and that unrelated images still pull. +1. Scans `/sys/class/block` for disks whose first partition, or whole disk when + unpartitioned, starts with the `DSTACK_VOLUME` magic. +2. Matches the full root advertised by the envelope against the measured root. +3. Opens p2 and p3 with `veritysetup` using the measured root. +4. Reads the first mapped block to force an initial integrity check. +5. Mounts the mapped filesystem read-only at the measured target. -The build side is proven too. `dstack verity` builds a volume daemonlessly and is byte-for-byte reproducible: independent runs of the same pinned image produce the same volume and the same `verity_root`, including a multi-layer image with whiteouts. The resulting store loads and runs in a stock docker with no pull, and the overlay2 whiteouts apply correctly (deleted files stay deleted). The filesystem choice was checked against the live guest kernel: it mounts uncompressed squashfs and reads it back correctly; erofs is not compiled in. +A required volume that is missing, malformed, or fails verification stops guest +preparation. dm-verity continues verifying blocks lazily as the application +reads them. -## Open questions +## Trust and limitations -Whether the guest-agent should handle `verity_volumes` natively instead of through a pre-launch helper; how the operator garbage-collects stale volumes and tracks roots across image updates; how volume files are distributed to hosts at scale (`--volume` references a name already in `volumes_dir`); and encrypted volumes for secret layers. +The root hash authenticates bytes, not availability. A malicious host can omit a +volume or cause I/O failures, but cannot silently replace its contents. Volumes +are unencrypted and are intended for public, shareable data; confidential data +requires a separate encryption design. diff --git a/dstack/Cargo.lock b/dstack/Cargo.lock index 1f2b158f2..b920c479b 100644 --- a/dstack/Cargo.lock +++ b/dstack/Cargo.lock @@ -14,7 +14,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" dependencies = [ - "crypto-common 0.1.7", + "crypto-common", "generic-array", ] @@ -525,15 +525,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "block-buffer" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" -dependencies = [ - "hybrid-array", -] - [[package]] name = "blowfish" version = "0.9.1" @@ -880,7 +871,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common 0.1.7", + "crypto-common", "inout", "zeroize", ] @@ -999,12 +990,6 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" -[[package]] -name = "const-oid" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" - [[package]] name = "const_format" version = "0.2.36" @@ -1228,15 +1213,6 @@ dependencies = [ "typenum", ] -[[package]] -name = "crypto-common" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" -dependencies = [ - "hybrid-array", -] - [[package]] name = "crypto-mac" version = "0.11.0" @@ -1415,7 +1391,7 @@ dependencies = [ "borsh", "byteorder", "chrono", - "const-oid 0.9.6", + "const-oid", "dcap-qvl-webpki", "der", "derive_more 2.1.1", @@ -1481,7 +1457,7 @@ version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ - "const-oid 0.9.6", + "const-oid", "der_derive", "flagset", "pem-rfc7468", @@ -1523,37 +1499,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "derive_builder" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" -dependencies = [ - "derive_builder_macro", -] - -[[package]] -name = "derive_builder_core" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" -dependencies = [ - "darling 0.20.11", - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "derive_builder_macro" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" -dependencies = [ - "derive_builder_core", - "syn 2.0.117", -] - [[package]] name = "derive_more" version = "1.0.0" @@ -1648,22 +1593,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", - "const-oid 0.9.6", - "crypto-common 0.1.7", + "const-oid", + "crypto-common", "subtle", ] -[[package]] -name = "digest" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" -dependencies = [ - "block-buffer 0.12.1", - "const-oid 0.10.2", - "crypto-common 0.2.2", -] - [[package]] name = "dirs" version = "5.0.1" @@ -2345,23 +2279,16 @@ dependencies = [ "binrw", "cmd_lib", "dstack-types", - "flate2", "fs-err", - "futures", "gpt", "hex", - "oci-client", - "rustix 0.38.44", - "serde", "serde_json", "sha2 0.10.9", - "tar", "tempfile", "tokio", "tracing", "tracing-subscriber", "uuid", - "xattr", ] [[package]] @@ -2957,17 +2884,6 @@ dependencies = [ "rand_core 0.6.4", ] -[[package]] -name = "getset" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cf442baaabe4213ce7d1239afc26c039180b6456da2cededa316ae2c8a77a77" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - [[package]] name = "ghash" version = "0.5.1" @@ -3329,15 +3245,6 @@ dependencies = [ "itoa", ] -[[package]] -name = "http-auth" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "150fa4a9462ef926824cf4519c84ed652ca8f4fbae34cb8af045b5cbcaf98822" -dependencies = [ - "memchr", -] - [[package]] name = "http-body" version = "1.0.1" @@ -3407,15 +3314,6 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" -[[package]] -name = "hybrid-array" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" -dependencies = [ - "typenum", -] - [[package]] name = "hyper" version = "1.10.1" @@ -3985,22 +3883,6 @@ dependencies = [ "regex-syntax", ] -[[package]] -name = "jsonwebtoken" -version = "10.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eba32bfb4ffdeaca3e34431072faf01745c9b26d25504aa7a6cf5684334fc4fc" -dependencies = [ - "aws-lc-rs", - "base64 0.22.1", - "getrandom 0.2.17", - "js-sys", - "serde", - "serde_json", - "signature", - "zeroize", -] - [[package]] name = "k256" version = "0.13.4" @@ -4793,50 +4675,6 @@ dependencies = [ "ruzstd", ] -[[package]] -name = "oci-client" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5261a7fb43d9c53b8e63e6d5e86860719dad253d015d022066c72d585125aed8" -dependencies = [ - "bytes", - "chrono", - "futures-util", - "hex", - "http", - "http-auth", - "jsonwebtoken", - "lazy_static", - "oci-spec", - "olpc-cjson", - "regex", - "reqwest", - "serde", - "serde_json", - "sha2 0.11.0", - "thiserror 2.0.18", - "tokio", - "tracing", - "unicase", -] - -[[package]] -name = "oci-spec" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8445a2631507cec628a15fdd6154b54a3ab3f20ed4fe9d73a3b8b7a4e1ba03a" -dependencies = [ - "const_format", - "derive_builder", - "getset", - "regex", - "serde", - "serde_json", - "strum", - "strum_macros", - "thiserror 2.0.18", -] - [[package]] name = "oid-registry" version = "0.7.1" @@ -4846,17 +4684,6 @@ dependencies = [ "asn1-rs", ] -[[package]] -name = "olpc-cjson" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "696183c9b5fe81a7715d074fd632e8bd46f4ccc0231a3ed7fc580a80de5f7083" -dependencies = [ - "serde", - "serde_json", - "unicode-normalization", -] - [[package]] name = "once_cell" version = "1.21.4" @@ -6030,14 +5857,12 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-rustls", - "tokio-util", "tower", "tower-http", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", - "wasm-streams", "web-sys", ] @@ -6243,7 +6068,7 @@ version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" dependencies = [ - "const-oid 0.9.6", + "const-oid", "digest 0.10.7", "num-bigint-dig", "num-integer", @@ -7023,17 +6848,6 @@ dependencies = [ "digest 0.10.7", ] -[[package]] -name = "sha2" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" -dependencies = [ - "cfg-if", - "cpufeatures 0.3.0", - "digest 0.11.3", -] - [[package]] name = "sha3" version = "0.10.9" @@ -7951,7 +7765,6 @@ version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ - "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -8067,15 +7880,6 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" -[[package]] -name = "unicode-normalization" -version = "0.1.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" -dependencies = [ - "tinyvec", -] - [[package]] name = "unicode-segmentation" version = "1.13.2" @@ -8094,7 +7898,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" dependencies = [ - "crypto-common 0.1.7", + "crypto-common", "subtle", ] @@ -8350,19 +8154,6 @@ dependencies = [ "wasmparser", ] -[[package]] -name = "wasm-streams" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" -dependencies = [ - "futures-util", - "js-sys", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] - [[package]] name = "wasmparser" version = "0.244.0" @@ -9037,7 +8828,7 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1301e935010a701ae5f8655edc0ad17c44bad3ac5ce8c39185f75453b720ae94" dependencies = [ - "const-oid 0.9.6", + "const-oid", "der", "spki", "tls_codec", diff --git a/dstack/crates/dstack-cli/src/main.rs b/dstack/crates/dstack-cli/src/main.rs index 2781bc59b..ffffd3fd7 100644 --- a/dstack/crates/dstack-cli/src/main.rs +++ b/dstack/crates/dstack-cli/src/main.rs @@ -108,24 +108,17 @@ enum Command { }, /// Scaffold a new app project in the current directory. Init, - /// Build a verity volume that pre-loads docker images (or a directory) into - /// a CVM. + /// Build a read-only verity data volume from a directory or filesystem image. /// - /// The CVM mounts the volume instead of pulling and unpacking the images, so - /// it starts in seconds. The build runs anywhere — no docker daemon, no TEE — - /// but needs root, `mksquashfs`, and `veritysetup`. It prints a verity_root - /// to paste into your compose. See docs/verity-volumes.md. + /// The build needs no daemon or TEE. It prints a verity_root to paste into + /// the deploy command. See docs/verity-volumes.md. Verity { - /// images to bake in, ideally pinned by digest (e.g. `repo@sha256:...`). - #[arg(value_name = "IMAGE")] - images: Vec, - /// pack this directory into a data volume instead of images (mounted at a - /// path you choose in the compose). - #[arg(long, value_name = "PATH", conflicts_with = "images")] + /// Pack this directory into a read-only data volume. + #[arg(long, value_name = "PATH")] dir: Option, /// wrap an existing filesystem image instead of building squashfs. The /// guest mounts it read-only after dm-verity verification. - #[arg(long = "fs-image", value_name = "PATH", conflicts_with_all = ["images", "dir"])] + #[arg(long = "fs-image", value_name = "PATH", conflicts_with = "dir")] fs_image: Option, /// where to write the volume. #[arg(long, short = 'o', default_value = "verity.img")] @@ -133,13 +126,6 @@ enum Command { /// squashfs compression: `none` (the default), `zstd`, or `gzip`. #[arg(long, default_value = "none")] compress: String, - /// image platform to fetch; must match the guest. `linux/amd64` today, - /// `linux/arm64` for arm64 hosts. - #[arg(long, default_value = "linux/amd64")] - platform: String, - /// allow plain-HTTP registries (loopback registries already use HTTP). - #[arg(long)] - plain_http: bool, }, } @@ -233,22 +219,16 @@ async fn main() -> Result<()> { Command::Info { .. } => stub("info"), Command::Init => stub("init"), Command::Verity { - images, dir, fs_image, output, compress, - platform, - plain_http, } => { cmd_verity( - &images, dir.as_deref(), fs_image.as_deref(), &output, &compress, - &platform, - plain_http, json, ) .await @@ -256,15 +236,11 @@ async fn main() -> Result<()> { } } -#[allow(clippy::too_many_arguments)] async fn cmd_verity( - images: &[String], dir: Option<&str>, fs_image: Option<&str>, output: &str, compress: &str, - platform: &str, - plain_http: bool, json: bool, ) -> Result<()> { let compress = match compress { @@ -274,13 +250,10 @@ async fn cmd_verity( other => bail!("unknown --compress '{other}' (use none|zstd|gzip)"), }; let result = dstack_volume::verity(dstack_volume::VerityOptions { - images: images.to_vec(), dir: dir.map(std::path::PathBuf::from), fs_image: fs_image.map(std::path::PathBuf::from), output: output.into(), compress, - platform: platform.to_string(), - plain_http, }) .await?; @@ -289,38 +262,17 @@ async fn cmd_verity( .len(); if json { - let imgs: Vec<_> = result - .images - .iter() - .map(|i| { - serde_json::json!({ - "reference": i.reference, - "manifestDigest": i.manifest_digest, - "configDigest": i.config_digest, - "topChainId": i.top_chain_id, - }) - }) - .collect(); print_json(&serde_json::json!({ "verityRoot": result.verity_root, "output": result.output.display().to_string(), "dataSize": result.data_size, "volumeSize": volume_size, - "images": imgs, })); return Ok(()); } let mib = volume_size as f64 / 1_048_576.0; println!("wrote {} ({mib:.1} MiB)", result.output.display()); - if !result.images.is_empty() { - // the manifest digest is what `image: repo@sha256:...` pins — not the - // config digest (the image id), which can't be pulled by digest. - println!("\nbaked images — pin each by digest in your compose:"); - for i in &result.images { - println!(" {} @ {}", i.reference, i.manifest_digest); - } - } let file = result .output .file_name() @@ -328,19 +280,13 @@ async fn cmd_verity( .unwrap_or_else(|| result.output.display().to_string()); // a data volume mounts at a path you choose; it must be writable (the guest // rootfs is read-only), e.g. under /run. - let target = if result.images.is_empty() { - "/run/models" - } else { - "docker" - }; + let target = "/run/models"; println!("\ncopy {file} into the vmm's volumes_dir, then deploy with:"); println!( " dstack deploy -c docker-compose.yaml --volume {file}:{}:{target}", result.verity_root ); - if result.images.is_empty() { - println!(" (change {target} to your mount path)"); - } + println!(" (change {target} to your mount path)"); Ok(()) } @@ -358,7 +304,7 @@ struct VolumeSpec { /// seeds content matching the attested root. `dstack verity` prints the exact /// spec to paste. /// -/// `TARGET` is `docker` (seed the image store) or an absolute mount path. +/// `TARGET` is an absolute read-only mount path in the guest. fn parse_volume(spec: &str) -> Result { let mut parts = spec.splitn(3, ':'); let name = parts.next().unwrap_or_default(); @@ -377,8 +323,8 @@ fn parse_volume(spec: &str) -> Result { if root.len() != 64 || !root.bytes().all(|b| b.is_ascii_hexdigit()) { bail!("verity_root '{root}' must be 64 hex chars (copy it from `dstack verity`)"); } - if target != "docker" && !target.starts_with('/') { - bail!("target '{target}' must be \"docker\" or an absolute path"); + if !target.starts_with('/') { + bail!("target '{target}' must be an absolute path"); } Ok(VolumeSpec { // verity volumes are read-only by construction; a writable one would let a @@ -728,19 +674,17 @@ mod tests { #[test] fn parses_volume_specs() { let root = "a".repeat(64); - let docker = parse_volume(&format!("images.img:{root}:docker")).unwrap(); - assert_eq!(docker.volume.source, "images.img"); - assert!(docker.volume.read_only); - assert_eq!(docker.verity_root, root); - assert_eq!(docker.target, "docker"); - let data = parse_volume(&format!("weights.img:{root}:/models/llama")).unwrap(); + assert_eq!(data.volume.source, "weights.img"); + assert!(data.volume.read_only); + assert_eq!(data.verity_root, root); assert_eq!(data.target, "/models/llama"); assert!(parse_volume("weights.img").is_err()); // missing verity_root:target assert!(parse_volume(&format!("weights.img:{root}")).is_err()); // missing target - assert!(parse_volume(&format!("../escape.img:{root}:docker")).is_err()); // path separator - assert!(parse_volume("x.img:nothex:docker").is_err()); // verity_root not hex + assert!(parse_volume(&format!("../escape.img:{root}:/models")).is_err()); // path separator + assert!(parse_volume("x.img:nothex:/models").is_err()); // verity_root not hex + assert!(parse_volume(&format!("x.img:{root}:docker")).is_err()); // docker seed removed assert!(parse_volume(&format!("x.img:{root}:relative/path")).is_err()); // bad target } @@ -782,13 +726,7 @@ mod tests { fn parses_verity_fs_image_flag() { let cli = Cli::parse_from(["dstack", "verity", "--fs-image", "rootfs.ext4"]); match cli.command { - Command::Verity { - images, - dir, - fs_image, - .. - } => { - assert!(images.is_empty()); + Command::Verity { dir, fs_image, .. } => { assert_eq!(dir, None); assert_eq!(fs_image.as_deref(), Some("rootfs.ext4")); } diff --git a/dstack/crates/dstack-volume/Cargo.toml b/dstack/crates/dstack-volume/Cargo.toml index 9b175bbea..d47798d55 100644 --- a/dstack/crates/dstack-volume/Cargo.toml +++ b/dstack/crates/dstack-volume/Cargo.toml @@ -14,21 +14,14 @@ binrw.workspace = true cmd_lib.workspace = true dstack-types.workspace = true tokio = { workspace = true, features = ["rt"] } -serde = { workspace = true, features = ["derive", "std"] } serde_json = { workspace = true, features = ["std"] } sha2 = { workspace = true, features = ["std"] } hex = { workspace = true, features = ["std"] } -flate2.workspace = true -futures.workspace = true fs-err.workspace = true gpt = "4.1.0" -tar.workspace = true tempfile.workspace = true tracing.workspace = true tracing-subscriber.workspace = true -rustix = { version = "0.38", features = ["fs"] } uuid.workspace = true -xattr = "1.5" -oci-client = { version = "0.17.0", default-features = false, features = ["rustls-tls"] } [dev-dependencies] diff --git a/dstack/crates/dstack-volume/src/bin/dstack-volume.rs b/dstack/crates/dstack-volume/src/bin/dstack-volume.rs index f4d1c6cc1..d510e6d4c 100644 --- a/dstack/crates/dstack-volume/src/bin/dstack-volume.rs +++ b/dstack/crates/dstack-volume/src/bin/dstack-volume.rs @@ -10,7 +10,6 @@ //! measured app compose as their source of policy and cryptographic identity. use std::collections::HashSet; -use std::ffi::OsStr; use std::io::Read; use std::os::unix::ffi::OsStrExt; use std::os::unix::fs::MetadataExt; @@ -18,16 +17,14 @@ use std::path::{Path, PathBuf}; use anyhow::{bail, Context, Result}; use cmd_lib::{run_cmd, run_fun}; -use dstack_types::{AppCompose, VerityVolume as RequestedVolume, VolumeTarget}; +use dstack_types::{AppCompose, VerityVolume as RequestedVolume}; use dstack_volume::volume_format::{ DstackVolumeHeader, DSTACK_VOLUME_HEADER_SIZE, DSTACK_VOLUME_KIND_VERITY, DSTACK_VOLUME_MAGIC, }; use fs_err::{self as fs, File}; -use serde_json::Value; use tracing::{info, warn}; const MAX_DISKS: usize = 64; -const DOCKER_STORE: &str = "/var/lib/docker"; #[derive(Clone, Debug, PartialEq, Eq)] struct BlockDisk { @@ -70,7 +67,7 @@ fn main() -> Result<()> { for (index, requested) in compose.verity_volumes.iter().enumerate() { if let Err(err) = activate_requested(index, requested, &volumes, &mut used) { failures += 1; - warn!(index, target = %display_target(&requested.target), error = %format_args!("{err:#}"), "failed to activate required volume"); + warn!(index, target = %requested.target.display(), error = %format_args!("{err:#}"), "failed to activate required volume"); } } if failures != 0 { @@ -231,7 +228,7 @@ fn activate_requested( if mapped.exists() { if mapping_root(&mapper_name)?.eq_ignore_ascii_case(&expected_root) { verify_first_block(&mapped)?; - mount_volume(index, requested, &mapped)?; + mount_volume(requested, &mapped)?; used.insert(candidate_index); info!(mapper = mapper_name, "reused active verity mapping"); return Ok(()); @@ -245,9 +242,7 @@ fn activate_requested( let hash = &candidate.hash; run_cmd!(veritysetup open $data $mapper_name $hash $expected_root) .context("opening dm-verity volume")?; - if let Err(err) = - verify_first_block(&mapped).and_then(|_| mount_volume(index, requested, &mapped)) - { + if let Err(err) = verify_first_block(&mapped).and_then(|_| mount_volume(requested, &mapped)) { let _ = run_cmd!(veritysetup close $mapper_name); return Err(err); } @@ -262,43 +257,19 @@ fn verify_first_block(path: &Path) -> Result<()> { .with_context(|| format!("verifying first block of {}", path.display())) } -fn mount_volume(index: usize, requested: &RequestedVolume, mapped: &Path) -> Result<()> { +fn mount_volume(requested: &RequestedVolume, mapped: &Path) -> Result<()> { let fs_type = run_fun!(blkid -o value -s TYPE $mapped).unwrap_or_default(); - match &requested.target { - VolumeTarget::DockerSeed => { - let mountpoint = PathBuf::from(format!("/run/dstack-verity/{index}")); - fs::create_dir_all(&mountpoint)?; - if is_mountpoint(&mountpoint)? { - ensure_mounted_from(&mountpoint, mapped)?; - } else { - mount_read_only(mapped, &mountpoint, fs_type.trim())?; - } - if let Err(err) = seed_docker(&mountpoint) { - let _ = run_cmd!(umount $mountpoint); - return Err(err); - } - info!(root = %hex::encode(requested.verity_root), "seeded docker from verity volume"); - } - VolumeTarget::Mount(target) => { - fs::create_dir_all(target)?; - if is_mountpoint(target)? { - ensure_mounted_from(target, mapped)?; - } else { - mount_read_only(mapped, target, fs_type.trim())?; - } - info!(root = %hex::encode(requested.verity_root), target = %target.display(), "mounted verity volume"); - } + let target = &requested.target; + fs::create_dir_all(target)?; + if is_mountpoint(target)? { + ensure_mounted_from(target, mapped)?; + } else { + mount_read_only(mapped, target, fs_type.trim())?; } + info!(root = %hex::encode(requested.verity_root), target = %target.display(), "mounted verity volume"); Ok(()) } -fn display_target(target: &VolumeTarget) -> std::borrow::Cow<'_, str> { - match target { - VolumeTarget::DockerSeed => "docker".into(), - VolumeTarget::Mount(path) => path.to_string_lossy(), - } -} - fn mount_read_only(device: &Path, target: &Path, fs_type: &str) -> Result<()> { let options = if matches!(fs_type, "ext3" | "ext4") { "ro,noload" @@ -314,131 +285,6 @@ fn mount_read_only(device: &Path, target: &Path, fs_type: &str) -> Result<()> { Ok(()) } -fn seed_docker(volume: &Path) -> Result<()> { - let store = Path::new(DOCKER_STORE); - let overlay = volume.join("overlay2"); - let mut bound = Vec::new(); - for layer in child_directories(&overlay)? { - if layer.file_name() == Some(OsStr::new("l")) { - continue; - } - let source = layer.join("diff"); - if !source.is_dir() { - continue; - } - let layer_id = layer - .file_name() - .context("docker layer path has no file name")?; - let target = store.join("overlay2").join(layer_id).join("diff"); - fs::create_dir_all(&target)?; - if !is_mountpoint(&target)? { - if let Err(err) = run_cmd!(mount --bind $source $target).context("binding docker layer") - { - unwind_binds(&bound); - return Err(err); - } - // A bind inherits neither the intended policy nor all mount flags. - if let Err(err) = - run_cmd!(mount -o remount,bind,ro $target).context("making docker layer read-only") - { - bound.push(target); - unwind_binds(&bound); - return Err(err); - } - bound.push(target); - } - } - - let copy_result = (|| -> Result<()> { - copy_contents( - &volume.join("image/overlay2/imagedb"), - &store.join("image/overlay2/imagedb"), - )?; - copy_contents( - &volume.join("image/overlay2/layerdb"), - &store.join("image/overlay2/layerdb"), - )?; - copy_contents(&volume.join("overlay2/l"), &store.join("overlay2/l"))?; - merge_repositories(volume, store)?; - copy_layer_metadata(volume, store) - })(); - if let Err(err) = copy_result { - unwind_binds(&bound); - return Err(err); - } - Ok(()) -} - -fn child_directories(path: &Path) -> Result> { - let mut result = Vec::new(); - for entry in fs::read_dir(path).with_context(|| format!("reading {}", path.display()))? { - let path = entry?.path(); - if path.is_dir() { - result.push(path); - } - } - result.sort(); - Ok(result) -} - -fn copy_contents(source: &Path, target: &Path) -> Result<()> { - fs::create_dir_all(target)?; - let source_contents = source.join("."); - run_cmd!(cp -a $source_contents $target).context("copying docker metadata")?; - Ok(()) -} - -fn merge_repositories(volume: &Path, store: &Path) -> Result<()> { - let source = volume.join("image/overlay2/repositories.json"); - let target = store.join("image/overlay2/repositories.json"); - if target.exists() { - let temporary = target.with_extension("json.dstack-tmp"); - let mut current: Value = serde_json::from_slice(&fs::read(&target)?)?; - let incoming: Value = serde_json::from_slice(&fs::read(&source)?)?; - merge_json(&mut current, incoming); - fs::write(&temporary, serde_json::to_vec(¤t)?)?; - fs::rename(&temporary, &target)?; - } else { - fs::copy(&source, &target)?; - } - Ok(()) -} - -fn merge_json(current: &mut Value, incoming: Value) { - match (current, incoming) { - (Value::Object(current), Value::Object(incoming)) => { - for (key, value) in incoming { - merge_json(current.entry(key).or_insert(Value::Null), value); - } - } - (current, incoming) => *current = incoming, - } -} - -fn copy_layer_metadata(volume: &Path, store: &Path) -> Result<()> { - for layer in child_directories(&volume.join("overlay2"))? { - let Some(id) = layer.file_name() else { - continue; - }; - if id == OsStr::new("l") { - continue; - } - let target = store.join("overlay2").join(id); - if target.join("link").exists() { - continue; - } - fs::create_dir_all(&target)?; - for entry in fs::read_dir(&layer)? { - let source = entry?.path(); - if source.file_name() == Some(OsStr::new("diff")) { - continue; - } - run_cmd!(cp -a $source $target).context("copying docker layer metadata")?; - } - } - Ok(()) -} - fn is_mountpoint(path: &Path) -> Result { Ok(mountpoint_device(path)?.is_some()) } @@ -503,12 +349,6 @@ fn unescape_mountinfo(value: &[u8]) -> Vec { decoded } -fn unwind_binds(paths: &[PathBuf]) { - for path in paths.iter().rev() { - let _ = run_cmd!(umount $path); - } -} - fn mapping_root(mapper_name: &str) -> Result { let status = run_fun!(veritysetup status $mapper_name)?; status @@ -562,19 +402,6 @@ mod tests { assert!(resolve_verity(&disk, header()).is_err()); } - #[test] - fn recursively_merges_json_objects() { - let mut current = serde_json::json!({"repo": {"old": 1, "same": 1}}); - merge_json( - &mut current, - serde_json::json!({"repo": {"new": 2, "same": 3}}), - ); - assert_eq!( - current, - serde_json::json!({"repo": {"old": 1, "new": 2, "same": 3}}) - ); - } - #[test] fn decodes_mountinfo_escapes() { assert_eq!(unescape_mountinfo(b"/run/my\\040volume"), b"/run/my volume"); diff --git a/dstack/crates/dstack-volume/src/lib.rs b/dstack/crates/dstack-volume/src/lib.rs index 44d07513f..5a8aa5f28 100644 --- a/dstack/crates/dstack-volume/src/lib.rs +++ b/dstack/crates/dstack-volume/src/lib.rs @@ -4,10 +4,8 @@ //! Build, describe, and activate dstack volumes. //! -//! The output is a reproducible, dm-verity-protected raw disk image: partition 1 -//! is the squashfs data filesystem, and partition 2 is the dm-verity superblock -//! plus hash tree. A `docker` volume seeds the overlay2 store; a data volume just -//! mounts at a path. +//! The output is a reproducible, dm-verity-protected raw disk image containing +//! a filesystem that the guest mounts read-only at a measured path. //! //! The build needs no docker daemon and no TEE, and it's reproducible: the same //! inputs always give the same `verity_root`. The first partition contains a @@ -19,10 +17,7 @@ use std::path::PathBuf; use anyhow::{bail, Context, Result}; -use fs_err as fs; -pub mod oci; -mod store; mod volume; pub mod volume_format; @@ -36,11 +31,7 @@ pub use volume::Compression; const VERITY_SALT: &str = "0000000000000000000000000000000000000000000000000000000000000000"; pub struct VerityOptions { - /// image references to bake in, e.g. `nvidia/cuda:12.4.1-runtime-ubuntu22.04`. - /// Empty when building a data volume from `dir`. - pub images: Vec, - /// build a data volume from this directory instead of docker images. The - /// resulting volume is mounted at a `target: "/path"` in the compose. + /// Build a volume from this directory. pub dir: Option, /// wrap an existing filesystem image as the verity data partition. This is /// for hand-built ext4/xfs/etc. images; `dstack verity --dir` still produces @@ -49,90 +40,23 @@ pub struct VerityOptions { pub output: PathBuf, /// squashfs compression (default: none — zero decompression at read time). pub compress: Compression, - /// allow plain-HTTP registries (loopback registries default to HTTP anyway). - pub plain_http: bool, - /// target platform `os/arch` for image pulls (e.g. `linux/amd64`). Explicit - /// so the build is reproducible and matches the guest's arch. - pub platform: String, } pub struct VerityResult { pub verity_root: String, pub data_size: u64, pub output: PathBuf, - pub images: Vec, -} - -pub struct ResolvedImage { - pub reference: String, - pub manifest_digest: String, - pub config_digest: String, - pub top_chain_id: String, } pub async fn verity(opts: VerityOptions) -> Result { - let source_count = - (!opts.images.is_empty()) as u8 + opts.dir.is_some() as u8 + opts.fs_image.is_some() as u8; - if source_count != 1 { - bail!("give exactly one source: images, --dir , or --fs-image "); - } - if let Some(dir) = &opts.dir { - verity_dir(dir.clone(), opts.output, opts.compress).await - } else if let Some(fs_image) = &opts.fs_image { - verity_fs_image(fs_image.clone(), opts.output).await - } else { - verity_docker(opts).await - } -} - -/// Bake docker images into a `docker`-target volume. -async fn verity_docker(opts: VerityOptions) -> Result { - // 1. pull every image (daemonless, verified against its digests). Dedup - // repeated references so `verity alpine alpine` doesn't pull twice. - let mut seen = std::collections::HashSet::new(); - let refs: Vec<&String> = opts.images.iter().filter(|r| seen.insert(*r)).collect(); - let mut pulled = Vec::with_capacity(refs.len()); - for r in refs { - tracing::info!("resolving {r}"); - pulled.push(oci::pull(r, opts.plain_http, &opts.platform).await?); + match (opts.dir, opts.fs_image) { + (Some(dir), None) => verity_dir(dir, opts.output, opts.compress).await, + (None, Some(fs_image)) => verity_fs_image(fs_image, opts.output).await, + _ => bail!("give exactly one source: --dir or --fs-image "), } - - // 2. lay out a deterministic overlay2 store, then pack + verity it. Both are - // synchronous and privileged (mknod / trusted xattr); do them off the async - // reactor. - let output = opts.output.clone(); - let compress = opts.compress; - let (built, resolved) = tokio::task::spawn_blocking(move || -> Result<_> { - let tmp = tempfile::tempdir().context("creating scratch dir")?; - let store_dir = tmp.path().join("store"); - fs::create_dir_all(&store_dir)?; - let tops = store::build_store(&pulled, &store_dir)?; - let built = volume::build_volume(&store_dir, &output, VERITY_SALT, compress)?; - let resolved = pulled - .iter() - .zip(&tops) - .map(|(img, top)| ResolvedImage { - reference: img.reference.clone(), - manifest_digest: img.manifest_digest.clone(), - config_digest: img.config_digest.clone(), - top_chain_id: top.clone(), - }) - .collect::>(); - Ok((built, resolved)) - }) - .await - .context("the build task failed")??; - - Ok(VerityResult { - verity_root: built.verity_root, - data_size: built.data_size, - output: opts.output, - images: resolved, - }) } -/// Bake a directory tree into a data volume (mounted at a `target: "/path"`). -/// No docker, no overlay2 layout — just a reproducible squashfs of the tree. +/// Bake a directory tree into a reproducible squashfs data volume. async fn verity_dir(dir: PathBuf, output: PathBuf, compress: Compression) -> Result { if !dir.is_dir() { bail!("--dir '{}' is not a directory", dir.display()); @@ -148,7 +72,6 @@ async fn verity_dir(dir: PathBuf, output: PathBuf, compress: Compression) -> Res verity_root: built.verity_root, data_size: built.data_size, output, - images: vec![], }) } @@ -168,6 +91,5 @@ async fn verity_fs_image(fs_image: PathBuf, output: PathBuf) -> Result -// -// SPDX-License-Identifier: Apache-2.0 - -//! Fetch an image from a registry, using the `oci-client` crate. -//! -//! Given a reference, this returns the image's config and per-layer blobs, byte -//! for byte, so the caller can lay out a deterministic overlay2 store. oci-client -//! does the fiddly parts: auth, Docker-Hub name normalization, picking the right -//! platform out of an image index, and verifying digests (including a `@sha256:` -//! pin). - -use anyhow::{bail, Context, Result}; -use oci_client::client::{ClientConfig, ClientProtocol}; -use oci_client::manifest; -use oci_client::secrets::RegistryAuth; -use oci_client::{Client, Reference}; -use serde::Deserialize; -use sha2::{Digest, Sha256}; - -/// How many layer blobs to download at once (order-preserving, see `pull`). -const MAX_CONCURRENT_LAYERS: usize = 4; - -/// A fully-resolved image: everything needed to build an overlay2 store. -pub struct PulledImage { - /// the reference as given by the user (for display). - pub reference: String, - /// docker's canonical repository name, e.g. `docker.io/library/alpine` - /// (the repositories.json map key). - pub repo_name: String, - /// the repositories.json key for this reference: `repo_name:tag`, or - /// `repo_name@sha256:...` for a digest pin. `docker run` of the same - /// reference looks this key up, so it's what makes the image a cache hit. - pub ref_key: String, - /// `sha256:...` digest of the manifest actually used (the resolved image). - pub manifest_digest: String, - /// `sha256:...` digest of the config blob — this is the docker image id. - pub config_digest: String, - /// the exact config blob bytes (stored verbatim in imagedb). - pub config_bytes: Vec, - /// `sha256:...` uncompressed digest per layer, bottom to top. - pub diff_ids: Vec, - /// layer blobs, bottom to top, aligned with `diff_ids`. - pub layers: Vec, -} - -pub struct PulledLayer { - pub media_type: String, - pub data: Vec, -} - -/// Pull `reference` for `platform`, returning its config and layers. -/// -/// Layers are fetched in parallel but returned in manifest order, which keeps the -/// store layout deterministic. A loopback registry (or `plain_http`) is fetched -/// over HTTP instead of HTTPS. -pub async fn pull(reference: &str, plain_http: bool, platform: &str) -> Result { - let r: Reference = reference - .parse() - .with_context(|| format!("invalid image reference '{reference}'"))?; - - let insecure = plain_http || is_loopback(r.resolve_registry()); - let protocol = if insecure { - ClientProtocol::Http - } else { - ClientProtocol::Https - }; - let client = Client::new(ClientConfig { - protocol, - // Pin the platform explicitly, not the build host's, so the build is - // reproducible and targets the guest's arch. Not always amd64: arm64 - // confidential hosts are coming (NVIDIA Vera, AWS Graviton/Nitro). - platform_resolver: Some(platform_resolver(platform)?), - ..Default::default() - }); - let auth = RegistryAuth::Anonymous; - - // oci-client verifies the manifest/config digests here (including a - // `@sha256:` pin), and returns the config as its exact raw bytes. - let (manifest, manifest_digest, config_json) = client - .pull_manifest_and_config(&r, &auth) - .await - .with_context(|| format!("resolving {reference}"))?; - let config_bytes = config_json.into_bytes(); - let config_digest = format!("sha256:{}", hex::encode(Sha256::digest(&config_bytes))); - let config: Config = serde_json::from_slice(&config_bytes).context("parsing image config")?; - - // the index resolver only runs for a multi-arch image; a single-arch manifest - // is baked as-is, so check its config arch matches --platform (else we'd bake - // a wrong-arch volume with a right-looking label). - let (want_os, want_arch) = parse_platform(platform)?; - if !config.architecture.is_empty() && config.architecture != want_arch { - bail!( - "image is {}/{}, but --platform is {want_os}/{want_arch} (pass --platform to match)", - if config.os.is_empty() { - "?" - } else { - &config.os - }, - config.architecture - ); - } - - for d in &manifest.layers { - if !accepted_layer_types().contains(&d.media_type.as_str()) { - bail!( - "unsupported layer media type '{}' (only gzip/plain tar; zstd is not supported yet)", - d.media_type - ); - } - } - if config.rootfs.diff_ids.len() != manifest.layers.len() { - bail!( - "manifest/config mismatch: {} layers vs {} diff_ids", - manifest.layers.len(), - config.rootfs.diff_ids.len() - ); - } - - // Download each layer blob (digest-verified by oci-client) concurrently but - // yielded in manifest order — `buffered` preserves input order, so the store - // layout is deterministic without giving up parallel download. - tracing::info!("pulling {} layer(s) of {reference}", manifest.layers.len()); - use futures::stream::{StreamExt, TryStreamExt}; - let layers: Vec = futures::stream::iter(manifest.layers.iter()) - .map(|d| { - let client = &client; - let r = &r; - async move { - // cap the speculative reservation: `d.size` is from the manifest - // and only digest-checked after download. pull_blob grows it as - // needed, so an inflated size can't force a huge up-front alloc. - let hint = (d.size.max(0) as usize).min(64 << 20); - let mut data = Vec::with_capacity(hint); - client - .pull_blob(r, d, &mut data) - .await - .with_context(|| format!("fetching layer {}", d.digest))?; - Ok::<_, anyhow::Error>(PulledLayer { - media_type: d.media_type.clone(), - data, - }) - } - }) - .buffered(MAX_CONCURRENT_LAYERS) - .try_collect() - .await?; - - let (repo_name, ref_key) = repo_and_key(&r); - Ok(PulledImage { - reference: reference.to_string(), - repo_name, - ref_key, - manifest_digest, - config_digest, - config_bytes, - diff_ids: config.rootfs.diff_ids, - layers, - }) -} - -type PlatformResolverFn = - dyn Fn(&[oci_client::manifest::ImageIndexEntry]) -> Option + Send + Sync; - -/// Split `os/arch` (default os `linux`), normalizing the common non-OCI arch -/// spellings (`x86_64` -> `amd64`, `aarch64` -> `arm64`). -fn parse_platform(platform: &str) -> Result<(String, String)> { - let (os, arch) = platform - .split_once('/') - .map(|(o, a)| (o.to_string(), a.to_string())) - .unwrap_or_else(|| ("linux".to_string(), platform.to_string())); - let arch = match arch.as_str() { - "x86_64" => "amd64".to_string(), - "aarch64" => "arm64".to_string(), - _ => arch, - }; - if os.is_empty() || arch.is_empty() { - bail!("invalid --platform '{platform}' (expected e.g. linux/amd64)"); - } - Ok((os, arch)) -} - -/// Build a resolver that picks the `os/arch` entry out of an image index. -fn platform_resolver(platform: &str) -> Result> { - let (os, arch) = parse_platform(platform)?; - Ok(Box::new( - move |manifests: &[oci_client::manifest::ImageIndexEntry]| { - manifests - .iter() - .find(|e| { - e.platform.as_ref().is_some_and(|p| { - p.os.to_string() == os && p.architecture.to_string() == arch - }) - }) - .map(|e| e.digest.clone()) - }, - )) -} - -/// The layer media types we can extract: gzip and plain tar. -/// -/// Checking against these lets us reject an unsupported layer (e.g. zstd) up -/// front, instead of failing later during extraction. -fn accepted_layer_types() -> [&'static str; 4] { - [ - manifest::IMAGE_LAYER_GZIP_MEDIA_TYPE, - manifest::IMAGE_DOCKER_LAYER_GZIP_MEDIA_TYPE, - manifest::IMAGE_LAYER_MEDIA_TYPE, - manifest::IMAGE_DOCKER_LAYER_TAR_MEDIA_TYPE, - ] -} - -/// Return `(repo_name, ref_key)` for a reference. -/// -/// `repo_name` is docker's canonical name (the repositories.json map key); -/// `ref_key` is the entry within it — `repo@digest` for a pin, `repo:tag` -/// otherwise. `Reference` already normalizes Docker-Hub names to -/// `docker.io/library/`. -fn repo_and_key(r: &Reference) -> (String, String) { - let repo = format!("{}/{}", r.registry(), r.repository()); - let key = match r.digest() { - Some(d) => format!("{repo}@{d}"), - None => format!("{repo}:{}", r.tag().unwrap_or("latest")), - }; - (repo, key) -} - -/// A loopback registry we default to plain HTTP for. Any other insecure -/// registry uses `--plain-http`. -fn is_loopback(host: &str) -> bool { - let h = host.split(':').next().unwrap_or(host); - h == "localhost" || h.starts_with("127.") -} - -#[derive(Deserialize)] -struct Config { - #[serde(default)] - architecture: String, - #[serde(default)] - os: String, - rootfs: RootFs, -} -#[derive(Deserialize)] -struct RootFs { - #[serde(rename = "diff_ids", default)] - diff_ids: Vec, -} - -#[cfg(test)] -mod tests { - use super::*; - - fn key(s: &str) -> (String, String) { - repo_and_key(&s.parse::().unwrap()) - } - - #[test] - fn hub_short_name_canonicalizes() { - let (repo, k) = key("alpine:3.20"); - assert_eq!(repo, "docker.io/library/alpine"); - assert_eq!(k, "docker.io/library/alpine:3.20"); - } - - #[test] - fn fully_qualified_hub_matches_short() { - // the doc tells users to fully-qualify; it must resolve identically. - assert_eq!(key("docker.io/library/alpine:3.20"), key("alpine:3.20")); - } - - #[test] - fn digest_pin_uses_at_key() { - let d = "sha256:1111111111111111111111111111111111111111111111111111111111111111"; - let (repo, k) = key(&format!("ghcr.io/foo/bar@{d}")); - assert_eq!(repo, "ghcr.io/foo/bar"); - assert_eq!(k, format!("ghcr.io/foo/bar@{d}")); - } - - #[test] - fn private_registry_with_port() { - let (repo, k) = key("10.0.2.2:5000/bench/alpine:3.20"); - assert_eq!(repo, "10.0.2.2:5000/bench/alpine"); - assert_eq!(k, "10.0.2.2:5000/bench/alpine:3.20"); - } - - #[test] - fn loopback_detection() { - assert!(is_loopback("localhost:5000")); - assert!(is_loopback("127.0.0.1:5000")); - assert!(!is_loopback("10.0.2.2:5000")); // qemu-guest alias, not the host - assert!(!is_loopback("ghcr.io")); - assert!(!is_loopback("index.docker.io")); - } - - #[test] - fn platform_resolver_picks_requested_arch() { - let entries: Vec = - serde_json::from_value(serde_json::json!([ - {"mediaType":"m","digest":"sha256:amd","size":1, - "platform":{"architecture":"amd64","os":"linux"}}, - {"mediaType":"m","digest":"sha256:arm","size":1, - "platform":{"architecture":"arm64","os":"linux"}} - ])) - .unwrap(); - assert_eq!( - platform_resolver("linux/amd64").unwrap()(&entries).as_deref(), - Some("sha256:amd") - ); - assert_eq!( - platform_resolver("linux/arm64").unwrap()(&entries).as_deref(), - Some("sha256:arm") - ); - // aarch64 alias normalizes to arm64. - assert_eq!( - platform_resolver("linux/aarch64").unwrap()(&entries).as_deref(), - Some("sha256:arm") - ); - // a platform with no matching entry resolves to nothing. - assert_eq!(platform_resolver("windows/amd64").unwrap()(&entries), None); - } -} diff --git a/dstack/crates/dstack-volume/src/store.rs b/dstack/crates/dstack-volume/src/store.rs deleted file mode 100644 index 88e8dfa8e..000000000 --- a/dstack/crates/dstack-volume/src/store.rs +++ /dev/null @@ -1,458 +0,0 @@ -// SPDX-FileCopyrightText: © 2026 Phala Network -// -// SPDX-License-Identifier: Apache-2.0 - -//! Build a docker overlay2 image store, laid out deterministically. -//! -//! The `oci` module pulls an image's layers; this module lays them out into the -//! on-disk overlay2 store that docker reads. -//! -//! docker's own store is non-deterministic in two ways: it gives each layer a -//! random cache-id, and it stamps wall-clock times. Here the cache-id is the -//! layer's chain-id instead, so the store bytes are a pure function of the -//! pinned layer digests. -//! -//! Whiteouts in the layer tars (the AUFS `.wh.` markers) become the form -//! overlay2 wants on disk: a `0:0` char device, or the `trusted.overlay.opaque` -//! xattr. Both need root. - -use std::collections::BTreeMap; -use std::collections::HashSet; -use std::io::Read; -use std::path::{Path, PathBuf}; - -use anyhow::{bail, Context, Result}; -use flate2::read::GzDecoder; -use fs_err as fs; -use serde_json::json; -use sha2::{Digest, Sha256}; - -use crate::oci::{PulledImage, PulledLayer}; - -/// Build the overlay2 store for `images` under `root`. -/// -/// Returns each image's top chain-id, in the same order as `images`. That -/// chain-id is what the verity volume ultimately vouches for. -pub fn build_store(images: &[PulledImage], root: &Path) -> Result> { - fs::create_dir_all(root.join("overlay2/l"))?; - fs::create_dir_all(root.join("image/overlay2/layerdb/sha256"))?; - fs::create_dir_all(root.join("image/overlay2/imagedb/content/sha256"))?; - - let mut built: HashSet = HashSet::new(); - // repo_name -> { "repo:tag" -> "sha256:imageid", "repo@digest" -> id } - let mut repositories: BTreeMap> = BTreeMap::new(); - let mut tops = Vec::with_capacity(images.len()); - - for img in images { - let mut parent: Option = None; - let mut parent_short: Option = None; - let mut chain = String::new(); - for (i, diff_id) in img.diff_ids.iter().enumerate() { - let diff_hex = strip_sha256(diff_id)?; - chain = match &parent { - None => diff_hex.to_string(), - Some(p) => chain_id(p, diff_hex), - }; - let short = short_link(&chain); - - // A layer shared by two images has the same chain-id; write it once. - if built.insert(chain.clone()) { - write_layer( - root, - &img.layers[i], - &img.reference, - i, - diff_id, - &chain, - &short, - parent.as_deref(), - parent_short.as_deref(), - )?; - } - parent = Some(chain.clone()); - parent_short = Some(short); - } - - // imagedb: the config blob verbatim, keyed by its digest (the image id). - let image_id = strip_sha256(&img.config_digest)?; - write_bytes( - root.join("image/overlay2/imagedb/content/sha256") - .join(image_id), - &img.config_bytes, - )?; - // repositories.json: map the exact canonical reference the user pinned - // (`repo:tag` or `repo@sha256:...`) to the image id, plus the resolved - // manifest digest as a secondary alias. - let entry = repositories.entry(img.repo_name.clone()).or_default(); - entry.insert(img.ref_key.clone(), img.config_digest.clone()); - entry.insert( - format!("{}@{}", img.repo_name, img.manifest_digest), - img.config_digest.clone(), - ); - tops.push(chain); - } - - write_bytes( - root.join("image/overlay2/repositories.json"), - serde_json::to_vec(&json!({ "Repositories": repositories }))?.as_slice(), - )?; - Ok(tops) -} - -/// Write one layer into the store, under its `chain` id. -/// -/// Extracts the layer's files, checks them against `diff_id`, then records the -/// overlay2 link and the layerdb entry. `parent` and `parent_short` describe the -/// layer directly below; both are `None` for the base layer. -#[allow(clippy::too_many_arguments)] -fn write_layer( - root: &Path, - layer: &PulledLayer, - reference: &str, - layer_index: usize, - diff_id: &str, - chain: &str, - short: &str, - parent: Option<&str>, - parent_short: Option<&str>, -) -> Result<()> { - let layer_dir = root.join("overlay2").join(chain); - let diff_dir = layer_dir.join("diff"); - fs::create_dir_all(&diff_dir)?; - - let (layer_size, actual_diff) = extract_layer(&layer.data, &layer.media_type, &diff_dir) - .with_context(|| format!("extracting layer {layer_index} of {reference}"))?; - - // The extracted files must hash to the diff_id the config claims. Otherwise a - // registry could hand us content that doesn't match its (pinned) config, and - // we'd bake it in anyway. - if actual_diff != diff_id { - bail!( - "layer {layer_index} of {reference} does not match its diff_id \ - (got {actual_diff}, config says {diff_id})" - ); - } - - // overlay2 finds a layer through the `l/` symlink and its `link` file. - symlink( - format!("../{chain}/diff"), - root.join("overlay2/l").join(short), - )?; - write(layer_dir.join("link"), short)?; - if let Some(ps) = parent_short { - write(layer_dir.join("lower"), &lower_chain(root, ps)?)?; - } - - let layerdb = root.join("image/overlay2/layerdb/sha256").join(chain); - fs::create_dir_all(&layerdb)?; - write(layerdb.join("diff"), diff_id)?; - write(layerdb.join("cache-id"), chain)?; - write(layerdb.join("size"), &layer_size.to_string())?; - if let Some(p) = parent { - write(layerdb.join("parent"), &format!("sha256:{p}"))?; - } - Ok(()) -} - -/// docker's chain-id for a layer: `sha256("sha256: sha256:")`. -fn chain_id(parent_chain_hex: &str, diff_hex: &str) -> String { - let mut h = Sha256::new(); - h.update(format!("sha256:{parent_chain_hex} sha256:{diff_hex}").as_bytes()); - hex::encode(h.finalize()) -} - -/// Deterministic 26-char [A-Z0-9] overlay2 link name (docker uses a random one). -fn short_link(chain_hex: &str) -> String { - let mut s = String::from("L"); - s.push_str(&chain_hex[..25].to_uppercase()); - s -} - -/// Build the `lower` file for a layer, given its parent's link name. -/// -/// overlay2 lists every ancestor there, nearest first. We build that by taking -/// the parent's own `lower` and prepending the parent. -fn lower_chain(root: &Path, parent_short: &str) -> Result { - // resolve the parent's chain dir from its l/ symlink to read its `lower`. - let parent_dir = fs::read_link(root.join("overlay2/l").join(parent_short))?; - // ..//diff -> - let chain = parent_dir - .parent() - .and_then(|p| p.file_name()) - .and_then(|s| s.to_str()) - .context("bad parent link")? - .to_string(); - let plower = root.join("overlay2").join(&chain).join("lower"); - let mut parts = vec![format!("l/{parent_short}")]; - if let Ok(existing) = fs::read_to_string(&plower) { - parts.extend(existing.split(':').map(str::to_string)); - } - Ok(parts.join(":")) -} - -/// Extract a layer tar into `dest`, converting AUFS whiteouts as it goes. -/// -/// Handles gzip and plain tar. Returns two things: the layer's uncompressed size -/// (what docker records), and its diff-id — the sha256 of the whole decompressed -/// tar, which the caller checks against the config. -fn extract_layer(data: &[u8], media_type: &str, dest: &Path) -> Result<(u64, String)> { - let is_gzip = - media_type.contains("gzip") || (data.len() >= 2 && data[0] == 0x1f && data[1] == 0x8b); - let decompressed: Box = if is_gzip { - Box::new(GzDecoder::new(data)) - } else { - Box::new(data) - }; - let mut ar = tar::Archive::new(HashingReader::new(decompressed)); - ar.set_preserve_permissions(true); - ar.set_preserve_mtime(true); - ar.set_unpack_xattrs(true); - ar.set_preserve_ownerships(true); - - let mut total = 0u64; - for entry in ar.entries()? { - let mut entry = entry?; - if entry.header().entry_type().is_file() { - total += entry.header().size().unwrap_or(0); - } - let path = entry.path()?.into_owned(); - let name = match path.file_name().and_then(|s| s.to_str()) { - Some(n) => n.to_string(), - None => continue, - }; - - if let Some(rest) = name.strip_prefix(".wh.") { - // Whiteouts are placed by hand (mknod / an xattr) rather than through - // the tar crate, so they need their own containment. Without it a - // hostile layer could escape `dest` — lexically, or through a - // symlinked parent it planted earlier — and we run as root. - // `safe_subpath` won't follow a symlink out of `dest`. - let out_parent = - match safe_subpath(dest, path.parent().unwrap_or_else(|| Path::new(""))) { - Some(p) => p, - None => continue, - }; - if name == ".wh..wh..opq" { - fs::create_dir_all(&out_parent)?; - xattr::set(&out_parent, "trusted.overlay.opaque", b"y") - .context("setting overlay opaque xattr (need root)")?; - } else if rest.starts_with(".wh.") { - // other reserved aufs metadata (e.g. `.wh..wh..plnk`) — not a - // whiteout for `rest`; ignore. - } else if rest.is_empty() || rest == "." || rest == ".." { - // a name like `.wh.`, `.wh..`, or `.wh...` would target the parent - // dir itself or escape it; ignore. - } else { - fs::create_dir_all(&out_parent)?; - let target = out_parent.join(rest); - let _ = remove_any(&target); - mknod_whiteout(&target).context("creating overlay whiteout (need root)")?; - } - continue; - } - // normal entry — the tar crate sandboxes this against `dest` (rejects - // `..`/absolute and creates parents itself). - entry.unpack_in(dest)?; - } - // drain any trailing padding the entry iterator didn't read, so the digest - // covers the whole tar, then finalize the diff-id. - let mut hr = ar.into_inner(); - std::io::copy(&mut hr, &mut std::io::sink())?; - let diff_id = format!("sha256:{}", hex::encode(hr.finalize())); - Ok((total, diff_id)) -} - -/// A reader that sha256's every byte passing through it. -/// -/// This lets us compute a layer's diff-id while extracting it, instead of -/// decompressing the layer a second time. -struct HashingReader { - inner: R, - hasher: Sha256, -} -impl HashingReader { - fn new(inner: R) -> Self { - Self { - inner, - hasher: Sha256::new(), - } - } - fn finalize(self) -> impl AsRef<[u8]> { - self.hasher.finalize() - } -} -impl Read for HashingReader { - fn read(&mut self, buf: &mut [u8]) -> std::io::Result { - let n = self.inner.read(buf)?; - self.hasher.update(&buf[..n]); - Ok(n) - } -} - -/// Resolve `rel` under `dest`, but only if it stays inside `dest`. -/// -/// Returns `None` for a `..` or absolute component. It also refuses to walk -/// through any component that already exists as a symlink, so a symlink a -/// hostile layer planted earlier can't be used to escape. (The tar crate gets -/// the same guarantee by canonicalizing.) -fn safe_subpath(dest: &Path, rel: &Path) -> Option { - use std::path::Component; - let mut cur = dest.to_path_buf(); - for comp in rel.components() { - match comp { - Component::CurDir => {} - Component::Normal(c) => { - cur.push(c); - if let Ok(md) = fs::symlink_metadata(&cur) { - if md.file_type().is_symlink() { - return None; - } - } - } - // absolute root, `..`, or a Windows prefix: reject outright. - _ => return None, - } - } - Some(cur) -} - -/// overlay2 whiteout: a character device with major/minor 0/0. -fn mknod_whiteout(path: &Path) -> Result<()> { - use rustix::fs::{mknodat, FileType, Mode, CWD}; - mknodat(CWD, path, FileType::CharacterDevice, Mode::empty(), 0)?; - Ok(()) -} - -fn remove_any(path: &Path) -> std::io::Result<()> { - match fs::symlink_metadata(path) { - Ok(m) if m.is_dir() => fs::remove_dir_all(path), - Ok(_) => fs::remove_file(path), - Err(e) => Err(e), - } -} - -fn strip_sha256(d: &str) -> Result<&str> { - let hex = d - .strip_prefix("sha256:") - .with_context(|| format!("expected sha256: digest, got '{d}'"))?; - // validate here so every downstream chain-id / link name is 64 hex chars, - // and slicing (e.g. short_link's [..25]) can't panic on hostile registry data. - if hex.len() != 64 || !hex.bytes().all(|b| b.is_ascii_hexdigit()) { - bail!("malformed sha256 digest '{d}' (expected 64 hex chars)"); - } - Ok(hex) -} - -fn symlink(target: impl AsRef, link: impl AsRef) -> std::io::Result<()> { - std::os::unix::fs::symlink(target, link) -} - -fn write(path: PathBuf, contents: &str) -> std::io::Result<()> { - fs::write(path, contents.as_bytes()) -} - -fn write_bytes(path: PathBuf, contents: &[u8]) -> std::io::Result<()> { - fs::write(path, contents) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn strip_sha256_validates_64_hex() { - let ok = "a".repeat(64); - assert_eq!(strip_sha256(&format!("sha256:{ok}")).unwrap(), ok); - assert!(strip_sha256("nothex").is_err()); // no sha256: prefix - assert!(strip_sha256("sha256:abc").is_err()); // too short (would panic short_link) - assert!(strip_sha256(&format!("sha256:{}", "g".repeat(64))).is_err()); // not hex - assert!(strip_sha256("sha256:€€€€€€€€€€€€€€€€€€€€€€€€€€€€€€€€").is_err()); - // multibyte - } - - #[test] - fn chain_id_matches_docker_formula() { - // base layer chain-id == diff-id. - let d = "0000000000000000000000000000000000000000000000000000000000000000"; - // second layer: sha256("sha256: sha256:") - let c = chain_id(d, d); - let mut h = Sha256::new(); - h.update(format!("sha256:{d} sha256:{d}").as_bytes()); - assert_eq!(c, hex::encode(h.finalize())); - } - - #[test] - fn short_link_is_26_upper_alnum() { - let s = short_link("abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"); - assert_eq!(s.len(), 26); - assert!(s.starts_with('L')); - assert!(s - .chars() - .all(|c| c.is_ascii_uppercase() || c.is_ascii_digit())); - } - - #[test] - fn safe_subpath_rejects_traversal_absolute_and_symlink() { - let dir = tempfile::tempdir().unwrap(); - let dest = dir.path(); - // plain nested paths resolve inside dest. - assert_eq!(safe_subpath(dest, Path::new("a/b")), Some(dest.join("a/b"))); - assert_eq!( - safe_subpath(dest, Path::new("./app")), - Some(dest.join("app")) - ); - // lexical escapes are refused. - assert!(safe_subpath(dest, Path::new("../etc")).is_none()); - assert!(safe_subpath(dest, Path::new("/etc")).is_none()); - assert!(safe_subpath(dest, Path::new("a/../../etc")).is_none()); - // a planted symlink parent must not be traversed (the real exploit). - std::os::unix::fs::symlink("/var", dest.join("evil")).unwrap(); - assert!(safe_subpath(dest, Path::new("evil")).is_none()); - assert!(safe_subpath(dest, Path::new("evil/lib")).is_none()); - } - - #[test] - fn whiteout_through_symlinked_parent_cannot_escape() { - // End-to-end: a hostile layer plants `evil -> ` then a - // whiteout `evil/.wh.keep`; without containment that would delete the - // victim file outside dest, as root. - use std::os::unix::fs::MetadataExt; - let dest = tempfile::tempdir().unwrap(); - let victim = tempfile::tempdir().unwrap(); - fs::write(victim.path().join("keep"), b"x").unwrap(); - // stamp the tar entries with our own uid/gid so unpacking the symlink - // doesn't need root to chown. - let md = fs::metadata(victim.path()).unwrap(); - let (uid, gid) = (md.uid() as u64, md.gid() as u64); - - let mut b = tar::Builder::new(Vec::new()); - let mut h = tar::Header::new_gnu(); - h.set_entry_type(tar::EntryType::Symlink); - h.set_size(0); - h.set_uid(uid); - h.set_gid(gid); - h.set_mode(0o777); - b.append_link(&mut h, "evil", victim.path()).unwrap(); - let mut h2 = tar::Header::new_gnu(); - h2.set_entry_type(tar::EntryType::Regular); - h2.set_size(0); - h2.set_uid(uid); - h2.set_gid(gid); - h2.set_mode(0o644); - b.append_data(&mut h2, "evil/.wh.keep", std::io::empty()) - .unwrap(); - let tar_bytes = b.into_inner().unwrap(); - - extract_layer( - &tar_bytes, - "application/vnd.oci.image.layer.v1.tar", - dest.path(), - ) - .unwrap(); - - assert!( - victim.path().join("keep").exists(), - "whiteout escaped dest through a symlinked parent" - ); - } -} diff --git a/dstack/crates/dstack-volume/src/volume.rs b/dstack/crates/dstack-volume/src/volume.rs index b6d37fc33..68303f958 100644 --- a/dstack/crates/dstack-volume/src/volume.rs +++ b/dstack/crates/dstack-volume/src/volume.rs @@ -68,8 +68,7 @@ pub struct BuiltVolume { /// Build `output`: a GPT disk image with the squashfs data partition and a /// separate dm-verity hash partition. /// -/// `store` is any directory — a docker overlay2 store, or plain data. `salt` -/// fixes the verity root. +/// `store` is the directory to pack; `salt` fixes the verity root. /// /// The verity UUID is derived from the squashfs bytes, so two different volumes /// get two different UUIDs. That matters because a VM can mount several volumes diff --git a/dstack/crates/dstack-volume/src/volume_format.rs b/dstack/crates/dstack-volume/src/volume_format.rs index 9f1be78da..89bf8c6a3 100644 --- a/dstack/crates/dstack-volume/src/volume_format.rs +++ b/dstack/crates/dstack-volume/src/volume_format.rs @@ -51,7 +51,7 @@ impl DstackVolumeHeader { #[cfg(test)] mod tests { use super::*; - use dstack_types::{VerityVolume, VolumeTarget}; + use dstack_types::VerityVolume; #[test] fn volume_header_round_trip() { @@ -69,7 +69,7 @@ mod tests { "target": "/run/models" }))?; assert_eq!(volume.verity_root, [0x5a; 32]); - assert_eq!(volume.target, VolumeTarget::Mount("/run/models".into())); + assert_eq!(volume.target, std::path::PathBuf::from("/run/models")); assert_eq!( serde_json::to_value(&volume)?["verity_root"], "5a".repeat(32) @@ -77,7 +77,7 @@ mod tests { assert!(serde_json::from_value::(serde_json::json!({ "verity_root": "abcd", - "target": "docker" + "target": "/run/models" })) .is_err()); assert!(serde_json::from_value::(serde_json::json!({ diff --git a/dstack/dstack-types/src/lib.rs b/dstack/dstack-types/src/lib.rs index 75a808adc..f7fa20748 100644 --- a/dstack/dstack-types/src/lib.rs +++ b/dstack/dstack-types/src/lib.rs @@ -134,47 +134,23 @@ pub struct VerityVolume { /// check. The guest matches attached devices against it. #[serde(with = "hex_bytes")] pub verity_root: [u8; 32], - /// `"docker"` (seed the docker overlay2 image store), or an absolute path - /// where the volume's filesystem is mounted (e.g. model weights). - pub target: VolumeTarget, + /// Absolute path where the volume's filesystem is mounted. + #[serde(deserialize_with = "deserialize_absolute_path")] + pub target: std::path::PathBuf, } -/// How a verified volume is exposed inside the guest. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum VolumeTarget { - DockerSeed, - Mount(std::path::PathBuf), -} - -impl Serialize for VolumeTarget { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - match self { - Self::DockerSeed => serializer.serialize_str("docker"), - Self::Mount(path) => serializer.serialize_str(&path.to_string_lossy()), - } - } -} - -impl<'de> Deserialize<'de> for VolumeTarget { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - let value = String::deserialize(deserializer)?; - if value == "docker" { - return Ok(Self::DockerSeed); - } - let path = std::path::PathBuf::from(&value); - if !path.is_absolute() { - return Err(serde::de::Error::custom(format!( - "volume target must be \"docker\" or an absolute path, got '{value}'" - ))); - } - Ok(Self::Mount(path)) - } +fn deserialize_absolute_path<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + let value = String::deserialize(deserializer)?; + let path = std::path::PathBuf::from(&value); + if !path.is_absolute() { + return Err(serde::de::Error::custom(format!( + "volume target must be an absolute path, got '{value}'" + ))); + } + Ok(path) } /// Canonical source for the policy used when `requirements.gpu_policy` is diff --git a/os/common/rootfs/dstack-prepare.sh b/os/common/rootfs/dstack-prepare.sh index 877674e13..85b64598d 100755 --- a/os/common/rootfs/dstack-prepare.sh +++ b/os/common/rootfs/dstack-prepare.sh @@ -300,9 +300,8 @@ echo "============================" cd /dstack -# Seed pre-baked verity volumes (docker images / data) before dockerd starts, so -# the images are already present with no pull. Fail-safe -- see dstack-volume. -/bin/dstack-volume app-compose.json || log "dstack-volume failed (continuing; images will pull normally)" +# Verify and mount required read-only data volumes before the application starts. +/bin/dstack-volume app-compose.json if [ "$(jq 'has("init_script")' app-compose.json)" == true ]; then log "Running init script" From 77ac69d7ee8311dc45a3bfa64a26ebf32f066bb1 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 21 Jul 2026 02:24:14 -0700 Subject: [PATCH 18/25] fix(volume): fail immediately on activation errors --- .../dstack-volume/src/bin/dstack-volume.rs | 14 +++--- dstack/vmm/src/main_service.rs | 44 +++---------------- 2 files changed, 11 insertions(+), 47 deletions(-) diff --git a/dstack/crates/dstack-volume/src/bin/dstack-volume.rs b/dstack/crates/dstack-volume/src/bin/dstack-volume.rs index d510e6d4c..2f486a2bd 100644 --- a/dstack/crates/dstack-volume/src/bin/dstack-volume.rs +++ b/dstack/crates/dstack-volume/src/bin/dstack-volume.rs @@ -63,15 +63,13 @@ fn main() -> Result<()> { "discovered dstack volumes" ); let mut used = HashSet::new(); - let mut failures = 0; for (index, requested) in compose.verity_volumes.iter().enumerate() { - if let Err(err) = activate_requested(index, requested, &volumes, &mut used) { - failures += 1; - warn!(index, target = %requested.target.display(), error = %format_args!("{err:#}"), "failed to activate required volume"); - } - } - if failures != 0 { - bail!("failed to activate {failures} required volume(s)"); + activate_requested(index, requested, &volumes, &mut used).with_context(|| { + format!( + "failed to activate required volume {index} at {}", + requested.target.display() + ) + })?; } Ok(()) } diff --git a/dstack/vmm/src/main_service.rs b/dstack/vmm/src/main_service.rs index 6f0d6da2e..4e5f4c208 100644 --- a/dstack/vmm/src/main_service.rs +++ b/dstack/vmm/src/main_service.rs @@ -11,12 +11,11 @@ use dstack_types::AppCompose; use dstack_vmm_rpc as rpc; use dstack_vmm_rpc::vmm_server::{VmmRpc, VmmServer}; use dstack_vmm_rpc::{ - AppId, ComposeHash as RpcComposeHash, DhcpLeaseRequest, GatewaySettings, GetInfoResponse, - GetMetaResponse, Id, ImageInfo as RpcImageInfo, ImageListResponse, KmsSettings, - ListGpusResponse, PublicKeyResponse, PullRegistryImageRequest, RegistryImageInfo, - RegistryImageListResponse, ReloadVmsResponse, ResizeVmRequest, ResourcesSettings, - StatusRequest, StatusResponse, SvListResponse, SvProcessInfo, UpdateVmRequest, VersionResponse, - VmConfiguration, + AppId, ComposeHash as RpcComposeHash, GatewaySettings, GetInfoResponse, GetMetaResponse, Id, + ImageInfo as RpcImageInfo, ImageListResponse, KmsSettings, ListGpusResponse, PublicKeyResponse, + PullRegistryImageRequest, RegistryImageInfo, RegistryImageListResponse, ReloadVmsResponse, + ResizeVmRequest, ResourcesSettings, StatusRequest, StatusResponse, SvListResponse, + SvProcessInfo, UpdateVmRequest, VersionResponse, VmConfiguration, }; use fs_err as fs; use or_panic::ResultOrPanic; @@ -285,7 +284,6 @@ fn networking_from_proto(proto: &rpc::NetworkingConfig) -> Result "bridge".to_string(), NetworkingMode::Custom => String::new(), }, - forward_service_enabled: default_networking.forward_service_enabled, default_bridge: default_networking.bridge.clone(), }), }) @@ -724,11 +718,6 @@ impl VmmRpc for RpcHandler { self.app.reload_vms_sync().await } - async fn report_dhcp_lease(self, request: DhcpLeaseRequest) -> Result<()> { - self.app.report_dhcp_lease(&request.mac, &request.ip).await; - Ok(()) - } - async fn sv_list(self) -> Result { use supervisor_client::supervisor::ProcessStatus; let list = self.app.supervisor.list().await?; @@ -991,29 +980,6 @@ mod tests { assert!(err.to_string().contains("custom networking mode")); } - #[test] - fn multiple_bridges_are_rejected_when_builtin_forwarding_is_enabled() { - let mut cvm_config = test_cvm_config(); - cvm_config.networking.forward_service_enabled = true; - let mut request = test_vm_configuration(); - request.networks = vec![ - rpc::NetworkingConfig { - mode: "bridge".to_string(), - bridge_name: "lo".to_string(), - }, - rpc::NetworkingConfig { - mode: "bridge".to_string(), - bridge_name: "lo".to_string(), - }, - ]; - - let err = create_manifest_from_vm_config(request, &cvm_config).unwrap_err(); - - assert!(err - .to_string() - .contains("built-in port forwarding supports only one bridge")); - } - #[test] fn resolve_volume_source_rejects_escape_symlink_and_qemu_metachars() -> Result<()> { let tmp = tempfile::tempdir()?; From b22f57580a2ddca84d96f1d94f9e764519f059e4 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 21 Jul 2026 03:28:28 -0700 Subject: [PATCH 19/25] fix(verity): measure attached volume device count --- dstack/dstack-mr/cli/src/main.rs | 5 +++++ dstack/dstack-mr/src/acpi.rs | 10 ++++++++++ dstack/dstack-mr/src/machine.rs | 3 +++ dstack/dstack-mr/src/tdx.rs | 3 +++ dstack/dstack-types/src/lib.rs | 28 +++++++++++++++++++++++++++- dstack/verifier/src/verification.rs | 1 + dstack/vmm/src/app.rs | 29 +++++++++++++++++++++++++++++ 7 files changed, 78 insertions(+), 1 deletion(-) diff --git a/dstack/dstack-mr/cli/src/main.rs b/dstack/dstack-mr/cli/src/main.rs index 146112b3d..258d99f5d 100644 --- a/dstack/dstack-mr/cli/src/main.rs +++ b/dstack/dstack-mr/cli/src/main.rs @@ -70,6 +70,10 @@ struct MachineConfig { #[arg(long, default_value = "1")] num_nics: u32, + /// Number of virtio-blk verity volumes + #[arg(long, default_value = "0")] + num_verity_volumes: u32, + /// Disable hotplug #[arg(long, default_value = "false")] hotplug_off: Bool, @@ -138,6 +142,7 @@ fn main() -> Result<()> { .num_gpus(config.num_gpus) .num_nvswitches(config.num_nvswitches) .num_nics(config.num_nics) + .num_verity_volumes(config.num_verity_volumes) .hotplug_off(config.hotplug_off) .root_verity(config.root_verity) .maybe_qemu_version(config.qemu_version.clone()) diff --git a/dstack/dstack-mr/src/acpi.rs b/dstack/dstack-mr/src/acpi.rs index db8f52845..f9df12a2f 100644 --- a/dstack/dstack-mr/src/acpi.rs +++ b/dstack/dstack-mr/src/acpi.rs @@ -57,6 +57,16 @@ impl Machine<'_> { "virtio-blk-pci,drive=hd1", ]); + // Verity volumes are emitted after hd1 and before networking, matching + // dstack-vmm's QEMU device order. + for i in 0..self.num_verity_volumes { + let id = format!("vol{i}"); + cmd.arg("-drive").arg(format!( + "file={dummy_disk},if=none,id={id},format=raw,readonly=on" + )); + cmd.arg("-device").arg(format!("virtio-blk-pci,drive={id}")); + } + // One virtio-net-pci per NIC. Emitted at the same position and, for the // default single-NIC case, with the exact same args as the previous // hardcoded layout so RTMR0 stays byte-for-byte unchanged. diff --git a/dstack/dstack-mr/src/machine.rs b/dstack/dstack-mr/src/machine.rs index 59d061084..59b91c088 100644 --- a/dstack/dstack-mr/src/machine.rs +++ b/dstack/dstack-mr/src/machine.rs @@ -33,6 +33,9 @@ pub struct Machine<'a> { /// predate this field keep the historical single-NIC layout. #[builder(default = 1)] pub num_nics: u32, + /// Number of virtio-blk verity volumes attached before the NICs. + #[builder(default)] + pub num_verity_volumes: u32, pub hotplug_off: bool, pub root_verity: bool, #[builder(default)] diff --git a/dstack/dstack-mr/src/tdx.rs b/dstack/dstack-mr/src/tdx.rs index abd6c7f99..e2243a584 100644 --- a/dstack/dstack-mr/src/tdx.rs +++ b/dstack/dstack-mr/src/tdx.rs @@ -83,6 +83,7 @@ fn select_mrtd(measurement: &TdxOsImageMeasurement, vm_config: &VmConfig) -> Res .hugepages(vm_config.hugepages) .num_gpus(vm_config.num_gpus) .num_nics(vm_config.num_nics) + .num_verity_volumes(vm_config.num_verity_volumes) .num_nvswitches(vm_config.num_nvswitches) .host_share_mode(vm_config.host_share_mode.clone()) .ovmf_variant(measurement.tdvf.ovmf_variant) @@ -474,6 +475,7 @@ pub fn tdx_measurements_for_image_dir_without_rtmr0( .hugepages(vm_config.hugepages) .num_gpus(vm_config.num_gpus) .num_nics(vm_config.num_nics) + .num_verity_volumes(vm_config.num_verity_volumes) .num_nvswitches(vm_config.num_nvswitches) .host_share_mode(vm_config.host_share_mode.clone()) .ovmf_variant(ovmf_variant) @@ -570,6 +572,7 @@ pub fn tdx_measurements_for_image_dir_with_acpi_hashes( .hugepages(vm_config.hugepages) .num_gpus(vm_config.num_gpus) .num_nics(vm_config.num_nics) + .num_verity_volumes(vm_config.num_verity_volumes) .num_nvswitches(vm_config.num_nvswitches) .host_share_mode(vm_config.host_share_mode.clone()) .ovmf_variant(ovmf_variant) diff --git a/dstack/dstack-types/src/lib.rs b/dstack/dstack-types/src/lib.rs index f7fa20748..73a7d66e6 100644 --- a/dstack/dstack-types/src/lib.rs +++ b/dstack/dstack-types/src/lib.rs @@ -792,6 +792,10 @@ fn is_default_num_nics(n: &u32) -> bool { *n == default_num_nics() } +fn is_zero(n: &u32) -> bool { + *n == 0 +} + #[derive(Deserialize, Serialize, Debug, Clone)] pub struct VmConfig { #[serde(with = "hex_bytes", default)] @@ -825,6 +829,11 @@ pub struct VmConfig { skip_serializing_if = "is_default_num_nics" )] pub num_nics: u32, + /// Number of read-only verity volume devices attached to the guest. Each + /// volume adds a virtio-blk PCI device before the NICs and therefore + /// changes the measured ACPI/DSDT layout. + #[serde(default, skip_serializing_if = "is_zero")] + pub num_verity_volumes: u32, #[serde(default)] pub hotplug_off: bool, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -1819,7 +1828,7 @@ mod platform_tests { } #[cfg(test)] -mod vm_config_num_nics_tests { +mod vm_config_device_count_tests { use super::VmConfig; fn legacy_json() -> serde_json::Value { @@ -1835,6 +1844,7 @@ mod vm_config_num_nics_tests { fn legacy_config_without_num_nics_defaults_to_one() { let cfg: VmConfig = serde_json::from_value(legacy_json()).unwrap(); assert_eq!(cfg.num_nics, 1); + assert_eq!(cfg.num_verity_volumes, 0); } #[test] @@ -1857,4 +1867,20 @@ mod vm_config_num_nics_tests { let serialized = serde_json::to_value(&cfg).unwrap(); assert_eq!(serialized.get("num_nics").and_then(|v| v.as_u64()), Some(2)); } + + #[test] + fn verity_volume_count_is_serialized_only_when_nonzero() { + let mut cfg: VmConfig = serde_json::from_value(legacy_json()).unwrap(); + let serialized = serde_json::to_value(&cfg).unwrap(); + assert!(serialized.get("num_verity_volumes").is_none()); + + cfg.num_verity_volumes = 2; + let serialized = serde_json::to_value(&cfg).unwrap(); + assert_eq!( + serialized + .get("num_verity_volumes") + .and_then(|v| v.as_u64()), + Some(2) + ); + } } diff --git a/dstack/verifier/src/verification.rs b/dstack/verifier/src/verification.rs index 2404ec163..f2c9dcbbf 100644 --- a/dstack/verifier/src/verification.rs +++ b/dstack/verifier/src/verification.rs @@ -341,6 +341,7 @@ impl CvmVerifier { .hugepages(vm_config.hugepages) .num_gpus(vm_config.num_gpus) .num_nics(vm_config.num_nics) + .num_verity_volumes(vm_config.num_verity_volumes) .num_nvswitches(vm_config.num_nvswitches) .host_share_mode(vm_config.host_share_mode.clone()) .ovmf_variant(ovmf_variant) diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index ea5c471a4..9f24baeae 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -1400,6 +1400,7 @@ fn make_vm_config( // ACPI/DSDT layout and therefore RTMR0. Measure the interface count so the // verifier reconstructs the exact device layout. let num_nics = resolved_networks(manifest, &cfg.cvm).len() as u32; + let num_verity_volumes = manifest.volumes.len() as u32; let mut config = serde_json::to_value(dstack_types::VmConfig { os_image_hash, cpu_count: effective_vcpus, @@ -1412,6 +1413,7 @@ fn make_vm_config( num_gpus: gpus.gpus.len() as u32, num_nvswitches: gpus.bridges.len() as u32, num_nics, + num_verity_volumes, host_share_mode: cfg.cvm.host_share_mode.clone(), hotplug_off: cfg.cvm.qemu_hotplug_off, image: Some(manifest.image.clone()), @@ -1732,6 +1734,33 @@ mod tests { Ok(()) } + #[test] + fn vm_measurement_config_includes_verity_volume_count() -> Result<()> { + let config = test_tdx_config()?; + let mut manifest = test_manifest(2048); + manifest.volumes = vec![ + VmVolume { + source: "/volumes/a.img".into(), + read_only: true, + }, + VmVolume { + source: "/volumes/b.img".into(), + read_only: true, + }, + ]; + let vm_config = make_vm_config( + &config, + &manifest, + &test_tdx_image(true), + &hex_of(0x22, 32), + None, + None, + )?; + + assert_eq!(vm_config["num_verity_volumes"], 2); + Ok(()) + } + #[test] fn tdx_auto_variant_uses_legacy_for_low_non_2g_memory() -> Result<()> { let config = test_tdx_config()?; From aa47b6b24d83be6a16a4b795a97624befa46d416 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 21 Jul 2026 03:49:06 -0700 Subject: [PATCH 20/25] refactor(volume): configure attachments in app compose --- docs/verity-volumes.md | 8 +- dstack/crates/dstack-cli-core/src/compose.rs | 6 +- dstack/crates/dstack-cli/src/main.rs | 20 +--- .../dstack-volume/src/bin/dstack-volume.rs | 103 ++++++++++++++++-- .../crates/dstack-volume/src/volume_format.rs | 3 + dstack/dstack-types/src/lib.rs | 2 + dstack/vmm/rpc/proto/vmm_rpc.proto | 12 +- dstack/vmm/src/app/vm_info.rs | 9 -- dstack/vmm/src/main_service.rs | 15 ++- dstack/vmm/src/vmm-cli.py | 27 ----- os/common/rootfs/dstack-prepare.sh | 2 +- 11 files changed, 125 insertions(+), 82 deletions(-) diff --git a/docs/verity-volumes.md b/docs/verity-volumes.md index d14f2f709..afb77d938 100644 --- a/docs/verity-volumes.md +++ b/docs/verity-volumes.md @@ -47,6 +47,7 @@ This adds the following measured entry to `app-compose.json`: { "verity_volumes": [ { + "source": "models.img", "verity_root": "a1b2c3d4...", "target": "/run/models" } @@ -59,7 +60,7 @@ The target must be an absolute path on a writable guest filesystem, such as ## Guest activation -Before the application starts, `dstack-volume`: +Before the application starts, `dstack-volume mount-all app-compose.json`: 1. Scans `/sys/class/block` for disks whose first partition, or whole disk when unpartitioned, starts with the `DSTACK_VOLUME` magic. @@ -72,6 +73,11 @@ A required volume that is missing, malformed, or fails verification stops guest preparation. dm-verity continues verifying blocks lazily as the application reads them. +For diagnostics, `dstack-volume scan` lists recognized disks and +`dstack-volume status app-compose.json` compares requested roots with attached +and active devices. A single entry can be activated with +`dstack-volume mount app-compose.json INDEX`. + ## Trust and limitations The root hash authenticates bytes, not availability. A malicious host can omit a diff --git a/dstack/crates/dstack-cli-core/src/compose.rs b/dstack/crates/dstack-cli-core/src/compose.rs index 7bf857505..db79fda31 100644 --- a/dstack/crates/dstack-cli-core/src/compose.rs +++ b/dstack/crates/dstack-cli-core/src/compose.rs @@ -13,14 +13,14 @@ use serde_json::json; /// `kms_enabled` selects KMS mode (deterministic, upgradeable per-app keys); /// gateway and local-key-provider are off for the direct-port single-node flow. /// -/// `verity_volumes` is a list of `(verity_root, target)` pairs. Each becomes a +/// `verity_volumes` is a list of `(source, verity_root, target)` tuples. Each becomes a /// measured `verity_volumes` entry, so the CVM only seeds content matching the /// attested root. Empty for a normal deploy. pub fn build_app_compose( name: &str, docker_compose_yaml: &str, kms_enabled: bool, - verity_volumes: &[(String, String)], + verity_volumes: &[(String, String, String)], ) -> String { let mut manifest = json!({ "manifest_version": 2, @@ -43,7 +43,7 @@ pub fn build_app_compose( if !verity_volumes.is_empty() { manifest["verity_volumes"] = json!(verity_volumes .iter() - .map(|(root, target)| json!({ "verity_root": root, "target": target })) + .map(|(source, root, target)| json!({ "source": source, "verity_root": root, "target": target })) .collect::>()); } // pretty-print via Value's Display (`{:#}`) — infallible, and byte-identical diff --git a/dstack/crates/dstack-cli/src/main.rs b/dstack/crates/dstack-cli/src/main.rs index ffffd3fd7..dd1bc8f8e 100644 --- a/dstack/crates/dstack-cli/src/main.rs +++ b/dstack/crates/dstack-cli/src/main.rs @@ -290,9 +290,9 @@ async fn cmd_verity( Ok(()) } -/// A parsed `--volume` spec: the disk to attach plus its measured verity entry. +/// A parsed `--volume` spec. All fields become part of measured app-compose. struct VolumeSpec { - volume: rpc::VmVolume, + source: String, verity_root: String, target: String, } @@ -327,12 +327,7 @@ fn parse_volume(spec: &str) -> Result { bail!("target '{target}' must be an absolute path"); } Ok(VolumeSpec { - // verity volumes are read-only by construction; a writable one would let a - // guest corrupt the shared backing file (the vmm also forces this). - volume: rpc::VmVolume { - source: name.to_string(), - read_only: true, - }, + source: name.to_string(), verity_root: root.to_string(), target: target.to_string(), }) @@ -430,12 +425,11 @@ async fn cmd_deploy( // each --volume declares a measured verity_volumes entry, so the built // app-compose (and thus app_id) binds the attested roots. - let verity_volumes: Vec<(String, String)> = parsed_volumes + let verity_volumes: Vec<(String, String, String)> = parsed_volumes .iter() - .map(|v| (v.verity_root.clone(), v.target.clone())) + .map(|v| (v.source.clone(), v.verity_root.clone(), v.target.clone())) .collect(); let app_compose = compose::build_app_compose(name, &yaml, !no_kms, &verity_volumes); - let volumes: Vec<_> = parsed_volumes.into_iter().map(|v| v.volume).collect(); let mut cfg = rpc::VmConfiguration { name: name.to_string(), @@ -445,7 +439,6 @@ async fn cmd_deploy( memory, disk_size: disk, ports: port_maps.clone(), - volumes, ..Default::default() }; @@ -675,8 +668,7 @@ mod tests { fn parses_volume_specs() { let root = "a".repeat(64); let data = parse_volume(&format!("weights.img:{root}:/models/llama")).unwrap(); - assert_eq!(data.volume.source, "weights.img"); - assert!(data.volume.read_only); + assert_eq!(data.source, "weights.img"); assert_eq!(data.verity_root, root); assert_eq!(data.target, "/models/llama"); diff --git a/dstack/crates/dstack-volume/src/bin/dstack-volume.rs b/dstack/crates/dstack-volume/src/bin/dstack-volume.rs index 2f486a2bd..5d665af21 100644 --- a/dstack/crates/dstack-volume/src/bin/dstack-volume.rs +++ b/dstack/crates/dstack-volume/src/bin/dstack-volume.rs @@ -41,22 +41,58 @@ struct VerityVolume { fn main() -> Result<()> { tracing_subscriber::fmt().init(); - let compose_path = std::env::args_os() - .nth(1) - .map(PathBuf::from) - .unwrap_or_else(|| PathBuf::from("app-compose.json")); + let mut args = std::env::args_os().skip(1); + let command = args + .next() + .context( + "usage: dstack-volume ", + )?; + match command.to_str() { + Some("mount-all") => mount_all( + args.next() + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from("app-compose.json")), + ), + Some("mount") => { + let compose = args.next().context("mount requires COMPOSE and INDEX")?; + let index = args + .next() + .context("mount requires COMPOSE and INDEX")? + .to_str() + .context("INDEX is not UTF-8")? + .parse() + .context("invalid volume INDEX")?; + mount_one(PathBuf::from(compose), index) + } + Some("scan") => scan(), + Some("status") => status( + args.next() + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from("app-compose.json")), + ), + _ => bail!("unknown command {:?}", command), + } +} + +fn read_compose(compose_path: &Path) -> Result { // Deserialize the complete shared type before probing any untrusted disk. // This validates roots and targets as part of parsing the measured compose. - let compose: AppCompose = serde_json::from_slice(&fs::read(&compose_path)?) - .with_context(|| format!("parsing {}", compose_path.display()))?; - if compose.verity_volumes.is_empty() { - return Ok(()); - } + serde_json::from_slice(&fs::read(compose_path)?) + .with_context(|| format!("parsing {}", compose_path.display())) +} +fn prepare_volumes() -> Result> { let _ = run_cmd!(modprobe dm-verity); let _ = run_cmd!(udevadm settle --timeout=5); + discover_volumes() +} - let volumes = discover_volumes()?; +fn mount_all(compose_path: PathBuf) -> Result<()> { + let compose = read_compose(&compose_path)?; + if compose.verity_volumes.is_empty() { + return Ok(()); + } + let volumes = prepare_volumes()?; info!( found = volumes.len(), requested = compose.verity_volumes.len(), @@ -74,6 +110,53 @@ fn main() -> Result<()> { Ok(()) } +fn mount_one(compose_path: PathBuf, index: usize) -> Result<()> { + let compose = read_compose(&compose_path)?; + let requested = compose + .verity_volumes + .get(index) + .with_context(|| format!("volume index {index} is out of range"))?; + let volumes = prepare_volumes()?; + activate_requested(index, requested, &volumes, &mut HashSet::new()).with_context(|| { + format!( + "failed to activate required volume {index} at {}", + requested.target.display() + ) + }) +} + +fn scan() -> Result<()> { + for volume in prepare_volumes()? { + println!( + "{}\tdata={}\thash={}", + hex::encode(volume.root_hash), + volume.data.display(), + volume.hash.display() + ); + } + Ok(()) +} + +fn status(compose_path: PathBuf) -> Result<()> { + let compose = read_compose(&compose_path)?; + let volumes = prepare_volumes()?; + for (index, requested) in compose.verity_volumes.iter().enumerate() { + let attached = volumes + .iter() + .any(|volume| volume.root_hash == requested.verity_root); + let mapper_name = format!("dstack-verity{index}"); + let active = Path::new("/dev/mapper").join(&mapper_name).exists() + && mapping_root(&mapper_name)? + .eq_ignore_ascii_case(&hex::encode(requested.verity_root)); + println!( + "{index}\troot={}\ttarget={}\tattached={attached}\tactive={active}", + hex::encode(requested.verity_root), + requested.target.display() + ); + } + Ok(()) +} + fn discover_volumes() -> Result> { let mut found = Vec::new(); let disks = list_disks()?; diff --git a/dstack/crates/dstack-volume/src/volume_format.rs b/dstack/crates/dstack-volume/src/volume_format.rs index 89bf8c6a3..df9fd4328 100644 --- a/dstack/crates/dstack-volume/src/volume_format.rs +++ b/dstack/crates/dstack-volume/src/volume_format.rs @@ -65,6 +65,7 @@ mod tests { fn verity_volume_validates_root_and_target_during_deserialization( ) -> Result<(), serde_json::Error> { let volume: VerityVolume = serde_json::from_value(serde_json::json!({ + "source": "models.img", "verity_root": "5a".repeat(32), "target": "/run/models" }))?; @@ -76,11 +77,13 @@ mod tests { ); assert!(serde_json::from_value::(serde_json::json!({ + "source": "models.img", "verity_root": "abcd", "target": "/run/models" })) .is_err()); assert!(serde_json::from_value::(serde_json::json!({ + "source": "models.img", "verity_root": "5a".repeat(32), "target": "relative/path" })) diff --git a/dstack/dstack-types/src/lib.rs b/dstack/dstack-types/src/lib.rs index 73a7d66e6..e585acdf6 100644 --- a/dstack/dstack-types/src/lib.rs +++ b/dstack/dstack-types/src/lib.rs @@ -130,6 +130,8 @@ pub struct AppCompose { /// A pre-baked, read-only dm-verity volume attached to the CVM. #[derive(Deserialize, Serialize, Debug, Clone)] pub struct VerityVolume { + /// Bare image file name resolved by the VMM under `cvm.volumes_dir`. + pub source: String, /// dm-verity root hash (hex): the volume's content identity and integrity /// check. The guest matches attached devices against it. #[serde(with = "hex_bytes")] diff --git a/dstack/vmm/rpc/proto/vmm_rpc.proto b/dstack/vmm/rpc/proto/vmm_rpc.proto index 44d584400..ff4b8ac25 100644 --- a/dstack/vmm/rpc/proto/vmm_rpc.proto +++ b/dstack/vmm/rpc/proto/vmm_rpc.proto @@ -118,17 +118,7 @@ message VmConfiguration { optional NetworkingConfig networking = 18; // Per-VM networking overrides. If set, wins over singular networking. repeated NetworkingConfig networks = 19; - // Extra volumes to attach (e.g. pre-baked verity volumes). The host only - // supplies bytes; the guest verifies content against the measured verity_root. - repeated VmVolume volumes = 20; -} - -// An extra disk attached to the VM, such as a pre-baked verity volume. -message VmVolume { - // Volume file name, resolved against the vmm's configured cvm.volumes_dir. - string source = 1; - // Attach the disk read-only. - bool read_only = 2; + reserved 20; } // Per-VM networking configuration. diff --git a/dstack/vmm/src/app/vm_info.rs b/dstack/vmm/src/app/vm_info.rs index 1bc39b9c8..d039418c8 100644 --- a/dstack/vmm/src/app/vm_info.rs +++ b/dstack/vmm/src/app/vm_info.rs @@ -169,15 +169,6 @@ impl VmInfo { no_tee, networking: configured_networking, networks: configured_networks, - volumes: self - .manifest - .volumes - .iter() - .map(|volume| pb::VmVolume { - source: volume.source.clone(), - read_only: volume.read_only, - }) - .collect(), }) }, app_url: self diff --git a/dstack/vmm/src/main_service.rs b/dstack/vmm/src/main_service.rs index 4e5f4c208..c3faa225c 100644 --- a/dstack/vmm/src/main_service.rs +++ b/dstack/vmm/src/main_service.rs @@ -184,7 +184,9 @@ pub fn create_manifest_from_vm_config( Some(gpus) => resolve_gpus_with_config(gpus, cvm_config)?, None => GpuConfig::default(), }; - let volumes = resolve_volumes(&request.volumes, cvm_config)?; + let app_compose: AppCompose = serde_json::from_str(&request.compose_file) + .context("invalid app-compose in VM configuration")?; + let volumes = resolve_volumes(&app_compose.verity_volumes, cvm_config)?; Ok(Manifest { id, @@ -211,7 +213,7 @@ pub fn create_manifest_from_vm_config( /// bare file name under that directory; the host attaches the bytes, and the /// guest verifies content against the measured `verity_root`. fn resolve_volumes( - reqs: &[rpc::VmVolume], + reqs: &[dstack_types::VerityVolume], cvm_config: &crate::config::CvmConfig, ) -> Result> { if reqs.is_empty() { @@ -903,7 +905,8 @@ mod tests { VmConfiguration { name: "vm-test".to_string(), image: "dstack-test".to_string(), - compose_file: "{}".to_string(), + compose_file: r#"{"manifest_version":2,"name":"test","runner":"docker-compose"}"# + .to_string(), vcpu: 1, memory: 1024, disk_size: 10, @@ -920,7 +923,6 @@ mod tests { no_tee: false, networking: None, networks: vec![], - volumes: vec![], } } @@ -1014,9 +1016,10 @@ mod tests { cvm_config.volumes_dir = tmp.path().to_string_lossy().into_owned(); let volumes = resolve_volumes( - &[rpc::VmVolume { + &[dstack_types::VerityVolume { source: "volume.img".into(), - read_only: false, + verity_root: [0; 32], + target: "/run/volume".into(), }], &cvm_config, )?; diff --git a/dstack/vmm/src/vmm-cli.py b/dstack/vmm/src/vmm-cli.py index 0d6605f0a..9671a025e 100755 --- a/dstack/vmm/src/vmm-cli.py +++ b/dstack/vmm/src/vmm-cli.py @@ -341,26 +341,6 @@ def parse_port_mapping(port_str: str) -> Dict: raise argparse.ArgumentTypeError(f"Invalid port mapping format: {port_str}") -def parse_volume(spec: str) -> Dict: - """Parse a volume spec "name" | "name:ro" into a dictionary. - - `name` is a bare file name resolved by the VMM against cvm.volumes_dir. - Verity volumes are read-only (the VMM enforces it regardless). - """ - parts = spec.split(":") - name = parts[0] - mode = parts[1] if len(parts) > 1 else "ro" - if len(parts) > 2 or mode != "ro": - raise argparse.ArgumentTypeError( - f'volume spec must be "name" or "name:ro" (verity volumes are read-only), got {spec!r}' - ) - if not name or "/" in name or ".." in name: - raise argparse.ArgumentTypeError( - f"volume name must be a bare file name (no '/' or '..'), got {name!r}" - ) - return {"source": name, "read_only": True} - - def read_utf8(filepath: str) -> str: """Read a file and return its contents as a UTF-8 string.""" with open(filepath, "rb") as f: @@ -912,7 +892,6 @@ def create_vm(self, args) -> None: "app_id": args.app_id, "user_config": user_config, "ports": [parse_port_mapping(port) for port in args.port or []], - "volumes": [parse_volume(v) for v in args.volume or []], "hugepages": args.hugepages, "pin_numa": args.pin_numa, "stopped": args.stopped, @@ -1776,12 +1755,6 @@ def _patched_format_help(): type=str, help="Port mapping in format: protocol[:address]:from:to", ) - deploy_parser.add_argument( - "--volume", - action="append", - type=str, - help='Attach a read-only volume by name from cvm.volumes_dir: "name" | "name:ro"', - ) deploy_parser.add_argument( "--gpu", action="append", diff --git a/os/common/rootfs/dstack-prepare.sh b/os/common/rootfs/dstack-prepare.sh index 85b64598d..313ba9eb3 100755 --- a/os/common/rootfs/dstack-prepare.sh +++ b/os/common/rootfs/dstack-prepare.sh @@ -301,7 +301,7 @@ echo "============================" cd /dstack # Verify and mount required read-only data volumes before the application starts. -/bin/dstack-volume app-compose.json +/bin/dstack-volume mount-all app-compose.json if [ "$(jq 'has("init_script")' app-compose.json)" == true ]; then log "Running init script" From 38087def48c4028d00417ff87d99b89f7ca895be Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 21 Jul 2026 04:02:25 -0700 Subject: [PATCH 21/25] refactor(volume): parse commands with clap derive --- dstack/Cargo.lock | 1 + dstack/crates/dstack-volume/Cargo.toml | 1 + .../dstack-volume/src/bin/dstack-volume.rs | 70 +++++++++++-------- 3 files changed, 42 insertions(+), 30 deletions(-) diff --git a/dstack/Cargo.lock b/dstack/Cargo.lock index b920c479b..19a347f08 100644 --- a/dstack/Cargo.lock +++ b/dstack/Cargo.lock @@ -2277,6 +2277,7 @@ version = "0.6.0" dependencies = [ "anyhow", "binrw", + "clap", "cmd_lib", "dstack-types", "fs-err", diff --git a/dstack/crates/dstack-volume/Cargo.toml b/dstack/crates/dstack-volume/Cargo.toml index d47798d55..76262b6fe 100644 --- a/dstack/crates/dstack-volume/Cargo.toml +++ b/dstack/crates/dstack-volume/Cargo.toml @@ -12,6 +12,7 @@ license.workspace = true anyhow = { workspace = true, features = ["std"] } binrw.workspace = true cmd_lib.workspace = true +clap.workspace = true dstack-types.workspace = true tokio = { workspace = true, features = ["rt"] } serde_json = { workspace = true, features = ["std"] } diff --git a/dstack/crates/dstack-volume/src/bin/dstack-volume.rs b/dstack/crates/dstack-volume/src/bin/dstack-volume.rs index 5d665af21..35d1b7b1a 100644 --- a/dstack/crates/dstack-volume/src/bin/dstack-volume.rs +++ b/dstack/crates/dstack-volume/src/bin/dstack-volume.rs @@ -16,6 +16,7 @@ use std::os::unix::fs::MetadataExt; use std::path::{Path, PathBuf}; use anyhow::{bail, Context, Result}; +use clap::{Parser, Subcommand}; use cmd_lib::{run_cmd, run_fun}; use dstack_types::{AppCompose, VerityVolume as RequestedVolume}; use dstack_volume::volume_format::{ @@ -39,38 +40,38 @@ struct VerityVolume { root_hash: [u8; 32], } +#[derive(Parser)] +#[command(about = "Discover and activate dstack verity volumes")] +struct Cli { + #[command(subcommand)] + command: VolumeCommand, +} + +#[derive(Subcommand)] +enum VolumeCommand { + /// Activate every verity volume required by the measured app compose. + MountAll { + #[arg(default_value = "app-compose.json")] + compose: PathBuf, + }, + /// Activate one required volume by its index in app compose. + Mount { compose: PathBuf, index: usize }, + /// List recognized dstack volume devices. + Scan, + /// Compare required volumes with attached and active devices. + Status { + #[arg(default_value = "app-compose.json")] + compose: PathBuf, + }, +} + fn main() -> Result<()> { tracing_subscriber::fmt().init(); - let mut args = std::env::args_os().skip(1); - let command = args - .next() - .context( - "usage: dstack-volume ", - )?; - match command.to_str() { - Some("mount-all") => mount_all( - args.next() - .map(PathBuf::from) - .unwrap_or_else(|| PathBuf::from("app-compose.json")), - ), - Some("mount") => { - let compose = args.next().context("mount requires COMPOSE and INDEX")?; - let index = args - .next() - .context("mount requires COMPOSE and INDEX")? - .to_str() - .context("INDEX is not UTF-8")? - .parse() - .context("invalid volume INDEX")?; - mount_one(PathBuf::from(compose), index) - } - Some("scan") => scan(), - Some("status") => status( - args.next() - .map(PathBuf::from) - .unwrap_or_else(|| PathBuf::from("app-compose.json")), - ), - _ => bail!("unknown command {:?}", command), + match Cli::parse().command { + VolumeCommand::MountAll { compose } => mount_all(compose), + VolumeCommand::Mount { compose, index } => mount_one(compose, index), + VolumeCommand::Scan => scan(), + VolumeCommand::Status { compose } => status(compose), } } @@ -447,6 +448,15 @@ fn mapping_root(mapper_name: &str) -> Result { mod tests { use super::*; + #[test] + fn cli_uses_default_compose_for_mount_all() { + let cli = Cli::try_parse_from(["dstack-volume", "mount-all"]).unwrap(); + let VolumeCommand::MountAll { compose } = cli.command else { + panic!("unexpected command"); + }; + assert_eq!(compose, Path::new("app-compose.json")); + } + fn header() -> DstackVolumeHeader { DstackVolumeHeader::new_verity([0x5a; 32]) } From 895e60014012bcbee718aa626a1098aae3e536d9 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 21 Jul 2026 04:04:52 -0700 Subject: [PATCH 22/25] refactor(volume): remove single-volume mount command --- docs/verity-volumes.md | 3 +-- .../dstack-volume/src/bin/dstack-volume.rs | 18 ------------------ 2 files changed, 1 insertion(+), 20 deletions(-) diff --git a/docs/verity-volumes.md b/docs/verity-volumes.md index afb77d938..e4ee122ba 100644 --- a/docs/verity-volumes.md +++ b/docs/verity-volumes.md @@ -75,8 +75,7 @@ reads them. For diagnostics, `dstack-volume scan` lists recognized disks and `dstack-volume status app-compose.json` compares requested roots with attached -and active devices. A single entry can be activated with -`dstack-volume mount app-compose.json INDEX`. +and active devices. ## Trust and limitations diff --git a/dstack/crates/dstack-volume/src/bin/dstack-volume.rs b/dstack/crates/dstack-volume/src/bin/dstack-volume.rs index 35d1b7b1a..4b2e9f0c3 100644 --- a/dstack/crates/dstack-volume/src/bin/dstack-volume.rs +++ b/dstack/crates/dstack-volume/src/bin/dstack-volume.rs @@ -54,8 +54,6 @@ enum VolumeCommand { #[arg(default_value = "app-compose.json")] compose: PathBuf, }, - /// Activate one required volume by its index in app compose. - Mount { compose: PathBuf, index: usize }, /// List recognized dstack volume devices. Scan, /// Compare required volumes with attached and active devices. @@ -69,7 +67,6 @@ fn main() -> Result<()> { tracing_subscriber::fmt().init(); match Cli::parse().command { VolumeCommand::MountAll { compose } => mount_all(compose), - VolumeCommand::Mount { compose, index } => mount_one(compose, index), VolumeCommand::Scan => scan(), VolumeCommand::Status { compose } => status(compose), } @@ -111,21 +108,6 @@ fn mount_all(compose_path: PathBuf) -> Result<()> { Ok(()) } -fn mount_one(compose_path: PathBuf, index: usize) -> Result<()> { - let compose = read_compose(&compose_path)?; - let requested = compose - .verity_volumes - .get(index) - .with_context(|| format!("volume index {index} is out of range"))?; - let volumes = prepare_volumes()?; - activate_requested(index, requested, &volumes, &mut HashSet::new()).with_context(|| { - format!( - "failed to activate required volume {index} at {}", - requested.target.display() - ) - }) -} - fn scan() -> Result<()> { for volume in prepare_volumes()? { println!( From b6575858deef0037fbff0a72d82045b42cda9f9c Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 21 Jul 2026 07:35:37 -0700 Subject: [PATCH 23/25] fix(volume): address CLI and verity review findings --- dstack/Cargo.lock | 3 + dstack/crates/dstack-cli-core/Cargo.toml | 1 + dstack/crates/dstack-cli-core/src/compose.rs | 12 +-- dstack/crates/dstack-cli/Cargo.toml | 2 + dstack/crates/dstack-cli/src/main.rs | 92 +++++++++++-------- .../dstack-volume/src/bin/dstack-volume.rs | 22 ++--- dstack/crates/dstack-volume/src/volume.rs | 3 +- .../recipes-core/dstack-guest/dstack-guest.bb | 2 +- 8 files changed, 74 insertions(+), 63 deletions(-) diff --git a/dstack/Cargo.lock b/dstack/Cargo.lock index 19a347f08..1b761177e 100644 --- a/dstack/Cargo.lock +++ b/dstack/Cargo.lock @@ -1773,8 +1773,10 @@ dependencies = [ "anyhow", "clap", "dstack-cli-core", + "dstack-types", "dstack-volume", "fs-err", + "hex", "serde_json", "tokio", "tracing", @@ -1786,6 +1788,7 @@ name = "dstack-cli-core" version = "0.6.0" dependencies = [ "anyhow", + "dstack-types", "dstack-vmm-rpc", "http-client", "rustix 0.38.44", diff --git a/dstack/crates/dstack-cli-core/Cargo.toml b/dstack/crates/dstack-cli-core/Cargo.toml index ea74358bc..720946063 100644 --- a/dstack/crates/dstack-cli-core/Cargo.toml +++ b/dstack/crates/dstack-cli-core/Cargo.toml @@ -12,6 +12,7 @@ license.workspace = true anyhow.workspace = true http-client = { workspace = true, features = ["prpc"] } dstack-vmm-rpc.workspace = true +dstack-types.workspace = true serde_json.workspace = true # advisory file locking (flock) for the allowlist/state read-modify-write; # already in the dependency tree transitively, so no extra compile cost. diff --git a/dstack/crates/dstack-cli-core/src/compose.rs b/dstack/crates/dstack-cli-core/src/compose.rs index db79fda31..acac40ab4 100644 --- a/dstack/crates/dstack-cli-core/src/compose.rs +++ b/dstack/crates/dstack-cli-core/src/compose.rs @@ -13,14 +13,13 @@ use serde_json::json; /// `kms_enabled` selects KMS mode (deterministic, upgradeable per-app keys); /// gateway and local-key-provider are off for the direct-port single-node flow. /// -/// `verity_volumes` is a list of `(source, verity_root, target)` tuples. Each becomes a -/// measured `verity_volumes` entry, so the CVM only seeds content matching the -/// attested root. Empty for a normal deploy. +/// Each `verity_volumes` entry is measured, so the CVM only mounts content +/// matching the attested root. Empty for a normal deploy. pub fn build_app_compose( name: &str, docker_compose_yaml: &str, kms_enabled: bool, - verity_volumes: &[(String, String, String)], + verity_volumes: &[dstack_types::VerityVolume], ) -> String { let mut manifest = json!({ "manifest_version": 2, @@ -41,10 +40,7 @@ pub fn build_app_compose( "secure_time": false, }); if !verity_volumes.is_empty() { - manifest["verity_volumes"] = json!(verity_volumes - .iter() - .map(|(source, root, target)| json!({ "source": source, "verity_root": root, "target": target })) - .collect::>()); + manifest["verity_volumes"] = json!(verity_volumes); } // pretty-print via Value's Display (`{:#}`) — infallible, and byte-identical // to serde_json::to_string_pretty (avoids an expect on an unfailable Result). diff --git a/dstack/crates/dstack-cli/Cargo.toml b/dstack/crates/dstack-cli/Cargo.toml index bf522939a..d1ef71268 100644 --- a/dstack/crates/dstack-cli/Cargo.toml +++ b/dstack/crates/dstack-cli/Cargo.toml @@ -19,7 +19,9 @@ anyhow.workspace = true clap.workspace = true dstack-cli-core.workspace = true dstack-volume.workspace = true +dstack-types.workspace = true fs-err.workspace = true +hex.workspace = true serde_json.workspace = true tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } tracing.workspace = true diff --git a/dstack/crates/dstack-cli/src/main.rs b/dstack/crates/dstack-cli/src/main.rs index dd1bc8f8e..d73016250 100644 --- a/dstack/crates/dstack-cli/src/main.rs +++ b/dstack/crates/dstack-cli/src/main.rs @@ -11,7 +11,7 @@ //! `logs`, a global `-j/--json`). use anyhow::{bail, Context, Result}; -use clap::{Parser, Subcommand}; +use clap::{Parser, Subcommand, ValueEnum}; use dstack_cli_core::layout::InstallLayout; use dstack_cli_core::vmm::{Vmm, DEFAULT_HOST}; use dstack_cli_core::{compose, ports, rpc}; @@ -77,7 +77,7 @@ enum Command { ports: Vec, /// attach a verity volume, as printed by `dstack verity`: the file name /// in the vmm's volumes_dir, its verity_root, and the target - /// (`docker` or a mount path). Repeatable. + /// (an absolute mount path). Repeatable. #[arg(long = "volume", value_name = "NAME:VERITY_ROOT:TARGET")] volumes: Vec, /// deploy in non-KMS mode (ephemeral keys; no KMS required). @@ -124,11 +124,29 @@ enum Command { #[arg(long, short = 'o', default_value = "verity.img")] output: String, /// squashfs compression: `none` (the default), `zstd`, or `gzip`. - #[arg(long, default_value = "none")] - compress: String, + #[arg(long, value_enum, default_value_t, conflicts_with = "fs_image")] + compress: CompressionArg, }, } +#[derive(Clone, Copy, Default, ValueEnum)] +enum CompressionArg { + #[default] + None, + Zstd, + Gzip, +} + +impl From for dstack_volume::Compression { + fn from(value: CompressionArg) -> Self { + match value { + CompressionArg::None => Self::None, + CompressionArg::Zstd => Self::Zstd, + CompressionArg::Gzip => Self::Gzip, + } + } +} + #[tokio::main] async fn main() -> Result<()> { // progress (e.g. `verity` pulling layers) goes to stderr so it never mixes @@ -223,16 +241,7 @@ async fn main() -> Result<()> { fs_image, output, compress, - } => { - cmd_verity( - dir.as_deref(), - fs_image.as_deref(), - &output, - &compress, - json, - ) - .await - } + } => cmd_verity(dir.as_deref(), fs_image.as_deref(), &output, compress, json).await, } } @@ -240,20 +249,14 @@ async fn cmd_verity( dir: Option<&str>, fs_image: Option<&str>, output: &str, - compress: &str, + compress: CompressionArg, json: bool, ) -> Result<()> { - let compress = match compress { - "none" => dstack_volume::Compression::None, - "zstd" => dstack_volume::Compression::Zstd, - "gzip" => dstack_volume::Compression::Gzip, - other => bail!("unknown --compress '{other}' (use none|zstd|gzip)"), - }; let result = dstack_volume::verity(dstack_volume::VerityOptions { dir: dir.map(std::path::PathBuf::from), fs_image: fs_image.map(std::path::PathBuf::from), output: output.into(), - compress, + compress: compress.into(), }) .await?; @@ -290,13 +293,6 @@ async fn cmd_verity( Ok(()) } -/// A parsed `--volume` spec. All fields become part of measured app-compose. -struct VolumeSpec { - source: String, - verity_root: String, - target: String, -} - /// Parse a `--volume` spec `NAME:VERITY_ROOT:TARGET`. /// /// `NAME` is the volume file in the vmm's volumes_dir. `VERITY_ROOT` and `TARGET` @@ -305,7 +301,7 @@ struct VolumeSpec { /// spec to paste. /// /// `TARGET` is an absolute read-only mount path in the guest. -fn parse_volume(spec: &str) -> Result { +fn parse_volume(spec: &str) -> Result { let mut parts = spec.splitn(3, ':'); let name = parts.next().unwrap_or_default(); let (root, target) = match (parts.next(), parts.next()) { @@ -326,10 +322,12 @@ fn parse_volume(spec: &str) -> Result { if !target.starts_with('/') { bail!("target '{target}' must be an absolute path"); } - Ok(VolumeSpec { + let mut verity_root = [0; 32]; + hex::decode_to_slice(root, &mut verity_root).context("decoding verity_root")?; + Ok(dstack_types::VerityVolume { source: name.to_string(), - verity_root: root.to_string(), - target: target.to_string(), + verity_root, + target: target.into(), }) } @@ -425,11 +423,7 @@ async fn cmd_deploy( // each --volume declares a measured verity_volumes entry, so the built // app-compose (and thus app_id) binds the attested roots. - let verity_volumes: Vec<(String, String, String)> = parsed_volumes - .iter() - .map(|v| (v.source.clone(), v.verity_root.clone(), v.target.clone())) - .collect(); - let app_compose = compose::build_app_compose(name, &yaml, !no_kms, &verity_volumes); + let app_compose = compose::build_app_compose(name, &yaml, !no_kms, &parsed_volumes); let mut cfg = rpc::VmConfiguration { name: name.to_string(), @@ -669,8 +663,8 @@ mod tests { let root = "a".repeat(64); let data = parse_volume(&format!("weights.img:{root}:/models/llama")).unwrap(); assert_eq!(data.source, "weights.img"); - assert_eq!(data.verity_root, root); - assert_eq!(data.target, "/models/llama"); + assert_eq!(data.verity_root, [0xaa; 32]); + assert_eq!(data.target, std::path::Path::new("/models/llama")); assert!(parse_volume("weights.img").is_err()); // missing verity_root:target assert!(parse_volume(&format!("weights.img:{root}")).is_err()); // missing target @@ -734,5 +728,23 @@ mod tests { "rootfs.ext4" ]) .is_err()); + assert!(Cli::try_parse_from([ + "dstack", + "verity", + "--fs-image", + "rootfs.ext4", + "--compress", + "zstd" + ]) + .is_err()); + assert!(Cli::try_parse_from([ + "dstack", + "verity", + "--dir", + "data", + "--compress", + "invalid" + ]) + .is_err()); } } diff --git a/dstack/crates/dstack-volume/src/bin/dstack-volume.rs b/dstack/crates/dstack-volume/src/bin/dstack-volume.rs index 4b2e9f0c3..ded54c880 100644 --- a/dstack/crates/dstack-volume/src/bin/dstack-volume.rs +++ b/dstack/crates/dstack-volume/src/bin/dstack-volume.rs @@ -9,7 +9,6 @@ //! table. Everything read from a disk is untrusted: kind handlers use the //! measured app compose as their source of policy and cryptographic identity. -use std::collections::HashSet; use std::io::Read; use std::os::unix::ffi::OsStrExt; use std::os::unix::fs::MetadataExt; @@ -96,9 +95,8 @@ fn mount_all(compose_path: PathBuf) -> Result<()> { requested = compose.verity_volumes.len(), "discovered dstack volumes" ); - let mut used = HashSet::new(); for (index, requested) in compose.verity_volumes.iter().enumerate() { - activate_requested(index, requested, &volumes, &mut used).with_context(|| { + activate_requested(index, requested, &volumes).with_context(|| { format!( "failed to activate required volume {index} at {}", requested.target.display() @@ -276,14 +274,10 @@ fn activate_requested( index: usize, requested: &RequestedVolume, volumes: &[VerityVolume], - used: &mut HashSet, ) -> Result<()> { - let (candidate_index, candidate) = volumes + let candidate = volumes .iter() - .enumerate() - .find(|(candidate_index, volume)| { - !used.contains(candidate_index) && volume.root_hash == requested.verity_root - }) + .find(|volume| volume.root_hash == requested.verity_root) .context("no attached volume advertises the measured root")?; let mapper_name = format!("dstack-verity{index}"); @@ -291,9 +285,12 @@ fn activate_requested( let expected_root = hex::encode(requested.verity_root); if mapped.exists() { if mapping_root(&mapper_name)?.eq_ignore_ascii_case(&expected_root) { - verify_first_block(&mapped)?; - mount_volume(requested, &mapped)?; - used.insert(candidate_index); + if let Err(err) = + verify_first_block(&mapped).and_then(|_| mount_volume(requested, &mapped)) + { + let _ = run_cmd!(veritysetup close $mapper_name); + return Err(err); + } info!(mapper = mapper_name, "reused active verity mapping"); return Ok(()); } @@ -310,7 +307,6 @@ fn activate_requested( let _ = run_cmd!(veritysetup close $mapper_name); return Err(err); } - used.insert(candidate_index); Ok(()) } diff --git a/dstack/crates/dstack-volume/src/volume.rs b/dstack/crates/dstack-volume/src/volume.rs index 68303f958..6c8175788 100644 --- a/dstack/crates/dstack-volume/src/volume.rs +++ b/dstack/crates/dstack-volume/src/volume.rs @@ -142,7 +142,8 @@ fn seal_data_image( let uuid = uuid_from_data(data_path, data_size)?; let out = run_fun!( veritysetup format --salt $salt_hex --uuid $uuid - --data-block-size 4096 --hash-block-size 4096 $data_path $hash_path + --hash sha256 --format 1 --data-block-size 4096 --hash-block-size 4096 + $data_path $hash_path ) .context("running veritysetup format")?; let verity_root = diff --git a/os/yocto/layers/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/os/yocto/layers/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index dd0097f4d..8dfd86e7e 100644 --- a/os/yocto/layers/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/os/yocto/layers/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -15,7 +15,7 @@ DSTACK_ROOTFS_SRC ?= "${DSTACK_MONOREPO_ROOT}/os/common/rootfs" S = "${UNPACKDIR}/repo/dstack" DSTACK_ROOTFS_FILES = "${UNPACKDIR}/repo/os/common/rootfs" -RDEPENDS:${PN} += "bash" +RDEPENDS:${PN} += "bash cryptsetup util-linux-blkid util-linux-mount" DEPENDS += "rsync-native tpm2-tss" DEPENDS += "cmake-native" From 5ffb0f38163e6cf106c3f8b9c7636518f5003938 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 21 Jul 2026 20:22:39 -0700 Subject: [PATCH 24/25] fix(volume): preserve compose compatibility and enforce invariants --- dstack/crates/dstack-cli/src/main.rs | 1 + .../dstack-volume/src/bin/dstack-volume.rs | 1 + dstack/dstack-types/src/lib.rs | 47 ++++++++++++++++ dstack/vmm/src/app.rs | 4 -- dstack/vmm/src/app/qemu.rs | 12 ++--- dstack/vmm/src/main_service.rs | 53 ++++++++++++++----- 6 files changed, 94 insertions(+), 24 deletions(-) diff --git a/dstack/crates/dstack-cli/src/main.rs b/dstack/crates/dstack-cli/src/main.rs index d73016250..77b62b9c0 100644 --- a/dstack/crates/dstack-cli/src/main.rs +++ b/dstack/crates/dstack-cli/src/main.rs @@ -420,6 +420,7 @@ async fn cmd_deploy( .iter() .map(|s| parse_volume(s)) .collect::>>()?; + dstack_types::validate_verity_volumes(&parsed_volumes).map_err(anyhow::Error::msg)?; // each --volume declares a measured verity_volumes entry, so the built // app-compose (and thus app_id) binds the attested roots. diff --git a/dstack/crates/dstack-volume/src/bin/dstack-volume.rs b/dstack/crates/dstack-volume/src/bin/dstack-volume.rs index ded54c880..a62f563cc 100644 --- a/dstack/crates/dstack-volume/src/bin/dstack-volume.rs +++ b/dstack/crates/dstack-volume/src/bin/dstack-volume.rs @@ -86,6 +86,7 @@ fn prepare_volumes() -> Result> { fn mount_all(compose_path: PathBuf) -> Result<()> { let compose = read_compose(&compose_path)?; + dstack_types::validate_verity_volumes(&compose.verity_volumes).map_err(anyhow::Error::msg)?; if compose.verity_volumes.is_empty() { return Ok(()); } diff --git a/dstack/dstack-types/src/lib.rs b/dstack/dstack-types/src/lib.rs index e585acdf6..dd59425b2 100644 --- a/dstack/dstack-types/src/lib.rs +++ b/dstack/dstack-types/src/lib.rs @@ -141,6 +141,29 @@ pub struct VerityVolume { pub target: std::path::PathBuf, } +/// Reject ambiguous volume declarations before any disk is attached or +/// activated. A root identifies one logical volume, and a mount target can +/// only be owned by one volume. +pub fn validate_verity_volumes(volumes: &[VerityVolume]) -> Result<(), String> { + let mut roots = std::collections::HashSet::new(); + let mut targets = std::collections::HashSet::new(); + for volume in volumes { + if !roots.insert(volume.verity_root) { + return Err(format!( + "duplicate verity root {}", + hex::encode(volume.verity_root) + )); + } + if !targets.insert(&volume.target) { + return Err(format!( + "duplicate verity volume target {}", + volume.target.display() + )); + } + } + Ok(()) +} + fn deserialize_absolute_path<'de, D>(deserializer: D) -> Result where D: serde::Deserializer<'de>, @@ -155,6 +178,30 @@ where Ok(path) } +#[cfg(test)] +mod verity_volume_tests { + use super::{validate_verity_volumes, VerityVolume}; + + fn volume(root: u8, target: &str) -> VerityVolume { + VerityVolume { + source: format!("{root}.img"), + verity_root: [root; 32], + target: target.into(), + } + } + + #[test] + fn rejects_duplicate_roots_and_targets() { + assert!(validate_verity_volumes(&[volume(1, "/a"), volume(1, "/b")]) + .unwrap_err() + .contains("duplicate verity root")); + assert!(validate_verity_volumes(&[volume(1, "/a"), volume(2, "/a")]) + .unwrap_err() + .contains("duplicate verity volume target")); + validate_verity_volumes(&[volume(1, "/a"), volume(2, "/b")]).unwrap(); + } +} + /// Canonical source for the policy used when `requirements.gpu_policy` is /// absent. Both typed defaults and measurement are derived from this JSON. pub const DEFAULT_GPU_POLICY: &str = "{}"; diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 9f24baeae..4b42b06c5 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -85,8 +85,6 @@ pub struct PortMapping { #[derive(Deserialize, Serialize, Debug, Clone)] pub struct VmVolume { pub source: String, - #[serde(default)] - pub read_only: bool, } #[derive(Deserialize, Serialize, Clone, Builder, Debug)] @@ -1741,11 +1739,9 @@ mod tests { manifest.volumes = vec![ VmVolume { source: "/volumes/a.img".into(), - read_only: true, }, VmVolume { source: "/volumes/b.img".into(), - read_only: true, }, ]; let vm_config = make_vm_config( diff --git a/dstack/vmm/src/app/qemu.rs b/dstack/vmm/src/app/qemu.rs index 30d9c8e5f..041784d8d 100644 --- a/dstack/vmm/src/app/qemu.rs +++ b/dstack/vmm/src/app/qemu.rs @@ -176,7 +176,6 @@ fn virtio_pci_device(device: &str, snp: bool) -> String { struct PreparedVolume { source: String, - read_only: bool, } struct PreparedQemuLaunch { @@ -214,7 +213,6 @@ impl PreparedQemuLaunch { .iter() .map(|volume| PreparedVolume { source: volume.source.clone(), - read_only: volume.read_only, }) .collect(); @@ -550,10 +548,10 @@ impl QemuCommandBuilder<'_> { // established device order. for (index, volume) in self.prepared.volumes.iter().enumerate() { let id = format!("vol{index}"); - let mut drive = format!("file={},if=none,id={id},format=raw", volume.source); - if volume.read_only { - drive.push_str(",readonly=on"); - } + let drive = format!( + "file={},if=none,id={id},format=raw,readonly=on", + volume.source + ); let device = format!("virtio-blk-pci,drive={id}"); command @@ -1048,7 +1046,6 @@ mod tests { networks: vec![], volumes: vec![VmVolume { source: "/does-not-exist/volume.img".into(), - read_only: true, }], }, image: Image { @@ -1086,7 +1083,6 @@ mod tests { networks: vec![config.cvm.networking.clone(), config.cvm.networking.clone()], volumes: vec![PreparedVolume { source: "/does-not-exist/volume.img".into(), - read_only: true, }], hugepage_numa_nodes: None, gpu_numa_nodes: HashMap::new(), diff --git a/dstack/vmm/src/main_service.rs b/dstack/vmm/src/main_service.rs index c3faa225c..4c51664d5 100644 --- a/dstack/vmm/src/main_service.rs +++ b/dstack/vmm/src/main_service.rs @@ -184,9 +184,9 @@ pub fn create_manifest_from_vm_config( Some(gpus) => resolve_gpus_with_config(gpus, cvm_config)?, None => GpuConfig::default(), }; - let app_compose: AppCompose = serde_json::from_str(&request.compose_file) - .context("invalid app-compose in VM configuration")?; - let volumes = resolve_volumes(&app_compose.verity_volumes, cvm_config)?; + let verity_volumes = extract_verity_volumes(&request.compose_file)?; + dstack_types::validate_verity_volumes(&verity_volumes).map_err(anyhow::Error::msg)?; + let volumes = resolve_volumes(&verity_volumes, cvm_config)?; Ok(Manifest { id, @@ -209,6 +209,19 @@ pub fn create_manifest_from_vm_config( }) } +/// Extract only the field understood by this VMM. Keep every other app-compose +/// field opaque so newer guest schemas and legacy third-party clients remain +/// compatible with older VMMs. +fn extract_verity_volumes(compose: &str) -> Result> { + let Ok(compose) = serde_json::from_str::(compose) else { + return Ok(vec![]); + }; + let Some(volumes) = compose.get("verity_volumes") else { + return Ok(vec![]); + }; + serde_json::from_value(volumes.clone()).context("invalid verity_volumes in app-compose") +} + /// Resolve requested volumes against `cvm.volumes_dir`. Each `source` must be a /// bare file name under that directory; the host attaches the bytes, and the /// guest verifies content against the measured `verity_root`. @@ -229,11 +242,6 @@ fn resolve_volumes( let real = resolve_volume_source(&base, &v.source)?; Ok(crate::app::VmVolume { source: real.to_string_lossy().into_owned(), - // Verity volumes are always read-only: the backing file is shared - // content-addressed data, so a writable attach could only let one - // guest corrupt it for every other tenant. Force it regardless of - // what the client asked for. - read_only: true, }) }) .collect() @@ -905,8 +913,7 @@ mod tests { VmConfiguration { name: "vm-test".to_string(), image: "dstack-test".to_string(), - compose_file: r#"{"manifest_version":2,"name":"test","runner":"docker-compose"}"# - .to_string(), + compose_file: "{}".to_string(), vcpu: 1, memory: 1024, disk_size: 10, @@ -934,6 +941,25 @@ mod tests { assert!(manifest.networks.is_empty()); } + #[test] + fn volume_extraction_keeps_other_compose_fields_opaque() -> Result<()> { + assert!(extract_verity_volumes("not json")?.is_empty()); + assert!(extract_verity_volumes(r#"{"future_manifest":true}"#)?.is_empty()); + + let compose = serde_json::json!({ + "unknown_future_field": { "anything": true }, + "verity_volumes": [{ + "source": "volume.img", + "verity_root": "11".repeat(32), + "target": "/run/volume" + }] + }); + let volumes = extract_verity_volumes(&compose.to_string())?; + assert_eq!(volumes.len(), 1); + assert_eq!(volumes[0].verity_root, [0x11; 32]); + Ok(()) + } + #[test] fn explicit_user_networking_is_resolved_before_persist() { let mut request = test_vm_configuration(); @@ -1009,7 +1035,7 @@ mod tests { } #[test] - fn resolve_volumes_forces_read_only() -> Result<()> { + fn resolve_volumes_resolves_measured_source() -> Result<()> { let tmp = tempfile::tempdir()?; fs::write(tmp.path().join("volume.img"), b"volume")?; let mut cvm_config = test_cvm_config(); @@ -1025,7 +1051,10 @@ mod tests { )?; assert_eq!(volumes.len(), 1); - assert!(volumes[0].read_only); + assert_eq!( + volumes[0].source, + tmp.path().join("volume.img").display().to_string() + ); Ok(()) } } From 517eec9526f8b67782326131e854e6f5f367de16 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 21 Jul 2026 20:30:49 -0700 Subject: [PATCH 25/25] fix(volume): share duplicate roots across mount targets --- docs/verity-volumes.md | 4 ++ .../dstack-volume/src/bin/dstack-volume.rs | 35 ++++++++++---- dstack/dstack-types/src/lib.rs | 19 ++------ dstack/vmm/src/main_service.rs | 47 +++++++++++++++++++ 4 files changed, 83 insertions(+), 22 deletions(-) diff --git a/docs/verity-volumes.md b/docs/verity-volumes.md index e4ee122ba..6c7dac295 100644 --- a/docs/verity-volumes.md +++ b/docs/verity-volumes.md @@ -73,6 +73,10 @@ A required volume that is missing, malformed, or fails verification stops guest preparation. dm-verity continues verifying blocks lazily as the application reads them. +The same `verity_root` may be declared more than once with different targets. +The VMM attaches one disk for that root, and the guest mounts the verified +content at each requested target. + For diagnostics, `dstack-volume scan` lists recognized disks and `dstack-volume status app-compose.json` compares requested roots with attached and active devices. diff --git a/dstack/crates/dstack-volume/src/bin/dstack-volume.rs b/dstack/crates/dstack-volume/src/bin/dstack-volume.rs index a62f563cc..367fc33fc 100644 --- a/dstack/crates/dstack-volume/src/bin/dstack-volume.rs +++ b/dstack/crates/dstack-volume/src/bin/dstack-volume.rs @@ -9,6 +9,7 @@ //! table. Everything read from a disk is untrusted: kind handlers use the //! measured app compose as their source of policy and cryptographic identity. +use std::collections::HashMap; use std::io::Read; use std::os::unix::ffi::OsStrExt; use std::os::unix::fs::MetadataExt; @@ -96,13 +97,24 @@ fn mount_all(compose_path: PathBuf) -> Result<()> { requested = compose.verity_volumes.len(), "discovered dstack volumes" ); + let mut active_roots: HashMap<[u8; 32], PathBuf> = HashMap::new(); for (index, requested) in compose.verity_volumes.iter().enumerate() { - activate_requested(index, requested, &volumes).with_context(|| { + if let Some(mapped) = active_roots.get(&requested.verity_root) { + mount_volume(requested, mapped).with_context(|| { + format!( + "failed to mount required volume {index} at {}", + requested.target.display() + ) + })?; + continue; + } + let mapped = activate_requested(index, requested, &volumes).with_context(|| { format!( "failed to activate required volume {index} at {}", requested.target.display() ) })?; + active_roots.insert(requested.verity_root, mapped); } Ok(()) } @@ -126,10 +138,17 @@ fn status(compose_path: PathBuf) -> Result<()> { let attached = volumes .iter() .any(|volume| volume.root_hash == requested.verity_root); - let mapper_name = format!("dstack-verity{index}"); - let active = Path::new("/dev/mapper").join(&mapper_name).exists() - && mapping_root(&mapper_name)? - .eq_ignore_ascii_case(&hex::encode(requested.verity_root)); + let expected_root = hex::encode(requested.verity_root); + let mut active = false; + for mapper_index in 0..compose.verity_volumes.len() { + let mapper_name = format!("dstack-verity{mapper_index}"); + if Path::new("/dev/mapper").join(&mapper_name).exists() + && mapping_root(&mapper_name)?.eq_ignore_ascii_case(&expected_root) + { + active = true; + break; + } + } println!( "{index}\troot={}\ttarget={}\tattached={attached}\tactive={active}", hex::encode(requested.verity_root), @@ -275,7 +294,7 @@ fn activate_requested( index: usize, requested: &RequestedVolume, volumes: &[VerityVolume], -) -> Result<()> { +) -> Result { let candidate = volumes .iter() .find(|volume| volume.root_hash == requested.verity_root) @@ -293,7 +312,7 @@ fn activate_requested( return Err(err); } info!(mapper = mapper_name, "reused active verity mapping"); - return Ok(()); + return Ok(mapped); } run_cmd!(veritysetup close $mapper_name).context("closing stale verity mapping")?; } @@ -308,7 +327,7 @@ fn activate_requested( let _ = run_cmd!(veritysetup close $mapper_name); return Err(err); } - Ok(()) + Ok(mapped) } fn verify_first_block(path: &Path) -> Result<()> { diff --git a/dstack/dstack-types/src/lib.rs b/dstack/dstack-types/src/lib.rs index dd59425b2..8edb4cf8f 100644 --- a/dstack/dstack-types/src/lib.rs +++ b/dstack/dstack-types/src/lib.rs @@ -141,19 +141,12 @@ pub struct VerityVolume { pub target: std::path::PathBuf, } -/// Reject ambiguous volume declarations before any disk is attached or -/// activated. A root identifies one logical volume, and a mount target can -/// only be owned by one volume. +/// Reject ambiguous mount declarations before any disk is attached or +/// activated. The same root may intentionally be mounted at multiple targets, +/// but a target can only be owned by one volume. pub fn validate_verity_volumes(volumes: &[VerityVolume]) -> Result<(), String> { - let mut roots = std::collections::HashSet::new(); let mut targets = std::collections::HashSet::new(); for volume in volumes { - if !roots.insert(volume.verity_root) { - return Err(format!( - "duplicate verity root {}", - hex::encode(volume.verity_root) - )); - } if !targets.insert(&volume.target) { return Err(format!( "duplicate verity volume target {}", @@ -191,10 +184,8 @@ mod verity_volume_tests { } #[test] - fn rejects_duplicate_roots_and_targets() { - assert!(validate_verity_volumes(&[volume(1, "/a"), volume(1, "/b")]) - .unwrap_err() - .contains("duplicate verity root")); + fn allows_duplicate_roots_but_rejects_duplicate_targets() { + validate_verity_volumes(&[volume(1, "/a"), volume(1, "/b")]).unwrap(); assert!(validate_verity_volumes(&[volume(1, "/a"), volume(2, "/a")]) .unwrap_err() .contains("duplicate verity volume target")); diff --git a/dstack/vmm/src/main_service.rs b/dstack/vmm/src/main_service.rs index 4c51664d5..42a9cc781 100644 --- a/dstack/vmm/src/main_service.rs +++ b/dstack/vmm/src/main_service.rs @@ -2,6 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 +use std::collections::HashSet; use std::ops::Deref; use std::path::{Component, Path, PathBuf}; use std::time::{SystemTime, UNIX_EPOCH}; @@ -237,7 +238,19 @@ fn resolve_volumes( bail!("volumes requested but cvm.volumes_dir is not configured"); } let base = fs::canonicalize(dir)?; + let mut roots = HashSet::new(); reqs.iter() + .filter(|volume| { + let first = roots.insert(volume.verity_root); + if !first { + warn!( + root = %hex::encode(volume.verity_root), + source = volume.source, + "not attaching duplicate verity root" + ); + } + first + }) .map(|v| { let real = resolve_volume_source(&base, &v.source)?; Ok(crate::app::VmVolume { @@ -1057,4 +1070,38 @@ mod tests { ); Ok(()) } + + #[test] + fn resolve_volumes_attaches_duplicate_root_once() -> Result<()> { + let tmp = tempfile::tempdir()?; + fs::write(tmp.path().join("first.img"), b"volume")?; + let mut cvm_config = test_cvm_config(); + cvm_config.volumes_dir = tmp.path().to_string_lossy().into_owned(); + let root = [7; 32]; + + let volumes = resolve_volumes( + &[ + dstack_types::VerityVolume { + source: "first.img".into(), + verity_root: root, + target: "/run/first".into(), + }, + dstack_types::VerityVolume { + // This source deliberately does not exist: the first entry + // owns the single attachment for this content root. + source: "duplicate.img".into(), + verity_root: root, + target: "/run/second".into(), + }, + ], + &cvm_config, + )?; + + assert_eq!(volumes.len(), 1); + assert_eq!( + volumes[0].source, + tmp.path().join("first.img").display().to_string() + ); + Ok(()) + } }