From 3a3aec6bd5e7b9c20ca114437fd8ce7090514ca4 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 20 Jul 2026 02:15:45 -0700 Subject: [PATCH 01/12] feat(http-client): support custom request headers and a prpc bearer token --- dstack/http-client/src/lib.rs | 32 ++++++++++++++++++++++++-------- dstack/http-client/src/prpc.rs | 20 +++++++++++++++++++- 2 files changed, 43 insertions(+), 9 deletions(-) diff --git a/dstack/http-client/src/lib.rs b/dstack/http-client/src/lib.rs index e33d89ae1..e616e1aae 100644 --- a/dstack/http-client/src/lib.rs +++ b/dstack/http-client/src/lib.rs @@ -34,6 +34,17 @@ pub async fn http_request( base: &str, path: &str, body: &[u8], +) -> Result<(u16, Vec)> { + http_request_with_headers(method, base, path, body, &[]).await +} + +/// Same as [`http_request`], with extra request headers (e.g. `Authorization`). +pub async fn http_request_with_headers( + method: &str, + base: &str, + path: &str, + body: &[u8], + headers: &[(&str, &str)], ) -> Result<(u16, Vec)> { debug!("Sending HTTP request to {base}, path={path}"); let mut response = if let Some(uds) = base.strip_prefix("unix:") { @@ -44,24 +55,29 @@ pub async fn http_request( }; let client: Client> = Client::unix(); let unix_uri: hyper::Uri = Uri::new(uds, &path).into(); - let req = Request::builder() - .method(method) - .uri(unix_uri) - .body(Full::new(Bytes::copy_from_slice(body)))?; + let mut builder = Request::builder().method(method).uri(unix_uri); + for (name, value) in headers { + builder = builder.header(*name, *value); + } + let req = builder.body(Full::new(Bytes::copy_from_slice(body)))?; client.request(req).await? } else if base.starts_with("vsock:") { let client = Client::vsock(); let uri = mk_url(base, path).parse::()?; - let req = Request::builder() - .method(method) - .uri(uri) - .body(Full::new(Bytes::copy_from_slice(body)))?; + let mut builder = Request::builder().method(method).uri(uri); + for (name, value) in headers { + builder = builder.header(*name, *value); + } + let req = builder.body(Full::new(Bytes::copy_from_slice(body)))?; client.request(req).await? } else { let uri = mk_url(base, path); let client = reqwest::Client::builder().build()?; let method = reqwest::Method::from_bytes(method.as_bytes())?; let mut request = client.request(method, uri); + for (name, value) in headers { + request = request.header(*name, *value); + } if !body.is_empty() { request = request.body(body.to_vec()); } diff --git a/dstack/http-client/src/prpc.rs b/dstack/http-client/src/prpc.rs index eacc03b1e..e26e5bb64 100644 --- a/dstack/http-client/src/prpc.rs +++ b/dstack/http-client/src/prpc.rs @@ -12,6 +12,7 @@ use serde::{de::DeserializeOwned, Serialize}; pub struct PrpcClient { base_url: String, path_append: String, + auth_token: Option, } impl PrpcClient { @@ -19,6 +20,7 @@ impl PrpcClient { Self { base_url, path_append: String::new(), + auth_token: None, } } @@ -29,8 +31,16 @@ impl PrpcClient { Self { base_url: format!("unix:{socket_path}"), path_append: path, + auth_token: None, } } + + /// Send `Authorization: Bearer ` with every request. + pub fn with_bearer_token(mut self, token: impl Into) -> Self { + let token = token.into(); + self.auth_token = (!token.is_empty()).then_some(token); + self + } } impl RequestClient for PrpcClient { @@ -41,7 +51,15 @@ impl RequestClient for PrpcClient { { let body = serde_json::to_vec(&body).context("Failed to serialize body")?; let path = format!("{}{path}?json", self.path_append); - let (status, body) = super::http_request("POST", &self.base_url, &path, &body).await?; + let auth_header; + let mut headers: Vec<(&str, &str)> = Vec::new(); + if let Some(token) = &self.auth_token { + auth_header = format!("Bearer {token}"); + headers.push(("Authorization", auth_header.as_str())); + } + let (status, body) = + super::http_request_with_headers("POST", &self.base_url, &path, &body, &headers) + .await?; if status != 200 { anyhow::bail!("Invalid status code: {status}, path={path}"); } From f87dc6af5fdd683c72b127111128bfeeb5d17a89 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 20 Jul 2026 02:15:45 -0700 Subject: [PATCH 02/12] feat(cli): authenticate to a VMM with a bearer token --- dstack/crates/dstack-cli-core/src/vmm.rs | 40 ++++++++++++++++--- dstack/crates/dstack-cli/src/main.rs | 49 +++++++++++++++++++----- 2 files changed, 73 insertions(+), 16 deletions(-) diff --git a/dstack/crates/dstack-cli-core/src/vmm.rs b/dstack/crates/dstack-cli-core/src/vmm.rs index 5703ee71b..cbec5d7d3 100644 --- a/dstack/crates/dstack-cli-core/src/vmm.rs +++ b/dstack/crates/dstack-cli-core/src/vmm.rs @@ -11,7 +11,7 @@ use anyhow::{anyhow, bail, Result}; use dstack_vmm_rpc::vmm_client::VmmClient; use dstack_vmm_rpc::{Id, StatusRequest, StatusResponse, VmConfiguration}; -use http_client::http_request; +use http_client::http_request_with_headers; use http_client::prpc::PrpcClient; /// default local VMM control socket (created by `dstackup install`). @@ -20,25 +20,46 @@ pub const DEFAULT_HOST: &str = "unix:/var/run/dstack/vmm.sock"; /// a connection to a VMM — local unix socket or remote http endpoint. pub struct Vmm { rpc: VmmClient, - /// base string usable with [`http_request`] for non-prpc endpoints. + /// base string usable with [`http_request_with_headers`] for non-prpc endpoints. base: String, + /// bearer token sent with every request when the VMM has `[auth]` enabled. + auth_token: Option, } impl Vmm { /// connect to a VMM addressed by `host`: /// `unix:/path/to/vmm.sock` (local) or `http(s)://host:port` (remote). pub fn connect(host: &str) -> Result { + Self::connect_with_token(host, None) + } + + /// like [`Vmm::connect`], additionally sending `Authorization: Bearer + /// ` with every request — required when the VMM has `[auth]` + /// enabled. An empty or `None` token sends no header. + pub fn connect_with_token(host: &str, token: Option<&str>) -> Result { let host = host.trim(); + let token = token.map(str::trim).filter(|t| !t.is_empty()); + let with_token = |client: PrpcClient| match token { + Some(token) => client.with_bearer_token(token), + None => client, + }; if let Some(sock) = host.strip_prefix("unix:") { - let rpc = VmmClient::new(PrpcClient::new_unix(sock.to_string(), "/prpc".to_string())); + let client = PrpcClient::new_unix(sock.to_string(), "/prpc".to_string()); + let rpc = VmmClient::new(with_token(client)); Ok(Self { rpc, base: format!("unix:{sock}"), + auth_token: token.map(str::to_string), }) } else if host.starts_with("http://") || host.starts_with("https://") { let base = host.trim_end_matches('/').to_string(); - let rpc = VmmClient::new(PrpcClient::new(format!("{base}/prpc"))); - Ok(Self { rpc, base }) + let client = PrpcClient::new(format!("{base}/prpc")); + let rpc = VmmClient::new(with_token(client)); + Ok(Self { + rpc, + base, + auth_token: token.map(str::to_string), + }) } else { bail!( "unsupported host '{host}': expected unix:/path/to/vmm.sock or http(s)://host:port" @@ -112,7 +133,14 @@ impl Vmm { /// socket and over the localhost HTTP endpoint written by `dstackup install`. pub async fn logs(&self, id: &str, lines: u32) -> Result { let path = format!("/logs?id={id}&follow=false&ansi=false&lines={lines}"); - let (status, body) = http_request("GET", &self.base, &path, b"").await?; + let auth_header; + let mut headers: Vec<(&str, &str)> = Vec::new(); + if let Some(token) = &self.auth_token { + auth_header = format!("Bearer {token}"); + headers.push(("Authorization", auth_header.as_str())); + } + let (status, body) = + http_request_with_headers("GET", &self.base, &path, b"", &headers).await?; if status != 200 { bail!("vmm /logs returned status {status}"); } diff --git a/dstack/crates/dstack-cli/src/main.rs b/dstack/crates/dstack-cli/src/main.rs index 1d31897d3..9ee0a662e 100644 --- a/dstack/crates/dstack-cli/src/main.rs +++ b/dstack/crates/dstack-cli/src/main.rs @@ -32,7 +32,9 @@ struct Cli { #[arg(long, global = true, value_name = "DIR")] prefix: Option, - /// auth token for a remote VMM. + /// auth token for a VMM with `[auth]` enabled (sent as `Authorization: + /// Bearer`). Falls back to `DSTACK_VMM_TOKEN`, then the token written by + /// `dstackup install`. #[arg(long, global = true)] token: Option, @@ -105,8 +107,6 @@ enum Command { #[tokio::main] async fn main() -> Result<()> { let cli = Cli::parse(); - // remote-auth wiring lands with the TLS+token transport. - let _ = &cli.token; let defaults = LocalDefaults::read(cli.prefix.as_deref()); let use_local_defaults = cli.host.is_none(); let host = cli @@ -114,11 +114,24 @@ async fn main() -> Result<()> { .clone() .or_else(|| defaults.as_ref().and_then(|d| d.client_url.clone())) .unwrap_or_else(|| DEFAULT_HOST.to_string()); + // auth token: --token, then DSTACK_VMM_TOKEN, then the token file written + // by `dstackup install` (local defaults only). + let token = cli + .token + .clone() + .or_else(|| std::env::var("DSTACK_VMM_TOKEN").ok()) + .filter(|t| !t.trim().is_empty()) + .or_else(|| { + use_local_defaults + .then(|| defaults.as_ref().and_then(LocalDefaults::token)) + .flatten() + }); + let token = token.as_deref(); let json = cli.json; match cli.command { - Command::Apps => cmd_apps(&host, json).await, - Command::Logs { id, lines } => cmd_logs(&host, &id, lines).await, + Command::Apps => cmd_apps(&host, token, json).await, + Command::Logs { id, lines } => cmd_logs(&host, token, &id, lines).await, Command::Deploy { compose, compose_file, @@ -149,6 +162,7 @@ async fn main() -> Result<()> { }; cmd_deploy( &host, + token, &compose, &name, image.as_deref(), @@ -178,6 +192,7 @@ fn resolve_compose_arg(positional: Option, flagged: Option) -> R struct LocalDefaults { client_url: Option, + client_token_path: Option, image: Option, allowlist_path: Option, } @@ -197,6 +212,11 @@ impl LocalDefaults { .and_then(|x| x.as_str()) .filter(|s| !s.is_empty()) .map(str::to_string), + client_token_path: v + .get("client_token_path") + .and_then(|x| x.as_str()) + .filter(|s| !s.is_empty()) + .map(str::to_string), image: v .get("image") .and_then(|x| x.as_str()) @@ -213,6 +233,14 @@ impl LocalDefaults { fn allowlist_path(&self) -> Option { self.allowlist_path.clone() } + + /// read the VMM API token from the file recorded by `dstackup install`. + fn token(&self) -> Option { + let path = self.client_token_path.as_ref()?; + let token = std::fs::read_to_string(path).ok()?; + let token = token.trim(); + (!token.is_empty()).then(|| token.to_string()) + } } #[cfg(test)] @@ -307,6 +335,7 @@ mod tests { #[allow(clippy::too_many_arguments)] async fn cmd_deploy( host: &str, + token: Option<&str>, compose_path: &str, name: &str, image: Option<&str>, @@ -339,7 +368,7 @@ async fn cmd_deploy( ..Default::default() }; - let vmm = Vmm::connect(host)?; + let vmm = Vmm::connect_with_token(host, token)?; let hash = vmm.get_compose_hash(&cfg).await?; let app_id = short(&hash, 40); cfg.app_id = Some(app_id.clone()); @@ -429,8 +458,8 @@ fn stub(name: &str) -> Result<()> { ) } -async fn cmd_apps(host: &str, json: bool) -> Result<()> { - let vmm = Vmm::connect(host)?; +async fn cmd_apps(host: &str, token: Option<&str>, json: bool) -> Result<()> { + let vmm = Vmm::connect_with_token(host, token)?; let resp = vmm.status().await?; if json { let arr: Vec<_> = resp @@ -470,8 +499,8 @@ async fn cmd_apps(host: &str, json: bool) -> Result<()> { Ok(()) } -async fn cmd_logs(host: &str, id: &str, lines: u32) -> Result<()> { - let vmm = Vmm::connect(host)?; +async fn cmd_logs(host: &str, token: Option<&str>, id: &str, lines: u32) -> Result<()> { + let vmm = Vmm::connect_with_token(host, token)?; let logs = vmm.logs(id, lines).await?; print!("{logs}"); Ok(()) From 3e6362c730575c5dce86d3a891601c2673a0f6e0 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 20 Jul 2026 02:15:45 -0700 Subject: [PATCH 03/12] feat(vmm-cli): send a bearer token and reject half-set Basic credentials --- dstack/vmm/src/vmm-cli.py | 61 ++++++++++++++++++++++++++++++--------- 1 file changed, 48 insertions(+), 13 deletions(-) diff --git a/dstack/vmm/src/vmm-cli.py b/dstack/vmm/src/vmm-cli.py index 290ee2440..9671a025e 100755 --- a/dstack/vmm/src/vmm-cli.py +++ b/dstack/vmm/src/vmm-cli.py @@ -63,7 +63,8 @@ def load_config() -> Dict[str, Any]: """Load configuration from the default config file. Returns: - Dictionary with configuration values (url, auth_user, auth_password) + Dictionary with configuration values (url, auth_user, auth_password, + auth_token) """ if not os.path.exists(DEFAULT_CONFIG_PATH): @@ -371,12 +372,29 @@ def __init__( base_url: str, auth_user: Optional[str] = None, auth_password: Optional[str] = None, + auth_token: Optional[str] = None, ): - """Initialize the client with a base URL and optional auth credentials.""" + """Initialize the client with a base URL and optional auth credentials. + + Two credential forms are supported for a VMM with `[auth]` enabled: + a bearer token (sent as `Authorization: Bearer `), or HTTP Basic + (user + password). A bearer token takes precedence when both are set. + """ self.base_url = base_url.rstrip("/") self.use_uds = self.base_url.startswith("unix:") - self.auth_user = auth_user - self.auth_password = auth_password + self.auth_user = auth_user or None + self.auth_password = auth_password or None + self.auth_token = auth_token or None + # fail fast on a half-configured Basic credential: silently sending no + # auth header would make requests fail against `[auth] enabled = true` + # in a way that is hard to diagnose. + if not self.auth_token and bool(self.auth_user) != bool(self.auth_password): + missing = "password" if self.auth_user else "username" + raise ValueError( + f"incomplete VMM Basic auth credentials: {missing} is missing. " + "set both username and password, or use a bearer token " + "(--token / DSTACK_VMM_TOKEN)." + ) if self.use_uds: self.uds_path = self.base_url[5:] # Remove 'unix:' prefix @@ -410,13 +428,17 @@ def request( if headers is None: headers = {} - # Add Basic Authentication header if credentials are provided - if self.auth_user and self.auth_password: - credentials = f"{self.auth_user}:{self.auth_password}" - encoded_credentials = base64.b64encode(credentials.encode("utf-8")).decode( - "ascii" - ) - headers["Authorization"] = f"Basic {encoded_credentials}" + # Add an auth header when credentials are provided. A bearer token wins + # over Basic when both happen to be set. + if "Authorization" not in headers: + if self.auth_token: + headers["Authorization"] = f"Bearer {self.auth_token}" + elif self.auth_user and self.auth_password: + credentials = f"{self.auth_user}:{self.auth_password}" + encoded_credentials = base64.b64encode( + credentials.encode("utf-8") + ).decode("ascii") + headers["Authorization"] = f"Basic {encoded_credentials}" # Prepare the body if isinstance(body, dict): @@ -480,11 +502,12 @@ def __init__( base_url: str, auth_user: Optional[str] = None, auth_password: Optional[str] = None, + auth_token: Optional[str] = None, ): """Initialize the CLI with a base URL and optional auth credentials.""" self.base_url = base_url.rstrip("/") self.headers = {"Content-Type": "application/json"} - self.client = VmmClient(base_url, auth_user, auth_password) + self.client = VmmClient(base_url, auth_user, auth_password, auth_token) def rpc_call(self, method: str, params: Optional[Dict] = None) -> Dict: """Make an RPC call to the dstack-vmm API.""" @@ -1519,6 +1542,7 @@ def main(): default_auth_password = os.environ.get( "DSTACK_VMM_AUTH_PASSWORD", config.get("auth_password") ) + default_auth_token = os.environ.get("DSTACK_VMM_TOKEN", config.get("auth_token")) parser.add_argument( "--url", @@ -1537,6 +1561,13 @@ def main(): default=default_auth_password, help="Basic auth password (can also be set via DSTACK_VMM_AUTH_PASSWORD env var or config file)", ) + parser.add_argument( + "--token", + default=default_auth_token, + help="bearer token for a VMM with `[auth]` enabled, sent as " + "`Authorization: Bearer` (can also be set via DSTACK_VMM_TOKEN env var " + "or config file). Takes precedence over --auth-user/--auth-password.", + ) subparsers = parser.add_subparsers(dest="command", help="Commands") @@ -1939,7 +1970,11 @@ def _patched_format_help(): # Resolve the URL with auto-discovery url = resolve_vmm_url(instances, config, args.url) - cli = VmmCLI(url, args.auth_user, args.auth_password) + try: + cli = VmmCLI(url, args.auth_user, args.auth_password, args.token) + except ValueError as e: + print(f"error: {e}", file=sys.stderr) + sys.exit(1) if args.command == "lsvm": cli.list_vms(args.verbose, args.json) From 7e322c8e4ab21afe622015d865fa472f8d496634 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 20 Jul 2026 02:15:45 -0700 Subject: [PATCH 04/12] feat(vmm): warn when the management API is exposed without auth --- dstack/vmm/src/main.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/dstack/vmm/src/main.rs b/dstack/vmm/src/main.rs index 6e12529ea..fb34bc362 100644 --- a/dstack/vmm/src/main.rs +++ b/dstack/vmm/src/main.rs @@ -205,6 +205,10 @@ async fn main() -> Result<()> { // Register this VMM instance for local discovery discovery::cleanup_stale_registrations(); + // whether the management API binds a TCP address reachable beyond the local + // host (i.e. not a Unix socket and not a loopback IP). Used to warn when the + // surface is exposed without authentication. + let mut listen_tcp_public = false; let listen_address = { // Use Rocket's Endpoint type to parse the address exactly as Rocket would, // then override the port with the figment's port value (matching Rocket's behavior). @@ -213,6 +217,7 @@ async fn main() -> Result<()> { match endpoint.tcp() { Some(addr) => { let port: u16 = figment.extract_inner("port").unwrap_or(addr.port()); + listen_tcp_public = !addr.ip().is_loopback(); format!("{}:{port}", addr.ip()) } None => endpoint.to_string(), @@ -241,6 +246,14 @@ async fn main() -> Result<()> { if config.auth.enabled && !config.auth.htpasswd_file.as_os_str().is_empty() { api_auth = api_auth.with_htpasswd_file(&config.auth.htpasswd_file)?; } + if !config.auth.enabled && listen_tcp_public { + warn!( + "the management API is bound to a non-loopback address ({listen_address}) with \ + `[auth] enabled = false`: the entire VMM control surface (create/stop VM, UI, \ + pRPC) is exposed WITHOUT authentication. set `[auth] enabled = true` with a \ + token, or bind `address` to localhost / a Unix socket." + ); + } let supervisor = { let cfg = &config.supervisor; let abs_exe = Path::new(&cfg.exe).absolutize()?; From 1d930bba521a06f93f6ec758419533d5354937ed Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 20 Jul 2026 02:15:45 -0700 Subject: [PATCH 05/12] feat(dstackup): generate and enable a VMM API token on install --- dstack/crates/dstack-cli-core/src/config.rs | 43 ++++++++-- dstack/crates/dstackup/src/destroy.rs | 4 +- dstack/crates/dstackup/src/install.rs | 93 +++++++++++++++++---- dstack/crates/dstackup/src/state.rs | 4 + 4 files changed, 120 insertions(+), 24 deletions(-) diff --git a/dstack/crates/dstack-cli-core/src/config.rs b/dstack/crates/dstack-cli-core/src/config.rs index 270c88b59..f62cba0ff 100644 --- a/dstack/crates/dstack-cli-core/src/config.rs +++ b/dstack/crates/dstack-cli-core/src/config.rs @@ -9,7 +9,8 @@ //! false` + a set `auto_bootstrap_domain`, the combination validated to make //! bootstrap hands-off). //! * `auth-allowlist.json` — read by the host-side Rust auth webhook. -//! * `vmm.toml` — the host VMM config (gateway + auth-token gating off). +//! * `vmm.toml` — the host VMM config (gateway off; management-API auth token +//! set by `dstackup install`). use crate::host::Platform; use anyhow::{Context, Result}; @@ -277,6 +278,11 @@ pub struct VmmRender { pub kms_urls: Vec, /// confidential-computing platform (selects qemu/share-mode for the CVMs). pub platform: Platform, + /// gate the management API behind a bearer/Basic token (`[auth] enabled`). + pub auth_enabled: bool, + /// the token accepted by the management API when `auth_enabled` is set. + /// Empty renders `tokens = []`. + pub auth_token: String, } impl Default for VmmRender { @@ -295,13 +301,16 @@ impl Default for VmmRender { key_provider_port: 3443, kms_urls: Vec::new(), platform: Platform::Tdx, + auth_enabled: false, + auth_token: String::new(), } } } -/// render the host `vmm.toml`. Gateway and auth-token gating are off -/// (single-node direct-port access); CVMs use user-mode networking with host -/// port mapping. +/// render the host `vmm.toml`. Gateway is off (single-node direct-port +/// access); management-API auth is gated by `r.auth_enabled`/`r.auth_token` +/// (`dstackup install` generates a token and enables it). CVMs use user-mode +/// networking with host port mapping. pub fn vmm_toml(r: &VmmRender) -> String { format!( r#"# generated by `dstackup install` @@ -375,9 +384,12 @@ base_domain = "localhost" port = 8082 agent_port = 8090 +# management API auth. `dstackup install` generates a token and enables this +# so the VMM control surface (create/stop VM, UI, pRPC) is not exposed +# unauthenticated. Clients send `Authorization: Bearer `. [auth] -enabled = false -tokens = [] +enabled = {auth_enabled} +tokens = [{auth_tokens}] [supervisor] exe = "{supervisor_exe}" @@ -422,6 +434,12 @@ port = {kp_port} host_api_port = r.host_api_port, kp_addr = r.key_provider_addr, kp_port = r.key_provider_port, + auth_enabled = r.auth_enabled, + auth_tokens = if r.auth_token.is_empty() { + String::new() + } else { + format!("\"{}\"", r.auth_token) + }, ) } @@ -435,15 +453,28 @@ mod tests { dashboard_addr: "tcp:127.0.0.1:19080".into(), cid_start: 2000, host_api_port: 10001, + auth_enabled: true, + auth_token: "deadbeef".into(), ..Default::default() }; let rendered = vmm_toml(&r); assert!(rendered.contains(r#"address = "tcp:127.0.0.1:19080""#)); assert!(rendered.contains("cid_start = 2000")); assert!(rendered.contains("port = 10001")); + assert!(rendered.contains("enabled = true")); + assert!(rendered.contains(r#"tokens = ["deadbeef"]"#)); toml::from_str::(&rendered).expect("vmm.toml must be valid TOML"); } + #[test] + fn vmm_toml_auth_disabled_renders_empty_tokens() { + let rendered = vmm_toml(&VmmRender::default()); + // default (no token) must still be valid TOML with an empty token list. + assert!(rendered.contains("tokens = []")); + let v: toml::Value = toml::from_str(&rendered).expect("vmm.toml must be valid TOML"); + assert_eq!(v["auth"]["enabled"].as_bool(), Some(false)); + } + #[test] fn kms_toml_has_single_node_invariants() { let cfg = HostConfig { diff --git a/dstack/crates/dstackup/src/destroy.rs b/dstack/crates/dstackup/src/destroy.rs index 2ff6ab4f3..bb6bcd18f 100644 --- a/dstack/crates/dstackup/src/destroy.rs +++ b/dstack/crates/dstackup/src/destroy.rs @@ -4,6 +4,7 @@ //! `dstackup destroy` — tear down what `install` started. +use crate::install::read_token_file; use crate::state::{read_state, state_path}; use crate::systemd::{remove_unit, systemctl, tool}; use anyhow::{Context, Result}; @@ -26,7 +27,8 @@ pub(crate) async fn cmd_destroy(prefix: Option<&str>, purge: bool) -> Result<()> // and the CVM qemu via the unit's cgroup. Look it up by recorded id // AND by name, so an install that died before persisting kms_vm_id // (or a torn state file) doesn't leave the CVM orphaned. - if let Ok(vmm) = Vmm::connect(&st.client_url) { + let token = read_token_file(Path::new(&st.client_token_path)); + if let Ok(vmm) = Vmm::connect_with_token(&st.client_url, token.as_deref()) { let mut target = st.kms_vm_id.clone(); if target.is_none() { if let Ok(s) = vmm.status().await { diff --git a/dstack/crates/dstackup/src/install.rs b/dstack/crates/dstackup/src/install.rs index 469fc51ae..dcad4bb19 100644 --- a/dstack/crates/dstackup/src/install.rs +++ b/dstack/crates/dstackup/src/install.rs @@ -29,16 +29,16 @@ const DAEMON_BINARIES: &[(&str, &str)] = &[ ]; pub(crate) async fn cmd_install(mut o: InstallOpts) -> Result<()> { - // --expose is not safe yet: the rendered vmm.toml binds the VM-control - // plane with neither TLS nor an auth token (the management RPCs are not - // behind an auth guard), so exposing it would hand deploy/destroy to anyone - // who can reach the IP. Refuse until the TLS+token transport lands; the - // supported path is localhost + an SSH tunnel. + // --expose is not safe yet: the rendered vmm.toml now gates the management + // API behind a generated token, but the transport is still plain HTTP, so + // exposing it would send that bearer token in cleartext to anyone on-path. + // Refuse until the TLS transport lands; the supported path is localhost + an + // SSH tunnel. if let Some(ip) = &o.expose { bail!( "--expose {ip} is not yet safe: it would bind the VM-control plane on \ - {ip}:{port} with no TLS and no auth. reach the dashboard over an SSH \ - tunnel instead: ssh -L {port}:127.0.0.1:{port} ", + {ip}:{port} over plain HTTP, leaking the API token in cleartext. reach \ + the dashboard over an SSH tunnel instead: ssh -L {port}:127.0.0.1:{port} ", port = o.dashboard_port ); } @@ -81,10 +81,24 @@ pub(crate) async fn cmd_install(mut o: InstallOpts) -> Result<()> { let client_url = format!("http://{bind}:{}", o.dashboard_port); let kms_port = resolve_kms_port(&o, &st)?; + // management-API token: reuse the one a prior install wrote (so re-runs and + // an already-running VMM keep matching credentials), else mint a fresh one + // below once the config dir exists. The existing token authenticates the + // preflight probe against an already-running, auth-enabled VMM. + let token_path = layout.config_dir.join("vmm-auth-token"); + let existing_token = read_token_file(&token_path); + // 4. preflight - fail BEFORE any side effect (image download, key provider, // dirs, units), so a CID/port clash can't half-install the host. let cid_start = pick_cid_start(o.cid_start, &host::occupied_cid_ranges())?; - let kms_owned = kms_port_owned(&st, &client_url, kms_port, o.no_kms).await; + let kms_owned = kms_port_owned( + &st, + &client_url, + existing_token.as_deref(), + kms_port, + o.no_kms, + ) + .await; let port_plan = tcp_port_plan(&o, &st, platform, &bind, &client_url, kms_port, kms_owned); preflight_ports(&port_plan)?; @@ -117,6 +131,15 @@ pub(crate) async fn cmd_install(mut o: InstallOpts) -> Result<()> { fs::create_dir_all(&dir).with_context(|| format!("creating {}", dir.display()))?; } + // ensure a management-API token exists on disk (0600) before rendering the + // config that references it. The local `dstack` CLI reads this path from the + // install state, so it authenticates automatically. + let vmm_token = match existing_token { + Some(t) => t, + None => generate_vmm_token()?, + }; + write_token_file(&token_path, &vmm_token)?; + // 8. resolve the key provider - run our own unless told to use an existing // one (TDX only; SNP has no SGX local provider). let (kp_addr, kp_port, kp_own_project) = @@ -144,6 +167,8 @@ pub(crate) async fn cmd_install(mut o: InstallOpts) -> Result<()> { key_provider_port: kp_port as u32, kms_urls: kms_urls.clone(), platform, + auth_enabled: true, + auth_token: vmm_token.clone(), ..Default::default() }); // the KMS-in-CVM reaches the host auth webhook at 10.0.2.2:. @@ -191,6 +216,7 @@ pub(crate) async fn cmd_install(mut o: InstallOpts) -> Result<()> { st.run_dir = layout.run_dir.display().to_string(); st.allowlist_path = allow_path.display().to_string(); st.client_url = client_url.clone(); + st.client_token_path = token_path.display().to_string(); st.auth_port = o.auth_port; st.platform = platform.vmm_str().to_string(); st.image = o.image.clone(); @@ -215,7 +241,7 @@ pub(crate) async fn cmd_install(mut o: InstallOpts) -> Result<()> { st.auth_unit = auth_unit.clone(); // 11. VMM systemd unit (idempotent). - if vmm_reachable(&client_url).await { + if vmm_reachable(&client_url, Some(&vmm_token)).await { println!(" [ok] VMM already serving at {client_url}"); } else { install_unit( @@ -225,7 +251,7 @@ pub(crate) async fn cmd_install(mut o: InstallOpts) -> Result<()> { .context("installing the VMM unit")?; println!(" [ok] started {vmm_unit}.service"); print!(" [..] waiting for VMM at {client_url} "); - if wait_ready(&client_url, Duration::from_secs(25)).await { + if wait_ready(&client_url, Some(&vmm_token), Duration::from_secs(25)).await { println!("=> ready"); } else { println!("=> not ready within timeout (journalctl -u {vmm_unit})"); @@ -241,7 +267,7 @@ pub(crate) async fn cmd_install(mut o: InstallOpts) -> Result<()> { if o.no_kms { println!(" (--no-kms: skipping KMS deploy)"); } else { - let vmm = Vmm::connect(&client_url)?; + let vmm = Vmm::connect_with_token(&client_url, Some(&vmm_token))?; let existing = match &st.kms_vm_id { Some(id) if vmm.has_vm(id).await => Some(id.clone()), _ => None, @@ -1007,7 +1033,13 @@ fn tcp_port_plan( ports } -async fn kms_port_owned(st: &State, client_url: &str, kms_port: u16, no_kms: bool) -> bool { +async fn kms_port_owned( + st: &State, + client_url: &str, + token: Option<&str>, + kms_port: u16, + no_kms: bool, +) -> bool { if no_kms || st.client_url != client_url || state_kms_port(st) != Some(kms_port) @@ -1019,7 +1051,7 @@ async fn kms_port_owned(st: &State, client_url: &str, kms_port: u16, no_kms: boo let Some(kms_vm_id) = &st.kms_vm_id else { return false; }; - match Vmm::connect(client_url) { + match Vmm::connect_with_token(client_url, token) { Ok(vmm) => vmm.has_vm(kms_vm_id).await, Err(_) => false, } @@ -1158,18 +1190,18 @@ fn kms_vm_boot_state( } /// one-shot liveness probe of the VMM. -async fn vmm_reachable(client_url: &str) -> bool { - match Vmm::connect(client_url) { +async fn vmm_reachable(client_url: &str, token: Option<&str>) -> bool { + match Vmm::connect_with_token(client_url, token) { Ok(vmm) => vmm.status().await.is_ok(), Err(_) => false, } } /// poll the VMM `Status` RPC until it succeeds or the deadline passes. -async fn wait_ready(client_url: &str, timeout: Duration) -> bool { +async fn wait_ready(client_url: &str, token: Option<&str>, timeout: Duration) -> bool { let deadline = tokio::time::Instant::now() + timeout; loop { - if let Ok(vmm) = Vmm::connect(client_url) { + if let Ok(vmm) = Vmm::connect_with_token(client_url, token) { if vmm.status().await.is_ok() { return true; } @@ -1181,6 +1213,33 @@ async fn wait_ready(client_url: &str, timeout: Duration) -> bool { } } +/// mint a 256-bit management-API token, hex-encoded, from the OS RNG. +fn generate_vmm_token() -> Result { + let mut buf = [0u8; 32]; + let mut f = fs::File::open("/dev/urandom").context("opening /dev/urandom")?; + std::io::Read::read_exact(&mut f, &mut buf).context("reading /dev/urandom")?; + Ok(hex::encode(buf)) +} + +/// read a previously written management-API token, if the file exists and is +/// non-empty. +pub(crate) fn read_token_file(path: &Path) -> Option { + let token = fs::read_to_string(path).ok()?; + let token = token.trim(); + (!token.is_empty()).then(|| token.to_string()) +} + +/// write the management-API token atomically with owner-only (0600) +/// permissions — it is a bearer credential. +fn write_token_file(path: &Path, token: &str) -> Result<()> { + use std::os::unix::fs::PermissionsExt; + dstack_cli_core::fsutil::write_atomic(path, token) + .with_context(|| format!("writing {}", path.display()))?; + fs::set_permissions(path, fs::Permissions::from_mode(0o600)) + .with_context(|| format!("setting 0600 on {}", path.display()))?; + Ok(()) +} + #[cfg(test)] mod tests { use super::*; diff --git a/dstack/crates/dstackup/src/state.rs b/dstack/crates/dstackup/src/state.rs index 3414ac818..96516eac4 100644 --- a/dstack/crates/dstackup/src/state.rs +++ b/dstack/crates/dstackup/src/state.rs @@ -30,6 +30,10 @@ pub(crate) struct State { #[serde(default)] pub(crate) platform: String, pub(crate) client_url: String, + /// path to the management-API bearer token file the local `dstack` CLI + /// reads to authenticate against the VMM. + #[serde(default)] + pub(crate) client_token_path: String, pub(crate) auth_port: u16, /// systemd unit names (without the `.service` suffix). #[serde(default)] From 0fc36ee5c9934b8eb14221c9c755ceda22bf46fb Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 20 Jul 2026 02:15:46 -0700 Subject: [PATCH 06/12] docs: document management and admin API authentication (#796) --- CHANGELOG.md | 5 +++ docs/deployment.md | 13 ++++++ docs/dstack-gateway.md | 25 ++++++++++++ docs/security/security-best-practices.md | 10 +++++ docs/tutorials/attestation-verification.md | 4 ++ docs/tutorials/gateway-build-configuration.md | 1 + docs/tutorials/gateway-service-setup.md | 2 + docs/tutorials/hello-world-app.md | 3 ++ docs/tutorials/kms-cvm-deployment.md | 3 +- .../troubleshooting-first-application.md | 1 + .../troubleshooting-gateway-deployment.md | 2 + .../troubleshooting-kms-deployment.md | 1 + docs/vmm-cli-user-guide.md | 40 +++++++++++++------ 13 files changed, 97 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c88d03653..9d712eb28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added +- shared API authentication (`dstack-api-auth`) protecting the full VMM HTTP/pRPC/UI surface and unifying Gateway/KMS admin auth: bearer/`X-Admin-Token`/HTTP Basic/bcrypt htpasswd, constant-time verification (#796) + ## [0.5.5] - 2025-10-20 ### Added diff --git a/docs/deployment.md b/docs/deployment.md index be439e30a..0de43ea92 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -74,6 +74,11 @@ reuse = true image_path = "./images" run_path = "./run/vm" +[auth] +enabled = true +# generate with: openssl rand -hex 32 +tokens = [""] + [cvm] kms_urls = [] gateway_urls = [] @@ -93,6 +98,10 @@ address = "vsock:2" port = 10000 ``` +> **VMM API authentication (required for non-localhost binds).** Since [PR #796](https://github.com/Dstack-TEE/dstack/pull/796), the `[auth]` token guards the *entire* VMM surface — creating and stopping CVMs, the web UI, and all pRPC calls — not just `/logs`. Because the example above binds `tcp:0.0.0.0:9080` (reachable off the host), you MUST enable `[auth]`; otherwise the whole control API is exposed unauthenticated. Auth is fail-closed: with `enabled = true` and no usable credential, requests are rejected. +> +> Clients authenticate with `Authorization: Bearer ` (or the `X-Admin-Token: ` header). Instead of inline `tokens`, you can point to an Apache bcrypt htpasswd file with `htpasswd_file = "/etc/dstack/admin.htpasswd"` (create it with `htpasswd -B -c /etc/dstack/admin.htpasswd admin`). Only bcrypt (`-B`) entries are accepted. + Download guest images from [dstack guest-OS releases](https://github.com/Dstack-TEE/dstack/releases) and extract to `./images/`. > For reproducible builds and verification, see the [Security Model](./security/security-model.md). @@ -113,6 +122,8 @@ Production KMS requires: #### Auth Server Options +> **Note:** the boot-authorization webhook below (auth-simple / auth-eth) is a *different* mechanism from the KMS admin-API authentication. The webhook allowlists which CVMs may boot and receive keys; the admin API (`admin_token_hash` in `kms.toml`) guards operator RPCs. See [dstack-kms admin authentication](../dstack/kms/README.md#admin-api-authentication). + | Server | Use Case | Configuration | |--------|----------|---------------| | [auth-simple](../dstack/kms/auth-simple/) | Config-file-based whitelisting | JSON config file | @@ -428,6 +439,8 @@ curl http://:9203/finish > > If you skip this, `Onboard.Onboard` or later trusted RPCs will fail with KMS authorization errors. +> **Admin authentication.** Onboarding itself is gated by attestation and the authorization backend above — not by a token. Separately, the KMS *admin* RPCs (for example clearing the image cache) are guarded by an admin token: set `admin_token_hash` (SHA-256 hex of your token) in the KMS `kms.toml` and send the token on admin requests via `Authorization: Bearer `. An empty hash denies all admin RPCs (fail-closed). See [dstack-kms admin authentication](../dstack/kms/README.md#admin-api-authentication). + --- ## Deploying Apps diff --git a/docs/dstack-gateway.md b/docs/dstack-gateway.md index 78ad911bf..2de511702 100644 --- a/docs/dstack-gateway.md +++ b/docs/dstack-gateway.md @@ -63,3 +63,28 @@ Open `vmm.toml` and adjust dstack-gateway configuration in the `gateway` section - `base_domain`: Same as `base_domain` from `gateway.toml`'s `core.proxy` section - `port`: Same as `listen_port` from `gateway.toml`'s `core.proxy` section + +## Admin API authentication + +The gateway exposes a separate admin API (used for sync, WireGuard peer management, and other operator RPCs). Configure it in the `core.admin` section of `gateway.toml`: + +```toml +[core.admin] +enabled = true +address = "0.0.0.0:9016" +# generate with: openssl rand -hex 32 +admin_token = "" +# alternatively, an Apache bcrypt htpasswd file (htpasswd -B -c admin.htpasswd admin) +# htpasswd_file = "/etc/dstack/gateway-admin.htpasswd" +insecure_no_auth = false +``` + +- `enabled`: enable the admin API server. +- `address`: bind address/port for the admin API. +- `admin_token`: shared admin token. It can also be supplied via the environment variables `DSTACK_GATEWAY_ADMIN_TOKEN` or `ADMIN_API_TOKEN` instead of the config file. +- `htpasswd_file`: path to an Apache bcrypt htpasswd file (create with `htpasswd -B -c admin.htpasswd admin`); only bcrypt entries are accepted. Can be used instead of, or alongside, `admin_token`. +- `insecure_no_auth`: development-only escape hatch that disables admin authentication. Never enable it on a network-reachable admin interface. + +The admin server is fail-closed: if it is enabled with no `admin_token` and no `htpasswd_file`, and `insecure_no_auth` is `false`, it refuses to start rather than exposing an unauthenticated admin API. + +Clients authenticate by sending `Authorization: Bearer ` or the `X-Admin-Token: ` header. diff --git a/docs/security/security-best-practices.md b/docs/security/security-best-practices.md index af3f94263..132501c10 100644 --- a/docs/security/security-best-practices.md +++ b/docs/security/security-best-practices.md @@ -93,6 +93,16 @@ The KMS TLS listener may keep `rpc.tls.mutual.mandatory = false` because bootstr App key release and KMS key handover still require verified caller attestation from the RA-TLS client certificate. Certificate signing verifies the CSR signature and embedded attestation before signing. +## Management/admin API authentication + +The VMM, gateway, and KMS management surfaces must have authentication enabled in production: + +- VMM: set `[auth] enabled = true` with `tokens` (or `htpasswd_file`) — this guards the entire VMM HTTP/pRPC/UI surface. Never bind to a non-localhost address without it. +- Gateway: set `[core.admin] admin_token` (or `htpasswd_file`) and keep `insecure_no_auth = false`. +- KMS: set `admin_token_hash` (SHA-256 hex of the admin token); an empty hash denies all admin RPCs. + +Only bcrypt htpasswd entries are accepted (create with `htpasswd -B`), and credential comparisons are constant-time. Clients authenticate with `Authorization: Bearer ` (or `X-Admin-Token`). All three are fail-closed. + ## Keep private material owner-only Secret-bearing files should be owner-only (`0600`) wherever possible, including app keys, decrypted env files, KMS root keys, gateway WireGuard/TLS keys, and ACME credentials. Preserve restrictive permissions when copying volumes, backing up `/etc/kms/certs`, or moving gateway and certbot state between hosts. Public issue [#606](https://github.com/Dstack-TEE/dstack/issues/606) tracks the remaining low-cost hardening work in dstack-managed file writes. diff --git a/docs/tutorials/attestation-verification.md b/docs/tutorials/attestation-verification.md index 63f4a6f22..b9661de2f 100644 --- a/docs/tutorials/attestation-verification.md +++ b/docs/tutorials/attestation-verification.md @@ -129,6 +129,7 @@ Verify you have a running CVM: ```bash cd ~/dstack/dstack/vmm +export DSTACK_VMM_AUTH_USER=admin export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) ./src/vmm-cli.py --url http://127.0.0.1:9080 lsvm ``` @@ -141,6 +142,7 @@ The VMM provides a `/guest/Info` endpoint that proxies into the CVM and retrieve ```bash cd ~/dstack/dstack/vmm +export DSTACK_VMM_AUTH_USER=admin export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) # Get the VM UUID for hello-world @@ -384,6 +386,7 @@ Compare the CVM's actual measurements against your expected values: # verify-measurements.sh cd ~/dstack/dstack/vmm +export DSTACK_VMM_AUTH_USER=admin export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) # Get VM UUID @@ -576,6 +579,7 @@ echo "Image: $IMAGE_VERSION" echo "" cd ~/dstack/dstack/vmm +export DSTACK_VMM_AUTH_USER=admin export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) # --- Step 1: Get VM UUID --- diff --git a/docs/tutorials/gateway-build-configuration.md b/docs/tutorials/gateway-build-configuration.md index 3a807f8df..8964385c9 100644 --- a/docs/tutorials/gateway-build-configuration.md +++ b/docs/tutorials/gateway-build-configuration.md @@ -460,6 +460,7 @@ Now generate the VMM deployment manifest: ```bash cd ~/dstack/dstack/vmm +export DSTACK_VMM_AUTH_USER=admin export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) ./src/vmm-cli.py --url http://127.0.0.1:9080 compose \ diff --git a/docs/tutorials/gateway-service-setup.md b/docs/tutorials/gateway-service-setup.md index ba693486e..9570975c3 100644 --- a/docs/tutorials/gateway-service-setup.md +++ b/docs/tutorials/gateway-service-setup.md @@ -85,6 +85,7 @@ cd ~/gateway-deploy set -a; source .env; set +a cd ~/dstack/dstack/vmm +export DSTACK_VMM_AUTH_USER=admin export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) ./src/vmm-cli.py --url http://127.0.0.1:9080 deploy \ @@ -408,6 +409,7 @@ Navigate to the VMM directory first: ```bash cd ~/dstack/dstack/vmm +export DSTACK_VMM_AUTH_USER=admin export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) ``` diff --git a/docs/tutorials/hello-world-app.md b/docs/tutorials/hello-world-app.md index 2792e5ac7..51d1ce656 100644 --- a/docs/tutorials/hello-world-app.md +++ b/docs/tutorials/hello-world-app.md @@ -184,6 +184,7 @@ Use `vmm-cli.py compose` to generate the encrypted deployment manifest. The `--g ```bash cd ~/dstack/dstack/vmm +export DSTACK_VMM_AUTH_USER=admin export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) ./src/vmm-cli.py --url http://127.0.0.1:9080 compose \ @@ -265,6 +266,7 @@ ssh user@your-server ```bash cd ~/dstack/dstack/vmm +export DSTACK_VMM_AUTH_USER=admin export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) SRV_DOMAIN=$(grep ^SRV_DOMAIN ~/gateway-deploy/.env | cut -d= -f2) @@ -373,6 +375,7 @@ Navigate to the VMM directory: ```bash cd ~/dstack/dstack/vmm +export DSTACK_VMM_AUTH_USER=admin export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) ``` diff --git a/docs/tutorials/kms-cvm-deployment.md b/docs/tutorials/kms-cvm-deployment.md index bf34ad6cd..a0dacaf5e 100644 --- a/docs/tutorials/kms-cvm-deployment.md +++ b/docs/tutorials/kms-cvm-deployment.md @@ -256,6 +256,7 @@ Use the VMM CLI tool to deploy the CVM: cd ~/dstack/dstack/vmm # Set VMM auth from saved token +export DSTACK_VMM_AUTH_USER=admin export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) # Generate app-compose.json with local key provider enabled @@ -306,7 +307,7 @@ curl -s -H "Authorization: Bearer $(cat ~/.dstack/secrets/vmm-auth-token)" \ "http://127.0.0.1:9080/logs?id=VM_ID&follow=true&ansi=false" ``` -> **Note:** The VMM logs endpoint requires Bearer token authentication. The `vmm-cli.py logs` command may not work with token auth — use curl directly as shown above. +> **Note:** The VMM logs endpoint requires authentication. `vmm-cli.py logs` sends credentials automatically when `DSTACK_VMM_TOKEN` (or `--token`) is set, so it works against an auth-enabled VMM; the curl form above is an equivalent alternative. Look for these log messages indicating KMS entered onboard mode: ``` diff --git a/docs/tutorials/troubleshooting-first-application.md b/docs/tutorials/troubleshooting-first-application.md index ac5c11647..5738ab06e 100644 --- a/docs/tutorials/troubleshooting-first-application.md +++ b/docs/tutorials/troubleshooting-first-application.md @@ -84,6 +84,7 @@ If `/guest/Info` returns empty or errors, check that the CVM is running: ```bash cd ~/dstack/dstack/vmm +export DSTACK_VMM_AUTH_USER=admin export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) ./src/vmm-cli.py --url http://127.0.0.1:9080 lsvm ``` diff --git a/docs/tutorials/troubleshooting-gateway-deployment.md b/docs/tutorials/troubleshooting-gateway-deployment.md index 1ef5846d1..650b30ba8 100644 --- a/docs/tutorials/troubleshooting-gateway-deployment.md +++ b/docs/tutorials/troubleshooting-gateway-deployment.md @@ -59,6 +59,7 @@ sudo systemctl restart dstack-vmm **"Authentication required"** — Set the auth token: ```bash +export DSTACK_VMM_AUTH_USER=admin export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) ``` @@ -72,6 +73,7 @@ export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) ```bash cd ~/dstack/dstack/vmm +export DSTACK_VMM_AUTH_USER=admin export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) # Get KMS VM ID and remove it diff --git a/docs/tutorials/troubleshooting-kms-deployment.md b/docs/tutorials/troubleshooting-kms-deployment.md index 7a4219ea0..7282db94d 100644 --- a/docs/tutorials/troubleshooting-kms-deployment.md +++ b/docs/tutorials/troubleshooting-kms-deployment.md @@ -231,6 +231,7 @@ cat ~/kms-deployment/docker-compose.yml | grep ports -A2 # Check CVM status via vmm-cli.py cd ~/dstack/dstack/vmm +export DSTACK_VMM_AUTH_USER=admin export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) ./src/vmm-cli.py --url http://127.0.0.1:9080 lsvm ``` diff --git a/docs/vmm-cli-user-guide.md b/docs/vmm-cli-user-guide.md index bb93315d9..938d0deb5 100644 --- a/docs/vmm-cli-user-guide.md +++ b/docs/vmm-cli-user-guide.md @@ -70,26 +70,43 @@ export DSTACK_VMM_URL=unix:/path/to/socket ### Authentication -If your dstack-vmm server requires authentication, you can provide credentials using: +When the dstack-vmm server has `[auth] enabled = true`, the token now guards the +*entire* management surface (listing/creating/stopping VMs, deploys, logs, and +the web UI) — not just the logs endpoint. Provide credentials in one of two +forms. -#### Environment Variables (Recommended) +#### Bearer token (Recommended) -```bash -# Set authentication credentials -export DSTACK_VMM_AUTH_USER=your-username -export DSTACK_VMM_AUTH_PASSWORD=your-password +Pass the VMM API token directly; it is sent as `Authorization: Bearer `. -# Then use CLI normally +```bash +# the token dstackup writes, or your configured `[auth] tokens` entry +export DSTACK_VMM_TOKEN=$(cat ~/.dstack/secrets/vmm-auth-token) ./vmm-cli.py lsvm + +# or as a flag +./vmm-cli.py --token "$DSTACK_VMM_TOKEN" lsvm ``` -#### Command Line Arguments +#### HTTP Basic + +The server also accepts HTTP Basic, where the password may be the shared token +(any username, e.g. `admin`) or an entry in the server's `htpasswd_file`. ```bash -./vmm-cli.py --auth-user your-username --auth-password your-password lsvm +export DSTACK_VMM_AUTH_USER=admin +export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) +./vmm-cli.py lsvm + +# or as flags +./vmm-cli.py --auth-user admin --auth-password "$DSTACK_VMM_AUTH_PASSWORD" lsvm ``` -**Note:** Environment variables take precedence over command line arguments for authentication. +**Note:** A bearer token takes precedence over Basic when both are set. +Environment variables take precedence over command line arguments. Setting only +one half of a Basic credential (e.g. `DSTACK_VMM_AUTH_PASSWORD` without +`DSTACK_VMM_AUTH_USER`) is rejected with an error rather than silently sending an +unauthenticated request. ## Basic Commands @@ -325,8 +342,7 @@ After successful deployment, verify your VM is running correctly: export DSTACK_VMM_URL=http://127.0.0.1:12000 # If authentication is required -export DSTACK_VMM_AUTH_USER=your-username -export DSTACK_VMM_AUTH_PASSWORD=your-password +export DSTACK_VMM_TOKEN=$(cat ~/.dstack/secrets/vmm-auth-token) # Create a basic docker-compose.yml cat > docker-compose.yml << 'EOF' From 84ae901bc13b3bd8d9b7efbfc4e7a9009e4ea7f1 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 20 Jul 2026 02:15:46 -0700 Subject: [PATCH 07/12] test: annotate insecure test-only credentials --- dstack/gateway/test-run/e2e/configs/gateway-1.toml | 2 ++ dstack/gateway/test-run/e2e/configs/gateway-2.toml | 2 ++ dstack/gateway/test-run/e2e/configs/gateway-3.toml | 2 ++ dstack/gateway/test-run/test_suite.sh | 1 + 4 files changed, 7 insertions(+) diff --git a/dstack/gateway/test-run/e2e/configs/gateway-1.toml b/dstack/gateway/test-run/e2e/configs/gateway-1.toml index 350b1d177..fc5f49243 100644 --- a/dstack/gateway/test-run/e2e/configs/gateway-1.toml +++ b/dstack/gateway/test-run/e2e/configs/gateway-1.toml @@ -19,9 +19,11 @@ rpc_domain = "gateway-1" enabled = true port = 9016 address = "0.0.0.0" +# TEST ONLY - do not use in production; weak/hardcoded credential auth_token = "e2e-admin-token" [core.debug] +# TEST ONLY - do not use in production; enables debug RPC insecure_enable_debug_rpc = true insecure_skip_attestation = false port = 9015 diff --git a/dstack/gateway/test-run/e2e/configs/gateway-2.toml b/dstack/gateway/test-run/e2e/configs/gateway-2.toml index 0838921bd..40a9491a1 100644 --- a/dstack/gateway/test-run/e2e/configs/gateway-2.toml +++ b/dstack/gateway/test-run/e2e/configs/gateway-2.toml @@ -19,9 +19,11 @@ rpc_domain = "gateway-2" enabled = true port = 9016 address = "0.0.0.0" +# TEST ONLY - do not use in production; weak/hardcoded credential auth_token = "e2e-admin-token" [core.debug] +# TEST ONLY - do not use in production; enables debug RPC insecure_enable_debug_rpc = true insecure_skip_attestation = false port = 9015 diff --git a/dstack/gateway/test-run/e2e/configs/gateway-3.toml b/dstack/gateway/test-run/e2e/configs/gateway-3.toml index 9be8d04e0..f71e43109 100644 --- a/dstack/gateway/test-run/e2e/configs/gateway-3.toml +++ b/dstack/gateway/test-run/e2e/configs/gateway-3.toml @@ -19,9 +19,11 @@ rpc_domain = "gateway-3" enabled = true port = 9016 address = "0.0.0.0" +# TEST ONLY - do not use in production; weak/hardcoded credential auth_token = "e2e-admin-token" [core.debug] +# TEST ONLY - do not use in production; enables debug RPC insecure_enable_debug_rpc = true insecure_skip_attestation = false port = 9015 diff --git a/dstack/gateway/test-run/test_suite.sh b/dstack/gateway/test-run/test_suite.sh index 36eb44ef8..0de50202b 100755 --- a/dstack/gateway/test-run/test_suite.sh +++ b/dstack/gateway/test-run/test_suite.sh @@ -100,6 +100,7 @@ address = "127.0.0.1:${debug_port}" [core.admin] enabled = true address = "127.0.0.1:${admin_port}" +# TEST ONLY - do not use in production; disables admin API authentication insecure_no_auth = true [core.sync] From 8757966db24bb2f9d75bbb98d201f3136fba2787 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 20 Jul 2026 02:30:22 -0700 Subject: [PATCH 08/12] test(http-client): verify auth headers are sent on the wire --- dstack/http-client/src/lib.rs | 58 +++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/dstack/http-client/src/lib.rs b/dstack/http-client/src/lib.rs index e616e1aae..0b83dda54 100644 --- a/dstack/http-client/src/lib.rs +++ b/dstack/http-client/src/lib.rs @@ -140,4 +140,62 @@ mod tests { ); Ok(()) } + + #[tokio::test] + async fn http_transport_sends_extra_headers() -> Result<(), Box> { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await?; + let mut buf = [0u8; 1024]; + let n = socket.read(&mut buf).await?; + let request = String::from_utf8_lossy(&buf[..n]).into_owned(); + socket + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok") + .await?; + Ok::<_, std::io::Error>(request) + }); + + let (status, _body) = http_request_with_headers( + "POST", + &format!("http://{addr}"), + "/prpc/Status", + b"{}", + &[("Authorization", "Bearer secret-token")], + ) + .await?; + assert_eq!(status, 200); + let request = server.await??; + assert!( + request.contains("authorization: Bearer secret-token") + || request.contains("Authorization: Bearer secret-token"), + "request is missing the Authorization header: {request:?}" + ); + Ok(()) + } + + #[tokio::test] + async fn http_transport_omits_headers_when_none() -> Result<(), Box> { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await?; + let mut buf = [0u8; 1024]; + let n = socket.read(&mut buf).await?; + let request = String::from_utf8_lossy(&buf[..n]).into_owned(); + socket + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok") + .await?; + Ok::<_, std::io::Error>(request) + }); + + let (status, _body) = http_request("GET", &format!("http://{addr}"), "/x", b"").await?; + assert_eq!(status, 200); + let request = server.await??; + assert!( + !request.to_lowercase().contains("authorization:"), + "unexpected Authorization header: {request:?}" + ); + Ok(()) + } } From e753d9c732277f48f30e161b695097c6c6b9a38a Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 20 Jul 2026 02:56:51 -0700 Subject: [PATCH 09/12] docs: correct KMS admin token transport (RPC field, not HTTP header) --- docs/deployment.md | 2 +- docs/security/security-best-practices.md | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/deployment.md b/docs/deployment.md index 0de43ea92..117ddef68 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -439,7 +439,7 @@ curl http://:9203/finish > > If you skip this, `Onboard.Onboard` or later trusted RPCs will fail with KMS authorization errors. -> **Admin authentication.** Onboarding itself is gated by attestation and the authorization backend above — not by a token. Separately, the KMS *admin* RPCs (for example clearing the image cache) are guarded by an admin token: set `admin_token_hash` (SHA-256 hex of your token) in the KMS `kms.toml` and send the token on admin requests via `Authorization: Bearer `. An empty hash denies all admin RPCs (fail-closed). See [dstack-kms admin authentication](../dstack/kms/README.md#admin-api-authentication). +> **Admin authentication.** Onboarding itself is gated by attestation and the authorization backend above — not by a token. Separately, the KMS *admin* RPCs (for example `ClearImageCache`) are guarded by an admin token: set `admin_token_hash` (SHA-256 hex of your token) in the KMS `kms.toml`. Unlike VMM/gateway, the KMS admin token is not an HTTP header — it is passed as the `token` field of the admin RPC request (the raw token; the server hashes and compares it). An empty hash denies all admin RPCs (fail-closed). See [dstack-kms admin authentication](../dstack/kms/README.md#admin-api-authentication). --- diff --git a/docs/security/security-best-practices.md b/docs/security/security-best-practices.md index 132501c10..4b33a781d 100644 --- a/docs/security/security-best-practices.md +++ b/docs/security/security-best-practices.md @@ -97,11 +97,11 @@ App key release and KMS key handover still require verified caller attestation f The VMM, gateway, and KMS management surfaces must have authentication enabled in production: -- VMM: set `[auth] enabled = true` with `tokens` (or `htpasswd_file`) — this guards the entire VMM HTTP/pRPC/UI surface. Never bind to a non-localhost address without it. -- Gateway: set `[core.admin] admin_token` (or `htpasswd_file`) and keep `insecure_no_auth = false`. -- KMS: set `admin_token_hash` (SHA-256 hex of the admin token); an empty hash denies all admin RPCs. +- VMM: set `[auth] enabled = true` with `tokens` (or `htpasswd_file`) — this guards the entire VMM HTTP/pRPC/UI surface. Never bind to a non-localhost address without it. Clients send `Authorization: Bearer ` or `X-Admin-Token`. +- Gateway: set `[core.admin] admin_token` (or `htpasswd_file`) and keep `insecure_no_auth = false`. Clients send `Authorization: Bearer ` or `X-Admin-Token`. +- KMS: set `admin_token_hash` (SHA-256 hex of the admin token); an empty hash denies all admin RPCs. The KMS admin token is not an HTTP header — it is passed as the `token` field of the admin RPC request. -Only bcrypt htpasswd entries are accepted (create with `htpasswd -B`), and credential comparisons are constant-time. Clients authenticate with `Authorization: Bearer ` (or `X-Admin-Token`). All three are fail-closed. +VMM and gateway share the HTTP authenticator (bcrypt-only htpasswd via `htpasswd -B`, constant-time comparison). The KMS admin token is compared as a constant-time SHA-256 match. All three are fail-closed. ## Keep private material owner-only From 05025bc4df0ad7c5317220e554b906f6a4656b56 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 20 Jul 2026 05:39:25 -0700 Subject: [PATCH 10/12] docs: align KMS admin auth with the dedicated admin listener (#802) --- docs/deployment.md | 4 ++-- docs/security/security-best-practices.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/deployment.md b/docs/deployment.md index 117ddef68..c50ac42e6 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -122,7 +122,7 @@ Production KMS requires: #### Auth Server Options -> **Note:** the boot-authorization webhook below (auth-simple / auth-eth) is a *different* mechanism from the KMS admin-API authentication. The webhook allowlists which CVMs may boot and receive keys; the admin API (`admin_token_hash` in `kms.toml`) guards operator RPCs. See [dstack-kms admin authentication](../dstack/kms/README.md#admin-api-authentication). +> **Note:** the boot-authorization webhook below (auth-simple / auth-eth) is a *different* mechanism from the KMS admin-API authentication. The webhook allowlists which CVMs may boot and receive keys; the admin API (`[core.admin]` in `kms.toml`) guards operator RPCs. See [dstack-kms admin authentication](../dstack/kms/README.md#admin-api-authentication). | Server | Use Case | Configuration | |--------|----------|---------------| @@ -439,7 +439,7 @@ curl http://:9203/finish > > If you skip this, `Onboard.Onboard` or later trusted RPCs will fail with KMS authorization errors. -> **Admin authentication.** Onboarding itself is gated by attestation and the authorization backend above — not by a token. Separately, the KMS *admin* RPCs (for example `ClearImageCache`) are guarded by an admin token: set `admin_token_hash` (SHA-256 hex of your token) in the KMS `kms.toml`. Unlike VMM/gateway, the KMS admin token is not an HTTP header — it is passed as the `token` field of the admin RPC request (the raw token; the server hashes and compares it). An empty hash denies all admin RPCs (fail-closed). See [dstack-kms admin authentication](../dstack/kms/README.md#admin-api-authentication). +> **Admin authentication.** Onboarding itself is gated by attestation and the authorization backend above — not by a token. Separately, the KMS *admin* RPCs (for example `ClearImageCache`) are served on a dedicated `[core.admin]` listener behind the shared HTTP authenticator, just like the VMM and gateway: set `[core.admin] enabled = true` with an `auth_token` (or the `DSTACK_KMS_ADMIN_TOKEN` / `ADMIN_API_TOKEN` env vars), and clients send `Authorization: Bearer ` or `X-Admin-Token`. Enabled with neither `auth_token` nor `htpasswd_file` (and `insecure_no_auth = false`) fails closed — the KMS refuses to start. See [dstack-kms admin authentication](../dstack/kms/README.md#admin-api-authentication). --- diff --git a/docs/security/security-best-practices.md b/docs/security/security-best-practices.md index 4b33a781d..73760ea82 100644 --- a/docs/security/security-best-practices.md +++ b/docs/security/security-best-practices.md @@ -99,9 +99,9 @@ The VMM, gateway, and KMS management surfaces must have authentication enabled i - VMM: set `[auth] enabled = true` with `tokens` (or `htpasswd_file`) — this guards the entire VMM HTTP/pRPC/UI surface. Never bind to a non-localhost address without it. Clients send `Authorization: Bearer ` or `X-Admin-Token`. - Gateway: set `[core.admin] admin_token` (or `htpasswd_file`) and keep `insecure_no_auth = false`. Clients send `Authorization: Bearer ` or `X-Admin-Token`. -- KMS: set `admin_token_hash` (SHA-256 hex of the admin token); an empty hash denies all admin RPCs. The KMS admin token is not an HTTP header — it is passed as the `token` field of the admin RPC request. +- KMS: enable `[core.admin]` with an `auth_token` (or `htpasswd_file`); the admin RPCs are served on a dedicated listener and clients send `Authorization: Bearer ` or `X-Admin-Token`. Enabled with no credential denies all admin RPCs (fail-closed). -VMM and gateway share the HTTP authenticator (bcrypt-only htpasswd via `htpasswd -B`, constant-time comparison). The KMS admin token is compared as a constant-time SHA-256 match. All three are fail-closed. +All three share the same HTTP authenticator: bcrypt-only htpasswd (via `htpasswd -B`), constant-time token comparison, and fail-closed behavior. ## Keep private material owner-only From c64861cede35f33a009fd28cd1772b5bd363d67c Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 20 Jul 2026 06:12:29 -0700 Subject: [PATCH 11/12] fix(dstackup): create the VMM token file 0600 up front (no chmod window) --- dstack/crates/dstack-cli-core/src/fsutil.rs | 47 ++++++++++++++++++++- dstack/crates/dstackup/src/install.rs | 11 ++--- 2 files changed, 49 insertions(+), 9 deletions(-) diff --git a/dstack/crates/dstack-cli-core/src/fsutil.rs b/dstack/crates/dstack-cli-core/src/fsutil.rs index e03567e0a..125a6eab3 100644 --- a/dstack/crates/dstack-cli-core/src/fsutil.rs +++ b/dstack/crates/dstack-cli-core/src/fsutil.rs @@ -31,9 +31,38 @@ fn sibling(path: &Path, suffix: &str) -> PathBuf { /// durable across a power loss. `tmp` and `path` are in the same directory so /// the rename is atomic. pub fn write_atomic(path: &Path, contents: &str) -> Result<()> { + write_atomic_inner(path, contents, None) +} + +/// like [`write_atomic`], but the temp file is created with `mode` (Unix +/// permission bits) *before* any content is written, so a secret never exists +/// on disk with broader-than-intended permissions — not even transiently +/// between the rename and a follow-up `chmod`. Use for credential files +/// (`0o600`). The final file keeps `mode` because `rename` preserves it. +pub fn write_atomic_mode(path: &Path, contents: &str, mode: u32) -> Result<()> { + write_atomic_inner(path, contents, Some(mode)) +} + +fn write_atomic_inner(path: &Path, contents: &str, mode: Option) -> Result<()> { let tmp = sibling(path, ".tmp"); - let mut f = - File::create(&tmp).with_context(|| format!("creating temp file {}", tmp.display()))?; + let mut opts = OpenOptions::new(); + opts.write(true).create(true).truncate(true); + #[cfg(unix)] + if let Some(mode) = mode { + use std::os::unix::fs::OpenOptionsExt; + opts.mode(mode); + } + let mut f = opts + .open(&tmp) + .with_context(|| format!("creating temp file {}", tmp.display()))?; + // if a stale temp file survived a crash, `create` reused it without + // resetting its mode; tighten it before writing the secret. + #[cfg(unix)] + if let Some(mode) = mode { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&tmp, std::fs::Permissions::from_mode(mode)) + .with_context(|| format!("setting mode on {}", tmp.display()))?; + } f.write_all(contents.as_bytes()) .with_context(|| format!("writing {}", tmp.display()))?; f.sync_all() @@ -86,6 +115,20 @@ mod tests { let _ = std::fs::remove_dir_all(&dir); } + #[cfg(unix)] + #[test] + fn atomic_write_mode_creates_owner_only_file() { + use std::os::unix::fs::PermissionsExt; + let dir = std::env::temp_dir().join(format!("dstack-fsmode-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let p = dir.join("token"); + write_atomic_mode(&p, "secret", 0o600).unwrap(); + assert_eq!(std::fs::read_to_string(&p).unwrap(), "secret"); + let mode = std::fs::metadata(&p).unwrap().permissions().mode() & 0o777; + assert_eq!(mode, 0o600, "credential file must be 0600, got {mode:o}"); + let _ = std::fs::remove_dir_all(&dir); + } + #[test] fn lock_is_reentrant_within_process_after_drop() { let dir = std::env::temp_dir().join(format!("dstack-fslock-{}", std::process::id())); diff --git a/dstack/crates/dstackup/src/install.rs b/dstack/crates/dstackup/src/install.rs index dcad4bb19..05155b957 100644 --- a/dstack/crates/dstackup/src/install.rs +++ b/dstack/crates/dstackup/src/install.rs @@ -1230,14 +1230,11 @@ pub(crate) fn read_token_file(path: &Path) -> Option { } /// write the management-API token atomically with owner-only (0600) -/// permissions — it is a bearer credential. +/// permissions — it is a bearer credential, so it is created 0600 up front +/// (never exposed with wider bits, even transiently). fn write_token_file(path: &Path, token: &str) -> Result<()> { - use std::os::unix::fs::PermissionsExt; - dstack_cli_core::fsutil::write_atomic(path, token) - .with_context(|| format!("writing {}", path.display()))?; - fs::set_permissions(path, fs::Permissions::from_mode(0o600)) - .with_context(|| format!("setting 0600 on {}", path.display()))?; - Ok(()) + dstack_cli_core::fsutil::write_atomic_mode(path, token, 0o600) + .with_context(|| format!("writing {}", path.display())) } #[cfg(test)] From 06b7599a4bd17ff7b682abd6b3018bf5f2294120 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 20 Jul 2026 06:13:29 -0700 Subject: [PATCH 12/12] docs(vmm-cli): fix token file path and credential precedence order --- docs/vmm-cli-user-guide.md | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/docs/vmm-cli-user-guide.md b/docs/vmm-cli-user-guide.md index 938d0deb5..7e4fc4aa6 100644 --- a/docs/vmm-cli-user-guide.md +++ b/docs/vmm-cli-user-guide.md @@ -80,8 +80,11 @@ forms. Pass the VMM API token directly; it is sent as `Authorization: Bearer `. ```bash -# the token dstackup writes, or your configured `[auth] tokens` entry -export DSTACK_VMM_TOKEN=$(cat ~/.dstack/secrets/vmm-auth-token) +# your VMM API token. `dstackup install` writes it to +# /vmm-auth-token (default /etc/dstack/vmm-auth-token) — and the +# local `dstack` CLI reads it automatically; a manual setup (see the VMM +# configuration tutorial) stores it at ~/.dstack/secrets/vmm-auth-token. +export DSTACK_VMM_TOKEN=$(cat /etc/dstack/vmm-auth-token) ./vmm-cli.py lsvm # or as a flag @@ -95,18 +98,18 @@ The server also accepts HTTP Basic, where the password may be the shared token ```bash export DSTACK_VMM_AUTH_USER=admin -export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) +export DSTACK_VMM_AUTH_PASSWORD=$(cat /etc/dstack/vmm-auth-token) ./vmm-cli.py lsvm # or as flags ./vmm-cli.py --auth-user admin --auth-password "$DSTACK_VMM_AUTH_PASSWORD" lsvm ``` -**Note:** A bearer token takes precedence over Basic when both are set. -Environment variables take precedence over command line arguments. Setting only -one half of a Basic credential (e.g. `DSTACK_VMM_AUTH_PASSWORD` without -`DSTACK_VMM_AUTH_USER`) is rejected with an error rather than silently sending an -unauthenticated request. +**Note:** A bearer token takes precedence over Basic when both are set. For each +setting, command-line flags take precedence over environment variables, which +take precedence over the config file. Setting only one half of a Basic +credential (e.g. `DSTACK_VMM_AUTH_PASSWORD` without `DSTACK_VMM_AUTH_USER`) is +rejected with an error rather than silently sending an unauthenticated request. ## Basic Commands