fix(setup): non-root + tailscale provisioning on fresh Debian 13

CHANGELOG.md

@@ -4,6 +4,37 @@ All notable changes to teploy are documented here. Format follows [Keep a Change
 
 ## [Unreleased]
 
+### Fixed
+
+Found provisioning a fresh OVH Debian 13 VPS as a non-root user with
+`--network tailscale` (2026-10-25):
+
+- SSH now offers only the host-key algorithms `known_hosts` holds for a
+  known host. A box first trusted by stock `ssh` (ed25519) failed every
+  connect with a "host key mismatch" when Go negotiated ecdsa.
+- Remote uploads apply the file mode after writing, not before. Read-only
+  modes (the 0440 sudoers audit drop-in) could not be written by a
+  non-root user, aborting `setup` hardening.
+- Hardening and setup probes for `/usr/sbin` tools (`ufw`, `auditctl`,
+  `visudo`, `ufw status`) run through sudo for non-root users. Before,
+  they missed the tools, so ufw and auditd were reinstalled on every run,
+  the firewall stage reported "No active firewall", and `visudo` was not found.
+- Tailscale and Headscale joins pass the key via `--auth-key=file:`. The
+  tailscale CLI ignores `TS_AUTHKEY`, so every join fell back to an
+  interactive login and timed out. The credential still never enters argv.
+- The tailscale `BackendState` check parses the JSON; the pretty-printed
+  output never matched the compact substring.
+- Re-running `setup --network tailscale|headscale` leaves an already-joined
+  node alone. It used to wipe `/var/lib/tailscale` every run and re-up with
+  the spent one-time key, logging the server out of the mesh.
+- A failed mesh join reports the provider's output (e.g. `invalid key`)
+  instead of a bare wait timeout.
+- `setup` installs cron (backup and autodeploy schedules need it; Debian 13
+  cloud images ship without it) and `doctor` checks for it.
+- The first-deploy DNS check accepts the server's own interface addresses,
+  so a server registered by its Tailscale IP no longer refuses a public
+  domain that points at its public IP.
+
 ## [0.1.47] - 2026-10-21
 
 ### Fixed

internal/cli/deploy.go

@@ -24,7 +24,6 @@ import (
 	"github.com/useteploy/teploy/internal/config"
 	"github.com/useteploy/teploy/internal/confighome"
 	"github.com/useteploy/teploy/internal/deploy"
-	"github.com/useteploy/teploy/internal/dns"
 	"github.com/useteploy/teploy/internal/docker"
 	"github.com/useteploy/teploy/internal/env"
 	"github.com/useteploy/teploy/internal/multideploy"
@@ -630,7 +629,7 @@ func deployAppConfig(flags *Flags, appCfg *config.AppConfig, serverName, image,
 		current, _ := state.Read(ctx, executor, appCfg.App)
 		if current == nil {
 			fmt.Println("Validating DNS...")
-			if err := dns.Validate(appCfg.Domain, host, nil); err != nil {
+			if err := validateDomainDNS(ctx, executor, appCfg.Domain, host); err != nil {
 				return err
 			}
 			fmt.Println("  DNS validated")

internal/cli/deployserver.go

@@ -14,7 +14,6 @@ import (
 	"github.com/useteploy/teploy/internal/build"
 	"github.com/useteploy/teploy/internal/config"
 	"github.com/useteploy/teploy/internal/confighome"
-	"github.com/useteploy/teploy/internal/dns"
 	"github.com/useteploy/teploy/internal/docker"
 	"github.com/useteploy/teploy/internal/gitsource"
 	"github.com/useteploy/teploy/internal/releasemeta"
@@ -180,7 +179,7 @@ func deployServerConfigLocked(ctx context.Context, executor ssh.Executor, host,
 		appCfg.Ingress != config.IngressHost && appCfg.Ingress != config.IngressExternal {
 		if current, _ := state.Read(ctx, executor, app); current == nil {
 			fmt.Println("Validating DNS...")
-			if err := dns.Validate(appCfg.Domain, host, nil); err != nil {
+			if err := validateDomainDNS(ctx, executor, appCfg.Domain, host); err != nil {
 				return err
 			}
 			fmt.Println("  DNS validated")

internal/cli/dnscheck.go

@@ -0,0 +1,28 @@
+package cli
+
+import (
+	"context"
+	"strings"
+
+	"github.com/useteploy/teploy/internal/dns"
+	"github.com/useteploy/teploy/internal/ssh"
+)
+
+// validateDomainDNS checks the domain points at the server. When the SSH
+// host does not match (a server reached over its mesh IP), the server's
+// own interface addresses are accepted too: a box registered by its
+// Tailscale IP otherwise refused every first deploy of a public domain
+// (2026-10-25 live finding, fresh OVH box set up with --network
+// tailscale). A NAT'd server whose public IP is not on an interface
+// still needs --skip-dns-check.
+func validateDomainDNS(ctx context.Context, executor ssh.Executor, domain, host string) error {
+	err := dns.Validate(domain, host, nil)
+	if err == nil || executor == nil {
+		return err
+	}
+	out, runErr := executor.Run(ctx, "hostname -I")
+	if runErr == nil && dns.Resolves(domain, strings.Fields(out), nil) {
+		return nil
+	}
+	return err
+}

internal/cli/doctor.go

@@ -395,7 +395,8 @@ func doctorDockerCheck(ctx context.Context, exec ssh.Executor) doctorCheck {
 // doctorServerToolsCheck verifies the server-side tools teploy shells out
 // to. python3 is required (fenced state writes, volume ownership, trigger
 // receipts), so its absence fails; git is a warning until a feature on the
-// box needs it. `teploy setup` installs both.
+// box needs it, as is cron (backup and autodeploy schedules). `teploy
+// setup` installs all three.
 func doctorServerToolsCheck(ctx context.Context, exec ssh.Executor) doctorCheck {
 	out, err := exec.Run(ctx, serverToolsProbe)
 	if err != nil {

internal/cli/setup.go

@@ -312,9 +312,22 @@ func setupNetwork(ctx context.Context, exec ssh.Executor, w io.Writer, providerN
 		return "", fmt.Errorf("installing %s: %w", providerName, err)
 	}
 
-	// Reset Tailscale state if present — cloned VMs inherit the previous machine's identity
-	// which causes IP conflicts. Stop tailscaled, wipe state, restart.
 	if providerName == "tailscale" || providerName == "headscale" {
+		// An already-joined node is left alone: the reset below used to run
+		// on every setup, so re-running `teploy setup --network` wiped a
+		// working node's identity and re-upped it with a (typically spent)
+		// one-time key — logging the server out of the mesh (2026-10-25
+		// live finding). A clone that inherited another machine's identity
+		// needs `tailscale logout` on it first.
+		if out, err := exec.Run(ctx, sudo+"tailscale status --json 2>/dev/null"); err == nil && network.TailscaleRunning(out) {
+			if vpnIP, err := provider.GetIP(ctx, exec); err == nil && net.ParseIP(vpnIP) != nil {
+				fmt.Fprintf(w, "  Already on the %s mesh (to re-enroll a cloned VM, run `tailscale logout` on it first)\n", providerName)
+				fmt.Fprintf(w, "  VPN IP: %s\n", vpnIP)
+				return vpnIP, nil
+			}
+		}
+		// Not joined: reset any inherited state — cloned VMs carry the
+		// previous machine's identity, which causes IP conflicts.
 		exec.Run(ctx, sudo+"systemctl stop tailscaled 2>/dev/null; "+sudo+"rm -rf /var/lib/tailscale; "+sudo+"systemctl start tailscaled 2>/dev/null; true")
 	}
 
@@ -336,7 +349,8 @@ func setupNetwork(ctx context.Context, exec ssh.Executor, w io.Writer, providerN
 	// Tailscale/Headscale modifies iptables which can kill the SSH connection,
 	// so detach the join and use the reconnecting authenticated executor to observe it.
 	fmt.Fprintf(w, "Joining %s mesh...\n", providerName)
-	if err := joinVPNMesh(ctx, exec, sudo, providerName, cfg); err != nil {
+	joinLog, err := joinVPNMesh(ctx, exec, sudo, providerName, cfg)
+	if err != nil {
 		return "", err
 	}
 
@@ -357,18 +371,22 @@ func setupNetwork(ctx context.Context, exec ssh.Executor, w io.Writer, providerN
 		}
 		select {
 		case <-waitCtx.Done():
+			if out, _ := exec.Run(ctx, "tail -n 3 "+joinLog+" 2>/dev/null"); strings.TrimSpace(out) != "" {
+				return "", fmt.Errorf("waiting for %s address: %w; join output: %s", providerName, waitCtx.Err(), strings.TrimSpace(out))
+			}
 			return "", fmt.Errorf("waiting for %s address: %w", providerName, waitCtx.Err())
 		case <-time.After(2 * time.Second):
 		}
 	}
 
+	_, _ = exec.Run(ctx, "rm -f -- "+joinLog)
 	fmt.Fprintf(w, "  VPN IP: %s\n", vpnIP)
 	return vpnIP, nil
 }
 
 // vpnCredential resolves the provider's join credential and the env var
-// its CLI documents for it (tailscale up reads TS_AUTHKEY, netbird up
-// reads NB_SETUP_KEY).
+// its CLI documents for it (netbird up reads NB_SETUP_KEY; the tailscale
+// name is unused — its CLI ignores TS_AUTHKEY, see TailscaleJoinShell).
 func vpnCredential(providerName string, cfg network.Config) (envVar, value string, err error) {
 	switch providerName {
 	case "tailscale", "headscale":
@@ -390,37 +408,46 @@ func vpnCredential(providerName string, cfg network.Config) (envVar, value strin
 // setup with the file path named (nothing was joined); a join failure
 // after that surfaces through the tailnet wait below, and the key file
 // is gone regardless — its only reader is the rm-ing shell itself.
-func joinVPNMesh(ctx context.Context, exec ssh.Executor, sudo, providerName string, cfg network.Config) error {
+func joinVPNMesh(ctx context.Context, exec ssh.Executor, sudo, providerName string, cfg network.Config) (string, error) {
 	_, credential, err := vpnCredential(providerName, cfg)
 	if err != nil {
-		return err
+		return "", err
 	}
 	if credential == "" {
-		return fmt.Errorf("%s join credential is empty", providerName)
+		return "", fmt.Errorf("%s join credential is empty", providerName)
 	}
 	keyPath, err := network.StageJoinCredential(ctx, exec, credential)
 	if err != nil {
-		return err
+		return "", err
 	}
 	_, _ = exec.Run(ctx, vpnJoinCommand(providerName, cfg, sudo, keyPath))
-	return nil
+	return vpnJoinLog(keyPath), nil
 }
 
-// vpnJoinCommand renders the detached join: `cat` the key file into the
-// provider's env var, remove the file, then exec the provider. The
+// vpnJoinLog is where the detached join's output lands, so a rejected
+// key surfaces as the provider's own error rather than a bare wait
+// timeout. Per run (derived from the random key path): a fixed /tmp name
+// owned by one user cannot be reopened by another under Debian's
+// fs.protected_regular, which would fail the redirect and the join with
+// it. It never holds the credential — the key is passed by file path.
+func vpnJoinLog(keyPath string) string { return keyPath + ".log" }
+
+// vpnJoinCommand renders the detached join: tailscale/headscale read the
+// key file via --auth-key=file: and remove it on exit; netbird `cat`s it
+// into NB_SETUP_KEY, removes it, then execs. The
 // login-server URL (headscale) is not a secret and stays a plain flag.
 func vpnJoinCommand(providerName string, cfg network.Config, sudo, keyPath string) string {
-	envVar, _, _ := vpnCredential(providerName, cfg)
-	var tail string
+	var join string
 	switch providerName {
 	case "tailscale":
-		tail = "tailscale up --accept-routes"
+		join = network.TailscaleJoinShell(keyPath, "--accept-routes")
 	case "headscale":
-		tail = "tailscale up --login-server=" + ssh.ShellQuote(cfg.Server) + " --accept-routes"
+		join = network.TailscaleJoinShell(keyPath, "--login-server="+ssh.ShellQuote(cfg.Server)+" --accept-routes")
 	case "netbird":
-		tail = "netbird up"
+		envVar, _, _ := vpnCredential(providerName, cfg)
+		join = network.EnvVarJoinShell(envVar, keyPath, "netbird up")
 	}
-	return sudo + "nohup " + network.EnvVarJoinShell(envVar, keyPath, tail) + " >/dev/null 2>&1 &"
+	return sudo + "nohup " + join + " >" + vpnJoinLog(keyPath) + " 2>&1 &"
 }
 
 // runLocal executes a command on the local machine and returns its output.
@@ -522,9 +549,9 @@ type setupEnv struct {
 
 // serverTools are the packages teploy needs on every server beyond docker
 // and rsync; serverToolsProbe prints the missing ones, space separated.
-var serverTools = []string{"git", "python3"}
+var serverTools = []string{"git", "python3", "cron"}
 
-const serverToolsProbe = "for t in git python3; do command -v $t >/dev/null 2>&1 || printf '%s ' $t; done; true"
+const serverToolsProbe = "for t in git python3; do command -v $t >/dev/null 2>&1 || printf '%s ' $t; done; command -v crontab >/dev/null 2>&1 || printf 'cron '; true"
 
 // setupStages is the ordered provisioning plan. The ORDER is part of
 // the contract: docker before the network that needs it, the network
@@ -604,17 +631,17 @@ func setupStages() []setupStage {
 			},
 		},
 		{
-			name:    "git and python3",
-			affects: "git and python3 packages (apt, if missing) — state writes need python3, server-side config and checkouts need git",
+			name:    "git, python3 and cron",
+			affects: "git, python3 and cron packages (apt, if missing) — state writes need python3, server-side config and checkouts need git, backup and autodeploy schedules need cron",
 			run: func(ctx context.Context, exec ssh.Executor, w io.Writer, env *setupEnv) error {
-				fmt.Fprintln(w, "Checking git and python3...")
+				fmt.Fprintln(w, "Checking git, python3 and cron...")
 				out, err := exec.Run(ctx, serverToolsProbe)
 				if err != nil {
 					return fmt.Errorf("probing server tools: %w", err)
 				}
 				missing := strings.Fields(out)
 				if len(missing) == 0 {
-					fmt.Fprintln(w, "  git and python3 already installed")
+					fmt.Fprintln(w, "  git, python3 and cron already installed")
 					return nil
 				}
 				fmt.Fprintf(w, "  Installing %s...\n", strings.Join(missing, ", "))
@@ -665,7 +692,7 @@ func setupStages() []setupStage {
 			run: func(ctx context.Context, exec ssh.Executor, w io.Writer, env *setupEnv) error {
 				// 3. Check firewall
 				fmt.Fprintln(w, "Checking firewall...")
-				ufwOutput, ufwErr := exec.Run(ctx, "ufw status 2>/dev/null")
+				ufwOutput, ufwErr := exec.Run(ctx, env.sudo+"ufw status 2>/dev/null")
 				if ufwErr == nil && strings.Contains(ufwOutput, "Status: active") {
 					_, err1 := exec.Run(ctx, env.sudo+"ufw allow 80/tcp")
 					_, err2 := exec.Run(ctx, env.sudo+"ufw allow 443/tcp")

internal/cli/setup_test.go

@@ -529,7 +529,7 @@ func TestInstallSudoViaSu_WrongPassword(t *testing.T) {
 }
 
 // TestVPNJoinCommand_NoCredentialInArgv pins the join shape: the key
-// file is read, removed, and fed via the provider env var — the literal
+// file is passed by path and removed on exit — the literal
 // credential appears nowhere in the command.
 func TestVPNJoinCommand_NoCredentialInArgv(t *testing.T) {
 	cfg := network.Config{Provider: "tailscale", AuthKey: "tskey-auth-secret123"}
@@ -539,10 +539,8 @@ func TestVPNJoinCommand_NoCredentialInArgv(t *testing.T) {
 	}
 	for _, want := range []string{
 		"nohup sh -c ",
-		"k=$(cat /tmp/teploy-vpn-key-abc)",
+		"tailscale up --auth-key=file:/tmp/teploy-vpn-key-abc --accept-routes",
 		"rm -f -- /tmp/teploy-vpn-key-abc",
-		`export TS_AUTHKEY="$k"`,
-		"exec tailscale up --accept-routes",
 	} {
 		if !strings.Contains(cmd, want) {
 			t.Errorf("join command missing %q:\n%s", want, cmd)
@@ -576,9 +574,13 @@ func TestJoinVPNMesh_StagesPrivateFileAndCleansUp(t *testing.T) {
 		ssh.MockCommand{Match: "nohup sh -c", Output: ""},
 	)
 	cfg := network.Config{Provider: "tailscale", AuthKey: "tskey-auth-x"}
-	if err := joinVPNMesh(context.Background(), mock, "", "tailscale", cfg); err != nil {
+	joinLog, err := joinVPNMesh(context.Background(), mock, "", "tailscale", cfg)
+	if err != nil {
 		t.Fatalf("joinVPNMesh: %v", err)
 	}
+	if !strings.HasPrefix(joinLog, "/tmp/teploy-vpn-key-") || !strings.HasSuffix(joinLog, ".log") {
+		t.Errorf("join log must be per run, beside the staged key: %q", joinLog)
+	}
 	uploads := 0
 	for _, call := range mock.Calls {
 		if strings.HasPrefix(call, "UPLOAD:") {
@@ -846,3 +848,24 @@ func TestInstallAuthorizedKey_GuardedAgainstDuplicateAppend(t *testing.T) {
 		}
 	}
 }
+
+// TestSetupNetwork_AlreadyJoinedTailscaleIsLeftAlone: a re-run of setup
+// against a node already on the tailnet must not wipe its state or re-up
+// it (the spent one-time key then logs the node out).
+func TestSetupNetwork_AlreadyJoinedTailscaleIsLeftAlone(t *testing.T) {
+	m := ssh.NewMockExecutor("h",
+		ssh.MockCommand{Match: "whoami", Output: "debian"},
+		ssh.MockCommand{Match: "which tailscale", Output: "/usr/bin/tailscale"},
+		ssh.MockCommand{Match: "sudo tailscale status --json", Output: "{\n  \"BackendState\": \"Running\"\n}"},
+		ssh.MockCommand{Match: "sudo tailscale ip -4", Output: "100.118.234.58\n"},
+	)
+	got, err := setupNetwork(context.Background(), m, &bytes.Buffer{}, "tailscale", "tskey-auth-spent")
+	if err != nil || got != "100.118.234.58" {
+		t.Fatalf("got %q, %v", got, err)
+	}
+	for _, call := range m.Calls {
+		if strings.Contains(call, "rm -rf /var/lib/tailscale") || strings.Contains(call, "tailscale up") || strings.Contains(call, "teploy-vpn-key-") {
+			t.Fatalf("already-joined node was reset or re-joined: %s", call)
+		}
+	}
+}

internal/cli/validate.go

@@ -14,7 +14,6 @@ import (
 	"github.com/useteploy/teploy/internal/build"
 	"github.com/useteploy/teploy/internal/caddy"
 	"github.com/useteploy/teploy/internal/config"
-	"github.com/useteploy/teploy/internal/dns"
 	"github.com/useteploy/teploy/internal/ssh"
 )
 
@@ -105,7 +104,7 @@ func runValidate(flags *Flags) error {
 				// front (Cloudflare Tunnel / nginx / ALB) serves the domain, so
 				// it legitimately resolves there, not to the server IP.
 				if appCfg.Domain != "" && appCfg.Ingress != config.IngressExternal {
-					if err := dns.Validate(appCfg.Domain, host, nil); err != nil {
+					if err := validateDomainDNS(ctx, executor, appCfg.Domain, host); err != nil {
 						result.Warns = append(result.Warns, fmt.Sprintf("DNS: %v", err))
 					}
 				}

internal/dns/resolves_test.go

@@ -0,0 +1,13 @@
+package dns
+
+import "testing"
+
+func TestResolvesMatchesAnyServerAddress(t *testing.T) {
+	resolve := func(host string) ([]string, error) { return []string{"158.69.206.240"}, nil }
+	if !Resolves("example.com", []string{"100.67.126.38", "158.69.206.240", "172.17.0.1"}, resolve) {
+		t.Fatal("domain on the public interface must match")
+	}
+	if Resolves("example.com", []string{"100.67.126.38"}, resolve) {
+		t.Fatal("mesh-only addresses must not match a public A record")
+	}
+}

internal/dns/validate.go

@@ -47,3 +47,26 @@ func Validate(domain, serverHost string, resolve Resolver) error {
 		domain, domainIPs[0], serverIPs[0], serverIPs[0],
 	)
 }
+
+// Resolves reports whether the domain resolves to any of ips. It is the
+// fallback when the server is addressed by a private or mesh IP (a
+// servers.yml host set to its Tailscale address by `teploy setup
+// --network`): public DNS points at the server's public interface, which
+// Validate's comparison against the SSH host can never match.
+func Resolves(domain string, ips []string, resolve Resolver) bool {
+	if resolve == nil {
+		resolve = net.LookupHost
+	}
+	domainIPs, err := resolve(domain)
+	if err != nil {
+		return false
+	}
+	for _, dip := range domainIPs {
+		for _, ip := range ips {
+			if dip == ip {
+				return true
+			}
+		}
+	}
+	return false
+}