feat(volumes): optional fail-fast guard that the volume path is on the expected filesystem (distills #3196)

packages/orchestrator/pkg/cfg/model.go

@@ -109,6 +109,11 @@ type Config struct {
 	NBDPoolSize                 int               `env:"NBD_POOL_SIZE"                 envDefault:"64"`
 	Services                    []string          `env:"ORCHESTRATOR_SERVICES"         envDefault:"orchestrator"`
 	PersistentVolumeMounts      map[string]string `env:"PERSISTENT_VOLUME_MOUNTS"`
+	// VolumeExpectedFSType, when set, makes volume create fail fast unless the
+	// volume path is on a mount of this filesystem type (e.g. "fuse.juicefs",
+	// "ceph"), guarding against the configured distributed FS not being mounted.
+	// Empty (default) keeps the local-disk behavior with no mount check.
+	VolumeExpectedFSType string `env:"VOLUME_EXPECTED_FSTYPE"`
 
 	// TLS is the mutual TLS settings of the shared port and the peer hop.
 	TLS TLSConfig

packages/orchestrator/pkg/volumes/mount_guard.go

@@ -0,0 +1,87 @@
+//go:build linux
+
+package volumes
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"strings"
+)
+
+// procMountsPath is the kernel's mount table. Overridable in tests.
+const procMountsPath = "/proc/mounts"
+
+// mountFSTypeFrom returns the filesystem type of the mount that contains target,
+// read from a mount table in /proc/mounts format. It selects the mount whose
+// mount point is the longest prefix of target, so a not-yet-created leaf resolves
+// to its enclosing mount. Returns ("", nil) if no mount point is a prefix.
+func mountFSTypeFrom(mountsPath, target string) (string, error) {
+	f, err := os.Open(mountsPath)
+	if err != nil {
+		return "", fmt.Errorf("open %q: %w", mountsPath, err)
+	}
+	defer f.Close()
+
+	bestLen := -1
+	bestType := ""
+
+	sc := bufio.NewScanner(f)
+	for sc.Scan() {
+		fields := strings.Fields(sc.Text())
+		if len(fields) < 3 {
+			continue
+		}
+		mp, fsType := fields[1], fields[2]
+		if !pathHasPrefix(target, mp) {
+			continue
+		}
+		if len(mp) > bestLen {
+			bestLen = len(mp)
+			bestType = fsType
+		}
+	}
+	if err := sc.Err(); err != nil {
+		return "", fmt.Errorf("scan %q: %w", mountsPath, err)
+	}
+
+	return bestType, nil
+}
+
+// pathHasPrefix reports whether mp is a path-component prefix of target (so
+// "/mnt/jfs" matches "/mnt/jfs/x" and "/mnt/jfs" itself, but not "/mnt/jfs-2").
+func pathHasPrefix(target, mp string) bool {
+	if mp == "/" {
+		return true
+	}
+	if target == mp {
+		return true
+	}
+
+	return strings.HasPrefix(target, mp+"/")
+}
+
+// verifyVolumeMount fails fast when expectedFSType is set and the mount
+// containing path is not of that type - the usual cause being that the
+// configured distributed filesystem (JuiceFS, CephFS, ...) is not actually
+// mounted, so volume data would silently land on the local root disk. Empty
+// expectedFSType disables the check, preserving the default local behavior.
+func verifyVolumeMount(path, expectedFSType string) error {
+	return verifyVolumeMountFrom(procMountsPath, path, expectedFSType)
+}
+
+func verifyVolumeMountFrom(mountsPath, path, expectedFSType string) error {
+	if expectedFSType == "" {
+		return nil
+	}
+
+	got, err := mountFSTypeFrom(mountsPath, path)
+	if err != nil {
+		return fmt.Errorf("verify volume mount for %q: %w", path, err)
+	}
+	if got != expectedFSType {
+		return fmt.Errorf("volume path %q is on a %q mount, expected %q - is the configured volume filesystem mounted?", path, got, expectedFSType)
+	}
+
+	return nil
+}

packages/orchestrator/pkg/volumes/mount_guard_test.go

@@ -0,0 +1,85 @@
+//go:build linux
+
+package volumes
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func writeMounts(t *testing.T, content string) string {
+	t.Helper()
+	p := filepath.Join(t.TempDir(), "mounts")
+	require.NoError(t, os.WriteFile(p, []byte(content), 0o644))
+	return p
+}
+
+// a realistic /proc/mounts: root local, a JuiceFS FUSE mount, and a
+// prefix-colliding sibling that must NOT match /mnt/jfs.
+const sampleMounts = `/dev/sda1 / ext4 rw,relatime 0 0
+JuiceFS:vol /mnt/jfs fuse.juicefs rw,nosuid,nodev 0 0
+/dev/sdb1 /mnt/jfs-backup ext4 rw,relatime 0 0
+`
+
+func TestMountFSTypeFrom(t *testing.T) {
+	t.Parallel()
+	mp := writeMounts(t, sampleMounts)
+
+	for name, tc := range map[string]struct {
+		target string
+		want   string
+	}{
+		"inside juicefs mount":       {"/mnt/jfs/team/vol", "fuse.juicefs"},
+		"juicefs mount point itself": {"/mnt/jfs", "fuse.juicefs"},
+		"prefix-colliding sibling":   {"/mnt/jfs-backup/x", "ext4"},
+		"falls back to root fs":      {"/var/lib/whatever", "ext4"},
+		"root itself":                {"/", "ext4"},
+	} {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+			got, err := mountFSTypeFrom(mp, tc.target)
+			require.NoError(t, err)
+			assert.Equal(t, tc.want, got)
+		})
+	}
+}
+
+func TestVerifyVolumeMountFrom(t *testing.T) {
+	t.Parallel()
+	mp := writeMounts(t, sampleMounts)
+
+	t.Run("empty expected type is a no-op (default)", func(t *testing.T) {
+		t.Parallel()
+		assert.NoError(t, verifyVolumeMountFrom(mp, "/anywhere/at/all", ""))
+	})
+
+	t.Run("path on expected fs passes", func(t *testing.T) {
+		t.Parallel()
+		assert.NoError(t, verifyVolumeMountFrom(mp, "/mnt/jfs/team/vol", "fuse.juicefs"))
+	})
+
+	t.Run("distributed FS not mounted -> path on local root rejected", func(t *testing.T) {
+		t.Parallel()
+		// JuiceFS expected but the volume path resolves to the ext4 root mount:
+		// exactly the silent-to-local-disk footgun the guard prevents.
+		err := verifyVolumeMountFrom(mp, "/mnt/jfs-not-mounted/team/vol", "fuse.juicefs")
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), "expected \"fuse.juicefs\"")
+	})
+
+	t.Run("wrong fs type rejected", func(t *testing.T) {
+		t.Parallel()
+		err := verifyVolumeMountFrom(mp, "/mnt/jfs-backup/vol", "fuse.juicefs")
+		require.Error(t, err)
+	})
+
+	t.Run("missing mounts file surfaces an error when enabled", func(t *testing.T) {
+		t.Parallel()
+		err := verifyVolumeMountFrom(filepath.Join(t.TempDir(), "nope"), "/mnt/jfs/v", "fuse.juicefs")
+		require.Error(t, err)
+	})
+}

packages/orchestrator/pkg/volumes/volume_create.go

@@ -32,6 +32,13 @@ func (s *Service) CreateVolume(
 		attribute.String("path", fullPath),
 	))
 
+	// Fail fast if a volume filesystem type is configured but the path is not on
+	// such a mount - otherwise MkdirAll would silently create the volume on the
+	// local root disk when the distributed FS is not mounted. No-op by default.
+	if err := verifyVolumeMount(fullPath, s.config.VolumeExpectedFSType); err != nil {
+		return nil, err
+	}
+
 	if err := os.MkdirAll(fullPath, 0o700); err != nil {
 		return nil, fmt.Errorf("failed to create volume: %w", err)
 	}