Better /etc/ remount

Author: kvinwang

scripts/install-sysbox-complete.sh

@@ -29,23 +29,28 @@ log_error() {
     echo -e "${RED}❌ $1${NC}"
 }
 
+# Helper function to run commands on host via nsenter
+hostrun() {
+    nsenter -t 1 -m -p -n "$@"
+}
+
 # Check if already installed
 check_existing() {
     log_info "Checking existing installation..."
 
     # Check if systemd services exist (in either /etc or /run)
-    if chroot /host systemctl list-unit-files | grep -q "sysbox-mgr.service" ||
+    if hostrun systemctl list-unit-files | grep -q "sysbox-mgr.service" ||
         [ -f /host/run/systemd/system/sysbox-mgr.service ] ||
         [ -f /host/etc/systemd/system/sysbox-mgr.service ]; then
         log_warning "Sysbox services already installed - skipping installation"
 
         # Show service status
         echo "Service status:"
-        chroot /host systemctl status sysbox-mgr.service --no-pager 2>/dev/null | head -5 || true
-        chroot /host systemctl status sysbox-fs.service --no-pager 2>/dev/null | head -5 || true
+        hostrun systemctl status sysbox-mgr.service --no-pager 2>/dev/null | head -5 || true
+        hostrun systemctl status sysbox-fs.service --no-pager 2>/dev/null | head -5 || true
 
         # Check if actually running
-        if chroot /host systemctl is-active sysbox-mgr.service >/dev/null 2>&1; then
+        if hostrun systemctl is-active sysbox-mgr.service >/dev/null 2>&1; then
             log_success "Sysbox is installed and running"
         else
             log_info "Sysbox is installed but not running. Start with:"
@@ -71,15 +76,15 @@ copy_binaries() {
     chmod +x /host/tmp/rsync-static /host/tmp/sysbox-*
 
     # Create symlinks for dependencies
-    chroot /host ln -sf /tmp/rsync-static /usr/bin/rsync 2>/dev/null || true
-    chroot /host ln -sf /usr/sbin/modprobe /usr/bin/modprobe 2>/dev/null || true
-    chroot /host ln -sf /usr/sbin/iptables /usr/bin/iptables 2>/dev/null || true
+    hostrun ln -sf /tmp/rsync-static /usr/bin/rsync 2>/dev/null || true
+    hostrun ln -sf /usr/sbin/modprobe /usr/bin/modprobe 2>/dev/null || true
+    hostrun ln -sf /usr/sbin/iptables /usr/bin/iptables 2>/dev/null || true
 
     # Handle fusermount/fusermount3 (Alpine has fusermount3, sysbox expects fusermount)
-    if ! chroot /host which fusermount >/dev/null 2>&1; then
-        if chroot /host which fusermount3 >/dev/null 2>&1; then
+    if ! hostrun which fusermount >/dev/null 2>&1; then
+        if hostrun which fusermount3 >/dev/null 2>&1; then
             log_info "Creating symlink: fusermount -> fusermount3"
-            chroot /host ln -sf /usr/bin/fusermount3 /usr/bin/fusermount
+            hostrun ln -sf /usr/bin/fusermount3 /usr/bin/fusermount
         else
             log_warning "Neither fusermount nor fusermount3 found - FUSE operations may fail"
         fi
@@ -88,17 +93,28 @@ copy_binaries() {
     log_success "Binaries copied and dependencies linked"
 }
 
-# Setup /etc configuration (subuid/subgid)
-setup_etc_config() {
-    log_info "Setting up /etc configuration..."
+# Setup /etc overlay and configuration
+setup_etc_overlay() {
+    log_info "Setting up /etc overlay..."
+
+    # Create persistent overlay directories
+    hostrun mkdir -p /dstack/persistent/sysbox-etc-overlay/upper /dstack/persistent/sysbox-etc-overlay/work
 
-    # Create subuid/subgid files
-    echo "sysbox:200000:65536" >/host/tmp/subuid.tmp
-    echo "sysbox:200000:65536" >/host/tmp/subgid.tmp
+    # Check if main overlay already exists
+    if hostrun mount | grep -q "/etc.*overlay.*sysbox-etc-overlay"; then
+        log_warning "/etc already has sysbox overlay mounted"
+    else
+        # Mount main /etc overlay
+        hostrun mount -t overlay overlay \
+            -o lowerdir=/etc,upperdir=/dstack/persistent/sysbox-etc-overlay/upper,workdir=/dstack/persistent/sysbox-etc-overlay/work \
+            /etc
+        log_success "Main /etc overlay mounted"
+    fi
 
-    # Note: The actual /etc overlay will be handled by systemd service
-    log_success "Created subuid/subgid configuration files"
-    log_info "These will be applied when the overlay service starts"
+    # Create subuid/subgid
+    hostrun sh -c 'echo "sysbox:200000:65536" > /etc/subuid'
+    hostrun sh -c 'echo "sysbox:200000:65536" > /etc/subgid'
+    log_success "Created subuid/subgid mappings"
 }
 
 # Configure Docker runtime
@@ -109,12 +125,12 @@ configure_docker() {
     # Currently overwrites daemon.json - should merge with existing runtimes/settings
 
     # Backup existing daemon.json if it exists
-    if chroot /host [ -f /etc/docker/daemon.json ]; then
-        chroot /host cp /etc/docker/daemon.json /etc/docker/daemon.json.backup
+    if hostrun [ -f /etc/docker/daemon.json ]; then
+        hostrun cp /etc/docker/daemon.json /etc/docker/daemon.json.backup
         log_info "Backed up existing Docker daemon.json (will be overwritten)"
     fi
 
-    chroot /host tee /etc/docker/daemon.json >/dev/null <<'DOCKEREOF'
+    hostrun tee /etc/docker/daemon.json >/dev/null <<'DOCKEREOF'
 {
   "log-driver": "json-file",
   "log-opts": {
@@ -137,31 +153,13 @@ create_systemd_services() {
     log_info "Creating systemd services..."
 
     # Use /run/systemd/system for runtime units (doesn't require persistent storage)
-    chroot /host mkdir -p /run/systemd/system
+    hostrun mkdir -p /run/systemd/system
 
-    # Copy all service files from container to host runtime directory
-    cp /usr/local/share/sysbox-etc-overlay.service /host/run/systemd/system/
+    # Copy service files from container to host runtime directory
     cp /usr/local/share/sysbox-mgr.service /host/run/systemd/system/
     cp /usr/local/share/sysbox-fs.service /host/run/systemd/system/
 
-    # Create a setup script for subuid/subgid
-    cat >/host/tmp/sysbox-setup.sh <<'EOF'
-#!/bin/sh
-# Apply subuid/subgid configuration after overlay mount
-if [ -f /tmp/subuid.tmp ]; then
-    cat /tmp/subuid.tmp > /etc/subuid
-    cat /tmp/subgid.tmp > /etc/subgid
-    rm -f /tmp/subuid.tmp /tmp/subgid.tmp
-fi
-EOF
-    chmod +x /host/tmp/sysbox-setup.sh
-
     # Verify files were copied
-    if [ ! -f /host/run/systemd/system/sysbox-etc-overlay.service ]; then
-        log_error "Failed to copy sysbox-etc-overlay.service to /run/systemd/system/"
-        return 1
-    fi
-
     if [ ! -f /host/run/systemd/system/sysbox-mgr.service ]; then
         log_error "Failed to copy sysbox-mgr.service to /run/systemd/system/"
         return 1
@@ -175,47 +173,36 @@ EOF
     log_success "Service files copied to /run/systemd/system/"
 
     # Reload systemd to pick up new service files
-    chroot /host systemctl daemon-reload
+    hostrun systemctl daemon-reload
 
-    # Note: We don't enable services as that requires writing to /etc/systemd/system/*.wants/
-    # Services in /run are transient and will be lost on reboot anyway
     log_success "Systemd services created (transient until reboot)"
-    log_info "Services: sysbox-etc-overlay, sysbox-mgr, sysbox-fs"
-    log_info "Services will be started without enabling (read-only /etc)"
+    log_info "Services: sysbox-mgr, sysbox-fs"
 }
 
 # Start Sysbox services
 start_sysbox() {
     log_info "Starting Sysbox services..."
 
     # Create data directory
-    chroot /host mkdir -p /dstack/persistent/sysbox-data
-
-    # Start services in order: overlay first, then sysbox-mgr, then sysbox-fs
-    log_info "Starting /etc overlay service..."
-    chroot /host systemctl start sysbox-etc-overlay.service
-    sleep 2
-
-    # Apply subuid/subgid configuration
-    chroot /host /tmp/sysbox-setup.sh
+    hostrun mkdir -p /dstack/persistent/sysbox-data
 
+    # Start services in order
     log_info "Starting Sysbox manager..."
-    chroot /host systemctl start sysbox-mgr.service
+    hostrun systemctl start sysbox-mgr.service
     sleep 3
 
     log_info "Starting Sysbox filesystem..."
-    chroot /host systemctl start sysbox-fs.service
+    hostrun systemctl start sysbox-fs.service
     sleep 2
 
     # Verify services are running
-    if chroot /host systemctl is-active sysbox-etc-overlay.service >/dev/null &&
-        chroot /host systemctl is-active sysbox-mgr.service >/dev/null &&
-        chroot /host systemctl is-active sysbox-fs.service >/dev/null; then
-        log_success "All Sysbox services started successfully"
+    if hostrun systemctl is-active sysbox-mgr.service >/dev/null &&
+        hostrun systemctl is-active sysbox-fs.service >/dev/null; then
+        log_success "Sysbox services started successfully"
     else
         log_warning "Some services may not have started correctly"
-        log_info "Check status with: systemctl status sysbox-etc-overlay sysbox-mgr sysbox-fs"
-        log_info "Check logs with: journalctl -u sysbox-etc-overlay -u sysbox-mgr -u sysbox-fs"
+        log_info "Check status with: systemctl status sysbox-mgr sysbox-fs"
+        log_info "Check logs with: journalctl -u sysbox-mgr -u sysbox-fs"
     fi
 }
 
@@ -227,8 +214,8 @@ show_status() {
     echo "=========================================="
     echo
     echo "📊 Status:"
-    echo "  • Sysbox Manager: $(chroot /host systemctl is-active sysbox-mgr.service)"
-    echo "  • Sysbox FS:      $(chroot /host systemctl is-active sysbox-fs.service)"
+    echo "  • Sysbox Manager: $(hostrun systemctl is-active sysbox-mgr.service)"
+    echo "  • Sysbox FS:      $(hostrun systemctl is-active sysbox-fs.service)"
     echo "  • Docker Runtime: Configured (restart required)"
     echo
     echo -e "${YELLOW}⚠️  IMPORTANT: Restart Docker to enable sysbox-runc runtime:${NC}"
@@ -253,7 +240,7 @@ show_status() {
 main() {
     check_existing
     copy_binaries
-    setup_etc_config
+    setup_etc_overlay
     configure_docker
     create_systemd_services
     start_sysbox