feat(agent): StartedAt timeout for stuck Starting VMs; hostPath socketDir default

Add StartedAt to ImpVMStatus to track when a VM entered the Starting phase.
handleStarting times out VMs stuck in Starting (default 5 min, configurable
via ImpVMReconciler.StartTimeout) by calling finishFailed. Clear StartedAt
on successful transition to Running. Flip socketDir.enabled to true in Helm
values so Firecracker sockets survive agent pod restarts, enabling PID-based
reattach for running VMs.

Author: syscod3

api/v1alpha1/impvm_types.go

@@ -147,6 +147,11 @@ type ImpVMStatus struct {
 	// +optional
 	RunningAt *metav1.Time `json:"runningAt,omitempty"`
 
+	// StartedAt is the time the VM last transitioned to phase Starting.
+	// Used to detect and time out stuck start attempts.
+	// +optional
+	StartedAt *metav1.Time `json:"startedAt,omitempty"`
+
 	// RestartCount is the cumulative number of times this VM has been restarted.
 	// +optional
 	RestartCount int32 `json:"restartCount,omitempty"`

api/v1alpha1/zz_generated.deepcopy.go

@@ -39,7 +39,10 @@ agent:
     kernelPath: ""
   hostPaths:
     socketDir:
-      enabled: false
+      # enabled: true ensures sockets survive agent pod restarts (required for
+      # PID-based reattach). Setting this to false will orphan Firecracker
+      # sockets on restart — breaking graceful stop for running VMs.
+      enabled: true
       path: /run/imp/sockets
     imageCache:
       enabled: false

charts/imp/values.yaml

@@ -747,6 +747,12 @@ spec:
                   a node.
                 format: date-time
                 type: string
+              startedAt:
+                description: |-
+                  StartedAt is the time the VM last transitioned to phase Starting.
+                  Used to detect and time out stuck start attempts.
+                format: date-time
+                type: string
             type: object
         type: object
     served: true

config/crd/bases/imp.dev_impvms.yaml

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/util/retry"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -35,6 +36,9 @@ type ImpVMReconciler struct {
 	// Alloc is the in-memory IP allocator. When non-nil, Reserve is called during
 	// lazy reattach to restore IP state after agent restart.
 	Alloc *network.Allocator
+	// StartTimeout is how long a VM may remain in Starting before being
+	// transitioned to Failed. Defaults to 5 minutes when zero.
+	StartTimeout time.Duration
 }
 
 // +kubebuilder:rbac:groups=imp.dev,resources=impvms,verbs=get;list;watch;update;patch
@@ -65,20 +69,39 @@ func (r *ImpVMReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl
 	case impdevv1alpha1.VMPhaseRunning:
 		return r.handleRunning(ctx, vm)
 	case impdevv1alpha1.VMPhaseStarting:
-		log.Info("VM is Starting — requeuing")
-		return ctrl.Result{RequeueAfter: 2 * time.Second}, nil
+		return r.handleStarting(ctx, vm)
 	default:
 		// Pending, Succeeded, Failed — not our concern.
 		return ctrl.Result{}, nil
 	}
 }
 
+func (r *ImpVMReconciler) startTimeout() time.Duration {
+	if r.StartTimeout > 0 {
+		return r.StartTimeout
+	}
+	return 5 * time.Minute
+}
+
+func (r *ImpVMReconciler) handleStarting(ctx context.Context, vm *impdevv1alpha1.ImpVM) (ctrl.Result, error) {
+	log := logf.FromContext(ctx)
+	if vm.Status.StartedAt != nil {
+		if elapsed := time.Since(vm.Status.StartedAt.Time); elapsed > r.startTimeout() {
+			log.Info("VM stuck in Starting — timing out", "elapsed", elapsed, "timeout", r.startTimeout())
+			return r.finishFailed(ctx, vm)
+		}
+	}
+	return ctrl.Result{RequeueAfter: 2 * time.Second}, nil
+}
+
 func (r *ImpVMReconciler) handleScheduled(ctx context.Context, vm *impdevv1alpha1.ImpVM) (ctrl.Result, error) {
 	log := logf.FromContext(ctx)
 
 	// Set phase=Starting before calling driver to make concurrent reconciles idempotent.
 	base := vm.DeepCopy()
 	vm.Status.Phase = impdevv1alpha1.VMPhaseStarting
+	now := metav1.Now()
+	vm.Status.StartedAt = &now
 	if err := r.Status().Patch(ctx, vm, client.MergeFrom(base)); err != nil {
 		if apierrors.IsConflict(err) {
 			return ctrl.Result{Requeue: true}, nil
@@ -100,6 +123,7 @@ func (r *ImpVMReconciler) handleScheduled(ctx context.Context, vm *impdevv1alpha
 
 	base = vm.DeepCopy()
 	vm.Status.Phase = impdevv1alpha1.VMPhaseRunning
+	vm.Status.StartedAt = nil
 	vm.Status.IP = state.IP
 	vm.Status.RuntimePID = pid
 	if err := r.Status().Patch(ctx, vm, client.MergeFrom(base)); err != nil {

internal/agent/reconciler.go

@@ -549,6 +549,109 @@ var _ = Describe("ImpVM Agent: registerVTEP — concurrent writes", func() {
 	})
 })
 
+var _ = Describe("ImpVM Agent: Starting — stuck timeout", func() {
+	ctx := context.Background()
+
+	It("transitions to Failed when StartedAt exceeds StartTimeout", func() {
+		r := &ImpVMReconciler{
+			Client:       k8sClient,
+			NodeName:     testNode,
+			Driver:       NewStubDriver(),
+			StartTimeout: 5 * time.Minute,
+		}
+
+		sixMinsAgo := metav1.NewTime(time.Now().Add(-6 * time.Minute))
+		vm := &impdevv1alpha1.ImpVM{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "tc-starting-timeout", Namespace: "default",
+				Finalizers: []string{"imp/finalizer"},
+			},
+			Spec: impdevv1alpha1.ImpVMSpec{NodeName: testNode},
+		}
+		Expect(k8sClient.Create(ctx, vm)).To(Succeed())
+		DeferCleanup(func() { k8sClient.Delete(ctx, vm) }) //nolint:errcheck
+
+		base := vm.DeepCopy()
+		vm.Status.Phase = impdevv1alpha1.VMPhaseStarting
+		vm.Status.StartedAt = &sixMinsAgo
+		Expect(k8sClient.Status().Patch(ctx, vm, client.MergeFrom(base))).To(Succeed())
+
+		_, err := r.Reconcile(ctx, reconcile.Request{
+			NamespacedName: types.NamespacedName{Name: "tc-starting-timeout", Namespace: "default"},
+		})
+		Expect(err).NotTo(HaveOccurred())
+
+		updated := &impdevv1alpha1.ImpVM{}
+		Expect(k8sClient.Get(ctx, types.NamespacedName{Name: "tc-starting-timeout", Namespace: "default"}, updated)).To(Succeed())
+		Expect(updated.Status.Phase).To(Equal(impdevv1alpha1.VMPhaseFailed))
+	})
+
+	It("stays in Starting when StartedAt is within StartTimeout", func() {
+		r := &ImpVMReconciler{
+			Client:       k8sClient,
+			NodeName:     testNode,
+			Driver:       NewStubDriver(),
+			StartTimeout: 5 * time.Minute,
+		}
+
+		oneMinsAgo := metav1.NewTime(time.Now().Add(-1 * time.Minute))
+		vm := &impdevv1alpha1.ImpVM{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "tc-starting-notimeout", Namespace: "default",
+				Finalizers: []string{"imp/finalizer"},
+			},
+			Spec: impdevv1alpha1.ImpVMSpec{NodeName: testNode},
+		}
+		Expect(k8sClient.Create(ctx, vm)).To(Succeed())
+		DeferCleanup(func() { k8sClient.Delete(ctx, vm) }) //nolint:errcheck
+
+		base := vm.DeepCopy()
+		vm.Status.Phase = impdevv1alpha1.VMPhaseStarting
+		vm.Status.StartedAt = &oneMinsAgo
+		Expect(k8sClient.Status().Patch(ctx, vm, client.MergeFrom(base))).To(Succeed())
+
+		result, err := r.Reconcile(ctx, reconcile.Request{
+			NamespacedName: types.NamespacedName{Name: "tc-starting-notimeout", Namespace: "default"},
+		})
+		Expect(err).NotTo(HaveOccurred())
+		Expect(result.RequeueAfter).To(Equal(2 * time.Second))
+
+		updated := &impdevv1alpha1.ImpVM{}
+		Expect(k8sClient.Get(ctx, types.NamespacedName{Name: "tc-starting-notimeout", Namespace: "default"}, updated)).To(Succeed())
+		Expect(updated.Status.Phase).To(Equal(impdevv1alpha1.VMPhaseStarting))
+	})
+
+	It("stays in Starting when StartedAt is nil (timeout not yet set)", func() {
+		r := &ImpVMReconciler{
+			Client:       k8sClient,
+			NodeName:     testNode,
+			Driver:       NewStubDriver(),
+			StartTimeout: 5 * time.Minute,
+		}
+
+		vm := &impdevv1alpha1.ImpVM{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "tc-starting-nostartedat", Namespace: "default",
+				Finalizers: []string{"imp/finalizer"},
+			},
+			Spec: impdevv1alpha1.ImpVMSpec{NodeName: testNode},
+		}
+		Expect(k8sClient.Create(ctx, vm)).To(Succeed())
+		DeferCleanup(func() { k8sClient.Delete(ctx, vm) }) //nolint:errcheck
+
+		base := vm.DeepCopy()
+		vm.Status.Phase = impdevv1alpha1.VMPhaseStarting
+		// StartedAt intentionally not set
+		Expect(k8sClient.Status().Patch(ctx, vm, client.MergeFrom(base))).To(Succeed())
+
+		result, err := r.Reconcile(ctx, reconcile.Request{
+			NamespacedName: types.NamespacedName{Name: "tc-starting-nostartedat", Namespace: "default"},
+		})
+		Expect(err).NotTo(HaveOccurred())
+		Expect(result.RequeueAfter).To(Equal(2 * time.Second))
+	})
+})
+
 var _ = Describe("ImpVM Agent: handleTerminating driver.Stop error", func() {
 	ctx := context.Background()