fix(metrics): clear stale state on SetVMState, proper server lifecycle via mgr.Add, wire Metrics in main

Author: syscod3

cmd/agent/main.go

@@ -73,6 +73,7 @@ func main() {
 		Scheme:   mgr.GetScheme(),
 		NodeName: nodeName,
 		Driver:   driver,
+		Metrics:  agent.NewVMMetricsCollector(),
 	}).SetupWithManager(mgr); err != nil {
 		log.Error(err, "Unable to set up ImpVMReconciler")
 		os.Exit(1)

internal/agent/metrics.go

@@ -49,8 +49,11 @@ func NewVMMetricsCollector() *VMMetricsCollector {
 }
 
 // SetVMState sets the imp_vm_state gauge for a VM. key = "namespace/name".
+// Clears any previous state series for this VM so only one state is active at a time.
 func (c *VMMetricsCollector) SetVMState(key, state, node string) {
 	ns, name := splitKey(key)
+	// Remove stale state series before setting the new one to avoid double-counting.
+	c.vmState.DeletePartialMatch(prometheus.Labels{"impvm": name, "namespace": ns, "node": node})
 	c.vmState.WithLabelValues(name, ns, node, state).Set(1)
 }
 

internal/agent/reconciler.go

@@ -185,17 +185,30 @@ func (r *ImpVMReconciler) clearOwnership(ctx context.Context, vm *impdevv1alpha1
 	return ctrl.Result{}, nil
 }
 
+// metricsServer is a controller-runtime Runnable that serves Prometheus metrics.
+// Registered with the manager so it shuts down cleanly when the manager stops.
+type metricsServer struct{ handler http.Handler }
+
+func (s *metricsServer) Start(ctx context.Context) error {
+	srv := &http.Server{Addr: metricsPort, Handler: s.handler, ReadHeaderTimeout: 10 * time.Second}
+	go func() {
+		<-ctx.Done()
+		_ = srv.Shutdown(context.Background()) //nolint:errcheck
+	}()
+	if err := srv.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) {
+		return err
+	}
+	return nil
+}
+
 // SetupWithManager registers the reconciler with the controller-runtime manager.
 func (r *ImpVMReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	if r.Metrics != nil {
-		go func() {
-			mux := http.NewServeMux()
-			mux.Handle("/metrics", NewMetricsHandlerWithCollector(r.Metrics))
-			srv := &http.Server{Addr: metricsPort, Handler: mux, ReadHeaderTimeout: 10 * time.Second}
-			if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
-				logf.Log.Error(err, "metrics server failed")
-			}
-		}()
+		mux := http.NewServeMux()
+		mux.Handle("/metrics", NewMetricsHandlerWithCollector(r.Metrics))
+		if err := mgr.Add(&metricsServer{handler: mux}); err != nil {
+			return err
+		}
 	}
 
 	return ctrl.NewControllerManagedBy(mgr).