feat: add prom metrics for failover reservations

cmd/manager/main.go

@@ -611,6 +611,12 @@ func main() {
 		// The scheduler client calls the nova external scheduler API to get placement decisions
 		schedulerClient := reservations.NewSchedulerClient(failoverConfig.SchedulerURL)
 
+		failoverMonitor := failover.NewFailoverMonitor()
+		if err := metrics.Registry.Register(failoverMonitor); err != nil {
+			setupLog.Error(err, "failed to register failover monitor metrics, continuing without metrics")
+			failoverMonitor = nil
+		}
+
 		// Defer the initialization of PostgresReader until the manager starts
 		// because the cache is not ready during setup
 		if err := mgr.Add(manager.RunnableFunc(func(ctx context.Context) error {
@@ -636,6 +642,7 @@ func main() {
 				vmSource,
 				failoverConfig,
 				schedulerClient,
+				failoverMonitor,
 			)
 
 			// Set up the watch-based reconciler for per-reservation reconciliation

internal/scheduling/reservations/failover/controller.go

@@ -42,15 +42,17 @@ type FailoverReservationController struct {
 	Config          FailoverConfig
 	SchedulerClient *reservations.SchedulerClient
 	Recorder        events.EventRecorder // Event recorder for emitting Kubernetes events
-	reconcileCount  int64                // Track reconciliation count for rotating VM selection
+	Monitor         *FailoverMonitor
+	reconcileCount  int64 // Track reconciliation count for rotating VM selection
 }
 
-func NewFailoverReservationController(c client.Client, vmSource VMSource, config FailoverConfig, schedulerClient *reservations.SchedulerClient) *FailoverReservationController {
+func NewFailoverReservationController(c client.Client, vmSource VMSource, config FailoverConfig, schedulerClient *reservations.SchedulerClient, monitor *FailoverMonitor) *FailoverReservationController {
 	return &FailoverReservationController{
 		Client:          c,
 		VMSource:        vmSource,
 		Config:          config,
 		SchedulerClient: schedulerClient,
+		Monitor:         monitor,
 	}
 }
 
@@ -229,6 +231,9 @@ func (c *FailoverReservationController) validateReservation(ctx context.Context,
 
 // reconcileSummary holds statistics from the reconciliation cycle.
 type reconcileSummary struct {
+	duration            time.Duration
+	totalVMs            int
+	totalReservations   int
 	vmsMissingFailover  int
 	vmsProcessed        int
 	reservationsNeeded  int
@@ -324,7 +329,9 @@ func (c *FailoverReservationController) ReconcilePeriodic(ctx context.Context) (
 	summary.totalFailed = assignSummary.totalFailed
 
 	// Log summary
-	duration := time.Since(startTime)
+	summary.duration = time.Since(startTime)
+	summary.totalVMs = len(vms)
+	summary.totalReservations = len(failoverReservations) + summary.totalCreated
 	requeueAfter := c.Config.ReconcileInterval.Duration
 	successCount := summary.totalCreated + summary.totalReused
 	madeProgress := successCount >= *c.Config.MinSuccessForShortInterval
@@ -334,10 +341,10 @@ func (c *FailoverReservationController) ReconcilePeriodic(ctx context.Context) (
 	}
 	logger.Info("periodic reconciliation completed",
 		"reconcileCount", c.reconcileCount,
-		"duration", duration.Round(time.Millisecond),
+		"duration", summary.duration.Round(time.Millisecond),
 		"requeueAfter", requeueAfter,
-		"totalVMs", len(vms),
-		"totalReservations", len(failoverReservations),
+		"totalVMs", summary.totalVMs,
+		"totalReservations", summary.totalReservations,
 		"vmsMissingFailover", summary.vmsMissingFailover,
 		"vmsProcessed", summary.vmsProcessed,
 		"reservationsNeeded", summary.reservationsNeeded,
@@ -347,6 +354,10 @@ func (c *FailoverReservationController) ReconcilePeriodic(ctx context.Context) (
 		"updated", summary.reservationsUpdated,
 		"deleted", summary.reservationsDeleted)
 
+	if c.Monitor != nil {
+		c.Monitor.RecordReconciliation(summary, "")
+	}
+
 	return ctrl.Result{RequeueAfter: requeueAfter}, nil
 }
 

internal/scheduling/reservations/failover/integration_test.go

@@ -693,6 +693,7 @@ func (env *IntegrationTestEnv) TriggerFailoverReconcile(flavorRequirements map[s
 		env.VMSource,
 		config,
 		schedulerClient,
+		nil,
 	)
 
 	_, err := controller.ReconcilePeriodic(context.Background())

internal/scheduling/reservations/failover/monitor.go

@@ -0,0 +1,150 @@
+// Copyright SAP SE
+// SPDX-License-Identifier: Apache-2.0
+
+package failover
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var azLabel = []string{"availability_zone"}
+
+// FailoverMonitor provides Prometheus metrics for the failover reconciliation controller.
+type FailoverMonitor struct {
+	reconciliationRuns     *prometheus.CounterVec
+	reconciliationDuration *prometheus.HistogramVec
+	totalVMs               *prometheus.GaugeVec
+	totalReservations      *prometheus.GaugeVec
+	vmsMissingFailover     *prometheus.GaugeVec
+	vmsProcessed           *prometheus.CounterVec
+	reservationsNeeded     *prometheus.CounterVec
+	reservationsReused     *prometheus.CounterVec
+	reservationsCreated    *prometheus.CounterVec
+	reservationsFailed     *prometheus.CounterVec
+	reservationsUpdated    *prometheus.CounterVec
+	reservationsDeleted    *prometheus.CounterVec
+}
+
+// NewFailoverMonitor creates a new monitor with Prometheus metrics.
+func NewFailoverMonitor() *FailoverMonitor {
+	m := &FailoverMonitor{
+		reconciliationRuns: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "cortex_failover_reconciliation_runs_total",
+			Help: "Total number of failover periodic reconciliation runs since pod restart",
+		}, azLabel),
+		reconciliationDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
+			Name:    "cortex_failover_reconciliation_duration_seconds",
+			Help:    "Duration of failover periodic reconciliation cycles",
+			Buckets: []float64{0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60},
+		}, azLabel),
+		totalVMs: prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Name: "cortex_failover_reconciliation_total_vms",
+			Help: "Total number of VMs seen during the last reconciliation",
+		}, azLabel),
+		totalReservations: prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Name: "cortex_failover_reconciliation_total_reservations",
+			Help: "Total number of failover reservations during the last reconciliation",
+		}, azLabel),
+		vmsMissingFailover: prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Name: "cortex_failover_reconciliation_vms_missing_failover",
+			Help: "Number of VMs missing required failover reservations during the last reconciliation",
+		}, azLabel),
+		vmsProcessed: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "cortex_failover_reconciliation_vms_processed_total",
+			Help: "Total number of VMs processed across all reconciliation cycles since pod restart",
+		}, azLabel),
+		reservationsNeeded: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "cortex_failover_reconciliation_reservations_needed_total",
+			Help: "Total number of reservations needed across all reconciliation cycles since pod restart",
+		}, azLabel),
+		reservationsReused: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "cortex_failover_reconciliation_reservations_reused_total",
+			Help: "Total number of reservations reused across all reconciliation cycles since pod restart",
+		}, azLabel),
+		reservationsCreated: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "cortex_failover_reconciliation_reservations_created_total",
+			Help: "Total number of reservations created across all reconciliation cycles since pod restart",
+		}, azLabel),
+		reservationsFailed: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "cortex_failover_reconciliation_reservations_failed_total",
+			Help: "Total number of failed reservation attempts across all reconciliation cycles since pod restart",
+		}, azLabel),
+		reservationsUpdated: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "cortex_failover_reconciliation_reservations_updated_total",
+			Help: "Total number of reservation allocation updates across all reconciliation cycles since pod restart",
+		}, azLabel),
+		reservationsDeleted: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "cortex_failover_reconciliation_reservations_deleted_total",
+			Help: "Total number of empty reservations deleted across all reconciliation cycles since pod restart",
+		}, azLabel),
+	}
+
+	// Pre-initialize the aggregate label so metrics appear even before the first reconciliation.
+	m.preInitialize("")
+
+	return m
+}
+
+func (m *FailoverMonitor) preInitialize(az string) {
+	m.reconciliationRuns.WithLabelValues(az)
+	m.reconciliationDuration.WithLabelValues(az)
+	m.totalVMs.WithLabelValues(az)
+	m.totalReservations.WithLabelValues(az)
+	m.vmsMissingFailover.WithLabelValues(az)
+	m.vmsProcessed.WithLabelValues(az)
+	m.reservationsNeeded.WithLabelValues(az)
+	m.reservationsReused.WithLabelValues(az)
+	m.reservationsCreated.WithLabelValues(az)
+	m.reservationsFailed.WithLabelValues(az)
+	m.reservationsUpdated.WithLabelValues(az)
+	m.reservationsDeleted.WithLabelValues(az)
+}
+
+// RecordReconciliation records all metrics from a single reconciliation cycle.
+// The availabilityZone parameter allows future per-AZ reporting; pass "" for aggregate.
+func (m *FailoverMonitor) RecordReconciliation(summary reconcileSummary, availabilityZone string) {
+	m.reconciliationRuns.WithLabelValues(availabilityZone).Inc()
+	m.reconciliationDuration.WithLabelValues(availabilityZone).Observe(summary.duration.Seconds())
+	m.totalVMs.WithLabelValues(availabilityZone).Set(float64(summary.totalVMs))
+	m.totalReservations.WithLabelValues(availabilityZone).Set(float64(summary.totalReservations))
+	m.vmsMissingFailover.WithLabelValues(availabilityZone).Set(float64(summary.vmsMissingFailover))
+	m.vmsProcessed.WithLabelValues(availabilityZone).Add(float64(summary.vmsProcessed))
+	m.reservationsNeeded.WithLabelValues(availabilityZone).Add(float64(summary.reservationsNeeded))
+	m.reservationsReused.WithLabelValues(availabilityZone).Add(float64(summary.totalReused))
+	m.reservationsCreated.WithLabelValues(availabilityZone).Add(float64(summary.totalCreated))
+	m.reservationsFailed.WithLabelValues(availabilityZone).Add(float64(summary.totalFailed))
+	m.reservationsUpdated.WithLabelValues(availabilityZone).Add(float64(summary.reservationsUpdated))
+	m.reservationsDeleted.WithLabelValues(availabilityZone).Add(float64(summary.reservationsDeleted))
+}
+
+// Describe implements prometheus.Collector.
+func (m *FailoverMonitor) Describe(ch chan<- *prometheus.Desc) {
+	m.reconciliationRuns.Describe(ch)
+	m.reconciliationDuration.Describe(ch)
+	m.totalVMs.Describe(ch)
+	m.totalReservations.Describe(ch)
+	m.vmsMissingFailover.Describe(ch)
+	m.vmsProcessed.Describe(ch)
+	m.reservationsNeeded.Describe(ch)
+	m.reservationsReused.Describe(ch)
+	m.reservationsCreated.Describe(ch)
+	m.reservationsFailed.Describe(ch)
+	m.reservationsUpdated.Describe(ch)
+	m.reservationsDeleted.Describe(ch)
+}
+
+// Collect implements prometheus.Collector.
+func (m *FailoverMonitor) Collect(ch chan<- prometheus.Metric) {
+	m.reconciliationRuns.Collect(ch)
+	m.reconciliationDuration.Collect(ch)
+	m.totalVMs.Collect(ch)
+	m.totalReservations.Collect(ch)
+	m.vmsMissingFailover.Collect(ch)
+	m.vmsProcessed.Collect(ch)
+	m.reservationsNeeded.Collect(ch)
+	m.reservationsReused.Collect(ch)
+	m.reservationsCreated.Collect(ch)
+	m.reservationsFailed.Collect(ch)
+	m.reservationsUpdated.Collect(ch)
+	m.reservationsDeleted.Collect(ch)
+}