Add application credential finalizer management

api/bases/watcher.openstack.org_watchers.yaml

@@ -737,6 +737,13 @@ spec:
                   from watcher-api
                 format: int32
                 type: integer
+              applicationCredentialSecret:
+                description: |-
+                  ApplicationCredentialSecret - the AC secret watcher is currently
+                  consuming and protecting with the openstack.org/watcher-ac-consumer
+                  finalizer. Tracked so the controller can remove its finalizer from the
+                  old secret when the openstack-operator rotates the reference.
+                type: string
               applierServiceReadyCount:
                 description: ApplierServiceReadyCount defines the number or replicas
                   ready from watcher-applier

api/go.mod

@@ -91,3 +91,5 @@ replace k8s.io/component-base => k8s.io/component-base v0.31.14 //allow-merging
 replace github.com/rabbitmq/cluster-operator/v2 => github.com/openstack-k8s-operators/rabbitmq-cluster-operator/v2 v2.6.1-0.20250929174222-a0d328fa4dec //allow-merging
 
 replace k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20250627150254-e9823e99808e //allow-merging
+
+replace github.com/openstack-k8s-operators/keystone-operator/api => github.com/Deydra71/keystone-operator/api v0.0.0-20260424093804-00a0ccdc9d20

api/v1beta1/watcher_types.go

@@ -61,6 +61,12 @@ type WatcherStatus struct {
 
 	// DecisionEngineServiceReadyCount defines the number or replicas ready from watcher-decision-engine
 	DecisionEngineServiceReadyCount int32 `json:"decisionengineServiceReadyCount,omitempty"`
+
+	// ApplicationCredentialSecret - the AC secret watcher is currently
+	// consuming and protecting with the openstack.org/watcher-ac-consumer
+	// finalizer. Tracked so the controller can remove its finalizer from the
+	// old secret when the openstack-operator rotates the reference.
+	ApplicationCredentialSecret string `json:"applicationCredentialSecret,omitempty"`
 }
 
 // WatcherDBPurge defines the parameters for the Watcher database purging cron job

config/crd/bases/watcher.openstack.org_watchers.yaml

@@ -737,6 +737,13 @@ spec:
                   from watcher-api
                 format: int32
                 type: integer
+              applicationCredentialSecret:
+                description: |-
+                  ApplicationCredentialSecret - the AC secret watcher is currently
+                  consuming and protecting with the openstack.org/watcher-ac-consumer
+                  finalizer. Tracked so the controller can remove its finalizer from the
+                  old secret when the openstack-operator rotates the reference.
+                type: string
               applierServiceReadyCount:
                 description: ApplierServiceReadyCount defines the number or replicas
                   ready from watcher-applier

go.mod

@@ -141,3 +141,5 @@ replace k8s.io/component-base => k8s.io/component-base v0.31.14 //allow-merging
 replace github.com/rabbitmq/cluster-operator/v2 => github.com/openstack-k8s-operators/rabbitmq-cluster-operator/v2 v2.6.1-0.20250929174222-a0d328fa4dec //allow-merging
 
 replace k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20250627150254-e9823e99808e //allow-merging
+
+replace github.com/openstack-k8s-operators/keystone-operator/api => github.com/Deydra71/keystone-operator/api v0.0.0-20260424093804-00a0ccdc9d20

go.sum

@@ -1,3 +1,5 @@
+github.com/Deydra71/keystone-operator/api v0.0.0-20260424093804-00a0ccdc9d20 h1:iyxfh2SDvQrOrsHItYAE3A3+8Ku9UnzWAq9jnLJDLjg=
+github.com/Deydra71/keystone-operator/api v0.0.0-20260424093804-00a0ccdc9d20/go.mod h1:SpO4CL7c5/1HG+61fP6kWhL2+3aqR+5SNatdZueKrz8=
 github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0=
 github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
 github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI=
@@ -120,8 +122,6 @@ github.com/openshift/api v0.0.0-20250711200046-c86d80652a9e h1:E1OdwSpqWuDPCedyU
 github.com/openshift/api v0.0.0-20250711200046-c86d80652a9e/go.mod h1:Shkl4HanLwDiiBzakv+con/aMGnVE2MAGvoKp5oyYUo=
 github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260508091801-73f228e6af31 h1:FWa0vNs175LpV1eSZ60YOGFdbJ3LqxQ1fxfprBRg7T4=
 github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260508091801-73f228e6af31/go.mod h1:/S2AN21zV70V1XuL0Of2dCjYWNkKwQSyNI8l/iQVrMs=
-github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20260507114237-f0b612d6c21f h1:28WYAUIef3uion0Pps6doCSSbgZtIcodGzwG6BHhCOw=
-github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20260507114237-f0b612d6c21f/go.mod h1:4ryvbSYuoN522BIPijnm0wMemPgJVKf7jCv8BNDq46I=
 github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260506154724-30a976ba8ef0 h1:vkFvn06Ns9qW4AbzFjFDu8ioosRmhkEZiDrO3DOQhLg=
 github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260506154724-30a976ba8ef0/go.mod h1:aIuG6lx3aS0vnXweRNdR/Q0SlfOsLIo0OzrqKK7C6xs=
 github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20260430090237-a4265c18a162 h1:kUfZlcl+EbUBEWe6EGLXjzlUeYj7xZ21QsPA5jMJlwE=

internal/controller/watcher_controller.go

@@ -478,6 +478,24 @@ func (r *WatcherReconciler) Reconcile(ctx context.Context, req ctrl.Request) (re
 		return ctrl.Result{}, err
 	}
 
+	// Add consumer finalizer to the new AC secret early, before deployment.
+	// The old secret's finalizer is removed later (after all services deploy)
+	// so that rapid rotations don't revoke a credential still in use by pods.
+	if instance.Spec.Auth.ApplicationCredentialSecret != "" {
+		if err := keystonev1.ManageACSecretFinalizer(ctx, helper, instance.Namespace,
+			instance.Spec.Auth.ApplicationCredentialSecret,
+			"",
+			watcher.ACConsumerFinalizer); err != nil {
+			instance.Status.Conditions.Set(condition.FalseCondition(
+				condition.ServiceConfigReadyCondition,
+				condition.ErrorReason,
+				condition.SeverityWarning,
+				condition.ServiceConfigReadyErrorMessage,
+				err.Error()))
+			return ctrl.Result{}, err
+		}
+	}
+
 	instance.Status.Conditions.MarkTrue(condition.ServiceConfigReadyCondition, condition.ServiceConfigReadyMessage)
 	// End of config generation for dbsync
 
@@ -523,6 +541,27 @@ func (r *WatcherReconciler) Reconcile(ctx context.Context, req ctrl.Request) (re
 	}
 	// End of Watcher Applier deploy
 
+	// Manage the old AC secret's finalizer and status tracking.
+	// On rotation (old != new), only remove the old secret's finalizer after
+	// all sub-services are ready with the new credentials. This prevents
+	// premature revocation during rapid rotations.
+	isRotation := instance.Status.ApplicationCredentialSecret != "" && instance.Status.ApplicationCredentialSecret != instance.Spec.Auth.ApplicationCredentialSecret
+
+	if isRotation {
+		allServicesReady := instance.Status.Conditions.IsTrue(watcherv1beta1.WatcherAPIReadyCondition) &&
+			instance.Status.Conditions.IsTrue(watcherv1beta1.WatcherApplierReadyCondition) &&
+			instance.Status.Conditions.IsTrue(watcherv1beta1.WatcherDecisionEngineReadyCondition)
+		if allServicesReady {
+			if err := keystonev1.RemoveACSecretConsumerFinalizer(ctx, helper, instance.Namespace,
+				instance.Status.ApplicationCredentialSecret, watcher.ACConsumerFinalizer); err != nil {
+				return ctrl.Result{}, err
+			}
+			instance.Status.ApplicationCredentialSecret = instance.Spec.Auth.ApplicationCredentialSecret
+		}
+	} else {
+		instance.Status.ApplicationCredentialSecret = instance.Spec.Auth.ApplicationCredentialSecret
+	}
+
 	//
 	// remove finalizers from unused MariaDBAccount records
 	// this assumes all database-depedendent deployments are up and
@@ -1347,6 +1386,17 @@ func (r *WatcherReconciler) reconcileDelete(ctx context.Context, instance *watch
 	}
 	//
 
+	// Remove consumer finalizer from AC secrets watcher was consuming.
+	for _, secretName := range []string{
+		instance.Status.ApplicationCredentialSecret,
+		instance.Spec.Auth.ApplicationCredentialSecret,
+	} {
+		if err := keystonev1.RemoveACSecretConsumerFinalizer(ctx, helper, instance.Namespace,
+			secretName, watcher.ACConsumerFinalizer); err != nil {
+			return ctrl.Result{}, err
+		}
+	}
+
 	controllerutil.RemoveFinalizer(instance, helper.GetFinalizer())
 	Log.Info(fmt.Sprintf("Reconciled Service '%s' delete successfully", instance.Name))
 	return ctrl.Result{}, nil

internal/watcher/constants.go

@@ -50,4 +50,7 @@ const (
 
 	// scriptVolume is the name of the volume used to ship scripts into pods
 	scriptVolume = "scripts-volume"
+
+	// ACConsumerFinalizer is added to AC secrets that watcher is actively consuming
+	ACConsumerFinalizer = "openstack.org/watcher-ac-consumer"
 )

test/functional/watcher_controller_test.go

@@ -17,6 +17,7 @@ import (
 	. "github.com/openstack-k8s-operators/lib-common/modules/common/test/helpers"
 	mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1"
 	watcherv1beta1 "github.com/openstack-k8s-operators/watcher-operator/api/v1beta1"
+	"github.com/openstack-k8s-operators/watcher-operator/internal/watcher"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -1858,6 +1859,188 @@ var _ = Describe("Watcher controller", func() {
 		})
 	})
 
+	When("ApplicationCredential consumer finalizer is managed", func() {
+		var acSecretName string
+
+		BeforeEach(func() {
+			acSecretName = "ac-watcher-consumer-fnz-secret" //nolint:gosec
+
+			acSecret := &corev1.Secret{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      acSecretName,
+					Namespace: watcherTest.Instance.Namespace,
+				},
+				Data: map[string][]byte{
+					keystonev1beta1.ACIDSecretKey:     []byte("consumer-test-ac-id"),
+					keystonev1beta1.ACSecretSecretKey: []byte("consumer-test-ac-secret"), //nolint:gosec
+				},
+			}
+			Expect(k8sClient.Create(ctx, acSecret)).To(Succeed())
+			DeferCleanup(k8sClient.Delete, ctx, acSecret)
+
+			DeferCleanup(k8sClient.Delete, ctx, CreateWatcherMessageBusSecret(watcherTest.Instance.Namespace, "rabbitmq-secret"))
+
+			memcachedSpec := memcachedv1.MemcachedSpec{
+				MemcachedSpecCore: memcachedv1.MemcachedSpecCore{
+					Replicas: ptr.To(int32(1)),
+				},
+			}
+			DeferCleanup(infra.DeleteMemcached, infra.CreateMemcached(watcherTest.Watcher.Namespace, MemcachedInstance, memcachedSpec))
+			infra.SimulateMemcachedReady(watcherTest.MemcachedNamespace)
+
+			DeferCleanup(keystone.DeleteKeystoneAPI, keystone.CreateKeystoneAPI(watcherTest.WatcherAPI.Namespace))
+
+			DeferCleanup(
+				k8sClient.Delete, ctx, th.CreateSecret(
+					types.NamespacedName{Namespace: watcherTest.Instance.Namespace, Name: "metric-storage-prometheus-endpoint"},
+					map[string][]byte{
+						"host": []byte("prometheus.example.com"),
+						"port": []byte("9090"),
+					},
+				))
+
+			DeferCleanup(
+				k8sClient.Delete, ctx, th.CreateSecret(
+					types.NamespacedName{Namespace: watcherTest.Instance.Namespace, Name: SecretName},
+					map[string][]byte{
+						"WatcherPassword": []byte("password"),
+					},
+				))
+
+			// Create Watcher CR after all secrets and dependencies are in place
+			// so sub-CR controllers don't enter long exponential backoff.
+			spec := GetDefaultWatcherSpec()
+			spec["auth"] = map[string]any{"applicationCredentialSecret": acSecretName}
+			DeferCleanup(th.DeleteInstance, CreateWatcher(watcherTest.Instance, spec))
+
+			DeferCleanup(
+				mariadb.DeleteDBService,
+				mariadb.CreateDBService(
+					watcherTest.Instance.Namespace,
+					*GetWatcher(watcherTest.Instance).Spec.DatabaseInstance,
+					corev1.ServiceSpec{
+						Ports: []corev1.ServicePort{{Port: 3306}},
+					},
+				),
+			)
+
+			mariadb.SimulateMariaDBAccountCompleted(watcherTest.WatcherDatabaseAccount)
+			mariadb.SimulateMariaDBDatabaseCompleted(watcherTest.WatcherDatabaseName)
+			infra.SimulateTransportURLReady(watcherTest.WatcherTransportURL)
+
+			keystone.SimulateKeystoneServiceReady(watcherTest.KeystoneServiceName)
+			th.SimulateJobSuccess(watcherTest.WatcherDBSync)
+		})
+
+		It("should add the consumer finalizer to the AC secret", func() {
+			Eventually(func(g Gomega) {
+				secret := th.GetSecret(types.NamespacedName{
+					Namespace: watcherTest.Instance.Namespace,
+					Name:      acSecretName,
+				})
+				g.Expect(secret.Finalizers).To(
+					ContainElement(watcher.ACConsumerFinalizer))
+			}, timeout, interval).Should(Succeed())
+		})
+
+		It("should track the consumed AC secret in status", func() {
+			Eventually(func(g Gomega) {
+				w := GetWatcher(watcherTest.Instance)
+				g.Expect(w.Status.ApplicationCredentialSecret).To(Equal(acSecretName))
+			}, timeout, interval).Should(Succeed())
+		})
+
+		It("should move the finalizer from the old to the new secret on rotation", func() {
+			Eventually(func(g Gomega) {
+				secret := th.GetSecret(types.NamespacedName{
+					Namespace: watcherTest.Instance.Namespace,
+					Name:      acSecretName,
+				})
+				g.Expect(secret.Finalizers).To(
+					ContainElement(watcher.ACConsumerFinalizer))
+			}, timeout, interval).Should(Succeed())
+
+			newACSecretName := "ac-watcher-consumer-rotated-secret" //nolint:gosec
+			newSecret := &corev1.Secret{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: watcherTest.Instance.Namespace,
+					Name:      newACSecretName,
+				},
+				Data: map[string][]byte{
+					keystonev1beta1.ACIDSecretKey:     []byte("rotated-ac-id"),
+					keystonev1beta1.ACSecretSecretKey: []byte("rotated-ac-secret-value"), //nolint:gosec
+				},
+			}
+			DeferCleanup(k8sClient.Delete, ctx, newSecret)
+			Expect(k8sClient.Create(ctx, newSecret)).To(Succeed())
+
+			Eventually(func(g Gomega) {
+				w := GetWatcher(watcherTest.Instance)
+				w.Spec.Auth.ApplicationCredentialSecret = newACSecretName
+				g.Expect(k8sClient.Update(ctx, w)).Should(Succeed())
+			}, timeout, interval).Should(Succeed())
+
+			// New secret gets the consumer finalizer immediately (early in reconcile)
+			Eventually(func(g Gomega) {
+				secret := th.GetSecret(types.NamespacedName{
+					Namespace: watcherTest.Instance.Namespace,
+					Name:      newACSecretName,
+				})
+				g.Expect(secret.Finalizers).To(
+					ContainElement(watcher.ACConsumerFinalizer))
+			}, timeout, interval).Should(Succeed())
+
+			// Old secret keeps the finalizer until all services deploy (split pattern)
+			secret := th.GetSecret(types.NamespacedName{
+				Namespace: watcherTest.Instance.Namespace,
+				Name:      acSecretName,
+			})
+			Expect(secret.Finalizers).To(
+				ContainElement(watcher.ACConsumerFinalizer))
+
+			// Simulate all watcher services deploying successfully
+			th.SimulateStatefulSetReplicaReady(watcherTest.WatcherAPIStatefulSet)
+			keystone.SimulateKeystoneEndpointReady(watcherTest.WatcherKeystoneEndpointName)
+			th.SimulateStatefulSetReplicaReady(watcherTest.WatcherApplierStatefulSet)
+			th.SimulateStatefulSetReplicaReady(watcherTest.WatcherDecisionEngineStatefulSet)
+
+			// Now the old secret's finalizer is removed and status updated
+			Eventually(func(g Gomega) {
+				secret := th.GetSecret(types.NamespacedName{
+					Namespace: watcherTest.Instance.Namespace,
+					Name:      acSecretName,
+				})
+				g.Expect(secret.Finalizers).NotTo(
+					ContainElement(watcher.ACConsumerFinalizer))
+			}, timeout, interval).Should(Succeed())
+
+			Eventually(func(g Gomega) {
+				w := GetWatcher(watcherTest.Instance)
+				g.Expect(w.Status.ApplicationCredentialSecret).To(Equal(newACSecretName))
+			}, timeout, interval).Should(Succeed())
+		})
+
+		It("should remove the consumer finalizer from AC secret on CR deletion", func() {
+			Eventually(func(g Gomega) {
+				secret := th.GetSecret(types.NamespacedName{
+					Namespace: watcherTest.Instance.Namespace,
+					Name:      acSecretName,
+				})
+				g.Expect(secret.Finalizers).To(
+					ContainElement(watcher.ACConsumerFinalizer))
+			}, timeout, interval).Should(Succeed())
+
+			th.DeleteInstance(GetWatcher(watcherTest.Instance))
+
+			secret := th.GetSecret(types.NamespacedName{
+				Namespace: watcherTest.Instance.Namespace,
+				Name:      acSecretName,
+			})
+			Expect(secret.Finalizers).NotTo(
+				ContainElement(watcher.ACConsumerFinalizer))
+		})
+	})
+
 	When("ApplicationCredential is adopted on existing deployment", func() {
 		var appCredSecretName string
 		var appCredID string

test/kuttl/test-suites/default/appcred-tests/02-assert.yaml

@@ -46,6 +46,23 @@ commands:
       fi
       echo "✓ watcher pods restarted after appcred secret became available"
 
+      echo "Checking consumer finalizer on AC secret..."
+      finalizers=$(oc get -n "${NS}" secret/ac-watcher-test-secret -o jsonpath='{.metadata.finalizers}')
+      if [[ "${finalizers}" != *"openstack.org/watcher-ac-consumer"* ]]; then
+        echo "ERROR: AC secret missing watcher consumer finalizer"
+        echo "  finalizers: ${finalizers}"
+        exit 1
+      fi
+      echo "✓ AC secret has openstack.org/watcher-ac-consumer finalizer"
+
+      echo "Checking watcher status tracks the consumed AC secret..."
+      status_ac=$(oc get -n "${NS}" watcher/watcher-kuttl -o jsonpath='{.status.applicationCredentialSecret}')
+      if [ "${status_ac}" != "ac-watcher-test-secret" ]; then
+        echo "ERROR: watcher.status.applicationCredentialSecret expected ac-watcher-test-secret, got ${status_ac}"
+        exit 1
+      fi
+      echo "✓ watcher.status.applicationCredentialSecret = ${status_ac}"
+
       echo "Checking watcher config contains application_credential_id..."
       oc exec -n "${NS}" pod/watcher-kuttl-api-0 -c watcher-api -- \
         bash -c "grep -q \"^application_credential_id = ${ac_id}$\" /etc/watcher/watcher.conf.d/00-default.conf"

test/kuttl/test-suites/default/appcred-tests/03-assert.yaml

@@ -58,6 +58,15 @@ commands:
       fi
       echo "✓ watcher pods restarted after rotation"
 
+      echo "Checking consumer finalizer still present on AC secret after rotation..."
+      finalizers=$(oc get -n "${NS}" secret/ac-watcher-test-secret -o jsonpath='{.metadata.finalizers}')
+      if [[ "${finalizers}" != *"openstack.org/watcher-ac-consumer"* ]]; then
+        echo "ERROR: AC secret lost watcher consumer finalizer after rotation"
+        echo "  finalizers: ${finalizers}"
+        exit 1
+      fi
+      echo "✓ AC secret still has openstack.org/watcher-ac-consumer finalizer after rotation"
+
       echo "Checking watcher config contains rotated application_credential_id..."
       oc exec -n "${NS}" pod/watcher-kuttl-api-0 -c watcher-api -- \
         bash -c "grep -q \"^application_credential_id = ${ac_id}$\" /etc/watcher/watcher.conf.d/00-default.conf"