From 3ee2df90cf2119fbcc66b467ee5c1b791fa8ae09 Mon Sep 17 00:00:00 2001 From: Binh Nguyen Date: Fri, 24 Jul 2026 16:23:35 +0700 Subject: [PATCH] feat(sentinel): configurable deployment strategy and PDB minAvailable Two rollout/availability knobs that were previously hardcoded: - sentinel.strategy overrides the sentinel Deployment update strategy (e.g. rollingUpdate maxSurge/maxUnavailable), avoiding a deadlocked rolling update when required anti-affinity plus replicas==nodes leaves no room to surge. - redis/sentinel.podDisruptionBudgetMinAvailable overrides the PDB minAvailable (previously fixed at 2, or 1 when replicas<=2). Regenerated CRD + deepcopy for the new fields. Refs upstream spotahome/redis-operator#662, #516, #598. --- README.md | 21 ++++ api/redisfailover/v1/types.go | 13 +++ api/redisfailover/v1/zz_generated.deepcopy.go | 12 +++ ...atabases.spotahome.com_redisfailovers.yaml | 67 ++++++++++++ ...atabases.spotahome.com_redisfailovers.yaml | 67 ++++++++++++ ...atabases.spotahome.com_redisfailovers.yaml | 67 ++++++++++++ operator/redisfailover/service/client.go | 9 +- .../service/deployment_strategy_pdb_test.go | 101 ++++++++++++++++++ operator/redisfailover/service/generator.go | 1 + 9 files changed, 355 insertions(+), 3 deletions(-) create mode 100644 operator/redisfailover/service/deployment_strategy_pdb_test.go diff --git a/README.md b/README.md index 68c4c53d8..3c6fd204e 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ Redis Operator creates/configures/manages redis-failovers atop Kubernetes. - [Usage](#usage) - [Reducing update churn (`--enable-hash`)](#reducing-update-churn---enable-hash) - [Reconcile interval (`--resync-period`)](#reconcile-interval---resync-period) + - [Sentinel update strategy and PodDisruptionBudget](#sentinel-update-strategy-and-poddisruptionbudget) - [Persistence](#persistence) - [NodeAffinity and Tolerations](#nodeaffinity-and-tolerations) - [Topology Spread Contraints](#topology-spread-contraints) @@ -174,6 +175,26 @@ interval so it can recover from missed events and correct drift. This defaults t tuned with `--resync-period` (any Go duration, e.g. `--resync-period=1m`). Larger values reduce API load at the cost of slower periodic healing; smaller values react faster but poll more often. +### Sentinel update strategy and PodDisruptionBudget + +The sentinel `Deployment` update strategy can be overridden via `sentinel.strategy` (e.g. to set +`rollingUpdate.maxSurge`/`maxUnavailable`). This helps when required anti-affinity plus +`replicas == nodes` would otherwise deadlock the default rolling update: + +```yaml +spec: + sentinel: + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 +``` + +The `PodDisruptionBudget` `minAvailable` for each component defaults to `2` (or `1` when redis +`replicas <= 2`). Override it per component with `redis.podDisruptionBudgetMinAvailable` / +`sentinel.podDisruptionBudgetMinAvailable` (an integer or percentage string such as `"60%"`). + ### Persistence The operator has the ability of add persistence to Redis data. By default an `emptyDir` will be used, so the data is not saved. diff --git a/api/redisfailover/v1/types.go b/api/redisfailover/v1/types.go index cd3dc667a..6ac903b13 100644 --- a/api/redisfailover/v1/types.go +++ b/api/redisfailover/v1/types.go @@ -1,8 +1,10 @@ package v1 import ( + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" ) // +genclient @@ -71,6 +73,9 @@ type RedisSettings struct { CustomReadinessProbe *corev1.Probe `json:"customReadinessProbe,omitempty"` CustomStartupProbe *corev1.Probe `json:"customStartupProbe,omitempty"` DisablePodDisruptionBudget bool `json:"disablePodDisruptionBudget,omitempty"` + // PodDisruptionBudgetMinAvailable overrides the PodDisruptionBudget + // minAvailable for the redis pods. Defaults to 2 (or 1 when replicas <= 2). + PodDisruptionBudgetMinAvailable *intstr.IntOrString `json:"podDisruptionBudgetMinAvailable,omitempty"` } // SentinelSettings defines the specification of the sentinel cluster @@ -105,6 +110,14 @@ type SentinelSettings struct { CustomReadinessProbe *corev1.Probe `json:"customReadinessProbe,omitempty"` CustomStartupProbe *corev1.Probe `json:"customStartupProbe,omitempty"` DisablePodDisruptionBudget bool `json:"disablePodDisruptionBudget,omitempty"` + // Strategy overrides the sentinel Deployment update strategy (e.g. to set + // rollingUpdate maxSurge/maxUnavailable). Defaults to the Kubernetes default + // RollingUpdate strategy when unset. + Strategy appsv1.DeploymentStrategy `json:"strategy,omitempty"` + // PodDisruptionBudgetMinAvailable overrides the PodDisruptionBudget + // minAvailable for the sentinel pods. Defaults to 2 (or 1 when redis + // replicas <= 2). + PodDisruptionBudgetMinAvailable *intstr.IntOrString `json:"podDisruptionBudgetMinAvailable,omitempty"` } // AuthSettings contains settings about auth diff --git a/api/redisfailover/v1/zz_generated.deepcopy.go b/api/redisfailover/v1/zz_generated.deepcopy.go index acc8b9002..3e61f0dc9 100644 --- a/api/redisfailover/v1/zz_generated.deepcopy.go +++ b/api/redisfailover/v1/zz_generated.deepcopy.go @@ -8,6 +8,7 @@ package v1 import ( corev1 "k8s.io/api/core/v1" runtime "k8s.io/apimachinery/pkg/runtime" + intstr "k8s.io/apimachinery/pkg/util/intstr" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. @@ -354,6 +355,11 @@ func (in *RedisSettings) DeepCopyInto(out *RedisSettings) { *out = new(corev1.Probe) (*in).DeepCopyInto(*out) } + if in.PodDisruptionBudgetMinAvailable != nil { + in, out := &in.PodDisruptionBudgetMinAvailable, &out.PodDisruptionBudgetMinAvailable + *out = new(intstr.IntOrString) + **out = **in + } return } @@ -528,6 +534,12 @@ func (in *SentinelSettings) DeepCopyInto(out *SentinelSettings) { *out = new(corev1.Probe) (*in).DeepCopyInto(*out) } + in.Strategy.DeepCopyInto(&out.Strategy) + if in.PodDisruptionBudgetMinAvailable != nil { + in, out := &in.PodDisruptionBudgetMinAvailable, &out.PodDisruptionBudgetMinAvailable + *out = new(intstr.IntOrString) + **out = **in + } return } diff --git a/charts/redisoperator/crds/databases.spotahome.com_redisfailovers.yaml b/charts/redisoperator/crds/databases.spotahome.com_redisfailovers.yaml index c786d3dc8..21a8b28e6 100644 --- a/charts/redisoperator/crds/databases.spotahome.com_redisfailovers.yaml +++ b/charts/redisoperator/crds/databases.spotahome.com_redisfailovers.yaml @@ -7157,6 +7157,14 @@ spec: additionalProperties: type: string type: object + podDisruptionBudgetMinAvailable: + anyOf: + - type: integer + - type: string + description: |- + PodDisruptionBudgetMinAvailable overrides the PodDisruptionBudget + minAvailable for the redis pods. Defaults to 2 (or 1 when replicas <= 2). + x-kubernetes-int-or-string: true port: format: int32 type: integer @@ -15437,6 +15445,15 @@ spec: additionalProperties: type: string type: object + podDisruptionBudgetMinAvailable: + anyOf: + - type: integer + - type: string + description: |- + PodDisruptionBudgetMinAvailable overrides the PodDisruptionBudget + minAvailable for the sentinel pods. Defaults to 2 (or 1 when redis + replicas <= 2). + x-kubernetes-int-or-string: true priorityClassName: type: string replicas: @@ -15745,6 +15762,56 @@ spec: type: object startupConfigMap: type: string + strategy: + description: |- + Strategy overrides the sentinel Deployment update strategy (e.g. to set + rollingUpdate maxSurge/maxUnavailable). Defaults to the Kubernetes default + RollingUpdate strategy when unset. + properties: + rollingUpdate: + description: |- + Rolling update config params. Present only if DeploymentStrategyType = + RollingUpdate. + properties: + maxSurge: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of pods that can be scheduled above the desired number of + pods. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + This can not be 0 if MaxUnavailable is 0. + Absolute number is calculated from percentage by rounding up. + Defaults to 25%. + Example: when this is set to 30%, the new ReplicaSet can be scaled up immediately when + the rolling update starts, such that the total number of old and new pods do not exceed + 130% of desired pods. Once old pods have been killed, + new ReplicaSet can be scaled up further, ensuring that total number of pods running + at any time during the update is at most 130% of desired pods. + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of pods that can be unavailable during the update. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + Absolute number is calculated from percentage by rounding down. + This can not be 0 if MaxSurge is 0. + Defaults to 25%. + Example: when this is set to 30%, the old ReplicaSet can be scaled down to 70% of desired pods + immediately when the rolling update starts. Once new pods are ready, old ReplicaSet + can be scaled down further, followed by scaling up the new ReplicaSet, ensuring + that the total number of pods available at all times during the update is at + least 70% of desired pods. + x-kubernetes-int-or-string: true + type: object + type: + description: Type of deployment. Can be "Recreate" or "RollingUpdate". + Default is RollingUpdate. + type: string + type: object tolerations: items: description: |- diff --git a/manifests/databases.spotahome.com_redisfailovers.yaml b/manifests/databases.spotahome.com_redisfailovers.yaml index c786d3dc8..21a8b28e6 100644 --- a/manifests/databases.spotahome.com_redisfailovers.yaml +++ b/manifests/databases.spotahome.com_redisfailovers.yaml @@ -7157,6 +7157,14 @@ spec: additionalProperties: type: string type: object + podDisruptionBudgetMinAvailable: + anyOf: + - type: integer + - type: string + description: |- + PodDisruptionBudgetMinAvailable overrides the PodDisruptionBudget + minAvailable for the redis pods. Defaults to 2 (or 1 when replicas <= 2). + x-kubernetes-int-or-string: true port: format: int32 type: integer @@ -15437,6 +15445,15 @@ spec: additionalProperties: type: string type: object + podDisruptionBudgetMinAvailable: + anyOf: + - type: integer + - type: string + description: |- + PodDisruptionBudgetMinAvailable overrides the PodDisruptionBudget + minAvailable for the sentinel pods. Defaults to 2 (or 1 when redis + replicas <= 2). + x-kubernetes-int-or-string: true priorityClassName: type: string replicas: @@ -15745,6 +15762,56 @@ spec: type: object startupConfigMap: type: string + strategy: + description: |- + Strategy overrides the sentinel Deployment update strategy (e.g. to set + rollingUpdate maxSurge/maxUnavailable). Defaults to the Kubernetes default + RollingUpdate strategy when unset. + properties: + rollingUpdate: + description: |- + Rolling update config params. Present only if DeploymentStrategyType = + RollingUpdate. + properties: + maxSurge: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of pods that can be scheduled above the desired number of + pods. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + This can not be 0 if MaxUnavailable is 0. + Absolute number is calculated from percentage by rounding up. + Defaults to 25%. + Example: when this is set to 30%, the new ReplicaSet can be scaled up immediately when + the rolling update starts, such that the total number of old and new pods do not exceed + 130% of desired pods. Once old pods have been killed, + new ReplicaSet can be scaled up further, ensuring that total number of pods running + at any time during the update is at most 130% of desired pods. + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of pods that can be unavailable during the update. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + Absolute number is calculated from percentage by rounding down. + This can not be 0 if MaxSurge is 0. + Defaults to 25%. + Example: when this is set to 30%, the old ReplicaSet can be scaled down to 70% of desired pods + immediately when the rolling update starts. Once new pods are ready, old ReplicaSet + can be scaled down further, followed by scaling up the new ReplicaSet, ensuring + that the total number of pods available at all times during the update is at + least 70% of desired pods. + x-kubernetes-int-or-string: true + type: object + type: + description: Type of deployment. Can be "Recreate" or "RollingUpdate". + Default is RollingUpdate. + type: string + type: object tolerations: items: description: |- diff --git a/manifests/kustomize/base/databases.spotahome.com_redisfailovers.yaml b/manifests/kustomize/base/databases.spotahome.com_redisfailovers.yaml index c786d3dc8..21a8b28e6 100644 --- a/manifests/kustomize/base/databases.spotahome.com_redisfailovers.yaml +++ b/manifests/kustomize/base/databases.spotahome.com_redisfailovers.yaml @@ -7157,6 +7157,14 @@ spec: additionalProperties: type: string type: object + podDisruptionBudgetMinAvailable: + anyOf: + - type: integer + - type: string + description: |- + PodDisruptionBudgetMinAvailable overrides the PodDisruptionBudget + minAvailable for the redis pods. Defaults to 2 (or 1 when replicas <= 2). + x-kubernetes-int-or-string: true port: format: int32 type: integer @@ -15437,6 +15445,15 @@ spec: additionalProperties: type: string type: object + podDisruptionBudgetMinAvailable: + anyOf: + - type: integer + - type: string + description: |- + PodDisruptionBudgetMinAvailable overrides the PodDisruptionBudget + minAvailable for the sentinel pods. Defaults to 2 (or 1 when redis + replicas <= 2). + x-kubernetes-int-or-string: true priorityClassName: type: string replicas: @@ -15745,6 +15762,56 @@ spec: type: object startupConfigMap: type: string + strategy: + description: |- + Strategy overrides the sentinel Deployment update strategy (e.g. to set + rollingUpdate maxSurge/maxUnavailable). Defaults to the Kubernetes default + RollingUpdate strategy when unset. + properties: + rollingUpdate: + description: |- + Rolling update config params. Present only if DeploymentStrategyType = + RollingUpdate. + properties: + maxSurge: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of pods that can be scheduled above the desired number of + pods. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + This can not be 0 if MaxUnavailable is 0. + Absolute number is calculated from percentage by rounding up. + Defaults to 25%. + Example: when this is set to 30%, the new ReplicaSet can be scaled up immediately when + the rolling update starts, such that the total number of old and new pods do not exceed + 130% of desired pods. Once old pods have been killed, + new ReplicaSet can be scaled up further, ensuring that total number of pods running + at any time during the update is at most 130% of desired pods. + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of pods that can be unavailable during the update. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + Absolute number is calculated from percentage by rounding down. + This can not be 0 if MaxSurge is 0. + Defaults to 25%. + Example: when this is set to 30%, the old ReplicaSet can be scaled down to 70% of desired pods + immediately when the rolling update starts. Once new pods are ready, old ReplicaSet + can be scaled down further, followed by scaling up the new ReplicaSet, ensuring + that the total number of pods available at all times during the update is at + least 70% of desired pods. + x-kubernetes-int-or-string: true + type: object + type: + description: Type of deployment. Can be "Recreate" or "RollingUpdate". + Default is RollingUpdate. + type: string + type: object tolerations: items: description: |- diff --git a/operator/redisfailover/service/client.go b/operator/redisfailover/service/client.go index d784e1911..1e9d217b9 100644 --- a/operator/redisfailover/service/client.go +++ b/operator/redisfailover/service/client.go @@ -86,7 +86,7 @@ func (r *RedisFailoverKubeClient) EnsureSentinelConfigMap(rf *redisfailoverv1.Re // EnsureSentinelDeployment makes sure the sentinel deployment exists in the desired state func (r *RedisFailoverKubeClient) EnsureSentinelDeployment(rf *redisfailoverv1.RedisFailover, labels map[string]string, ownerRefs []metav1.OwnerReference) error { if !rf.Spec.Sentinel.DisablePodDisruptionBudget { - if err := r.ensurePodDisruptionBudget(rf, sentinelName, sentinelRoleName, labels, ownerRefs); err != nil { + if err := r.ensurePodDisruptionBudget(rf, sentinelName, sentinelRoleName, rf.Spec.Sentinel.PodDisruptionBudgetMinAvailable, labels, ownerRefs); err != nil { return err } } @@ -100,7 +100,7 @@ func (r *RedisFailoverKubeClient) EnsureSentinelDeployment(rf *redisfailoverv1.R // EnsureRedisStatefulset makes sure the redis statefulset exists in the desired state func (r *RedisFailoverKubeClient) EnsureRedisStatefulset(rf *redisfailoverv1.RedisFailover, labels map[string]string, ownerRefs []metav1.OwnerReference) error { if !rf.Spec.Redis.DisablePodDisruptionBudget { - if err := r.ensurePodDisruptionBudget(rf, redisName, redisRoleName, labels, ownerRefs); err != nil { + if err := r.ensurePodDisruptionBudget(rf, redisName, redisRoleName, rf.Spec.Redis.PodDisruptionBudgetMinAvailable, labels, ownerRefs); err != nil { return err } } @@ -188,7 +188,7 @@ func (r *RedisFailoverKubeClient) EnsureRedisSlaveService(rf *redisfailoverv1.Re } // EnsureRedisStatefulset makes sure the pdb exists in the desired state -func (r *RedisFailoverKubeClient) ensurePodDisruptionBudget(rf *redisfailoverv1.RedisFailover, name string, component string, labels map[string]string, ownerRefs []metav1.OwnerReference) error { +func (r *RedisFailoverKubeClient) ensurePodDisruptionBudget(rf *redisfailoverv1.RedisFailover, name string, component string, minAvailableOverride *intstr.IntOrString, labels map[string]string, ownerRefs []metav1.OwnerReference) error { name = generateName(name, rf.Name) namespace := rf.Namespace @@ -196,6 +196,9 @@ func (r *RedisFailoverKubeClient) ensurePodDisruptionBudget(rf *redisfailoverv1. if rf.Spec.Redis.Replicas <= 2 { minAvailable = intstr.FromInt(1) } + if minAvailableOverride != nil { + minAvailable = *minAvailableOverride + } labels = util.MergeLabels(labels, generateSelectorLabels(component, rf.Name)) diff --git a/operator/redisfailover/service/deployment_strategy_pdb_test.go b/operator/redisfailover/service/deployment_strategy_pdb_test.go new file mode 100644 index 000000000..efac88689 --- /dev/null +++ b/operator/redisfailover/service/deployment_strategy_pdb_test.go @@ -0,0 +1,101 @@ +package service_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + appsv1 "k8s.io/api/apps/v1" + policyv1 "k8s.io/api/policy/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + + "github.com/dnse-tech/redis-operator/log" + "github.com/dnse-tech/redis-operator/metrics" + mK8SService "github.com/dnse-tech/redis-operator/mocks/service/k8s" + rfservice "github.com/dnse-tech/redis-operator/operator/redisfailover/service" +) + +func TestSentinelDeploymentStrategy(t *testing.T) { + assert := assert.New(t) + + maxSurge := intstr.FromInt(1) + maxUnavailable := intstr.FromInt(0) + strategy := appsv1.DeploymentStrategy{ + Type: appsv1.RollingUpdateDeploymentStrategyType, + RollingUpdate: &appsv1.RollingUpdateDeployment{ + MaxSurge: &maxSurge, + MaxUnavailable: &maxUnavailable, + }, + } + + rf := generateRF() + rf.Spec.Sentinel.Strategy = strategy + + var gotStrategy appsv1.DeploymentStrategy + ms := &mK8SService.Services{} + ms.On("CreateOrUpdatePodDisruptionBudget", namespace, mock.Anything).Once().Return(nil, nil) + ms.On("CreateOrUpdateDeployment", namespace, mock.Anything).Once().Run(func(args mock.Arguments) { + gotStrategy = args.Get(1).(*appsv1.Deployment).Spec.Strategy + }).Return(nil) + + client := rfservice.NewRedisFailoverKubeClient(ms, log.Dummy, metrics.Dummy) + err := client.EnsureSentinelDeployment(rf, nil, []metav1.OwnerReference{}) + + assert.NoError(err) + assert.Equal(strategy, gotStrategy) +} + +func TestPodDisruptionBudgetMinAvailableOverride(t *testing.T) { + tests := []struct { + name string + override *intstr.IntOrString + redisReplic int32 + expected intstr.IntOrString + }{ + { + name: "default with >2 replicas is 2", + override: nil, + redisReplic: 3, + expected: intstr.FromInt(2), + }, + { + name: "default with <=2 replicas is 1", + override: nil, + redisReplic: 2, + expected: intstr.FromInt(1), + }, + { + name: "explicit override wins", + override: ptrIOS(intstr.FromString("60%")), + redisReplic: 3, + expected: intstr.FromString("60%"), + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + assert := assert.New(t) + + rf := generateRF() + rf.Spec.Redis.Replicas = test.redisReplic + rf.Spec.Redis.PodDisruptionBudgetMinAvailable = test.override + + var gotMinAvailable *intstr.IntOrString + ms := &mK8SService.Services{} + ms.On("CreateOrUpdatePodDisruptionBudget", namespace, mock.Anything).Once().Run(func(args mock.Arguments) { + gotMinAvailable = args.Get(1).(*policyv1.PodDisruptionBudget).Spec.MinAvailable + }).Return(nil) + ms.On("CreateOrUpdateStatefulSet", namespace, mock.Anything).Once().Return(nil) + + client := rfservice.NewRedisFailoverKubeClient(ms, log.Dummy, metrics.Dummy) + err := client.EnsureRedisStatefulset(rf, nil, []metav1.OwnerReference{}) + + assert.NoError(err) + assert.NotNil(gotMinAvailable) + assert.Equal(test.expected, *gotMinAvailable) + }) + } +} + +func ptrIOS(v intstr.IntOrString) *intstr.IntOrString { return &v } diff --git a/operator/redisfailover/service/generator.go b/operator/redisfailover/service/generator.go index a4547f717..9fb6bcd4e 100644 --- a/operator/redisfailover/service/generator.go +++ b/operator/redisfailover/service/generator.go @@ -568,6 +568,7 @@ func generateSentinelDeployment(rf *redisfailoverv1.RedisFailover, labels map[st }, Spec: appsv1.DeploymentSpec{ Replicas: &rf.Spec.Sentinel.Replicas, + Strategy: rf.Spec.Sentinel.Strategy, Selector: &metav1.LabelSelector{ MatchLabels: selectorLabels, },