From dcac2e22c6832620e82e1a85668c2ac182dbaeda Mon Sep 17 00:00:00 2001 From: Boris Urbanik Date: Tue, 21 Apr 2026 10:27:51 +0100 Subject: [PATCH 1/2] Prevent perpetual reconcile by hardcoding K8s-defaulted fields in component specs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The K8s API server fills zero-value fields with defaults when a resource is written. Mutators using reflect.DeepEqual then see a mismatch between the desired spec (Go zero values) and the live spec (K8s-filled defaults), triggering an update on every reconcile cycle. Explicitly set all K8s-defaulted fields in every component Deployment spec: - Probe fields: Scheme, TimeoutSeconds, PeriodSeconds, SuccessThreshold, FailureThreshold - Container / init-container fields: TerminationMessagePath, TerminationMessagePolicy, ImagePullPolicy (init containers are most sensitive — DeploymentPodInitContainerMutator does a full struct DeepEqual) - Pod spec fields: RestartPolicy, DNSPolicy, SecurityContext, TerminationGracePeriodSeconds, SchedulerName - Volume source fields: DefaultMode on Secret, ConfigMap, and Projected volume sources - Use nil (not []T{}) for optional volume/volumemount slices so reflect.DeepEqual treats K8s-normalised absent and locally-absent the same Extend UpdateResource to log the object's namespace and APIManager owner name as structured fields, enabling the integration test's ReconcileCounter to attribute each Deployment update to the correct CR instance. Add ReconcileCounter and verifyNoDeploymentUpdates to the integration test suite to assert ≤50 total Deployment update calls per APIManager install, providing a regression test for this class of bug. Co-Authored-By: Claude Sonnet 4.6 --- controllers/apps/apimanager_controller.go | 6 +- go.mod | 2 +- pkg/3scale/amp/component/apicast.go | 126 ++++++++----- pkg/3scale/amp/component/backend.go | 145 +++++++++------ .../amp/component/backend_redis_tls_test.go | 11 +- pkg/3scale/amp/component/memcached.go | 34 ++-- pkg/3scale/amp/component/system.go | 162 ++++++++++------- .../amp/component/system_redis_tls_test.go | 7 + pkg/3scale/amp/component/system_searchd.go | 45 +++-- pkg/3scale/amp/component/zync.go | 92 +++++++--- pkg/reconcilers/base_reconciler.go | 7 + .../integration/apimanager_controller_test.go | 32 ++++ test/integration/apimanager_suite_test.go | 36 +++- test/integration/reconcile_counter_test.go | 172 ++++++++++++++++++ ...verify_no_perpetual_reconciliation_test.go | 76 ++++++++ 15 files changed, 726 insertions(+), 227 deletions(-) create mode 100644 test/integration/reconcile_counter_test.go create mode 100644 test/integration/verify_no_perpetual_reconciliation_test.go diff --git a/controllers/apps/apimanager_controller.go b/controllers/apps/apimanager_controller.go index 4e29a63dd..397ee1e5a 100644 --- a/controllers/apps/apimanager_controller.go +++ b/controllers/apps/apimanager_controller.go @@ -160,7 +160,7 @@ func (r *APIManagerReconciler) Reconcile(ctx context.Context, req ctrl.Request) return res, nil } - specResult, specErr := r.reconcileAPIManagerLogic(instance) + specResult, specErr := r.reconcileAPIManagerLogic(r.WithRequest(req), instance) statusResult, statusErr := r.reconcileAPIManagerStatus(instance, preflightChecksError) if statusErr != nil { return ctrl.Result{}, statusErr @@ -402,8 +402,8 @@ func (r *APIManagerReconciler) setAPIManagerDefaults(cr *appsv1alpha1.APIManager return ctrl.Result{Requeue: updated}, err } -func (r *APIManagerReconciler) reconcileAPIManagerLogic(cr *appsv1alpha1.APIManager) (reconcile.Result, error) { - baseAPIManagerLogicReconciler := operator.NewBaseAPIManagerLogicReconciler(r.BaseReconciler, cr) +func (r *APIManagerReconciler) reconcileAPIManagerLogic(b *reconcilers.BaseReconciler, cr *appsv1alpha1.APIManager) (reconcile.Result, error) { + baseAPIManagerLogicReconciler := operator.NewBaseAPIManagerLogicReconciler(b, cr) imageReconciler := operator.NewAMPImagesReconciler(baseAPIManagerLogicReconciler) result, err := imageReconciler.Reconcile() if err != nil || result.Requeue { diff --git a/go.mod b/go.mod index 8003c19be..e01e873f0 100644 --- a/go.mod +++ b/go.mod @@ -28,6 +28,7 @@ require ( github.com/spf13/cobra v1.7.0 github.com/spf13/viper v1.7.0 github.com/stretchr/testify v1.11.1 + go.uber.org/zap v1.26.0 golang.org/x/mod v0.27.0 k8s.io/api v0.29.0 k8s.io/apimachinery v0.29.0 @@ -101,7 +102,6 @@ require ( github.com/subosito/gotenv v1.2.0 // indirect go.mongodb.org/mongo-driver v1.14.0 // indirect go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.26.0 // indirect golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect golang.org/x/net v0.43.0 // indirect golang.org/x/oauth2 v0.16.0 // indirect diff --git a/pkg/3scale/amp/component/apicast.go b/pkg/3scale/amp/component/apicast.go index b2daa6e43..021ae47ee 100644 --- a/pkg/3scale/amp/component/apicast.go +++ b/pkg/3scale/amp/component/apicast.go @@ -17,6 +17,7 @@ import ( policyv1 "k8s.io/api/policy/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -139,36 +140,49 @@ func (apicast *Apicast) StagingDeployment(ctx context.Context, k8sclient client. Annotations: apicast.stagingPodAnnotations(watchedSecretAnnotations), }, Spec: v1.PodSpec{ - Affinity: apicast.Options.StagingAffinity, - Tolerations: apicast.Options.StagingTolerations, - ServiceAccountName: "amp", - Volumes: apicast.stagingVolumes(), + Affinity: apicast.Options.StagingAffinity, + Tolerations: apicast.Options.StagingTolerations, + ServiceAccountName: "amp", + RestartPolicy: v1.RestartPolicyAlways, + DNSPolicy: v1.DNSClusterFirst, + SecurityContext: &v1.PodSecurityContext{}, + TerminationGracePeriodSeconds: ptr.To(int64(v1.DefaultTerminationGracePeriodSeconds)), + SchedulerName: v1.DefaultSchedulerName, + Volumes: apicast.stagingVolumes(), Containers: []v1.Container{ { - Ports: apicast.stagingContainerPorts(), - Env: apicast.buildApicastStagingEnv(), - Image: containerImage, - ImagePullPolicy: v1.PullIfNotPresent, - Name: ApicastStagingName, - Resources: apicast.Options.StagingResourceRequirements, - VolumeMounts: apicast.stagingVolumeMounts(), + Ports: apicast.stagingContainerPorts(), + Env: apicast.buildApicastStagingEnv(), + Image: containerImage, + ImagePullPolicy: v1.PullIfNotPresent, + Name: ApicastStagingName, + Resources: apicast.Options.StagingResourceRequirements, + VolumeMounts: apicast.stagingVolumeMounts(), + TerminationMessagePath: v1.TerminationMessagePathDefault, + TerminationMessagePolicy: v1.TerminationMessageReadFile, LivenessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{HTTPGet: &v1.HTTPGetAction{ - Path: "/status/live", - Port: intstr.FromInt32(8090), + Path: "/status/live", + Port: intstr.FromInt32(8090), + Scheme: v1.URISchemeHTTP, }}, InitialDelaySeconds: 10, TimeoutSeconds: 5, PeriodSeconds: 10, + SuccessThreshold: 1, + FailureThreshold: 3, }, ReadinessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{HTTPGet: &v1.HTTPGetAction{ - Path: "/status/ready", - Port: intstr.FromInt32(8090), + Path: "/status/ready", + Port: intstr.FromInt32(8090), + Scheme: v1.URISchemeHTTP, }}, InitialDelaySeconds: 15, TimeoutSeconds: 5, PeriodSeconds: 30, + SuccessThreshold: 1, + FailureThreshold: 3, }, }, }, @@ -219,15 +233,23 @@ func (apicast *Apicast) ProductionDeployment(ctx context.Context, k8sclient clie Annotations: apicast.productionPodAnnotations(watchedSecretAnnotations), }, Spec: v1.PodSpec{ - Affinity: apicast.Options.ProductionAffinity, - Tolerations: apicast.Options.ProductionTolerations, - ServiceAccountName: "amp", - Volumes: apicast.productionVolumes(), + Affinity: apicast.Options.ProductionAffinity, + Tolerations: apicast.Options.ProductionTolerations, + ServiceAccountName: "amp", + RestartPolicy: v1.RestartPolicyAlways, + DNSPolicy: v1.DNSClusterFirst, + SecurityContext: &v1.PodSecurityContext{}, + TerminationGracePeriodSeconds: ptr.To(int64(v1.DefaultTerminationGracePeriodSeconds)), + SchedulerName: v1.DefaultSchedulerName, + Volumes: apicast.productionVolumes(), InitContainers: []v1.Container{ { - Name: ApicastProductionInitContainerName, - Image: containerImage, - Command: []string{"sh", "-c", "until $(curl --output /dev/null --silent --fail --head http://system-master:3000/status); do sleep $SLEEP_SECONDS; done"}, + Name: ApicastProductionInitContainerName, + Image: containerImage, + ImagePullPolicy: v1.PullIfNotPresent, + Command: []string{"sh", "-c", "until $(curl --output /dev/null --silent --fail --head http://system-master:3000/status); do sleep $SLEEP_SECONDS; done"}, + TerminationMessagePath: v1.TerminationMessagePathDefault, + TerminationMessagePolicy: v1.TerminationMessageReadFile, Env: []v1.EnvVar{ { Name: "SLEEP_SECONDS", @@ -238,30 +260,38 @@ func (apicast *Apicast) ProductionDeployment(ctx context.Context, k8sclient clie }, Containers: []v1.Container{ { - Ports: apicast.productionContainerPorts(), - Env: apicast.buildApicastProductionEnv(), - Image: containerImage, - ImagePullPolicy: v1.PullIfNotPresent, - Name: ApicastProductionName, - Resources: apicast.Options.ProductionResourceRequirements, - VolumeMounts: apicast.productionVolumeMounts(), + Ports: apicast.productionContainerPorts(), + Env: apicast.buildApicastProductionEnv(), + Image: containerImage, + ImagePullPolicy: v1.PullIfNotPresent, + Name: ApicastProductionName, + Resources: apicast.Options.ProductionResourceRequirements, + VolumeMounts: apicast.productionVolumeMounts(), + TerminationMessagePath: v1.TerminationMessagePathDefault, + TerminationMessagePolicy: v1.TerminationMessageReadFile, LivenessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{HTTPGet: &v1.HTTPGetAction{ - Path: "/status/live", - Port: intstr.FromInt32(8090), + Path: "/status/live", + Port: intstr.FromInt32(8090), + Scheme: v1.URISchemeHTTP, }}, InitialDelaySeconds: 10, TimeoutSeconds: 5, PeriodSeconds: 10, + SuccessThreshold: 1, + FailureThreshold: 3, }, ReadinessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{HTTPGet: &v1.HTTPGetAction{ - Path: "/status/ready", - Port: intstr.FromInt32(8090), + Path: "/status/ready", + Port: intstr.FromInt32(8090), + Scheme: v1.URISchemeHTTP, }}, InitialDelaySeconds: 15, TimeoutSeconds: 5, PeriodSeconds: 30, + SuccessThreshold: 1, + FailureThreshold: 3, }, }, }, @@ -618,7 +648,8 @@ func (apicast *Apicast) productionVolumes() []v1.Volume { Name: customPolicy.VolumeName(), VolumeSource: v1.VolumeSource{ Secret: &v1.SecretVolumeSource{ - SecretName: customPolicy.Secret.Name, + SecretName: customPolicy.Secret.Name, + DefaultMode: ptr.To(v1.SecretVolumeSourceDefaultMode), }, }, }) @@ -636,6 +667,7 @@ func (apicast *Apicast) productionVolumes() []v1.Volume { Path: apicast.Options.ProductionTracingConfig.VolumeName(), }, }, + DefaultMode: ptr.To(v1.SecretVolumeSourceDefaultMode), }, }, }) @@ -646,8 +678,9 @@ func (apicast *Apicast) productionVolumes() []v1.Volume { Name: OpentelemetryConfigurationVolumeName, VolumeSource: v1.VolumeSource{ Secret: &v1.SecretVolumeSource{ - SecretName: apicast.Options.ProductionOpentelemetry.Secret.Name, - Optional: &[]bool{false}[0], + SecretName: apicast.Options.ProductionOpentelemetry.Secret.Name, + Optional: &[]bool{false}[0], + DefaultMode: ptr.To(v1.SecretVolumeSourceDefaultMode), }, }, }) @@ -658,7 +691,8 @@ func (apicast *Apicast) productionVolumes() []v1.Volume { Name: customEnvVolumeName(customEnvSecret), VolumeSource: v1.VolumeSource{ Secret: &v1.SecretVolumeSource{ - SecretName: customEnvSecret.GetName(), + SecretName: customEnvSecret.GetName(), + DefaultMode: ptr.To(v1.SecretVolumeSourceDefaultMode), }, }, }) @@ -669,7 +703,8 @@ func (apicast *Apicast) productionVolumes() []v1.Volume { Name: HTTPSCertificatesVolumeName, VolumeSource: v1.VolumeSource{ Secret: &v1.SecretVolumeSource{ - SecretName: *apicast.Options.ProductionHTTPSCertificateSecretName, + SecretName: *apicast.Options.ProductionHTTPSCertificateSecretName, + DefaultMode: ptr.To(v1.SecretVolumeSourceDefaultMode), }, }, }) @@ -686,7 +721,8 @@ func (apicast *Apicast) stagingVolumes() []v1.Volume { Name: customPolicy.VolumeName(), VolumeSource: v1.VolumeSource{ Secret: &v1.SecretVolumeSource{ - SecretName: customPolicy.Secret.Name, + SecretName: customPolicy.Secret.Name, + DefaultMode: ptr.To(v1.SecretVolumeSourceDefaultMode), }, }, }) @@ -704,6 +740,7 @@ func (apicast *Apicast) stagingVolumes() []v1.Volume { Path: apicast.Options.StagingTracingConfig.VolumeName(), }, }, + DefaultMode: ptr.To(v1.SecretVolumeSourceDefaultMode), }, }, }) @@ -714,8 +751,9 @@ func (apicast *Apicast) stagingVolumes() []v1.Volume { Name: OpentelemetryConfigurationVolumeName, VolumeSource: v1.VolumeSource{ Secret: &v1.SecretVolumeSource{ - SecretName: apicast.Options.StagingOpentelemetry.Secret.Name, - Optional: &[]bool{false}[0], + SecretName: apicast.Options.StagingOpentelemetry.Secret.Name, + Optional: &[]bool{false}[0], + DefaultMode: ptr.To(v1.SecretVolumeSourceDefaultMode), }, }, }) @@ -726,7 +764,8 @@ func (apicast *Apicast) stagingVolumes() []v1.Volume { Name: customEnvVolumeName(customEnvSecret), VolumeSource: v1.VolumeSource{ Secret: &v1.SecretVolumeSource{ - SecretName: customEnvSecret.GetName(), + SecretName: customEnvSecret.GetName(), + DefaultMode: ptr.To(v1.SecretVolumeSourceDefaultMode), }, }, }) @@ -737,7 +776,8 @@ func (apicast *Apicast) stagingVolumes() []v1.Volume { Name: HTTPSCertificatesVolumeName, VolumeSource: v1.VolumeSource{ Secret: &v1.SecretVolumeSource{ - SecretName: *apicast.Options.StagingHTTPSCertificateSecretName, + SecretName: *apicast.Options.StagingHTTPSCertificateSecretName, + DefaultMode: ptr.To(v1.SecretVolumeSourceDefaultMode), }, }, }) diff --git a/pkg/3scale/amp/component/backend.go b/pkg/3scale/amp/component/backend.go index 0722ac353..30b6d524e 100644 --- a/pkg/3scale/amp/component/backend.go +++ b/pkg/3scale/amp/component/backend.go @@ -15,6 +15,7 @@ import ( policyv1 "k8s.io/api/policy/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" ) const ( @@ -130,33 +131,43 @@ func (backend *Backend) WorkerDeployment(ctx context.Context, k8sclient client.C Annotations: deploymentAnnotations, }, Spec: v1.PodSpec{ - Affinity: backend.Options.WorkerAffinity, - Tolerations: backend.Options.WorkerTolerations, - Volumes: backend.backendVolumes(), + Affinity: backend.Options.WorkerAffinity, + Tolerations: backend.Options.WorkerTolerations, + RestartPolicy: v1.RestartPolicyAlways, + DNSPolicy: v1.DNSClusterFirst, + SecurityContext: &v1.PodSecurityContext{}, + TerminationGracePeriodSeconds: ptr.To(int64(v1.DefaultTerminationGracePeriodSeconds)), + SchedulerName: v1.DefaultSchedulerName, + Volumes: backend.backendVolumes(), InitContainers: []v1.Container{ { - Name: "backend-redis-svc", - Image: containerImage, + Name: "backend-redis-svc", + Image: containerImage, + ImagePullPolicy: v1.PullIfNotPresent, Command: []string{ "/opt/app/entrypoint.sh", "sh", "-c", "until rake connectivity:redis_storage_queue_check; do sleep $SLEEP_SECONDS; done", }, - VolumeMounts: backend.backendContainerVolumeMounts(), - Env: append(backend.buildBackendCommonEnv(), helper.EnvVarFromValue("SLEEP_SECONDS", "1")), + VolumeMounts: backend.backendContainerVolumeMounts(), + Env: append(backend.buildBackendCommonEnv(), helper.EnvVarFromValue("SLEEP_SECONDS", "1")), + TerminationMessagePath: v1.TerminationMessagePathDefault, + TerminationMessagePolicy: v1.TerminationMessageReadFile, }, }, Containers: []v1.Container{ { - Name: BackendWorkerName, - Image: containerImage, - Args: []string{"bin/3scale_backend_worker", "run"}, - Env: backend.buildBackendWorkerEnv(), - Resources: backend.Options.WorkerResourceRequirements, - VolumeMounts: backend.backendContainerVolumeMounts(), - ImagePullPolicy: v1.PullIfNotPresent, - Ports: backend.workerPorts(), + Name: BackendWorkerName, + Image: containerImage, + Args: []string{"bin/3scale_backend_worker", "run"}, + Env: backend.buildBackendWorkerEnv(), + Resources: backend.Options.WorkerResourceRequirements, + VolumeMounts: backend.backendContainerVolumeMounts(), + ImagePullPolicy: v1.PullIfNotPresent, + Ports: backend.workerPorts(), + TerminationMessagePath: v1.TerminationMessagePathDefault, + TerminationMessagePolicy: v1.TerminationMessageReadFile, LivenessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{ HTTPGet: &v1.HTTPGetAction{ @@ -222,32 +233,42 @@ func (backend *Backend) CronDeployment(ctx context.Context, k8sclient client.Cli Annotations: deploymentAnnotations, }, Spec: v1.PodSpec{ - Affinity: backend.Options.CronAffinity, - Tolerations: backend.Options.CronTolerations, - Volumes: backend.backendVolumes(), + Affinity: backend.Options.CronAffinity, + Tolerations: backend.Options.CronTolerations, + RestartPolicy: v1.RestartPolicyAlways, + DNSPolicy: v1.DNSClusterFirst, + SecurityContext: &v1.PodSecurityContext{}, + TerminationGracePeriodSeconds: ptr.To(int64(v1.DefaultTerminationGracePeriodSeconds)), + SchedulerName: v1.DefaultSchedulerName, + Volumes: backend.backendVolumes(), InitContainers: []v1.Container{ { - Name: "backend-redis-svc", - Image: containerImage, + Name: "backend-redis-svc", + Image: containerImage, + ImagePullPolicy: v1.PullIfNotPresent, Command: []string{ "/opt/app/entrypoint.sh", "sh", "-c", "until rake connectivity:redis_storage_queue_check; do sleep $SLEEP_SECONDS; done", }, - VolumeMounts: backend.backendContainerVolumeMounts(), - Env: append(backend.buildBackendCommonEnv(), helper.EnvVarFromValue("SLEEP_SECONDS", "1")), + VolumeMounts: backend.backendContainerVolumeMounts(), + Env: append(backend.buildBackendCommonEnv(), helper.EnvVarFromValue("SLEEP_SECONDS", "1")), + TerminationMessagePath: v1.TerminationMessagePathDefault, + TerminationMessagePolicy: v1.TerminationMessageReadFile, }, }, Containers: []v1.Container{ { - Name: "backend-cron", - Image: containerImage, - Args: []string{"touch /tmp/healthy && backend-cron"}, - Env: backend.buildBackendCronEnv(), - VolumeMounts: backend.backendContainerVolumeMounts(), - Resources: backend.Options.CronResourceRequirements, - ImagePullPolicy: v1.PullIfNotPresent, + Name: "backend-cron", + Image: containerImage, + Args: []string{"touch /tmp/healthy && backend-cron"}, + Env: backend.buildBackendCronEnv(), + VolumeMounts: backend.backendContainerVolumeMounts(), + Resources: backend.Options.CronResourceRequirements, + ImagePullPolicy: v1.PullIfNotPresent, + TerminationMessagePath: v1.TerminationMessagePathDefault, + TerminationMessagePolicy: v1.TerminationMessageReadFile, LivenessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{ Exec: &v1.ExecAction{ @@ -256,6 +277,9 @@ func (backend *Backend) CronDeployment(ctx context.Context, k8sclient client.Cli }, InitialDelaySeconds: 30, PeriodSeconds: 5, + TimeoutSeconds: 1, + SuccessThreshold: 1, + FailureThreshold: 3, }, }, }, @@ -308,18 +332,26 @@ func (backend *Backend) ListenerDeployment(ctx context.Context, k8sclient client Annotations: deploymentAnnotations, }, Spec: v1.PodSpec{ - Affinity: backend.Options.ListenerAffinity, - Tolerations: backend.Options.ListenerTolerations, - Volumes: backend.backendVolumes(), + Affinity: backend.Options.ListenerAffinity, + Tolerations: backend.Options.ListenerTolerations, + RestartPolicy: v1.RestartPolicyAlways, + DNSPolicy: v1.DNSClusterFirst, + SecurityContext: &v1.PodSecurityContext{}, + TerminationGracePeriodSeconds: ptr.To(int64(v1.DefaultTerminationGracePeriodSeconds)), + SchedulerName: v1.DefaultSchedulerName, + Volumes: backend.backendVolumes(), Containers: []v1.Container{ { - Name: BackendListenerName, - Image: containerImage, - Args: backend.backendListenerRunArgs(), - Ports: backend.listenerPorts(), - Env: backend.buildBackendListenerEnv(), - Resources: backend.Options.ListenerResourceRequirements, - VolumeMounts: backend.backendContainerVolumeMounts(), + Name: BackendListenerName, + Image: containerImage, + ImagePullPolicy: v1.PullIfNotPresent, + Args: backend.backendListenerRunArgs(), + Ports: backend.listenerPorts(), + Env: backend.buildBackendListenerEnv(), + Resources: backend.Options.ListenerResourceRequirements, + VolumeMounts: backend.backendContainerVolumeMounts(), + TerminationMessagePath: v1.TerminationMessagePathDefault, + TerminationMessagePolicy: v1.TerminationMessageReadFile, LivenessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{ TCPSocket: &v1.TCPSocketAction{ @@ -330,28 +362,25 @@ func (backend *Backend) ListenerDeployment(ctx context.Context, k8sclient client }, }, InitialDelaySeconds: 30, - TimeoutSeconds: 0, + TimeoutSeconds: 1, PeriodSeconds: 10, - SuccessThreshold: 0, - FailureThreshold: 0, + SuccessThreshold: 1, + FailureThreshold: 3, }, ReadinessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{ HTTPGet: &v1.HTTPGetAction{ - Path: "/status", - Port: intstr.IntOrString{ - Type: intstr.Int, - IntVal: 3000, - }, + Path: "/status", + Port: intstr.IntOrString{Type: intstr.Int, IntVal: 3000}, + Scheme: v1.URISchemeHTTP, }, }, InitialDelaySeconds: 30, TimeoutSeconds: 5, - PeriodSeconds: 0, - SuccessThreshold: 0, - FailureThreshold: 0, + PeriodSeconds: 10, + SuccessThreshold: 1, + FailureThreshold: 3, }, - ImagePullPolicy: v1.PullIfNotPresent, }, }, ServiceAccountName: "amp", @@ -668,7 +697,7 @@ func (backend *Backend) QueuesRedisTLSEnvVars() []v1.EnvVar { } func (backend *Backend) backendVolumes() []v1.Volume { - res := []v1.Volume{} + var res []v1.Volume if backend.Options.BackendRedisTLS.Enabled { items := []v1.KeyToPath{} if backend.Options.BackendRedisTLS.HasCA() { @@ -682,8 +711,9 @@ func (backend *Backend) backendVolumes() []v1.Volume { Name: "backend-redis-tls", VolumeSource: v1.VolumeSource{ Secret: &v1.SecretVolumeSource{ - SecretName: BackendSecretBackendRedisSecretName, - Items: items, + SecretName: BackendSecretBackendRedisSecretName, + Items: items, + DefaultMode: ptr.To(v1.SecretVolumeSourceDefaultMode), }, }, } @@ -703,8 +733,9 @@ func (backend *Backend) backendVolumes() []v1.Volume { Name: "queues-redis-tls", VolumeSource: v1.VolumeSource{ Secret: &v1.SecretVolumeSource{ - SecretName: BackendSecretBackendRedisSecretName, - Items: items, + SecretName: BackendSecretBackendRedisSecretName, + Items: items, + DefaultMode: ptr.To(v1.SecretVolumeSourceDefaultMode), }, }, } @@ -725,7 +756,7 @@ func (backend *Backend) backendListenerRunArgs() []string { } func (backend *Backend) backendContainerVolumeMounts() []v1.VolumeMount { - res := []v1.VolumeMount{} + var res []v1.VolumeMount if backend.Options.BackendRedisTLS.Enabled { res = append(res, backend.backendRedisContainerVolumeMounts()) } diff --git a/pkg/3scale/amp/component/backend_redis_tls_test.go b/pkg/3scale/amp/component/backend_redis_tls_test.go index 5b0c49298..732a0f9cd 100644 --- a/pkg/3scale/amp/component/backend_redis_tls_test.go +++ b/pkg/3scale/amp/component/backend_redis_tls_test.go @@ -5,6 +5,7 @@ import ( "github.com/google/go-cmp/cmp" v1 "k8s.io/api/core/v1" + "k8s.io/utils/ptr" "github.com/3scale/3scale-operator/pkg/helper" ) @@ -163,7 +164,7 @@ func TestBackendComponentRedisTLSVolumes(t *testing.T) { BackendRedisTLS: TLSConfig{Enabled: false}, BackendRedisQueuesTLS: TLSConfig{Enabled: false}, }, - []v1.Volume{}, + nil, }, { "StorageOnly_OneWayTLS", @@ -183,6 +184,7 @@ func TestBackendComponentRedisTLSVolumes(t *testing.T) { Items: []v1.KeyToPath{ {Key: "REDIS_SSL_CA", Path: "backend-redis-ca.crt"}, }, + DefaultMode: ptr.To(v1.SecretVolumeSourceDefaultMode), }, }, }, @@ -206,6 +208,7 @@ func TestBackendComponentRedisTLSVolumes(t *testing.T) { Items: []v1.KeyToPath{ {Key: "REDIS_SSL_QUEUES_CA", Path: "backend-redis-queues-ca.crt"}, }, + DefaultMode: ptr.To(v1.SecretVolumeSourceDefaultMode), }, }, }, @@ -238,6 +241,7 @@ func TestBackendComponentRedisTLSVolumes(t *testing.T) { {Key: "REDIS_SSL_CERT", Path: "backend-redis-client.crt"}, {Key: "REDIS_SSL_KEY", Path: "backend-redis-private.key"}, }, + DefaultMode: ptr.To(v1.SecretVolumeSourceDefaultMode), }, }, }, @@ -251,6 +255,7 @@ func TestBackendComponentRedisTLSVolumes(t *testing.T) { {Key: "REDIS_SSL_QUEUES_CERT", Path: "backend-redis-queues-client.crt"}, {Key: "REDIS_SSL_QUEUES_KEY", Path: "backend-redis-queues-private.key"}, }, + DefaultMode: ptr.To(v1.SecretVolumeSourceDefaultMode), }, }, }, @@ -281,6 +286,7 @@ func TestBackendComponentRedisTLSVolumes(t *testing.T) { {Key: "REDIS_SSL_CERT", Path: "backend-redis-client.crt"}, {Key: "REDIS_SSL_KEY", Path: "backend-redis-private.key"}, }, + DefaultMode: ptr.To(v1.SecretVolumeSourceDefaultMode), }, }, }, @@ -292,6 +298,7 @@ func TestBackendComponentRedisTLSVolumes(t *testing.T) { Items: []v1.KeyToPath{ {Key: "REDIS_SSL_QUEUES_CA", Path: "backend-redis-queues-ca.crt"}, }, + DefaultMode: ptr.To(v1.SecretVolumeSourceDefaultMode), }, }, }, @@ -322,7 +329,7 @@ func TestBackendComponentRedisTLSVolumeMounts(t *testing.T) { BackendRedisTLS: TLSConfig{Enabled: false}, BackendRedisQueuesTLS: TLSConfig{Enabled: false}, }, - []v1.VolumeMount{}, + nil, }, { "StorageOnly", diff --git a/pkg/3scale/amp/component/memcached.go b/pkg/3scale/amp/component/memcached.go index 3e413b58c..be44946ae 100644 --- a/pkg/3scale/amp/component/memcached.go +++ b/pkg/3scale/amp/component/memcached.go @@ -7,6 +7,7 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" ) const ( @@ -47,14 +48,20 @@ func (m *Memcached) Deployment(containerImage string) *k8sappsv1.Deployment { Annotations: m.Options.PodTemplateAnnotations, }, Spec: v1.PodSpec{ - Affinity: m.Options.Affinity, - Tolerations: m.Options.Tolerations, - ServiceAccountName: "amp", // TODO make this configurable via flag + Affinity: m.Options.Affinity, + Tolerations: m.Options.Tolerations, + ServiceAccountName: "amp", // TODO make this configurable via flag + RestartPolicy: v1.RestartPolicyAlways, + DNSPolicy: v1.DNSClusterFirst, + SecurityContext: &v1.PodSecurityContext{}, + TerminationGracePeriodSeconds: ptr.To(int64(v1.DefaultTerminationGracePeriodSeconds)), + SchedulerName: v1.DefaultSchedulerName, Containers: []v1.Container{ { - Name: "memcache", - Image: containerImage, - Command: []string{"memcached", "-m", "64"}, + Name: "memcache", + Image: containerImage, + ImagePullPolicy: v1.PullIfNotPresent, + Command: []string{"memcached", "-m", "64"}, Ports: []v1.ContainerPort{ { HostPort: 0, @@ -62,7 +69,9 @@ func (m *Memcached) Deployment(containerImage string) *k8sappsv1.Deployment { Protocol: v1.ProtocolTCP, }, }, - Resources: m.Options.ResourceRequirements, + Resources: m.Options.ResourceRequirements, + TerminationMessagePath: v1.TerminationMessagePathDefault, + TerminationMessagePolicy: v1.TerminationMessageReadFile, LivenessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{ TCPSocket: &v1.TCPSocketAction{ @@ -73,10 +82,10 @@ func (m *Memcached) Deployment(containerImage string) *k8sappsv1.Deployment { }, }, InitialDelaySeconds: 10, - TimeoutSeconds: 0, + TimeoutSeconds: 1, PeriodSeconds: 10, - SuccessThreshold: 0, - FailureThreshold: 0, + SuccessThreshold: 1, + FailureThreshold: 3, }, ReadinessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{ @@ -90,10 +99,9 @@ func (m *Memcached) Deployment(containerImage string) *k8sappsv1.Deployment { InitialDelaySeconds: 10, TimeoutSeconds: 5, PeriodSeconds: 30, - SuccessThreshold: 0, - FailureThreshold: 0, + SuccessThreshold: 1, + FailureThreshold: 3, }, - ImagePullPolicy: v1.PullIfNotPresent, }, }, PriorityClassName: m.Options.PriorityClassName, diff --git a/pkg/3scale/amp/component/system.go b/pkg/3scale/amp/component/system.go index e06777d64..d5188f395 100644 --- a/pkg/3scale/amp/component/system.go +++ b/pkg/3scale/amp/component/system.go @@ -14,6 +14,7 @@ import ( policyv1 "k8s.io/api/policy/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" "github.com/3scale/3scale-operator/apis/apps" "github.com/3scale/3scale-operator/pkg/helper" @@ -569,6 +570,7 @@ func (system *System) appPodVolumes() []v1.Volume { Path: "service_discovery.yml", }, }, + DefaultMode: ptr.To(v1.ConfigMapVolumeSourceDefaultMode), }, }, } @@ -594,6 +596,7 @@ func (system *System) appPodVolumes() []v1.Volume { Path: "tls.key", // Map the secret key to the tls.key file in the container }, }, + DefaultMode: ptr.To(v1.SecretVolumeSourceDefaultMode), }, }, } @@ -624,6 +627,7 @@ func (system *System) appPodVolumes() []v1.Volume { }, }, }, + DefaultMode: ptr.To(v1.ProjectedVolumeSourceDefaultMode), }, }, } @@ -672,19 +676,27 @@ func (system *System) AppDeployment(ctx context.Context, k8sclient client.Client Annotations: system.appPodAnnotations(watchedSecretAnnotations), }, Spec: v1.PodSpec{ - Affinity: system.Options.AppAffinity, - Tolerations: system.Options.AppTolerations, - Volumes: system.appPodVolumes(), - InitContainers: system.systemInit(containerImage), + Affinity: system.Options.AppAffinity, + Tolerations: system.Options.AppTolerations, + RestartPolicy: v1.RestartPolicyAlways, + DNSPolicy: v1.DNSClusterFirst, + SecurityContext: &v1.PodSecurityContext{}, + TerminationGracePeriodSeconds: ptr.To(int64(v1.DefaultTerminationGracePeriodSeconds)), + SchedulerName: v1.DefaultSchedulerName, + Volumes: system.appPodVolumes(), + InitContainers: system.systemInit(containerImage), Containers: []v1.Container{ { - Name: SystemAppMasterContainerName, - Image: containerImage, - Args: []string{"env", "TENANT_MODE=master", "PORT=3002", "container-entrypoint", "bundle", "exec", "unicorn", "-c", "config/unicorn.rb", "-E", "production", "config.ru"}, - Ports: system.appMasterPorts(), - Env: system.buildAppMasterContainerEnv(), - Resources: *system.Options.AppMasterContainerResourceRequirements, - VolumeMounts: system.appMasterContainerVolumeMounts(), + Name: SystemAppMasterContainerName, + Image: containerImage, + ImagePullPolicy: v1.PullIfNotPresent, + Args: []string{"env", "TENANT_MODE=master", "PORT=3002", "container-entrypoint", "bundle", "exec", "unicorn", "-c", "config/unicorn.rb", "-E", "production", "config.ru"}, + Ports: system.appMasterPorts(), + Env: system.buildAppMasterContainerEnv(), + Resources: *system.Options.AppMasterContainerResourceRequirements, + VolumeMounts: system.appMasterContainerVolumeMounts(), + TerminationMessagePath: v1.TerminationMessagePathDefault, + TerminationMessagePolicy: v1.TerminationMessageReadFile, LivenessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{ TCPSocket: &v1.TCPSocketAction{ @@ -697,7 +709,7 @@ func (system *System) AppDeployment(ctx context.Context, k8sclient client.Client InitialDelaySeconds: 40, TimeoutSeconds: 10, PeriodSeconds: 10, - SuccessThreshold: 0, + SuccessThreshold: 1, FailureThreshold: 40, }, ReadinessProbe: &v1.Probe{ @@ -720,22 +732,24 @@ func (system *System) AppDeployment(ctx context.Context, k8sclient client.Client InitialDelaySeconds: 60, TimeoutSeconds: 10, PeriodSeconds: 30, - SuccessThreshold: 0, + SuccessThreshold: 1, FailureThreshold: 10, }, - ImagePullPolicy: v1.PullIfNotPresent, - Stdin: false, - StdinOnce: false, - TTY: false, + Stdin: false, + StdinOnce: false, + TTY: false, }, { - Name: SystemAppProviderContainerName, - Image: containerImage, - Args: []string{"env", "TENANT_MODE=provider", "PORT=3000", "container-entrypoint", "bundle", "exec", "unicorn", "-c", "config/unicorn.rb", "-E", "production", "config.ru"}, - Ports: system.appProviderPorts(), - Env: system.buildAppProviderContainerEnv(), - Resources: *system.Options.AppProviderContainerResourceRequirements, - VolumeMounts: system.appProviderContainerVolumeMounts(), + Name: SystemAppProviderContainerName, + Image: containerImage, + ImagePullPolicy: v1.PullIfNotPresent, + Args: []string{"env", "TENANT_MODE=provider", "PORT=3000", "container-entrypoint", "bundle", "exec", "unicorn", "-c", "config/unicorn.rb", "-E", "production", "config.ru"}, + Ports: system.appProviderPorts(), + Env: system.buildAppProviderContainerEnv(), + Resources: *system.Options.AppProviderContainerResourceRequirements, + VolumeMounts: system.appProviderContainerVolumeMounts(), + TerminationMessagePath: v1.TerminationMessagePathDefault, + TerminationMessagePolicy: v1.TerminationMessageReadFile, LivenessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{ TCPSocket: &v1.TCPSocketAction{ @@ -748,7 +762,7 @@ func (system *System) AppDeployment(ctx context.Context, k8sclient client.Client InitialDelaySeconds: 40, TimeoutSeconds: 10, PeriodSeconds: 10, - SuccessThreshold: 0, + SuccessThreshold: 1, FailureThreshold: 40, }, ReadinessProbe: &v1.Probe{ @@ -771,22 +785,24 @@ func (system *System) AppDeployment(ctx context.Context, k8sclient client.Client InitialDelaySeconds: 60, TimeoutSeconds: 10, PeriodSeconds: 30, - SuccessThreshold: 0, + SuccessThreshold: 1, FailureThreshold: 10, }, - ImagePullPolicy: v1.PullIfNotPresent, - Stdin: false, - StdinOnce: false, - TTY: false, + Stdin: false, + StdinOnce: false, + TTY: false, }, { - Name: SystemAppDeveloperContainerName, - Image: containerImage, - Args: []string{"env", "PORT=3001", "container-entrypoint", "bundle", "exec", "unicorn", "-c", "config/unicorn.rb", "-E", "production", "config.ru"}, - Ports: system.appDeveloperPorts(), - Env: system.buildAppDeveloperContainerEnv(), - Resources: *system.Options.AppDeveloperContainerResourceRequirements, - VolumeMounts: system.appDeveloperContainerVolumeMounts(), + Name: SystemAppDeveloperContainerName, + Image: containerImage, + ImagePullPolicy: v1.PullIfNotPresent, + Args: []string{"env", "PORT=3001", "container-entrypoint", "bundle", "exec", "unicorn", "-c", "config/unicorn.rb", "-E", "production", "config.ru"}, + Ports: system.appDeveloperPorts(), + Env: system.buildAppDeveloperContainerEnv(), + Resources: *system.Options.AppDeveloperContainerResourceRequirements, + VolumeMounts: system.appDeveloperContainerVolumeMounts(), + TerminationMessagePath: v1.TerminationMessagePathDefault, + TerminationMessagePolicy: v1.TerminationMessageReadFile, LivenessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{ TCPSocket: &v1.TCPSocketAction{ @@ -799,7 +815,7 @@ func (system *System) AppDeployment(ctx context.Context, k8sclient client.Client InitialDelaySeconds: 40, TimeoutSeconds: 10, PeriodSeconds: 10, - SuccessThreshold: 0, + SuccessThreshold: 1, FailureThreshold: 40, }, ReadinessProbe: &v1.Probe{ @@ -822,10 +838,9 @@ func (system *System) AppDeployment(ctx context.Context, k8sclient client.Client InitialDelaySeconds: 60, TimeoutSeconds: 10, PeriodSeconds: 30, - SuccessThreshold: 0, + SuccessThreshold: 1, FailureThreshold: 10, }, - ImagePullPolicy: v1.PullIfNotPresent, }, }, ServiceAccountName: "amp", @@ -962,6 +977,7 @@ func (system *System) SidekiqPodVolumes() []v1.Volume { Path: "service_discovery.yml", }, }, + DefaultMode: ptr.To(v1.ConfigMapVolumeSourceDefaultMode), }, }, } @@ -988,6 +1004,7 @@ func (system *System) SidekiqPodVolumes() []v1.Volume { Path: "tls.key", // Map the secret key to the tls.key file in the container }, }, + DefaultMode: ptr.To(v1.SecretVolumeSourceDefaultMode), }, }, } @@ -1015,6 +1032,7 @@ func (system *System) SidekiqPodVolumes() []v1.Volume { }, }, }, + DefaultMode: ptr.To(v1.ProjectedVolumeSourceDefaultMode), }, }, } @@ -1064,20 +1082,27 @@ func (system *System) SidekiqDeployment(ctx context.Context, k8sclient client.Cl Annotations: system.sidekiqPodAnnotations(watchedSecretAnnotations), }, Spec: v1.PodSpec{ - Affinity: system.Options.SidekiqAffinity, - Tolerations: system.Options.SidekiqTolerations, - Volumes: system.SidekiqPodVolumes(), - InitContainers: system.sidekiqInit(containerImage), + Affinity: system.Options.SidekiqAffinity, + Tolerations: system.Options.SidekiqTolerations, + RestartPolicy: v1.RestartPolicyAlways, + DNSPolicy: v1.DNSClusterFirst, + SecurityContext: &v1.PodSecurityContext{}, + TerminationGracePeriodSeconds: ptr.To(int64(v1.DefaultTerminationGracePeriodSeconds)), + SchedulerName: v1.DefaultSchedulerName, + Volumes: system.SidekiqPodVolumes(), + InitContainers: system.sidekiqInit(containerImage), Containers: []v1.Container{ { - Name: SystemSidekiqName, - Image: containerImage, - Args: []string{"rake", "sidekiq:worker", "RAILS_MAX_THREADS=25"}, - Env: system.buildSystemSidekiqContainerEnv(), - Resources: *system.Options.SidekiqContainerResourceRequirements, - VolumeMounts: system.sidekiqContainerVolumeMounts(), - ImagePullPolicy: v1.PullIfNotPresent, - Ports: system.sideKiqPorts(), + Name: SystemSidekiqName, + Image: containerImage, + Args: []string{"rake", "sidekiq:worker", "RAILS_MAX_THREADS=25"}, + Env: system.buildSystemSidekiqContainerEnv(), + Resources: *system.Options.SidekiqContainerResourceRequirements, + VolumeMounts: system.sidekiqContainerVolumeMounts(), + ImagePullPolicy: v1.PullIfNotPresent, + Ports: system.sideKiqPorts(), + TerminationMessagePath: v1.TerminationMessagePathDefault, + TerminationMessagePolicy: v1.TerminationMessageReadFile, LivenessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{ TCPSocket: &v1.TCPSocketAction{ @@ -1498,13 +1523,16 @@ func (system *System) systemInit(containerImage string) []v1.Container { if system.Options.SystemDbTLSEnabled { return []v1.Container{ { - Name: "set-permissions", - Image: containerImage, // Minimal image for chmod + Name: "set-permissions", + Image: containerImage, // Minimal image for chmod + ImagePullPolicy: v1.PullIfNotPresent, Command: []string{ "sh", "-c", "cp /tls/* /writable-tls/ && chmod 0600 /writable-tls/*", }, + TerminationMessagePath: v1.TerminationMessagePathDefault, + TerminationMessagePolicy: v1.TerminationMessageReadFile, VolumeMounts: []v1.VolumeMount{ { Name: "tls-secret", @@ -1528,14 +1556,17 @@ func (system *System) sidekiqInit(containerImage string) []v1.Container { var containers []v1.Container // Base init container setup initContainer := v1.Container{ - Name: SystemSideKiqInitContainerName, - Image: containerImage, + Name: SystemSideKiqInitContainerName, + Image: containerImage, + ImagePullPolicy: v1.PullIfNotPresent, Command: []string{ "bash", "-c", "bundle exec sh -c \"until rake boot:redis && curl --output /dev/null --silent --fail --head http://system-master:3000/status; do sleep $SLEEP_SECONDS; done\"", }, - Env: append(system.SystemRedisEnvVars(), helper.EnvVarFromValue("SLEEP_SECONDS", "1")), + Env: append(system.SystemRedisEnvVars(), helper.EnvVarFromValue("SLEEP_SECONDS", "1")), + TerminationMessagePath: v1.TerminationMessagePathDefault, + TerminationMessagePolicy: v1.TerminationMessageReadFile, } // Append Redis TLS volume mounts if Redis TLS is enabled @@ -1545,13 +1576,16 @@ func (system *System) sidekiqInit(containerImage string) []v1.Container { if system.Options.SystemDbTLSEnabled { // Set-permissions container for DB TLS containers = append(containers, v1.Container{ - Name: "set-permissions", - Image: containerImage, + Name: "set-permissions", + Image: containerImage, + ImagePullPolicy: v1.PullIfNotPresent, Command: []string{ "sh", "-c", "cp /tls/* /writable-tls/ && chmod 0600 /writable-tls/*", }, + TerminationMessagePath: v1.TerminationMessagePathDefault, + TerminationMessagePolicy: v1.TerminationMessageReadFile, VolumeMounts: []v1.VolumeMount{ { Name: "tls-secret", @@ -1683,8 +1717,9 @@ func (system *System) redisTLSVolumes() []v1.Volume { Name: "system-redis-tls", VolumeSource: v1.VolumeSource{ Secret: &v1.SecretVolumeSource{ - SecretName: SystemSecretSystemRedisSecretName, - Items: items, + SecretName: SystemSecretSystemRedisSecretName, + Items: items, + DefaultMode: ptr.To(v1.SecretVolumeSourceDefaultMode), }, }, } @@ -1704,8 +1739,9 @@ func (system *System) redisTLSVolumes() []v1.Volume { Name: "backend-redis-tls", VolumeSource: v1.VolumeSource{ Secret: &v1.SecretVolumeSource{ - SecretName: BackendSecretBackendRedisSecretName, - Items: items, + SecretName: BackendSecretBackendRedisSecretName, + Items: items, + DefaultMode: ptr.To(v1.SecretVolumeSourceDefaultMode), }, }, } diff --git a/pkg/3scale/amp/component/system_redis_tls_test.go b/pkg/3scale/amp/component/system_redis_tls_test.go index 60faa16fe..1865c999f 100644 --- a/pkg/3scale/amp/component/system_redis_tls_test.go +++ b/pkg/3scale/amp/component/system_redis_tls_test.go @@ -5,6 +5,7 @@ import ( "github.com/google/go-cmp/cmp" v1 "k8s.io/api/core/v1" + "k8s.io/utils/ptr" "github.com/3scale/3scale-operator/pkg/helper" ) @@ -187,6 +188,7 @@ func TestRedisTLSVolumes(t *testing.T) { Items: []v1.KeyToPath{ {Key: "REDIS_SSL_CA", Path: "system-redis-ca.crt"}, }, + DefaultMode: ptr.To(v1.SecretVolumeSourceDefaultMode), }, }, }, @@ -210,6 +212,7 @@ func TestRedisTLSVolumes(t *testing.T) { Items: []v1.KeyToPath{ {Key: "REDIS_SSL_CA", Path: "backend-redis-ca.crt"}, }, + DefaultMode: ptr.To(v1.SecretVolumeSourceDefaultMode), }, }, }, @@ -242,6 +245,7 @@ func TestRedisTLSVolumes(t *testing.T) { {Key: "REDIS_SSL_CERT", Path: "system-redis-client.crt"}, {Key: "REDIS_SSL_KEY", Path: "system-redis-private.key"}, }, + DefaultMode: ptr.To(v1.SecretVolumeSourceDefaultMode), }, }, }, @@ -255,6 +259,7 @@ func TestRedisTLSVolumes(t *testing.T) { {Key: "REDIS_SSL_CERT", Path: "backend-redis-client.crt"}, {Key: "REDIS_SSL_KEY", Path: "backend-redis-private.key"}, }, + DefaultMode: ptr.To(v1.SecretVolumeSourceDefaultMode), }, }, }, @@ -285,6 +290,7 @@ func TestRedisTLSVolumes(t *testing.T) { {Key: "REDIS_SSL_CERT", Path: "system-redis-client.crt"}, {Key: "REDIS_SSL_KEY", Path: "system-redis-private.key"}, }, + DefaultMode: ptr.To(v1.SecretVolumeSourceDefaultMode), }, }, }, @@ -296,6 +302,7 @@ func TestRedisTLSVolumes(t *testing.T) { Items: []v1.KeyToPath{ {Key: "REDIS_SSL_CA", Path: "backend-redis-ca.crt"}, }, + DefaultMode: ptr.To(v1.SecretVolumeSourceDefaultMode), }, }, }, diff --git a/pkg/3scale/amp/component/system_searchd.go b/pkg/3scale/amp/component/system_searchd.go index b0bf23190..9119c7beb 100644 --- a/pkg/3scale/amp/component/system_searchd.go +++ b/pkg/3scale/amp/component/system_searchd.go @@ -9,6 +9,7 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -83,17 +84,24 @@ func (s *SystemSearchd) Deployment(ctx context.Context, k8sclient client.Client, Annotations: s.searchdPodAnnotations(watchedSecretAnnotations), }, Spec: v1.PodSpec{ - InitContainers: s.searchdInit(containerImage), - Affinity: s.Options.Affinity, - Tolerations: s.Options.Tolerations, - ServiceAccountName: "amp", - Volumes: s.searchdVolume(), + InitContainers: s.searchdInit(containerImage), + Affinity: s.Options.Affinity, + Tolerations: s.Options.Tolerations, + ServiceAccountName: "amp", + RestartPolicy: v1.RestartPolicyAlways, + DNSPolicy: v1.DNSClusterFirst, + SecurityContext: &v1.PodSecurityContext{}, + TerminationGracePeriodSeconds: ptr.To(int64(v1.DefaultTerminationGracePeriodSeconds)), + SchedulerName: v1.DefaultSchedulerName, + Volumes: s.searchdVolume(), Containers: []v1.Container{ { - Name: SystemSearchdDeploymentName, - Image: containerImage, - ImagePullPolicy: v1.PullIfNotPresent, - VolumeMounts: s.searchDVolumeMounts(), + Name: SystemSearchdDeploymentName, + Image: containerImage, + ImagePullPolicy: v1.PullIfNotPresent, + VolumeMounts: s.searchDVolumeMounts(), + TerminationMessagePath: v1.TerminationMessagePathDefault, + TerminationMessagePolicy: v1.TerminationMessageReadFile, LivenessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{ TCPSocket: &v1.TCPSocketAction{ @@ -101,7 +109,10 @@ func (s *SystemSearchd) Deployment(ctx context.Context, k8sclient client.Client, }, }, InitialDelaySeconds: 60, + TimeoutSeconds: 1, PeriodSeconds: 10, + SuccessThreshold: 1, + FailureThreshold: 3, }, ReadinessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{ @@ -172,13 +183,15 @@ func (s *SystemSearchd) ReindexingJob(containerImage string, system *System) *ba InitContainers: s.searchdInit(containerImage), Containers: []v1.Container{ { - Name: SystemSearchdReindexJobName, - Image: containerImage, - Args: []string{"bash", "-c", "bundle exec rake searchd:optimal_index"}, - Env: system.buildSystemBaseEnv(), - Resources: s.Options.ContainerResourceRequirements, - ImagePullPolicy: v1.PullIfNotPresent, - VolumeMounts: s.searchdManticoreVolumeMounts(), + Name: SystemSearchdReindexJobName, + Image: containerImage, + Args: []string{"bash", "-c", "bundle exec rake searchd:optimal_index"}, + Env: system.buildSystemBaseEnv(), + Resources: s.Options.ContainerResourceRequirements, + ImagePullPolicy: v1.PullIfNotPresent, + VolumeMounts: s.searchdManticoreVolumeMounts(), + TerminationMessagePath: v1.TerminationMessagePathDefault, + TerminationMessagePolicy: v1.TerminationMessageReadFile, }, }, Volumes: s.searchdJobVolume(), diff --git a/pkg/3scale/amp/component/zync.go b/pkg/3scale/amp/component/zync.go index 38764d75a..258391b09 100644 --- a/pkg/3scale/amp/component/zync.go +++ b/pkg/3scale/amp/component/zync.go @@ -11,6 +11,7 @@ import ( rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -222,16 +223,24 @@ func (zync *Zync) Deployment(ctx context.Context, k8sclient client.Client, conta Annotations: zync.zyncPodAnnotations(watchedSecretAnnotations), }, Spec: v1.PodSpec{ - Affinity: zync.Options.ZyncAffinity, - Tolerations: zync.Options.ZyncTolerations, - ServiceAccountName: "amp", - InitContainers: zync.zyncInit(containerImage), + Affinity: zync.Options.ZyncAffinity, + Tolerations: zync.Options.ZyncTolerations, + ServiceAccountName: "amp", + RestartPolicy: v1.RestartPolicyAlways, + DNSPolicy: v1.DNSClusterFirst, + SecurityContext: &v1.PodSecurityContext{}, + TerminationGracePeriodSeconds: ptr.To(int64(v1.DefaultTerminationGracePeriodSeconds)), + SchedulerName: v1.DefaultSchedulerName, + InitContainers: zync.zyncInit(containerImage), Containers: []v1.Container{ { - Name: ZyncName, - Image: containerImage, - Ports: zync.zyncPorts(), - Env: zync.commonZyncEnvVars(), + Name: ZyncName, + Image: containerImage, + ImagePullPolicy: v1.PullIfNotPresent, + Ports: zync.zyncPorts(), + Env: zync.commonZyncEnvVars(), + TerminationMessagePath: v1.TerminationMessagePathDefault, + TerminationMessagePolicy: v1.TerminationMessageReadFile, LivenessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{ HTTPGet: &v1.HTTPGetAction{ @@ -379,15 +388,20 @@ func (zync *Zync) QueDeployment(ctx context.Context, k8sclient client.Client, co Tolerations: zync.Options.ZyncQueTolerations, ServiceAccountName: "zync-que-sa", RestartPolicy: v1.RestartPolicyAlways, - TerminationGracePeriodSeconds: &[]int64{30}[0], + DNSPolicy: v1.DNSClusterFirst, + SecurityContext: &v1.PodSecurityContext{}, + TerminationGracePeriodSeconds: ptr.To(int64(v1.DefaultTerminationGracePeriodSeconds)), + SchedulerName: v1.DefaultSchedulerName, InitContainers: zync.zyncQueInit(containerImage), Containers: []v1.Container{ { - Name: "que", - Command: []string{"/usr/bin/bash"}, - Args: []string{"-c", "bundle exec rake 'que[--worker-count 10]'"}, - Image: containerImage, - ImagePullPolicy: v1.PullAlways, + Name: "que", + Command: []string{"/usr/bin/bash"}, + Args: []string{"-c", "bundle exec rake 'que[--worker-count 10]'"}, + Image: containerImage, + ImagePullPolicy: v1.PullAlways, + TerminationMessagePath: v1.TerminationMessagePathDefault, + TerminationMessagePolicy: v1.TerminationMessageReadFile, LivenessProbe: &v1.Probe{ FailureThreshold: 3, InitialDelaySeconds: 10, @@ -448,10 +462,14 @@ func (zync *Zync) DatabaseDeployment(containerImage string) *k8sappsv1.Deploymen Annotations: zync.Options.ZyncDatabasePodTemplateAnnotations, }, Spec: v1.PodSpec{ - Affinity: zync.Options.ZyncDatabaseAffinity, - Tolerations: zync.Options.ZyncDatabaseTolerations, - RestartPolicy: v1.RestartPolicyAlways, - ServiceAccountName: "amp", + Affinity: zync.Options.ZyncDatabaseAffinity, + Tolerations: zync.Options.ZyncDatabaseTolerations, + RestartPolicy: v1.RestartPolicyAlways, + DNSPolicy: v1.DNSClusterFirst, + SecurityContext: &v1.PodSecurityContext{}, + TerminationGracePeriodSeconds: ptr.To(int64(v1.DefaultTerminationGracePeriodSeconds)), + SchedulerName: v1.DefaultSchedulerName, + ServiceAccountName: "amp", Containers: []v1.Container{ { Name: "postgresql", @@ -468,7 +486,9 @@ func (zync *Zync) DatabaseDeployment(containerImage string) *k8sappsv1.Deploymen MountPath: "/var/lib/pgsql/data", }, }, - ImagePullPolicy: v1.PullIfNotPresent, + ImagePullPolicy: v1.PullIfNotPresent, + TerminationMessagePath: v1.TerminationMessagePathDefault, + TerminationMessagePolicy: v1.TerminationMessageReadFile, Env: []v1.EnvVar{ { Name: "POSTGRESQL_USER", @@ -500,17 +520,23 @@ func (zync *Zync) DatabaseDeployment(containerImage string) *k8sappsv1.Deploymen Port: intstr.FromInt32(5432), }, }, - TimeoutSeconds: 1, InitialDelaySeconds: 30, + TimeoutSeconds: 1, + PeriodSeconds: 10, + SuccessThreshold: 1, + FailureThreshold: 3, }, ReadinessProbe: &v1.Probe{ - TimeoutSeconds: 1, - InitialDelaySeconds: 5, ProbeHandler: v1.ProbeHandler{ Exec: &v1.ExecAction{ Command: []string{"/bin/sh", "-i", "-c", "psql -h 127.0.0.1 -U zync -q -d zync_production -c 'SELECT 1'"}, }, }, + InitialDelaySeconds: 5, + TimeoutSeconds: 1, + PeriodSeconds: 10, + SuccessThreshold: 1, + FailureThreshold: 3, }, Resources: zync.Options.DatabaseContainerResourceRequirements, }, @@ -635,13 +661,16 @@ func (zync *Zync) zyncInit(containerImage string) []v1.Container { if zync.Options.ZyncDbTLSEnabled { return []v1.Container{ { - Name: "set-permissions", - Image: containerImage, + Name: "set-permissions", + Image: containerImage, + ImagePullPolicy: v1.PullIfNotPresent, Command: []string{ "sh", "-c", "cp /tls/* /writable-tls/ && chmod 0600 /writable-tls/*", }, + TerminationMessagePath: v1.TerminationMessagePathDefault, + TerminationMessagePolicy: v1.TerminationMessageReadFile, VolumeMounts: []v1.VolumeMount{ { Name: "tls-secret", @@ -656,13 +685,16 @@ func (zync *Zync) zyncInit(containerImage string) []v1.Container { }, }, { - Name: ZyncInitContainerName, - Image: containerImage, + Name: ZyncInitContainerName, + Image: containerImage, + ImagePullPolicy: v1.PullIfNotPresent, Command: []string{ "bash", "-c", "bundle exec sh -c \"until rake boot:db; do sleep $SLEEP_SECONDS; done\"", }, + TerminationMessagePath: v1.TerminationMessagePathDefault, + TerminationMessagePolicy: v1.TerminationMessageReadFile, Env: []v1.EnvVar{ { Name: "SLEEP_SECONDS", @@ -697,13 +729,16 @@ func (zync *Zync) zyncInit(containerImage string) []v1.Container { } else { return []v1.Container{ { - Name: ZyncInitContainerName, - Image: containerImage, + Name: ZyncInitContainerName, + Image: containerImage, + ImagePullPolicy: v1.PullIfNotPresent, Command: []string{ "bash", "-c", "bundle exec sh -c \"until rake boot:db; do sleep $SLEEP_SECONDS; done\"", }, + TerminationMessagePath: v1.TerminationMessagePathDefault, + TerminationMessagePolicy: v1.TerminationMessageReadFile, Env: []v1.EnvVar{ { Name: "SLEEP_SECONDS", @@ -763,6 +798,7 @@ func (zync *Zync) zyncVolume() []v1.Volume { Path: "tls.key", // Map the secret key to the tls.key file in the container }, }, + DefaultMode: ptr.To(v1.SecretVolumeSourceDefaultMode), }, }, }, diff --git a/pkg/reconcilers/base_reconciler.go b/pkg/reconcilers/base_reconciler.go index 14ad55633..a1ff7c9e5 100644 --- a/pkg/reconcilers/base_reconciler.go +++ b/pkg/reconcilers/base_reconciler.go @@ -67,6 +67,13 @@ func (b *BaseReconciler) Reconcile(ctx context.Context, req reconcile.Request) ( return reconcile.Result{}, nil } +// WithRequest returns a shallow copy with a logger enriched with the request's namespace/name. +func (b *BaseReconciler) WithRequest(req reconcile.Request) *BaseReconciler { + c := *b + c.logger = b.logger.WithValues("namespace", req.Namespace, "name", req.Name) + return &c +} + // Client returns a split client that reads objects from // the cache and writes to the Kubernetes APIServer func (b *BaseReconciler) Client() client.Client { diff --git a/test/integration/apimanager_controller_test.go b/test/integration/apimanager_controller_test.go index b0b0c4bd8..a2fa6b6ae 100644 --- a/test/integration/apimanager_controller_test.go +++ b/test/integration/apimanager_controller_test.go @@ -211,6 +211,8 @@ var _ = Describe("APIManager controller", func() { waitForAPIManagerAvailableCondition(5*time.Second, 15*time.Minute, apimanager, GinkgoWriter) fmt.Fprintf(GinkgoWriter, "APIManager 'Available' condition is true\n") + triggerSyntheticDeploymentUpdate(testNamespace, GinkgoWriter) + verifyNoDeploymentUpdates(apimanager.Namespace, apimanager.Name, GinkgoWriter) elapsed := time.Since(start) fmt.Fprintf(GinkgoWriter, "APIManager creation and availability took '%s'\n", elapsed) }) @@ -1053,6 +1055,36 @@ func testCustomEnvironmentContent() string { ` } +// triggerSyntheticDeploymentUpdate patches a monitored deployment with a dummy +// image tag, then waits for the operator to reconcile it back. This guarantees +// at least one Deployment UPDATE in the ReconcileCounter, proving the counter +// is wired correctly and not silently counting nothing. +func triggerSyntheticDeploymentUpdate(namespace string, w io.Writer) { + const deploymentName = "system-memcache" + + dep := &appsv1.Deployment{} + Expect(testK8sClient.Get(context.Background(), + types.NamespacedName{Name: deploymentName, Namespace: namespace}, dep)).To(Succeed()) + + originalImage := dep.Spec.Template.Spec.Containers[0].Image + dep.Spec.Template.Spec.Containers[0].Image = originalImage + "-synthetic-test-trigger" + Expect(testK8sClient.Update(context.Background(), dep)).To(Succeed()) + fmt.Fprintf(w, "Synthetic image change applied to %s; waiting for operator to reconcile back\n", deploymentName) + + // Use testK8sAPIClient (direct API server read) to avoid the cached client + // returning the pre-update image before the operator has processed the change. + Eventually(func() bool { + d := &appsv1.Deployment{} + if err := testK8sAPIClient.Get(context.Background(), + types.NamespacedName{Name: deploymentName, Namespace: namespace}, d); err != nil { + return false + } + return d.Spec.Template.Spec.Containers[0].Image == originalImage + }, 2*time.Minute, 2*time.Second).Should(BeTrue(), + fmt.Sprintf("operator did not revert synthetic image change on %s within 2 minutes", deploymentName)) + fmt.Fprintf(w, "Operator reconciled %s back to desired image\n", deploymentName) +} + func testGetCustomEnvironmentSecret(namespace string) *corev1.Secret { customEnvironmentSecret := corev1.Secret{ TypeMeta: metav1.TypeMeta{ diff --git a/test/integration/apimanager_suite_test.go b/test/integration/apimanager_suite_test.go index 3427c6e48..fe7f69442 100644 --- a/test/integration/apimanager_suite_test.go +++ b/test/integration/apimanager_suite_test.go @@ -25,6 +25,8 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + uberzap "go.uber.org/zap" + "go.uber.org/zap/zapcore" "k8s.io/client-go/discovery" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" @@ -57,6 +59,7 @@ var ( testK8sClient client.Client testK8sAPIClient client.Reader testEnv *envtest.Environment + reconcileCounter *ReconcileCounter ) func TestAPIManager(t *testing.T) { @@ -65,7 +68,38 @@ func TestAPIManager(t *testing.T) { } var _ = BeforeSuite(func() { - logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + // List of operator-managed deployments to monitor for updates + monitoredDeployments := []string{ + "apicast-production", + "apicast-staging", + "backend-cron", + "backend-listener", + "backend-worker", + "system-app", + "system-memcache", + "system-sidekiq", + "system-searchd", + "zync", + "zync-que", + "zync-database", + } + + // Build a base zapcore.Core writing to GinkgoWriter so ReconcileCounter can wrap it. + baseCore := zapcore.NewCore( + zapcore.NewConsoleEncoder(uberzap.NewDevelopmentEncoderConfig()), + zapcore.AddSync(GinkgoWriter), + zapcore.DebugLevel, + ) + reconcileCounter = NewReconcileCounter(baseCore, monitoredDeployments) + + // Inject the wrapped core into the controller-runtime logger via RawZapOpts. + logf.SetLogger(zap.New( + zap.WriteTo(GinkgoWriter), + zap.UseDevMode(true), + zap.RawZapOpts(uberzap.WrapCore(func(_ zapcore.Core) zapcore.Core { + return reconcileCounter + })), + )) By("bootstrapping test environment") diff --git a/test/integration/reconcile_counter_test.go b/test/integration/reconcile_counter_test.go new file mode 100644 index 000000000..9d0cb93f8 --- /dev/null +++ b/test/integration/reconcile_counter_test.go @@ -0,0 +1,172 @@ +package integration + +import ( + "fmt" + "strings" + "sync" + + "go.uber.org/zap/zapcore" +) + +// counterState is the single shared mutable core of a ReconcileCounter tree. +// All ReconcileCounter instances produced by With() point to the same counterState, +// so counts accumulate correctly regardless of which logger variant does the write. +// mu protects updateCounts; deploymentNames is read-only after construction. +type counterState struct { + deploymentNames map[string]bool + // updateCounts is keyed by "namespace/name" (the APIManager CR instance), + // then by deployment name. This allows parallel specs targeting different + // CR instances to accumulate counts independently. + updateCounts map[string]map[string]int + mu sync.Mutex +} + +// ReconcileCounter wraps a zapcore.Core to count deployment update events per +// APIManager CR instance. It overrides With and Check so the counter survives +// logger.WithName/WithValues chains, and captures the "namespace" and "name" +// fields that controller-runtime adds to the reconciler logger context. +type ReconcileCounter struct { + zapcore.Core + state *counterState + namespace string // captured from With() chain + crName string // captured from With() chain +} + +func (rc *ReconcileCounter) crKey() string { + if rc.namespace == "" || rc.crName == "" { + return "" + } + return rc.namespace + "/" + rc.crName +} + +// NewReconcileCounter creates a new ReconcileCounter wrapping the given core. +func NewReconcileCounter(core zapcore.Core, deploymentNames []string) *ReconcileCounter { + nameMap := make(map[string]bool) + for _, name := range deploymentNames { + nameMap[name] = true + } + return &ReconcileCounter{ + Core: core, + state: &counterState{ + deploymentNames: nameMap, + updateCounts: make(map[string]map[string]int), + }, + } +} + +// With returns a new ReconcileCounter wrapping the inner core-with-fields, +// sharing the same counter state. It captures "namespace" and "name" fields +// so Write can key counts by CR instance. +func (rc *ReconcileCounter) With(fields []zapcore.Field) zapcore.Core { + ns := rc.namespace + name := rc.crName + for _, f := range fields { + if f.Type == zapcore.StringType { + switch f.Key { + case "namespace": + ns = f.String + case "name": + name = f.String + } + } + } + return &ReconcileCounter{ + Core: rc.Core.With(fields), + state: rc.state, + namespace: ns, + crName: name, + } +} + +// Check ensures rc.Write is called (not the inner core's Write) for entries +// that pass the level check. +func (rc *ReconcileCounter) Check(entry zapcore.Entry, ce *zapcore.CheckedEntry) *zapcore.CheckedEntry { + if rc.Core.Enabled(entry.Level) { + return ce.AddCore(entry, rc) + } + return ce +} + +// Write intercepts log entries and counts deployment updates keyed by CR instance. +// The key is resolved in priority order: +// 1. With() chain (context logger): controller-runtime injects "namespace"/"name" before Reconcile. +// 2. Explicit fields on the log entry: UpdateResource logs obj.GetNamespace() as "namespace" and +// the APIManager owner reference name as "name". +func (rc *ReconcileCounter) Write(entry zapcore.Entry, fields []zapcore.Field) error { + if !strings.Contains(entry.Message, "Updated object 'v1.Deployment/") { + return rc.Core.Write(entry, fields) + } + parts := strings.Split(entry.Message, "v1.Deployment/") + if len(parts) == 2 { + deploymentName := strings.TrimSuffix(parts[1], "'") + if rc.state.deploymentNames[deploymentName] { + key := rc.crKey() + if key == "" { + var ns, name string + for _, f := range fields { + if f.Type == zapcore.StringType { + switch f.Key { + case "namespace": + ns = f.String + case "name": + name = f.String + } + } + } + if ns != "" && name != "" { + key = ns + "/" + name + } + } + if key != "" { + rc.state.mu.Lock() + if rc.state.updateCounts[key] == nil { + rc.state.updateCounts[key] = make(map[string]int) + } + rc.state.updateCounts[key][deploymentName]++ + rc.state.mu.Unlock() + } + } + } + return rc.Core.Write(entry, fields) +} + +// GetUpdateCounts returns a copy of per-deployment counts for the given CR instance. +func (rc *ReconcileCounter) GetUpdateCounts(namespace, name string) map[string]int { + key := namespace + "/" + name + rc.state.mu.Lock() + defer rc.state.mu.Unlock() + counts := make(map[string]int) + for k, v := range rc.state.updateCounts[key] { + counts[k] = v + } + return counts +} + +// GetTotalUpdates returns the total deployment update count for the given CR instance. +func (rc *ReconcileCounter) GetTotalUpdates(namespace, name string) int { + key := namespace + "/" + name + rc.state.mu.Lock() + defer rc.state.mu.Unlock() + total := 0 + for _, count := range rc.state.updateCounts[key] { + total += count + } + return total +} + +// GetReport returns a formatted breakdown for the given CR instance. +func (rc *ReconcileCounter) GetReport(namespace, name string) string { + key := namespace + "/" + name + rc.state.mu.Lock() + defer rc.state.mu.Unlock() + counts := rc.state.updateCounts[key] + if len(counts) == 0 { + return fmt.Sprintf("No deployment updates detected for %s", key) + } + var sb strings.Builder + fmt.Fprintf(&sb, "Deployment update counts for %s:\n", key) + for depName, count := range counts { + fmt.Fprintf(&sb, " %s: %d updates\n", depName, count) + } + return sb.String() +} diff --git a/test/integration/verify_no_perpetual_reconciliation_test.go b/test/integration/verify_no_perpetual_reconciliation_test.go new file mode 100644 index 000000000..21618a768 --- /dev/null +++ b/test/integration/verify_no_perpetual_reconciliation_test.go @@ -0,0 +1,76 @@ +package integration + +import ( + "fmt" + "io" + "sort" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// minTotalDeploymentUpdates is a floor that guards against a silent counter +// failure. A synthetic image change is applied to system-memcache before +// calling verifyNoDeploymentUpdates (see triggerSyntheticDeploymentUpdate), +// which guarantees the operator issues at least one real UPDATE reconciling +// the deployment back to the desired image. A count of 0 therefore means the +// counter is broken (namespace/name fields not captured) rather than the +// install being genuinely update-free. +const minTotalDeploymentUpdates = 1 + +// maxTotalDeploymentUpdates is the ceiling on the total number of Deployment +// update calls for a single APIManager CR instance over the full test session. +// +// Rationale: during a normal install each deployment receives 1–3 legitimate +// updates as pods roll out and probes settle, giving ~12–36 total for 12 +// deployments. The ceiling is set at 50 to absorb timing variance while +// staying well below what the perpetual-reconcile bug produced (~7 deployments +// × many cycles = hundreds of updates per install). +const maxTotalDeploymentUpdates = 50 + +// verifyNoDeploymentUpdates asserts that the total deployment update count +// for the given APIManager CR over the full test session is within the ceiling. +// On failure it lists each deployment's individual count to aid diagnosis. +func verifyNoDeploymentUpdates(namespace, name string, w io.Writer) { + updateCounts := reconcileCounter.GetUpdateCounts(namespace, name) + totalUpdates := reconcileCounter.GetTotalUpdates(namespace, name) + + fmt.Fprintf(w, "\n=== Deployment Update Report (%s/%s, session ceiling %d) ===\n", + namespace, name, maxTotalDeploymentUpdates) + fmt.Fprintf(w, "Total: %d\n", totalUpdates) + + names := make([]string, 0, len(updateCounts)) + for n := range updateCounts { + names = append(names, n) + } + sort.Strings(names) + for _, n := range names { + fmt.Fprintf(w, " %s: %d\n", n, updateCounts[n]) + } + fmt.Fprintf(w, "=============================================================\n\n") + + Expect(totalUpdates).To(BeNumerically(">=", minTotalDeploymentUpdates), + fmt.Sprintf("%s/%s: total deployment updates is 0 — counter is likely misconfigured (namespace/name fields not captured)", namespace, name)) + Expect(totalUpdates).To(BeNumerically("<=", maxTotalDeploymentUpdates), + deploymentUpdateDetail(namespace, name, updateCounts, totalUpdates)) +} + +// deploymentUpdateDetail builds a human-readable breakdown for use in a Gomega +// failure message so the offending deployments are immediately visible. +func deploymentUpdateDetail(namespace, name string, counts map[string]int, total int) string { + var sb strings.Builder + fmt.Fprintf(&sb, "%s/%s: total deployment updates %d exceeded ceiling %d; per-deployment breakdown:\n", + namespace, name, total, maxTotalDeploymentUpdates) + names := make([]string, 0, len(counts)) + for n := range counts { + names = append(names, n) + } + sort.Strings(names) + for _, n := range names { + fmt.Fprintf(&sb, " %s: %d\n", n, counts[n]) + } + return sb.String() +} + +var _ = Describe // suppress unused import lint for ginkgo dot-import From a5ce829877ed71c147399c1c354ad8bbbcc3963a Mon Sep 17 00:00:00 2001 From: Boris Urbanik Date: Mon, 25 May 2026 00:37:37 +0100 Subject: [PATCH 2/2] Address review feedback: remove cosmetic defaults and simplify test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove WithRequest, pass BaseReconciler directly - Remove cosmetic K8s defaults (RestartPolicy, DNSPolicy, etc.) that have no corresponding mutator — keep only fields compared by DeploymentPodInitContainerMutator, DeploymentVolumesMutator, and DeploymentProbesMutator - Revert apicast.go and memcached.go fully to master (no mutators require defaults in those components) - Simplify integration test to assert exact update counts (0 before synthetic trigger, 1 after) instead of ceiling/floor ranges Co-Authored-By: Claude Sonnet 4.6 --- controllers/apps/apimanager_controller.go | 2 +- pkg/3scale/amp/component/apicast.go | 126 ++++++---------- pkg/3scale/amp/component/backend.go | 114 ++++++-------- pkg/3scale/amp/component/memcached.go | 34 ++--- pkg/3scale/amp/component/system.go | 142 ++++++++---------- pkg/3scale/amp/component/system_searchd.go | 42 ++---- pkg/3scale/amp/component/zync.go | 90 +++++------ pkg/reconcilers/base_reconciler.go | 7 - .../integration/apimanager_controller_test.go | 8 +- ...verify_no_perpetual_reconciliation_test.go | 55 +++---- 10 files changed, 240 insertions(+), 380 deletions(-) diff --git a/controllers/apps/apimanager_controller.go b/controllers/apps/apimanager_controller.go index 397ee1e5a..0849dac79 100644 --- a/controllers/apps/apimanager_controller.go +++ b/controllers/apps/apimanager_controller.go @@ -160,7 +160,7 @@ func (r *APIManagerReconciler) Reconcile(ctx context.Context, req ctrl.Request) return res, nil } - specResult, specErr := r.reconcileAPIManagerLogic(r.WithRequest(req), instance) + specResult, specErr := r.reconcileAPIManagerLogic(r.BaseReconciler, instance) statusResult, statusErr := r.reconcileAPIManagerStatus(instance, preflightChecksError) if statusErr != nil { return ctrl.Result{}, statusErr diff --git a/pkg/3scale/amp/component/apicast.go b/pkg/3scale/amp/component/apicast.go index 021ae47ee..b2daa6e43 100644 --- a/pkg/3scale/amp/component/apicast.go +++ b/pkg/3scale/amp/component/apicast.go @@ -17,7 +17,6 @@ import ( policyv1 "k8s.io/api/policy/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -140,49 +139,36 @@ func (apicast *Apicast) StagingDeployment(ctx context.Context, k8sclient client. Annotations: apicast.stagingPodAnnotations(watchedSecretAnnotations), }, Spec: v1.PodSpec{ - Affinity: apicast.Options.StagingAffinity, - Tolerations: apicast.Options.StagingTolerations, - ServiceAccountName: "amp", - RestartPolicy: v1.RestartPolicyAlways, - DNSPolicy: v1.DNSClusterFirst, - SecurityContext: &v1.PodSecurityContext{}, - TerminationGracePeriodSeconds: ptr.To(int64(v1.DefaultTerminationGracePeriodSeconds)), - SchedulerName: v1.DefaultSchedulerName, - Volumes: apicast.stagingVolumes(), + Affinity: apicast.Options.StagingAffinity, + Tolerations: apicast.Options.StagingTolerations, + ServiceAccountName: "amp", + Volumes: apicast.stagingVolumes(), Containers: []v1.Container{ { - Ports: apicast.stagingContainerPorts(), - Env: apicast.buildApicastStagingEnv(), - Image: containerImage, - ImagePullPolicy: v1.PullIfNotPresent, - Name: ApicastStagingName, - Resources: apicast.Options.StagingResourceRequirements, - VolumeMounts: apicast.stagingVolumeMounts(), - TerminationMessagePath: v1.TerminationMessagePathDefault, - TerminationMessagePolicy: v1.TerminationMessageReadFile, + Ports: apicast.stagingContainerPorts(), + Env: apicast.buildApicastStagingEnv(), + Image: containerImage, + ImagePullPolicy: v1.PullIfNotPresent, + Name: ApicastStagingName, + Resources: apicast.Options.StagingResourceRequirements, + VolumeMounts: apicast.stagingVolumeMounts(), LivenessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{HTTPGet: &v1.HTTPGetAction{ - Path: "/status/live", - Port: intstr.FromInt32(8090), - Scheme: v1.URISchemeHTTP, + Path: "/status/live", + Port: intstr.FromInt32(8090), }}, InitialDelaySeconds: 10, TimeoutSeconds: 5, PeriodSeconds: 10, - SuccessThreshold: 1, - FailureThreshold: 3, }, ReadinessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{HTTPGet: &v1.HTTPGetAction{ - Path: "/status/ready", - Port: intstr.FromInt32(8090), - Scheme: v1.URISchemeHTTP, + Path: "/status/ready", + Port: intstr.FromInt32(8090), }}, InitialDelaySeconds: 15, TimeoutSeconds: 5, PeriodSeconds: 30, - SuccessThreshold: 1, - FailureThreshold: 3, }, }, }, @@ -233,23 +219,15 @@ func (apicast *Apicast) ProductionDeployment(ctx context.Context, k8sclient clie Annotations: apicast.productionPodAnnotations(watchedSecretAnnotations), }, Spec: v1.PodSpec{ - Affinity: apicast.Options.ProductionAffinity, - Tolerations: apicast.Options.ProductionTolerations, - ServiceAccountName: "amp", - RestartPolicy: v1.RestartPolicyAlways, - DNSPolicy: v1.DNSClusterFirst, - SecurityContext: &v1.PodSecurityContext{}, - TerminationGracePeriodSeconds: ptr.To(int64(v1.DefaultTerminationGracePeriodSeconds)), - SchedulerName: v1.DefaultSchedulerName, - Volumes: apicast.productionVolumes(), + Affinity: apicast.Options.ProductionAffinity, + Tolerations: apicast.Options.ProductionTolerations, + ServiceAccountName: "amp", + Volumes: apicast.productionVolumes(), InitContainers: []v1.Container{ { - Name: ApicastProductionInitContainerName, - Image: containerImage, - ImagePullPolicy: v1.PullIfNotPresent, - Command: []string{"sh", "-c", "until $(curl --output /dev/null --silent --fail --head http://system-master:3000/status); do sleep $SLEEP_SECONDS; done"}, - TerminationMessagePath: v1.TerminationMessagePathDefault, - TerminationMessagePolicy: v1.TerminationMessageReadFile, + Name: ApicastProductionInitContainerName, + Image: containerImage, + Command: []string{"sh", "-c", "until $(curl --output /dev/null --silent --fail --head http://system-master:3000/status); do sleep $SLEEP_SECONDS; done"}, Env: []v1.EnvVar{ { Name: "SLEEP_SECONDS", @@ -260,38 +238,30 @@ func (apicast *Apicast) ProductionDeployment(ctx context.Context, k8sclient clie }, Containers: []v1.Container{ { - Ports: apicast.productionContainerPorts(), - Env: apicast.buildApicastProductionEnv(), - Image: containerImage, - ImagePullPolicy: v1.PullIfNotPresent, - Name: ApicastProductionName, - Resources: apicast.Options.ProductionResourceRequirements, - VolumeMounts: apicast.productionVolumeMounts(), - TerminationMessagePath: v1.TerminationMessagePathDefault, - TerminationMessagePolicy: v1.TerminationMessageReadFile, + Ports: apicast.productionContainerPorts(), + Env: apicast.buildApicastProductionEnv(), + Image: containerImage, + ImagePullPolicy: v1.PullIfNotPresent, + Name: ApicastProductionName, + Resources: apicast.Options.ProductionResourceRequirements, + VolumeMounts: apicast.productionVolumeMounts(), LivenessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{HTTPGet: &v1.HTTPGetAction{ - Path: "/status/live", - Port: intstr.FromInt32(8090), - Scheme: v1.URISchemeHTTP, + Path: "/status/live", + Port: intstr.FromInt32(8090), }}, InitialDelaySeconds: 10, TimeoutSeconds: 5, PeriodSeconds: 10, - SuccessThreshold: 1, - FailureThreshold: 3, }, ReadinessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{HTTPGet: &v1.HTTPGetAction{ - Path: "/status/ready", - Port: intstr.FromInt32(8090), - Scheme: v1.URISchemeHTTP, + Path: "/status/ready", + Port: intstr.FromInt32(8090), }}, InitialDelaySeconds: 15, TimeoutSeconds: 5, PeriodSeconds: 30, - SuccessThreshold: 1, - FailureThreshold: 3, }, }, }, @@ -648,8 +618,7 @@ func (apicast *Apicast) productionVolumes() []v1.Volume { Name: customPolicy.VolumeName(), VolumeSource: v1.VolumeSource{ Secret: &v1.SecretVolumeSource{ - SecretName: customPolicy.Secret.Name, - DefaultMode: ptr.To(v1.SecretVolumeSourceDefaultMode), + SecretName: customPolicy.Secret.Name, }, }, }) @@ -667,7 +636,6 @@ func (apicast *Apicast) productionVolumes() []v1.Volume { Path: apicast.Options.ProductionTracingConfig.VolumeName(), }, }, - DefaultMode: ptr.To(v1.SecretVolumeSourceDefaultMode), }, }, }) @@ -678,9 +646,8 @@ func (apicast *Apicast) productionVolumes() []v1.Volume { Name: OpentelemetryConfigurationVolumeName, VolumeSource: v1.VolumeSource{ Secret: &v1.SecretVolumeSource{ - SecretName: apicast.Options.ProductionOpentelemetry.Secret.Name, - Optional: &[]bool{false}[0], - DefaultMode: ptr.To(v1.SecretVolumeSourceDefaultMode), + SecretName: apicast.Options.ProductionOpentelemetry.Secret.Name, + Optional: &[]bool{false}[0], }, }, }) @@ -691,8 +658,7 @@ func (apicast *Apicast) productionVolumes() []v1.Volume { Name: customEnvVolumeName(customEnvSecret), VolumeSource: v1.VolumeSource{ Secret: &v1.SecretVolumeSource{ - SecretName: customEnvSecret.GetName(), - DefaultMode: ptr.To(v1.SecretVolumeSourceDefaultMode), + SecretName: customEnvSecret.GetName(), }, }, }) @@ -703,8 +669,7 @@ func (apicast *Apicast) productionVolumes() []v1.Volume { Name: HTTPSCertificatesVolumeName, VolumeSource: v1.VolumeSource{ Secret: &v1.SecretVolumeSource{ - SecretName: *apicast.Options.ProductionHTTPSCertificateSecretName, - DefaultMode: ptr.To(v1.SecretVolumeSourceDefaultMode), + SecretName: *apicast.Options.ProductionHTTPSCertificateSecretName, }, }, }) @@ -721,8 +686,7 @@ func (apicast *Apicast) stagingVolumes() []v1.Volume { Name: customPolicy.VolumeName(), VolumeSource: v1.VolumeSource{ Secret: &v1.SecretVolumeSource{ - SecretName: customPolicy.Secret.Name, - DefaultMode: ptr.To(v1.SecretVolumeSourceDefaultMode), + SecretName: customPolicy.Secret.Name, }, }, }) @@ -740,7 +704,6 @@ func (apicast *Apicast) stagingVolumes() []v1.Volume { Path: apicast.Options.StagingTracingConfig.VolumeName(), }, }, - DefaultMode: ptr.To(v1.SecretVolumeSourceDefaultMode), }, }, }) @@ -751,9 +714,8 @@ func (apicast *Apicast) stagingVolumes() []v1.Volume { Name: OpentelemetryConfigurationVolumeName, VolumeSource: v1.VolumeSource{ Secret: &v1.SecretVolumeSource{ - SecretName: apicast.Options.StagingOpentelemetry.Secret.Name, - Optional: &[]bool{false}[0], - DefaultMode: ptr.To(v1.SecretVolumeSourceDefaultMode), + SecretName: apicast.Options.StagingOpentelemetry.Secret.Name, + Optional: &[]bool{false}[0], }, }, }) @@ -764,8 +726,7 @@ func (apicast *Apicast) stagingVolumes() []v1.Volume { Name: customEnvVolumeName(customEnvSecret), VolumeSource: v1.VolumeSource{ Secret: &v1.SecretVolumeSource{ - SecretName: customEnvSecret.GetName(), - DefaultMode: ptr.To(v1.SecretVolumeSourceDefaultMode), + SecretName: customEnvSecret.GetName(), }, }, }) @@ -776,8 +737,7 @@ func (apicast *Apicast) stagingVolumes() []v1.Volume { Name: HTTPSCertificatesVolumeName, VolumeSource: v1.VolumeSource{ Secret: &v1.SecretVolumeSource{ - SecretName: *apicast.Options.StagingHTTPSCertificateSecretName, - DefaultMode: ptr.To(v1.SecretVolumeSourceDefaultMode), + SecretName: *apicast.Options.StagingHTTPSCertificateSecretName, }, }, }) diff --git a/pkg/3scale/amp/component/backend.go b/pkg/3scale/amp/component/backend.go index 30b6d524e..432878fc6 100644 --- a/pkg/3scale/amp/component/backend.go +++ b/pkg/3scale/amp/component/backend.go @@ -131,43 +131,33 @@ func (backend *Backend) WorkerDeployment(ctx context.Context, k8sclient client.C Annotations: deploymentAnnotations, }, Spec: v1.PodSpec{ - Affinity: backend.Options.WorkerAffinity, - Tolerations: backend.Options.WorkerTolerations, - RestartPolicy: v1.RestartPolicyAlways, - DNSPolicy: v1.DNSClusterFirst, - SecurityContext: &v1.PodSecurityContext{}, - TerminationGracePeriodSeconds: ptr.To(int64(v1.DefaultTerminationGracePeriodSeconds)), - SchedulerName: v1.DefaultSchedulerName, - Volumes: backend.backendVolumes(), + Affinity: backend.Options.WorkerAffinity, + Tolerations: backend.Options.WorkerTolerations, + Volumes: backend.backendVolumes(), InitContainers: []v1.Container{ { - Name: "backend-redis-svc", - Image: containerImage, - ImagePullPolicy: v1.PullIfNotPresent, + Name: "backend-redis-svc", + Image: containerImage, Command: []string{ "/opt/app/entrypoint.sh", "sh", "-c", "until rake connectivity:redis_storage_queue_check; do sleep $SLEEP_SECONDS; done", }, - VolumeMounts: backend.backendContainerVolumeMounts(), - Env: append(backend.buildBackendCommonEnv(), helper.EnvVarFromValue("SLEEP_SECONDS", "1")), - TerminationMessagePath: v1.TerminationMessagePathDefault, - TerminationMessagePolicy: v1.TerminationMessageReadFile, + VolumeMounts: backend.backendContainerVolumeMounts(), + Env: append(backend.buildBackendCommonEnv(), helper.EnvVarFromValue("SLEEP_SECONDS", "1")), }, }, Containers: []v1.Container{ { - Name: BackendWorkerName, - Image: containerImage, - Args: []string{"bin/3scale_backend_worker", "run"}, - Env: backend.buildBackendWorkerEnv(), - Resources: backend.Options.WorkerResourceRequirements, - VolumeMounts: backend.backendContainerVolumeMounts(), - ImagePullPolicy: v1.PullIfNotPresent, - Ports: backend.workerPorts(), - TerminationMessagePath: v1.TerminationMessagePathDefault, - TerminationMessagePolicy: v1.TerminationMessageReadFile, + Name: BackendWorkerName, + Image: containerImage, + Args: []string{"bin/3scale_backend_worker", "run"}, + Env: backend.buildBackendWorkerEnv(), + Resources: backend.Options.WorkerResourceRequirements, + VolumeMounts: backend.backendContainerVolumeMounts(), + ImagePullPolicy: v1.PullIfNotPresent, + Ports: backend.workerPorts(), LivenessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{ HTTPGet: &v1.HTTPGetAction{ @@ -233,42 +223,32 @@ func (backend *Backend) CronDeployment(ctx context.Context, k8sclient client.Cli Annotations: deploymentAnnotations, }, Spec: v1.PodSpec{ - Affinity: backend.Options.CronAffinity, - Tolerations: backend.Options.CronTolerations, - RestartPolicy: v1.RestartPolicyAlways, - DNSPolicy: v1.DNSClusterFirst, - SecurityContext: &v1.PodSecurityContext{}, - TerminationGracePeriodSeconds: ptr.To(int64(v1.DefaultTerminationGracePeriodSeconds)), - SchedulerName: v1.DefaultSchedulerName, - Volumes: backend.backendVolumes(), + Affinity: backend.Options.CronAffinity, + Tolerations: backend.Options.CronTolerations, + Volumes: backend.backendVolumes(), InitContainers: []v1.Container{ { - Name: "backend-redis-svc", - Image: containerImage, - ImagePullPolicy: v1.PullIfNotPresent, + Name: "backend-redis-svc", + Image: containerImage, Command: []string{ "/opt/app/entrypoint.sh", "sh", "-c", "until rake connectivity:redis_storage_queue_check; do sleep $SLEEP_SECONDS; done", }, - VolumeMounts: backend.backendContainerVolumeMounts(), - Env: append(backend.buildBackendCommonEnv(), helper.EnvVarFromValue("SLEEP_SECONDS", "1")), - TerminationMessagePath: v1.TerminationMessagePathDefault, - TerminationMessagePolicy: v1.TerminationMessageReadFile, + VolumeMounts: backend.backendContainerVolumeMounts(), + Env: append(backend.buildBackendCommonEnv(), helper.EnvVarFromValue("SLEEP_SECONDS", "1")), }, }, Containers: []v1.Container{ { - Name: "backend-cron", - Image: containerImage, - Args: []string{"touch /tmp/healthy && backend-cron"}, - Env: backend.buildBackendCronEnv(), - VolumeMounts: backend.backendContainerVolumeMounts(), - Resources: backend.Options.CronResourceRequirements, - ImagePullPolicy: v1.PullIfNotPresent, - TerminationMessagePath: v1.TerminationMessagePathDefault, - TerminationMessagePolicy: v1.TerminationMessageReadFile, + Name: "backend-cron", + Image: containerImage, + Args: []string{"touch /tmp/healthy && backend-cron"}, + Env: backend.buildBackendCronEnv(), + VolumeMounts: backend.backendContainerVolumeMounts(), + Resources: backend.Options.CronResourceRequirements, + ImagePullPolicy: v1.PullIfNotPresent, LivenessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{ Exec: &v1.ExecAction{ @@ -332,26 +312,18 @@ func (backend *Backend) ListenerDeployment(ctx context.Context, k8sclient client Annotations: deploymentAnnotations, }, Spec: v1.PodSpec{ - Affinity: backend.Options.ListenerAffinity, - Tolerations: backend.Options.ListenerTolerations, - RestartPolicy: v1.RestartPolicyAlways, - DNSPolicy: v1.DNSClusterFirst, - SecurityContext: &v1.PodSecurityContext{}, - TerminationGracePeriodSeconds: ptr.To(int64(v1.DefaultTerminationGracePeriodSeconds)), - SchedulerName: v1.DefaultSchedulerName, - Volumes: backend.backendVolumes(), + Affinity: backend.Options.ListenerAffinity, + Tolerations: backend.Options.ListenerTolerations, + Volumes: backend.backendVolumes(), Containers: []v1.Container{ { - Name: BackendListenerName, - Image: containerImage, - ImagePullPolicy: v1.PullIfNotPresent, - Args: backend.backendListenerRunArgs(), - Ports: backend.listenerPorts(), - Env: backend.buildBackendListenerEnv(), - Resources: backend.Options.ListenerResourceRequirements, - VolumeMounts: backend.backendContainerVolumeMounts(), - TerminationMessagePath: v1.TerminationMessagePathDefault, - TerminationMessagePolicy: v1.TerminationMessageReadFile, + Name: BackendListenerName, + Image: containerImage, + Args: backend.backendListenerRunArgs(), + Ports: backend.listenerPorts(), + Env: backend.buildBackendListenerEnv(), + Resources: backend.Options.ListenerResourceRequirements, + VolumeMounts: backend.backendContainerVolumeMounts(), LivenessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{ TCPSocket: &v1.TCPSocketAction{ @@ -370,8 +342,11 @@ func (backend *Backend) ListenerDeployment(ctx context.Context, k8sclient client ReadinessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{ HTTPGet: &v1.HTTPGetAction{ - Path: "/status", - Port: intstr.IntOrString{Type: intstr.Int, IntVal: 3000}, + Path: "/status", + Port: intstr.IntOrString{ + Type: intstr.Int, + IntVal: 3000, + }, Scheme: v1.URISchemeHTTP, }, }, @@ -381,6 +356,7 @@ func (backend *Backend) ListenerDeployment(ctx context.Context, k8sclient client SuccessThreshold: 1, FailureThreshold: 3, }, + ImagePullPolicy: v1.PullIfNotPresent, }, }, ServiceAccountName: "amp", diff --git a/pkg/3scale/amp/component/memcached.go b/pkg/3scale/amp/component/memcached.go index be44946ae..3e413b58c 100644 --- a/pkg/3scale/amp/component/memcached.go +++ b/pkg/3scale/amp/component/memcached.go @@ -7,7 +7,6 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/utils/ptr" ) const ( @@ -48,20 +47,14 @@ func (m *Memcached) Deployment(containerImage string) *k8sappsv1.Deployment { Annotations: m.Options.PodTemplateAnnotations, }, Spec: v1.PodSpec{ - Affinity: m.Options.Affinity, - Tolerations: m.Options.Tolerations, - ServiceAccountName: "amp", // TODO make this configurable via flag - RestartPolicy: v1.RestartPolicyAlways, - DNSPolicy: v1.DNSClusterFirst, - SecurityContext: &v1.PodSecurityContext{}, - TerminationGracePeriodSeconds: ptr.To(int64(v1.DefaultTerminationGracePeriodSeconds)), - SchedulerName: v1.DefaultSchedulerName, + Affinity: m.Options.Affinity, + Tolerations: m.Options.Tolerations, + ServiceAccountName: "amp", // TODO make this configurable via flag Containers: []v1.Container{ { - Name: "memcache", - Image: containerImage, - ImagePullPolicy: v1.PullIfNotPresent, - Command: []string{"memcached", "-m", "64"}, + Name: "memcache", + Image: containerImage, + Command: []string{"memcached", "-m", "64"}, Ports: []v1.ContainerPort{ { HostPort: 0, @@ -69,9 +62,7 @@ func (m *Memcached) Deployment(containerImage string) *k8sappsv1.Deployment { Protocol: v1.ProtocolTCP, }, }, - Resources: m.Options.ResourceRequirements, - TerminationMessagePath: v1.TerminationMessagePathDefault, - TerminationMessagePolicy: v1.TerminationMessageReadFile, + Resources: m.Options.ResourceRequirements, LivenessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{ TCPSocket: &v1.TCPSocketAction{ @@ -82,10 +73,10 @@ func (m *Memcached) Deployment(containerImage string) *k8sappsv1.Deployment { }, }, InitialDelaySeconds: 10, - TimeoutSeconds: 1, + TimeoutSeconds: 0, PeriodSeconds: 10, - SuccessThreshold: 1, - FailureThreshold: 3, + SuccessThreshold: 0, + FailureThreshold: 0, }, ReadinessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{ @@ -99,9 +90,10 @@ func (m *Memcached) Deployment(containerImage string) *k8sappsv1.Deployment { InitialDelaySeconds: 10, TimeoutSeconds: 5, PeriodSeconds: 30, - SuccessThreshold: 1, - FailureThreshold: 3, + SuccessThreshold: 0, + FailureThreshold: 0, }, + ImagePullPolicy: v1.PullIfNotPresent, }, }, PriorityClassName: m.Options.PriorityClassName, diff --git a/pkg/3scale/amp/component/system.go b/pkg/3scale/amp/component/system.go index d5188f395..37285d0b7 100644 --- a/pkg/3scale/amp/component/system.go +++ b/pkg/3scale/amp/component/system.go @@ -570,7 +570,6 @@ func (system *System) appPodVolumes() []v1.Volume { Path: "service_discovery.yml", }, }, - DefaultMode: ptr.To(v1.ConfigMapVolumeSourceDefaultMode), }, }, } @@ -676,27 +675,19 @@ func (system *System) AppDeployment(ctx context.Context, k8sclient client.Client Annotations: system.appPodAnnotations(watchedSecretAnnotations), }, Spec: v1.PodSpec{ - Affinity: system.Options.AppAffinity, - Tolerations: system.Options.AppTolerations, - RestartPolicy: v1.RestartPolicyAlways, - DNSPolicy: v1.DNSClusterFirst, - SecurityContext: &v1.PodSecurityContext{}, - TerminationGracePeriodSeconds: ptr.To(int64(v1.DefaultTerminationGracePeriodSeconds)), - SchedulerName: v1.DefaultSchedulerName, - Volumes: system.appPodVolumes(), - InitContainers: system.systemInit(containerImage), + Affinity: system.Options.AppAffinity, + Tolerations: system.Options.AppTolerations, + Volumes: system.appPodVolumes(), + InitContainers: system.systemInit(containerImage), Containers: []v1.Container{ { - Name: SystemAppMasterContainerName, - Image: containerImage, - ImagePullPolicy: v1.PullIfNotPresent, - Args: []string{"env", "TENANT_MODE=master", "PORT=3002", "container-entrypoint", "bundle", "exec", "unicorn", "-c", "config/unicorn.rb", "-E", "production", "config.ru"}, - Ports: system.appMasterPorts(), - Env: system.buildAppMasterContainerEnv(), - Resources: *system.Options.AppMasterContainerResourceRequirements, - VolumeMounts: system.appMasterContainerVolumeMounts(), - TerminationMessagePath: v1.TerminationMessagePathDefault, - TerminationMessagePolicy: v1.TerminationMessageReadFile, + Name: SystemAppMasterContainerName, + Image: containerImage, + Args: []string{"env", "TENANT_MODE=master", "PORT=3002", "container-entrypoint", "bundle", "exec", "unicorn", "-c", "config/unicorn.rb", "-E", "production", "config.ru"}, + Ports: system.appMasterPorts(), + Env: system.buildAppMasterContainerEnv(), + Resources: *system.Options.AppMasterContainerResourceRequirements, + VolumeMounts: system.appMasterContainerVolumeMounts(), LivenessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{ TCPSocket: &v1.TCPSocketAction{ @@ -709,7 +700,7 @@ func (system *System) AppDeployment(ctx context.Context, k8sclient client.Client InitialDelaySeconds: 40, TimeoutSeconds: 10, PeriodSeconds: 10, - SuccessThreshold: 1, + SuccessThreshold: 0, FailureThreshold: 40, }, ReadinessProbe: &v1.Probe{ @@ -732,24 +723,22 @@ func (system *System) AppDeployment(ctx context.Context, k8sclient client.Client InitialDelaySeconds: 60, TimeoutSeconds: 10, PeriodSeconds: 30, - SuccessThreshold: 1, + SuccessThreshold: 0, FailureThreshold: 10, }, - Stdin: false, - StdinOnce: false, - TTY: false, + ImagePullPolicy: v1.PullIfNotPresent, + Stdin: false, + StdinOnce: false, + TTY: false, }, { - Name: SystemAppProviderContainerName, - Image: containerImage, - ImagePullPolicy: v1.PullIfNotPresent, - Args: []string{"env", "TENANT_MODE=provider", "PORT=3000", "container-entrypoint", "bundle", "exec", "unicorn", "-c", "config/unicorn.rb", "-E", "production", "config.ru"}, - Ports: system.appProviderPorts(), - Env: system.buildAppProviderContainerEnv(), - Resources: *system.Options.AppProviderContainerResourceRequirements, - VolumeMounts: system.appProviderContainerVolumeMounts(), - TerminationMessagePath: v1.TerminationMessagePathDefault, - TerminationMessagePolicy: v1.TerminationMessageReadFile, + Name: SystemAppProviderContainerName, + Image: containerImage, + Args: []string{"env", "TENANT_MODE=provider", "PORT=3000", "container-entrypoint", "bundle", "exec", "unicorn", "-c", "config/unicorn.rb", "-E", "production", "config.ru"}, + Ports: system.appProviderPorts(), + Env: system.buildAppProviderContainerEnv(), + Resources: *system.Options.AppProviderContainerResourceRequirements, + VolumeMounts: system.appProviderContainerVolumeMounts(), LivenessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{ TCPSocket: &v1.TCPSocketAction{ @@ -762,7 +751,7 @@ func (system *System) AppDeployment(ctx context.Context, k8sclient client.Client InitialDelaySeconds: 40, TimeoutSeconds: 10, PeriodSeconds: 10, - SuccessThreshold: 1, + SuccessThreshold: 0, FailureThreshold: 40, }, ReadinessProbe: &v1.Probe{ @@ -785,24 +774,22 @@ func (system *System) AppDeployment(ctx context.Context, k8sclient client.Client InitialDelaySeconds: 60, TimeoutSeconds: 10, PeriodSeconds: 30, - SuccessThreshold: 1, + SuccessThreshold: 0, FailureThreshold: 10, }, - Stdin: false, - StdinOnce: false, - TTY: false, + ImagePullPolicy: v1.PullIfNotPresent, + Stdin: false, + StdinOnce: false, + TTY: false, }, { - Name: SystemAppDeveloperContainerName, - Image: containerImage, - ImagePullPolicy: v1.PullIfNotPresent, - Args: []string{"env", "PORT=3001", "container-entrypoint", "bundle", "exec", "unicorn", "-c", "config/unicorn.rb", "-E", "production", "config.ru"}, - Ports: system.appDeveloperPorts(), - Env: system.buildAppDeveloperContainerEnv(), - Resources: *system.Options.AppDeveloperContainerResourceRequirements, - VolumeMounts: system.appDeveloperContainerVolumeMounts(), - TerminationMessagePath: v1.TerminationMessagePathDefault, - TerminationMessagePolicy: v1.TerminationMessageReadFile, + Name: SystemAppDeveloperContainerName, + Image: containerImage, + Args: []string{"env", "PORT=3001", "container-entrypoint", "bundle", "exec", "unicorn", "-c", "config/unicorn.rb", "-E", "production", "config.ru"}, + Ports: system.appDeveloperPorts(), + Env: system.buildAppDeveloperContainerEnv(), + Resources: *system.Options.AppDeveloperContainerResourceRequirements, + VolumeMounts: system.appDeveloperContainerVolumeMounts(), LivenessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{ TCPSocket: &v1.TCPSocketAction{ @@ -815,7 +802,7 @@ func (system *System) AppDeployment(ctx context.Context, k8sclient client.Client InitialDelaySeconds: 40, TimeoutSeconds: 10, PeriodSeconds: 10, - SuccessThreshold: 1, + SuccessThreshold: 0, FailureThreshold: 40, }, ReadinessProbe: &v1.Probe{ @@ -838,9 +825,10 @@ func (system *System) AppDeployment(ctx context.Context, k8sclient client.Client InitialDelaySeconds: 60, TimeoutSeconds: 10, PeriodSeconds: 30, - SuccessThreshold: 1, + SuccessThreshold: 0, FailureThreshold: 10, }, + ImagePullPolicy: v1.PullIfNotPresent, }, }, ServiceAccountName: "amp", @@ -977,7 +965,6 @@ func (system *System) SidekiqPodVolumes() []v1.Volume { Path: "service_discovery.yml", }, }, - DefaultMode: ptr.To(v1.ConfigMapVolumeSourceDefaultMode), }, }, } @@ -1082,27 +1069,20 @@ func (system *System) SidekiqDeployment(ctx context.Context, k8sclient client.Cl Annotations: system.sidekiqPodAnnotations(watchedSecretAnnotations), }, Spec: v1.PodSpec{ - Affinity: system.Options.SidekiqAffinity, - Tolerations: system.Options.SidekiqTolerations, - RestartPolicy: v1.RestartPolicyAlways, - DNSPolicy: v1.DNSClusterFirst, - SecurityContext: &v1.PodSecurityContext{}, - TerminationGracePeriodSeconds: ptr.To(int64(v1.DefaultTerminationGracePeriodSeconds)), - SchedulerName: v1.DefaultSchedulerName, - Volumes: system.SidekiqPodVolumes(), - InitContainers: system.sidekiqInit(containerImage), + Affinity: system.Options.SidekiqAffinity, + Tolerations: system.Options.SidekiqTolerations, + Volumes: system.SidekiqPodVolumes(), + InitContainers: system.sidekiqInit(containerImage), Containers: []v1.Container{ { - Name: SystemSidekiqName, - Image: containerImage, - Args: []string{"rake", "sidekiq:worker", "RAILS_MAX_THREADS=25"}, - Env: system.buildSystemSidekiqContainerEnv(), - Resources: *system.Options.SidekiqContainerResourceRequirements, - VolumeMounts: system.sidekiqContainerVolumeMounts(), - ImagePullPolicy: v1.PullIfNotPresent, - Ports: system.sideKiqPorts(), - TerminationMessagePath: v1.TerminationMessagePathDefault, - TerminationMessagePolicy: v1.TerminationMessageReadFile, + Name: SystemSidekiqName, + Image: containerImage, + Args: []string{"rake", "sidekiq:worker", "RAILS_MAX_THREADS=25"}, + Env: system.buildSystemSidekiqContainerEnv(), + Resources: *system.Options.SidekiqContainerResourceRequirements, + VolumeMounts: system.sidekiqContainerVolumeMounts(), + ImagePullPolicy: v1.PullIfNotPresent, + Ports: system.sideKiqPorts(), LivenessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{ TCPSocket: &v1.TCPSocketAction{ @@ -1523,14 +1503,14 @@ func (system *System) systemInit(containerImage string) []v1.Container { if system.Options.SystemDbTLSEnabled { return []v1.Container{ { - Name: "set-permissions", - Image: containerImage, // Minimal image for chmod - ImagePullPolicy: v1.PullIfNotPresent, + Name: "set-permissions", + Image: containerImage, // Minimal image for chmod Command: []string{ "sh", "-c", "cp /tls/* /writable-tls/ && chmod 0600 /writable-tls/*", }, + ImagePullPolicy: v1.PullIfNotPresent, TerminationMessagePath: v1.TerminationMessagePathDefault, TerminationMessagePolicy: v1.TerminationMessageReadFile, VolumeMounts: []v1.VolumeMount{ @@ -1556,15 +1536,15 @@ func (system *System) sidekiqInit(containerImage string) []v1.Container { var containers []v1.Container // Base init container setup initContainer := v1.Container{ - Name: SystemSideKiqInitContainerName, - Image: containerImage, - ImagePullPolicy: v1.PullIfNotPresent, + Name: SystemSideKiqInitContainerName, + Image: containerImage, Command: []string{ "bash", "-c", "bundle exec sh -c \"until rake boot:redis && curl --output /dev/null --silent --fail --head http://system-master:3000/status; do sleep $SLEEP_SECONDS; done\"", }, Env: append(system.SystemRedisEnvVars(), helper.EnvVarFromValue("SLEEP_SECONDS", "1")), + ImagePullPolicy: v1.PullIfNotPresent, TerminationMessagePath: v1.TerminationMessagePathDefault, TerminationMessagePolicy: v1.TerminationMessageReadFile, } @@ -1576,14 +1556,14 @@ func (system *System) sidekiqInit(containerImage string) []v1.Container { if system.Options.SystemDbTLSEnabled { // Set-permissions container for DB TLS containers = append(containers, v1.Container{ - Name: "set-permissions", - Image: containerImage, - ImagePullPolicy: v1.PullIfNotPresent, + Name: "set-permissions", + Image: containerImage, Command: []string{ "sh", "-c", "cp /tls/* /writable-tls/ && chmod 0600 /writable-tls/*", }, + ImagePullPolicy: v1.PullIfNotPresent, TerminationMessagePath: v1.TerminationMessagePathDefault, TerminationMessagePolicy: v1.TerminationMessageReadFile, VolumeMounts: []v1.VolumeMount{ diff --git a/pkg/3scale/amp/component/system_searchd.go b/pkg/3scale/amp/component/system_searchd.go index 9119c7beb..16dd9fd1a 100644 --- a/pkg/3scale/amp/component/system_searchd.go +++ b/pkg/3scale/amp/component/system_searchd.go @@ -9,7 +9,6 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -84,24 +83,17 @@ func (s *SystemSearchd) Deployment(ctx context.Context, k8sclient client.Client, Annotations: s.searchdPodAnnotations(watchedSecretAnnotations), }, Spec: v1.PodSpec{ - InitContainers: s.searchdInit(containerImage), - Affinity: s.Options.Affinity, - Tolerations: s.Options.Tolerations, - ServiceAccountName: "amp", - RestartPolicy: v1.RestartPolicyAlways, - DNSPolicy: v1.DNSClusterFirst, - SecurityContext: &v1.PodSecurityContext{}, - TerminationGracePeriodSeconds: ptr.To(int64(v1.DefaultTerminationGracePeriodSeconds)), - SchedulerName: v1.DefaultSchedulerName, - Volumes: s.searchdVolume(), + InitContainers: s.searchdInit(containerImage), + Affinity: s.Options.Affinity, + Tolerations: s.Options.Tolerations, + ServiceAccountName: "amp", + Volumes: s.searchdVolume(), Containers: []v1.Container{ { - Name: SystemSearchdDeploymentName, - Image: containerImage, - ImagePullPolicy: v1.PullIfNotPresent, - VolumeMounts: s.searchDVolumeMounts(), - TerminationMessagePath: v1.TerminationMessagePathDefault, - TerminationMessagePolicy: v1.TerminationMessageReadFile, + Name: SystemSearchdDeploymentName, + Image: containerImage, + ImagePullPolicy: v1.PullIfNotPresent, + VolumeMounts: s.searchDVolumeMounts(), LivenessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{ TCPSocket: &v1.TCPSocketAction{ @@ -183,15 +175,13 @@ func (s *SystemSearchd) ReindexingJob(containerImage string, system *System) *ba InitContainers: s.searchdInit(containerImage), Containers: []v1.Container{ { - Name: SystemSearchdReindexJobName, - Image: containerImage, - Args: []string{"bash", "-c", "bundle exec rake searchd:optimal_index"}, - Env: system.buildSystemBaseEnv(), - Resources: s.Options.ContainerResourceRequirements, - ImagePullPolicy: v1.PullIfNotPresent, - VolumeMounts: s.searchdManticoreVolumeMounts(), - TerminationMessagePath: v1.TerminationMessagePathDefault, - TerminationMessagePolicy: v1.TerminationMessageReadFile, + Name: SystemSearchdReindexJobName, + Image: containerImage, + Args: []string{"bash", "-c", "bundle exec rake searchd:optimal_index"}, + Env: system.buildSystemBaseEnv(), + Resources: s.Options.ContainerResourceRequirements, + ImagePullPolicy: v1.PullIfNotPresent, + VolumeMounts: s.searchdManticoreVolumeMounts(), }, }, Volumes: s.searchdJobVolume(), diff --git a/pkg/3scale/amp/component/zync.go b/pkg/3scale/amp/component/zync.go index 258391b09..326c295de 100644 --- a/pkg/3scale/amp/component/zync.go +++ b/pkg/3scale/amp/component/zync.go @@ -223,24 +223,16 @@ func (zync *Zync) Deployment(ctx context.Context, k8sclient client.Client, conta Annotations: zync.zyncPodAnnotations(watchedSecretAnnotations), }, Spec: v1.PodSpec{ - Affinity: zync.Options.ZyncAffinity, - Tolerations: zync.Options.ZyncTolerations, - ServiceAccountName: "amp", - RestartPolicy: v1.RestartPolicyAlways, - DNSPolicy: v1.DNSClusterFirst, - SecurityContext: &v1.PodSecurityContext{}, - TerminationGracePeriodSeconds: ptr.To(int64(v1.DefaultTerminationGracePeriodSeconds)), - SchedulerName: v1.DefaultSchedulerName, - InitContainers: zync.zyncInit(containerImage), + Affinity: zync.Options.ZyncAffinity, + Tolerations: zync.Options.ZyncTolerations, + ServiceAccountName: "amp", + InitContainers: zync.zyncInit(containerImage), Containers: []v1.Container{ { - Name: ZyncName, - Image: containerImage, - ImagePullPolicy: v1.PullIfNotPresent, - Ports: zync.zyncPorts(), - Env: zync.commonZyncEnvVars(), - TerminationMessagePath: v1.TerminationMessagePathDefault, - TerminationMessagePolicy: v1.TerminationMessageReadFile, + Name: ZyncName, + Image: containerImage, + Ports: zync.zyncPorts(), + Env: zync.commonZyncEnvVars(), LivenessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{ HTTPGet: &v1.HTTPGetAction{ @@ -388,20 +380,15 @@ func (zync *Zync) QueDeployment(ctx context.Context, k8sclient client.Client, co Tolerations: zync.Options.ZyncQueTolerations, ServiceAccountName: "zync-que-sa", RestartPolicy: v1.RestartPolicyAlways, - DNSPolicy: v1.DNSClusterFirst, - SecurityContext: &v1.PodSecurityContext{}, - TerminationGracePeriodSeconds: ptr.To(int64(v1.DefaultTerminationGracePeriodSeconds)), - SchedulerName: v1.DefaultSchedulerName, + TerminationGracePeriodSeconds: &[]int64{30}[0], InitContainers: zync.zyncQueInit(containerImage), Containers: []v1.Container{ { - Name: "que", - Command: []string{"/usr/bin/bash"}, - Args: []string{"-c", "bundle exec rake 'que[--worker-count 10]'"}, - Image: containerImage, - ImagePullPolicy: v1.PullAlways, - TerminationMessagePath: v1.TerminationMessagePathDefault, - TerminationMessagePolicy: v1.TerminationMessageReadFile, + Name: "que", + Command: []string{"/usr/bin/bash"}, + Args: []string{"-c", "bundle exec rake 'que[--worker-count 10]'"}, + Image: containerImage, + ImagePullPolicy: v1.PullAlways, LivenessProbe: &v1.Probe{ FailureThreshold: 3, InitialDelaySeconds: 10, @@ -462,14 +449,10 @@ func (zync *Zync) DatabaseDeployment(containerImage string) *k8sappsv1.Deploymen Annotations: zync.Options.ZyncDatabasePodTemplateAnnotations, }, Spec: v1.PodSpec{ - Affinity: zync.Options.ZyncDatabaseAffinity, - Tolerations: zync.Options.ZyncDatabaseTolerations, - RestartPolicy: v1.RestartPolicyAlways, - DNSPolicy: v1.DNSClusterFirst, - SecurityContext: &v1.PodSecurityContext{}, - TerminationGracePeriodSeconds: ptr.To(int64(v1.DefaultTerminationGracePeriodSeconds)), - SchedulerName: v1.DefaultSchedulerName, - ServiceAccountName: "amp", + Affinity: zync.Options.ZyncDatabaseAffinity, + Tolerations: zync.Options.ZyncDatabaseTolerations, + RestartPolicy: v1.RestartPolicyAlways, + ServiceAccountName: "amp", Containers: []v1.Container{ { Name: "postgresql", @@ -486,9 +469,7 @@ func (zync *Zync) DatabaseDeployment(containerImage string) *k8sappsv1.Deploymen MountPath: "/var/lib/pgsql/data", }, }, - ImagePullPolicy: v1.PullIfNotPresent, - TerminationMessagePath: v1.TerminationMessagePathDefault, - TerminationMessagePolicy: v1.TerminationMessageReadFile, + ImagePullPolicy: v1.PullIfNotPresent, Env: []v1.EnvVar{ { Name: "POSTGRESQL_USER", @@ -520,23 +501,17 @@ func (zync *Zync) DatabaseDeployment(containerImage string) *k8sappsv1.Deploymen Port: intstr.FromInt32(5432), }, }, - InitialDelaySeconds: 30, TimeoutSeconds: 1, - PeriodSeconds: 10, - SuccessThreshold: 1, - FailureThreshold: 3, + InitialDelaySeconds: 30, }, ReadinessProbe: &v1.Probe{ + TimeoutSeconds: 1, + InitialDelaySeconds: 5, ProbeHandler: v1.ProbeHandler{ Exec: &v1.ExecAction{ Command: []string{"/bin/sh", "-i", "-c", "psql -h 127.0.0.1 -U zync -q -d zync_production -c 'SELECT 1'"}, }, }, - InitialDelaySeconds: 5, - TimeoutSeconds: 1, - PeriodSeconds: 10, - SuccessThreshold: 1, - FailureThreshold: 3, }, Resources: zync.Options.DatabaseContainerResourceRequirements, }, @@ -661,14 +636,14 @@ func (zync *Zync) zyncInit(containerImage string) []v1.Container { if zync.Options.ZyncDbTLSEnabled { return []v1.Container{ { - Name: "set-permissions", - Image: containerImage, - ImagePullPolicy: v1.PullIfNotPresent, + Name: "set-permissions", + Image: containerImage, Command: []string{ "sh", "-c", "cp /tls/* /writable-tls/ && chmod 0600 /writable-tls/*", }, + ImagePullPolicy: v1.PullIfNotPresent, TerminationMessagePath: v1.TerminationMessagePathDefault, TerminationMessagePolicy: v1.TerminationMessageReadFile, VolumeMounts: []v1.VolumeMount{ @@ -685,14 +660,14 @@ func (zync *Zync) zyncInit(containerImage string) []v1.Container { }, }, { - Name: ZyncInitContainerName, - Image: containerImage, - ImagePullPolicy: v1.PullIfNotPresent, + Name: ZyncInitContainerName, + Image: containerImage, Command: []string{ "bash", "-c", "bundle exec sh -c \"until rake boot:db; do sleep $SLEEP_SECONDS; done\"", }, + ImagePullPolicy: v1.PullIfNotPresent, TerminationMessagePath: v1.TerminationMessagePathDefault, TerminationMessagePolicy: v1.TerminationMessageReadFile, Env: []v1.EnvVar{ @@ -729,14 +704,14 @@ func (zync *Zync) zyncInit(containerImage string) []v1.Container { } else { return []v1.Container{ { - Name: ZyncInitContainerName, - Image: containerImage, - ImagePullPolicy: v1.PullIfNotPresent, + Name: ZyncInitContainerName, + Image: containerImage, Command: []string{ "bash", "-c", "bundle exec sh -c \"until rake boot:db; do sleep $SLEEP_SECONDS; done\"", }, + ImagePullPolicy: v1.PullIfNotPresent, TerminationMessagePath: v1.TerminationMessagePathDefault, TerminationMessagePolicy: v1.TerminationMessageReadFile, Env: []v1.EnvVar{ @@ -825,6 +800,9 @@ func (zync *Zync) zyncQueInit(containerImage string) []v1.Container { "-c", "cp /tls/* /writable-tls/ && chmod 0600 /writable-tls/*", }, + ImagePullPolicy: v1.PullIfNotPresent, + TerminationMessagePath: v1.TerminationMessagePathDefault, + TerminationMessagePolicy: v1.TerminationMessageReadFile, VolumeMounts: []v1.VolumeMount{ { Name: "tls-secret", diff --git a/pkg/reconcilers/base_reconciler.go b/pkg/reconcilers/base_reconciler.go index a1ff7c9e5..14ad55633 100644 --- a/pkg/reconcilers/base_reconciler.go +++ b/pkg/reconcilers/base_reconciler.go @@ -67,13 +67,6 @@ func (b *BaseReconciler) Reconcile(ctx context.Context, req reconcile.Request) ( return reconcile.Result{}, nil } -// WithRequest returns a shallow copy with a logger enriched with the request's namespace/name. -func (b *BaseReconciler) WithRequest(req reconcile.Request) *BaseReconciler { - c := *b - c.logger = b.logger.WithValues("namespace", req.Namespace, "name", req.Name) - return &c -} - // Client returns a split client that reads objects from // the cache and writes to the Kubernetes APIServer func (b *BaseReconciler) Client() client.Client { diff --git a/test/integration/apimanager_controller_test.go b/test/integration/apimanager_controller_test.go index a2fa6b6ae..7b53f9735 100644 --- a/test/integration/apimanager_controller_test.go +++ b/test/integration/apimanager_controller_test.go @@ -211,8 +211,14 @@ var _ = Describe("APIManager controller", func() { waitForAPIManagerAvailableCondition(5*time.Second, 15*time.Minute, apimanager, GinkgoWriter) fmt.Fprintf(GinkgoWriter, "APIManager 'Available' condition is true\n") + // Verify no perpetual reconciliation occurred during initial deployment. + verifyNoDeploymentUpdates(apimanager.Namespace, apimanager.Name, 0, GinkgoWriter) + + // Trigger a synthetic change and confirm the operator corrects it exactly once. triggerSyntheticDeploymentUpdate(testNamespace, GinkgoWriter) - verifyNoDeploymentUpdates(apimanager.Namespace, apimanager.Name, GinkgoWriter) + time.Sleep(settlingPeriod) + verifyNoDeploymentUpdates(apimanager.Namespace, apimanager.Name, 1, GinkgoWriter) + elapsed := time.Since(start) fmt.Fprintf(GinkgoWriter, "APIManager creation and availability took '%s'\n", elapsed) }) diff --git a/test/integration/verify_no_perpetual_reconciliation_test.go b/test/integration/verify_no_perpetual_reconciliation_test.go index 21618a768..33dc2a829 100644 --- a/test/integration/verify_no_perpetual_reconciliation_test.go +++ b/test/integration/verify_no_perpetual_reconciliation_test.go @@ -4,40 +4,27 @@ import ( "fmt" "io" "sort" - "strings" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) -// minTotalDeploymentUpdates is a floor that guards against a silent counter -// failure. A synthetic image change is applied to system-memcache before -// calling verifyNoDeploymentUpdates (see triggerSyntheticDeploymentUpdate), -// which guarantees the operator issues at least one real UPDATE reconciling -// the deployment back to the desired image. A count of 0 therefore means the -// counter is broken (namespace/name fields not captured) rather than the -// install being genuinely update-free. -const minTotalDeploymentUpdates = 1 +// settlingPeriod is the time to wait after the synthetic update is reconciled +// to confirm no further updates occur. +const settlingPeriod = 30 * time.Second -// maxTotalDeploymentUpdates is the ceiling on the total number of Deployment -// update calls for a single APIManager CR instance over the full test session. -// -// Rationale: during a normal install each deployment receives 1–3 legitimate -// updates as pods roll out and probes settle, giving ~12–36 total for 12 -// deployments. The ceiling is set at 50 to absorb timing variance while -// staying well below what the perpetual-reconcile bug produced (~7 deployments -// × many cycles = hundreds of updates per install). -const maxTotalDeploymentUpdates = 50 - -// verifyNoDeploymentUpdates asserts that the total deployment update count -// for the given APIManager CR over the full test session is within the ceiling. -// On failure it lists each deployment's individual count to aid diagnosis. -func verifyNoDeploymentUpdates(namespace, name string, w io.Writer) { +// verifyNoDeploymentUpdates asserts that the reconcile counter recorded exactly +// the expected number of deployment updates. Before the synthetic trigger this +// should be 0 (no perpetual reconcile during initial deployment). After the +// synthetic trigger it should be exactly 1 (the operator corrected the drift +// and stopped). +func verifyNoDeploymentUpdates(namespace, name string, expected int, w io.Writer) { updateCounts := reconcileCounter.GetUpdateCounts(namespace, name) totalUpdates := reconcileCounter.GetTotalUpdates(namespace, name) - fmt.Fprintf(w, "\n=== Deployment Update Report (%s/%s, session ceiling %d) ===\n", - namespace, name, maxTotalDeploymentUpdates) + fmt.Fprintf(w, "\n=== Deployment Update Report (%s/%s, expected %d) ===\n", + namespace, name, expected) fmt.Fprintf(w, "Total: %d\n", totalUpdates) names := make([]string, 0, len(updateCounts)) @@ -50,27 +37,25 @@ func verifyNoDeploymentUpdates(namespace, name string, w io.Writer) { } fmt.Fprintf(w, "=============================================================\n\n") - Expect(totalUpdates).To(BeNumerically(">=", minTotalDeploymentUpdates), - fmt.Sprintf("%s/%s: total deployment updates is 0 — counter is likely misconfigured (namespace/name fields not captured)", namespace, name)) - Expect(totalUpdates).To(BeNumerically("<=", maxTotalDeploymentUpdates), - deploymentUpdateDetail(namespace, name, updateCounts, totalUpdates)) + Expect(totalUpdates).To(Equal(expected), + deploymentUpdateDetail(namespace, name, updateCounts, totalUpdates, expected)) } // deploymentUpdateDetail builds a human-readable breakdown for use in a Gomega // failure message so the offending deployments are immediately visible. -func deploymentUpdateDetail(namespace, name string, counts map[string]int, total int) string { - var sb strings.Builder - fmt.Fprintf(&sb, "%s/%s: total deployment updates %d exceeded ceiling %d; per-deployment breakdown:\n", - namespace, name, total, maxTotalDeploymentUpdates) +func deploymentUpdateDetail(namespace, name string, counts map[string]int, total, expected int) string { + var sb string + sb = fmt.Sprintf("%s/%s: total deployment updates %d, expected %d; per-deployment breakdown:\n", + namespace, name, total, expected) names := make([]string, 0, len(counts)) for n := range counts { names = append(names, n) } sort.Strings(names) for _, n := range names { - fmt.Fprintf(&sb, " %s: %d\n", n, counts[n]) + sb += fmt.Sprintf(" %s: %d\n", n, counts[n]) } - return sb.String() + return sb } var _ = Describe // suppress unused import lint for ginkgo dot-import