diff --git a/CONTEXT.md b/CONTEXT.md new file mode 100644 index 0000000..ff29d99 --- /dev/null +++ b/CONTEXT.md @@ -0,0 +1,23 @@ +# Understudy + +Understudy prepares temporary replacement capacity for announced node disruptions. + +## Language + +**Desired baseline**: The capacity the service author requests outside a disruption. +_Avoid_: Effective replicas, surge capacity + +**Effective replicas**: The capacity currently requested from the workload controller, +including any temporary surge. + +**Handover baseline**: The baseline committed to an active disruption handover; +later desired-baseline changes do not alter its readiness requirement. + +**Handover**: The interval in which Understudy prepares replacement capacity and +allows departing pods to leave, ending when the disruption clears or a release takes over. + +**Pending baseline**: The latest desired baseline awaiting completion of a handover. +Intermediate requests are superseded rather than queued. + +**Release**: A change to the application's revision, managed by its workload controller. +_Avoid_: Handover, node drain diff --git a/README.md b/README.md index 4b0f254..00a4a9c 100644 --- a/README.md +++ b/README.md @@ -184,6 +184,7 @@ someone runs them on GKE and AKS. |---|---|---| | `targetRef.kind` | `Deployment` | `Deployment` (`apps/v1`) or `Rollout` (`argoproj.io/v1alpha1`) | | `targetRef.name` | required | The workload to protect, in the same namespace | +| `baselineReplicas` | unset | Opt into capacity management: desired steady-state replicas; changes wait during handover. Supports zero. See [GitOps capacity](docs/gitops-capacity.md) | | `surgeReplicas` | 1 | How many stand-ins to add | | `minSurgeTimeSeconds` | 60 | How long this workload realistically takes to start serving. If a termination deadline is nearer than this, the eviction is released immediately instead of being held | | `readinessDeadlineSeconds` | 600 | How long to hold an eviction with no progress before giving up and letting it through | @@ -236,14 +237,28 @@ revision (or the blue/green active revision) count as stand-ins. Ready preview pods and old revisions cannot release a production pod's eviction hold. During an update, pause, or abort, Understudy stands down and admits evictions. -If a release begins during an Understudy surge, it restores the original replica -count and relinquishes the hold. Progressive delivery remains Argo's responsibility. +If a release begins during an Understudy surge, it removes the surge and +relinquishes the hold. With `baselineReplicas`, it applies the latest requested +baseline; otherwise it restores the original replica count. Progressive delivery +remains Argo's responsibility. Argo Rollouts must already be installed. Its API is detected at operator startup; restart Understudy if you install the Rollout CRD later. Clusters without Argo Rollouts continue to support Deployments. As with Deployments, another reconciler (such as an HPA or GitOps tool) must not overwrite the temporary replica increase. +### GitOps capacity + +Argo CD keeps Kubernetes aligned with Git; Argo Rollouts manages application +releases. These are separate jobs. For either a Deployment or a Rollout managed +by Argo CD, set `spec.baselineReplicas` on the Understudy resource and delegate +the workload's effective `spec.replicas` to Understudy. + +Git capacity changes then converge automatically. During a drain handover, +Understudy preserves its captured baseline plus surge and applies the latest +requested baseline afterward. `BaselineApplied=False` reports a pending change. +See [the complete ownership configuration and lifecycle](docs/gitops-capacity.md). + ## How it works Briefly: answer the eviction at the door with HTTP 429, surge when its node is diff --git a/UPGRADING.md b/UPGRADING.md index 95993be..9f00f24 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -2,6 +2,29 @@ Version-to-version notes. Anything not mentioned here needs no action. +## 0.4.3 to 0.4.4 + +`spec.baselineReplicas` opts into continuous capacity management. Existing +resources without it retain reactive-only protection. Install the updated +operator and CRD together before enabling it; v0.4.3 does not implement this +contract. [GitOps capacity](docs/gitops-capacity.md) documents the exact Argo +ownership settings and lifecycle for Deployments and Rollouts. + +Helm does not upgrade files in `crds/`. Apply the v0.4.4 schema first, then +upgrade the chart and operator: + +```sh +kubectl apply --server-side -f \ + https://raw.githubusercontent.com/mitis-cloud/understudy/v0.4.4/charts/understudy/crds/apps.understudy.sh_understudies.yaml + +helm upgrade understudy oci://ghcr.io/kylan11/charts/understudy \ + --namespace understudy-system --version 0.4.4 +``` + +Then configure the desired baseline and Argo ignore rules together. Do not +enable only the ignore rules: without the new +baseline input, intentional Git capacity changes would still be ignored. + ## 0.4.2 to 0.4.3 Argo Rollouts are now supported with `targetRef.kind: Rollout`. Both canary and diff --git a/api/v1alpha1/understudy_types.go b/api/v1alpha1/understudy_types.go index 1126a57..e17d3b2 100644 --- a/api/v1alpha1/understudy_types.go +++ b/api/v1alpha1/understudy_types.go @@ -45,6 +45,7 @@ const ( ConditionProtected = "Protected" ConditionSurgeReady = "SurgeReady" ConditionZeroDowntimeStrategy = "ZeroDowntimeUpdateStrategy" + ConditionBaselineApplied = "BaselineApplied" ) type TargetReference struct { @@ -62,6 +63,11 @@ type UnderstudySpec struct { // +required TargetRef TargetReference `json:"targetRef"` + // BaselineReplicas opts into capacity management; changes wait during handover. + // +kubebuilder:validation:Minimum=0 + // +optional + BaselineReplicas *int32 `json:"baselineReplicas,omitempty"` + // +kubebuilder:default=1 // +kubebuilder:validation:Minimum=1 // +optional @@ -85,6 +91,10 @@ type UnderstudySpec struct { } type UnderstudyStatus struct { + // BaselineReplicas is the applied baseline, frozen while a handover is active. + // +optional + BaselineReplicas *int32 `json:"baselineReplicas,omitempty"` + // +optional Phase Phase `json:"phase,omitempty"` diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index a298afe..a11cc8e 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -103,6 +103,11 @@ func (in *UnderstudyList) DeepCopyObject() runtime.Object { func (in *UnderstudySpec) DeepCopyInto(out *UnderstudySpec) { *out = *in out.TargetRef = in.TargetRef + if in.BaselineReplicas != nil { + in, out := &in.BaselineReplicas, &out.BaselineReplicas + *out = new(int32) + **out = **in + } if in.SurgeReplicas != nil { in, out := &in.SurgeReplicas, &out.SurgeReplicas *out = new(int32) @@ -133,6 +138,11 @@ func (in *UnderstudySpec) DeepCopy() *UnderstudySpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *UnderstudyStatus) DeepCopyInto(out *UnderstudyStatus) { *out = *in + if in.BaselineReplicas != nil { + in, out := &in.BaselineReplicas, &out.BaselineReplicas + *out = new(int32) + **out = **in + } if in.LastSurgeTime != nil { in, out := &in.LastSurgeTime, &out.LastSurgeTime *out = (*in).DeepCopy() diff --git a/charts/understudy/Chart.yaml b/charts/understudy/Chart.yaml index 2b4eb1a..7e74ed4 100644 --- a/charts/understudy/Chart.yaml +++ b/charts/understudy/Chart.yaml @@ -5,8 +5,8 @@ description: >- downtime from announced node disruptions (drains, upgrades, consolidation, spot reclaims) without a standing second replica. type: application -version: 0.4.3 -appVersion: "0.4.3" +version: 0.4.4 +appVersion: "0.4.4" kubeVersion: ">=1.34.0-0" keywords: - availability @@ -17,6 +17,7 @@ keywords: - karpenter - cluster-autoscaler - argo-rollouts + - gitops home: https://github.com/kylan11/understudy sources: - https://github.com/kylan11/understudy @@ -41,7 +42,7 @@ annotations: url: https://github.com/kylan11/understudy/issues artifacthub.io/images: | - name: understudy - image: ghcr.io/kylan11/understudy:0.4.3 + image: ghcr.io/kylan11/understudy:0.4.4 artifacthub.io/crds: | - kind: Understudy version: v1alpha1 @@ -65,6 +66,8 @@ annotations: readinessDeadlineSeconds: 600 artifacthub.io/changes: | - kind: added - description: Protect fully promoted Argo Rollouts with canary and blue/green strategies using targetRef.kind Rollout + description: Optional baselineReplicas separates Git-owned desired capacity from effective Deployment and Rollout replicas, deferring capacity changes until handover completes + - kind: added + description: BaselineApplied status and documented Argo CD field delegation expose and preserve pending capacity changes - kind: fixed - description: Admit evictions while standing down for progressive delivery and exclude preview revisions from Rollout replacement readiness + description: Detect Deployment releases during an active surge and apply the latest baseline when application releases take over diff --git a/charts/understudy/crds/apps.understudy.sh_understudies.yaml b/charts/understudy/crds/apps.understudy.sh_understudies.yaml index 0a4a204..7cceb63 100644 --- a/charts/understudy/crds/apps.understudy.sh_understudies.yaml +++ b/charts/understudy/crds/apps.understudy.sh_understudies.yaml @@ -53,6 +53,12 @@ spec: type: object spec: properties: + baselineReplicas: + description: BaselineReplicas opts into capacity management; changes + wait during handover. + format: int32 + minimum: 0 + type: integer holdMode: enum: - always @@ -97,6 +103,11 @@ spec: type: object status: properties: + baselineReplicas: + description: BaselineReplicas is the applied baseline, frozen while + a handover is active. + format: int32 + type: integer blockedSince: format: date-time type: string diff --git a/config/crd/bases/apps.understudy.sh_understudies.yaml b/config/crd/bases/apps.understudy.sh_understudies.yaml index 0a4a204..7cceb63 100644 --- a/config/crd/bases/apps.understudy.sh_understudies.yaml +++ b/config/crd/bases/apps.understudy.sh_understudies.yaml @@ -53,6 +53,12 @@ spec: type: object spec: properties: + baselineReplicas: + description: BaselineReplicas opts into capacity management; changes + wait during handover. + format: int32 + minimum: 0 + type: integer holdMode: enum: - always @@ -97,6 +103,11 @@ spec: type: object status: properties: + baselineReplicas: + description: BaselineReplicas is the applied baseline, frozen while + a handover is active. + format: int32 + type: integer blockedSince: format: date-time type: string diff --git a/docs/adr/0001-declarative-baseline-capacity.md b/docs/adr/0001-declarative-baseline-capacity.md new file mode 100644 index 0000000..9376fe8 --- /dev/null +++ b/docs/adr/0001-declarative-baseline-capacity.md @@ -0,0 +1,27 @@ +--- +status: accepted +--- + +# Separate desired baseline from effective replicas + +GitOps synchronization and Understudy's temporary surge both write workload +replicas. Permanently ignoring replicas prevents Git capacity changes, while +conditional ignores do not preserve the surge through Argo CD 3.5.5's sync +preprocessing. We therefore add opt-in `spec.baselineReplicas`: Git owns the +desired baseline, and Understudy continuously reconciles effective replicas. + +During handover, the captured baseline remains the readiness threshold and the +latest desired baseline waits in the resource's spec. Completion, cancellation, +or release takeover applies that latest value. Application release stand-down +disables disruption surging and holding, not ordinary baseline reconciliation. + +Argo delegates the effective replica field and Understudy's handover annotations +using its native, statically scoped `ignoreDifferences` and +`RespectIgnoreDifferences=true`. No synchronization wrapper or additional +controller is introduced. Omitting the baseline retains reactive-only +protection; the opt-in contract requires a single capacity writer. + +This boundary expansion was approved in Q38, originating from DBRID-2851's +service-chart integration requirement. The operator owns its implementation; +consumer charts derive the desired baseline and first-install replica count +from the same service capacity value. diff --git a/docs/gitops-capacity.md b/docs/gitops-capacity.md new file mode 100644 index 0000000..5a0175c --- /dev/null +++ b/docs/gitops-capacity.md @@ -0,0 +1,129 @@ +# GitOps capacity contract + +Available starting with **Understudy v0.4.4**, Helm chart **0.4.4**, and its +`apps.understudy.sh/v1alpha1` CRD. Upgrade the CRD and operator together; +v0.4.3 does not support the field. + +Git supplies the **desired baseline**. Understudy owns **effective replicas**: +the normal baseline plus temporary capacity while handing over departing pods. +This applies equally to Kubernetes Deployments and Argo Rollouts. + +## Configure the two owners + +Enable capacity management on the Understudy resource: + +```yaml +apiVersion: apps.understudy.sh/v1alpha1 +kind: Understudy +metadata: + name: worker + namespace: workers +spec: + targetRef: + kind: Deployment # or Rollout + name: worker + baselineReplicas: 1 +``` + +In the Argo CD Application, delegate the effective field and handover markers +for that specific target: + +```yaml +spec: + ignoreDifferences: + - group: apps # argoproj.io for a Rollout + kind: Deployment # Rollout for an Argo Rollout + name: worker + namespace: workers + jsonPointers: + - /spec/replicas + - /metadata/annotations/understudy.sh~1base-replicas + - /metadata/annotations/understudy.sh~1target-revision + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - RespectIgnoreDifferences=true +``` + +Do not ignore `Understudy.spec.baselineReplicas`. Argo continues to synchronize +that input, application releases, and unrelated settings normally. The ignore +rules are static and scoped to the protected target: no conditional JQ, +managed-field ownership inference, or manual sync pause is needed. + +Consumer charts should render the workload's initial `spec.replicas` and the +Understudy's `spec.baselineReplicas` from **one service capacity value**. On first +creation, Argo applies the workload as rendered; `RespectIgnoreDifferences` +only preserves ignored fields on resources that already exist. Understudy then +corrects any mismatch once both resources exist. A CR created before its target +waits for the target to appear. + +## Lifecycle + +| Situation | Effective replicas and protection | +|---|---| +| No handover | Understudy reconciles effective replicas to the desired baseline, including live drift. | +| Handover starts | Captures the currently applied capacity as the handover baseline and adds `surgeReplicas`. | +| Baseline changes during handover | Keeps the captured baseline and surge; the latest requested value waits in the CR spec. Intermediate requests are superseded. | +| Replacement becomes ready | The readiness threshold remains the captured baseline, not a pending larger or smaller request. | +| Departing pods leave, or the disruption clears | Removes handover markers and applies the latest desired baseline. | +| Readiness deadline expires | Relaxes the eviction hold; keeps surge capacity until departing pods leave or the disruption clears, then applies the latest baseline. | +| Application release starts during handover | Ends the disruption surge, clears deletion-cost hints, admits evictions, and applies the latest baseline. | +| Application release is updating, paused, or aborted | Disruption protection stands down; ordinary baseline reconciliation continues. Argo Rollouts still owns promotion and traffic routing. | +| Desired baseline is zero | Scales to zero. A zero requested during handover waits like any other change; an already-zero workload does not start a disruption surge. | + +The baseline and revision captured for handover live in annotations on the +target. The pending request lives in `Understudy.spec.baselineReplicas`, so a +lost status update cannot discard it. On restart the operator reconstructs +disruption signals and resumes reconciliation. Webhook availability still has +the existing fail-open behavior while the operator is unavailable. + +## Observe the result + +- `spec.baselineReplicas`: latest Git request. +- `status.baselineReplicas`: applied baseline, frozen during handover. +- `BaselineApplied=False`, reason `HandoverInProgress`: desired baseline is pending. +- `BaselineApplied=True`: baseline has been applied to the workload's replica + request. This does **not** claim that every requested pod is Ready. +- `SurgeReady` and `Protected`: replacement readiness and eviction-hold behavior. +- Target `spec.replicas`: effective replicas, including a possible surge. + +Conditions include `observedGeneration`, so a client can distinguish the latest +request from an older status observation. + +## Enable and retire ownership + +Use one Understudy resource per target and one capacity writer. An HPA, KEDA, +another Understudy resource, or another GitOps application must not concurrently +write that target's effective replicas. Workloads must still tolerate temporary +overlap; baseline management does not make overlap-unsafe singletons safe. + +Omitting `baselineReplicas` keeps reactive-only protection. Removing it stops +continuous baseline reconciliation; an active handover finishes at its captured +baseline. Deleting the CR does not restore replicas: it removes the controller's +input. To retire protection, finish or cancel the handover first, align the Git +replica count with the intended capacity, then remove the CR and its Argo ignore +rules together. Keep a target reference fixed through handover; retargeting a CR +does not clean up the old target. + +The documented Argo configuration uses its ordinary apply synchronization path. +Force/replace synchronization is outside this contract because it can recreate +resources and discard the runtime state. Argo's native documentation explains +[RespectIgnoreDifferences](https://argo-cd.readthedocs.io/en/stable/user-guide/sync-options/#respect-ignore-differences-configs). + +For Helm upgrades, update the CRD explicitly before using the new field: Helm +does not upgrade CRDs in a chart's `crds/` directory. See [upgrade notes](../UPGRADING.md). + +## Verified integration + +Validated with Argo CD **3.5.5**, Argo Rollouts **1.10.1**, and Kubernetes +**1.36.4** in isolated Kind clusters, with automatic sync and self-heal enabled. +Deployment, canary, and blue/green targets passed initial installation, +unrelated sync during surge, repeated pending baseline updates, cold operator +restart and leadership recovery, eviction hold/release, final convergence, +steady scaling, HTTP serving, and release takeover. Blue/green also converged a +pending scale-to-zero request and subsequently scaled up from Git. + +The test used the ordinary Argo apply path and the scoped ignore rules above. +Server-side apply and force/replace sync were not part of this verification. diff --git a/docs/how-it-works.md b/docs/how-it-works.md index 9300145..9660036 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -25,7 +25,8 @@ Readiness gates therefore include load balancer registration. Once enough healthy stand-ins exist, the webhook admits the drainer's next retry. **5. Scale back.** After the old pod leaves, the workload returns to its -original replica count. The deletion cost makes the old pod the preferred +original replica count, or the latest requested `baselineReplicas` when capacity +management is enabled. The deletion cost makes the old pod the preferred scale-down target. ```mermaid @@ -57,12 +58,26 @@ blue/green strategies. Pod selection additionally checks the `status.blueGreen.activeSelector` for blue/green. Preview pods cannot substitute for production capacity. During progressive delivery, pauses, or aborts, the controller and webhook stand down together. A release that starts during a surge -restores the saved replica count and clears deletion-cost hints. +removes it and clears deletion-cost hints. It applies the latest desired baseline +when configured, otherwise the saved replica count. The operator uses Argo's native replica scaling behavior and does not manage its Services, traffic weights, analysis runs, or promotion steps. The Rollout watch is registered only when the API is discovered at startup. +## Declarative baseline capacity + +With `spec.baselineReplicas`, Understudy continuously reconciles the workload's +effective replica count. During handover, `understudy.sh/base-replicas` freezes +the baseline used for both the surge and the eviction readiness threshold. +`understudy.sh/target-revision` records the release revision so a new release +can take over an active handover. A pending baseline remains in the CR spec, +and completion applies its latest value rather than restoring an outdated one. + +Baseline reconciliation also runs during release stand-down: only disruption +surging and holding stop. See [GitOps capacity](gitops-capacity.md) for the Argo +field-ownership configuration, first-install behavior, status, and retirement. + ## Signals The core consumes one provider-neutral value: diff --git a/docs/roadmap.md b/docs/roadmap.md index ee36f39..ffa1281 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -1,5 +1,17 @@ # Roadmap +## v0.4.4 - GitOps baseline capacity + +`spec.baselineReplicas` lets Git supply the desired baseline while Understudy +owns effective Deployment or Rollout replicas. Changes during handover remain +pending and the latest value converges afterward, including scale-to-zero. +`BaselineApplied` reports that state. Application release takeover applies the +latest baseline while disruption protection stands down. + +The [Argo CD ownership contract](gitops-capacity.md) was validated against +Argo CD 3.5.5 and Argo Rollouts 1.10.1 with automatic sync and self-heal enabled, +including unrelated sync, operator restart, and release-during-handover checks. + ## v0.4.3 - Argo Rollouts Fully promoted canary and blue/green Rollouts can now be protected using @@ -46,8 +58,8 @@ v0.4.x and will be removed in v0.5.0. readiness instead of merely observed. - **GCP and Azure live validation.** Both sentinels remain experimental until exercised against their real metadata services. -- **GitOps coexistence.** Document Argo CD and Flux `ignoreDifferences` - recipes, or move surge state to a separate standby Deployment. +- **Flux coexistence.** Validate and document Flux field delegation for the + declarative baseline capacity contract. - **EvictionRequest migration.** KEP-4563 is the future interception point. When drainers adopt it, the same state machine can respond before acking without changing the product boundary. diff --git a/internal/controller/capacity_test.go b/internal/controller/capacity_test.go new file mode 100644 index 0000000..4a98812 --- /dev/null +++ b/internal/controller/capacity_test.go @@ -0,0 +1,203 @@ +/* +Copyright 2026 The Understudy Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "os" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + + appsv1alpha1 "github.com/kylan11/understudy/api/v1alpha1" + "github.com/kylan11/understudy/internal/workload" +) + +func setBaseline(f *fixture, replicas int32) { + EventuallyWithOffset(1, func() error { + cr := getCR(f) + cr.Spec.BaselineReplicas = ptr.To(replicas) + return k8sClient.Update(ctx, cr) + }).Should(Succeed()) +} + +func capacityTarget(f *fixture) *workload.Target { + cr := getCR(f) + target, err := workload.Resolve(ctx, k8sClient, client.ObjectKey{Namespace: f.ns, Name: cr.Spec.TargetRef.Name}, cr.Spec.TargetRef.Kind) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + return target +} + +func expectBaselineApplied(f *fixture, replicas int32) { + EventuallyWithOffset(1, func() *int32 { return getCR(f).Status.BaselineReplicas }).Should(Equal(ptr.To(replicas))) + EventuallyWithOffset(1, func() *metav1.Condition { return findCond(getCR(f), appsv1alpha1.ConditionBaselineApplied) }).Should( + HaveField("Status", metav1.ConditionTrue)) +} + +func newCapacityFixture(kind string) *fixture { + if kind != workload.KindDeployment { + if os.Getenv("ARGO_ROLLOUTS_CRD") == "" { + Skip("run make test to install the upstream Rollout CRD") + } + f, _ := newRolloutFixture(kind == "BlueGreen") + setBaseline(f, 1) + expectBaselineApplied(f, 1) + return f + } + f := newFixture(func(cr *appsv1alpha1.Understudy) { cr.Spec.BaselineReplicas = ptr.To(int32(1)) }) + Eventually(func() error { + dep := getDeployment(f) + dep.Annotations = map[string]string{"deployment.kubernetes.io/revision": "1"} + return k8sClient.Update(ctx, dep) + }).Should(Succeed()) + settleDeployment(f.dep) + expectBaselineApplied(f, 1) + return f +} + +var _ = Describe("Declarative baseline capacity", func() { + It("converges an initial baseline, subsequent capacity changes, and live replica drift", func() { + f := newFixture(func(cr *appsv1alpha1.Understudy) { cr.Spec.BaselineReplicas = ptr.To(int32(3)) }) + Eventually(func() int32 { return capacityTarget(f).Replicas() }).Should(Equal(int32(3))) + expectBaselineApplied(f, 3) + setBaseline(f, 2) + Eventually(func() int32 { return capacityTarget(f).Replicas() }).Should(Equal(int32(2))) + expectBaselineApplied(f, 2) + Expect(capacityTarget(f).PatchReplicas(ctx, k8sClient, 4, nil)).To(Succeed()) + Eventually(func() int32 { return capacityTarget(f).Replicas() }).Should(Equal(int32(2))) + setBaseline(f, 0) + Eventually(func() int32 { return capacityTarget(f).Replicas() }).Should(BeZero()) + cordonNode(f.node) + Consistently(func() int32 { return capacityTarget(f).Replicas() }).Should(BeZero()) + Expect(evict(f.pod, false)).To(Succeed()) + setBaseline(f, 1) + Eventually(func() int32 { return capacityTarget(f).Replicas() }).Should(Equal(int32(1))) + expectBaselineApplied(f, 1) + }) + + DescribeTable("keeps the handover baseline and applies the latest request after eviction", + func(kind string, desired int32) { + f := newCapacityFixture(kind) + cordonNode(f.node) + Eventually(func() int32 { return capacityTarget(f).Replicas() }).Should(Equal(int32(2))) + setBaseline(f, 4) + setBaseline(f, desired) + Eventually(func() *metav1.Condition { return findCond(getCR(f), appsv1alpha1.ConditionBaselineApplied) }).Should( + And(HaveField("Status", metav1.ConditionFalse), HaveField("Reason", "HandoverInProgress"))) + Expect(getCR(f).Status.BaselineReplicas).To(Equal(ptr.To(int32(1)))) + Consistently(func() int32 { return capacityTarget(f).Replicas() }).Should(Equal(int32(2))) + Expect(apierrors.IsTooManyRequests(evict(f.pod, false))).To(BeTrue()) + + By("repairing effective replica drift without changing the captured baseline") + target := capacityTarget(f) + Expect(target.PatchReplicas(ctx, k8sClient, 1, ptr.To(int32(1)))).To(Succeed()) + Eventually(func() int32 { return capacityTarget(f).Replicas() }).Should(Equal(int32(2))) + + By("recovering from a lost status write using the persisted handover marker") + Eventually(func() error { + cr := getCR(f) + cr.Status = appsv1alpha1.UnderstudyStatus{} + return k8sClient.Status().Update(ctx, cr) + }).Should(Succeed()) + Eventually(func() *int32 { return getCR(f).Status.BaselineReplicas }).Should(Equal(ptr.To(int32(1)))) + Expect(capacityTarget(f).Replicas()).To(Equal(int32(2))) + + By("using the frozen readiness threshold even when a larger baseline is pending") + healthy := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: f.ns + "-healthy"}} + Expect(k8sClient.Create(ctx, healthy)).To(Succeed()) + standIn := makePod(f.ns, "stand-in", f.dep.Name, healthy.Name, true) + if kind != workload.KindDeployment { + labelRolloutPod(standIn, stableHash) + } + Eventually(func() error { return evict(f.pod, false) }).Should(Succeed()) + Eventually(func() int32 { return capacityTarget(f).Replicas() }).Should(Equal(desired)) + Eventually(func() map[string]string { return capacityTarget(f).GetAnnotations() }).ShouldNot(HaveKey(AnnotationBaseReplicas)) + expectBaselineApplied(f, desired) + }, Entry("Deployment scale-up", workload.KindDeployment, int32(3)), + Entry("Deployment scale-to-zero", workload.KindDeployment, int32(0)), + Entry("canary scale-up", workload.KindRollout, int32(3)), + Entry("blue/green scale-down", "BlueGreen", int32(0))) + + It("applies the pending baseline when the disruption is cancelled", func() { + f := newCapacityFixture(workload.KindDeployment) + cordonNode(f.node) + Eventually(func() int32 { return capacityTarget(f).Replicas() }).Should(Equal(int32(2))) + setBaseline(f, 3) + Eventually(func() error { + node := &corev1.Node{} + if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(f.node), node); err != nil { + return err + } + node.Spec.Unschedulable = false + return k8sClient.Update(ctx, node) + }).Should(Succeed()) + Eventually(func() int32 { return capacityTarget(f).Replicas() }).Should(Equal(int32(3))) + expectBaselineApplied(f, 3) + }) + + It("hands a Deployment release the latest baseline and releases its eviction hold", func() { + f := newCapacityFixture(workload.KindDeployment) + cordonNode(f.node) + Eventually(func() int32 { return capacityTarget(f).Replicas() }).Should(Equal(int32(2))) + setBaseline(f, 3) + Eventually(func() error { + dep := getDeployment(f) + dep.Status.Conditions = []appsv1.DeploymentCondition{{ + Type: appsv1.DeploymentProgressing, Status: corev1.ConditionTrue, Reason: "ReplicaSetUpdated", + }} + return k8sClient.Status().Update(ctx, dep) + }).Should(Succeed()) + Eventually(func() error { + dep := getDeployment(f) + dep.Annotations["deployment.kubernetes.io/revision"] = "2" + return k8sClient.Update(ctx, dep) + }).Should(Succeed()) + Eventually(func() int32 { return capacityTarget(f).Replicas() }).Should(Equal(int32(3))) + Eventually(func() map[string]string { return capacityTarget(f).GetAnnotations() }).ShouldNot(HaveKey(AnnotationBaseReplicas)) + expectBaselineApplied(f, 3) + Eventually(func() error { return evict(f.pod, false) }).Should(Succeed()) + }) + + It("continues baseline management during a paused Rollout release", func() { + f := newCapacityFixture(workload.KindRollout) + ro, err := workload.Resolve(ctx, k8sClient, client.ObjectKey{Namespace: f.ns, Name: f.dep.Name}, workload.KindRollout) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient.Patch(ctx, ro.Object, client.RawPatch(types.MergePatchType, []byte(`{"spec":{"paused":true}}`)))).To(Succeed()) + setBaseline(f, 3) + Eventually(func() int32 { return capacityTarget(f).Replicas() }).Should(Equal(int32(3))) + expectBaselineApplied(f, 3) + Eventually(func() *metav1.Condition { return findCond(getCR(f), appsv1alpha1.ConditionProtected) }).Should(HaveField("Status", metav1.ConditionFalse)) + Expect(evict(f.pod, false)).To(Succeed()) + }) + + It("rejects a negative desired baseline at admission", func() { + f := newFixture(nil) + cr := &appsv1alpha1.Understudy{ + ObjectMeta: metav1.ObjectMeta{Name: "invalid", Namespace: f.ns}, + Spec: appsv1alpha1.UnderstudySpec{ + TargetRef: f.cr.Spec.TargetRef, BaselineReplicas: ptr.To(int32(-1)), + }, + } + Expect(apierrors.IsInvalid(k8sClient.Create(ctx, cr))).To(BeTrue()) + }) +}) diff --git a/internal/controller/understudy_controller.go b/internal/controller/understudy_controller.go index ea93ec3..82a0318 100644 --- a/internal/controller/understudy_controller.go +++ b/internal/controller/understudy_controller.go @@ -31,6 +31,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/events" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/handler" @@ -121,6 +122,11 @@ func (r *UnderstudyReconciler) Reconcile(ctx context.Context, req ctrl.Request) setCond(&status, &cr, appsv1alpha1.ConditionTargetSupported, metav1.ConditionTrue, target.Kind(), "target resolved") base, surgeActive := target.BaseReplicas() + desiredBase := base + if cr.Spec.BaselineReplicas != nil { + desiredBase = *cr.Spec.BaselineReplicas + } + setBaselineCondition(&status, &cr, base) zeroDowntime, reason, message := target.ZeroDowntimeStrategy(base) setCond(&status, &cr, appsv1alpha1.ConditionZeroDowntimeStrategy, boolToStatus(zeroDowntime), reason, message) @@ -141,10 +147,13 @@ func (r *UnderstudyReconciler) Reconcile(ctx context.Context, req ctrl.Request) if err := r.clearDeletionCosts(ctx, assessment.all); err != nil { return ctrl.Result{}, err } - if err := target.PatchReplicas(ctx, r.Client, base, nil); err != nil { + } + if surgeActive || cr.Spec.BaselineReplicas != nil && target.Replicas() != desiredBase { + if err := target.PatchReplicas(ctx, r.Client, desiredBase, nil); err != nil { return ctrl.Result{}, err } } + setBaselineCondition(&status, &cr, desiredBase) status.Phase = appsv1alpha1.PhaseIdle status.BlockedSince = nil metrics.UntrackBlocked(req.String()) @@ -160,15 +169,21 @@ func (r *UnderstudyReconciler) Reconcile(ctx context.Context, req ctrl.Request) setProtectedCondition(&status, &cr, mode, relaxReason) switch { - case len(assessment.doomed) == 0 && !surgeActive: + case !surgeActive && (len(assessment.doomed) == 0 || base == 0): + if cr.Spec.BaselineReplicas != nil && target.Replicas() != desiredBase { + if err := target.PatchReplicas(ctx, r.Client, desiredBase, nil); err != nil { + return ctrl.Result{}, err + } + } + setBaselineCondition(&status, &cr, desiredBase) status.Phase = appsv1alpha1.PhaseIdle status.BlockedSince = nil metrics.UntrackBlocked(req.String()) - setCond(&status, &cr, appsv1alpha1.ConditionSurgeReady, boolToStatus(surgeReady), "Steady", "no disruption in sight") + setCond(&status, &cr, appsv1alpha1.ConditionSurgeReady, boolToStatus(assessment.viable >= int(desiredBase)), "Steady", "no disruption in sight") return ctrl.Result{}, r.patchStatus(ctx, &cr, status) case len(assessment.doomed) == 0 && surgeActive: - return r.reconcileScaleBack(ctx, &cr, target, status, assessment, base, req) + return r.reconcileScaleBack(ctx, &cr, target, status, assessment, desiredBase, req) default: return r.reconcileDoomed(ctx, &cr, target, status, assessment, base, surgeActive, surgeReady, relaxReason, ttlExpired, req) @@ -247,6 +262,7 @@ func (r *UnderstudyReconciler) reconcileScaleBack(ctx context.Context, cr *appsv status.LastReleaseTime = &now status.BlockedSince = nil status.Phase = appsv1alpha1.PhaseIdle + setBaselineCondition(&status, cr, base) metrics.UntrackBlocked(req.String()) r.event(cr, corev1.EventTypeNormal, "ScaledBack", "Release", fmt.Sprintf("predicament over; scaled %s back to %d replicas", target.GetName(), base)) @@ -264,10 +280,12 @@ func (r *UnderstudyReconciler) reconcileDoomed(ctx context.Context, cr *appsv1al } metrics.TrackBlocked(req.String(), status.BlockedSince.Time) - if !surgeActive { + if !surgeActive || cr.Spec.BaselineReplicas != nil && target.Replicas() != base+cr.SurgeReplicasOrDefault() { if err := target.PatchReplicas(ctx, r.Client, base+cr.SurgeReplicasOrDefault(), &base); err != nil { return ctrl.Result{}, err } + } + if !surgeActive { now := metav1.Now() status.LastSurgeTime = &now class := signal.VoluntaryUnbounded @@ -368,6 +386,22 @@ func setProtectedCondition(status *appsv1alpha1.UnderstudyStatus, cr *appsv1alph setCond(status, cr, appsv1alpha1.ConditionProtected, boolToStatus(mode != appsv1alpha1.HoldOff && relaxReason == ""), reason, msg) } +func setBaselineCondition(status *appsv1alpha1.UnderstudyStatus, cr *appsv1alpha1.Understudy, base int32) { + if cr.Spec.BaselineReplicas == nil { + status.BaselineReplicas = nil + meta.RemoveStatusCondition(&status.Conditions, appsv1alpha1.ConditionBaselineApplied) + return + } + status.BaselineReplicas = ptr.To(base) + if *cr.Spec.BaselineReplicas != base { + setCond(status, cr, appsv1alpha1.ConditionBaselineApplied, metav1.ConditionFalse, + "HandoverInProgress", fmt.Sprintf("baseline %d is pending; handover retains baseline %d", *cr.Spec.BaselineReplicas, base)) + return + } + setCond(status, cr, appsv1alpha1.ConditionBaselineApplied, metav1.ConditionTrue, + "Applied", fmt.Sprintf("baseline %d is applied; effective replicas may include a temporary surge", base)) +} + func (r *UnderstudyReconciler) patchStatus(ctx context.Context, cr *appsv1alpha1.Understudy, status appsv1alpha1.UnderstudyStatus) error { if statusEqual(&cr.Status, &status) { return nil @@ -487,14 +521,14 @@ func boolToStatus(b bool) metav1.ConditionStatus { } func statusEqual(a, b *appsv1alpha1.UnderstudyStatus) bool { - if a.Phase != b.Phase || a.Mode != b.Mode || !timePtrEqual(a.LastSurgeTime, b.LastSurgeTime) || + if !ptr.Equal(a.BaselineReplicas, b.BaselineReplicas) || a.Phase != b.Phase || a.Mode != b.Mode || !timePtrEqual(a.LastSurgeTime, b.LastSurgeTime) || !timePtrEqual(a.LastReleaseTime, b.LastReleaseTime) || !timePtrEqual(a.BlockedSince, b.BlockedSince) || len(a.Conditions) != len(b.Conditions) { return false } for i := range a.Conditions { ca, cb := a.Conditions[i], b.Conditions[i] - if ca.Type != cb.Type || ca.Status != cb.Status || ca.Reason != cb.Reason || ca.Message != cb.Message { + if ca.Type != cb.Type || ca.Status != cb.Status || ca.Reason != cb.Reason || ca.Message != cb.Message || ca.ObservedGeneration != cb.ObservedGeneration { return false } } diff --git a/internal/workload/target.go b/internal/workload/target.go index 44e2ee8..11f235e 100644 --- a/internal/workload/target.go +++ b/internal/workload/target.go @@ -31,9 +31,10 @@ import ( ) const ( - AnnotationBaseReplicas = "understudy.sh/base-replicas" - KindDeployment = "Deployment" - KindRollout = "Rollout" + AnnotationBaseReplicas = "understudy.sh/base-replicas" + AnnotationTargetRevision = "understudy.sh/target-revision" + KindDeployment = "Deployment" + KindRollout = "Rollout" ) type Target struct { @@ -73,6 +74,10 @@ func (t *Target) BaseReplicas() (int32, bool) { return int32(parsed), true } } + return t.Replicas(), false +} + +func (t *Target) Replicas() int32 { var replicas *int32 switch obj := t.Object.(type) { case *appsv1.Deployment: @@ -81,9 +86,16 @@ func (t *Target) BaseReplicas() (int32, bool) { replicas = obj.Spec.Replicas } if replicas == nil { - return 1, false + return 1 + } + return *replicas +} + +func (t *Target) revision() string { + if ro, ok := t.Object.(*rolloutsv1alpha1.Rollout); ok { + return ro.Status.CurrentPodHash } - return *replicas, false + return t.GetAnnotations()["deployment.kubernetes.io/revision"] } func (t *Target) Selector() (labels.Selector, error) { @@ -125,8 +137,12 @@ func (t *Target) PatchReplicas(ctx context.Context, c client.Client, replicas in } if base == nil { delete(annotations, AnnotationBaseReplicas) + delete(annotations, AnnotationTargetRevision) } else { annotations[AnnotationBaseReplicas] = strconv.FormatInt(int64(*base), 10) + if _, active := t.BaseReplicas(); !active && t.revision() != "" { + annotations[AnnotationTargetRevision] = t.revision() + } } patched.SetAnnotations(annotations) switch obj := patched.(type) { @@ -140,11 +156,17 @@ func (t *Target) PatchReplicas(ctx context.Context, c client.Client, replicas in func (t *Target) StandDown() bool { base, surging := t.BaseReplicas() + if revision := t.GetAnnotations()[AnnotationTargetRevision]; surging && revision != "" && revision != t.revision() { + return true + } switch obj := t.Object.(type) { case *appsv1.Deployment: - if surging || obj.Status.ObservedGeneration < obj.Generation { + if surging { return false } + if obj.Status.ObservedGeneration < obj.Generation { + return true + } for _, c := range obj.Status.Conditions { if c.Type == appsv1.DeploymentProgressing && c.Status == corev1.ConditionTrue && c.Reason != "NewReplicaSetAvailable" { return true