From 58608e661410afbaa63be8cfb105ee817a577dc2 Mon Sep 17 00:00:00 2001 From: Anna Williamson Date: Mon, 22 Jun 2026 08:47:51 -0700 Subject: [PATCH 1/8] Manage Karpenter CRDs in lockstep with the controller Install the karpenter-crd chart as a second helm.cattle.io/v1 HelmChart pinned to the same karpenter_version as the controller, so a controller bump can never run against stale, hand-applied CRDs. Pre-stamp the existing CRDs with Helm ownership metadata so the chart can adopt them on existing clusters; the stamp is a no-op on greenfield. Order CR creation with dependsOn (CRD chart first). Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/steps/helm_aws.go | 83 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 80 insertions(+), 3 deletions(-) diff --git a/lib/steps/helm_aws.go b/lib/steps/helm_aws.go index fbf8518..a9e62a2 100644 --- a/lib/steps/helm_aws.go +++ b/lib/steps/helm_aws.go @@ -1366,12 +1366,86 @@ func awsHelmKarpenter(ctx *pulumi.Context, k8sOpt pulumi.ResourceOption, compoun }, } + // ── Karpenter CRDs ───────────────────────────────────────────────────────── + // Manage the Karpenter CRDs in lockstep with the controller via the + // karpenter-crd chart, pinned to the SAME `version` (single source of truth — + // both driven by `karpenter_version`). This prevents a controller bump from + // running against stale, hand-applied CRDs. + // + // Adoption of pre-existing CRDs: every cluster created before this change + // already has the Karpenter CRDs applied WITHOUT Helm ownership metadata, so a + // fresh karpenter-crd install would fail with "exists and cannot be imported + // into the current release — invalid ownership metadata". The pinned + // helm-controller (v0.16.10) does not yet expose spec.takeOwnership, so we + // pre-stamp the Helm ownership label + annotations onto each CRD (the same + // helm-adopt pattern used for the Calico FelixConfiguration in eks_helpers.go). + // The stamp is a no-op on greenfield clusters where the CRD does not yet exist + // (the patch simply has nothing to adopt and the chart creates it cleanly). + // The release name/namespace must match the karpenter-crd HelmChart release: + // for a helm.cattle.io/v1 HelmChart the release name is metadata.name and the + // release namespace is targetNamespace. + const karpenterCRDReleaseName = "karpenter-crd" + karpenterCRDNames := []string{ + "nodepools.karpenter.sh", + "nodeclaims.karpenter.sh", + "nodeoverlays.karpenter.sh", + "ec2nodeclasses.karpenter.k8s.aws", + } + adoptDeps := make([]pulumi.Resource, 0, len(karpenterCRDNames)) + for _, crdName := range karpenterCRDNames { + adoptResourceName := compoundName + "-karpenter-crd-helm-adopt-" + crdName + adopt, err := apiextensions.NewCustomResourcePatch(ctx, adoptResourceName, &apiextensions.CustomResourcePatchArgs{ + ApiVersion: pulumi.String("apiextensions.k8s.io/v1"), + Kind: pulumi.String("CustomResourceDefinition"), + Metadata: &metav1.ObjectMetaArgs{ + Name: pulumi.String(crdName), + Labels: pulumi.StringMap{"app.kubernetes.io/managed-by": pulumi.String("Helm")}, + Annotations: pulumi.StringMap{ + "meta.helm.sh/release-name": pulumi.String(karpenterCRDReleaseName), + "meta.helm.sh/release-namespace": pulumi.String(clustersKubeSystemNamespace), + }, + }, + }, k8sOpt, pulumi.IgnoreChanges([]string{"metadata"}), + withAlias("kubernetes:apiextensions.k8s.io/v1:CustomResourceDefinitionPatch", adoptResourceName)) + if err != nil { + return err + } + adoptDeps = append(adoptDeps, adopt) + } + + crdChartResourceName := compoundName + "-karpenter-crd-helm-release" + crdChart, err := apiextensions.NewCustomResource(ctx, crdChartResourceName, &apiextensions.CustomResourceArgs{ + ApiVersion: pulumi.String("helm.cattle.io/v1"), + Kind: pulumi.String("HelmChart"), + Metadata: metav1.ObjectMetaArgs{ + Name: pulumi.String(karpenterCRDReleaseName), + Namespace: pulumi.String(clustersHelmControllerNamespace), + Labels: pulumi.StringMap{"posit.team/managed-by": pulumi.String("ptd.pulumi_resources.aws_workload_helm")}, + }, + OtherFields: kubernetes.UntypedArgs{ + "spec": pulumi.Map{ + "chart": pulumi.String("oci://public.ecr.aws/karpenter/karpenter-crd"), + "targetNamespace": pulumi.String(clustersKubeSystemNamespace), + "version": pulumi.String(version), + }, + }, + }, k8sOpt, pulumi.DependsOn(adoptDeps), + withAlias("kubernetes:helm.cattle.io/v1:HelmChart", crdChartResourceName)) + if err != nil { + return err + } + chartResourceName := compoundName + "-karpenter-helm-release" valuesYAML, err := marshalYAML(values) if err != nil { return err } + // The controller chart depends on the CRD chart so Pulumi creates the CRD + // HelmChart CR first. Note: helm-controller reconciles HelmChart CRs + // asynchronously, so dependsOn orders CR *creation*, not the underlying helm + // jobs — see PR notes on the mitigation (CRDs are additive/backward-compatible, + // and the controller chart re-templates CRs that retry until the CRDs exist). _, err = apiextensions.NewCustomResource(ctx, chartResourceName, &apiextensions.CustomResourceArgs{ ApiVersion: pulumi.String("helm.cattle.io/v1"), Kind: pulumi.String("HelmChart"), @@ -1388,7 +1462,8 @@ func awsHelmKarpenter(ctx *pulumi.Context, k8sOpt pulumi.ResourceOption, compoun "valuesContent": pulumi.String(string(valuesYAML)), }, }, - }, k8sOpt, withAlias("kubernetes:helm.cattle.io/v1:HelmChart", chartResourceName)) + }, k8sOpt, pulumi.DependsOn([]pulumi.Resource{crdChart}), + withAlias("kubernetes:helm.cattle.io/v1:HelmChart", chartResourceName)) if err != nil { return err } @@ -1502,7 +1577,8 @@ func awsHelmKarpenter(ctx *pulumi.Context, k8sOpt pulumi.ResourceOption, compoun OtherFields: kubernetes.UntypedArgs{ "spec": nodepoolSpec, }, - }, k8sOpt, withAlias("kubernetes:karpenter.sh/v1:NodePool", npResourceName)) + }, k8sOpt, pulumi.DependsOn([]pulumi.Resource{crdChart}), + withAlias("kubernetes:karpenter.sh/v1:NodePool", npResourceName)) if err != nil { return err } @@ -1549,7 +1625,8 @@ func awsHelmKarpenter(ctx *pulumi.Context, k8sOpt pulumi.ResourceOption, compoun }, }, }, - }, k8sOpt, withAlias("kubernetes:karpenter.k8s.aws/v1:EC2NodeClass", ec2ResourceName)) + }, k8sOpt, pulumi.DependsOn([]pulumi.Resource{crdChart}), + withAlias("kubernetes:karpenter.k8s.aws/v1:EC2NodeClass", ec2ResourceName)) if err != nil { return err } From 086b492935ed6533fb9e1b75afcfd6827e455aa1 Mon Sep 17 00:00:00 2001 From: Anna Williamson Date: Mon, 22 Jun 2026 14:06:07 -0700 Subject: [PATCH 2/8] Adopt Karpenter CRDs via helm-controller takeOwnership Replace the manual CustomResourcePatch ownership pre-stamp with the helm-controller's native spec.takeOwnership (helm --take-ownership). Bump helm-controller v0.16.10 -> v0.16.14 (adds takeOwnership) on both AWS and Azure clusters, and add the takeOwnership boolean to PTD's embedded HelmChart CRD schema so the field is not pruned. Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/steps/clusters_aws.go | 2 +- lib/steps/clusters_azure.go | 2 +- lib/steps/clusters_helpers.go | 1 + lib/steps/helm_aws.go | 58 ++++++++++------------------------- 4 files changed, 19 insertions(+), 44 deletions(-) diff --git a/lib/steps/clusters_aws.go b/lib/steps/clusters_aws.go index 33d9cbe..0674c27 100644 --- a/lib/steps/clusters_aws.go +++ b/lib/steps/clusters_aws.go @@ -699,7 +699,7 @@ func awsClustersDeploy(ctx *pulumi.Context, _ types.Target, params awsClustersPa Containers: corev1.ContainerArray{ &corev1.ContainerArgs{ Name: pulumi.String("helm-controller"), - Image: pulumi.String("ghcr.io/k3s-io/helm-controller:v0.16.10"), + Image: pulumi.String("ghcr.io/k3s-io/helm-controller:v0.16.14"), Command: pulumi.StringArray{pulumi.String("helm-controller")}, Args: pulumi.StringArray{ pulumi.String("--namespace"), diff --git a/lib/steps/clusters_azure.go b/lib/steps/clusters_azure.go index aacff8d..acc8c04 100644 --- a/lib/steps/clusters_azure.go +++ b/lib/steps/clusters_azure.go @@ -429,7 +429,7 @@ func azureClustersDeploy(ctx *pulumi.Context, _ types.Target, params azureCluste Containers: corev1.ContainerArray{ &corev1.ContainerArgs{ Name: pulumi.String("helm-controller"), - Image: pulumi.String("ghcr.io/k3s-io/helm-controller:v0.16.10"), + Image: pulumi.String("ghcr.io/k3s-io/helm-controller:v0.16.14"), Command: pulumi.StringArray{pulumi.String("helm-controller")}, Args: pulumi.StringArray{ pulumi.String("--namespace"), diff --git a/lib/steps/clusters_helpers.go b/lib/steps/clusters_helpers.go index 1c945fe..3d1729e 100644 --- a/lib/steps/clusters_helpers.go +++ b/lib/steps/clusters_helpers.go @@ -158,6 +158,7 @@ var helmChartsCRDSpec = map[string]interface{}{ }, }, "set": map[string]interface{}{"nullable": true, "type": "object", "additionalProperties": map[string]interface{}{"x-kubernetes-int-or-string": true}}, + "takeOwnership": map[string]interface{}{"type": "boolean"}, "targetNamespace": map[string]interface{}{"nullable": true, "type": "string"}, "timeout": map[string]interface{}{"nullable": true, "type": "string"}, "valuesContent": map[string]interface{}{"nullable": true, "type": "string"}, diff --git a/lib/steps/helm_aws.go b/lib/steps/helm_aws.go index a9e62a2..11cf7cd 100644 --- a/lib/steps/helm_aws.go +++ b/lib/steps/helm_aws.go @@ -1373,52 +1373,25 @@ func awsHelmKarpenter(ctx *pulumi.Context, k8sOpt pulumi.ResourceOption, compoun // running against stale, hand-applied CRDs. // // Adoption of pre-existing CRDs: every cluster created before this change - // already has the Karpenter CRDs applied WITHOUT Helm ownership metadata, so a - // fresh karpenter-crd install would fail with "exists and cannot be imported - // into the current release — invalid ownership metadata". The pinned - // helm-controller (v0.16.10) does not yet expose spec.takeOwnership, so we - // pre-stamp the Helm ownership label + annotations onto each CRD (the same - // helm-adopt pattern used for the Calico FelixConfiguration in eks_helpers.go). - // The stamp is a no-op on greenfield clusters where the CRD does not yet exist - // (the patch simply has nothing to adopt and the chart creates it cleanly). - // The release name/namespace must match the karpenter-crd HelmChart release: - // for a helm.cattle.io/v1 HelmChart the release name is metadata.name and the - // release namespace is targetNamespace. - const karpenterCRDReleaseName = "karpenter-crd" - karpenterCRDNames := []string{ - "nodepools.karpenter.sh", - "nodeclaims.karpenter.sh", - "nodeoverlays.karpenter.sh", - "ec2nodeclasses.karpenter.k8s.aws", - } - adoptDeps := make([]pulumi.Resource, 0, len(karpenterCRDNames)) - for _, crdName := range karpenterCRDNames { - adoptResourceName := compoundName + "-karpenter-crd-helm-adopt-" + crdName - adopt, err := apiextensions.NewCustomResourcePatch(ctx, adoptResourceName, &apiextensions.CustomResourcePatchArgs{ - ApiVersion: pulumi.String("apiextensions.k8s.io/v1"), - Kind: pulumi.String("CustomResourceDefinition"), - Metadata: &metav1.ObjectMetaArgs{ - Name: pulumi.String(crdName), - Labels: pulumi.StringMap{"app.kubernetes.io/managed-by": pulumi.String("Helm")}, - Annotations: pulumi.StringMap{ - "meta.helm.sh/release-name": pulumi.String(karpenterCRDReleaseName), - "meta.helm.sh/release-namespace": pulumi.String(clustersKubeSystemNamespace), - }, - }, - }, k8sOpt, pulumi.IgnoreChanges([]string{"metadata"}), - withAlias("kubernetes:apiextensions.k8s.io/v1:CustomResourceDefinitionPatch", adoptResourceName)) - if err != nil { - return err - } - adoptDeps = append(adoptDeps, adopt) - } - + // already has the Karpenter CRDs installed (the controller chart's bundled + // crds/ created them) WITHOUT karpenter-crd Helm ownership metadata, so a + // fresh karpenter-crd install would otherwise fail with "exists and cannot be + // imported into the current release — invalid ownership metadata". We adopt + // them via the helm-controller's native spec.takeOwnership, which maps to + // `helm upgrade --take-ownership`: it rewrites the ownership label + + // release-name/namespace annotations on the existing objects so the + // karpenter-crd release becomes their owner. This is scoped to exactly the + // CRDs this chart renders, and is a no-op on greenfield clusters where the + // CRDs do not yet exist (the chart simply creates them cleanly). + // Requires helm-controller ≥ v0.16.14 (adds spec.takeOwnership) and Helm + // ≥ 3.17 in the job image (--take-ownership); both are satisfied by PTD's + // pinned versions. crdChartResourceName := compoundName + "-karpenter-crd-helm-release" crdChart, err := apiextensions.NewCustomResource(ctx, crdChartResourceName, &apiextensions.CustomResourceArgs{ ApiVersion: pulumi.String("helm.cattle.io/v1"), Kind: pulumi.String("HelmChart"), Metadata: metav1.ObjectMetaArgs{ - Name: pulumi.String(karpenterCRDReleaseName), + Name: pulumi.String("karpenter-crd"), Namespace: pulumi.String(clustersHelmControllerNamespace), Labels: pulumi.StringMap{"posit.team/managed-by": pulumi.String("ptd.pulumi_resources.aws_workload_helm")}, }, @@ -1427,9 +1400,10 @@ func awsHelmKarpenter(ctx *pulumi.Context, k8sOpt pulumi.ResourceOption, compoun "chart": pulumi.String("oci://public.ecr.aws/karpenter/karpenter-crd"), "targetNamespace": pulumi.String(clustersKubeSystemNamespace), "version": pulumi.String(version), + "takeOwnership": pulumi.Bool(true), }, }, - }, k8sOpt, pulumi.DependsOn(adoptDeps), + }, k8sOpt, withAlias("kubernetes:helm.cattle.io/v1:HelmChart", crdChartResourceName)) if err != nil { return err From 810f908c2a7d8be1a8773358d26ce7b8ed3c81b1 Mon Sep 17 00:00:00 2001 From: Anna Williamson Date: Tue, 23 Jun 2026 08:38:14 -0700 Subject: [PATCH 3/8] Move Karpenter CRD rationale to docs, trim code comment Condense the inline comment in awsHelmKarpenter to the essentials and move the full rationale (CRD staleness, takeOwnership adoption, version requirements, limitations, rollout) to docs/infrastructure/karpenter-crds.md. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/infrastructure/karpenter-crds.md | 73 +++++++++++++++++++++++++++ lib/steps/helm_aws.go | 25 ++------- 2 files changed, 78 insertions(+), 20 deletions(-) create mode 100644 docs/infrastructure/karpenter-crds.md diff --git a/docs/infrastructure/karpenter-crds.md b/docs/infrastructure/karpenter-crds.md new file mode 100644 index 0000000..4c70854 --- /dev/null +++ b/docs/infrastructure/karpenter-crds.md @@ -0,0 +1,73 @@ +# Karpenter CRD management + +How PTD manages Karpenter's CRDs, and why it's done this way. Implemented in +`awsHelmKarpenter` (`lib/steps/helm_aws.go`). + +## The problem + +The Karpenter controller chart (`oci://public.ecr.aws/karpenter/karpenter`) +bundles its CRDs in the chart's `crds/` directory. Helm installs `crds/` +resources **once, on first install, and never upgrades them** on subsequent +`helm upgrade`s (this is intentional, documented Helm behavior). + +So historically a `karpenter_version` bump upgraded the controller while leaving +the CRDs frozen at whatever version was first installed. When a controller +>= 1.9 then emits a `Gte`/`Lte` requirement — normalized from a NodePool +`Gt`/`Lt` instance floor — against a pre-1.9 CRD whose schema enum lacks those +operators, the API server rejects the NodeClaim and no nodes provision. The +break needs all three: controller >= 1.9, a NodePool using a `Gt`/`Lt` floor, +and a CRD missing `Gte`/`Lte`. + +## The fix + +Manage the CRDs with the dedicated **`karpenter-crd`** chart, installed as a +second `helm.cattle.io/v1` HelmChart pinned to the **same `karpenter_version`** +as the controller. The `karpenter-crd` chart templates the CRDs (rather than +shipping them in `crds/`), so Helm upgrades them on every version bump. This is +Karpenter's officially recommended way to manage CRD lifecycle. + +### Adopting pre-existing CRDs + +Every cluster created before this change already has the Karpenter CRDs — the +controller chart's `crds/` created them — but **without** `karpenter-crd` Helm +ownership metadata. A plain `karpenter-crd` install would therefore fail with +`exists and cannot be imported into the current release — invalid ownership +metadata`. + +We resolve this with the helm-controller's native **`spec.takeOwnership`** +(maps to `helm upgrade --take-ownership`). It rewrites the ownership label and +`release-name`/`-namespace` annotations on the existing objects so the +`karpenter-crd` release becomes their owner. It is scoped to exactly the CRDs +the `karpenter-crd` chart renders, and is a no-op on greenfield clusters (the +chart just creates them). This replaced an earlier approach that manually +pre-stamped ownership metadata via `CustomResourcePatch`. + +## Version requirements + +- **helm-controller >= v0.16.14** — adds `spec.takeOwnership` (PTD sets this image + in `clusters_aws.go` / `clusters_azure.go`, and the field must also be present + in PTD's embedded HelmChart CRD schema in `clusters_helpers.go`, or the API + server prunes it). +- **Helm >= 3.17** in the job image — `--take-ownership` (satisfied by the + `klipper-helm` job image PTD pins). + +## Known limitations + +- **The controller chart's bundled `crds/` are left in place.** The k3s + helm-controller has no `skipCRDs`/`crds` option and no way to pass Helm's + `--skip-crds` (verified against its `HelmChartSpec`), so the controller chart's + CRDs can't be disabled. This is benign on existing clusters: Helm skips `crds/` + resources that already exist. +- **Ordering is best-effort.** `dependsOn` orders CR *creation* (CRD chart before + the controller chart and the NodePool/EC2NodeClass CRs), but the helm-controller + reconciles HelmChart CRs asynchronously, so it doesn't strictly guarantee the + CRD helm job finishes first. The CRD changes are additive/backward-compatible + and the dependent CRs retry until the CRDs exist. + +## Rollout + +The helm-controller image bump and the embedded HelmChart CRD schema change apply +to **every** cluster through the `clusters` step, not just Karpenter clusters. +Validate on a staging cluster (ideally one already in the stale-CRD state) before +broad rollout — confirm both the brownfield adoption path and a greenfield +install. diff --git a/lib/steps/helm_aws.go b/lib/steps/helm_aws.go index 11cf7cd..abd65b2 100644 --- a/lib/steps/helm_aws.go +++ b/lib/steps/helm_aws.go @@ -1366,26 +1366,11 @@ func awsHelmKarpenter(ctx *pulumi.Context, k8sOpt pulumi.ResourceOption, compoun }, } - // ── Karpenter CRDs ───────────────────────────────────────────────────────── - // Manage the Karpenter CRDs in lockstep with the controller via the - // karpenter-crd chart, pinned to the SAME `version` (single source of truth — - // both driven by `karpenter_version`). This prevents a controller bump from - // running against stale, hand-applied CRDs. - // - // Adoption of pre-existing CRDs: every cluster created before this change - // already has the Karpenter CRDs installed (the controller chart's bundled - // crds/ created them) WITHOUT karpenter-crd Helm ownership metadata, so a - // fresh karpenter-crd install would otherwise fail with "exists and cannot be - // imported into the current release — invalid ownership metadata". We adopt - // them via the helm-controller's native spec.takeOwnership, which maps to - // `helm upgrade --take-ownership`: it rewrites the ownership label + - // release-name/namespace annotations on the existing objects so the - // karpenter-crd release becomes their owner. This is scoped to exactly the - // CRDs this chart renders, and is a no-op on greenfield clusters where the - // CRDs do not yet exist (the chart simply creates them cleanly). - // Requires helm-controller ≥ v0.16.14 (adds spec.takeOwnership) and Helm - // ≥ 3.17 in the job image (--take-ownership); both are satisfied by PTD's - // pinned versions. + // Manage the Karpenter CRDs via the karpenter-crd chart, pinned to the same + // karpenter_version as the controller so they never drift. takeOwnership adopts + // the controller chart's pre-existing, Helm-unowned CRDs (needs helm-controller + // >= v0.16.14 and Helm >= 3.17 in the job image). + // Full rationale: docs/infrastructure/karpenter-crds.md crdChartResourceName := compoundName + "-karpenter-crd-helm-release" crdChart, err := apiextensions.NewCustomResource(ctx, crdChartResourceName, &apiextensions.CustomResourceArgs{ ApiVersion: pulumi.String("helm.cattle.io/v1"), From fb887e9719144fc154b7540f01c2d24190f708a8 Mon Sep 17 00:00:00 2001 From: Anna Williamson Date: Wed, 22 Jul 2026 08:47:29 -0700 Subject: [PATCH 4/8] Adopt Karpenter CRDs via one-time manual stamp (drop takeOwnership) The helm job image is pinned to Helm 3.16.4 (klipper-helm:v0.9.5-build20250306), which predates --take-ownership, so spec.takeOwnership can't adopt the controller chart's pre-existing, Helm-unowned CRDs. Drop the takeOwnership field and the helm-controller v0.16.14 bump; adopt via a documented one-time manual kubectl stamp instead (docs/infrastructure/karpenter-crds.md). The karpenter-crd chart and dependsOn ordering are unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/infrastructure/karpenter-crds.md | 157 +++++++++++++++----------- lib/steps/clusters_aws.go | 2 +- lib/steps/clusters_azure.go | 2 +- lib/steps/clusters_helpers.go | 1 - lib/steps/helm_aws.go | 13 ++- 5 files changed, 103 insertions(+), 72 deletions(-) diff --git a/docs/infrastructure/karpenter-crds.md b/docs/infrastructure/karpenter-crds.md index 4c70854..42778a9 100644 --- a/docs/infrastructure/karpenter-crds.md +++ b/docs/infrastructure/karpenter-crds.md @@ -1,73 +1,102 @@ # Karpenter CRD management -How PTD manages Karpenter's CRDs, and why it's done this way. Implemented in -`awsHelmKarpenter` (`lib/steps/helm_aws.go`). +How PTD manages Karpenter's CRDs, and the one-time migration existing clusters +need. Implemented in `awsHelmKarpenter` (`lib/steps/helm_aws.go`). -## The problem +## Problem The Karpenter controller chart (`oci://public.ecr.aws/karpenter/karpenter`) -bundles its CRDs in the chart's `crds/` directory. Helm installs `crds/` -resources **once, on first install, and never upgrades them** on subsequent -`helm upgrade`s (this is intentional, documented Helm behavior). +bundles its CRDs in `crds/`. Helm installs `crds/` **once, on first install, and +never upgrades them**. So a `karpenter_version` bump upgrades the controller but +leaves the CRDs frozen. When a controller >= 1.9 emits a `Gte`/`Lte` requirement +(normalized from a NodePool `Gt`/`Lt` floor) against a pre-1.9 CRD, the API +server rejects the NodeClaim and no nodes provision. -So historically a `karpenter_version` bump upgraded the controller while leaving -the CRDs frozen at whatever version was first installed. When a controller ->= 1.9 then emits a `Gte`/`Lte` requirement — normalized from a NodePool -`Gt`/`Lt` instance floor — against a pre-1.9 CRD whose schema enum lacks those -operators, the API server rejects the NodeClaim and no nodes provision. The -break needs all three: controller >= 1.9, a NodePool using a `Gt`/`Lt` floor, -and a CRD missing `Gte`/`Lte`. - -## The fix +## Fix Manage the CRDs with the dedicated **`karpenter-crd`** chart, installed as a second `helm.cattle.io/v1` HelmChart pinned to the **same `karpenter_version`** -as the controller. The `karpenter-crd` chart templates the CRDs (rather than -shipping them in `crds/`), so Helm upgrades them on every version bump. This is -Karpenter's officially recommended way to manage CRD lifecycle. - -### Adopting pre-existing CRDs - -Every cluster created before this change already has the Karpenter CRDs — the -controller chart's `crds/` created them — but **without** `karpenter-crd` Helm -ownership metadata. A plain `karpenter-crd` install would therefore fail with -`exists and cannot be imported into the current release — invalid ownership -metadata`. - -We resolve this with the helm-controller's native **`spec.takeOwnership`** -(maps to `helm upgrade --take-ownership`). It rewrites the ownership label and -`release-name`/`-namespace` annotations on the existing objects so the -`karpenter-crd` release becomes their owner. It is scoped to exactly the CRDs -the `karpenter-crd` chart renders, and is a no-op on greenfield clusters (the -chart just creates them). This replaced an earlier approach that manually -pre-stamped ownership metadata via `CustomResourcePatch`. - -## Version requirements - -- **helm-controller >= v0.16.14** — adds `spec.takeOwnership` (PTD sets this image - in `clusters_aws.go` / `clusters_azure.go`, and the field must also be present - in PTD's embedded HelmChart CRD schema in `clusters_helpers.go`, or the API - server prunes it). -- **Helm >= 3.17** in the job image — `--take-ownership` (satisfied by the - `klipper-helm` job image PTD pins). - -## Known limitations - -- **The controller chart's bundled `crds/` are left in place.** The k3s - helm-controller has no `skipCRDs`/`crds` option and no way to pass Helm's - `--skip-crds` (verified against its `HelmChartSpec`), so the controller chart's - CRDs can't be disabled. This is benign on existing clusters: Helm skips `crds/` - resources that already exist. -- **Ordering is best-effort.** `dependsOn` orders CR *creation* (CRD chart before - the controller chart and the NodePool/EC2NodeClass CRs), but the helm-controller - reconciles HelmChart CRs asynchronously, so it doesn't strictly guarantee the - CRD helm job finishes first. The CRD changes are additive/backward-compatible - and the dependent CRs retry until the CRDs exist. - -## Rollout - -The helm-controller image bump and the embedded HelmChart CRD schema change apply -to **every** cluster through the `clusters` step, not just Karpenter clusters. -Validate on a staging cluster (ideally one already in the stale-CRD state) before -broad rollout — confirm both the brownfield adoption path and a greenfield -install. +as the controller. Its CRDs are templated, so Helm upgrades them on every bump. +This is Karpenter's recommended way to manage CRD lifecycle. + +## Why adoption is a manual, one-time step + +Every existing cluster already has the Karpenter CRDs — the controller chart's +`crds/` created them — but **without Helm ownership metadata**. A plain +`karpenter-crd` install therefore fails: + +``` +Unable to continue with install: CustomResourceDefinition "nodepools.karpenter.sh" +in namespace "" exists and cannot be imported into the current release: invalid +ownership metadata; ... +``` + +Helm's normal escape hatch is `--take-ownership`, but PTD's helm job image is +pinned to `klipper-helm:v0.9.5-build20250306` (**Helm 3.16.4**), which predates +that flag. (The pin is deliberate — newer, Helm-4 klipper-helm builds regressed +install/upgrade detection.) So adoption can't be automated in the chart. + +Instead, adopt the CRDs once per existing cluster by writing the ownership +metadata Helm expects, so the subsequent `karpenter-crd` install adopts them +cleanly. This is a **one-time migration**, not steady-state IaC: greenfield +clusters have no CRDs to adopt and skip it entirely. + +> ⚠️ **Rollout order matters.** Once this ships, the `helm` step tries to install +> `karpenter-crd` on every cluster. On an un-migrated existing cluster that +> install **fails** until the stamp below is applied — non-destructive (running +> Karpenter is unaffected; the chart just won't reconcile). Migrate each existing +> cluster before, or immediately alongside, its next `helm`-step apply. + +## Migration procedure (existing clusters, run once each) + +Run against the target cluster (e.g. inside `ptd workon `). + +**1. Confirm the CRDs exist and are not yet Helm-owned** (empty output = unowned): + +```bash +kubectl get crd nodeclaims.karpenter.sh \ + -o jsonpath='{.metadata.labels.app\.kubernetes\.io/managed-by}{"\n"}' +``` + +**2. Stamp Helm ownership.** Release name is the HelmChart CR name +(`karpenter-crd`); release namespace is its `targetNamespace` (`kube-system`). +Discovering the CRDs dynamically covers installs that lack `nodeoverlays`: + +```bash +CRDS=$(kubectl get crd -o name | grep 'karpenter\.' | sed 's|customresourcedefinition.apiextensions.k8s.io/||') +kubectl label crd $CRDS app.kubernetes.io/managed-by=Helm --overwrite +kubectl annotate crd $CRDS meta.helm.sh/release-name=karpenter-crd --overwrite +kubectl annotate crd $CRDS meta.helm.sh/release-namespace=kube-system --overwrite +``` + +**3. Apply the chart** — the `karpenter-crd` install now adopts and upgrades the +CRDs: + +```bash +ptd ensure --only-steps helm --dry-run # review: new karpenter-crd HelmChart +ptd ensure --only-steps helm --auto-apply +``` + +**4. Verify:** + +```bash +# adopted → karpenter-crd +kubectl get crd nodeclaims.karpenter.sh -o jsonpath='{.metadata.annotations.meta\.helm\.sh/release-name}{"\n"}' +# CRDs upgraded → enum now includes Gte/Lte +kubectl get crd nodeclaims.karpenter.sh -o jsonpath='{.spec.versions[*].schema.openAPIV3Schema.properties.spec.properties.requirements.items.properties.operator.enum}{"\n"}' +# existing Karpenter resources intact +kubectl get nodepools,ec2nodeclasses +``` + +**Rollback (before step 3 only):** the stamp is additive metadata; remove it with +`kubectl label crd $CRDS app.kubernetes.io/managed-by-` and +`kubectl annotate crd $CRDS meta.helm.sh/release-name- meta.helm.sh/release-namespace-`. +After step 3 the CRDs are owned by the `karpenter-crd` release. + +## Future: dropping the manual step + +When the fleet's helm job image moves to a fixed Helm-4 klipper-helm build +(`>= v0.11.1-build20260615`, which restored install/upgrade detection) paired +with helm-controller `>= v0.16.14`, set `spec.takeOwnership: true` on the +`karpenter-crd` HelmChart and adoption becomes automatic — the manual migration +above is no longer needed. That executor upgrade is tracked separately. diff --git a/lib/steps/clusters_aws.go b/lib/steps/clusters_aws.go index fa2d85c..c5d626f 100644 --- a/lib/steps/clusters_aws.go +++ b/lib/steps/clusters_aws.go @@ -699,7 +699,7 @@ func awsClustersDeploy(ctx *pulumi.Context, _ types.Target, params awsClustersPa Containers: corev1.ContainerArray{ &corev1.ContainerArgs{ Name: pulumi.String("helm-controller"), - Image: pulumi.String("ghcr.io/k3s-io/helm-controller:v0.16.14"), + Image: pulumi.String("ghcr.io/k3s-io/helm-controller:v0.16.10"), Command: pulumi.StringArray{pulumi.String("helm-controller")}, Args: pulumi.StringArray{ pulumi.String("--namespace"), diff --git a/lib/steps/clusters_azure.go b/lib/steps/clusters_azure.go index b2fe983..f8c6adc 100644 --- a/lib/steps/clusters_azure.go +++ b/lib/steps/clusters_azure.go @@ -485,7 +485,7 @@ func azureClustersDeploy(ctx *pulumi.Context, _ types.Target, params azureCluste Containers: corev1.ContainerArray{ &corev1.ContainerArgs{ Name: pulumi.String("helm-controller"), - Image: pulumi.String("ghcr.io/k3s-io/helm-controller:v0.16.14"), + Image: pulumi.String("ghcr.io/k3s-io/helm-controller:v0.16.10"), Command: pulumi.StringArray{pulumi.String("helm-controller")}, Args: pulumi.StringArray{ pulumi.String("--namespace"), diff --git a/lib/steps/clusters_helpers.go b/lib/steps/clusters_helpers.go index 3d1729e..1c945fe 100644 --- a/lib/steps/clusters_helpers.go +++ b/lib/steps/clusters_helpers.go @@ -158,7 +158,6 @@ var helmChartsCRDSpec = map[string]interface{}{ }, }, "set": map[string]interface{}{"nullable": true, "type": "object", "additionalProperties": map[string]interface{}{"x-kubernetes-int-or-string": true}}, - "takeOwnership": map[string]interface{}{"type": "boolean"}, "targetNamespace": map[string]interface{}{"nullable": true, "type": "string"}, "timeout": map[string]interface{}{"nullable": true, "type": "string"}, "valuesContent": map[string]interface{}{"nullable": true, "type": "string"}, diff --git a/lib/steps/helm_aws.go b/lib/steps/helm_aws.go index 721b4e4..85f0895 100644 --- a/lib/steps/helm_aws.go +++ b/lib/steps/helm_aws.go @@ -1411,10 +1411,14 @@ func awsHelmKarpenter(ctx *pulumi.Context, k8sOpt pulumi.ResourceOption, compoun } // Manage the Karpenter CRDs via the karpenter-crd chart, pinned to the same - // karpenter_version as the controller so they never drift. takeOwnership adopts - // the controller chart's pre-existing, Helm-unowned CRDs (needs helm-controller - // >= v0.16.14 and Helm >= 3.17 in the job image). - // Full rationale: docs/infrastructure/karpenter-crds.md + // karpenter_version as the controller so they never drift. + // + // On existing clusters the CRDs already exist (installed by the controller + // chart's crds/) without Helm ownership, and must be adopted with a ONE-TIME + // manual step before this chart first applies — see the migration procedure in + // docs/infrastructure/karpenter-crds.md. Adoption can't be automated here: the + // pinned helm job image runs Helm 3.16.4, which predates --take-ownership. + // Greenfield clusters need no migration (the chart just creates the CRDs). crdChartResourceName := compoundName + "-karpenter-crd-helm-release" crdChart, err := apiextensions.NewCustomResource(ctx, crdChartResourceName, &apiextensions.CustomResourceArgs{ ApiVersion: pulumi.String("helm.cattle.io/v1"), @@ -1429,7 +1433,6 @@ func awsHelmKarpenter(ctx *pulumi.Context, k8sOpt pulumi.ResourceOption, compoun "chart": pulumi.String("oci://public.ecr.aws/karpenter/karpenter-crd"), "targetNamespace": pulumi.String(clustersKubeSystemNamespace), "version": pulumi.String(version), - "takeOwnership": pulumi.Bool(true), }, }, }, k8sOpt, From 5cae1088200361963277ab80ac56cad8f2135e00 Mon Sep 17 00:00:00 2001 From: Anna Williamson Date: Wed, 22 Jul 2026 09:06:29 -0700 Subject: [PATCH 5/8] Trim Karpenter CRD code comment to a pointer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Condense the inline comment to what/where — the full rationale and migration procedure live in docs/infrastructure/karpenter-crds.md. Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/steps/helm_aws.go | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/lib/steps/helm_aws.go b/lib/steps/helm_aws.go index 85f0895..b46037c 100644 --- a/lib/steps/helm_aws.go +++ b/lib/steps/helm_aws.go @@ -1410,15 +1410,9 @@ func awsHelmKarpenter(ctx *pulumi.Context, k8sOpt pulumi.ResourceOption, compoun }, } - // Manage the Karpenter CRDs via the karpenter-crd chart, pinned to the same - // karpenter_version as the controller so they never drift. - // - // On existing clusters the CRDs already exist (installed by the controller - // chart's crds/) without Helm ownership, and must be adopted with a ONE-TIME - // manual step before this chart first applies — see the migration procedure in - // docs/infrastructure/karpenter-crds.md. Adoption can't be automated here: the - // pinned helm job image runs Helm 3.16.4, which predates --take-ownership. - // Greenfield clusters need no migration (the chart just creates the CRDs). + // Manage the Karpenter CRDs via the karpenter-crd chart, pinned to karpenter_version. + // NOTE: existing clusters need a ONE-TIME manual CRD adoption before first apply — + // see docs/infrastructure/karpenter-crds.md. crdChartResourceName := compoundName + "-karpenter-crd-helm-release" crdChart, err := apiextensions.NewCustomResource(ctx, crdChartResourceName, &apiextensions.CustomResourceArgs{ ApiVersion: pulumi.String("helm.cattle.io/v1"), From 507326d00330db676c3e1539dd382dd330aa4b15 Mon Sep 17 00:00:00 2001 From: Anna Williamson Date: Wed, 22 Jul 2026 09:28:26 -0700 Subject: [PATCH 6/8] Make CRD-adoption commands shell-robust in migration doc Use a for loop instead of an unquoted $CRDS variable, which zsh does not word-split (bash does), so the label/annotate ran against one mashed-together argument. The loop works in both shells. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/infrastructure/karpenter-crds.md | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/docs/infrastructure/karpenter-crds.md b/docs/infrastructure/karpenter-crds.md index 42778a9..b2e262d 100644 --- a/docs/infrastructure/karpenter-crds.md +++ b/docs/infrastructure/karpenter-crds.md @@ -63,10 +63,11 @@ kubectl get crd nodeclaims.karpenter.sh \ Discovering the CRDs dynamically covers installs that lack `nodeoverlays`: ```bash -CRDS=$(kubectl get crd -o name | grep 'karpenter\.' | sed 's|customresourcedefinition.apiextensions.k8s.io/||') -kubectl label crd $CRDS app.kubernetes.io/managed-by=Helm --overwrite -kubectl annotate crd $CRDS meta.helm.sh/release-name=karpenter-crd --overwrite -kubectl annotate crd $CRDS meta.helm.sh/release-namespace=kube-system --overwrite +for crd in $(kubectl get crd -o name | grep 'karpenter\.'); do + kubectl label "$crd" app.kubernetes.io/managed-by=Helm --overwrite + kubectl annotate "$crd" meta.helm.sh/release-name=karpenter-crd --overwrite + kubectl annotate "$crd" meta.helm.sh/release-namespace=kube-system --overwrite +done ``` **3. Apply the chart** — the `karpenter-crd` install now adopts and upgrades the @@ -88,10 +89,16 @@ kubectl get crd nodeclaims.karpenter.sh -o jsonpath='{.spec.versions[*].schema.o kubectl get nodepools,ec2nodeclasses ``` -**Rollback (before step 3 only):** the stamp is additive metadata; remove it with -`kubectl label crd $CRDS app.kubernetes.io/managed-by-` and -`kubectl annotate crd $CRDS meta.helm.sh/release-name- meta.helm.sh/release-namespace-`. -After step 3 the CRDs are owned by the `karpenter-crd` release. +**Rollback (before step 3 only):** the stamp is additive metadata; undo it (the +trailing `-` deletes each key). After step 3 the CRDs are owned by the +`karpenter-crd` release. + +```bash +for crd in $(kubectl get crd -o name | grep 'karpenter\.'); do + kubectl label "$crd" app.kubernetes.io/managed-by- + kubectl annotate "$crd" meta.helm.sh/release-name- meta.helm.sh/release-namespace- +done +``` ## Future: dropping the manual step From 615926bd3f7b47aea50ed666fcb34ff18270df3b Mon Sep 17 00:00:00 2001 From: Anna Williamson Date: Wed, 22 Jul 2026 11:02:25 -0700 Subject: [PATCH 7/8] Gloss nodeoverlays CRD in migration doc Add a one-phrase explanation for readers unfamiliar with the NodeOverlay CRD (added in Karpenter v1.7.1; absent on older installs). Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/infrastructure/karpenter-crds.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/infrastructure/karpenter-crds.md b/docs/infrastructure/karpenter-crds.md index b2e262d..60a8d82 100644 --- a/docs/infrastructure/karpenter-crds.md +++ b/docs/infrastructure/karpenter-crds.md @@ -60,7 +60,8 @@ kubectl get crd nodeclaims.karpenter.sh \ **2. Stamp Helm ownership.** Release name is the HelmChart CR name (`karpenter-crd`); release namespace is its `targetNamespace` (`kube-system`). -Discovering the CRDs dynamically covers installs that lack `nodeoverlays`: +Discovering the CRDs dynamically covers installs that lack `nodeoverlays` (a +Karpenter CRD added in v1.7.1, so clusters set up on older Karpenter don't have it): ```bash for crd in $(kubectl get crd -o name | grep 'karpenter\.'); do From d2ae2e6073aac571c9ee5fa549392ae5e31741da Mon Sep 17 00:00:00 2001 From: Anna Williamson Date: Wed, 22 Jul 2026 13:43:59 -0700 Subject: [PATCH 8/8] Guard Karpenter CRDs with helm.sh/resource-policy: keep MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stamp the karpenter-crd chart's CRDs with resource-policy: keep (via its additionalAnnotations value) so a helm uninstall — the HelmChart CR being deleted, renamed, or the code reverted — never cascade-deletes the CRDs and every NodePool/NodeClaim/EC2NodeClass. Restores the delete-safety the controller chart's bundled crds/ had, while keeping upgrade management. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/infrastructure/karpenter-crds.md | 6 ++++++ lib/steps/helm_aws.go | 3 +++ 2 files changed, 9 insertions(+) diff --git a/docs/infrastructure/karpenter-crds.md b/docs/infrastructure/karpenter-crds.md index 60a8d82..70e63af 100644 --- a/docs/infrastructure/karpenter-crds.md +++ b/docs/infrastructure/karpenter-crds.md @@ -19,6 +19,12 @@ second `helm.cattle.io/v1` HelmChart pinned to the **same `karpenter_version`** as the controller. Its CRDs are templated, so Helm upgrades them on every bump. This is Karpenter's recommended way to manage CRD lifecycle. +The chart's CRDs are stamped `helm.sh/resource-policy: keep` (via its +`additionalAnnotations` value) so that a `helm uninstall` — e.g. the HelmChart CR +being deleted, renamed, or the code reverted — never cascade-deletes the CRDs and +every NodePool/NodeClaim/EC2NodeClass. Uninstall leaves the CRDs in place (the safe +default, matching how the controller chart's bundled `crds/` always behaved). + ## Why adoption is a manual, one-time step Every existing cluster already has the Karpenter CRDs — the controller chart's diff --git a/lib/steps/helm_aws.go b/lib/steps/helm_aws.go index b46037c..b7fafe5 100644 --- a/lib/steps/helm_aws.go +++ b/lib/steps/helm_aws.go @@ -1427,6 +1427,9 @@ func awsHelmKarpenter(ctx *pulumi.Context, k8sOpt pulumi.ResourceOption, compoun "chart": pulumi.String("oci://public.ecr.aws/karpenter/karpenter-crd"), "targetNamespace": pulumi.String(clustersKubeSystemNamespace), "version": pulumi.String(version), + // resource-policy: keep so a helm uninstall (e.g. this HelmChart CR being + // deleted) never cascade-deletes the CRDs and every NodePool/NodeClaim/EC2NodeClass. + "valuesContent": pulumi.String("additionalAnnotations:\n helm.sh/resource-policy: keep\n"), }, }, }, k8sOpt,