From b236a94bec07a2a6f733e3e3367884a35dcb2ea9 Mon Sep 17 00:00:00 2001 From: Anthony Byrne Date: Thu, 23 Jul 2026 10:14:09 -0400 Subject: [PATCH] feat: offload DP/DRA management to KMM Replace direct DevicePlugin and DRA DaemonSet management with a single KMM (Kernel Module Management) Module CR. Two controllers collapse into one, and the operator no longer needs to build DaemonSets, manage RBAC, handle pod readiness, or construct OpenShift SCCs at runtime. RBAC, SCC, ValidatingAdmissionPolicy, and DeviceClass resources have been moved from operator-managed code into Helm templates and kustomize overlays. A new optional kernelModule CRD field enables OOT driver loading via KMM when set. Signed-off-by: Anthony Byrne --- api/v1alpha1/clusterpolicy_types.go | 100 +++ api/v1alpha1/clusterpolicy_webhook.go | 109 +++ api/v1alpha1/clusterpolicy_webhook_test.go | 222 +++++ api/v1alpha1/zz_generated.deepcopy.go | 103 +++ .../templates/clusterpolicy.yaml | 55 ++ charts/gpu-base-operator-policy/values.yaml | 26 + .../crds/clusterpolicies.yaml | 146 ++++ .../gpu-base-operator/templates/dp_scc.yaml | 38 + .../templates/dp_scc_role.yaml | 14 + .../templates/dp_scc_rolebinding.yaml | 17 + .../templates/dp_serviceaccount.yaml | 8 + .../templates/dra_admission_policy.yaml | 34 + .../dra_admission_policy_binding.yaml | 10 + .../templates/dra_clusterrole.yaml | 17 + .../templates/dra_clusterrolebinding.yaml | 15 + .../gpu-base-operator/templates/dra_scc.yaml | 37 + .../templates/dra_scc_role.yaml | 14 + .../templates/dra_scc_rolebinding.yaml | 17 + .../templates/dra_serviceaccount.yaml | 8 + .../gpu-base-operator/templates/manager.yaml | 4 + charts/gpu-base-operator/templates/role.yaml | 18 + charts/gpu-base-operator/values.yaml | 9 + cmd/main.go | 37 +- .../crd/bases/intel.com_clusterpolicies.yaml | 146 ++++ config/deployments/deployments.go | 150 ---- config/deployments/deployments_test.go | 197 ----- config/deployments/dp/dp.yaml | 80 -- .../dra/admissionpolicybinding.yaml | 7 - .../deployments/dra/clusterrolebinding.yaml | 13 - config/deployments/dra/daemonset.yaml | 90 --- config/deployments/dra/device-class-vfio.yaml | 12 - config/deployments/dra/device-class.yaml | 12 - config/deployments/dra/serviceaccount.yaml | 5 - config/manager/manager.yaml | 4 + config/rbac/dp_scc.yaml | 35 + config/rbac/dp_scc_role.yaml | 12 + config/rbac/dp_scc_rolebinding.yaml | 14 + config/rbac/dp_serviceaccount.yaml | 7 + .../dra_admission_policy.yaml} | 6 +- config/rbac/dra_admission_policy_binding.yaml | 9 + .../dra_clusterrole.yaml} | 4 +- config/rbac/dra_clusterrolebinding.yaml | 14 + config/rbac/dra_scc.yaml | 34 + config/rbac/dra_scc_role.yaml | 12 + config/rbac/dra_scc_rolebinding.yaml | 14 + config/rbac/dra_serviceaccount.yaml | 7 + config/rbac/kustomization.yaml | 21 + config/rbac/module_loader_scc.yaml | 37 + config/rbac/module_loader_scc_role.yaml | 12 + .../rbac/module_loader_scc_rolebinding.yaml | 14 + config/rbac/module_loader_serviceaccount.yaml | 7 + config/rbac/role.yaml | 18 + go.mod | 11 +- go.sum | 22 +- .../controller/clusterpolicy_controller.go | 126 +-- .../clusterpolicy_controller_test.go | 199 +---- .../controller/deviceplugin_controller.go | 354 -------- .../deviceplugin_controller_test.go | 721 ----------------- internal/controller/dra_controller.go | 582 -------------- internal/controller/dra_controller_test.go | 534 ------------ internal/controller/kmm_controller.go | 467 +++++++++++ internal/controller/kmm_controller_test.go | 761 ++++++++++++++++++ internal/controller/openshift.go | 51 -- internal/controller/openshift_test.go | 44 +- internal/controller/suite_test.go | 15 + internal/controller/xpumanager_controller.go | 14 +- 66 files changed, 2803 insertions(+), 3149 deletions(-) create mode 100644 charts/gpu-base-operator/templates/dp_scc.yaml create mode 100644 charts/gpu-base-operator/templates/dp_scc_role.yaml create mode 100644 charts/gpu-base-operator/templates/dp_scc_rolebinding.yaml create mode 100644 charts/gpu-base-operator/templates/dp_serviceaccount.yaml create mode 100644 charts/gpu-base-operator/templates/dra_admission_policy.yaml create mode 100644 charts/gpu-base-operator/templates/dra_admission_policy_binding.yaml create mode 100644 charts/gpu-base-operator/templates/dra_clusterrole.yaml create mode 100644 charts/gpu-base-operator/templates/dra_clusterrolebinding.yaml create mode 100644 charts/gpu-base-operator/templates/dra_scc.yaml create mode 100644 charts/gpu-base-operator/templates/dra_scc_role.yaml create mode 100644 charts/gpu-base-operator/templates/dra_scc_rolebinding.yaml create mode 100644 charts/gpu-base-operator/templates/dra_serviceaccount.yaml delete mode 100644 config/deployments/dp/dp.yaml delete mode 100644 config/deployments/dra/admissionpolicybinding.yaml delete mode 100644 config/deployments/dra/clusterrolebinding.yaml delete mode 100644 config/deployments/dra/daemonset.yaml delete mode 100644 config/deployments/dra/device-class-vfio.yaml delete mode 100644 config/deployments/dra/device-class.yaml delete mode 100644 config/deployments/dra/serviceaccount.yaml create mode 100644 config/rbac/dp_scc.yaml create mode 100644 config/rbac/dp_scc_role.yaml create mode 100644 config/rbac/dp_scc_rolebinding.yaml create mode 100644 config/rbac/dp_serviceaccount.yaml rename config/{deployments/dra/admissionpolicy.yaml => rbac/dra_admission_policy.yaml} (88%) create mode 100644 config/rbac/dra_admission_policy_binding.yaml rename config/{deployments/dra/clusterrole.yaml => rbac/dra_clusterrole.yaml} (80%) create mode 100644 config/rbac/dra_clusterrolebinding.yaml create mode 100644 config/rbac/dra_scc.yaml create mode 100644 config/rbac/dra_scc_role.yaml create mode 100644 config/rbac/dra_scc_rolebinding.yaml create mode 100644 config/rbac/dra_serviceaccount.yaml create mode 100644 config/rbac/module_loader_scc.yaml create mode 100644 config/rbac/module_loader_scc_role.yaml create mode 100644 config/rbac/module_loader_scc_rolebinding.yaml create mode 100644 config/rbac/module_loader_serviceaccount.yaml delete mode 100644 internal/controller/deviceplugin_controller.go delete mode 100644 internal/controller/deviceplugin_controller_test.go delete mode 100644 internal/controller/dra_controller.go delete mode 100644 internal/controller/dra_controller_test.go create mode 100644 internal/controller/kmm_controller.go create mode 100644 internal/controller/kmm_controller_test.go diff --git a/api/v1alpha1/clusterpolicy_types.go b/api/v1alpha1/clusterpolicy_types.go index a58a3a1..08772da 100644 --- a/api/v1alpha1/clusterpolicy_types.go +++ b/api/v1alpha1/clusterpolicy_types.go @@ -69,6 +69,12 @@ type ClusterPolicySpec struct { // +kubebuilder:validation:Range=0:4 // +kubebuilder:validation:Default=2 LogLevel int32 `json:"logLevel,omitempty"` + + // KernelModule configures out-of-tree kernel module loading via KMM. + // When set, KMM loads the specified OOT driver module on each node. + // When nil, the in-tree kernel driver is used. + // +optional + KernelModule *KernelModuleSpec `json:"kernelModule,omitempty"` } // DynamicResourceAllocationSpec defines the desired state of DynamicResourceAllocation. @@ -147,11 +153,104 @@ type XpuManagerSpec struct { MonitoringResource string `json:"monitoringResource,omitempty"` } +// KernelModuleSpec configures out-of-tree kernel module loading via KMM. +type KernelModuleSpec struct { + // ModuleName is the kernel module to load (e.g., "xe"). + // Also used as the default InTreeModulesToRemove entry. + ModuleName string `json:"moduleName"` + + // Image is the default container image for the OOT driver module. + // When KernelMappings is empty, generates a single wildcard mapping. + // When KernelMappings is non-empty, serves as the KMM-level fallback + // for mappings that omit ContainerImage. + // +optional + Image string `json:"image,omitempty"` + + // KernelMappings maps kernel version patterns to container images or + // build specifications. Translates directly to KMM KernelMapping + // objects. When empty, a single wildcard mapping (regexp "^.+$") is + // generated from Image. + // +optional + KernelMappings []KernelMappingSpec `json:"kernelMappings,omitempty"` + + // InTreeModulesToRemove lists in-tree modules to unload before + // inserting the OOT module. ModuleName is always included implicitly. + // +optional + InTreeModulesToRemove []string `json:"inTreeModulesToRemove,omitempty"` + + // ModulesLoadingOrder specifies softdep-style loading order for + // multi-module drivers. First element must be ModuleName; KMM loads + // in order and unloads in reverse. Must have >=2 entries if set. + // +optional + ModulesLoadingOrder []string `json:"modulesLoadingOrder,omitempty"` + + // FirmwarePath is the in-container path where firmware files are stored. + // +optional + FirmwarePath string `json:"firmwarePath,omitempty"` + + // SkipTLSVerify disables TLS certificate verification when pulling + // OOT driver images. For air-gapped or internal registries. + // +optional + SkipTLSVerify bool `json:"skipTLSVerify,omitempty"` +} + +// KernelMappingSpec maps a kernel version pattern to a container image +// or build specification. +type KernelMappingSpec struct { + // Regexp is a regular expression matched against node kernel versions. + // Exactly one of Regexp or Literal must be set. + // +optional + Regexp string `json:"regexp,omitempty"` + + // Literal is an exact kernel version string to match. + // Exactly one of Regexp or Literal must be set. + // +optional + Literal string `json:"literal,omitempty"` + + // ContainerImage is the full image reference for this kernel version. + // Required when Build is nil and no parent-level Image is set. + // +optional + ContainerImage string `json:"containerImage,omitempty"` + + // Build configures in-cluster building of the driver image via KMM. + // When set, KMM builds the image if it doesn't exist in the registry. + // +optional + Build *KernelModuleBuildSpec `json:"build,omitempty"` + + // InTreeModulesToRemove overrides the parent-level list for this + // specific kernel mapping. + // +optional + InTreeModulesToRemove []string `json:"inTreeModulesToRemove,omitempty"` +} + +// KernelModuleBuildSpec configures in-cluster driver image building. +type KernelModuleBuildSpec struct { + // DockerfileConfigMap references a ConfigMap containing the Dockerfile. + DockerfileConfigMap v1.LocalObjectReference `json:"dockerfileConfigMap"` + + // BuildArgs are key-value pairs passed to the image builder. + // +optional + BuildArgs []BuildArg `json:"buildArgs,omitempty"` + + // Secrets are made available during the build (e.g., for private + // source repos). Not for registry auth -- use pullSecret on + // ClusterPolicySpec. + // +optional + Secrets []v1.LocalObjectReference `json:"secrets,omitempty"` +} + +// BuildArg is a key-value pair passed as a build argument. +type BuildArg struct { + Name string `json:"name"` + Value string `json:"value"` +} + // ClusterPolicyStatus defines the observed state of ClusterPolicy. type ClusterPolicyStatus struct { DevicePluginStatus string `json:"devicePluginStatus,omitempty"` DRAStatus string `json:"draStatus,omitempty"` XPUManagerStatus string `json:"xpuManagerStatus,omitempty"` + KMMStatus string `json:"kmmStatus,omitempty"` Errors []string `json:"errors,omitempty"` } @@ -183,6 +282,7 @@ type LocalQueueSpec struct { // +kubebuilder:printcolumn:name="DP",type=string,JSONPath=`.status.devicePluginStatus` // +kubebuilder:printcolumn:name="DRA",type=string,JSONPath=`.status.draStatus` // +kubebuilder:printcolumn:name="XPU",type=string,JSONPath=`.status.xpuManagerStatus` +// +kubebuilder:printcolumn:name="KMM",type=string,JSONPath=`.status.kmmStatus` // +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` // +operator-sdk:csv:customresourcedefinitions:displayName="Intel GPU Cluster Policy" // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/api/v1alpha1/clusterpolicy_webhook.go b/api/v1alpha1/clusterpolicy_webhook.go index 2b58d53..e55e849 100644 --- a/api/v1alpha1/clusterpolicy_webhook.go +++ b/api/v1alpha1/clusterpolicy_webhook.go @@ -95,6 +95,7 @@ func validateClusterPolicySpec(spec *ClusterPolicySpec) (admission.Warnings, err errs = append(errs, validatePullSecret(spec)...) errs = append(errs, validateConfigMapOverride(spec)...) errs = append(errs, validateKueueSpec(spec)...) + errs = append(errs, validateKernelModuleSpec(spec)...) if w := warnForSpecProblems(spec); w != "" { warnings = append(warnings, w) @@ -223,6 +224,114 @@ func validateKueueSpec(spec *ClusterPolicySpec) []error { return errs } +func validateKernelModuleSpec(spec *ClusterPolicySpec) []error { + if spec.KernelModule == nil { + return nil + } + + km := spec.KernelModule + var errs []error + + if km.ModuleName == "" { + errs = append(errs, fmt.Errorf("kernelModule.moduleName is required")) + } + + if km.Image != "" { + if err := validateImageTagOrDigest("kernelModule.image", km.Image); err != nil { + errs = append(errs, err) + } + } + + if len(km.KernelMappings) == 0 { + if km.Image == "" { + errs = append(errs, fmt.Errorf("kernelModule.image is required when kernelMappings is empty")) + } + } else { + for i, m := range km.KernelMappings { + prefix := fmt.Sprintf("kernelModule.kernelMappings[%d]", i) + errs = append(errs, validateKernelMapping(prefix, &m, km.Image)...) + } + } + + if len(km.ModulesLoadingOrder) > 0 { + errs = append(errs, validateModulesLoadingOrder(km.ModuleName, km.ModulesLoadingOrder)...) + } + + return errs +} + +func validateImageTagOrDigest(field, image string) error { + if _, err := reference.ParseNormalizedNamed(image); err != nil { + return fmt.Errorf("invalid image reference in %s: %q: %w", field, image, err) + } + + if !strings.Contains(image, ":") && !strings.Contains(image, "@") { + return fmt.Errorf("%s: image %q must include an explicit tag or digest", field, image) + } + + return nil +} + +func validateKernelMapping(prefix string, m *KernelMappingSpec, parentImage string) []error { + var errs []error + + hasRegexp := m.Regexp != "" + hasLiteral := m.Literal != "" + + if hasRegexp && hasLiteral { + errs = append(errs, fmt.Errorf("%s: regexp and literal are mutually exclusive", prefix)) + } else if !hasRegexp && !hasLiteral { + errs = append(errs, fmt.Errorf("%s: exactly one of regexp or literal must be set", prefix)) + } + + if hasRegexp { + if _, err := regexp.Compile(m.Regexp); err != nil { + errs = append(errs, fmt.Errorf("%s.regexp: invalid regular expression: %w", prefix, err)) + } + } + + if m.ContainerImage != "" { + if err := validateImageTagOrDigest(prefix+".containerImage", m.ContainerImage); err != nil { + errs = append(errs, err) + } + } + + if m.ContainerImage == "" && m.Build == nil && parentImage == "" { + errs = append(errs, fmt.Errorf("%s: one of containerImage, build, or parent-level image must be set", prefix)) + } + + if m.Build != nil { + if m.Build.DockerfileConfigMap.Name == "" { + errs = append(errs, fmt.Errorf("%s.build.dockerfileConfigMap.name is required", prefix)) + } + } + + return errs +} + +func validateModulesLoadingOrder(moduleName string, order []string) []error { + var errs []error + + if len(order) < 2 { + errs = append(errs, fmt.Errorf("kernelModule.modulesLoadingOrder must have at least 2 entries")) + } + + if len(order) > 0 && moduleName != "" && order[0] != moduleName { + errs = append(errs, fmt.Errorf("kernelModule.modulesLoadingOrder[0] must be moduleName %q", moduleName)) + } + + seen := make(map[string]bool, len(order)) + for i, entry := range order { + if seen[entry] { + errs = append(errs, fmt.Errorf("kernelModule.modulesLoadingOrder[%d]: duplicate entry %q", i, entry)) + break + } + seen[entry] = true + } + + return errs +} + // warnForSpecProblems returns a warning message if some old or deprecated option is set. func warnForSpecProblems(spec *ClusterPolicySpec) string { if spec.ResourceRegistration == "dp" && spec.DevicePluginSpec.LevelzeroImage != "" { diff --git a/api/v1alpha1/clusterpolicy_webhook_test.go b/api/v1alpha1/clusterpolicy_webhook_test.go index 45967d1..2c56e40 100644 --- a/api/v1alpha1/clusterpolicy_webhook_test.go +++ b/api/v1alpha1/clusterpolicy_webhook_test.go @@ -387,6 +387,228 @@ var _ = Describe("ClusterPolicy Webhook", func() { }) }) + Context("kernelModule validation", func() { + It("accepts nil kernelModule (in-tree mode)", func() { + obj.Spec.KernelModule = nil + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).NotTo(HaveOccurred()) + }) + + It("accepts valid kernelModule with moduleName + image only", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + Image: "registry.example.com/xe-driver:1.0", + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).NotTo(HaveOccurred()) + }) + + It("rejects kernelModule with empty moduleName", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + Image: "registry.example.com/xe-driver:1.0", + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("moduleName is required")) + }) + + It("rejects kernelModule without image when kernelMappings is empty", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("image is required when kernelMappings is empty")) + }) + + It("accepts kernelMappings with each mapping having its own containerImage", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + {Regexp: "^5\\.14\\..*", ContainerImage: "registry.example.com/xe-rhel9:1.0"}, + {Regexp: "^6\\.12\\..*", ContainerImage: "registry.example.com/xe-rhel10:1.0"}, + }, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).NotTo(HaveOccurred()) + }) + + It("accepts image as fallback when mapping omits containerImage", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + Image: "registry.example.com/xe-driver:1.0", + KernelMappings: []KernelMappingSpec{ + {Regexp: "^5\\.14\\..*"}, + }, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).NotTo(HaveOccurred()) + }) + + It("rejects mapping with neither containerImage, build, nor parent image", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + {Regexp: "^5\\.14\\..*"}, + }, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("one of containerImage, build, or parent-level image must be set")) + }) + + It("rejects mapping with both regexp and literal set", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + {Regexp: "^5\\.14\\..*", Literal: "5.14.0-687.el9.x86_64", ContainerImage: "registry.example.com/xe:1.0"}, + }, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("regexp and literal are mutually exclusive")) + }) + + It("rejects mapping with neither regexp nor literal set", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + {ContainerImage: "registry.example.com/xe:1.0"}, + }, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("exactly one of regexp or literal must be set")) + }) + + It("rejects mapping with invalid regexp", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + {Regexp: "[invalid", ContainerImage: "registry.example.com/xe:1.0"}, + }, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("invalid regular expression")) + }) + + It("rejects image without tag or digest", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + Image: "registry.example.com/xe-driver", + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("must include an explicit tag or digest")) + }) + + It("rejects containerImage without tag or digest", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + {Regexp: "^5\\.14\\..*", ContainerImage: "registry.example.com/xe-driver"}, + }, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("must include an explicit tag or digest")) + }) + + It("accepts image with tag", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + Image: "registry.example.com/xe-driver:1.0", + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).NotTo(HaveOccurred()) + }) + + It("accepts image with digest", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + Image: "registry.example.com/xe-driver@sha256:abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890", + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).NotTo(HaveOccurred()) + }) + + It("accepts valid build spec with dockerfileConfigMap", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + { + Regexp: "^5\\.14\\..*", + Build: &KernelModuleBuildSpec{ + DockerfileConfigMap: v1.LocalObjectReference{Name: "xe-dockerfile"}, + BuildArgs: []BuildArg{{Name: "XE_TAG", Value: "v1.0"}}, + }, + }, + }, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).NotTo(HaveOccurred()) + }) + + It("rejects build spec with empty dockerfileConfigMap name", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + { + Regexp: "^5\\.14\\..*", + Build: &KernelModuleBuildSpec{}, + }, + }, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("dockerfileConfigMap.name is required")) + }) + + It("accepts valid modulesLoadingOrder with moduleName first", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + Image: "registry.example.com/xe-driver:1.0", + ModulesLoadingOrder: []string{"xe", "drm_buddy"}, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).NotTo(HaveOccurred()) + }) + + It("rejects modulesLoadingOrder with fewer than 2 entries", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + Image: "registry.example.com/xe-driver:1.0", + ModulesLoadingOrder: []string{"xe"}, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("at least 2 entries")) + }) + + It("rejects modulesLoadingOrder where first entry is not moduleName", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + Image: "registry.example.com/xe-driver:1.0", + ModulesLoadingOrder: []string{"drm_buddy", "xe"}, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("must be moduleName")) + }) + + It("rejects modulesLoadingOrder with duplicate entries", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + Image: "registry.example.com/xe-driver:1.0", + ModulesLoadingOrder: []string{"xe", "drm_buddy", "drm_buddy"}, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("duplicate entry")) + }) + }) + Context("spec warning for Levelzero", func() { It("emits a warning when levelzero image is set in DP mode", func() { obj.Spec.ResourceRegistration = dpName diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 2470cf4..2b3162d 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -25,6 +25,21 @@ import ( "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BuildArg) DeepCopyInto(out *BuildArg) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BuildArg. +func (in *BuildArg) DeepCopy() *BuildArg { + if in == nil { + return nil + } + out := new(BuildArg) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClusterPolicy) DeepCopyInto(out *ClusterPolicy) { *out = *in @@ -149,6 +164,11 @@ func (in *ClusterPolicySpec) DeepCopyInto(out *ClusterPolicySpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.KernelModule != nil { + in, out := &in.KernelModule, &out.KernelModule + *out = new(KernelModuleSpec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterPolicySpec. @@ -466,6 +486,89 @@ func (in *HealthinessSpec) DeepCopy() *HealthinessSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KernelMappingSpec) DeepCopyInto(out *KernelMappingSpec) { + *out = *in + if in.Build != nil { + in, out := &in.Build, &out.Build + *out = new(KernelModuleBuildSpec) + (*in).DeepCopyInto(*out) + } + if in.InTreeModulesToRemove != nil { + in, out := &in.InTreeModulesToRemove, &out.InTreeModulesToRemove + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KernelMappingSpec. +func (in *KernelMappingSpec) DeepCopy() *KernelMappingSpec { + if in == nil { + return nil + } + out := new(KernelMappingSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KernelModuleBuildSpec) DeepCopyInto(out *KernelModuleBuildSpec) { + *out = *in + out.DockerfileConfigMap = in.DockerfileConfigMap + if in.BuildArgs != nil { + in, out := &in.BuildArgs, &out.BuildArgs + *out = make([]BuildArg, len(*in)) + copy(*out, *in) + } + if in.Secrets != nil { + in, out := &in.Secrets, &out.Secrets + *out = make([]v1.LocalObjectReference, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KernelModuleBuildSpec. +func (in *KernelModuleBuildSpec) DeepCopy() *KernelModuleBuildSpec { + if in == nil { + return nil + } + out := new(KernelModuleBuildSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KernelModuleSpec) DeepCopyInto(out *KernelModuleSpec) { + *out = *in + if in.KernelMappings != nil { + in, out := &in.KernelMappings, &out.KernelMappings + *out = make([]KernelMappingSpec, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.InTreeModulesToRemove != nil { + in, out := &in.InTreeModulesToRemove, &out.InTreeModulesToRemove + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.ModulesLoadingOrder != nil { + in, out := &in.ModulesLoadingOrder, &out.ModulesLoadingOrder + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KernelModuleSpec. +func (in *KernelModuleSpec) DeepCopy() *KernelModuleSpec { + if in == nil { + return nil + } + out := new(KernelModuleSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KueueQueueSpec) DeepCopyInto(out *KueueQueueSpec) { *out = *in diff --git a/charts/gpu-base-operator-policy/templates/clusterpolicy.yaml b/charts/gpu-base-operator-policy/templates/clusterpolicy.yaml index 737355d..8716f82 100644 --- a/charts/gpu-base-operator-policy/templates/clusterpolicy.yaml +++ b/charts/gpu-base-operator-policy/templates/clusterpolicy.yaml @@ -62,6 +62,61 @@ spec: configMapOverride: {{ .Values.xpu.configMapOverride | default "" }} {{- end }} +{{- if .Values.kernelModule }} + kernelModule: + moduleName: {{ .Values.kernelModule.moduleName }} + {{- if .Values.kernelModule.image }} + image: {{ .Values.kernelModule.image }} + {{- end }} + {{- if .Values.kernelModule.kernelMappings }} + kernelMappings: + {{- range .Values.kernelModule.kernelMappings }} + - {{- if .regexp }} + regexp: {{ .regexp | quote }} + {{- end }} + {{- if .literal }} + literal: {{ .literal | quote }} + {{- end }} + {{- if .containerImage }} + containerImage: {{ .containerImage }} + {{- end }} + {{- if .build }} + build: + dockerfileConfigMap: + name: {{ .build.dockerfileConfigMap.name }} + {{- if .build.buildArgs }} + buildArgs: + {{- range .build.buildArgs }} + - name: {{ .name }} + value: {{ .value | quote }} + {{- end }} + {{- end }} + {{- if .build.secrets }} + secrets: + {{- range .build.secrets }} + - name: {{ .name }} + {{- end }} + {{- end }} + {{- end }} + {{- if .inTreeModulesToRemove }} + inTreeModulesToRemove: {{ toYaml .inTreeModulesToRemove | nindent 10 }} + {{- end }} + {{- end }} + {{- end }} + {{- if .Values.kernelModule.inTreeModulesToRemove }} + inTreeModulesToRemove: {{ toYaml .Values.kernelModule.inTreeModulesToRemove | nindent 6 }} + {{- end }} + {{- if .Values.kernelModule.modulesLoadingOrder }} + modulesLoadingOrder: {{ toYaml .Values.kernelModule.modulesLoadingOrder | nindent 6 }} + {{- end }} + {{- if .Values.kernelModule.firmwarePath }} + firmwarePath: {{ .Values.kernelModule.firmwarePath }} + {{- end }} + {{- if .Values.kernelModule.skipTLSVerify }} + skipTLSVerify: {{ .Values.kernelModule.skipTLSVerify }} + {{- end }} +{{- end }} + {{- if .Values.pullSecret }} pullSecret: {{ .Values.pullSecret }} {{- end }} diff --git a/charts/gpu-base-operator-policy/values.yaml b/charts/gpu-base-operator-policy/values.yaml index fc465f2..01a4d50 100644 --- a/charts/gpu-base-operator-policy/values.yaml +++ b/charts/gpu-base-operator-policy/values.yaml @@ -35,6 +35,32 @@ kueue: - name: gpu-queue namespace: default +# kernelModule: +# moduleName: xe +# image: registry.example.com/xe-driver:1.0 +# +# # Multi-kernel support: +# # kernelMappings: +# # - regexp: "^5\\.14\\.0-.*\\.el9.*\\.x86_64$" +# # containerImage: registry.example.com/xe-driver-rhel9:1.0 +# # - regexp: "^6\\.12\\..*" +# # containerImage: registry.example.com/xe-driver-rhel10:1.0 +# # +# # In-cluster build: +# # kernelMappings: +# # - regexp: "^5\\.14\\..*" +# # build: +# # dockerfileConfigMap: +# # name: xe-dockerfile +# # buildArgs: +# # - name: XE_TAG +# # value: v1.0 +# # +# # inTreeModulesToRemove: ["i915"] +# # modulesLoadingOrder: ["xe", "drm_buddy", "drm_ttm_helper"] +# # firmwarePath: /opt/lib/firmware/xe +# # skipTLSVerify: false + pullSecret: null nodeSelector: {} tolerations: [] diff --git a/charts/gpu-base-operator/crds/clusterpolicies.yaml b/charts/gpu-base-operator/crds/clusterpolicies.yaml index 870f9c5..af78116 100644 --- a/charts/gpu-base-operator/crds/clusterpolicies.yaml +++ b/charts/gpu-base-operator/crds/clusterpolicies.yaml @@ -24,6 +24,9 @@ spec: - jsonPath: .status.xpuManagerStatus name: XPU type: string + - jsonPath: .status.kmmStatus + name: KMM + type: string - jsonPath: .metadata.creationTimestamp name: Age type: date @@ -136,6 +139,147 @@ spec: minimum: 1 type: integer type: object + kernelModule: + description: |- + KernelModule configures out-of-tree kernel module loading via KMM. + When set, KMM loads the specified OOT driver module on each node. + When nil, the in-tree kernel driver is used. + properties: + firmwarePath: + description: FirmwarePath is the in-container path where firmware + files are stored. + type: string + image: + description: |- + Image is the default container image for the OOT driver module. + When KernelMappings is empty, generates a single wildcard mapping. + When KernelMappings is non-empty, serves as the KMM-level fallback + for mappings that omit ContainerImage. + type: string + inTreeModulesToRemove: + description: |- + InTreeModulesToRemove lists in-tree modules to unload before + inserting the OOT module. ModuleName is always included implicitly. + items: + type: string + type: array + kernelMappings: + description: |- + KernelMappings maps kernel version patterns to container images or + build specifications. Translates directly to KMM KernelMapping + objects. When empty, a single wildcard mapping (regexp "^.+$") is + generated from Image. + items: + description: |- + KernelMappingSpec maps a kernel version pattern to a container image + or build specification. + properties: + build: + description: |- + Build configures in-cluster building of the driver image via KMM. + When set, KMM builds the image if it doesn't exist in the registry. + properties: + buildArgs: + description: BuildArgs are key-value pairs passed to + the image builder. + items: + description: BuildArg is a key-value pair passed as + a build argument. + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + dockerfileConfigMap: + description: DockerfileConfigMap references a ConfigMap + containing the Dockerfile. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + secrets: + description: |- + Secrets are made available during the build (e.g., for private + source repos). Not for registry auth -- use pullSecret on + ClusterPolicySpec. + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + type: array + required: + - dockerfileConfigMap + type: object + containerImage: + description: |- + ContainerImage is the full image reference for this kernel version. + Required when Build is nil and no parent-level Image is set. + type: string + inTreeModulesToRemove: + description: |- + InTreeModulesToRemove overrides the parent-level list for this + specific kernel mapping. + items: + type: string + type: array + literal: + description: |- + Literal is an exact kernel version string to match. + Exactly one of Regexp or Literal must be set. + type: string + regexp: + description: |- + Regexp is a regular expression matched against node kernel versions. + Exactly one of Regexp or Literal must be set. + type: string + type: object + type: array + moduleName: + description: |- + ModuleName is the kernel module to load (e.g., "xe"). + Also used as the default InTreeModulesToRemove entry. + type: string + modulesLoadingOrder: + description: |- + ModulesLoadingOrder specifies softdep-style loading order for + multi-module drivers. First element must be ModuleName; KMM loads + in order and unloads in reverse. Must have >=2 entries if set. + items: + type: string + type: array + skipTLSVerify: + description: |- + SkipTLSVerify disables TLS certificate verification when pulling + OOT driver images. For air-gapped or internal registries. + type: boolean + required: + - moduleName + type: object kueue: description: Define Kueue queues properties: @@ -289,6 +433,8 @@ spec: items: type: string type: array + kmmStatus: + type: string xpuManagerStatus: type: string type: object diff --git a/charts/gpu-base-operator/templates/dp_scc.yaml b/charts/gpu-base-operator/templates/dp_scc.yaml new file mode 100644 index 0000000..de6ae1a --- /dev/null +++ b/charts/gpu-base-operator/templates/dp_scc.yaml @@ -0,0 +1,38 @@ +{{- if .Values.openshift.enabled }} +apiVersion: security.openshift.io/v1 +kind: SecurityContextConstraints +metadata: + name: {{ .Release.Name }}-dp-scc + labels: + app.kubernetes.io/name: intel-gpu-base-operator + app.kubernetes.io/managed-by: {{ .Release.Service }} +allowPrivilegedContainer: true +allowHostDirVolumePlugin: true +allowHostIPC: false +allowHostNetwork: false +allowHostPID: false +allowHostPorts: false +allowPrivilegeEscalation: true +allowedCapabilities: null +defaultAddCapabilities: null +fsGroup: + type: RunAsAny +readOnlyRootFilesystem: false +requiredDropCapabilities: null +runAsUser: + type: RunAsAny +seLinuxContext: + type: RunAsAny +seccompProfiles: +- "*" +supplementalGroups: + type: RunAsAny +volumes: +- hostPath +- emptyDir +- projected +- secret +- configMap +users: [] +groups: [] +{{- end }} diff --git a/charts/gpu-base-operator/templates/dp_scc_role.yaml b/charts/gpu-base-operator/templates/dp_scc_role.yaml new file mode 100644 index 0000000..1776506 --- /dev/null +++ b/charts/gpu-base-operator/templates/dp_scc_role.yaml @@ -0,0 +1,14 @@ +{{- if .Values.openshift.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Release.Name }}-dp-scc-role + labels: + app.kubernetes.io/name: intel-gpu-base-operator + app.kubernetes.io/managed-by: {{ .Release.Service }} +rules: +- apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + resourceNames: ["{{ .Release.Name }}-dp-scc"] + verbs: ["use"] +{{- end }} diff --git a/charts/gpu-base-operator/templates/dp_scc_rolebinding.yaml b/charts/gpu-base-operator/templates/dp_scc_rolebinding.yaml new file mode 100644 index 0000000..6b5f39e --- /dev/null +++ b/charts/gpu-base-operator/templates/dp_scc_rolebinding.yaml @@ -0,0 +1,17 @@ +{{- if .Values.openshift.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Release.Name }}-dp-scc-binding + labels: + app.kubernetes.io/name: intel-gpu-base-operator + app.kubernetes.io/managed-by: {{ .Release.Service }} +subjects: +- kind: ServiceAccount + name: {{ .Values.dp.serviceAccountName }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ .Release.Name }}-dp-scc-role + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/charts/gpu-base-operator/templates/dp_serviceaccount.yaml b/charts/gpu-base-operator/templates/dp_serviceaccount.yaml new file mode 100644 index 0000000..fba0ef6 --- /dev/null +++ b/charts/gpu-base-operator/templates/dp_serviceaccount.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.dp.serviceAccountName }} + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: intel-gpu-base-operator + app.kubernetes.io/managed-by: {{ .Release.Service }} diff --git a/charts/gpu-base-operator/templates/dra_admission_policy.yaml b/charts/gpu-base-operator/templates/dra_admission_policy.yaml new file mode 100644 index 0000000..0b68093 --- /dev/null +++ b/charts/gpu-base-operator/templates/dra_admission_policy.yaml @@ -0,0 +1,34 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: {{ .Release.Name }}-dra-resourceslices + labels: + app.kubernetes.io/name: intel-gpu-base-operator + app.kubernetes.io/managed-by: {{ .Release.Service }} +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: ["resource.k8s.io"] + apiVersions: ["v1beta1"] + operations: ["CREATE", "UPDATE", "DELETE"] + resources: ["resourceslices"] + matchConditions: + - name: isRestrictedUser + expression: >- + request.userInfo.username == "system:serviceaccount:{{ .Release.Namespace }}:{{ .Values.dra.serviceAccountName }}" + variables: + - name: userNodeName + expression: >- + request.userInfo.extra[?'authentication.kubernetes.io/node-name'][0].orValue('') + - name: objectNodeName + expression: >- + (request.operation == "DELETE" ? oldObject : object).spec.?nodeName.orValue("") + validations: + - expression: variables.userNodeName != "" + message: >- + no node association found for user, this user must run in a pod on a node and ServiceAccountTokenPodNodeInfo must be enabled + - expression: variables.userNodeName == variables.objectNodeName + messageExpression: >- + "this user running on node '"+variables.userNodeName+"' may not modify " + + (variables.objectNodeName == "" ?"cluster resourceslices" : "resourceslices on node '"+variables.objectNodeName+"'") diff --git a/charts/gpu-base-operator/templates/dra_admission_policy_binding.yaml b/charts/gpu-base-operator/templates/dra_admission_policy_binding.yaml new file mode 100644 index 0000000..09e5243 --- /dev/null +++ b/charts/gpu-base-operator/templates/dra_admission_policy_binding.yaml @@ -0,0 +1,10 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: {{ .Release.Name }}-dra-resourceslices + labels: + app.kubernetes.io/name: intel-gpu-base-operator + app.kubernetes.io/managed-by: {{ .Release.Service }} +spec: + policyName: {{ .Release.Name }}-dra-resourceslices + validationActions: [Deny] diff --git a/charts/gpu-base-operator/templates/dra_clusterrole.yaml b/charts/gpu-base-operator/templates/dra_clusterrole.yaml new file mode 100644 index 0000000..d908ec5 --- /dev/null +++ b/charts/gpu-base-operator/templates/dra_clusterrole.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Release.Name }}-dra + labels: + app.kubernetes.io/name: intel-gpu-base-operator + app.kubernetes.io/managed-by: {{ .Release.Service }} +rules: +- apiGroups: [""] + resources: ["nodes"] + verbs: ["get"] +- apiGroups: ["resource.k8s.io"] + resources: ["resourceslices"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +- apiGroups: ["resource.k8s.io"] + resources: ["resourceclaims"] + verbs: ["get"] diff --git a/charts/gpu-base-operator/templates/dra_clusterrolebinding.yaml b/charts/gpu-base-operator/templates/dra_clusterrolebinding.yaml new file mode 100644 index 0000000..8941072 --- /dev/null +++ b/charts/gpu-base-operator/templates/dra_clusterrolebinding.yaml @@ -0,0 +1,15 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Release.Name }}-dra + labels: + app.kubernetes.io/name: intel-gpu-base-operator + app.kubernetes.io/managed-by: {{ .Release.Service }} +subjects: +- kind: ServiceAccount + name: {{ .Values.dra.serviceAccountName }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ .Release.Name }}-dra + apiGroup: rbac.authorization.k8s.io diff --git a/charts/gpu-base-operator/templates/dra_scc.yaml b/charts/gpu-base-operator/templates/dra_scc.yaml new file mode 100644 index 0000000..e842d01 --- /dev/null +++ b/charts/gpu-base-operator/templates/dra_scc.yaml @@ -0,0 +1,37 @@ +{{- if .Values.openshift.enabled }} +apiVersion: security.openshift.io/v1 +kind: SecurityContextConstraints +metadata: + name: {{ .Release.Name }}-dra-scc + labels: + app.kubernetes.io/name: intel-gpu-base-operator + app.kubernetes.io/managed-by: {{ .Release.Service }} +allowPrivilegedContainer: true +allowHostDirVolumePlugin: true +allowHostIPC: false +allowHostNetwork: true +allowHostPID: false +allowHostPorts: false +allowPrivilegeEscalation: true +allowedCapabilities: null +defaultAddCapabilities: null +fsGroup: + type: RunAsAny +readOnlyRootFilesystem: false +requiredDropCapabilities: null +runAsUser: + type: RunAsAny +seLinuxContext: + type: RunAsAny +seccompProfiles: +- "*" +supplementalGroups: + type: RunAsAny +volumes: +- hostPath +- projected +- secret +- configMap +users: [] +groups: [] +{{- end }} diff --git a/charts/gpu-base-operator/templates/dra_scc_role.yaml b/charts/gpu-base-operator/templates/dra_scc_role.yaml new file mode 100644 index 0000000..9fb338b --- /dev/null +++ b/charts/gpu-base-operator/templates/dra_scc_role.yaml @@ -0,0 +1,14 @@ +{{- if .Values.openshift.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Release.Name }}-dra-scc-role + labels: + app.kubernetes.io/name: intel-gpu-base-operator + app.kubernetes.io/managed-by: {{ .Release.Service }} +rules: +- apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + resourceNames: ["{{ .Release.Name }}-dra-scc"] + verbs: ["use"] +{{- end }} diff --git a/charts/gpu-base-operator/templates/dra_scc_rolebinding.yaml b/charts/gpu-base-operator/templates/dra_scc_rolebinding.yaml new file mode 100644 index 0000000..c84bedf --- /dev/null +++ b/charts/gpu-base-operator/templates/dra_scc_rolebinding.yaml @@ -0,0 +1,17 @@ +{{- if .Values.openshift.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Release.Name }}-dra-scc-binding + labels: + app.kubernetes.io/name: intel-gpu-base-operator + app.kubernetes.io/managed-by: {{ .Release.Service }} +subjects: +- kind: ServiceAccount + name: {{ .Values.dra.serviceAccountName }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ .Release.Name }}-dra-scc-role + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/charts/gpu-base-operator/templates/dra_serviceaccount.yaml b/charts/gpu-base-operator/templates/dra_serviceaccount.yaml new file mode 100644 index 0000000..f75b865 --- /dev/null +++ b/charts/gpu-base-operator/templates/dra_serviceaccount.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.dra.serviceAccountName }} + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: intel-gpu-base-operator + app.kubernetes.io/managed-by: {{ .Release.Service }} diff --git a/charts/gpu-base-operator/templates/manager.yaml b/charts/gpu-base-operator/templates/manager.yaml index bec40f0..4fda946 100644 --- a/charts/gpu-base-operator/templates/manager.yaml +++ b/charts/gpu-base-operator/templates/manager.yaml @@ -64,6 +64,10 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: DP_SERVICE_ACCOUNT_NAME + value: {{ .Values.dp.serviceAccountName }} + - name: DRA_SERVICE_ACCOUNT_NAME + value: {{ .Values.dra.serviceAccountName }} {{- if .Values.privateRegistry.token }} - name: OPERATOR_SECRET value: "{{ .Release.Name }}-operator-private-registry" diff --git a/charts/gpu-base-operator/templates/role.yaml b/charts/gpu-base-operator/templates/role.yaml index c556fbd..51b89f7 100644 --- a/charts/gpu-base-operator/templates/role.yaml +++ b/charts/gpu-base-operator/templates/role.yaml @@ -78,6 +78,24 @@ rules: - get - patch - update +- apiGroups: + - kmm.sigs.x-k8s.io + resources: + - modules + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kmm.sigs.x-k8s.io + resources: + - modules/status + verbs: + - get - apiGroups: - kueue.x-k8s.io resources: diff --git a/charts/gpu-base-operator/values.yaml b/charts/gpu-base-operator/values.yaml index 8f7a648..85d90cc 100644 --- a/charts/gpu-base-operator/values.yaml +++ b/charts/gpu-base-operator/values.yaml @@ -19,6 +19,15 @@ operator: cpu: 100m memory: 256Mi +dp: + serviceAccountName: intel-gpu-dp + +dra: + serviceAccountName: intel-gpu-dra + +openshift: + enabled: false + privateRegistry: url: "" user: "" diff --git a/cmd/main.go b/cmd/main.go index 59dea7c..48ba33a 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -45,6 +45,7 @@ import ( "k8s.io/client-go/discovery" "k8s.io/client-go/rest" + kmmv1beta1 "github.com/kubernetes-sigs/kernel-module-management/api/v1beta1" prometheusv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" corev1 "k8s.io/api/core/v1" resv1 "k8s.io/api/resource/v1" @@ -63,6 +64,7 @@ var ( setupLog = ctrl.Log.WithName("setup") openShiftGroups = []string{"route.openshift.io", "security.openshift.io"} draGroups = []string{"resource.k8s.io"} + kmmGroups = []string{"kmm.sigs.x-k8s.io"} ) const ( @@ -70,6 +72,7 @@ const ( openshiftCluster = "OpenShift" draCluster = "DRA" + kmmCluster = "KMM" ) func init() { @@ -80,6 +83,7 @@ func init() { utilruntime.Must(prometheusv1.AddToScheme(scheme)) utilruntime.Must(resv1.AddToScheme(scheme)) utilruntime.Must(kueuev1.AddToScheme(scheme)) + utilruntime.Must(kmmv1beta1.AddToScheme(scheme)) // +kubebuilder:scaffold:scheme } @@ -103,6 +107,7 @@ func detectClusterFeatures() (map[string]bool, error) { features := map[string]bool{ openshiftCluster: false, draCluster: false, + kmmCluster: false, } for _, group := range apiGroups.Groups { @@ -112,6 +117,9 @@ func detectClusterFeatures() (map[string]bool, error) { if slices.Contains(draGroups, group.Name) { features[draCluster] = true } + if slices.Contains(kmmGroups, group.Name) { + features[kmmCluster] = true + } } return features, nil @@ -323,12 +331,31 @@ func main() { setupLog.Info("Operator secret supplied", "secret", secret) } + dpSAName := os.Getenv("DP_SERVICE_ACCOUNT_NAME") + if dpSAName == "" { + dpSAName = "intel-gpu-dp" + } + + draSAName := os.Getenv("DRA_SERVICE_ACCOUNT_NAME") + if draSAName == "" { + draSAName = "intel-gpu-dra" + } + + moduleLoaderSAName := os.Getenv("MODULE_LOADER_SERVICE_ACCOUNT_NAME") + if moduleLoaderSAName == "" { + moduleLoaderSAName = "intel-gpu-module-loader" + } + copts := controller.ControllerOpts{ - Namespace: ns, - SecretName: secret, - RequeueDelay: time.Second * 5, - DRAEnable: features[draCluster], - OpenShift: features[openshiftCluster], + Namespace: ns, + SecretName: secret, + DPServiceAccountName: dpSAName, + DRAServiceAccountName: draSAName, + ModuleLoaderServiceAccountName: moduleLoaderSAName, + RequeueDelay: time.Second * 5, + DRAEnable: features[draCluster], + KMMEnable: features[kmmCluster], + OpenShift: features[openshiftCluster], } if err := (&controller.ClusterPolicyReconciler{ diff --git a/config/crd/bases/intel.com_clusterpolicies.yaml b/config/crd/bases/intel.com_clusterpolicies.yaml index 870f9c5..af78116 100644 --- a/config/crd/bases/intel.com_clusterpolicies.yaml +++ b/config/crd/bases/intel.com_clusterpolicies.yaml @@ -24,6 +24,9 @@ spec: - jsonPath: .status.xpuManagerStatus name: XPU type: string + - jsonPath: .status.kmmStatus + name: KMM + type: string - jsonPath: .metadata.creationTimestamp name: Age type: date @@ -136,6 +139,147 @@ spec: minimum: 1 type: integer type: object + kernelModule: + description: |- + KernelModule configures out-of-tree kernel module loading via KMM. + When set, KMM loads the specified OOT driver module on each node. + When nil, the in-tree kernel driver is used. + properties: + firmwarePath: + description: FirmwarePath is the in-container path where firmware + files are stored. + type: string + image: + description: |- + Image is the default container image for the OOT driver module. + When KernelMappings is empty, generates a single wildcard mapping. + When KernelMappings is non-empty, serves as the KMM-level fallback + for mappings that omit ContainerImage. + type: string + inTreeModulesToRemove: + description: |- + InTreeModulesToRemove lists in-tree modules to unload before + inserting the OOT module. ModuleName is always included implicitly. + items: + type: string + type: array + kernelMappings: + description: |- + KernelMappings maps kernel version patterns to container images or + build specifications. Translates directly to KMM KernelMapping + objects. When empty, a single wildcard mapping (regexp "^.+$") is + generated from Image. + items: + description: |- + KernelMappingSpec maps a kernel version pattern to a container image + or build specification. + properties: + build: + description: |- + Build configures in-cluster building of the driver image via KMM. + When set, KMM builds the image if it doesn't exist in the registry. + properties: + buildArgs: + description: BuildArgs are key-value pairs passed to + the image builder. + items: + description: BuildArg is a key-value pair passed as + a build argument. + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + dockerfileConfigMap: + description: DockerfileConfigMap references a ConfigMap + containing the Dockerfile. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + secrets: + description: |- + Secrets are made available during the build (e.g., for private + source repos). Not for registry auth -- use pullSecret on + ClusterPolicySpec. + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + type: array + required: + - dockerfileConfigMap + type: object + containerImage: + description: |- + ContainerImage is the full image reference for this kernel version. + Required when Build is nil and no parent-level Image is set. + type: string + inTreeModulesToRemove: + description: |- + InTreeModulesToRemove overrides the parent-level list for this + specific kernel mapping. + items: + type: string + type: array + literal: + description: |- + Literal is an exact kernel version string to match. + Exactly one of Regexp or Literal must be set. + type: string + regexp: + description: |- + Regexp is a regular expression matched against node kernel versions. + Exactly one of Regexp or Literal must be set. + type: string + type: object + type: array + moduleName: + description: |- + ModuleName is the kernel module to load (e.g., "xe"). + Also used as the default InTreeModulesToRemove entry. + type: string + modulesLoadingOrder: + description: |- + ModulesLoadingOrder specifies softdep-style loading order for + multi-module drivers. First element must be ModuleName; KMM loads + in order and unloads in reverse. Must have >=2 entries if set. + items: + type: string + type: array + skipTLSVerify: + description: |- + SkipTLSVerify disables TLS certificate verification when pulling + OOT driver images. For air-gapped or internal registries. + type: boolean + required: + - moduleName + type: object kueue: description: Define Kueue queues properties: @@ -289,6 +433,8 @@ spec: items: type: string type: array + kmmStatus: + type: string xpuManagerStatus: type: string type: object diff --git a/config/deployments/deployments.go b/config/deployments/deployments.go index 9f61eb5..156d2ec 100644 --- a/config/deployments/deployments.go +++ b/config/deployments/deployments.go @@ -16,13 +16,10 @@ package deployments import ( _ "embed" - "fmt" - adreg "k8s.io/api/admissionregistration/v1" apps "k8s.io/api/apps/v1" batch "k8s.io/api/batch/v1" core "k8s.io/api/core/v1" - rbac "k8s.io/api/rbac/v1" prometheusv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" resv1 "k8s.io/api/resource/v1" @@ -31,11 +28,6 @@ import ( "sigs.k8s.io/yaml" ) -const ( - vfioExpression = `device.attributes["gpu.intel.com"].driver == 'vfio-pci'` - xeVfioExpression = `device.attributes["gpu.intel.com"].driver == 'xe-vfio-pci'` -) - // XPU Manager //go:embed xpum/xpum.yaml @@ -52,84 +44,8 @@ func XpuManagerOTelConfig() *OTelConfig { return getOTelConfig(contentXpumOTelConfig) } -// Device Plugin - -//go:embed dp/dp.yaml -var contentDPDs []byte - -func DevicePluginDaemonset() *apps.DaemonSet { - return getDaemonset(contentDPDs).DeepCopy() -} - // DRA -//go:embed dra/daemonset.yaml -var contentDRADs []byte - -func DynamicResourceAllocationDaemonset() *apps.DaemonSet { - return getDaemonset(contentDRADs).DeepCopy() -} - -//go:embed dra/clusterrole.yaml -var contentDRACR []byte - -func DynamicResourceAllocationClusterRole() *rbac.ClusterRole { - return getClusterRole(contentDRACR).DeepCopy() -} - -//go:embed dra/clusterrolebinding.yaml -var contentDRACRB []byte - -func DynamicResourceAllocationClusterRoleBinding() *rbac.ClusterRoleBinding { - return getClusterRoleBinding(contentDRACRB).DeepCopy() -} - -//go:embed dra/serviceaccount.yaml -var contentDRASA []byte - -func DynamicResourceAllocationServiceAccount() *core.ServiceAccount { - return getServiceAccount(contentDRASA).DeepCopy() -} - -//go:embed dra/device-class.yaml -var contentDRADC []byte - -func DynamicResourceAllocationDeviceClass() *resv1.DeviceClass { - return getDeviceClass(contentDRADC).DeepCopy() -} - -//go:embed dra/device-class-vfio.yaml -var contentDRADCVfio []byte - -func DynamicResourceAllocationDeviceClassVfio(limitToVfio bool) *resv1.DeviceClass { - dc := getDeviceClass(contentDRADCVfio).DeepCopy() - - // Limit VFIO device class to only VFIO-bound devices. - if limitToVfio { - dc.Spec.Selectors = append(dc.Spec.Selectors, resv1.DeviceSelector{ - CEL: &resv1.CELDeviceSelector{ - Expression: fmt.Sprintf("%s || %s", vfioExpression, xeVfioExpression), - }, - }) - } - - return dc -} - -//go:embed dra/admissionpolicy.yaml -var contentDRAAP []byte - -func DynamicResourceAllocationValidatingAdmissionPolicy() *adreg.ValidatingAdmissionPolicy { - return getAdmissionPolicy(contentDRAAP).DeepCopy() -} - -//go:embed dra/admissionpolicybinding.yaml -var contentDRAAPB []byte - -func DynamicResourceAllocationValidatingAdmissionPolicyBinding() *adreg.ValidatingAdmissionPolicyBinding { - return getAdmissionPolicyBinding(contentDRAAPB).DeepCopy() -} - //go:embed dra/monitorclaimtemplate.yaml var contentDRAMCT []byte @@ -193,72 +109,6 @@ func getService(content []byte) *core.Service { return &result } -func getServiceAccount(content []byte) *core.ServiceAccount { - var result core.ServiceAccount - - err := yaml.Unmarshal(content, &result) - if err != nil { - panic(err) - } - - return &result -} - -func getClusterRole(content []byte) *rbac.ClusterRole { - var result rbac.ClusterRole - - err := yaml.Unmarshal(content, &result) - if err != nil { - panic(err) - } - - return &result -} - -func getClusterRoleBinding(content []byte) *rbac.ClusterRoleBinding { - var result rbac.ClusterRoleBinding - - err := yaml.Unmarshal(content, &result) - if err != nil { - panic(err) - } - - return &result -} - -func getAdmissionPolicy(content []byte) *adreg.ValidatingAdmissionPolicy { - var result adreg.ValidatingAdmissionPolicy - - err := yaml.Unmarshal(content, &result) - if err != nil { - panic(err) - } - - return &result -} - -func getAdmissionPolicyBinding(content []byte) *adreg.ValidatingAdmissionPolicyBinding { - var result adreg.ValidatingAdmissionPolicyBinding - - err := yaml.Unmarshal(content, &result) - if err != nil { - panic(err) - } - - return &result -} - -func getDeviceClass(content []byte) *resv1.DeviceClass { - var result resv1.DeviceClass - - err := yaml.Unmarshal(content, &result) - if err != nil { - panic(err) - } - - return &result -} - func getResourceClaimTemplate(content []byte) *resv1.ResourceClaimTemplate { var result resv1.ResourceClaimTemplate diff --git a/config/deployments/deployments_test.go b/config/deployments/deployments_test.go index 17784cc..5825d26 100644 --- a/config/deployments/deployments_test.go +++ b/config/deployments/deployments_test.go @@ -51,69 +51,6 @@ func TestXpuManagerDaemonset(t *testing.T) { } } -func TestDevicePluginDaemonset(t *testing.T) { - ds := DevicePluginDaemonset() - if ds == nil { - t.Error("DevicePluginDaemonset returned nil") - } -} - -func TestDynamicResourceAllocationDaemonset(t *testing.T) { - ds := DynamicResourceAllocationDaemonset() - if ds == nil { - t.Error("DynamicResourceAllocationDaemonset returned nil") - } -} - -func TestDynamicResourceAllocationClusterRole(t *testing.T) { - cr := DynamicResourceAllocationClusterRole() - if cr == nil { - t.Error("DynamicResourceAllocationClusterRole returned nil") - } -} - -func TestDynamicResourceAllocationClusterRoleBinding(t *testing.T) { - crb := DynamicResourceAllocationClusterRoleBinding() - if crb == nil { - t.Error("DynamicResourceAllocationClusterRoleBinding returned nil") - } -} - -func TestDynamicResourceAllocationServiceAccount(t *testing.T) { - sa := DynamicResourceAllocationServiceAccount() - if sa == nil { - t.Error("DynamicResourceAllocationServiceAccount returned nil") - } -} - -func TestDynamicResourceAllocationDeviceClass(t *testing.T) { - dc := DynamicResourceAllocationDeviceClass() - if dc == nil { - t.Error("DynamicResourceAllocationDeviceClass returned nil") - } -} - -func TestDynamicResourceAllocationDeviceClassVfio(t *testing.T) { - dc := DynamicResourceAllocationDeviceClassVfio(false) - if dc == nil { - t.Error("DynamicResourceAllocationDeviceClassVfio returned nil") - } -} - -func TestDynamicResourceAllocationValidatingAdmissionPolicy(t *testing.T) { - ap := DynamicResourceAllocationValidatingAdmissionPolicy() - if ap == nil { - t.Error("DynamicResourceAllocationValidatingAdmissionPolicy returned nil") - } -} - -func TestDynamicResourceAllocationValidatingAdmissionPolicyBinding(t *testing.T) { - apb := DynamicResourceAllocationValidatingAdmissionPolicyBinding() - if apb == nil { - t.Error("DynamicResourceAllocationValidatingAdmissionPolicyBinding returned nil") - } -} - func TestDynamicResourceAllocationMonitorClaimTemplate(t *testing.T) { mct := DynamicResourceAllocationMonitorClaimTemplate() if mct == nil { @@ -156,33 +93,6 @@ func TestOTelConfig(t *testing.T) { } } -func TestDevicePluginDaemonset_AutomountServiceAccountToken(t *testing.T) { - ds := DevicePluginDaemonset() - if ds.Spec.Template.Spec.AutomountServiceAccountToken == nil { - t.Fatal("automountServiceAccountToken must be set (non-nil)") - } - if *ds.Spec.Template.Spec.AutomountServiceAccountToken != false { - t.Error("automountServiceAccountToken must be false") - } -} - -func TestDynamicResourceAllocationDaemonset_KubeletPluginAllowPrivilegeEscalation(t *testing.T) { - ds := DynamicResourceAllocationDaemonset() - c := findContainer(ds.Spec.Template.Spec.Containers, "kubelet-plugin") - if c == nil { - t.Fatal("kubelet-plugin container not found") - } - if c.SecurityContext == nil { - t.Fatal("SecurityContext must be set on kubelet-plugin") - } - if c.SecurityContext.AllowPrivilegeEscalation == nil { - t.Fatal("AllowPrivilegeEscalation must be set (non-nil) on kubelet-plugin") - } - if *c.SecurityContext.AllowPrivilegeEscalation != false { - t.Error("AllowPrivilegeEscalation must be false on kubelet-plugin") - } -} - func TestXpuManagerDaemonset_AutomountServiceAccountToken(t *testing.T) { ds := XpuManagerDaemonset() if ds.Spec.Template.Spec.AutomountServiceAccountToken == nil { @@ -275,85 +185,6 @@ func TestXpuFwUpdateJob_UpdaterContainerSecurityContext(t *testing.T) { } } -func TestDevicePluginDaemonset_PluginContainerSecurityContext(t *testing.T) { - ds := DevicePluginDaemonset() - c := findContainer(ds.Spec.Template.Spec.Containers, "intel-gpu-plugin") - if c == nil { - t.Fatal("intel-gpu-plugin container not found") - } - if c.SecurityContext == nil { - t.Fatal("SecurityContext must be set on intel-gpu-plugin") - } - if c.SecurityContext.AllowPrivilegeEscalation == nil { - t.Fatal("AllowPrivilegeEscalation must be set (non-nil) on intel-gpu-plugin") - } - if *c.SecurityContext.AllowPrivilegeEscalation { - t.Error("AllowPrivilegeEscalation must be false on intel-gpu-plugin") - } - if c.SecurityContext.ReadOnlyRootFilesystem == nil { - t.Fatal("ReadOnlyRootFilesystem must be set (non-nil) on intel-gpu-plugin") - } - if !*c.SecurityContext.ReadOnlyRootFilesystem { - t.Error("ReadOnlyRootFilesystem must be true on intel-gpu-plugin") - } - if c.SecurityContext.Capabilities == nil { - t.Fatal("Capabilities must be set on intel-gpu-plugin") - } - found := false - for _, cap := range c.SecurityContext.Capabilities.Drop { - if cap == allCaps { - found = true - break - } - } - if !found { - t.Error("capabilities.drop must contain ALL on intel-gpu-plugin") - } - if c.SecurityContext.SeccompProfile == nil { - t.Fatal("SeccompProfile must be set on intel-gpu-plugin") - } - if c.SecurityContext.SeccompProfile.Type != core.SeccompProfileTypeRuntimeDefault { - t.Errorf("SeccompProfile.Type: got %v, want RuntimeDefault on intel-gpu-plugin", - c.SecurityContext.SeccompProfile.Type) - } -} - -func TestDynamicResourceAllocationDaemonset_KubeletPluginFullSecurityContext(t *testing.T) { - ds := DynamicResourceAllocationDaemonset() - c := findContainer(ds.Spec.Template.Spec.Containers, "kubelet-plugin") - if c == nil { - t.Fatal("kubelet-plugin container not found") - } - if c.SecurityContext == nil { - t.Fatal("SecurityContext must be set on kubelet-plugin") - } - if c.SecurityContext.ReadOnlyRootFilesystem == nil { - t.Fatal("ReadOnlyRootFilesystem must be set (non-nil) on kubelet-plugin") - } - if !*c.SecurityContext.ReadOnlyRootFilesystem { - t.Error("ReadOnlyRootFilesystem must be true on kubelet-plugin") - } - if c.SecurityContext.Capabilities == nil { - t.Fatal("Capabilities must be set on kubelet-plugin") - } - found := false - for _, cap := range c.SecurityContext.Capabilities.Drop { - if cap == allCaps { - found = true - break - } - } - if !found { - t.Error("capabilities.drop must contain ALL on kubelet-plugin") - } - if c.SecurityContext.SeccompProfile == nil { - t.Fatal("SeccompProfile must be set on kubelet-plugin") - } - if c.SecurityContext.SeccompProfile.Type != core.SeccompProfileTypeRuntimeDefault { - t.Errorf("SeccompProfile.Type: got %v, want RuntimeDefault on kubelet-plugin", c.SecurityContext.SeccompProfile.Type) - } -} - func TestXpuManagerDaemonset_XpumdContainerSecurityContext(t *testing.T) { ds := XpuManagerDaemonset() c := findContainer(ds.Spec.Template.Spec.Containers, "xpumd") @@ -399,31 +230,3 @@ func TestXpuManagerDaemonset_XpumdContainerSecurityContext(t *testing.T) { t.Error("capabilities.add must contain SYS_ADMIN on xpumd (required for engine utilization metrics)") } } - -func TestDRAClusterRole_NoWildcardsAndNoSecrets(t *testing.T) { - cr := DynamicResourceAllocationClusterRole() - - for _, rule := range cr.Rules { - for _, apiGroup := range rule.APIGroups { - if apiGroup == "*" { - t.Errorf("DRA ClusterRole rule has wildcard apiGroup: %+v", rule) - } - } - - for _, resource := range rule.Resources { - if resource == "*" { - t.Errorf("DRA ClusterRole rule has wildcard resource: %+v", rule) - } - - if resource == "secrets" { - t.Errorf("DRA ClusterRole must not grant access to secrets: %+v", rule) - } - } - - for _, verb := range rule.Verbs { - if verb == "*" { - t.Errorf("DRA ClusterRole rule has wildcard verb: %+v", rule) - } - } - } -} diff --git a/config/deployments/dp/dp.yaml b/config/deployments/dp/dp.yaml deleted file mode 100644 index f41c46e..0000000 --- a/config/deployments/dp/dp.yaml +++ /dev/null @@ -1,80 +0,0 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - labels: - app: intel-gpu-plugin - name: intel-gpu-plugin -spec: - selector: - matchLabels: - app: intel-gpu-plugin - template: - metadata: - labels: - app: intel-gpu-plugin - spec: - automountServiceAccountToken: false - containers: - - args: - - -health-management - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: HOST_IP - valueFrom: - fieldRef: - fieldPath: status.hostIP - image: intel/intel-gpu-plugin:devel - imagePullPolicy: IfNotPresent - name: intel-gpu-plugin - resources: - limits: - cpu: 100m - memory: 90Mi - requests: - cpu: 40m - memory: 45Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - seLinuxOptions: - type: container_device_plugin_t - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /dev/dri - name: devfs - readOnly: true - - mountPath: /sys/class/drm - name: sysfsdrm - readOnly: true - - mountPath: /var/lib/kubelet/device-plugins - name: kubeletsockets - - mountPath: /var/run/cdi - name: cdipath - nodeSelector: - kubernetes.io/arch: amd64 - volumes: - - hostPath: - path: /dev/dri - name: devfs - - hostPath: - path: /sys/class/drm - name: sysfsdrm - - hostPath: - path: /var/lib/kubelet/device-plugins - name: kubeletsockets - - hostPath: - path: /var/run/cdi - type: DirectoryOrCreate - name: cdipath - updateStrategy: - rollingUpdate: - maxSurge: 0 - maxUnavailable: 1 - type: RollingUpdate diff --git a/config/deployments/dra/admissionpolicybinding.yaml b/config/deployments/dra/admissionpolicybinding.yaml deleted file mode 100644 index d7fbece..0000000 --- a/config/deployments/dra/admissionpolicybinding.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingAdmissionPolicyBinding -metadata: - name: resourceslices-policy-dra-kubelet-plugin-gpu -spec: - policyName: resourceslices-policy-dra-kubelet-plugin-gpu - validationActions: [Deny] diff --git a/config/deployments/dra/clusterrolebinding.yaml b/config/deployments/dra/clusterrolebinding.yaml deleted file mode 100644 index 145b8cc..0000000 --- a/config/deployments/dra/clusterrolebinding.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: intel-gpu-resource-driver-role-binding - namespace: intel-gpu-resource-driver -subjects: -- kind: ServiceAccount - name: intel-gpu-resource-driver-service-account - namespace: intel-gpu-resource-driver -roleRef: - kind: ClusterRole - name: intel-gpu-resource-driver-role - apiGroup: rbac.authorization.k8s.io diff --git a/config/deployments/dra/daemonset.yaml b/config/deployments/dra/daemonset.yaml deleted file mode 100644 index 89dce71..0000000 --- a/config/deployments/dra/daemonset.yaml +++ /dev/null @@ -1,90 +0,0 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: intel-gpu-resource-driver-kubelet-plugin - namespace: intel-gpu-resource-driver - labels: - app: intel-gpu-resource-driver-kubelet-plugin -spec: - selector: - matchLabels: - app: intel-gpu-resource-driver-kubelet-plugin - template: - metadata: - labels: - app: intel-gpu-resource-driver-kubelet-plugin - spec: - serviceAccount: intel-gpu-resource-driver-service-account - serviceAccountName: intel-gpu-resource-driver-service-account - containers: - - name: kubelet-plugin - image: ghcr.io/intel/intel-resource-drivers-for-kubernetes/intel-gpu-resource-driver:v0.9.0 - imagePullPolicy: IfNotPresent - command: ["/kubelet-gpu-plugin"] - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: SYSFS_ROOT - value: "/sysfs" - # Only use devdri when using fake devfs with device-faker. - # Use this to tell kubelet-plugin where the fake DRI devices nodes are. - # This will be prefix for CDI devices, runtime will try to mount devices - # with this prefix into workloads, so the path should be equal to where - # fake DRI files are in real host. - #- name: DEV_DRI_PATH - # value: "/tmp/test-0123456789/dev/dri" - volumeMounts: - - name: plugins-registry - mountPath: /var/lib/kubelet/plugins_registry - - name: plugins - mountPath: /var/lib/kubelet/plugins - - name: cdi - mountPath: /etc/cdi - - name: varruncdi - mountPath: /var/run/cdi - - name: xpumdrundir - mountPath: /run/xpumd - # when using fake sysfs - mount at the same place as on host - - name: sysfs - mountPath: "/sysfs" - # Only use devdri when using fake devfs with device-faker - #- name: devdri - # mountPath: "/tmp/test-0123456789/dev/dri" - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: [ "ALL" ] - readOnlyRootFilesystem: true - runAsUser: 0 - seccompProfile: - type: RuntimeDefault - volumes: - - name: plugins-registry - hostPath: - path: /var/lib/kubelet/plugins_registry - - name: plugins - hostPath: - path: /var/lib/kubelet/plugins - - name: cdi - hostPath: - path: /etc/cdi - - name: varruncdi - hostPath: - path: /var/run/cdi - - name: sysfs - hostPath: - path: /sys - - name: xpumdrundir - hostPath: - path: /run/xpumd - type: DirectoryOrCreate - # Only use devdri when using fake devfs with device-faker - #- name: devdri - # hostPath: - # path: /tmp/test-0123456789/dev/dri diff --git a/config/deployments/dra/device-class-vfio.yaml b/config/deployments/dra/device-class-vfio.yaml deleted file mode 100644 index 0922522..0000000 --- a/config/deployments/dra/device-class-vfio.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: resource.k8s.io/v1 -kind: DeviceClass -metadata: - name: gpu-vfio.intel.com -spec: - selectors: - - cel: - expression: device.driver == "gpu.intel.com" - # Uncomment if active binding management is disabled in kubelet-plugin, to prevent non-VFIO bound GPUs - # from being allocated to VM-based workloads. - #- cel: - # expression: device.attributes["gpu.intel.com"].driver == 'vfio-pci' || device.attributes["gpu.intel.com"].driver == 'xe-vfio-pci' diff --git a/config/deployments/dra/device-class.yaml b/config/deployments/dra/device-class.yaml deleted file mode 100644 index 64f62c0..0000000 --- a/config/deployments/dra/device-class.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: resource.k8s.io/v1 -kind: DeviceClass -metadata: - name: gpu.intel.com - -spec: - selectors: - - cel: - expression: device.driver == "gpu.intel.com" - # Available in K8s v1.34 requires feature gate enabled - # See https://github.com/kubernetes/enhancements/tree/master/keps/sig-scheduling/5004-dra-extended-resource - extendedResourceName: intel.com/gpu diff --git a/config/deployments/dra/serviceaccount.yaml b/config/deployments/dra/serviceaccount.yaml deleted file mode 100644 index 008d32f..0000000 --- a/config/deployments/dra/serviceaccount.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: intel-gpu-resource-driver-service-account - namespace: intel-gpu-resource-driver diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 201c738..857e886 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -69,6 +69,10 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: DP_SERVICE_ACCOUNT_NAME + value: intel-gpu-base-operator-intel-gpu-dp + - name: DRA_SERVICE_ACCOUNT_NAME + value: intel-gpu-base-operator-intel-gpu-dra image: ghcr.io/intel/intel-gpu-base-operator:devel imagePullPolicy: IfNotPresent name: manager diff --git a/config/rbac/dp_scc.yaml b/config/rbac/dp_scc.yaml new file mode 100644 index 0000000..ddccd03 --- /dev/null +++ b/config/rbac/dp_scc.yaml @@ -0,0 +1,35 @@ +apiVersion: security.openshift.io/v1 +kind: SecurityContextConstraints +metadata: + name: intel-gpu-dp-scc + labels: + app.kubernetes.io/name: intel-gpu-base-operator +allowPrivilegedContainer: true +allowHostDirVolumePlugin: true +allowHostIPC: false +allowHostNetwork: false +allowHostPID: false +allowHostPorts: false +allowPrivilegeEscalation: true +allowedCapabilities: null +defaultAddCapabilities: null +fsGroup: + type: RunAsAny +readOnlyRootFilesystem: false +requiredDropCapabilities: null +runAsUser: + type: RunAsAny +seLinuxContext: + type: RunAsAny +seccompProfiles: +- "*" +supplementalGroups: + type: RunAsAny +volumes: +- hostPath +- emptyDir +- projected +- secret +- configMap +users: [] +groups: [] diff --git a/config/rbac/dp_scc_role.yaml b/config/rbac/dp_scc_role.yaml new file mode 100644 index 0000000..8bc4aeb --- /dev/null +++ b/config/rbac/dp_scc_role.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: intel-gpu-dp-scc-role + labels: + app.kubernetes.io/name: intel-gpu-base-operator +rules: +- apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + # Hardcoded with kustomize namePrefix — kustomize does not transform resourceNames values. + resourceNames: ["intel-gpu-base-operator-intel-gpu-dp-scc"] + verbs: ["use"] diff --git a/config/rbac/dp_scc_rolebinding.yaml b/config/rbac/dp_scc_rolebinding.yaml new file mode 100644 index 0000000..79019f9 --- /dev/null +++ b/config/rbac/dp_scc_rolebinding.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: intel-gpu-dp-scc-binding + labels: + app.kubernetes.io/name: intel-gpu-base-operator +subjects: +- kind: ServiceAccount + name: intel-gpu-dp + namespace: system +roleRef: + kind: ClusterRole + name: intel-gpu-dp-scc-role + apiGroup: rbac.authorization.k8s.io diff --git a/config/rbac/dp_serviceaccount.yaml b/config/rbac/dp_serviceaccount.yaml new file mode 100644 index 0000000..48146aa --- /dev/null +++ b/config/rbac/dp_serviceaccount.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: intel-gpu-dp + namespace: system + labels: + app.kubernetes.io/name: intel-gpu-base-operator diff --git a/config/deployments/dra/admissionpolicy.yaml b/config/rbac/dra_admission_policy.yaml similarity index 88% rename from config/deployments/dra/admissionpolicy.yaml rename to config/rbac/dra_admission_policy.yaml index 65c5553..dd858b8 100644 --- a/config/deployments/dra/admissionpolicy.yaml +++ b/config/rbac/dra_admission_policy.yaml @@ -1,7 +1,9 @@ apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingAdmissionPolicy metadata: - name: resourceslices-policy-dra-kubelet-plugin-gpu + name: intel-gpu-dra-resourceslices + labels: + app.kubernetes.io/name: intel-gpu-base-operator spec: failurePolicy: Fail matchConstraints: @@ -13,7 +15,7 @@ spec: matchConditions: - name: isRestrictedUser expression: >- - request.userInfo.username == "system:serviceaccount:intel-gpu-base-operator:gpu-policy-sample-gpu-dra" + request.userInfo.username == "system:serviceaccount:intel-gpu-base-operator:intel-gpu-base-operator-intel-gpu-dra" variables: - name: userNodeName expression: >- diff --git a/config/rbac/dra_admission_policy_binding.yaml b/config/rbac/dra_admission_policy_binding.yaml new file mode 100644 index 0000000..fa1da0d --- /dev/null +++ b/config/rbac/dra_admission_policy_binding.yaml @@ -0,0 +1,9 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: intel-gpu-dra-resourceslices + labels: + app.kubernetes.io/name: intel-gpu-base-operator +spec: + policyName: intel-gpu-dra-resourceslices + validationActions: [Deny] diff --git a/config/deployments/dra/clusterrole.yaml b/config/rbac/dra_clusterrole.yaml similarity index 80% rename from config/deployments/dra/clusterrole.yaml rename to config/rbac/dra_clusterrole.yaml index 21d6e2b..9714dab 100644 --- a/config/deployments/dra/clusterrole.yaml +++ b/config/rbac/dra_clusterrole.yaml @@ -1,7 +1,9 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: intel-gpu-resource-driver-role + name: intel-gpu-dra + labels: + app.kubernetes.io/name: intel-gpu-base-operator rules: - apiGroups: [""] resources: ["nodes"] diff --git a/config/rbac/dra_clusterrolebinding.yaml b/config/rbac/dra_clusterrolebinding.yaml new file mode 100644 index 0000000..79b8f97 --- /dev/null +++ b/config/rbac/dra_clusterrolebinding.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: intel-gpu-dra + labels: + app.kubernetes.io/name: intel-gpu-base-operator +subjects: +- kind: ServiceAccount + name: intel-gpu-dra + namespace: system +roleRef: + kind: ClusterRole + name: intel-gpu-dra + apiGroup: rbac.authorization.k8s.io diff --git a/config/rbac/dra_scc.yaml b/config/rbac/dra_scc.yaml new file mode 100644 index 0000000..6a6bec6 --- /dev/null +++ b/config/rbac/dra_scc.yaml @@ -0,0 +1,34 @@ +apiVersion: security.openshift.io/v1 +kind: SecurityContextConstraints +metadata: + name: intel-gpu-dra-scc + labels: + app.kubernetes.io/name: intel-gpu-base-operator +allowPrivilegedContainer: true +allowHostDirVolumePlugin: true +allowHostIPC: false +allowHostNetwork: true +allowHostPID: false +allowHostPorts: false +allowPrivilegeEscalation: true +allowedCapabilities: null +defaultAddCapabilities: null +fsGroup: + type: RunAsAny +readOnlyRootFilesystem: false +requiredDropCapabilities: null +runAsUser: + type: RunAsAny +seLinuxContext: + type: RunAsAny +seccompProfiles: +- "*" +supplementalGroups: + type: RunAsAny +volumes: +- hostPath +- projected +- secret +- configMap +users: [] +groups: [] diff --git a/config/rbac/dra_scc_role.yaml b/config/rbac/dra_scc_role.yaml new file mode 100644 index 0000000..a9b6a61 --- /dev/null +++ b/config/rbac/dra_scc_role.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: intel-gpu-dra-scc-role + labels: + app.kubernetes.io/name: intel-gpu-base-operator +rules: +- apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + # Hardcoded with kustomize namePrefix — kustomize does not transform resourceNames values. + resourceNames: ["intel-gpu-base-operator-intel-gpu-dra-scc"] + verbs: ["use"] diff --git a/config/rbac/dra_scc_rolebinding.yaml b/config/rbac/dra_scc_rolebinding.yaml new file mode 100644 index 0000000..c641a94 --- /dev/null +++ b/config/rbac/dra_scc_rolebinding.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: intel-gpu-dra-scc-binding + labels: + app.kubernetes.io/name: intel-gpu-base-operator +subjects: +- kind: ServiceAccount + name: intel-gpu-dra + namespace: system +roleRef: + kind: ClusterRole + name: intel-gpu-dra-scc-role + apiGroup: rbac.authorization.k8s.io diff --git a/config/rbac/dra_serviceaccount.yaml b/config/rbac/dra_serviceaccount.yaml new file mode 100644 index 0000000..6870872 --- /dev/null +++ b/config/rbac/dra_serviceaccount.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: intel-gpu-dra + namespace: system + labels: + app.kubernetes.io/name: intel-gpu-base-operator diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index 955856d..a87f00c 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -30,4 +30,25 @@ resources: - clusterpolicy_admin_role.yaml - clusterpolicy_editor_role.yaml - clusterpolicy_viewer_role.yaml +# DP RBAC — ServiceAccount and OpenShift SCC for KMM-managed DP pods (SCC is no-op on non-OpenShift) +- dp_serviceaccount.yaml +- dp_scc.yaml +- dp_scc_role.yaml +- dp_scc_rolebinding.yaml +# Module loader RBAC — ServiceAccount and OpenShift SCC for KMM module loader worker pods +- module_loader_serviceaccount.yaml +- module_loader_scc.yaml +- module_loader_scc_role.yaml +- module_loader_scc_rolebinding.yaml +# DRA RBAC — ServiceAccount, ClusterRole, ClusterRoleBinding for KMM-managed DRA pods +- dra_serviceaccount.yaml +- dra_clusterrole.yaml +- dra_clusterrolebinding.yaml +# DRA SCC — OpenShift SecurityContextConstraints (no-op on non-OpenShift) +- dra_scc.yaml +- dra_scc_role.yaml +- dra_scc_rolebinding.yaml +# DRA admission policy — restricts ResourceSlice modifications to the DRA SA's node +- dra_admission_policy.yaml +- dra_admission_policy_binding.yaml diff --git a/config/rbac/module_loader_scc.yaml b/config/rbac/module_loader_scc.yaml new file mode 100644 index 0000000..933358e --- /dev/null +++ b/config/rbac/module_loader_scc.yaml @@ -0,0 +1,37 @@ +apiVersion: security.openshift.io/v1 +kind: SecurityContextConstraints +metadata: + name: intel-gpu-module-loader-scc + labels: + app.kubernetes.io/name: intel-gpu-base-operator +allowPrivilegedContainer: true +allowHostDirVolumePlugin: true +allowHostIPC: false +allowHostNetwork: false +allowHostPID: false +allowHostPorts: false +allowPrivilegeEscalation: true +allowedCapabilities: +- SYS_MODULE +defaultAddCapabilities: null +fsGroup: + type: RunAsAny +readOnlyRootFilesystem: false +requiredDropCapabilities: null +runAsUser: + type: RunAsAny +seLinuxContext: + type: RunAsAny +seccompProfiles: +- "*" +supplementalGroups: + type: RunAsAny +volumes: +- hostPath +- emptyDir +- projected +- secret +- configMap +- downwardAPI +users: [] +groups: [] diff --git a/config/rbac/module_loader_scc_role.yaml b/config/rbac/module_loader_scc_role.yaml new file mode 100644 index 0000000..638066c --- /dev/null +++ b/config/rbac/module_loader_scc_role.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: intel-gpu-module-loader-scc-role + labels: + app.kubernetes.io/name: intel-gpu-base-operator +rules: +- apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + # Hardcoded with kustomize namePrefix — kustomize does not transform resourceNames values. + resourceNames: ["intel-gpu-base-operator-intel-gpu-module-loader-scc"] + verbs: ["use"] diff --git a/config/rbac/module_loader_scc_rolebinding.yaml b/config/rbac/module_loader_scc_rolebinding.yaml new file mode 100644 index 0000000..e4ed2bc --- /dev/null +++ b/config/rbac/module_loader_scc_rolebinding.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: intel-gpu-module-loader-scc-binding + labels: + app.kubernetes.io/name: intel-gpu-base-operator +subjects: +- kind: ServiceAccount + name: intel-gpu-module-loader + namespace: system +roleRef: + kind: ClusterRole + name: intel-gpu-module-loader-scc-role + apiGroup: rbac.authorization.k8s.io diff --git a/config/rbac/module_loader_serviceaccount.yaml b/config/rbac/module_loader_serviceaccount.yaml new file mode 100644 index 0000000..654cd27 --- /dev/null +++ b/config/rbac/module_loader_serviceaccount.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: intel-gpu-module-loader + namespace: system + labels: + app.kubernetes.io/name: intel-gpu-base-operator diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index e691942..b68ee6b 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -78,6 +78,24 @@ rules: - get - patch - update +- apiGroups: + - kmm.sigs.x-k8s.io + resources: + - modules + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kmm.sigs.x-k8s.io + resources: + - modules/status + verbs: + - get - apiGroups: - kueue.x-k8s.io resources: diff --git a/go.mod b/go.mod index f692c28..1883686 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/distribution/reference v0.6.0 github.com/google/go-cmp v0.7.0 github.com/google/go-containerregistry v0.21.7 + github.com/kubernetes-sigs/kernel-module-management v0.0.0-20260707212627-01f137c54ff8 github.com/onsi/ginkgo/v2 v2.32.0 github.com/onsi/gomega v1.42.1 github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.92.1 @@ -98,13 +99,13 @@ require ( golang.org/x/sync v0.21.0 // indirect golang.org/x/sys v0.46.0 // indirect golang.org/x/term v0.44.0 // indirect - golang.org/x/text v0.39.0 // indirect + golang.org/x/text v0.38.0 // indirect golang.org/x/time v0.15.0 // indirect - golang.org/x/tools v0.47.0 // indirect + golang.org/x/tools v0.46.0 // indirect gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect - google.golang.org/grpc v1.82.1 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 // indirect + google.golang.org/grpc v1.80.0 // indirect google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/go.sum b/go.sum index 5332c20..002d09b 100644 --- a/go.sum +++ b/go.sum @@ -117,6 +117,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kubernetes-sigs/kernel-module-management v0.0.0-20260707212627-01f137c54ff8 h1:jTT17VOVcJPwXsSTH9f2gw6VSwnvKgXsBqWewaQYZjg= +github.com/kubernetes-sigs/kernel-module-management v0.0.0-20260707212627-01f137c54ff8/go.mod h1:tqr/yRJMo+6VzLB9B/25H+wWvB+VMr95CYHD4ObyByA= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= @@ -231,22 +233,22 @@ golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= -golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus= -golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM= +golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= +golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= -golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= -golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= +golang.org/x/tools v0.46.0 h1:7jTurBkPZu4moS/Uy4OQT1M+QBlsj3wejyZwsT8Z7rk= +golang.org/x/tools v0.46.0/go.mod h1:FrD85F8l+NWL+9XWBSyVSHO6Ne4jutsfIFba7AWQ5Ys= gomodules.xyz/jsonpatch/v2 v2.5.0 h1:JELs8RLM12qJGXU4u/TO3V25KW8GreMKl9pdkk14RM0= gomodules.xyz/jsonpatch/v2 v2.5.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 h1:yQugLulqltosq0B/f8l4w9VryjV+N/5gcW0jQ3N8Qec= -google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478/go.mod h1:C6ADNqOxbgdUUeRTU+LCHDPB9ttAMCTff6auwCVa4uc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 h1:RmoJA1ujG+/lRGNfUnOMfhCy5EipVMyvUE+KNbPbTlw= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= -google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= +google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 h1:merA0rdPeUV3YIIfHHcH4qBkiQAc1nfCKSI7lB4cV2M= +google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409/go.mod h1:fl8J1IvUjCilwZzQowmw2b7HQB2eAuYBabMXzWurF+I= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 h1:ndE4FoJqsIceKP2oYSnUZqhTdYufCYYkqwtFzfrhI7w= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= +google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/controller/clusterpolicy_controller.go b/internal/controller/clusterpolicy_controller.go index 8ecd2ee..a73041a 100644 --- a/internal/controller/clusterpolicy_controller.go +++ b/internal/controller/clusterpolicy_controller.go @@ -24,22 +24,16 @@ import ( "time" apps "k8s.io/api/apps/v1" - core "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" "k8s.io/klog/v2" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sigs.k8s.io/controller-runtime/pkg/event" - "sigs.k8s.io/controller-runtime/pkg/handler" logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/predicate" - "sigs.k8s.io/controller-runtime/pkg/reconcile" v1alpha "github.com/intel/gpu-base-operator/api/v1alpha1" + kmmv1beta1 "github.com/kubernetes-sigs/kernel-module-management/api/v1beta1" ) // ClusterPolicyReconciler reconciles a ClusterPolicy object @@ -50,12 +44,16 @@ type ClusterPolicyReconciler struct { } type ControllerOpts struct { - ReqName string - Namespace string - SecretName string - RequeueDelay time.Duration - DRAEnable bool - OpenShift bool + ReqName string + Namespace string + SecretName string + DPServiceAccountName string + DRAServiceAccountName string + ModuleLoaderServiceAccountName string + RequeueDelay time.Duration + DRAEnable bool + KMMEnable bool + OpenShift bool } type requeueReconcileErr struct { @@ -67,10 +65,9 @@ type SubControllerInterface interface { } const ( - appLabel = "app" ownerKey = "owner" - draNotEnabledMsg = "DRA is not enabled in the cluster, but ClusterPolicy requests it." + xpumdVolumeName = "runxpumd" resourceModeDRA = "dra" resourceModeDP = "dp" @@ -84,6 +81,22 @@ const ( maxKeptErrors = 10 ) +func gpuNodeSelector(cp *v1alpha.ClusterPolicy) map[string]string { + selector := map[string]string{ + "kubernetes.io/arch": "amd64", + } + + for k, v := range cp.Spec.NodeSelector { + selector[k] = v + } + + if cp.Spec.UseNFDLabeling { + selector["intel.feature.node.kubernetes.io/gpu"] = trueValue + } + + return selector +} + func addIfMissing(slice *[]string, s string) { if slices.Contains(*slice, s) { return @@ -161,13 +174,10 @@ func (r *ClusterPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Reques opts := r.Opts opts.ReqName = req.Name - subControllers := make([]SubControllerInterface, 0, 4) + subControllers := make([]SubControllerInterface, 0, 3) - // Initialize sub-controllers - subControllers = append(subControllers, &DevicePluginReconciler{Client: r.Client, Scheme: r.Scheme, Opts: opts}) + subControllers = append(subControllers, &KMMReconciler{Client: r.Client, Scheme: r.Scheme, Opts: opts}) subControllers = append(subControllers, &XpuManagerReconciler{Client: r.Client, Scheme: r.Scheme, Opts: opts}) - // Include DRA subcontroller even though cluster might not be configured to use DRA, so it can report a status correctly. - subControllers = append(subControllers, &DRAReconciler{Client: r.Client, Scheme: r.Scheme, Opts: opts}) subControllers = append(subControllers, &MiscReconciler{Client: r.Client, Scheme: r.Scheme, Opts: opts}) // Ensure finalizer is present on live (non-deleted) ClusterPolicy objects. @@ -239,72 +249,6 @@ func (r *ClusterPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Reques return ctrl.Result{}, retErr } -// draPodToClusterPolicy maps any DRA pod event to reconcile requests for all existing -// ClusterPolicy objects. This avoids relying on r.Opts.ReqName (startup config) as a -// source of truth — instead it queries the actual state of the cluster. -func (r *ClusterPolicyReconciler) draPodToClusterPolicy(ctx context.Context, _ client.Object) []reconcile.Request { - cpList := &v1alpha.ClusterPolicyList{} - if err := r.List(ctx, cpList); err != nil { - klog.Error(err, "failed to list ClusterPolicies for DRA pod event") - return nil - } - - reqs := make([]reconcile.Request, len(cpList.Items)) - for i := range cpList.Items { - reqs[i] = reconcile.Request{ - NamespacedName: types.NamespacedName{Name: cpList.Items[i].Name}, - } - } - - return reqs -} - -// draPodReadinessPredicate returns a predicate that passes only DRA pod events where -// the Ready condition has changed (or the pod was created/deleted). This avoids -// spurious reconciles while still refreshing ClusterPolicy status when a DRA pod -// health check transitions between passing and failing. -func draPodReadinessPredicate() predicate.Predicate { - isDRAPod := func(obj client.Object) bool { - return obj.GetLabels()[appLabel] == draValue - } - - isPodReady := func(pod *core.Pod) bool { - for _, c := range pod.Status.Conditions { - if c.Type == core.PodReady { - return c.Status == core.ConditionTrue - } - } - - return false - } - - return predicate.Funcs{ - CreateFunc: func(e event.CreateEvent) bool { - return isDRAPod(e.Object) - }, - UpdateFunc: func(e event.UpdateEvent) bool { - if !isDRAPod(e.ObjectNew) { - return false - } - - oldPod, ok1 := e.ObjectOld.(*core.Pod) - newPod, ok2 := e.ObjectNew.(*core.Pod) - - if !ok1 || !ok2 { - return false - } - - return isPodReady(oldPod) != isPodReady(newPod) - }, - DeleteFunc: func(e event.DeleteEvent) bool { - return isDRAPod(e.Object) - }, - GenericFunc: func(e event.GenericEvent) bool { - return false - }, - } -} - // SetupWithManager sets up the controller with the Manager. func (r *ClusterPolicyReconciler) SetupWithManager(mgr ctrl.Manager, opts ControllerOpts) error { r.Opts = opts @@ -314,14 +258,8 @@ func (r *ClusterPolicyReconciler) SetupWithManager(mgr ctrl.Manager, opts Contro Named("clusterpolicy"). Owns(&apps.DaemonSet{}) - // Only watch DRA pods when DRA is enabled in the cluster, to avoid unnecessary - // pod list/watch permissions and reconcile noise when DRA is not in use. - if opts.DRAEnable { - b = b.Watches( - &core.Pod{}, - handler.EnqueueRequestsFromMapFunc(r.draPodToClusterPolicy), - builder.WithPredicates(draPodReadinessPredicate()), - ) + if opts.KMMEnable { + b = b.Owns(&kmmv1beta1.Module{}) } return b.Complete(r) diff --git a/internal/controller/clusterpolicy_controller_test.go b/internal/controller/clusterpolicy_controller_test.go index 3a91490..daed29c 100644 --- a/internal/controller/clusterpolicy_controller_test.go +++ b/internal/controller/clusterpolicy_controller_test.go @@ -18,7 +18,6 @@ package controller import ( "context" - "fmt" "time" . "github.com/onsi/ginkgo/v2" @@ -26,12 +25,8 @@ import ( apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - "sigs.k8s.io/controller-runtime/pkg/client/interceptor" - "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/yaml" @@ -39,6 +34,7 @@ import ( v1alpha "github.com/intel/gpu-base-operator/api/v1alpha1" "github.com/intel/gpu-base-operator/config/deployments" + kmmv1beta1 "github.com/kubernetes-sigs/kernel-module-management/api/v1beta1" nfdcrd "sigs.k8s.io/node-feature-discovery/api/nfd/v1alpha1" ) @@ -96,9 +92,11 @@ var _ = Describe("ClusterPolicy Controller", func() { Client: k8sClient, Scheme: k8sClient.Scheme(), Opts: ControllerOpts{ - Namespace: defaultNamespace, - DRAEnable: true, - RequeueDelay: time.Millisecond * 50, + Namespace: defaultNamespace, + DRAEnable: true, + KMMEnable: true, + DRAServiceAccountName: "intel-gpu-dra", + RequeueDelay: time.Millisecond * 50, }, } @@ -115,22 +113,20 @@ var _ = Describe("ClusterPolicy Controller", func() { Expect(err).NotTo(HaveOccurred()) Expect(ret.RequeueAfter).To(BeZero()) + By("verifying KMM Module CR was created") + mod := &kmmv1beta1.Module{} + modKey := types.NamespacedName{Name: resourceName + "-gpu", Namespace: defaultNamespace} + Expect(k8sClient.Get(ctx, modKey, mod)).To(Succeed()) + Expect(mod.Spec.DRA).NotTo(BeNil()) + Expect(mod.Spec.DRA.Container.Image).To(Equal("dra-image:v1.2.3")) + + By("verifying XPUM DaemonSet was created") daemonSets := apps.DaemonSetList{} err = k8sClient.List(ctx, &daemonSets, client.InNamespace(defaultNamespace)) Expect(err).NotTo(HaveOccurred()) - - Expect(daemonSets.Items).To(HaveLen(2)) - - for _, ds := range daemonSets.Items { - switch ds.Name { - case "test-resource-dra-xpum-xpum-gpu-dra": - Expect(ds.Spec.Template.Spec.Containers[0].Image).To(Equal("dra-image:v1.2.3")) - case "test-resource-dra-xpum-xpum-xpu-manager": - Expect(ds.Spec.Template.Spec.Containers[0].Image).To(Equal("xpum-image:v1.2.3")) - default: - Fail("Unexpected DaemonSet found: " + ds.Name) - } - } + Expect(daemonSets.Items).To(HaveLen(1)) + Expect(daemonSets.Items[0].Name).To(Equal(resourceName + "-xpu-manager")) + Expect(daemonSets.Items[0].Spec.Template.Spec.Containers[0].Image).To(Equal("xpum-image:v1.2.3")) nfr := nfdcrd.NodeFeatureRule{} err = k8sClient.Get(ctx, types.NamespacedName{Name: "intel-gpu-devices"}, &nfr) @@ -150,15 +146,14 @@ var _ = Describe("ClusterPolicy Controller", func() { err = k8sClient.List(ctx, &daemonSets, client.InNamespace(defaultNamespace)) Expect(err).NotTo(HaveOccurred()) - - Expect(daemonSets.Items).To(HaveLen(2)) + Expect(daemonSets.Items).To(HaveLen(1)) configmaps := v1.ConfigMapList{} err = k8sClient.List(ctx, &configmaps, client.InNamespace(defaultNamespace)) Expect(err).NotTo(HaveOccurred()) Expect(configmaps.Items).To(HaveLen(1)) - Expect(configmaps.Items[0].Name).To(Equal("test-resource-dra-xpum-xpum-xpumanager-otel-config")) + Expect(configmaps.Items[0].Name).To(Equal(resourceName + "-xpumanager-otel-config")) clusterpolicy.Spec.HealthinessSpec = &v1alpha.HealthinessSpec{ CheckIntervalSeconds: 17, @@ -172,7 +167,6 @@ var _ = Describe("ClusterPolicy Controller", func() { Expect(ret.RequeueAfter).To(BeZero()) // Create XPUM config override - xpumConfig := deployments.XpuManagerOTelConfig() data, err := yaml.Marshal(xpumConfig) Expect(err).NotTo(HaveOccurred()) @@ -199,26 +193,18 @@ var _ = Describe("ClusterPolicy Controller", func() { err = k8sClient.List(ctx, &daemonSets, client.InNamespace(defaultNamespace)) Expect(err).NotTo(HaveOccurred()) + Expect(daemonSets.Items).To(HaveLen(1)) - Expect(daemonSets.Items).To(HaveLen(2)) - - for _, ds := range daemonSets.Items { - switch ds.Name { - case "test-resource-dra-xpum-xpum-gpu-dra": - case "test-resource-dra-xpum-xpum-xpu-manager": - checked := false - for _, v := range ds.Spec.Template.Spec.Volumes { - if v.Name == "config" { - Expect(v.VolumeSource.ConfigMap).NotTo(BeNil()) - Expect(v.VolumeSource.ConfigMap.Name).To(Equal("test-xpum-cm")) - checked = true - } - } - Expect(checked).To(BeTrue()) - default: - Fail("Unexpected DaemonSet found: " + ds.Name) + By("verifying XPUM config override was applied") + checked := false + for _, v := range daemonSets.Items[0].Spec.Template.Spec.Volumes { + if v.Name == "config" { + Expect(v.VolumeSource.ConfigMap).NotTo(BeNil()) + Expect(v.VolumeSource.ConfigMap.Name).To(Equal("test-xpum-cm")) + checked = true } } + Expect(checked).To(BeTrue()) err = k8sClient.Delete(ctx, clusterpolicy) Expect(err).NotTo(HaveOccurred()) @@ -240,136 +226,7 @@ var _ = Describe("ClusterPolicy Controller", func() { err = k8sClient.List(ctx, &daemonSets, client.InNamespace(defaultNamespace)) Expect(err).NotTo(HaveOccurred()) - // Both DaemonSets (DRA and XPUM) are explicitly deleted by the reconciler - // during the finalizer cleanup — we no longer rely on the K8s GC. Expect(daemonSets.Items).To(BeEmpty()) }) }) }) - -var _ = Describe("draPodReadinessPredicate", func() { - var pred = draPodReadinessPredicate() - - draLabels := map[string]string{appLabel: draValue} - - draReadyPod := func(ready bool) *v1.Pod { - status := v1.ConditionFalse - if ready { - status = v1.ConditionTrue - } - return &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{Labels: draLabels}, - Status: v1.PodStatus{ - Conditions: []v1.PodCondition{ - {Type: v1.PodReady, Status: status}, - }, - }, - } - } - - Context("CreateFunc", func() { - It("passes DRA pods", func() { - Expect(pred.Create(event.CreateEvent{Object: draReadyPod(false)})).To(BeTrue()) - }) - - It("filters out non-DRA pods", func() { - pod := &v1.Pod{} - Expect(pred.Create(event.CreateEvent{Object: pod})).To(BeFalse()) - }) - }) - - Context("DeleteFunc", func() { - It("passes DRA pods", func() { - Expect(pred.Delete(event.DeleteEvent{Object: draReadyPod(true)})).To(BeTrue()) - }) - - It("filters out non-DRA pods", func() { - Expect(pred.Delete(event.DeleteEvent{Object: &v1.Pod{}})).To(BeFalse()) - }) - }) - - Context("GenericFunc", func() { - It("always filters events", func() { - Expect(pred.Generic(event.GenericEvent{Object: draReadyPod(true)})).To(BeFalse()) - }) - }) - - Context("UpdateFunc", func() { - It("passes when DRA pod Ready condition changes from false to true", func() { - e := event.UpdateEvent{ObjectOld: draReadyPod(false), ObjectNew: draReadyPod(true)} - Expect(pred.Update(e)).To(BeTrue()) - }) - - It("passes when DRA pod Ready condition changes from true to false", func() { - e := event.UpdateEvent{ObjectOld: draReadyPod(true), ObjectNew: draReadyPod(false)} - Expect(pred.Update(e)).To(BeTrue()) - }) - - It("filters when DRA pod Ready condition is unchanged (both ready)", func() { - e := event.UpdateEvent{ObjectOld: draReadyPod(true), ObjectNew: draReadyPod(true)} - Expect(pred.Update(e)).To(BeFalse()) - }) - - It("filters when DRA pod Ready condition is unchanged (both not ready)", func() { - e := event.UpdateEvent{ObjectOld: draReadyPod(false), ObjectNew: draReadyPod(false)} - Expect(pred.Update(e)).To(BeFalse()) - }) - - It("filters non-DRA pods", func() { - nonDRA := &v1.Pod{} - e := event.UpdateEvent{ObjectOld: nonDRA, ObjectNew: nonDRA} - Expect(pred.Update(e)).To(BeFalse()) - }) - - It("filters when objects are not pods", func() { - notAPod := &v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Labels: draLabels}} - e := event.UpdateEvent{ObjectOld: notAPod, ObjectNew: notAPod} - Expect(pred.Update(e)).To(BeFalse()) - }) - }) -}) - -var _ = Describe("draPodToClusterPolicy", func() { - newScheme := func() *runtime.Scheme { - s := runtime.NewScheme() - Expect(v1alpha.AddToScheme(s)).To(Succeed()) - return s - } - - It("returns one request per existing ClusterPolicy", func() { - cp1 := &v1alpha.ClusterPolicy{ObjectMeta: metav1.ObjectMeta{Name: "policy-one"}} - cp2 := &v1alpha.ClusterPolicy{ObjectMeta: metav1.ObjectMeta{Name: "policy-two"}} - - r := &ClusterPolicyReconciler{ - Client: fake.NewClientBuilder().WithScheme(newScheme()).WithObjects(cp1, cp2).Build(), - } - - reqs := r.draPodToClusterPolicy(context.Background(), nil) - Expect(reqs).To(HaveLen(2)) - - names := []string{reqs[0].Name, reqs[1].Name} - Expect(names).To(ConsistOf("policy-one", "policy-two")) - }) - - It("returns an empty slice when no ClusterPolicies exist", func() { - r := &ClusterPolicyReconciler{ - Client: fake.NewClientBuilder().WithScheme(newScheme()).Build(), - } - - reqs := r.draPodToClusterPolicy(context.Background(), nil) - Expect(reqs).To(BeEmpty()) - }) - - It("returns nil when the List call fails", func() { - r := &ClusterPolicyReconciler{ - Client: fake.NewClientBuilder().WithScheme(newScheme()).WithInterceptorFuncs(interceptor.Funcs{ - List: func(_ context.Context, _ client.WithWatch, _ client.ObjectList, _ ...client.ListOption) error { - return fmt.Errorf("list error") - }, - }).Build(), - } - - reqs := r.draPodToClusterPolicy(context.Background(), nil) - Expect(reqs).To(BeNil()) - }) -}) diff --git a/internal/controller/deviceplugin_controller.go b/internal/controller/deviceplugin_controller.go deleted file mode 100644 index 21588f6..0000000 --- a/internal/controller/deviceplugin_controller.go +++ /dev/null @@ -1,354 +0,0 @@ -/* -Copyright 2025 Intel Corporation. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "context" - "fmt" - "slices" - "strings" - - apps "k8s.io/api/apps/v1" - core "k8s.io/api/core/v1" - "k8s.io/klog/v2" - - "k8s.io/apimachinery/pkg/runtime" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - logf "sigs.k8s.io/controller-runtime/pkg/log" - - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - v1alpha "github.com/intel/gpu-base-operator/api/v1alpha1" - "github.com/intel/gpu-base-operator/config/deployments" -) - -type DevicePluginReconciler struct { - client.Client - Scheme *runtime.Scheme - Opts ControllerOpts -} - -const ( - dpValue = "intel-gpu-plugin" - xpumdVolumeName = "runxpumd" - - dpResourcePart = "gpu-dp" -) - -func logLevelForDp(spec *v1alpha.ClusterPolicy) int32 { - logLevel := int32(0) - logLevel = max(logLevel, spec.Spec.LogLevel) - logLevel = max(logLevel, spec.Spec.DevicePluginSpec.LogLevel) - - return logLevel -} - -func hexArgStr(s []string) string { - a := []string{} - for _, str := range s { - if !strings.HasPrefix(str, "0x") { - str = "0x" + str - } - - a = append(a, str) - } - return strings.Join(a, ",") -} - -func addXpumdMounts(spec *core.PodSpec) { - for _, v := range spec.Volumes { - if v.Name == xpumdVolumeName { - return - } - } - - dirOrCreate := core.HostPathDirectoryOrCreate - - spec.Volumes = append(spec.Volumes, core.Volume{ - Name: xpumdVolumeName, - VolumeSource: core.VolumeSource{ - HostPath: &core.HostPathVolumeSource{ - Path: "/run/xpumd", - Type: &dirOrCreate, - }, - }, - }) - - spec.Containers[0].VolumeMounts = append(spec.Containers[0].VolumeMounts, - core.VolumeMount{ - Name: xpumdVolumeName, - MountPath: "/run/xpumd", - }, - ) -} - -func removeXpumdMounts(spec *core.PodSpec) { - for i, v := range spec.Volumes { - if v.Name == xpumdVolumeName { - spec.Volumes = slices.Delete(spec.Volumes, i, i+1) - break - } - } - - for i, vm := range spec.Containers[0].VolumeMounts { - if vm.Name == xpumdVolumeName { - spec.Containers[0].VolumeMounts = slices.Delete(spec.Containers[0].VolumeMounts, i, i+1) - break - } - } -} - -func dpArgs(spec *v1alpha.ClusterPolicy) []string { - args := []string{} - - dpspec := spec.Spec.DevicePluginSpec - - if spec.Spec.ResourceMonitoring { - args = append(args, - "-enable-monitoring", - "-xpumd-endpoint=/run/xpumd/intelxpuinfo.sock") - } - - logLevel := logLevelForDp(spec) - if logLevel > 0 { - args = append(args, fmt.Sprintf("-v=%d", logLevel)) - } - - if len(dpspec.ByPathMode) > 0 { - args = append(args, fmt.Sprintf("-bypath=%s", dpspec.ByPathMode)) - } - - if len(dpspec.AllowIDs) > 0 { - args = append(args, fmt.Sprintf("-allow-ids=%s", hexArgStr(dpspec.AllowIDs))) - } - - if len(dpspec.DenyIDs) > 0 { - args = append(args, fmt.Sprintf("-deny-ids=%s", hexArgStr(dpspec.DenyIDs))) - } - - return args -} - -func (r *DevicePluginReconciler) updateDaemonSetObject(ds *apps.DaemonSet, spec *v1alpha.ClusterPolicy) { - name := fmt.Sprintf("%s-device-plugin", spec.Name) - - ds.Name = name - ds.Namespace = r.Opts.Namespace - ds.Labels[ownerKey] = spec.Name - - dspec := &spec.Spec.DevicePluginSpec - - ds.Spec.Template.Spec.Containers[0].Image = dspec.PluginImage - - ds.Spec.Template.Spec.Containers[0].Args = dpArgs(spec) - - ds.Spec.Template.Spec.NodeSelector = map[string]string{ - "kubernetes.io/arch": "amd64", - } - - if len(spec.Spec.NodeSelector) > 0 { - for k, v := range spec.Spec.NodeSelector { - ds.Spec.Template.Spec.NodeSelector[k] = v - } - } - - if spec.Spec.UseNFDLabeling { - ds.Spec.Template.Spec.NodeSelector["intel.feature.node.kubernetes.io/gpu"] = trueValue - } - - if len(spec.Spec.Tolerations) > 0 { - ds.Spec.Template.Spec.Tolerations = spec.Spec.Tolerations - } else { - ds.Spec.Template.Spec.Tolerations = nil - } - - cspec := &ds.Spec.Template.Spec - - secrets := []core.LocalObjectReference{} - if r.Opts.SecretName != "" { - secrets = append(secrets, core.LocalObjectReference{Name: r.Opts.SecretName}) - } - if spec.Spec.PullSecret != nil { - secrets = append(secrets, *spec.Spec.PullSecret) - } - - if len(secrets) > 0 { - cspec.ImagePullSecrets = secrets - } else { - cspec.ImagePullSecrets = nil - } - - if spec.Spec.ResourceMonitoring { - addXpumdMounts(cspec) - } else { - removeXpumdMounts(cspec) - } - - if r.Opts.OpenShift { - _, _, _, saName := buildOpenShiftNames(spec.Name, dpResourcePart) - cspec.ServiceAccountName = saName - } -} - -func (r *DevicePluginReconciler) createOpenShiftResourcesIfNotExists(ctx context.Context, cpName string) error { - sccName, roleName, bindingName, saName := buildOpenShiftNames(cpName, dpResourcePart) - - if err := createServiceAccount(ctx, r.Client, saName, r.Opts.Namespace); err != nil { - return fmt.Errorf("failed to ensure DP ServiceAccount: %w", err) - } - - if err := ensureSCC(ctx, r.Client, buildDevicePluginSCC(sccName)); err != nil { - return fmt.Errorf("failed to ensure DP SCC: %w", err) - } - - if err := createSCCRole(ctx, r.Client, roleName, sccName); err != nil { - return fmt.Errorf("failed to ensure DP SCC ClusterRole: %w", err) - } - - if err := createSCCRoleBinding(ctx, r.Client, bindingName, roleName, saName, r.Opts.Namespace); err != nil { - return fmt.Errorf("failed to ensure DP SCC ClusterRoleBinding: %w", err) - } - - return nil -} - -func (r *DevicePluginReconciler) cleanupOpenShiftResources(ctx context.Context, cpName string) { - sccName, roleName, bindingName, saName := buildOpenShiftNames(cpName, dpResourcePart) - - deleteOpenShiftSCCResources(ctx, r.Client, sccName, roleName, bindingName, saName, r.Opts.Namespace) -} - -func (r *DevicePluginReconciler) createDaemonSet(ctx context.Context, obj client.Object) (ctrl.Result, error) { - spec := obj.(*v1alpha.ClusterPolicy) - - ds := deployments.DevicePluginDaemonset() - - r.updateDaemonSetObject(ds, spec) - - if err := ctrl.SetControllerReference(obj, ds, r.Scheme); err != nil { - klog.Error(err, "unable to set controller reference") - - return ctrl.Result{}, err - } - - if err := r.Create(ctx, ds); err != nil { - klog.Error(err, "unable to create DaemonSet") - - return ctrl.Result{}, err - } - - return ctrl.Result{}, nil -} - -func (r *DevicePluginReconciler) removeDeploymentIfExists(ctx context.Context) (ctrl.Result, error) { - klog.V(4).Info("Removing Device Plugin deployment") - - crName := r.Opts.ReqName - - if r.Opts.OpenShift { - r.cleanupOpenShiftResources(ctx, crName) - } - - dss := &apps.DaemonSetList{} - labels := client.MatchingLabels{ - appLabel: dpValue, - ownerKey: crName, - } - - if err := r.List(ctx, dss, client.InNamespace(r.Opts.Namespace), labels); err != nil { - klog.Error(err, "unable to list child DaemonSets") - - return ctrl.Result{}, err - } - - if len(dss.Items) == 0 { - klog.V(4).Info("No DevicePlugin deployment found, nothing to do") - - return ctrl.Result{}, nil - } - - if err := r.Delete(ctx, &dss.Items[0]); err != nil { - return ctrl.Result{}, err - } - - klog.V(4).Info("DevicePlugin deployment removed") - - return ctrl.Result{}, nil -} - -func (r *DevicePluginReconciler) Reconcile(ctx context.Context, cp *v1alpha.ClusterPolicy) (ctrl.Result, error) { - _ = logf.FromContext(ctx) - - if cp == nil || !cp.DeletionTimestamp.IsZero() { - return r.removeDeploymentIfExists(ctx) - } - - if cp.Spec.ResourceRegistration != "dp" { - cp.Status.DevicePluginStatus = notAvailableStatus - - return r.removeDeploymentIfExists(ctx) - } - - var olderDs apps.DaemonSetList - if err := r.List(ctx, &olderDs, client.InNamespace(r.Opts.Namespace), client.MatchingLabels{appLabel: dpValue}); err != nil { - klog.Error(err, "unable to list child DaemonSets") - - return ctrl.Result{}, err - } - - if r.Opts.OpenShift { - if err := r.createOpenShiftResourcesIfNotExists(ctx, cp.Name); err != nil { - klog.Error(err, "unable to ensure OpenShift resources for DP") - - return ctrl.Result{}, err - } - } - - if len(olderDs.Items) == 0 { - return r.createDaemonSet(ctx, cp) - } - - // Update DaemonSet - - ds := &olderDs.Items[0] - originalDs := ds.DeepCopy() - - r.updateDaemonSetObject(ds, cp) - - dsDiff := cmp.Diff(originalDs.Spec.Template.Spec, ds.Spec.Template.Spec, cmpopts.EquateEmpty()) - if len(dsDiff) > 0 { - klog.Info("DS difference", "diff", dsDiff) - - if err := r.Update(ctx, ds); err != nil { - klog.Error(err, "unable to update daemonset", "DaemonSet", ds) - - return ctrl.Result{}, err - } - } - - if err := r.List(ctx, &olderDs, client.InNamespace(r.Opts.Namespace), client.MatchingLabels{appLabel: dpValue}); err != nil { - klog.Error(err, "unable to list child DaemonSets") - - return ctrl.Result{}, err - } - - cp.Status.DevicePluginStatus = fmt.Sprintf("%d/%d", - olderDs.Items[0].Status.NumberReady, olderDs.Items[0].Status.DesiredNumberScheduled) - - return ctrl.Result{}, nil -} diff --git a/internal/controller/deviceplugin_controller_test.go b/internal/controller/deviceplugin_controller_test.go deleted file mode 100644 index 79964e4..0000000 --- a/internal/controller/deviceplugin_controller_test.go +++ /dev/null @@ -1,721 +0,0 @@ -/* -Copyright 2025 Intel Corporation. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "context" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - apps "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" - rbac "k8s.io/api/rbac/v1" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - - v1alpha "github.com/intel/gpu-base-operator/api/v1alpha1" - "github.com/intel/gpu-base-operator/config/deployments" -) - -var _ = Describe("ClusterPolicy Controller for Device Plugin", func() { - - Context("When reconciling Device Plugin and XPUM", func() { - defaultNamespace := "foobar" - const resourceName = "test-resource" - - ctx := context.Background() - - typeNamespacedName := types.NamespacedName{ - Name: resourceName, - } - clusterpolicy := &v1alpha.ClusterPolicy{} - - BeforeEach(func() { - ns := &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaultNamespace, - }, - } - - Expect(k8sClient.Create(ctx, ns)).To(Succeed()) - }) - - AfterEach(func() { - resource := &v1alpha.ClusterPolicy{} - err := k8sClient.Get(ctx, typeNamespacedName, resource) - Expect(err).NotTo(HaveOccurred()) - - By("Cleanup the specific resource instance ClusterPolicy") - Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) - }) - - It("should successfully reconcile the resource", func() { - By("creating the custom resource for the Kind ClusterPolicy") - err := k8sClient.Get(ctx, typeNamespacedName, clusterpolicy) - if err != nil && errors.IsNotFound(err) { - resource := &v1alpha.ClusterPolicy{ - ObjectMeta: metav1.ObjectMeta{ - Name: resourceName, - }, - Spec: v1alpha.ClusterPolicySpec{ - ResourceRegistration: "dp", - ResourceMonitoring: true, - UseNFDLabeling: true, - DevicePluginSpec: v1alpha.DevicePluginSpec{ - PluginImage: "intel/intel-gpu-plugin:0.32.0", - }, - XpuManagerSpec: v1alpha.XpuManagerSpec{ - Image: "intel/xpumanager:v1.2.27", - MonitoringResource: "monitoring", - }, - }, - } - Expect(k8sClient.Create(ctx, resource)).To(Succeed()) - } - - By("Reconciling the created resource") - controllerReconciler := &ClusterPolicyReconciler{ - Client: k8sClient, - Scheme: k8sClient.Scheme(), - Opts: ControllerOpts{ - Namespace: defaultNamespace, - DRAEnable: true, - }, - } - - _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ - NamespacedName: typeNamespacedName, - }) - Expect(err).NotTo(HaveOccurred()) - - By("Checking that the DaemonSet has been created") - dpDs := apps.DaemonSetList{} - err = k8sClient.List(ctx, &dpDs, client.InNamespace(defaultNamespace)) - Expect(err).NotTo(HaveOccurred()) - - Expect(dpDs.Items).To(HaveLen(2)) - Expect(dpDs.Items[0].Name).To(Equal("test-resource-device-plugin")) - Expect(dpDs.Items[0].Spec.Template.Spec.Containers[0].Args).To(ContainElement("-enable-monitoring")) - Expect(dpDs.Items[0].Spec.Template.Spec.NodeSelector).To(HaveKeyWithValue("intel.feature.node.kubernetes.io/gpu", "true")) - Expect(dpDs.Items[0].Spec.Template.Spec.NodeSelector).To(HaveKeyWithValue("kubernetes.io/arch", "amd64")) - Expect(dpDs.Items[1].Name).To(Equal("test-resource-xpu-manager")) - Expect(dpDs.Items[1].Spec.Template.Spec.NodeSelector).To(HaveKeyWithValue("intel.feature.node.kubernetes.io/gpu", "true")) - - resource := &v1alpha.ClusterPolicy{} - err = k8sClient.Get(ctx, typeNamespacedName, resource) - Expect(err).NotTo(HaveOccurred()) - - resource.Spec.UseNFDLabeling = false - resource.Spec.NodeSelector = map[string]string{"foo": "bar"} - - Expect(k8sClient.Update(ctx, resource)).To(Succeed()) - - _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ - NamespacedName: typeNamespacedName, - }) - Expect(err).NotTo(HaveOccurred()) - - err = k8sClient.List(ctx, &dpDs, client.InNamespace(defaultNamespace)) - Expect(err).NotTo(HaveOccurred()) - - Expect(dpDs.Items).To(HaveLen(2)) - Expect(dpDs.Items[0].Spec.Template.Spec.NodeSelector).To(HaveKeyWithValue("foo", "bar")) - Expect(dpDs.Items[1].Spec.Template.Spec.NodeSelector).To(HaveKeyWithValue("foo", "bar")) - - err = k8sClient.Get(ctx, typeNamespacedName, resource) - Expect(err).NotTo(HaveOccurred()) - - resource.Spec.ResourceMonitoring = false - resource.Spec.DevicePluginSpec.LogLevel = 4 - - Expect(k8sClient.Update(ctx, resource)).To(Succeed()) - - _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ - NamespacedName: typeNamespacedName, - }) - Expect(err).NotTo(HaveOccurred()) - - err = k8sClient.List(ctx, &dpDs, client.InNamespace(defaultNamespace)) - Expect(err).NotTo(HaveOccurred()) - - Expect(dpDs.Items).To(HaveLen(1)) - Expect(dpDs.Items[0].Name).To(Equal("test-resource-device-plugin")) - Expect(dpDs.Items[0].Spec.Template.Spec.Containers[0].Args).To(ContainElement("-v=4")) - Expect(dpDs.Items[0].Spec.Template.Spec.Containers[0].Args).NotTo(ContainElement("-enable-monitoring")) - }) - }) - - Context("When reconciling DP and XPUM", func() { - defaultNamespace := "foobar-xpum" - const resourceName = "test-resource-xpum" - - ctx := context.Background() - - typeNamespacedName := types.NamespacedName{ - Name: resourceName, - } - clusterpolicy := &v1alpha.ClusterPolicy{ - ObjectMeta: metav1.ObjectMeta{ - Name: resourceName, - }, - Spec: v1alpha.ClusterPolicySpec{ - ResourceRegistration: "dp", - ResourceMonitoring: true, - DevicePluginSpec: v1alpha.DevicePluginSpec{ - PluginImage: "intel/intel-gpu-plugin:0.32.0", - }, - XpuManagerSpec: v1alpha.XpuManagerSpec{ - Image: "xpum-image:v1.2.3", - LogLevel: 3, - MonitoringResource: "monitoring", - }, - }, - } - - BeforeEach(func() { - ns := &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaultNamespace, - }, - } - - Expect(k8sClient.Create(ctx, ns)).To(Succeed()) - }) - - AfterEach(func() { - resource := &v1alpha.ClusterPolicy{} - err := k8sClient.Get(ctx, typeNamespacedName, resource) - Expect(err).NotTo(HaveOccurred()) - - By("Cleanup the specific resource instance ClusterPolicy") - Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) - }) - - It("should successfully reconcile the resource", func() { - By("creating the custom resource for the Kind ClusterPolicy") - err := k8sClient.Get(ctx, typeNamespacedName, clusterpolicy) - if err != nil && errors.IsNotFound(err) { - Expect(k8sClient.Create(ctx, clusterpolicy)).To(Succeed()) - } - - By("Reconciling the created resource") - controllerReconciler := &ClusterPolicyReconciler{ - Client: k8sClient, - Scheme: k8sClient.Scheme(), - Opts: ControllerOpts{ - Namespace: defaultNamespace, - DRAEnable: true, - RequeueDelay: time.Millisecond * 50, - }, - } - - _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ - NamespacedName: typeNamespacedName, - }) - Expect(err).NotTo(HaveOccurred()) - - daemonSets := apps.DaemonSetList{} - err = k8sClient.List(ctx, &daemonSets, client.InNamespace(defaultNamespace)) - Expect(err).NotTo(HaveOccurred()) - - Expect(daemonSets.Items).To(HaveLen(2)) - - for _, ds := range daemonSets.Items { - switch ds.Name { - case "test-resource-xpum-device-plugin": - case "test-resource-xpum-xpu-manager": - default: - Fail("Unexpected DaemonSet found: " + ds.Name) - } - } - }) - }) - - Context("When reconciling from DP to DRA", func() { - defaultNamespace := "foobar-dp-to-dra" - const resourceName = "test-resource-dp-to-dra" - - ctx := context.Background() - - typeNamespacedName := types.NamespacedName{ - Name: resourceName, - } - clusterpolicy := &v1alpha.ClusterPolicy{ - ObjectMeta: metav1.ObjectMeta{ - Name: resourceName, - }, - Spec: v1alpha.ClusterPolicySpec{ - ResourceRegistration: "dp", - ResourceMonitoring: false, - DevicePluginSpec: v1alpha.DevicePluginSpec{ - PluginImage: "intel/intel-gpu-plugin:0.32.0", - }, - DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ - Image: "intel/gpu-dra-driver:1.2.3", - }, - }, - } - - BeforeEach(func() { - ns := &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaultNamespace, - }, - } - - Expect(k8sClient.Create(ctx, ns)).To(Succeed()) - }) - - AfterEach(func() { - resource := &v1alpha.ClusterPolicy{} - err := k8sClient.Get(ctx, typeNamespacedName, resource) - Expect(err).NotTo(HaveOccurred()) - - By("Cleanup the specific resource instance ClusterPolicy") - Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) - }) - - It("should successfully reconcile the resource", func() { - By("creating the custom resource for the Kind ClusterPolicy") - err := k8sClient.Get(ctx, typeNamespacedName, clusterpolicy) - if err != nil && errors.IsNotFound(err) { - Expect(k8sClient.Create(ctx, clusterpolicy)).To(Succeed()) - } - - By("Reconciling the created resource") - controllerReconciler := &ClusterPolicyReconciler{ - Client: k8sClient, - Scheme: k8sClient.Scheme(), - Opts: ControllerOpts{ - Namespace: defaultNamespace, - DRAEnable: true, - RequeueDelay: time.Millisecond * 50, - }, - } - - _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ - NamespacedName: typeNamespacedName, - }) - Expect(err).NotTo(HaveOccurred()) - - daemonSets := apps.DaemonSetList{} - err = k8sClient.List(ctx, &daemonSets, client.InNamespace(defaultNamespace)) - Expect(err).NotTo(HaveOccurred()) - - Expect(daemonSets.Items).To(HaveLen(1)) - - for _, ds := range daemonSets.Items { - switch ds.Name { - case "test-resource-dp-to-dra-device-plugin": - default: - Fail("Unexpected DaemonSet found: " + ds.Name) - } - } - - err = k8sClient.Get(ctx, typeNamespacedName, clusterpolicy) - Expect(err).NotTo(HaveOccurred()) - - clusterpolicy.Spec.ResourceRegistration = "dra" - Expect(k8sClient.Update(ctx, clusterpolicy)).To(Succeed()) - - _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ - NamespacedName: typeNamespacedName, - }) - Expect(err).NotTo(HaveOccurred()) - - err = k8sClient.List(ctx, &daemonSets, client.InNamespace(defaultNamespace)) - Expect(err).NotTo(HaveOccurred()) - - Expect(daemonSets.Items).To(HaveLen(1)) - - for _, ds := range daemonSets.Items { - switch ds.Name { - case "test-resource-dp-to-dra-gpu-dra": - default: - Fail("Unexpected DaemonSet found: " + ds.Name) - } - } - }) - }) - - Context("When reconciling Device Plugin and XPUM on OpenShift", func() { - defaultNamespace := "foobar-openshift" - const resourceName = "test-resource-ocp" - - ctx := context.Background() - - typeNamespacedName := types.NamespacedName{Name: resourceName} - - BeforeEach(func() { - Expect(k8sClient.Create(ctx, &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{Name: defaultNamespace}, - })).To(Succeed()) - }) - - AfterEach(func() { - resource := &v1alpha.ClusterPolicy{} - Expect(k8sClient.Get(ctx, typeNamespacedName, resource)).To(Succeed()) - Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) - - // Clean up cluster-scoped OpenShift resources created by the reconciler. - deleteOpenShiftSCCResources(ctx, k8sClient, - resourceName+"-gpu-dp-scc", - resourceName+"-gpu-dp-scc-role", - resourceName+"-gpu-dp-scc-binding", - resourceName+"-gpu-dp", - defaultNamespace) - deleteOpenShiftSCCResources(ctx, k8sClient, - resourceName+"-xpu-manager-scc", - resourceName+"-xpu-manager-scc-role", - resourceName+"-xpu-manager-scc-binding", - resourceName+"-xpu-manager", - defaultNamespace) - }) - - It("creates SCC resources for DP and XPUM and sets ServiceAccountName on DaemonSets", func() { - By("creating the ClusterPolicy") - Expect(k8sClient.Create(ctx, &v1alpha.ClusterPolicy{ - ObjectMeta: metav1.ObjectMeta{Name: resourceName}, - Spec: v1alpha.ClusterPolicySpec{ - ResourceRegistration: "dp", - ResourceMonitoring: true, - DevicePluginSpec: v1alpha.DevicePluginSpec{ - PluginImage: "intel/intel-gpu-plugin:test", - }, - XpuManagerSpec: v1alpha.XpuManagerSpec{ - Image: "intel/xpumanager:test", - }, - }, - })).To(Succeed()) - - reconciler := &ClusterPolicyReconciler{ - Client: k8sClient, - Scheme: k8sClient.Scheme(), - Opts: ControllerOpts{ - Namespace: defaultNamespace, - OpenShift: true, - RequeueDelay: time.Millisecond * 50, - }, - } - - By("first reconcile creates SCC resources") - _, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) - Expect(err).NotTo(HaveOccurred()) - - By("DP ServiceAccount is created") - dpSA := &v1.ServiceAccount{} - Expect(k8sClient.Get(ctx, client.ObjectKey{Name: resourceName + "-gpu-dp-sa", Namespace: defaultNamespace}, dpSA)).To(Succeed()) - - By("DP SCC is created") - dpSCC := &unstructured.Unstructured{} - dpSCC.SetAPIVersion(sccAPIVersion) - dpSCC.SetKind(sccKind) - Expect(k8sClient.Get(ctx, client.ObjectKey{Name: resourceName + "-gpu-dp-scc"}, dpSCC)).To(Succeed()) - - By("DP ClusterRole is created") - Expect(k8sClient.Get(ctx, client.ObjectKey{Name: resourceName + "-gpu-dp-scc-role"}, &rbac.ClusterRole{})).To(Succeed()) - - By("DP ClusterRoleBinding is created") - Expect(k8sClient.Get(ctx, client.ObjectKey{Name: resourceName + "-gpu-dp-scc-binding"}, &rbac.ClusterRoleBinding{})).To(Succeed()) - - By("XPUM ServiceAccount is created") - Expect(k8sClient.Get(ctx, client.ObjectKey{Name: resourceName + "-xpu-manager-sa", Namespace: defaultNamespace}, &v1.ServiceAccount{})).To(Succeed()) - - By("XPUM SCC is created") - xpumSCC := &unstructured.Unstructured{} - xpumSCC.SetAPIVersion(sccAPIVersion) - xpumSCC.SetKind(sccKind) - Expect(k8sClient.Get(ctx, client.ObjectKey{Name: resourceName + "-xpu-manager-scc"}, xpumSCC)).To(Succeed()) - - By("DaemonSets have correct ServiceAccountNames") - dsList := &apps.DaemonSetList{} - Expect(k8sClient.List(ctx, dsList, client.InNamespace(defaultNamespace))).To(Succeed()) - Expect(dsList.Items).To(HaveLen(2)) - for _, ds := range dsList.Items { - switch ds.Name { - case resourceName + "-device-plugin": - Expect(ds.Spec.Template.Spec.ServiceAccountName).To(Equal(resourceName + "-gpu-dp-sa")) - case resourceName + "-xpu-manager": - Expect(ds.Spec.Template.Spec.ServiceAccountName).To(Equal(resourceName + "-xpu-manager-sa")) - default: - Fail("Unexpected DaemonSet: " + ds.Name) - } - } - - By("second reconcile is idempotent") - _, err = reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) - Expect(err).NotTo(HaveOccurred()) - }) - }) - -}) - -var _ = Describe("Device Plugin", func() { - - Context("Arguments creation", func() { - It("with defaults", func() { - spec := &v1alpha.ClusterPolicy{ - Spec: v1alpha.ClusterPolicySpec{ - DevicePluginSpec: v1alpha.DevicePluginSpec{}, - }, - } - args := dpArgs(spec) - Expect(args).To(BeEmpty()) - }) - - It("Resource monitoring enabled", func() { - spec := &v1alpha.ClusterPolicy{ - Spec: v1alpha.ClusterPolicySpec{ - ResourceMonitoring: true, - DevicePluginSpec: v1alpha.DevicePluginSpec{}, - }, - } - args := dpArgs(spec) - Expect(args).To(ContainElement("-enable-monitoring")) - Expect(args).To(ContainElement("-xpumd-endpoint=/run/xpumd/intelxpuinfo.sock")) - }) - - It("Log level set", func() { - spec := &v1alpha.ClusterPolicy{ - Spec: v1alpha.ClusterPolicySpec{ - LogLevel: 2, - DevicePluginSpec: v1alpha.DevicePluginSpec{ - LogLevel: 3, - }, - }, - } - args := dpArgs(spec) - Expect(args).To(ContainElement("-v=3")) - }) - - It("ByPath set", func() { - spec := &v1alpha.ClusterPolicy{ - Spec: v1alpha.ClusterPolicySpec{ - DevicePluginSpec: v1alpha.DevicePluginSpec{ - ByPathMode: "all", - }, - }, - } - args := dpArgs(spec) - Expect(args).To(ContainElement("-bypath=all")) - }) - - It("Allow IDs set", func() { - spec := &v1alpha.ClusterPolicy{ - Spec: v1alpha.ClusterPolicySpec{ - DevicePluginSpec: v1alpha.DevicePluginSpec{ - AllowIDs: []string{"0xabcd", "0xdefa"}, - }, - }, - } - args := dpArgs(spec) - Expect(args).To(ContainElement("-allow-ids=0xabcd,0xdefa")) - }) - - It("Deny IDs set", func() { - spec := &v1alpha.ClusterPolicy{ - Spec: v1alpha.ClusterPolicySpec{ - DevicePluginSpec: v1alpha.DevicePluginSpec{ - DenyIDs: []string{"0x0xyz", "0x0123"}, - }, - }, - } - args := dpArgs(spec) - Expect(args).To(ContainElement("-deny-ids=0x0xyz,0x0123")) - }) - - It("All options set", func() { - spec := &v1alpha.ClusterPolicy{ - Spec: v1alpha.ClusterPolicySpec{ - ResourceMonitoring: true, - LogLevel: 1, - DevicePluginSpec: v1alpha.DevicePluginSpec{ - LogLevel: 3, - AllowIDs: []string{"0x1id1"}, - DenyIDs: []string{"0x2id2"}, - }, - }, - } - args := dpArgs(spec) - Expect(args).To(HaveLen(5)) - Expect(args).To(ContainElement("-enable-monitoring")) - Expect(args).To(ContainElement("-v=3")) - Expect(args).To(ContainElement("-xpumd-endpoint=/run/xpumd/intelxpuinfo.sock")) - Expect(args).To(ContainElement("-allow-ids=0x1id1")) - Expect(args).To(ContainElement("-deny-ids=0x2id2")) - }) - }) - - Context("DaemonSet object update", func() { - It("with tolerations and pullsecret", func() { - cp := &v1alpha.ClusterPolicy{ - Spec: v1alpha.ClusterPolicySpec{ - LogLevel: 2, - DevicePluginSpec: v1alpha.DevicePluginSpec{ - LogLevel: 3, - }, - Tolerations: []v1.Toleration{ - { - Key: "my-toleration", - Operator: v1.TolerationOpExists, - Effect: v1.TaintEffectNoExecute, - }, - }, - PullSecret: &v1.LocalObjectReference{ - Name: "my-pull-secret", - }, - }, - } - controller := &DevicePluginReconciler{} - - ds := deployments.DevicePluginDaemonset() - controller.updateDaemonSetObject(ds, cp) - - Expect(ds.Spec.Template.Spec.Tolerations[0].Key).To(Equal("my-toleration")) - Expect(ds.Spec.Template.Spec.Tolerations[0].Operator).To(Equal(v1.TolerationOpExists)) - Expect(ds.Spec.Template.Spec.Tolerations[0].Effect).To(Equal(v1.TaintEffectNoExecute)) - - Expect(ds.Spec.Template.Spec.ImagePullSecrets[0].Name).To(Equal("my-pull-secret")) - - cp.Spec.PullSecret = nil - cp.Spec.Tolerations = nil - controller.updateDaemonSetObject(ds, cp) - - Expect(ds.Spec.Template.Spec.Tolerations).To(BeEmpty()) - Expect(ds.Spec.Template.Spec.ImagePullSecrets).To(BeEmpty()) - }) - - It("preserves pod spec and container security context from YAML", func() { - cp := &v1alpha.ClusterPolicy{ - Spec: v1alpha.ClusterPolicySpec{ - DevicePluginSpec: v1alpha.DevicePluginSpec{ - PluginImage: "intel/intel-gpu-plugin:test", - }, - }, - } - controller := &DevicePluginReconciler{} - - ds := deployments.DevicePluginDaemonset() - controller.updateDaemonSetObject(ds, cp) - - By("automountServiceAccountToken is false") - Expect(ds.Spec.Template.Spec.AutomountServiceAccountToken).NotTo(BeNil()) - Expect(*ds.Spec.Template.Spec.AutomountServiceAccountToken).To(BeFalse()) - }) - - It("adds runxpumd volume and mount when ResourceMonitoring is true", func() { - cp := &v1alpha.ClusterPolicy{ - Spec: v1alpha.ClusterPolicySpec{ - ResourceMonitoring: true, - DevicePluginSpec: v1alpha.DevicePluginSpec{ - PluginImage: "intel/intel-gpu-plugin:test", - }, - }, - } - controller := &DevicePluginReconciler{} - - ds := deployments.DevicePluginDaemonset() - controller.updateDaemonSetObject(ds, cp) - - volNames := []string{} - for _, v := range ds.Spec.Template.Spec.Volumes { - volNames = append(volNames, v.Name) - } - Expect(volNames).To(ContainElement(xpumdVolumeName)) - - mountNames := []string{} - for _, vm := range ds.Spec.Template.Spec.Containers[0].VolumeMounts { - mountNames = append(mountNames, vm.Name) - } - Expect(mountNames).To(ContainElement(xpumdVolumeName)) - }) - - It("does not duplicate runxpumd volume when updateDaemonSetObject is called twice", func() { - cp := &v1alpha.ClusterPolicy{ - Spec: v1alpha.ClusterPolicySpec{ - ResourceMonitoring: true, - DevicePluginSpec: v1alpha.DevicePluginSpec{ - PluginImage: "intel/intel-gpu-plugin:test", - }, - }, - } - controller := &DevicePluginReconciler{} - - ds := deployments.DevicePluginDaemonset() - controller.updateDaemonSetObject(ds, cp) - controller.updateDaemonSetObject(ds, cp) - - count := 0 - for _, v := range ds.Spec.Template.Spec.Volumes { - if v.Name == xpumdVolumeName { - count++ - } - } - Expect(count).To(Equal(1)) - }) - - It("removes runxpumd volume and mount when ResourceMonitoring is disabled", func() { - cp := &v1alpha.ClusterPolicy{ - Spec: v1alpha.ClusterPolicySpec{ - ResourceMonitoring: true, - DevicePluginSpec: v1alpha.DevicePluginSpec{ - PluginImage: "intel/intel-gpu-plugin:test", - }, - }, - } - controller := &DevicePluginReconciler{} - - ds := deployments.DevicePluginDaemonset() - controller.updateDaemonSetObject(ds, cp) - - cp.Spec.ResourceMonitoring = false - controller.updateDaemonSetObject(ds, cp) - - for _, v := range ds.Spec.Template.Spec.Volumes { - Expect(v.Name).NotTo(Equal(xpumdVolumeName)) - } - for _, vm := range ds.Spec.Template.Spec.Containers[0].VolumeMounts { - Expect(vm.Name).NotTo(Equal(xpumdVolumeName)) - } - }) - - It("removeXpumdMounts is a no-op when runxpumd is not present", func() { - cp := &v1alpha.ClusterPolicy{ - Spec: v1alpha.ClusterPolicySpec{ - ResourceMonitoring: false, - DevicePluginSpec: v1alpha.DevicePluginSpec{ - PluginImage: "intel/intel-gpu-plugin:test", - }, - }, - } - controller := &DevicePluginReconciler{} - - ds := deployments.DevicePluginDaemonset() - initialVolCount := len(ds.Spec.Template.Spec.Volumes) - initialMountCount := len(ds.Spec.Template.Spec.Containers[0].VolumeMounts) - - Expect(func() { controller.updateDaemonSetObject(ds, cp) }).NotTo(Panic()) - Expect(ds.Spec.Template.Spec.Volumes).To(HaveLen(initialVolCount)) - Expect(ds.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(initialMountCount)) - }) - }) -}) diff --git a/internal/controller/dra_controller.go b/internal/controller/dra_controller.go deleted file mode 100644 index 1f3fee3..0000000 --- a/internal/controller/dra_controller.go +++ /dev/null @@ -1,582 +0,0 @@ -/* -Copyright 2025 Intel Corporation. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "context" - "fmt" - - adreg "k8s.io/api/admissionregistration/v1" - apps "k8s.io/api/apps/v1" - core "k8s.io/api/core/v1" - rbac "k8s.io/api/rbac/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/klog/v2" - - resv1 "k8s.io/api/resource/v1" - - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - logf "sigs.k8s.io/controller-runtime/pkg/log" - - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - v1alpha "github.com/intel/gpu-base-operator/api/v1alpha1" - "github.com/intel/gpu-base-operator/config/deployments" -) - -type DRAReconciler struct { - client.Client - Scheme *runtime.Scheme - Opts ControllerOpts -} - -const ( - draValue = "intel-gpu-resource-driver-kubelet-plugin" - healthCheckPort = 51516 - gpuDeviceClass = "gpu.intel.com" - vfioGpuDeviceClass = "gpu-vfio.intel.com" - - draResourcePart = "gpu-dra" -) - -func (r *DRAReconciler) createAll(ctx context.Context, cp *v1alpha.ClusterPolicy) error { - objects := []client.Object{} - - objName := fmt.Sprintf("%s-gpu-dra", cp.Name) - - // Service account - sa := deployments.DynamicResourceAllocationServiceAccount() - sa.Name = objName - sa.Namespace = r.Opts.Namespace - objects = append(objects, sa) - - // Cluster role - cr := deployments.DynamicResourceAllocationClusterRole() - cr.Name = objName - objects = append(objects, cr) - - // Cluster role binding - crb := deployments.DynamicResourceAllocationClusterRoleBinding() - crb.Name = objName - crb.Namespace = r.Opts.Namespace - - for i := range crb.Subjects { - crb.Subjects[i].Namespace = r.Opts.Namespace - crb.Subjects[i].Name = objName - } - crb.RoleRef.Name = objName - objects = append(objects, crb) - - // Device classes - objects = append(objects, deployments.DynamicResourceAllocationDeviceClass()) - - // Device Class for VFIO and configure it based on the ManageBinding setting in the CR. - mb := cp.Spec.DynamicResourceAllocationSpec.ManageBinding - objects = append(objects, deployments.DynamicResourceAllocationDeviceClassVfio(!mb)) - - // Validating admission policy - ap := deployments.DynamicResourceAllocationValidatingAdmissionPolicy() - ap.Name = objName - ap.Spec.MatchConditions[0].Expression = fmt.Sprintf("request.userInfo.username == \"system:serviceaccount:%s:%s\"", r.Opts.Namespace, objName) - objects = append(objects, ap) - - // Validating admission policy binding - apb := deployments.DynamicResourceAllocationValidatingAdmissionPolicyBinding() - apb.Name = objName - apb.Spec.PolicyName = apb.Name - objects = append(objects, apb) - - for _, o := range objects { - if err := r.Create(ctx, o); err != nil { - if errors.IsAlreadyExists(err) { - klog.Info("object already exists: "+o.GetName(), o) - - continue - } - - klog.Error(err, "unable to create object", "object", o) - - return err - } - } - - return r.createDaemonSet(ctx, cp) -} - -func (r *DRAReconciler) ensureVfioDeviceClass(ctx context.Context, manageBinding bool) { - existing := deployments.DynamicResourceAllocationDeviceClassVfio(!manageBinding) - - if err := r.Get(ctx, client.ObjectKey{Name: existing.Name}, existing); err != nil { - if errors.IsNotFound(err) { - klog.Info("VFIO device class not found, creating") - - if err := r.Create(ctx, existing); err != nil { - klog.Error(err, "unable to create VFIO device class") - } - - return - } - - klog.Error(err, "unable to get VFIO device class") - return - } - - desired := deployments.DynamicResourceAllocationDeviceClassVfio(!manageBinding) - - diff := cmp.Diff(existing.Spec.Selectors, desired.Spec.Selectors, cmpopts.EquateEmpty()) - if len(diff) > 0 { - klog.V(2).Info("Updating VFIO device class due to ManageBinding change", "diff", diff) - - existing.Spec.Selectors = desired.Spec.Selectors - - if err := r.Update(ctx, existing); err != nil { - klog.Error(err, "unable to update VFIO device class") - } - } -} - -func (r *DRAReconciler) deleteAll(ctx context.Context, crName string) error { - klog.Info("Deleting all DRA related objects") - - objName := fmt.Sprintf("%s-gpu-dra", crName) - - objects := []client.Object{ - &core.ServiceAccount{ - ObjectMeta: metav1.ObjectMeta{ - Name: objName, - Namespace: r.Opts.Namespace, - }, - }, - &rbac.ClusterRole{ - ObjectMeta: metav1.ObjectMeta{ - Name: objName, - }, - }, - &rbac.ClusterRoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: objName, - Namespace: r.Opts.Namespace, - }, - }, - &resv1.DeviceClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: gpuDeviceClass, - }, - }, - &resv1.DeviceClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: vfioGpuDeviceClass, - }, - }, - &adreg.ValidatingAdmissionPolicy{ - ObjectMeta: metav1.ObjectMeta{ - Name: objName, - }, - }, - &adreg.ValidatingAdmissionPolicyBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: objName, - }, - }, - &apps.DaemonSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: objName, - Namespace: r.Opts.Namespace, - }, - }, - } - - if r.Opts.OpenShift { - sccName, roleName, bindingName, _ := buildOpenShiftNames(crName, draResourcePart) - - scc := &unstructured.Unstructured{} - scc.SetAPIVersion(sccAPIVersion) - scc.SetKind(sccKind) - scc.SetName(sccName) - - objects = append(objects, - scc, - &rbac.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: roleName}}, - &rbac.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: bindingName}}, - ) - } - - for _, o := range objects { - if err := r.Delete(ctx, o); err != nil { - if errors.IsNotFound(err) { - klog.Warningf("unable to delete object, not found: %+v (%+v)", err, o) - - continue - } - - klog.Error(err, "unable to delete object", "object", o) - - return err - } - } - - klog.Info("Objects deleted") - - return nil -} - -func (r *DRAReconciler) createOpenShiftResourcesIfNotExists(ctx context.Context, crName string) error { - sccName, roleName, bindingName, _ := buildOpenShiftNames(crName, draResourcePart) - // Use the same service account name as the DaemonSet, which is based on the CR name. - saName := fmt.Sprintf("%s-gpu-dra", crName) - - if err := ensureSCC(ctx, r.Client, buildDRASCC(sccName)); err != nil { - klog.Error(err, "unable to ensure DRA SCC") - - return err - } - - if err := createSCCRole(ctx, r.Client, roleName, sccName); err != nil { - klog.Error(err, "unable to ensure DRA SCC ClusterRole") - - return err - } - - if err := createSCCRoleBinding(ctx, r.Client, bindingName, roleName, saName, r.Opts.Namespace); err != nil { - klog.Error(err, "unable to ensure DRA SCC ClusterRoleBinding") - - return err - } - - return nil -} - -func (r *DRAReconciler) anyAllocatedResourceClaims(ctx context.Context) bool { - var rcList resv1.ResourceClaimList - - klog.Info("Checking for allocated ResourceClaims that would prevent DRA removal") - - if err := r.List(ctx, &rcList); err != nil { - klog.Error(err, "unable to list ResourceClaims") - - return false - } - - klog.Infof("Found %d ResourceClaims", len(rcList.Items)) - for _, claim := range rcList.Items { - alloc := claim.Status.Allocation - - if alloc == nil { - continue - } - if len(alloc.Devices.Results) == 0 { - continue - } - - for _, dev := range alloc.Devices.Results { - if dev.Driver == gpuDeviceClass { - klog.Infof("Found allocated ResourceClaim with GPU device: %s", claim.Name) - - return true - } - } - } - - return false -} - -func addHealthCheckIfMissing(container *core.Container, port int32) { - for _, p := range container.Ports { - if p.ContainerPort == port { - // If port is already there, we assume the health check is already set up. - return - } - } - - service := "liveness" - - container.Ports = append(container.Ports, core.ContainerPort{ - Name: "healthcheck", - ContainerPort: port, - }) - - container.StartupProbe = &core.Probe{ - FailureThreshold: 60, - PeriodSeconds: 10, - TimeoutSeconds: 10, - ProbeHandler: core.ProbeHandler{ - GRPC: &core.GRPCAction{ - Port: port, - Service: &service, - }, - }, - } - - container.LivenessProbe = &core.Probe{ - FailureThreshold: 3, - PeriodSeconds: 30, - TimeoutSeconds: 10, - ProbeHandler: core.ProbeHandler{ - GRPC: &core.GRPCAction{ - Port: port, - Service: &service, - }, - }, - } -} - -func removeHealthCheckIfExists(container *core.Container) { - found := false - - for i, p := range container.Ports { - if p.Name == "healthcheck" { - container.Ports = append(container.Ports[:i], container.Ports[i+1:]...) - found = true - break - } - } - - if !found { - return - } - - container.StartupProbe = nil - container.LivenessProbe = nil -} - -func (r *DRAReconciler) updateDaemonSetObject(ds *apps.DaemonSet, spec *v1alpha.ClusterPolicy) { - name := fmt.Sprintf("%s-gpu-dra", spec.Name) - - ds.Name = name - ds.Namespace = r.Opts.Namespace - ds.Labels[ownerKey] = spec.Name - - ds.Spec.Template.Spec.ServiceAccountName = name - - dspec := spec.Spec.DynamicResourceAllocationSpec - - ds.Spec.Template.Spec.Containers[0].Image = dspec.Image - ds.Spec.Template.Spec.Containers[0].Args = r.generateArgs(spec) - - ds.Spec.Template.Spec.NodeSelector = map[string]string{ - "kubernetes.io/arch": "amd64", - } - - if len(spec.Spec.NodeSelector) > 0 { - for k, v := range spec.Spec.NodeSelector { - ds.Spec.Template.Spec.NodeSelector[k] = v - } - } - - if spec.Spec.UseNFDLabeling { - ds.Spec.Template.Spec.NodeSelector["intel.feature.node.kubernetes.io/gpu"] = trueValue - } - - if len(spec.Spec.Tolerations) > 0 { - ds.Spec.Template.Spec.Tolerations = spec.Spec.Tolerations - } else { - ds.Spec.Template.Spec.Tolerations = nil - } - - cspec := &ds.Spec.Template.Spec - - // Enable health check for the DRA Pod - if spec.Spec.DynamicResourceAllocationSpec.PodHealthCheck { - addHealthCheckIfMissing(&cspec.Containers[0], healthCheckPort) - } else { - removeHealthCheckIfExists(&cspec.Containers[0]) - } - - secrets := []core.LocalObjectReference{} - if r.Opts.SecretName != "" { - secrets = append(secrets, core.LocalObjectReference{Name: r.Opts.SecretName}) - } - if spec.Spec.PullSecret != nil { - secrets = append(secrets, *spec.Spec.PullSecret) - } - - if len(secrets) > 0 { - cspec.ImagePullSecrets = secrets - } else { - cspec.ImagePullSecrets = nil - } - - if r.Opts.OpenShift { - // On OpenShift, SELinux labels the container process as container_t which cannot - // write to host directories (e.g. /etc/cdi labeled etc_t). spc_t bypasses SELinux - // confinement for the container so it can write CDI specs to the host filesystem. - if cspec.Containers[0].SecurityContext == nil { - cspec.Containers[0].SecurityContext = &core.SecurityContext{} - } - - cspec.Containers[0].SecurityContext.SELinuxOptions = &core.SELinuxOptions{ - Type: "spc_t", - } - } -} - -func (r *DRAReconciler) createDaemonSet(ctx context.Context, spec *v1alpha.ClusterPolicy) error { - ds := deployments.DynamicResourceAllocationDaemonset() - - r.updateDaemonSetObject(ds, spec) - - if err := r.Create(ctx, ds); err != nil { - klog.Error(err, "unable to create DaemonSet") - - return err - } - - return nil -} - -func (r *DRAReconciler) removeDeploymentIfExists(ctx context.Context) (ctrl.Result, error) { - crName := r.Opts.ReqName - - dss := &apps.DaemonSetList{} - labels := client.MatchingLabels{ - appLabel: draValue, - ownerKey: crName, - } - - if err := r.List(ctx, dss, client.InNamespace(r.Opts.Namespace), labels); err != nil { - klog.Error(err, "unable to list child DaemonSets") - - return ctrl.Result{}, err - } - - if len(dss.Items) == 0 { - return ctrl.Result{}, nil - } - - // If there are any allocated ResourceClaims, removal of DRA will cause - // the Pods using them to be stuck at Terminating. - // Requeue and try again later. - if r.anyAllocatedResourceClaims(ctx) { - return ctrl.Result{RequeueAfter: r.Opts.RequeueDelay}, requeueReconcileErr{} - } - - if err := r.deleteAll(ctx, r.Opts.ReqName); err != nil { - return ctrl.Result{}, err - } - - klog.Info("DRA deployment removed") - - return ctrl.Result{}, nil -} - -func (r *DRAReconciler) generateArgs(spec *v1alpha.ClusterPolicy) []string { - targetLevel := int32(0) - - targetLevel = max(targetLevel, spec.Spec.DynamicResourceAllocationSpec.LogLevel) - targetLevel = max(targetLevel, spec.Spec.LogLevel) - - args := []string{fmt.Sprintf("-v=%d", targetLevel)} - - if spec.Spec.HealthinessSpec != nil { - args = append(args, "--health-monitoring=true") - } - - if spec.Spec.DynamicResourceAllocationSpec.PodHealthCheck { - args = append(args, fmt.Sprintf("--healthcheck-port=%d", healthCheckPort)) - } else { - args = append(args, "--healthcheck-port=-1") - } - - manageBinding := "false" - if spec.Spec.DynamicResourceAllocationSpec.ManageBinding { - manageBinding = "true" - } - - args = append(args, fmt.Sprintf("--manage-binding=%s", manageBinding)) - - return args -} - -func (r *DRAReconciler) Reconcile(ctx context.Context, cp *v1alpha.ClusterPolicy) (ctrl.Result, error) { - _ = logf.FromContext(ctx) - - // If DRA is not enable in the cluster, we shouldn't cause errors in trying to do things. - if !r.Opts.DRAEnable { - if cp != nil && cp.Spec.ResourceRegistration == resourceModeDRA { - addIfMissing(&cp.Status.Errors, draNotEnabledMsg) - } - - return ctrl.Result{}, nil - } - - if cp == nil || !cp.DeletionTimestamp.IsZero() { - return r.removeDeploymentIfExists(ctx) - } - - // DRA not selected, remove existing deployment if exists - if cp.Spec.ResourceRegistration != resourceModeDRA { - cp.Status.DRAStatus = notAvailableStatus - - return r.removeDeploymentIfExists(ctx) - } - - labels := client.MatchingLabels{appLabel: draValue} - - var olderDs apps.DaemonSetList - if err := r.List(ctx, &olderDs, client.InNamespace(r.Opts.Namespace), labels); err != nil { - klog.Error(err, "unable to list child DaemonSets") - - return ctrl.Result{}, err - } - - if r.Opts.OpenShift { - if err := r.createOpenShiftResourcesIfNotExists(ctx, cp.Name); err != nil { - klog.Error(err, "unable to ensure OpenShift resources for DRA") - - return ctrl.Result{}, err - } - } - - if len(olderDs.Items) == 0 { - return ctrl.Result{}, r.createAll(ctx, cp) - } - - // Update DaemonSet - - ds := &olderDs.Items[0] - originalDs := ds.DeepCopy() - - r.updateDaemonSetObject(ds, cp) - - dsDiff := cmp.Diff(originalDs.Spec.Template.Spec, ds.Spec.Template.Spec, cmpopts.EquateEmpty()) - if len(dsDiff) > 0 { - klog.Info("DRA difference", "diff", dsDiff) - - if err := r.Update(ctx, ds); err != nil { - klog.Error(err, "unable to update daemonset", "DaemonSet", ds) - - return ctrl.Result{}, err - } - } - - r.ensureVfioDeviceClass(ctx, cp.Spec.DynamicResourceAllocationSpec.ManageBinding) - - if err := r.List(ctx, &olderDs, client.InNamespace(r.Opts.Namespace), labels); err != nil { - klog.Error(err, "unable to list child DaemonSets") - - return ctrl.Result{}, err - } - - cp.Status.DRAStatus = fmt.Sprintf("%d/%d", - olderDs.Items[0].Status.NumberReady, olderDs.Items[0].Status.DesiredNumberScheduled) - - return ctrl.Result{}, nil -} diff --git a/internal/controller/dra_controller_test.go b/internal/controller/dra_controller_test.go deleted file mode 100644 index d34a237..0000000 --- a/internal/controller/dra_controller_test.go +++ /dev/null @@ -1,534 +0,0 @@ -/* -Copyright 2025 Intel Corporation. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "context" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - apps "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" - rbac "k8s.io/api/rbac/v1" - resv1 "k8s.io/api/resource/v1" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - - v1alpha "github.com/intel/gpu-base-operator/api/v1alpha1" - "github.com/intel/gpu-base-operator/config/deployments" -) - -var _ = Describe("ClusterPolicy Controller for DRA", func() { - - Context("When reconciling DRA", func() { - defaultNamespace := "foobar-dra" - const resourceName = "test-resource-dra" - - ctx := context.Background() - - typeNamespacedName := types.NamespacedName{ - Name: resourceName, - } - clusterpolicy := &v1alpha.ClusterPolicy{} - - BeforeEach(func() { - ns := &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaultNamespace, - }, - } - - Expect(k8sClient.Create(ctx, ns)).To(Succeed()) - }) - - AfterEach(func() { - resource := &v1alpha.ClusterPolicy{} - err := k8sClient.Get(ctx, typeNamespacedName, resource) - Expect(err).NotTo(HaveOccurred()) - - By("Cleanup the specific resource instance ClusterPolicy") - Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) - }) - - It("should successfully reconcile the resource", func() { - By("creating the custom resource for the Kind ClusterPolicy") - err := k8sClient.Get(ctx, typeNamespacedName, clusterpolicy) - if err != nil && errors.IsNotFound(err) { - resource := &v1alpha.ClusterPolicy{ - ObjectMeta: metav1.ObjectMeta{ - Name: resourceName, - }, - Spec: v1alpha.ClusterPolicySpec{ - ResourceRegistration: "dra", - DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ - Image: "dra-image:v1.2.3", - }, - }, - } - Expect(k8sClient.Create(ctx, resource)).To(Succeed()) - } - - By("Reconciling the created resource") - controllerReconciler := &ClusterPolicyReconciler{ - Client: k8sClient, - Scheme: k8sClient.Scheme(), - Opts: ControllerOpts{ - Namespace: defaultNamespace, - DRAEnable: true, - RequeueDelay: time.Millisecond * 50, - }, - } - - _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ - NamespacedName: typeNamespacedName, - }) - - Expect(err).NotTo(HaveOccurred()) - - daemonSets := apps.DaemonSetList{} - err = k8sClient.List(ctx, &daemonSets, client.InNamespace(defaultNamespace)) - Expect(err).NotTo(HaveOccurred()) - - Expect(daemonSets.Items).To(HaveLen(1)) - - for _, ds := range daemonSets.Items { - switch ds.Name { - case "test-resource-dra-gpu-dra": - Expect(ds.Spec.Template.Spec.Containers[0].Image).To(Equal("dra-image:v1.2.3")) - default: - Fail("Unexpected DaemonSet found: " + ds.Name) - } - } - }) - }) - - Context("When reconciling DRA and XPUM", func() { - defaultNamespace := "foobar-dra-xpum" - const resourceName = "test-resource-dra-xpum" - - ctx := context.Background() - - typeNamespacedName := types.NamespacedName{ - Name: resourceName, - } - clusterpolicy := &v1alpha.ClusterPolicy{} - - BeforeEach(func() { - ns := &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaultNamespace, - Labels: map[string]string{ - "resource.kubernetes.io/admin-access": "true", - }, - }, - } - - Expect(k8sClient.Create(ctx, ns)).To(Succeed()) - }) - - AfterEach(func() { - resource := &v1alpha.ClusterPolicy{} - err := k8sClient.Get(ctx, typeNamespacedName, resource) - Expect(err).NotTo(HaveOccurred()) - - By("Cleanup the specific resource instance ClusterPolicy") - Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) - }) - - It("should successfully reconcile the resource", func() { - By("creating the custom resource for the Kind ClusterPolicy") - err := k8sClient.Get(ctx, typeNamespacedName, clusterpolicy) - if err != nil && errors.IsNotFound(err) { - resource := &v1alpha.ClusterPolicy{ - ObjectMeta: metav1.ObjectMeta{ - Name: resourceName, - }, - Spec: v1alpha.ClusterPolicySpec{ - ResourceRegistration: "dra", - ResourceMonitoring: true, - UseNFDLabeling: true, - DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ - Image: "dra-image:v1.2.3", - }, - XpuManagerSpec: v1alpha.XpuManagerSpec{ - Image: "xpumanager:v3.2.1", - LogLevel: 3, - }, - }, - } - Expect(k8sClient.Create(ctx, resource)).To(Succeed()) - } - - By("Reconciling the created resource") - controllerReconciler := &ClusterPolicyReconciler{ - Client: k8sClient, - Scheme: k8sClient.Scheme(), - Opts: ControllerOpts{ - Namespace: defaultNamespace, - DRAEnable: true, - RequeueDelay: time.Millisecond * 50, - }, - } - - _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ - NamespacedName: typeNamespacedName, - }) - - Expect(err).NotTo(HaveOccurred()) - - daemonSets := apps.DaemonSetList{} - err = k8sClient.List(ctx, &daemonSets, client.InNamespace(defaultNamespace)) - Expect(err).NotTo(HaveOccurred()) - - Expect(daemonSets.Items).To(HaveLen(2)) - - for _, ds := range daemonSets.Items { - switch ds.Name { - case "test-resource-dra-xpum-gpu-dra": - Expect(ds.Spec.Template.Spec.Containers[0].Image).To(Equal("dra-image:v1.2.3")) - Expect(ds.Spec.Template.Spec.NodeSelector).To(HaveKeyWithValue("intel.feature.node.kubernetes.io/gpu", "true")) - Expect(ds.Spec.Template.Spec.NodeSelector).To(HaveKeyWithValue("kubernetes.io/arch", "amd64")) - case "test-resource-dra-xpum-xpu-manager": - Expect(ds.Spec.Template.Spec.Containers[0].Image).To(Equal("xpumanager:v3.2.1")) - Expect(ds.Spec.Template.Spec.NodeSelector).To(HaveKeyWithValue("intel.feature.node.kubernetes.io/gpu", "true")) - Expect(ds.Spec.Template.Spec.NodeSelector).To(HaveKeyWithValue("kubernetes.io/arch", "amd64")) - default: - Fail("Unexpected DaemonSet found: " + ds.Name) - } - } - - dcList := &resv1.DeviceClassList{} - err = k8sClient.List(ctx, dcList) - Expect(err).NotTo(HaveOccurred()) - Expect(dcList.Items).To(HaveLen(2)) - for _, dc := range dcList.Items { - switch dc.Name { - case "gpu.intel.com": - case "gpu-vfio.intel.com": - Expect(dc.Spec.Selectors).To(HaveLen(2)) - default: - Fail("Unexpected DeviceClass found: " + dc.Name) - } - } - - Expect(k8sClient.Get(ctx, typeNamespacedName, clusterpolicy)).To(Succeed(), "Failed to get ClusterPolicy after reconciliation") - - clusterpolicy.Spec.DynamicResourceAllocationSpec.ManageBinding = true - Expect(k8sClient.Update(ctx, clusterpolicy)).To(Succeed(), "Failed to update ClusterPolicy with ManageBinding change") - - _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ - NamespacedName: typeNamespacedName, - }) - Expect(err).NotTo(HaveOccurred()) - - err = k8sClient.List(ctx, dcList) - Expect(err).NotTo(HaveOccurred()) - Expect(dcList.Items).To(HaveLen(2)) - - for _, dc := range dcList.Items { - switch dc.Name { - case "gpu.intel.com": - case "gpu-vfio.intel.com": - Expect(dc.Spec.Selectors).To(HaveLen(1)) - default: - Fail("Unexpected DeviceClass found: " + dc.Name) - } - } - }) - }) - - Context("When reconciling DRA on OpenShift", func() { - defaultNamespace := "foobar-dra-ocp" - const resourceName = "test-resource-dra-ocp" - - ctx := context.Background() - - typeNamespacedName := types.NamespacedName{Name: resourceName} - - BeforeEach(func() { - Expect(k8sClient.Create(ctx, &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{Name: defaultNamespace}, - })).To(Succeed()) - }) - - AfterEach(func() { - resource := &v1alpha.ClusterPolicy{} - Expect(k8sClient.Get(ctx, typeNamespacedName, resource)).To(Succeed()) - Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) - - deleteOpenShiftSCCResources(ctx, k8sClient, - resourceName+"-gpu-dra-scc", - resourceName+"-gpu-dra-scc-role", - resourceName+"-gpu-dra-scc-binding", - "", "") - }) - - It("creates DRA SCC resources and cleans them up on DRA removal", func() { - By("creating the ClusterPolicy with DRA mode") - Expect(k8sClient.Create(ctx, &v1alpha.ClusterPolicy{ - ObjectMeta: metav1.ObjectMeta{Name: resourceName}, - Spec: v1alpha.ClusterPolicySpec{ - ResourceRegistration: "dra", - DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ - Image: "intel/gpu-dra-driver:test", - }, - }, - })).To(Succeed()) - - reconciler := &ClusterPolicyReconciler{ - Client: k8sClient, - Scheme: k8sClient.Scheme(), - Opts: ControllerOpts{ - Namespace: defaultNamespace, - DRAEnable: true, - OpenShift: true, - RequeueDelay: time.Millisecond * 50, - }, - } - - By("reconcile creates DRA SCC resources") - _, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) - Expect(err).NotTo(HaveOccurred()) - - By("DRA SCC is created") - draSCC := &unstructured.Unstructured{} - draSCC.SetAPIVersion(sccAPIVersion) - draSCC.SetKind(sccKind) - Expect(k8sClient.Get(ctx, client.ObjectKey{Name: resourceName + "-gpu-dra-scc"}, draSCC)).To(Succeed()) - - By("DRA SCC ClusterRole is created") - Expect(k8sClient.Get(ctx, client.ObjectKey{Name: resourceName + "-gpu-dra-scc-role"}, &rbac.ClusterRole{})).To(Succeed()) - - By("DRA SCC ClusterRoleBinding is created") - Expect(k8sClient.Get(ctx, client.ObjectKey{Name: resourceName + "-gpu-dra-scc-binding"}, &rbac.ClusterRoleBinding{})).To(Succeed()) - - By("second reconcile is idempotent") - _, err = reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) - Expect(err).NotTo(HaveOccurred()) - - By("creating a ResourceClaim exercises anyAllocatedResourceClaims loop") - resourceClaim := &resv1.ResourceClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-rc-coverage", - Namespace: defaultNamespace, - }, - Spec: resv1.ResourceClaimSpec{ - Devices: resv1.DeviceClaim{ - Requests: []resv1.DeviceRequest{ - {Name: "gpu", Exactly: &resv1.ExactDeviceRequest{DeviceClassName: gpuDeviceClass}}, - }, - }, - }, - } - Expect(k8sClient.Create(ctx, resourceClaim)).To(Succeed()) - DeferCleanup(func() { _ = k8sClient.Delete(ctx, resourceClaim) }) - - By("switching to DP mode removes DRA SCC resources") - cp := &v1alpha.ClusterPolicy{} - Expect(k8sClient.Get(ctx, typeNamespacedName, cp)).To(Succeed()) - cp.Spec.ResourceRegistration = "dp" - cp.Spec.DevicePluginSpec = v1alpha.DevicePluginSpec{ - PluginImage: "intel/intel-gpu-plugin:test", - } - Expect(k8sClient.Update(ctx, cp)).To(Succeed()) - - _, err = reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) - Expect(err).NotTo(HaveOccurred()) - - draSCCCheck := &unstructured.Unstructured{} - draSCCCheck.SetAPIVersion(sccAPIVersion) - draSCCCheck.SetKind(sccKind) - Expect(k8sClient.Get(ctx, client.ObjectKey{Name: resourceName + "-gpu-dra-scc"}, draSCCCheck)). - To(Satisfy(errors.IsNotFound)) - }) - }) - - Context("DaemonSet object update", func() { - It("with tolerations, nodeSelector and pullsecret", func() { - cp := &v1alpha.ClusterPolicy{ - Spec: v1alpha.ClusterPolicySpec{ - LogLevel: 2, - DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ - LogLevel: 3, - PodHealthCheck: true, - }, - Tolerations: []v1.Toleration{ - { - Key: "my-toleration", - Operator: v1.TolerationOpExists, - Effect: v1.TaintEffectNoExecute, - }, - }, - PullSecret: &v1.LocalObjectReference{ - Name: "my-pull-secret", - }, - NodeSelector: map[string]string{ - "foo": "bar", - }, - }, - } - controller := &DRAReconciler{} - - ds := deployments.DynamicResourceAllocationDaemonset() - controller.updateDaemonSetObject(ds, cp) - - Expect(ds.Spec.Template.Spec.NodeSelector).To(HaveKeyWithValue("foo", "bar")) - - Expect(ds.Spec.Template.Spec.Tolerations[0].Key).To(Equal("my-toleration")) - Expect(ds.Spec.Template.Spec.Tolerations[0].Operator).To(Equal(v1.TolerationOpExists)) - Expect(ds.Spec.Template.Spec.Tolerations[0].Effect).To(Equal(v1.TaintEffectNoExecute)) - - Expect(ds.Spec.Template.Spec.ImagePullSecrets[0].Name).To(Equal("my-pull-secret")) - - Expect(ds.Spec.Template.Spec.Containers[0].Args).To(ContainElement("--healthcheck-port=51516")) - Expect(ds.Spec.Template.Spec.Containers[0].Ports).To(ContainElement(v1.ContainerPort{ - Name: "healthcheck", - ContainerPort: 51516, - })) - - Expect(ds.Spec.Template.Spec.Containers[0].StartupProbe).ToNot(BeNil()) - Expect(ds.Spec.Template.Spec.Containers[0].StartupProbe.GRPC).ToNot(BeNil()) - Expect(ds.Spec.Template.Spec.Containers[0].StartupProbe.GRPC.Port).To(Equal(int32(51516))) - - Expect(ds.Spec.Template.Spec.Containers[0].LivenessProbe).ToNot(BeNil()) - Expect(ds.Spec.Template.Spec.Containers[0].LivenessProbe.GRPC).ToNot(BeNil()) - Expect(ds.Spec.Template.Spec.Containers[0].LivenessProbe.GRPC.Port).To(Equal(int32(51516))) - - // Update with no changes - controller.updateDaemonSetObject(ds, cp) - Expect(ds.Spec.Template.Spec.Containers[0].StartupProbe).ToNot(BeNil()) - Expect(ds.Spec.Template.Spec.Containers[0].LivenessProbe).ToNot(BeNil()) - - cp.Spec.PullSecret = nil - cp.Spec.Tolerations = nil - cp.Spec.DynamicResourceAllocationSpec.PodHealthCheck = false - controller.updateDaemonSetObject(ds, cp) - - Expect(ds.Spec.Template.Spec.Tolerations).To(BeEmpty()) - Expect(ds.Spec.Template.Spec.ImagePullSecrets).To(BeEmpty()) - Expect(ds.Spec.Template.Spec.Containers[0].Args).To(ContainElement("--healthcheck-port=-1")) - Expect(ds.Spec.Template.Spec.Containers[0].StartupProbe).To(BeNil()) - Expect(ds.Spec.Template.Spec.Containers[0].LivenessProbe).To(BeNil()) - }) - - It("preserves kubelet-plugin container security context from YAML", func() { - cp := &v1alpha.ClusterPolicy{ - Spec: v1alpha.ClusterPolicySpec{ - DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ - Image: "intel/gpu-dra-driver:test", - }, - }, - } - controller := &DRAReconciler{} - - ds := deployments.DynamicResourceAllocationDaemonset() - controller.updateDaemonSetObject(ds, cp) - - var kubeletPlugin *v1.Container - for i := range ds.Spec.Template.Spec.Containers { - if ds.Spec.Template.Spec.Containers[i].Name == "kubelet-plugin" { - kubeletPlugin = &ds.Spec.Template.Spec.Containers[i] - break - } - } - Expect(kubeletPlugin).NotTo(BeNil()) - Expect(kubeletPlugin.SecurityContext).NotTo(BeNil()) - Expect(kubeletPlugin.SecurityContext.AllowPrivilegeEscalation).NotTo(BeNil()) - Expect(*kubeletPlugin.SecurityContext.AllowPrivilegeEscalation).To(BeFalse()) - }) - }) - - Context("DRA arguments handling", func() { - It("with health monitoring", func() { - cp := &v1alpha.ClusterPolicy{ - Spec: v1alpha.ClusterPolicySpec{ - LogLevel: 2, - DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ - LogLevel: 3, - PodHealthCheck: false, - }, - HealthinessSpec: &v1alpha.HealthinessSpec{ - CheckIntervalSeconds: 67, - CoreTemperatureThreshold: 42, - MemoryTemperatureThreshold: 45, - }, - }, - } - controller := &DRAReconciler{} - - args := controller.generateArgs(cp) - - Expect(args).To(HaveLen(4)) - Expect(args).To(ContainElement("-v=3")) - Expect(args).To(ContainElement("--health-monitoring=true")) - Expect(args).To(ContainElement("--healthcheck-port=-1")) - Expect(args).To(ContainElement("--manage-binding=false")) - }) - - It("with device taint", func() { - cp := &v1alpha.ClusterPolicy{ - Spec: v1alpha.ClusterPolicySpec{ - LogLevel: 2, - DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ - LogLevel: 3, - PodHealthCheck: true, - }, - HealthinessSpec: &v1alpha.HealthinessSpec{ - CheckIntervalSeconds: 67, - CoreTemperatureThreshold: 42, - MemoryTemperatureThreshold: 45, - }, - }, - } - controller := &DRAReconciler{} - - args := controller.generateArgs(cp) - - Expect(args).To(HaveLen(4)) - Expect(args).To(ContainElement("-v=3")) - Expect(args).To(ContainElement("--health-monitoring=true")) - Expect(args).To(ContainElement("--healthcheck-port=51516")) - Expect(args).To(ContainElement("--manage-binding=false")) - }) - - It("with driver bind management", func() { - cp := &v1alpha.ClusterPolicy{ - Spec: v1alpha.ClusterPolicySpec{ - LogLevel: 2, - DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ - LogLevel: 3, - PodHealthCheck: true, - ManageBinding: true, - }, - }, - } - controller := &DRAReconciler{} - - args := controller.generateArgs(cp) - - Expect(args).To(HaveLen(3)) - Expect(args).To(ContainElement("-v=3")) - Expect(args).To(ContainElement("--healthcheck-port=51516")) - Expect(args).To(ContainElement("--manage-binding=true")) - }) - }) -}) diff --git a/internal/controller/kmm_controller.go b/internal/controller/kmm_controller.go new file mode 100644 index 0000000..44bd8cf --- /dev/null +++ b/internal/controller/kmm_controller.go @@ -0,0 +1,467 @@ +/* +Copyright 2026 Intel Corporation. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "fmt" + "strconv" + "strings" + + v1 "k8s.io/api/core/v1" + resourcev1 "k8s.io/api/resource/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + v1alpha "github.com/intel/gpu-base-operator/api/v1alpha1" + kmmv1beta1 "github.com/kubernetes-sigs/kernel-module-management/api/v1beta1" +) + +// +kubebuilder:rbac:groups=kmm.sigs.x-k8s.io,resources=modules,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=kmm.sigs.x-k8s.io,resources=modules/status,verbs=get + +// KMMReconciler manages a KMM Module CR that delegates DP/DRA DaemonSet lifecycle +// to the Kernel Module Management operator, replacing direct DaemonSet management. +type KMMReconciler struct { + client.Client + Scheme *runtime.Scheme + Opts ControllerOpts +} + +const ( + kmmModuleSuffix = "-gpu" + + healthCheckPort = 51516 + gpuDeviceClass = "gpu.intel.com" + vfioGpuDeviceClass = "gpu-vfio.intel.com" + + kmmNotEnabledMsg = "KMM is not installed in the cluster." +) + +var hostPathDirOrCreate = v1.HostPathDirectoryOrCreate + +func logLevelForDp(spec *v1alpha.ClusterPolicy) int32 { + return max(spec.Spec.LogLevel, spec.Spec.DevicePluginSpec.LogLevel) +} + +func hexArgStr(s []string) string { + a := make([]string, 0, len(s)) + for _, str := range s { + if !strings.HasPrefix(str, "0x") { + str = "0x" + str + } + + a = append(a, str) + } + return strings.Join(a, ",") +} + +func dpArgs(spec *v1alpha.ClusterPolicy) []string { + var args []string + + dpspec := spec.Spec.DevicePluginSpec + + if spec.Spec.ResourceMonitoring { + args = append(args, + "-enable-monitoring", + "-xpumd-endpoint=/run/xpumd/intelxpuinfo.sock") + } + + logLevel := logLevelForDp(spec) + if logLevel > 0 { + args = append(args, fmt.Sprintf("-v=%d", logLevel)) + } + + if len(dpspec.ByPathMode) > 0 { + args = append(args, fmt.Sprintf("-bypath=%s", dpspec.ByPathMode)) + } + + if len(dpspec.AllowIDs) > 0 { + args = append(args, fmt.Sprintf("-allow-ids=%s", hexArgStr(dpspec.AllowIDs))) + } + + if len(dpspec.DenyIDs) > 0 { + args = append(args, fmt.Sprintf("-deny-ids=%s", hexArgStr(dpspec.DenyIDs))) + } + + return args +} + +func kmmModuleName(cpName string) string { + return cpName + kmmModuleSuffix +} + +func (r *KMMReconciler) Reconcile(ctx context.Context, cp *v1alpha.ClusterPolicy) (ctrl.Result, error) { + moduleName := kmmModuleName(r.Opts.ReqName) + + if !r.Opts.KMMEnable { + if cp != nil { + addIfMissing(&cp.Status.Errors, kmmNotEnabledMsg) + } + + return ctrl.Result{}, nil + } + + if !r.Opts.DRAEnable && cp != nil && cp.Spec.ResourceRegistration == resourceModeDRA { + addIfMissing(&cp.Status.Errors, "DRA is not enabled in the cluster, but ClusterPolicy requests it.") + return ctrl.Result{}, nil + } + + if cp == nil || !cp.DeletionTimestamp.IsZero() { + return r.deleteModuleIfExists(ctx, moduleName) + } + + mod := &kmmv1beta1.Module{ + ObjectMeta: metav1.ObjectMeta{ + Name: moduleName, + Namespace: r.Opts.Namespace, + }, + } + + result, err := controllerutil.CreateOrPatch(ctx, r.Client, mod, func() error { + return r.setModuleDesiredState(mod, cp) + }) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to reconcile KMM Module %s for ClusterPolicy %s: %w", moduleName, cp.Name, err) + } + + klog.Infof("KMM Module %s %s", moduleName, result) + + r.updateStatus(cp, mod) + + return ctrl.Result{}, nil +} + +func (r *KMMReconciler) setModuleDesiredState(mod *kmmv1beta1.Module, cp *v1alpha.ClusterPolicy) error { + if err := ctrl.SetControllerReference(cp, mod, r.Scheme); err != nil { + return fmt.Errorf("failed to set controller reference: %w", err) + } + + mod.Spec.Selector = r.buildNodeSelector(cp) + + mod.Spec.ImageRepoSecret = cp.Spec.PullSecret + mod.Spec.Tolerations = cp.Spec.Tolerations + + r.setModuleLoader(mod, cp) + + switch cp.Spec.ResourceRegistration { + case resourceModeDRA: + r.setDRA(mod, cp) + case resourceModeDP: + r.setDevicePlugin(mod, cp) + } + + return nil +} + +func (r *KMMReconciler) buildNodeSelector(cp *v1alpha.ClusterPolicy) map[string]string { + return gpuNodeSelector(cp) +} + +func (r *KMMReconciler) setModuleLoader(mod *kmmv1beta1.Module, cp *v1alpha.ClusterPolicy) { + if cp.Spec.KernelModule == nil { + mod.Spec.ModuleLoader = nil + return + } + + km := cp.Spec.KernelModule + + container := kmmv1beta1.ModuleLoaderContainerSpec{ + Modprobe: kmmv1beta1.ModprobeSpec{ + ModuleName: km.ModuleName, + FirmwarePath: km.FirmwarePath, + ModulesLoadingOrder: km.ModulesLoadingOrder, + }, + ContainerImage: km.Image, + InTreeModulesToRemove: dedupeStrings(append([]string{km.ModuleName}, km.InTreeModulesToRemove...)), + ImagePullPolicy: v1.PullAlways, + RegistryTLS: kmmv1beta1.TLSOptions{ + InsecureSkipTLSVerify: km.SkipTLSVerify, + }, + } + + if len(km.KernelMappings) == 0 { + container.KernelMappings = []kmmv1beta1.KernelMapping{ + {Regexp: "^.+$"}, + } + } else { + mappings := make([]kmmv1beta1.KernelMapping, 0, len(km.KernelMappings)) + for _, m := range km.KernelMappings { + mapping := kmmv1beta1.KernelMapping{ + Regexp: m.Regexp, + Literal: m.Literal, + ContainerImage: m.ContainerImage, + InTreeModulesToRemove: m.InTreeModulesToRemove, + } + if m.Build != nil { + mapping.Build = convertBuildSpec(m.Build) + } + mappings = append(mappings, mapping) + } + container.KernelMappings = mappings + } + + mod.Spec.ModuleLoader = &kmmv1beta1.ModuleLoaderSpec{ + Container: container, + ServiceAccountName: r.Opts.ModuleLoaderServiceAccountName, + } +} + +func convertBuildSpec(src *v1alpha.KernelModuleBuildSpec) *kmmv1beta1.Build { + build := &kmmv1beta1.Build{ + DockerfileConfigMap: &v1.LocalObjectReference{Name: src.DockerfileConfigMap.Name}, + Secrets: src.Secrets, + } + + if len(src.BuildArgs) > 0 { + args := make([]kmmv1beta1.BuildArg, len(src.BuildArgs)) + for i, a := range src.BuildArgs { + args[i] = kmmv1beta1.BuildArg{Name: a.Name, Value: a.Value} + } + build.BuildArgs = args + } + + return build +} + +func dedupeStrings(s []string) []string { + seen := make(map[string]bool, len(s)) + result := make([]string, 0, len(s)) + + for _, v := range s { + if !seen[v] { + seen[v] = true + result = append(result, v) + } + } + + return result +} + +func (r *KMMReconciler) setDRA(mod *kmmv1beta1.Module, cp *v1alpha.ClusterPolicy) { + mod.Spec.DevicePlugin = nil + + celExpression := fmt.Sprintf("device.driver == %q", gpuDeviceClass) + + args := r.generateDRAArgs(cp) + + // KMM's DRA reconciler automatically adds: kubelet-plugins, kubelet-plugins-registry, + // cdi (/var/run/cdi) volumes/mounts and NODE_NAME, CDI_ROOT, POD_UID env vars. + // We only include what KMM doesn't provide. + mod.Spec.DRA = &kmmv1beta1.DRASpec{ + DriverName: gpuDeviceClass, + ServiceAccountName: r.Opts.DRAServiceAccountName, + Container: kmmv1beta1.CommonContainerSpec{ + Image: cp.Spec.DynamicResourceAllocationSpec.Image, + ImagePullPolicy: v1.PullIfNotPresent, + Command: []string{"/kubelet-gpu-plugin"}, + Args: args, + Env: []v1.EnvVar{ + { + Name: "POD_NAMESPACE", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, + }, + }, + { + Name: "SYSFS_ROOT", + Value: "/sysfs", + }, + }, + VolumeMounts: []v1.VolumeMount{ + {Name: "xpumdrundir", MountPath: "/var/run/xpumd"}, + {Name: "sysfs", MountPath: "/sysfs"}, + }, + }, + Volumes: draExtraVolumes(), + DeviceClasses: []kmmv1beta1.DeviceClassSpec{ + { + Name: gpuDeviceClass, + Selectors: []resourcev1.DeviceSelector{ + {CEL: &resourcev1.CELDeviceSelector{Expression: celExpression}}, + }, + }, + { + Name: vfioGpuDeviceClass, + Selectors: []resourcev1.DeviceSelector{ + {CEL: &resourcev1.CELDeviceSelector{Expression: celExpression}}, + }, + }, + }, + } +} + +func (r *KMMReconciler) generateDRAArgs(cp *v1alpha.ClusterPolicy) []string { + targetLevel := max(cp.Spec.DynamicResourceAllocationSpec.LogLevel, cp.Spec.LogLevel) + + args := []string{fmt.Sprintf("-v=%d", targetLevel)} + + if cp.Spec.HealthinessSpec != nil { + args = append(args, "--health-monitoring=true") + } + + if cp.Spec.DynamicResourceAllocationSpec.PodHealthCheck { + args = append(args, fmt.Sprintf("--healthcheck-port=%d", healthCheckPort)) + } else { + args = append(args, "--healthcheck-port=-1") + } + + args = append(args, fmt.Sprintf("--manage-binding=%s", strconv.FormatBool(cp.Spec.DynamicResourceAllocationSpec.ManageBinding))) + + return args +} + +// draExtraVolumes returns only the volumes KMM's DRA reconciler doesn't add automatically. +// KMM auto-adds: kubelet-plugins, kubelet-plugins-registry, cdi (/var/run/cdi). +func draExtraVolumes() []v1.Volume { + return []v1.Volume{ + {Name: "sysfs", VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/sys"}}}, + {Name: "xpumdrundir", VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/var/run/xpumd", Type: &hostPathDirOrCreate}}}, + } +} + +func (r *KMMReconciler) setDevicePlugin(mod *kmmv1beta1.Module, cp *v1alpha.ClusterPolicy) { + mod.Spec.DRA = nil + + mod.Spec.DevicePlugin = &kmmv1beta1.DevicePluginSpec{ + ServiceAccountName: r.Opts.DPServiceAccountName, + Container: kmmv1beta1.CommonContainerSpec{ + Image: cp.Spec.DevicePluginSpec.PluginImage, + ImagePullPolicy: v1.PullIfNotPresent, + Args: dpArgs(cp), + Env: []v1.EnvVar{ + { + Name: "NODE_NAME", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, + }, + }, + { + Name: "HOST_IP", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{FieldPath: "status.hostIP"}, + }, + }, + }, + VolumeMounts: []v1.VolumeMount{ + {Name: "devfs", MountPath: "/dev/dri", ReadOnly: true}, + {Name: "sysfsdrm", MountPath: "/sys/class/drm", ReadOnly: true}, + {Name: "cdipath", MountPath: "/var/run/cdi"}, + }, + }, + Volumes: []v1.Volume{ + {Name: "devfs", VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/dev/dri"}}}, + {Name: "sysfsdrm", VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/sys/class/drm"}}}, + {Name: "cdipath", VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/var/run/cdi", Type: &hostPathDirOrCreate}}}, + }, + } + + if cp.Spec.ResourceMonitoring { + mod.Spec.DevicePlugin.Volumes = append(mod.Spec.DevicePlugin.Volumes, v1.Volume{ + Name: xpumdVolumeName, + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/run/xpumd", + Type: &hostPathDirOrCreate, + }, + }, + }) + mod.Spec.DevicePlugin.Container.VolumeMounts = append(mod.Spec.DevicePlugin.Container.VolumeMounts, v1.VolumeMount{ + Name: xpumdVolumeName, + MountPath: "/run/xpumd", + }) + } +} + +func (r *KMMReconciler) deleteModuleIfExists(ctx context.Context, name string) (ctrl.Result, error) { + mod := &kmmv1beta1.Module{} + key := types.NamespacedName{Name: name, Namespace: r.Opts.Namespace} + + if err := r.Get(ctx, key, mod); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{}, nil + } + + return ctrl.Result{}, fmt.Errorf("failed to get KMM Module %s: %w", name, err) + } + + if mod.Spec.DRA != nil && r.anyAllocatedResourceClaims(ctx, mod.Spec.DRA.DriverName) { + return ctrl.Result{RequeueAfter: r.Opts.RequeueDelay}, requeueReconcileErr{} + } + + klog.Infof("Deleting KMM Module %s", name) + + if err := r.Delete(ctx, mod); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, fmt.Errorf("failed to delete KMM Module %s: %w", name, err) + } + + return ctrl.Result{}, nil +} + +func (r *KMMReconciler) anyAllocatedResourceClaims(ctx context.Context, driverName string) bool { + var rcList resourcev1.ResourceClaimList + + klog.Info("Checking for allocated ResourceClaims that would prevent DRA removal") + + // Fail safe: assume claims exist so we don't delete the Module while pods may still be using GPUs. + if err := r.List(ctx, &rcList); err != nil { + klog.Error(err, "unable to list ResourceClaims, assuming allocated claims exist") + return true + } + + for _, claim := range rcList.Items { + alloc := claim.Status.Allocation + if alloc == nil || len(alloc.Devices.Results) == 0 { + continue + } + + for _, dev := range alloc.Devices.Results { + if dev.Driver == driverName { + klog.Infof("Found allocated ResourceClaim with GPU device: %s", claim.Name) + return true + } + } + } + + return false +} + +func (r *KMMReconciler) updateStatus(cp *v1alpha.ClusterPolicy, mod *kmmv1beta1.Module) { + switch cp.Spec.ResourceRegistration { + case resourceModeDRA: + dsStatus := mod.Status.DRA + cp.Status.DRAStatus = fmt.Sprintf("%d/%d", dsStatus.AvailableNumber, dsStatus.DesiredNumber) + cp.Status.DevicePluginStatus = notAvailableStatus + case resourceModeDP: + dsStatus := mod.Status.DevicePlugin + cp.Status.DevicePluginStatus = fmt.Sprintf("%d/%d", dsStatus.AvailableNumber, dsStatus.DesiredNumber) + cp.Status.DRAStatus = notAvailableStatus + } + + if cp.Spec.KernelModule != nil { + mlStatus := mod.Status.ModuleLoader + cp.Status.KMMStatus = fmt.Sprintf("%d/%d", mlStatus.AvailableNumber, mlStatus.DesiredNumber) + } else { + cp.Status.KMMStatus = notAvailableStatus + } +} diff --git a/internal/controller/kmm_controller_test.go b/internal/controller/kmm_controller_test.go new file mode 100644 index 0000000..bb62ba5 --- /dev/null +++ b/internal/controller/kmm_controller_test.go @@ -0,0 +1,761 @@ +/* +Copyright 2026 Intel Corporation. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + v1alpha "github.com/intel/gpu-base-operator/api/v1alpha1" + kmmv1beta1 "github.com/kubernetes-sigs/kernel-module-management/api/v1beta1" +) + +var _ = Describe("KMM Controller", func() { + Context("When creating a KMM Module in DRA mode", func() { + const ( + namespace = "kmm-dra-create" + resourceName = "kmm-dra-create" + ) + + ctx := context.Background() + typeNamespacedName := types.NamespacedName{Name: resourceName} + + BeforeEach(func() { + Expect(k8sClient.Create(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: namespace}, + })).To(Succeed()) + }) + + AfterEach(func() { + resource := &v1alpha.ClusterPolicy{} + if err := k8sClient.Get(ctx, typeNamespacedName, resource); err == nil { + Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + } + }) + + It("should create a KMM Module CR with DRA spec", func() { + By("creating a ClusterPolicy with DRA") + cp := &v1alpha.ClusterPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: resourceName}, + Spec: v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + UseNFDLabeling: true, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + PodHealthCheck: true, + }, + }, + } + Expect(k8sClient.Create(ctx, cp)).To(Succeed()) + + By("reconciling") + reconciler := &ClusterPolicyReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + Opts: ControllerOpts{ + Namespace: namespace, + DRAEnable: true, + KMMEnable: true, + DRAServiceAccountName: "intel-gpu-dra", + }, + } + + _, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) + Expect(err).NotTo(HaveOccurred()) + + By("verifying KMM Module was created") + mod := &kmmv1beta1.Module{} + modKey := types.NamespacedName{Name: resourceName + kmmModuleSuffix, Namespace: namespace} + Expect(k8sClient.Get(ctx, modKey, mod)).To(Succeed()) + + By("verifying selector includes NFD label and arch") + Expect(mod.Spec.Selector).To(HaveKeyWithValue("intel.feature.node.kubernetes.io/gpu", "true")) + Expect(mod.Spec.Selector).To(HaveKeyWithValue("kubernetes.io/arch", "amd64")) + + By("verifying ModuleLoader is nil for in-tree driver") + Expect(mod.Spec.ModuleLoader).To(BeNil()) + + By("verifying DRA spec is set") + Expect(mod.Spec.DRA).NotTo(BeNil()) + Expect(mod.Spec.DRA.DriverName).To(Equal("gpu.intel.com")) + Expect(mod.Spec.DRA.Container.Image).To(Equal("ghcr.io/intel/gpu-dra:v0.11.0")) + Expect(mod.Spec.DRA.Container.Command).To(Equal([]string{"/kubelet-gpu-plugin"})) + + By("verifying DevicePlugin is nil") + Expect(mod.Spec.DevicePlugin).To(BeNil()) + + By("verifying DRA device classes") + Expect(mod.Spec.DRA.DeviceClasses).To(HaveLen(2)) + Expect(mod.Spec.DRA.DeviceClasses[0].Name).To(Equal("gpu.intel.com")) + Expect(mod.Spec.DRA.DeviceClasses[1].Name).To(Equal("gpu-vfio.intel.com")) + + By("verifying DRA extra volumes (KMM adds plugins/registry/cdi automatically)") + Expect(mod.Spec.DRA.Volumes).To(HaveLen(2)) + + By("verifying owner reference is set") + Expect(mod.OwnerReferences).To(HaveLen(1)) + Expect(mod.OwnerReferences[0].Name).To(Equal(resourceName)) + }) + }) + + Context("When updating a KMM Module after ClusterPolicy changes", func() { + const ( + namespace = "kmm-dra-update" + resourceName = "kmm-dra-update" + ) + + ctx := context.Background() + typeNamespacedName := types.NamespacedName{Name: resourceName} + + BeforeEach(func() { + Expect(k8sClient.Create(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: namespace}, + })).To(Succeed()) + }) + + AfterEach(func() { + resource := &v1alpha.ClusterPolicy{} + if err := k8sClient.Get(ctx, typeNamespacedName, resource); err == nil { + Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + } + }) + + It("should update the Module selector", func() { + cp := &v1alpha.ClusterPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: resourceName}, + Spec: v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + PodHealthCheck: true, + }, + }, + } + Expect(k8sClient.Create(ctx, cp)).To(Succeed()) + + reconciler := &ClusterPolicyReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + Opts: ControllerOpts{ + Namespace: namespace, + DRAEnable: true, + KMMEnable: true, + DRAServiceAccountName: "intel-gpu-dra", + }, + } + + _, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) + Expect(err).NotTo(HaveOccurred()) + + By("updating the ClusterPolicy to add NodeSelector") + Expect(k8sClient.Get(ctx, typeNamespacedName, cp)).To(Succeed()) + cp.Spec.NodeSelector = map[string]string{"zone": "gpu-pool"} + Expect(k8sClient.Update(ctx, cp)).To(Succeed()) + + _, err = reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) + Expect(err).NotTo(HaveOccurred()) + + By("verifying Module selector was updated") + mod := &kmmv1beta1.Module{} + modKey := types.NamespacedName{Name: resourceName + kmmModuleSuffix, Namespace: namespace} + Expect(k8sClient.Get(ctx, modKey, mod)).To(Succeed()) + Expect(mod.Spec.Selector).To(HaveKeyWithValue("zone", "gpu-pool")) + }) + }) + + Context("When creating a KMM Module in DP mode", func() { + const ( + namespace = "kmm-dp-create" + resourceName = "kmm-dp-create" + ) + + ctx := context.Background() + typeNamespacedName := types.NamespacedName{Name: resourceName} + + BeforeEach(func() { + Expect(k8sClient.Create(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: namespace}, + })).To(Succeed()) + }) + + AfterEach(func() { + resource := &v1alpha.ClusterPolicy{} + if err := k8sClient.Get(ctx, typeNamespacedName, resource); err == nil { + Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + } + }) + + It("should create a KMM Module CR with DevicePlugin spec", func() { + cp := &v1alpha.ClusterPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: resourceName}, + Spec: v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dp", + ResourceMonitoring: true, + DevicePluginSpec: v1alpha.DevicePluginSpec{ + PluginImage: "intel/intel-gpu-plugin:0.36.0", + }, + XpuManagerSpec: v1alpha.XpuManagerSpec{ + Image: "intel/xpumanager:v2.0.0", + MonitoringResource: "monitoring", + }, + }, + } + Expect(k8sClient.Create(ctx, cp)).To(Succeed()) + + reconciler := &ClusterPolicyReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + Opts: ControllerOpts{ + Namespace: namespace, + DRAEnable: true, + KMMEnable: true, + DRAServiceAccountName: "intel-gpu-dra", + }, + } + + _, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) + Expect(err).NotTo(HaveOccurred()) + + By("verifying KMM Module was created with DevicePlugin spec") + mod := &kmmv1beta1.Module{} + modKey := types.NamespacedName{Name: resourceName + kmmModuleSuffix, Namespace: namespace} + Expect(k8sClient.Get(ctx, modKey, mod)).To(Succeed()) + + Expect(mod.Spec.DevicePlugin).NotTo(BeNil()) + Expect(mod.Spec.DevicePlugin.Container.Image).To(Equal("intel/intel-gpu-plugin:0.36.0")) + Expect(mod.Spec.DRA).To(BeNil()) + + By("verifying DP volumes include device paths") + volNames := []string{} + for _, vol := range mod.Spec.DevicePlugin.Volumes { + volNames = append(volNames, vol.Name) + } + Expect(volNames).To(ContainElement("devfs")) + Expect(volNames).To(ContainElement("sysfsdrm")) + + By("verifying xpumd volume is present when monitoring is enabled") + Expect(volNames).To(ContainElement(xpumdVolumeName)) + }) + }) + + Context("When configuring an OOT driver via KernelModule", func() { + const ( + namespace = "kmm-oot-create" + resourceName = "kmm-oot-create" + ) + + ctx := context.Background() + typeNamespacedName := types.NamespacedName{Name: resourceName} + + BeforeEach(func() { + Expect(k8sClient.Create(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: namespace}, + })).To(Succeed()) + }) + + AfterEach(func() { + resource := &v1alpha.ClusterPolicy{} + if err := k8sClient.Get(ctx, typeNamespacedName, resource); err == nil { + Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + } + }) + + It("should configure ModuleLoader for OOT driver", func() { + cp := &v1alpha.ClusterPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: resourceName}, + Spec: v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + Image: "registry.example.com/xe-driver:1.0", + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + } + Expect(k8sClient.Create(ctx, cp)).To(Succeed()) + + reconciler := &ClusterPolicyReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + Opts: ControllerOpts{ + Namespace: namespace, + DRAEnable: true, + KMMEnable: true, + DRAServiceAccountName: "intel-gpu-dra", + }, + } + + _, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) + Expect(err).NotTo(HaveOccurred()) + + mod := &kmmv1beta1.Module{} + modKey := types.NamespacedName{Name: resourceName + kmmModuleSuffix, Namespace: namespace} + Expect(k8sClient.Get(ctx, modKey, mod)).To(Succeed()) + + Expect(mod.Spec.ModuleLoader).NotTo(BeNil()) + Expect(mod.Spec.ModuleLoader.Container.Modprobe.ModuleName).To(Equal("xe")) + Expect(mod.Spec.ModuleLoader.Container.ContainerImage).To(Equal("registry.example.com/xe-driver:1.0")) + Expect(mod.Spec.ModuleLoader.Container.InTreeModulesToRemove).To(ContainElement("xe")) + Expect(mod.Spec.ModuleLoader.Container.KernelMappings).To(HaveLen(1)) + Expect(mod.Spec.ModuleLoader.Container.KernelMappings[0].Regexp).To(Equal("^.+$")) + }) + }) + + DescribeTable("expanded KernelModule fields", func( + ns string, + cpSpec v1alpha.ClusterPolicySpec, + assertFn func(mod *kmmv1beta1.Module, cp *v1alpha.ClusterPolicy), + ) { + ctx := context.Background() + resName := ns + + Expect(k8sClient.Create(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: ns}, + })).To(Succeed()) + + cp := &v1alpha.ClusterPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: resName}, + Spec: cpSpec, + } + Expect(k8sClient.Create(ctx, cp)).To(Succeed()) + + reconciler := &ClusterPolicyReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + Opts: ControllerOpts{ + Namespace: ns, + DRAEnable: true, + KMMEnable: true, + DRAServiceAccountName: "intel-gpu-dra", + }, + } + _, err := reconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{Name: resName}, + }) + Expect(err).NotTo(HaveOccurred()) + + mod := &kmmv1beta1.Module{} + modKey := types.NamespacedName{Name: resName + kmmModuleSuffix, Namespace: ns} + Expect(k8sClient.Get(ctx, modKey, mod)).To(Succeed()) + + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: resName}, cp)).To(Succeed()) + + assertFn(mod, cp) + }, + Entry("should configure ModuleLoader with multiple kernel mappings", + "kmm-exp-multi", + v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []v1alpha.KernelMappingSpec{ + {Regexp: "^5\\.14\\.0-.*\\.el9", ContainerImage: "registry.example.com/xe-rhel9:1.0"}, + {Regexp: "^6\\.12\\..*", ContainerImage: "registry.example.com/xe-rhel10:1.0"}, + }, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + func(mod *kmmv1beta1.Module, _ *v1alpha.ClusterPolicy) { + Expect(mod.Spec.ModuleLoader.Container.KernelMappings).To(HaveLen(2)) + Expect(mod.Spec.ModuleLoader.Container.KernelMappings[0].Regexp).To(Equal("^5\\.14\\.0-.*\\.el9")) + Expect(mod.Spec.ModuleLoader.Container.KernelMappings[0].ContainerImage).To(Equal("registry.example.com/xe-rhel9:1.0")) + Expect(mod.Spec.ModuleLoader.Container.KernelMappings[1].Regexp).To(Equal("^6\\.12\\..*")) + Expect(mod.Spec.ModuleLoader.Container.KernelMappings[1].ContainerImage).To(Equal("registry.example.com/xe-rhel10:1.0")) + Expect(mod.Spec.ModuleLoader.Container.ContainerImage).To(BeEmpty()) + }, + ), + Entry("should use Image as fallback when mapping omits ContainerImage", + "kmm-exp-fallback", + v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + Image: "registry.example.com/xe-driver:1.0", + KernelMappings: []v1alpha.KernelMappingSpec{ + {Regexp: "^5\\.14\\..*"}, + }, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + func(mod *kmmv1beta1.Module, _ *v1alpha.ClusterPolicy) { + Expect(mod.Spec.ModuleLoader.Container.ContainerImage).To(Equal("registry.example.com/xe-driver:1.0")) + Expect(mod.Spec.ModuleLoader.Container.KernelMappings[0].ContainerImage).To(BeEmpty()) + }, + ), + Entry("should configure Build on kernel mapping", + "kmm-exp-build", + v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []v1alpha.KernelMappingSpec{ + { + Regexp: "^5\\.14\\..*", + Build: &v1alpha.KernelModuleBuildSpec{ + DockerfileConfigMap: v1.LocalObjectReference{Name: "xe-dockerfile"}, + BuildArgs: []v1alpha.BuildArg{{Name: "XE_TAG", Value: "v1.0"}}, + Secrets: []v1.LocalObjectReference{{Name: "private-repo"}}, + }, + }, + }, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + func(mod *kmmv1beta1.Module, _ *v1alpha.ClusterPolicy) { + kmmBuild := mod.Spec.ModuleLoader.Container.KernelMappings[0].Build + Expect(kmmBuild).NotTo(BeNil()) + Expect(kmmBuild.DockerfileConfigMap).NotTo(BeNil()) + Expect(kmmBuild.DockerfileConfigMap.Name).To(Equal("xe-dockerfile")) + Expect(kmmBuild.BuildArgs).To(HaveLen(1)) + Expect(kmmBuild.BuildArgs[0].Name).To(Equal("XE_TAG")) + Expect(kmmBuild.BuildArgs[0].Value).To(Equal("v1.0")) + Expect(kmmBuild.Secrets).To(HaveLen(1)) + Expect(kmmBuild.Secrets[0].Name).To(Equal("private-repo")) + }, + ), + Entry("should propagate FirmwarePath to ModprobeSpec", + "kmm-exp-firmware", + v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + Image: "registry.example.com/xe-driver:1.0", + FirmwarePath: "/opt/lib/firmware/xe", + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + func(mod *kmmv1beta1.Module, _ *v1alpha.ClusterPolicy) { + Expect(mod.Spec.ModuleLoader.Container.Modprobe.FirmwarePath).To(Equal("/opt/lib/firmware/xe")) + }, + ), + Entry("should propagate ModulesLoadingOrder to ModprobeSpec", + "kmm-exp-loadorder", + v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + Image: "registry.example.com/xe-driver:1.0", + ModulesLoadingOrder: []string{"xe", "drm_buddy", "drm_ttm_helper"}, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + func(mod *kmmv1beta1.Module, _ *v1alpha.ClusterPolicy) { + Expect(mod.Spec.ModuleLoader.Container.Modprobe.ModulesLoadingOrder).To(Equal([]string{"xe", "drm_buddy", "drm_ttm_helper"})) + }, + ), + Entry("should set SkipTLSVerify on RegistryTLS", + "kmm-exp-tls", + v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + Image: "registry.example.com/xe-driver:1.0", + SkipTLSVerify: true, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + func(mod *kmmv1beta1.Module, _ *v1alpha.ClusterPolicy) { + Expect(mod.Spec.ModuleLoader.Container.RegistryTLS.InsecureSkipTLSVerify).To(BeTrue()) + }, + ), + Entry("should merge InTreeModulesToRemove with ModuleName", + "kmm-exp-intree", + v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + Image: "registry.example.com/xe-driver:1.0", + InTreeModulesToRemove: []string{"i915"}, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + func(mod *kmmv1beta1.Module, _ *v1alpha.ClusterPolicy) { + Expect(mod.Spec.ModuleLoader.Container.InTreeModulesToRemove).To(ContainElement("xe")) + Expect(mod.Spec.ModuleLoader.Container.InTreeModulesToRemove).To(ContainElement("i915")) + }, + ), + Entry("should deduplicate InTreeModulesToRemove", + "kmm-exp-dedup", + v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + Image: "registry.example.com/xe-driver:1.0", + InTreeModulesToRemove: []string{"xe", "i915"}, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + func(mod *kmmv1beta1.Module, _ *v1alpha.ClusterPolicy) { + xeCount := 0 + for _, m := range mod.Spec.ModuleLoader.Container.InTreeModulesToRemove { + if m == "xe" { + xeCount++ + } + } + Expect(xeCount).To(Equal(1)) + Expect(mod.Spec.ModuleLoader.Container.InTreeModulesToRemove).To(ContainElement("i915")) + }, + ), + Entry("should set per-mapping InTreeModulesToRemove override", + "kmm-exp-permapping", + v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []v1alpha.KernelMappingSpec{ + { + Regexp: "^5\\.14\\..*", + ContainerImage: "registry.example.com/xe:1.0", + InTreeModulesToRemove: []string{"old_xe"}, + }, + }, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + func(mod *kmmv1beta1.Module, _ *v1alpha.ClusterPolicy) { + Expect(mod.Spec.ModuleLoader.Container.KernelMappings[0].InTreeModulesToRemove).To(Equal([]string{"old_xe"})) + Expect(mod.Spec.ModuleLoader.Container.InTreeModulesToRemove).To(ContainElement("xe")) + }, + ), + Entry("should set KMMStatus to N/A when KernelModule is nil", + "kmm-exp-nkmm", + v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + func(_ *kmmv1beta1.Module, cp *v1alpha.ClusterPolicy) { + Expect(cp.Status.KMMStatus).To(Equal("N/A")) + }, + ), + ) + + Context("When deleting a KMM Module", func() { + const ( + namespace = "kmm-del-create" + resourceName = "kmm-del-create" + ) + + ctx := context.Background() + typeNamespacedName := types.NamespacedName{Name: resourceName} + + BeforeEach(func() { + Expect(k8sClient.Create(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: namespace}, + })).To(Succeed()) + }) + + AfterEach(func() { + resource := &v1alpha.ClusterPolicy{} + if err := k8sClient.Get(ctx, typeNamespacedName, resource); err == nil { + Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + } + }) + + It("should delete Module when ClusterPolicy is deleted", func() { + cp := &v1alpha.ClusterPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: resourceName}, + Spec: v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + } + Expect(k8sClient.Create(ctx, cp)).To(Succeed()) + + reconciler := &ClusterPolicyReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + Opts: ControllerOpts{ + Namespace: namespace, + DRAEnable: true, + KMMEnable: true, + DRAServiceAccountName: "intel-gpu-dra", + }, + } + + _, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) + Expect(err).NotTo(HaveOccurred()) + + modKey := types.NamespacedName{Name: resourceName + kmmModuleSuffix, Namespace: namespace} + Expect(k8sClient.Get(ctx, modKey, &kmmv1beta1.Module{})).To(Succeed()) + + By("deleting the ClusterPolicy") + Expect(k8sClient.Delete(ctx, cp)).To(Succeed()) + + By("reconciling the deletion via KMM reconciler directly") + kmmReconciler := &KMMReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + Opts: ControllerOpts{ + ReqName: resourceName, + Namespace: namespace, + KMMEnable: true, + }, + } + _, err = kmmReconciler.Reconcile(ctx, nil) + Expect(err).NotTo(HaveOccurred()) + + By("verifying Module was deleted") + err = k8sClient.Get(ctx, modKey, &kmmv1beta1.Module{}) + Expect(errors.IsNotFound(err)).To(BeTrue()) + }) + }) + + Context("When ClusterPolicy has tolerations and pull secret", func() { + const ( + namespace = "kmm-opts-create" + resourceName = "kmm-opts-create" + ) + + ctx := context.Background() + typeNamespacedName := types.NamespacedName{Name: resourceName} + + BeforeEach(func() { + Expect(k8sClient.Create(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: namespace}, + })).To(Succeed()) + }) + + AfterEach(func() { + resource := &v1alpha.ClusterPolicy{} + if err := k8sClient.Get(ctx, typeNamespacedName, resource); err == nil { + Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + } + }) + + It("should propagate tolerations and pull secret to Module", func() { + cp := &v1alpha.ClusterPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: resourceName}, + Spec: v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + Tolerations: []v1.Toleration{ + { + Key: "gpu-dedicated", + Operator: v1.TolerationOpExists, + Effect: v1.TaintEffectNoSchedule, + }, + }, + PullSecret: &v1.LocalObjectReference{ + Name: "my-registry-secret", + }, + }, + } + Expect(k8sClient.Create(ctx, cp)).To(Succeed()) + + reconciler := &ClusterPolicyReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + Opts: ControllerOpts{ + Namespace: namespace, + DRAEnable: true, + KMMEnable: true, + DRAServiceAccountName: "intel-gpu-dra", + }, + } + + _, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) + Expect(err).NotTo(HaveOccurred()) + + mod := &kmmv1beta1.Module{} + modKey := types.NamespacedName{Name: resourceName + kmmModuleSuffix, Namespace: namespace} + Expect(k8sClient.Get(ctx, modKey, mod)).To(Succeed()) + + By("verifying tolerations") + Expect(mod.Spec.Tolerations).To(HaveLen(1)) + Expect(mod.Spec.Tolerations[0].Key).To(Equal("gpu-dedicated")) + + By("verifying pull secret") + Expect(mod.Spec.ImageRepoSecret).NotTo(BeNil()) + Expect(mod.Spec.ImageRepoSecret.Name).To(Equal("my-registry-secret")) + }) + }) +}) + +var _ = Describe("KMM DRA args generation", func() { + It("generates correct args with all options", func() { + cp := &v1alpha.ClusterPolicy{ + Spec: v1alpha.ClusterPolicySpec{ + LogLevel: 2, + HealthinessSpec: &v1alpha.HealthinessSpec{ + CheckIntervalSeconds: 5, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + LogLevel: 3, + PodHealthCheck: true, + ManageBinding: true, + }, + }, + } + + r := &KMMReconciler{} + args := r.generateDRAArgs(cp) + + Expect(args).To(ContainElement("-v=3")) + Expect(args).To(ContainElement("--health-monitoring=true")) + Expect(args).To(ContainElement("--healthcheck-port=51516")) + Expect(args).To(ContainElement("--manage-binding=true")) + }) + + It("disables health check port when PodHealthCheck is false", func() { + cp := &v1alpha.ClusterPolicy{ + Spec: v1alpha.ClusterPolicySpec{ + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + PodHealthCheck: false, + }, + }, + } + + r := &KMMReconciler{} + args := r.generateDRAArgs(cp) + + Expect(args).To(ContainElement("--healthcheck-port=-1")) + Expect(args).To(ContainElement("--manage-binding=false")) + }) +}) diff --git a/internal/controller/openshift.go b/internal/controller/openshift.go index 74118ed..55879d0 100644 --- a/internal/controller/openshift.go +++ b/internal/controller/openshift.go @@ -48,31 +48,6 @@ func buildSCC(name string, spec map[string]interface{}) *unstructured.Unstructur return scc } -// buildDevicePluginSCC returns the SCC for the GPU device plugin daemonset. -func buildDevicePluginSCC(name string) *unstructured.Unstructured { - return buildSCC(name, map[string]interface{}{ - "allowPrivilegedContainer": false, - "allowHostDirVolumePlugin": true, - "allowHostIPC": false, - "allowHostNetwork": false, - "allowHostPID": false, - "allowHostPorts": false, - "allowPrivilegeEscalation": false, - "allowedCapabilities": nil, - "defaultAddCapabilities": nil, - "fsGroup": map[string]interface{}{"type": "RunAsAny"}, - "readOnlyRootFilesystem": false, - "requiredDropCapabilities": []interface{}{"ALL"}, - "runAsUser": map[string]interface{}{"type": "RunAsAny"}, - "seLinuxContext": map[string]interface{}{"type": "RunAsAny"}, - "seccompProfiles": []interface{}{"*"}, - "supplementalGroups": map[string]interface{}{"type": "RunAsAny"}, - "volumes": []interface{}{"hostPath", "emptyDir"}, - "users": []interface{}{}, - "groups": []interface{}{}, - }) -} - // buildXpuManagerSCC returns the SCC for the XPU Manager daemonset. // The daemonset runs as root and requires SYS_ADMIN capability. func buildXpuManagerSCC(name string) *unstructured.Unstructured { @@ -99,32 +74,6 @@ func buildXpuManagerSCC(name string) *unstructured.Unstructured { }) } -// buildDRASCC returns the SCC for the GPU DRA driver daemonset. -// The daemonset uses hostPath volumes and mounts a service account token (projected volume). -func buildDRASCC(name string) *unstructured.Unstructured { - return buildSCC(name, map[string]interface{}{ - "allowPrivilegedContainer": false, - "allowHostDirVolumePlugin": true, - "allowHostIPC": false, - "allowHostNetwork": false, - "allowHostPID": false, - "allowHostPorts": false, - "allowPrivilegeEscalation": false, - "allowedCapabilities": nil, - "defaultAddCapabilities": nil, - "fsGroup": map[string]interface{}{"type": "RunAsAny"}, - "readOnlyRootFilesystem": false, - "requiredDropCapabilities": []interface{}{"ALL"}, - "runAsUser": map[string]interface{}{"type": "RunAsAny"}, - "seLinuxContext": map[string]interface{}{"type": "RunAsAny"}, - "seccompProfiles": []interface{}{"*"}, - "supplementalGroups": map[string]interface{}{"type": "RunAsAny"}, - "volumes": []interface{}{"hostPath", "projected"}, - "users": []interface{}{}, - "groups": []interface{}{}, - }) -} - // buildFWUpdateSCC returns the SCC for the GPU firmware update Job pods. // The updater container runs privileged as root to access GPU firmware interfaces. func buildFWUpdateSCC(name string) *unstructured.Unstructured { diff --git a/internal/controller/openshift_test.go b/internal/controller/openshift_test.go index 24b85f8..c5ff107 100644 --- a/internal/controller/openshift_test.go +++ b/internal/controller/openshift_test.go @@ -34,26 +34,6 @@ var _ = Describe("OpenShift SCC helpers", func() { ctx := context.Background() Context("SCC builder functions", func() { - It("buildDevicePluginSCC sets correct fields", func() { - scc := buildDevicePluginSCC("dp-builder-test") - - Expect(scc.GetName()).To(Equal("dp-builder-test")) - Expect(scc.GetKind()).To(Equal("SecurityContextConstraints")) - Expect(scc.GetAPIVersion()).To(Equal("security.openshift.io/v1")) - Expect(scc.Object["allowPrivilegedContainer"]).To(BeFalse()) - Expect(scc.Object["allowPrivilegeEscalation"]).To(BeFalse()) - Expect(scc.Object["allowHostDirVolumePlugin"]).To(BeTrue()) - Expect(scc.Object["allowHostNetwork"]).To(BeFalse()) - - drops, ok := scc.Object["requiredDropCapabilities"].([]interface{}) - Expect(ok).To(BeTrue()) - Expect(drops).To(ContainElement("ALL")) - - vols, ok := scc.Object["volumes"].([]interface{}) - Expect(ok).To(BeTrue()) - Expect(vols).To(ContainElements("hostPath", "emptyDir")) - }) - It("buildXpuManagerSCC sets correct fields", func() { scc := buildXpuManagerSCC("xpum-builder-test") @@ -73,22 +53,6 @@ var _ = Describe("OpenShift SCC helpers", func() { Expect(vols).To(ContainElements("hostPath", "configMap")) }) - It("buildDRASCC sets correct fields", func() { - scc := buildDRASCC("dra-builder-test") - - Expect(scc.GetName()).To(Equal("dra-builder-test")) - Expect(scc.Object["allowPrivilegedContainer"]).To(BeFalse()) - Expect(scc.Object["allowPrivilegeEscalation"]).To(BeFalse()) - - drops, ok := scc.Object["requiredDropCapabilities"].([]interface{}) - Expect(ok).To(BeTrue()) - Expect(drops).To(ContainElement("ALL")) - - vols, ok := scc.Object["volumes"].([]interface{}) - Expect(ok).To(BeTrue()) - Expect(vols).To(ContainElements("hostPath", "projected")) - }) - It("buildFWUpdateSCC sets correct fields", func() { scc := buildFWUpdateSCC("fwupdate-builder-test") @@ -109,7 +73,7 @@ var _ = Describe("OpenShift SCC helpers", func() { Context("ensureSCC", func() { It("creates SCC when not found and is idempotent", func() { name := "test-ensure-scc-create" - scc := buildDevicePluginSCC(name) + scc := buildXpuManagerSCC(name) By("first call creates the SCC") Expect(ensureSCC(ctx, k8sClient, scc)).To(Succeed()) @@ -120,7 +84,7 @@ var _ = Describe("OpenShift SCC helpers", func() { Expect(k8sClient.Get(ctx, client.ObjectKey{Name: name}, existing)).To(Succeed()) By("second call is idempotent") - Expect(ensureSCC(ctx, k8sClient, buildDevicePluginSCC(name))).To(Succeed()) + Expect(ensureSCC(ctx, k8sClient, buildXpuManagerSCC(name))).To(Succeed()) DeferCleanup(func() { scc := &unstructured.Unstructured{} @@ -212,7 +176,7 @@ var _ = Describe("OpenShift SCC helpers", func() { saName := "test-delete-sa" By("creating the resources") - Expect(ensureSCC(ctx, k8sClient, buildDevicePluginSCC(sccName))).To(Succeed()) + Expect(ensureSCC(ctx, k8sClient, buildXpuManagerSCC(sccName))).To(Succeed()) Expect(createSCCRole(ctx, k8sClient, roleName, sccName)).To(Succeed()) Expect(createSCCRoleBinding(ctx, k8sClient, bindingName, roleName, saName, deleteNs)).To(Succeed()) Expect(createServiceAccount(ctx, k8sClient, saName, deleteNs)).To(Succeed()) @@ -243,7 +207,7 @@ var _ = Describe("OpenShift SCC helpers", func() { saName := "test-skip-sa" By("creating the resources") - Expect(ensureSCC(ctx, k8sClient, buildDevicePluginSCC(sccName))).To(Succeed()) + Expect(ensureSCC(ctx, k8sClient, buildXpuManagerSCC(sccName))).To(Succeed()) Expect(createSCCRole(ctx, k8sClient, roleName, sccName)).To(Succeed()) Expect(createSCCRoleBinding(ctx, k8sClient, bindingName, roleName, saName, deleteNs)).To(Succeed()) Expect(createServiceAccount(ctx, k8sClient, saName, deleteNs)).To(Succeed()) diff --git a/internal/controller/suite_test.go b/internal/controller/suite_test.go index 22daa3c..9bf4bc6 100644 --- a/internal/controller/suite_test.go +++ b/internal/controller/suite_test.go @@ -18,8 +18,11 @@ package controller import ( "context" + "fmt" "os" + "os/exec" "path/filepath" + "strings" "testing" . "github.com/onsi/ginkgo/v2" @@ -32,6 +35,7 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" + kmmv1beta1 "github.com/kubernetes-sigs/kernel-module-management/api/v1beta1" prometheusv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" corev1 "k8s.io/api/core/v1" resv1 "k8s.io/api/resource/v1" @@ -79,6 +83,8 @@ var _ = BeforeSuite(func() { Expect(err).NotTo(HaveOccurred()) err = corev1.AddToScheme(s) Expect(err).NotTo(HaveOccurred()) + err = kmmv1beta1.AddToScheme(s) + Expect(err).NotTo(HaveOccurred()) // +kubebuilder:scaffold:scheme @@ -88,6 +94,7 @@ var _ = BeforeSuite(func() { CRDDirectoryPaths: []string{ filepath.Join("..", "..", "config", "crd", "bases"), filepath.Join("..", "..", "config", "deployments", "nfd", "crds"), + filepath.Join(goModDir("github.com/kubernetes-sigs/kernel-module-management"), "config", "crd", "bases"), "testdata", }, ErrorIfCRDPathMissing: true, @@ -127,6 +134,14 @@ var _ = AfterSuite(func() { // This function streamlines the process by finding the required binaries, similar to // setting the 'KUBEBUILDER_ASSETS' environment variable. To ensure the binaries are // properly set up, run 'make setup-envtest' beforehand. +func goModDir(module string) string { + out, err := exec.Command("go", "list", "-m", "-f", "{{.Dir}}", module).Output() + if err != nil { + panic(fmt.Sprintf("failed to resolve module directory for %s: %v", module, err)) + } + return strings.TrimSpace(string(out)) +} + func getFirstFoundEnvTestBinaryDir() string { basePath := filepath.Join("..", "..", "bin", "k8s") entries, err := os.ReadDir(basePath) diff --git a/internal/controller/xpumanager_controller.go b/internal/controller/xpumanager_controller.go index 7adbff4..f9fd3f1 100644 --- a/internal/controller/xpumanager_controller.go +++ b/internal/controller/xpumanager_controller.go @@ -199,19 +199,7 @@ func processContainerResources(ds *apps.DaemonSet, spec *v1alpha.ClusterPolicy, } func processNodeSelectors(ds *apps.DaemonSet, spec *v1alpha.ClusterPolicy) { - ds.Spec.Template.Spec.NodeSelector = map[string]string{ - "kubernetes.io/arch": "amd64", - } - - if len(spec.Spec.NodeSelector) > 0 { - for k, v := range spec.Spec.NodeSelector { - ds.Spec.Template.Spec.NodeSelector[k] = v - } - } - - if spec.Spec.UseNFDLabeling { - ds.Spec.Template.Spec.NodeSelector["intel.feature.node.kubernetes.io/gpu"] = trueValue - } + ds.Spec.Template.Spec.NodeSelector = gpuNodeSelector(spec) } // Convert the integer based log level to a string based log level for the OTel config.