diff --git a/api/v1alpha1/clusterpolicy_types.go b/api/v1alpha1/clusterpolicy_types.go index a58a3a1..08772da 100644 --- a/api/v1alpha1/clusterpolicy_types.go +++ b/api/v1alpha1/clusterpolicy_types.go @@ -69,6 +69,12 @@ type ClusterPolicySpec struct { // +kubebuilder:validation:Range=0:4 // +kubebuilder:validation:Default=2 LogLevel int32 `json:"logLevel,omitempty"` + + // KernelModule configures out-of-tree kernel module loading via KMM. + // When set, KMM loads the specified OOT driver module on each node. + // When nil, the in-tree kernel driver is used. + // +optional + KernelModule *KernelModuleSpec `json:"kernelModule,omitempty"` } // DynamicResourceAllocationSpec defines the desired state of DynamicResourceAllocation. @@ -147,11 +153,104 @@ type XpuManagerSpec struct { MonitoringResource string `json:"monitoringResource,omitempty"` } +// KernelModuleSpec configures out-of-tree kernel module loading via KMM. +type KernelModuleSpec struct { + // ModuleName is the kernel module to load (e.g., "xe"). + // Also used as the default InTreeModulesToRemove entry. + ModuleName string `json:"moduleName"` + + // Image is the default container image for the OOT driver module. + // When KernelMappings is empty, generates a single wildcard mapping. + // When KernelMappings is non-empty, serves as the KMM-level fallback + // for mappings that omit ContainerImage. + // +optional + Image string `json:"image,omitempty"` + + // KernelMappings maps kernel version patterns to container images or + // build specifications. Translates directly to KMM KernelMapping + // objects. When empty, a single wildcard mapping (regexp "^.+$") is + // generated from Image. + // +optional + KernelMappings []KernelMappingSpec `json:"kernelMappings,omitempty"` + + // InTreeModulesToRemove lists in-tree modules to unload before + // inserting the OOT module. ModuleName is always included implicitly. + // +optional + InTreeModulesToRemove []string `json:"inTreeModulesToRemove,omitempty"` + + // ModulesLoadingOrder specifies softdep-style loading order for + // multi-module drivers. First element must be ModuleName; KMM loads + // in order and unloads in reverse. Must have >=2 entries if set. + // +optional + ModulesLoadingOrder []string `json:"modulesLoadingOrder,omitempty"` + + // FirmwarePath is the in-container path where firmware files are stored. + // +optional + FirmwarePath string `json:"firmwarePath,omitempty"` + + // SkipTLSVerify disables TLS certificate verification when pulling + // OOT driver images. For air-gapped or internal registries. + // +optional + SkipTLSVerify bool `json:"skipTLSVerify,omitempty"` +} + +// KernelMappingSpec maps a kernel version pattern to a container image +// or build specification. +type KernelMappingSpec struct { + // Regexp is a regular expression matched against node kernel versions. + // Exactly one of Regexp or Literal must be set. + // +optional + Regexp string `json:"regexp,omitempty"` + + // Literal is an exact kernel version string to match. + // Exactly one of Regexp or Literal must be set. + // +optional + Literal string `json:"literal,omitempty"` + + // ContainerImage is the full image reference for this kernel version. + // Required when Build is nil and no parent-level Image is set. + // +optional + ContainerImage string `json:"containerImage,omitempty"` + + // Build configures in-cluster building of the driver image via KMM. + // When set, KMM builds the image if it doesn't exist in the registry. + // +optional + Build *KernelModuleBuildSpec `json:"build,omitempty"` + + // InTreeModulesToRemove overrides the parent-level list for this + // specific kernel mapping. + // +optional + InTreeModulesToRemove []string `json:"inTreeModulesToRemove,omitempty"` +} + +// KernelModuleBuildSpec configures in-cluster driver image building. +type KernelModuleBuildSpec struct { + // DockerfileConfigMap references a ConfigMap containing the Dockerfile. + DockerfileConfigMap v1.LocalObjectReference `json:"dockerfileConfigMap"` + + // BuildArgs are key-value pairs passed to the image builder. + // +optional + BuildArgs []BuildArg `json:"buildArgs,omitempty"` + + // Secrets are made available during the build (e.g., for private + // source repos). Not for registry auth -- use pullSecret on + // ClusterPolicySpec. + // +optional + Secrets []v1.LocalObjectReference `json:"secrets,omitempty"` +} + +// BuildArg is a key-value pair passed as a build argument. +type BuildArg struct { + Name string `json:"name"` + Value string `json:"value"` +} + // ClusterPolicyStatus defines the observed state of ClusterPolicy. type ClusterPolicyStatus struct { DevicePluginStatus string `json:"devicePluginStatus,omitempty"` DRAStatus string `json:"draStatus,omitempty"` XPUManagerStatus string `json:"xpuManagerStatus,omitempty"` + KMMStatus string `json:"kmmStatus,omitempty"` Errors []string `json:"errors,omitempty"` } @@ -183,6 +282,7 @@ type LocalQueueSpec struct { // +kubebuilder:printcolumn:name="DP",type=string,JSONPath=`.status.devicePluginStatus` // +kubebuilder:printcolumn:name="DRA",type=string,JSONPath=`.status.draStatus` // +kubebuilder:printcolumn:name="XPU",type=string,JSONPath=`.status.xpuManagerStatus` +// +kubebuilder:printcolumn:name="KMM",type=string,JSONPath=`.status.kmmStatus` // +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` // +operator-sdk:csv:customresourcedefinitions:displayName="Intel GPU Cluster Policy" // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/api/v1alpha1/clusterpolicy_webhook.go b/api/v1alpha1/clusterpolicy_webhook.go index 2b58d53..e55e849 100644 --- a/api/v1alpha1/clusterpolicy_webhook.go +++ b/api/v1alpha1/clusterpolicy_webhook.go @@ -95,6 +95,7 @@ func validateClusterPolicySpec(spec *ClusterPolicySpec) (admission.Warnings, err errs = append(errs, validatePullSecret(spec)...) errs = append(errs, validateConfigMapOverride(spec)...) errs = append(errs, validateKueueSpec(spec)...) + errs = append(errs, validateKernelModuleSpec(spec)...) if w := warnForSpecProblems(spec); w != "" { warnings = append(warnings, w) @@ -223,6 +224,114 @@ func validateKueueSpec(spec *ClusterPolicySpec) []error { return errs } +func validateKernelModuleSpec(spec *ClusterPolicySpec) []error { + if spec.KernelModule == nil { + return nil + } + + km := spec.KernelModule + var errs []error + + if km.ModuleName == "" { + errs = append(errs, fmt.Errorf("kernelModule.moduleName is required")) + } + + if km.Image != "" { + if err := validateImageTagOrDigest("kernelModule.image", km.Image); err != nil { + errs = append(errs, err) + } + } + + if len(km.KernelMappings) == 0 { + if km.Image == "" { + errs = append(errs, fmt.Errorf("kernelModule.image is required when kernelMappings is empty")) + } + } else { + for i, m := range km.KernelMappings { + prefix := fmt.Sprintf("kernelModule.kernelMappings[%d]", i) + errs = append(errs, validateKernelMapping(prefix, &m, km.Image)...) + } + } + + if len(km.ModulesLoadingOrder) > 0 { + errs = append(errs, validateModulesLoadingOrder(km.ModuleName, km.ModulesLoadingOrder)...) + } + + return errs +} + +func validateImageTagOrDigest(field, image string) error { + if _, err := reference.ParseNormalizedNamed(image); err != nil { + return fmt.Errorf("invalid image reference in %s: %q: %w", field, image, err) + } + + if !strings.Contains(image, ":") && !strings.Contains(image, "@") { + return fmt.Errorf("%s: image %q must include an explicit tag or digest", field, image) + } + + return nil +} + +func validateKernelMapping(prefix string, m *KernelMappingSpec, parentImage string) []error { + var errs []error + + hasRegexp := m.Regexp != "" + hasLiteral := m.Literal != "" + + if hasRegexp && hasLiteral { + errs = append(errs, fmt.Errorf("%s: regexp and literal are mutually exclusive", prefix)) + } else if !hasRegexp && !hasLiteral { + errs = append(errs, fmt.Errorf("%s: exactly one of regexp or literal must be set", prefix)) + } + + if hasRegexp { + if _, err := regexp.Compile(m.Regexp); err != nil { + errs = append(errs, fmt.Errorf("%s.regexp: invalid regular expression: %w", prefix, err)) + } + } + + if m.ContainerImage != "" { + if err := validateImageTagOrDigest(prefix+".containerImage", m.ContainerImage); err != nil { + errs = append(errs, err) + } + } + + if m.ContainerImage == "" && m.Build == nil && parentImage == "" { + errs = append(errs, fmt.Errorf("%s: one of containerImage, build, or parent-level image must be set", prefix)) + } + + if m.Build != nil { + if m.Build.DockerfileConfigMap.Name == "" { + errs = append(errs, fmt.Errorf("%s.build.dockerfileConfigMap.name is required", prefix)) + } + } + + return errs +} + +func validateModulesLoadingOrder(moduleName string, order []string) []error { + var errs []error + + if len(order) < 2 { + errs = append(errs, fmt.Errorf("kernelModule.modulesLoadingOrder must have at least 2 entries")) + } + + if len(order) > 0 && moduleName != "" && order[0] != moduleName { + errs = append(errs, fmt.Errorf("kernelModule.modulesLoadingOrder[0] must be moduleName %q", moduleName)) + } + + seen := make(map[string]bool, len(order)) + for i, entry := range order { + if seen[entry] { + errs = append(errs, fmt.Errorf("kernelModule.modulesLoadingOrder[%d]: duplicate entry %q", i, entry)) + break + } + seen[entry] = true + } + + return errs +} + // warnForSpecProblems returns a warning message if some old or deprecated option is set. func warnForSpecProblems(spec *ClusterPolicySpec) string { if spec.ResourceRegistration == "dp" && spec.DevicePluginSpec.LevelzeroImage != "" { diff --git a/api/v1alpha1/clusterpolicy_webhook_test.go b/api/v1alpha1/clusterpolicy_webhook_test.go index 45967d1..2c56e40 100644 --- a/api/v1alpha1/clusterpolicy_webhook_test.go +++ b/api/v1alpha1/clusterpolicy_webhook_test.go @@ -387,6 +387,228 @@ var _ = Describe("ClusterPolicy Webhook", func() { }) }) + Context("kernelModule validation", func() { + It("accepts nil kernelModule (in-tree mode)", func() { + obj.Spec.KernelModule = nil + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).NotTo(HaveOccurred()) + }) + + It("accepts valid kernelModule with moduleName + image only", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + Image: "registry.example.com/xe-driver:1.0", + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).NotTo(HaveOccurred()) + }) + + It("rejects kernelModule with empty moduleName", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + Image: "registry.example.com/xe-driver:1.0", + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("moduleName is required")) + }) + + It("rejects kernelModule without image when kernelMappings is empty", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("image is required when kernelMappings is empty")) + }) + + It("accepts kernelMappings with each mapping having its own containerImage", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + {Regexp: "^5\\.14\\..*", ContainerImage: "registry.example.com/xe-rhel9:1.0"}, + {Regexp: "^6\\.12\\..*", ContainerImage: "registry.example.com/xe-rhel10:1.0"}, + }, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).NotTo(HaveOccurred()) + }) + + It("accepts image as fallback when mapping omits containerImage", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + Image: "registry.example.com/xe-driver:1.0", + KernelMappings: []KernelMappingSpec{ + {Regexp: "^5\\.14\\..*"}, + }, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).NotTo(HaveOccurred()) + }) + + It("rejects mapping with neither containerImage, build, nor parent image", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + {Regexp: "^5\\.14\\..*"}, + }, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("one of containerImage, build, or parent-level image must be set")) + }) + + It("rejects mapping with both regexp and literal set", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + {Regexp: "^5\\.14\\..*", Literal: "5.14.0-687.el9.x86_64", ContainerImage: "registry.example.com/xe:1.0"}, + }, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("regexp and literal are mutually exclusive")) + }) + + It("rejects mapping with neither regexp nor literal set", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + {ContainerImage: "registry.example.com/xe:1.0"}, + }, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("exactly one of regexp or literal must be set")) + }) + + It("rejects mapping with invalid regexp", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + {Regexp: "[invalid", ContainerImage: "registry.example.com/xe:1.0"}, + }, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("invalid regular expression")) + }) + + It("rejects image without tag or digest", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + Image: "registry.example.com/xe-driver", + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("must include an explicit tag or digest")) + }) + + It("rejects containerImage without tag or digest", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + {Regexp: "^5\\.14\\..*", ContainerImage: "registry.example.com/xe-driver"}, + }, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("must include an explicit tag or digest")) + }) + + It("accepts image with tag", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + Image: "registry.example.com/xe-driver:1.0", + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).NotTo(HaveOccurred()) + }) + + It("accepts image with digest", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + Image: "registry.example.com/xe-driver@sha256:abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890", + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).NotTo(HaveOccurred()) + }) + + It("accepts valid build spec with dockerfileConfigMap", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + { + Regexp: "^5\\.14\\..*", + Build: &KernelModuleBuildSpec{ + DockerfileConfigMap: v1.LocalObjectReference{Name: "xe-dockerfile"}, + BuildArgs: []BuildArg{{Name: "XE_TAG", Value: "v1.0"}}, + }, + }, + }, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).NotTo(HaveOccurred()) + }) + + It("rejects build spec with empty dockerfileConfigMap name", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + { + Regexp: "^5\\.14\\..*", + Build: &KernelModuleBuildSpec{}, + }, + }, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("dockerfileConfigMap.name is required")) + }) + + It("accepts valid modulesLoadingOrder with moduleName first", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + Image: "registry.example.com/xe-driver:1.0", + ModulesLoadingOrder: []string{"xe", "drm_buddy"}, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).NotTo(HaveOccurred()) + }) + + It("rejects modulesLoadingOrder with fewer than 2 entries", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + Image: "registry.example.com/xe-driver:1.0", + ModulesLoadingOrder: []string{"xe"}, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("at least 2 entries")) + }) + + It("rejects modulesLoadingOrder where first entry is not moduleName", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + Image: "registry.example.com/xe-driver:1.0", + ModulesLoadingOrder: []string{"drm_buddy", "xe"}, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("must be moduleName")) + }) + + It("rejects modulesLoadingOrder with duplicate entries", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + Image: "registry.example.com/xe-driver:1.0", + ModulesLoadingOrder: []string{"xe", "drm_buddy", "drm_buddy"}, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("duplicate entry")) + }) + }) + Context("spec warning for Levelzero", func() { It("emits a warning when levelzero image is set in DP mode", func() { obj.Spec.ResourceRegistration = dpName diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 2470cf4..2b3162d 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -25,6 +25,21 @@ import ( "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BuildArg) DeepCopyInto(out *BuildArg) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BuildArg. +func (in *BuildArg) DeepCopy() *BuildArg { + if in == nil { + return nil + } + out := new(BuildArg) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClusterPolicy) DeepCopyInto(out *ClusterPolicy) { *out = *in @@ -149,6 +164,11 @@ func (in *ClusterPolicySpec) DeepCopyInto(out *ClusterPolicySpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.KernelModule != nil { + in, out := &in.KernelModule, &out.KernelModule + *out = new(KernelModuleSpec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterPolicySpec. @@ -466,6 +486,89 @@ func (in *HealthinessSpec) DeepCopy() *HealthinessSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KernelMappingSpec) DeepCopyInto(out *KernelMappingSpec) { + *out = *in + if in.Build != nil { + in, out := &in.Build, &out.Build + *out = new(KernelModuleBuildSpec) + (*in).DeepCopyInto(*out) + } + if in.InTreeModulesToRemove != nil { + in, out := &in.InTreeModulesToRemove, &out.InTreeModulesToRemove + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KernelMappingSpec. +func (in *KernelMappingSpec) DeepCopy() *KernelMappingSpec { + if in == nil { + return nil + } + out := new(KernelMappingSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KernelModuleBuildSpec) DeepCopyInto(out *KernelModuleBuildSpec) { + *out = *in + out.DockerfileConfigMap = in.DockerfileConfigMap + if in.BuildArgs != nil { + in, out := &in.BuildArgs, &out.BuildArgs + *out = make([]BuildArg, len(*in)) + copy(*out, *in) + } + if in.Secrets != nil { + in, out := &in.Secrets, &out.Secrets + *out = make([]v1.LocalObjectReference, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KernelModuleBuildSpec. +func (in *KernelModuleBuildSpec) DeepCopy() *KernelModuleBuildSpec { + if in == nil { + return nil + } + out := new(KernelModuleBuildSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KernelModuleSpec) DeepCopyInto(out *KernelModuleSpec) { + *out = *in + if in.KernelMappings != nil { + in, out := &in.KernelMappings, &out.KernelMappings + *out = make([]KernelMappingSpec, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.InTreeModulesToRemove != nil { + in, out := &in.InTreeModulesToRemove, &out.InTreeModulesToRemove + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.ModulesLoadingOrder != nil { + in, out := &in.ModulesLoadingOrder, &out.ModulesLoadingOrder + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KernelModuleSpec. +func (in *KernelModuleSpec) DeepCopy() *KernelModuleSpec { + if in == nil { + return nil + } + out := new(KernelModuleSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KueueQueueSpec) DeepCopyInto(out *KueueQueueSpec) { *out = *in diff --git a/charts/gpu-base-operator-policy/templates/clusterpolicy.yaml b/charts/gpu-base-operator-policy/templates/clusterpolicy.yaml index 737355d..8716f82 100644 --- a/charts/gpu-base-operator-policy/templates/clusterpolicy.yaml +++ b/charts/gpu-base-operator-policy/templates/clusterpolicy.yaml @@ -62,6 +62,61 @@ spec: configMapOverride: {{ .Values.xpu.configMapOverride | default "" }} {{- end }} +{{- if .Values.kernelModule }} + kernelModule: + moduleName: {{ .Values.kernelModule.moduleName }} + {{- if .Values.kernelModule.image }} + image: {{ .Values.kernelModule.image }} + {{- end }} + {{- if .Values.kernelModule.kernelMappings }} + kernelMappings: + {{- range .Values.kernelModule.kernelMappings }} + - {{- if .regexp }} + regexp: {{ .regexp | quote }} + {{- end }} + {{- if .literal }} + literal: {{ .literal | quote }} + {{- end }} + {{- if .containerImage }} + containerImage: {{ .containerImage }} + {{- end }} + {{- if .build }} + build: + dockerfileConfigMap: + name: {{ .build.dockerfileConfigMap.name }} + {{- if .build.buildArgs }} + buildArgs: + {{- range .build.buildArgs }} + - name: {{ .name }} + value: {{ .value | quote }} + {{- end }} + {{- end }} + {{- if .build.secrets }} + secrets: + {{- range .build.secrets }} + - name: {{ .name }} + {{- end }} + {{- end }} + {{- end }} + {{- if .inTreeModulesToRemove }} + inTreeModulesToRemove: {{ toYaml .inTreeModulesToRemove | nindent 10 }} + {{- end }} + {{- end }} + {{- end }} + {{- if .Values.kernelModule.inTreeModulesToRemove }} + inTreeModulesToRemove: {{ toYaml .Values.kernelModule.inTreeModulesToRemove | nindent 6 }} + {{- end }} + {{- if .Values.kernelModule.modulesLoadingOrder }} + modulesLoadingOrder: {{ toYaml .Values.kernelModule.modulesLoadingOrder | nindent 6 }} + {{- end }} + {{- if .Values.kernelModule.firmwarePath }} + firmwarePath: {{ .Values.kernelModule.firmwarePath }} + {{- end }} + {{- if .Values.kernelModule.skipTLSVerify }} + skipTLSVerify: {{ .Values.kernelModule.skipTLSVerify }} + {{- end }} +{{- end }} + {{- if .Values.pullSecret }} pullSecret: {{ .Values.pullSecret }} {{- end }} diff --git a/charts/gpu-base-operator-policy/values.yaml b/charts/gpu-base-operator-policy/values.yaml index fc465f2..01a4d50 100644 --- a/charts/gpu-base-operator-policy/values.yaml +++ b/charts/gpu-base-operator-policy/values.yaml @@ -35,6 +35,32 @@ kueue: - name: gpu-queue namespace: default +# kernelModule: +# moduleName: xe +# image: registry.example.com/xe-driver:1.0 +# +# # Multi-kernel support: +# # kernelMappings: +# # - regexp: "^5\\.14\\.0-.*\\.el9.*\\.x86_64$" +# # containerImage: registry.example.com/xe-driver-rhel9:1.0 +# # - regexp: "^6\\.12\\..*" +# # containerImage: registry.example.com/xe-driver-rhel10:1.0 +# # +# # In-cluster build: +# # kernelMappings: +# # - regexp: "^5\\.14\\..*" +# # build: +# # dockerfileConfigMap: +# # name: xe-dockerfile +# # buildArgs: +# # - name: XE_TAG +# # value: v1.0 +# # +# # inTreeModulesToRemove: ["i915"] +# # modulesLoadingOrder: ["xe", "drm_buddy", "drm_ttm_helper"] +# # firmwarePath: /opt/lib/firmware/xe +# # skipTLSVerify: false + pullSecret: null nodeSelector: {} tolerations: [] diff --git a/charts/gpu-base-operator/crds/clusterpolicies.yaml b/charts/gpu-base-operator/crds/clusterpolicies.yaml index 870f9c5..af78116 100644 --- a/charts/gpu-base-operator/crds/clusterpolicies.yaml +++ b/charts/gpu-base-operator/crds/clusterpolicies.yaml @@ -24,6 +24,9 @@ spec: - jsonPath: .status.xpuManagerStatus name: XPU type: string + - jsonPath: .status.kmmStatus + name: KMM + type: string - jsonPath: .metadata.creationTimestamp name: Age type: date @@ -136,6 +139,147 @@ spec: minimum: 1 type: integer type: object + kernelModule: + description: |- + KernelModule configures out-of-tree kernel module loading via KMM. + When set, KMM loads the specified OOT driver module on each node. + When nil, the in-tree kernel driver is used. + properties: + firmwarePath: + description: FirmwarePath is the in-container path where firmware + files are stored. + type: string + image: + description: |- + Image is the default container image for the OOT driver module. + When KernelMappings is empty, generates a single wildcard mapping. + When KernelMappings is non-empty, serves as the KMM-level fallback + for mappings that omit ContainerImage. + type: string + inTreeModulesToRemove: + description: |- + InTreeModulesToRemove lists in-tree modules to unload before + inserting the OOT module. ModuleName is always included implicitly. + items: + type: string + type: array + kernelMappings: + description: |- + KernelMappings maps kernel version patterns to container images or + build specifications. Translates directly to KMM KernelMapping + objects. When empty, a single wildcard mapping (regexp "^.+$") is + generated from Image. + items: + description: |- + KernelMappingSpec maps a kernel version pattern to a container image + or build specification. + properties: + build: + description: |- + Build configures in-cluster building of the driver image via KMM. + When set, KMM builds the image if it doesn't exist in the registry. + properties: + buildArgs: + description: BuildArgs are key-value pairs passed to + the image builder. + items: + description: BuildArg is a key-value pair passed as + a build argument. + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + dockerfileConfigMap: + description: DockerfileConfigMap references a ConfigMap + containing the Dockerfile. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + secrets: + description: |- + Secrets are made available during the build (e.g., for private + source repos). Not for registry auth -- use pullSecret on + ClusterPolicySpec. + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + type: array + required: + - dockerfileConfigMap + type: object + containerImage: + description: |- + ContainerImage is the full image reference for this kernel version. + Required when Build is nil and no parent-level Image is set. + type: string + inTreeModulesToRemove: + description: |- + InTreeModulesToRemove overrides the parent-level list for this + specific kernel mapping. + items: + type: string + type: array + literal: + description: |- + Literal is an exact kernel version string to match. + Exactly one of Regexp or Literal must be set. + type: string + regexp: + description: |- + Regexp is a regular expression matched against node kernel versions. + Exactly one of Regexp or Literal must be set. + type: string + type: object + type: array + moduleName: + description: |- + ModuleName is the kernel module to load (e.g., "xe"). + Also used as the default InTreeModulesToRemove entry. + type: string + modulesLoadingOrder: + description: |- + ModulesLoadingOrder specifies softdep-style loading order for + multi-module drivers. First element must be ModuleName; KMM loads + in order and unloads in reverse. Must have >=2 entries if set. + items: + type: string + type: array + skipTLSVerify: + description: |- + SkipTLSVerify disables TLS certificate verification when pulling + OOT driver images. For air-gapped or internal registries. + type: boolean + required: + - moduleName + type: object kueue: description: Define Kueue queues properties: @@ -289,6 +433,8 @@ spec: items: type: string type: array + kmmStatus: + type: string xpuManagerStatus: type: string type: object diff --git a/charts/gpu-base-operator/templates/dp_scc.yaml b/charts/gpu-base-operator/templates/dp_scc.yaml new file mode 100644 index 0000000..de6ae1a --- /dev/null +++ b/charts/gpu-base-operator/templates/dp_scc.yaml @@ -0,0 +1,38 @@ +{{- if .Values.openshift.enabled }} +apiVersion: security.openshift.io/v1 +kind: SecurityContextConstraints +metadata: + name: {{ .Release.Name }}-dp-scc + labels: + app.kubernetes.io/name: intel-gpu-base-operator + app.kubernetes.io/managed-by: {{ .Release.Service }} +allowPrivilegedContainer: true +allowHostDirVolumePlugin: true +allowHostIPC: false +allowHostNetwork: false +allowHostPID: false +allowHostPorts: false +allowPrivilegeEscalation: true +allowedCapabilities: null +defaultAddCapabilities: null +fsGroup: + type: RunAsAny +readOnlyRootFilesystem: false +requiredDropCapabilities: null +runAsUser: + type: RunAsAny +seLinuxContext: + type: RunAsAny +seccompProfiles: +- "*" +supplementalGroups: + type: RunAsAny +volumes: +- hostPath +- emptyDir +- projected +- secret +- configMap +users: [] +groups: [] +{{- end }} diff --git a/charts/gpu-base-operator/templates/dp_scc_role.yaml b/charts/gpu-base-operator/templates/dp_scc_role.yaml new file mode 100644 index 0000000..1776506 --- /dev/null +++ b/charts/gpu-base-operator/templates/dp_scc_role.yaml @@ -0,0 +1,14 @@ +{{- if .Values.openshift.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Release.Name }}-dp-scc-role + labels: + app.kubernetes.io/name: intel-gpu-base-operator + app.kubernetes.io/managed-by: {{ .Release.Service }} +rules: +- apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + resourceNames: ["{{ .Release.Name }}-dp-scc"] + verbs: ["use"] +{{- end }} diff --git a/charts/gpu-base-operator/templates/dp_scc_rolebinding.yaml b/charts/gpu-base-operator/templates/dp_scc_rolebinding.yaml new file mode 100644 index 0000000..6b5f39e --- /dev/null +++ b/charts/gpu-base-operator/templates/dp_scc_rolebinding.yaml @@ -0,0 +1,17 @@ +{{- if .Values.openshift.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Release.Name }}-dp-scc-binding + labels: + app.kubernetes.io/name: intel-gpu-base-operator + app.kubernetes.io/managed-by: {{ .Release.Service }} +subjects: +- kind: ServiceAccount + name: {{ .Values.dp.serviceAccountName }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ .Release.Name }}-dp-scc-role + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/charts/gpu-base-operator/templates/dp_serviceaccount.yaml b/charts/gpu-base-operator/templates/dp_serviceaccount.yaml new file mode 100644 index 0000000..fba0ef6 --- /dev/null +++ b/charts/gpu-base-operator/templates/dp_serviceaccount.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.dp.serviceAccountName }} + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: intel-gpu-base-operator + app.kubernetes.io/managed-by: {{ .Release.Service }} diff --git a/charts/gpu-base-operator/templates/dra_admission_policy.yaml b/charts/gpu-base-operator/templates/dra_admission_policy.yaml new file mode 100644 index 0000000..0b68093 --- /dev/null +++ b/charts/gpu-base-operator/templates/dra_admission_policy.yaml @@ -0,0 +1,34 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: {{ .Release.Name }}-dra-resourceslices + labels: + app.kubernetes.io/name: intel-gpu-base-operator + app.kubernetes.io/managed-by: {{ .Release.Service }} +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: ["resource.k8s.io"] + apiVersions: ["v1beta1"] + operations: ["CREATE", "UPDATE", "DELETE"] + resources: ["resourceslices"] + matchConditions: + - name: isRestrictedUser + expression: >- + request.userInfo.username == "system:serviceaccount:{{ .Release.Namespace }}:{{ .Values.dra.serviceAccountName }}" + variables: + - name: userNodeName + expression: >- + request.userInfo.extra[?'authentication.kubernetes.io/node-name'][0].orValue('') + - name: objectNodeName + expression: >- + (request.operation == "DELETE" ? oldObject : object).spec.?nodeName.orValue("") + validations: + - expression: variables.userNodeName != "" + message: >- + no node association found for user, this user must run in a pod on a node and ServiceAccountTokenPodNodeInfo must be enabled + - expression: variables.userNodeName == variables.objectNodeName + messageExpression: >- + "this user running on node '"+variables.userNodeName+"' may not modify " + + (variables.objectNodeName == "" ?"cluster resourceslices" : "resourceslices on node '"+variables.objectNodeName+"'") diff --git a/charts/gpu-base-operator/templates/dra_admission_policy_binding.yaml b/charts/gpu-base-operator/templates/dra_admission_policy_binding.yaml new file mode 100644 index 0000000..09e5243 --- /dev/null +++ b/charts/gpu-base-operator/templates/dra_admission_policy_binding.yaml @@ -0,0 +1,10 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: {{ .Release.Name }}-dra-resourceslices + labels: + app.kubernetes.io/name: intel-gpu-base-operator + app.kubernetes.io/managed-by: {{ .Release.Service }} +spec: + policyName: {{ .Release.Name }}-dra-resourceslices + validationActions: [Deny] diff --git a/charts/gpu-base-operator/templates/dra_clusterrole.yaml b/charts/gpu-base-operator/templates/dra_clusterrole.yaml new file mode 100644 index 0000000..d908ec5 --- /dev/null +++ b/charts/gpu-base-operator/templates/dra_clusterrole.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Release.Name }}-dra + labels: + app.kubernetes.io/name: intel-gpu-base-operator + app.kubernetes.io/managed-by: {{ .Release.Service }} +rules: +- apiGroups: [""] + resources: ["nodes"] + verbs: ["get"] +- apiGroups: ["resource.k8s.io"] + resources: ["resourceslices"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +- apiGroups: ["resource.k8s.io"] + resources: ["resourceclaims"] + verbs: ["get"] diff --git a/charts/gpu-base-operator/templates/dra_clusterrolebinding.yaml b/charts/gpu-base-operator/templates/dra_clusterrolebinding.yaml new file mode 100644 index 0000000..8941072 --- /dev/null +++ b/charts/gpu-base-operator/templates/dra_clusterrolebinding.yaml @@ -0,0 +1,15 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Release.Name }}-dra + labels: + app.kubernetes.io/name: intel-gpu-base-operator + app.kubernetes.io/managed-by: {{ .Release.Service }} +subjects: +- kind: ServiceAccount + name: {{ .Values.dra.serviceAccountName }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ .Release.Name }}-dra + apiGroup: rbac.authorization.k8s.io diff --git a/charts/gpu-base-operator/templates/dra_scc.yaml b/charts/gpu-base-operator/templates/dra_scc.yaml new file mode 100644 index 0000000..e842d01 --- /dev/null +++ b/charts/gpu-base-operator/templates/dra_scc.yaml @@ -0,0 +1,37 @@ +{{- if .Values.openshift.enabled }} +apiVersion: security.openshift.io/v1 +kind: SecurityContextConstraints +metadata: + name: {{ .Release.Name }}-dra-scc + labels: + app.kubernetes.io/name: intel-gpu-base-operator + app.kubernetes.io/managed-by: {{ .Release.Service }} +allowPrivilegedContainer: true +allowHostDirVolumePlugin: true +allowHostIPC: false +allowHostNetwork: true +allowHostPID: false +allowHostPorts: false +allowPrivilegeEscalation: true +allowedCapabilities: null +defaultAddCapabilities: null +fsGroup: + type: RunAsAny +readOnlyRootFilesystem: false +requiredDropCapabilities: null +runAsUser: + type: RunAsAny +seLinuxContext: + type: RunAsAny +seccompProfiles: +- "*" +supplementalGroups: + type: RunAsAny +volumes: +- hostPath +- projected +- secret +- configMap +users: [] +groups: [] +{{- end }} diff --git a/charts/gpu-base-operator/templates/dra_scc_role.yaml b/charts/gpu-base-operator/templates/dra_scc_role.yaml new file mode 100644 index 0000000..9fb338b --- /dev/null +++ b/charts/gpu-base-operator/templates/dra_scc_role.yaml @@ -0,0 +1,14 @@ +{{- if .Values.openshift.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Release.Name }}-dra-scc-role + labels: + app.kubernetes.io/name: intel-gpu-base-operator + app.kubernetes.io/managed-by: {{ .Release.Service }} +rules: +- apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + resourceNames: ["{{ .Release.Name }}-dra-scc"] + verbs: ["use"] +{{- end }} diff --git a/charts/gpu-base-operator/templates/dra_scc_rolebinding.yaml b/charts/gpu-base-operator/templates/dra_scc_rolebinding.yaml new file mode 100644 index 0000000..c84bedf --- /dev/null +++ b/charts/gpu-base-operator/templates/dra_scc_rolebinding.yaml @@ -0,0 +1,17 @@ +{{- if .Values.openshift.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Release.Name }}-dra-scc-binding + labels: + app.kubernetes.io/name: intel-gpu-base-operator + app.kubernetes.io/managed-by: {{ .Release.Service }} +subjects: +- kind: ServiceAccount + name: {{ .Values.dra.serviceAccountName }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ .Release.Name }}-dra-scc-role + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/charts/gpu-base-operator/templates/dra_serviceaccount.yaml b/charts/gpu-base-operator/templates/dra_serviceaccount.yaml new file mode 100644 index 0000000..f75b865 --- /dev/null +++ b/charts/gpu-base-operator/templates/dra_serviceaccount.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.dra.serviceAccountName }} + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: intel-gpu-base-operator + app.kubernetes.io/managed-by: {{ .Release.Service }} diff --git a/charts/gpu-base-operator/templates/manager.yaml b/charts/gpu-base-operator/templates/manager.yaml index bec40f0..4fda946 100644 --- a/charts/gpu-base-operator/templates/manager.yaml +++ b/charts/gpu-base-operator/templates/manager.yaml @@ -64,6 +64,10 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: DP_SERVICE_ACCOUNT_NAME + value: {{ .Values.dp.serviceAccountName }} + - name: DRA_SERVICE_ACCOUNT_NAME + value: {{ .Values.dra.serviceAccountName }} {{- if .Values.privateRegistry.token }} - name: OPERATOR_SECRET value: "{{ .Release.Name }}-operator-private-registry" diff --git a/charts/gpu-base-operator/templates/role.yaml b/charts/gpu-base-operator/templates/role.yaml index c556fbd..51b89f7 100644 --- a/charts/gpu-base-operator/templates/role.yaml +++ b/charts/gpu-base-operator/templates/role.yaml @@ -78,6 +78,24 @@ rules: - get - patch - update +- apiGroups: + - kmm.sigs.x-k8s.io + resources: + - modules + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kmm.sigs.x-k8s.io + resources: + - modules/status + verbs: + - get - apiGroups: - kueue.x-k8s.io resources: diff --git a/charts/gpu-base-operator/values.yaml b/charts/gpu-base-operator/values.yaml index 8f7a648..85d90cc 100644 --- a/charts/gpu-base-operator/values.yaml +++ b/charts/gpu-base-operator/values.yaml @@ -19,6 +19,15 @@ operator: cpu: 100m memory: 256Mi +dp: + serviceAccountName: intel-gpu-dp + +dra: + serviceAccountName: intel-gpu-dra + +openshift: + enabled: false + privateRegistry: url: "" user: "" diff --git a/cmd/main.go b/cmd/main.go index 59dea7c..48ba33a 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -45,6 +45,7 @@ import ( "k8s.io/client-go/discovery" "k8s.io/client-go/rest" + kmmv1beta1 "github.com/kubernetes-sigs/kernel-module-management/api/v1beta1" prometheusv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" corev1 "k8s.io/api/core/v1" resv1 "k8s.io/api/resource/v1" @@ -63,6 +64,7 @@ var ( setupLog = ctrl.Log.WithName("setup") openShiftGroups = []string{"route.openshift.io", "security.openshift.io"} draGroups = []string{"resource.k8s.io"} + kmmGroups = []string{"kmm.sigs.x-k8s.io"} ) const ( @@ -70,6 +72,7 @@ const ( openshiftCluster = "OpenShift" draCluster = "DRA" + kmmCluster = "KMM" ) func init() { @@ -80,6 +83,7 @@ func init() { utilruntime.Must(prometheusv1.AddToScheme(scheme)) utilruntime.Must(resv1.AddToScheme(scheme)) utilruntime.Must(kueuev1.AddToScheme(scheme)) + utilruntime.Must(kmmv1beta1.AddToScheme(scheme)) // +kubebuilder:scaffold:scheme } @@ -103,6 +107,7 @@ func detectClusterFeatures() (map[string]bool, error) { features := map[string]bool{ openshiftCluster: false, draCluster: false, + kmmCluster: false, } for _, group := range apiGroups.Groups { @@ -112,6 +117,9 @@ func detectClusterFeatures() (map[string]bool, error) { if slices.Contains(draGroups, group.Name) { features[draCluster] = true } + if slices.Contains(kmmGroups, group.Name) { + features[kmmCluster] = true + } } return features, nil @@ -323,12 +331,31 @@ func main() { setupLog.Info("Operator secret supplied", "secret", secret) } + dpSAName := os.Getenv("DP_SERVICE_ACCOUNT_NAME") + if dpSAName == "" { + dpSAName = "intel-gpu-dp" + } + + draSAName := os.Getenv("DRA_SERVICE_ACCOUNT_NAME") + if draSAName == "" { + draSAName = "intel-gpu-dra" + } + + moduleLoaderSAName := os.Getenv("MODULE_LOADER_SERVICE_ACCOUNT_NAME") + if moduleLoaderSAName == "" { + moduleLoaderSAName = "intel-gpu-module-loader" + } + copts := controller.ControllerOpts{ - Namespace: ns, - SecretName: secret, - RequeueDelay: time.Second * 5, - DRAEnable: features[draCluster], - OpenShift: features[openshiftCluster], + Namespace: ns, + SecretName: secret, + DPServiceAccountName: dpSAName, + DRAServiceAccountName: draSAName, + ModuleLoaderServiceAccountName: moduleLoaderSAName, + RequeueDelay: time.Second * 5, + DRAEnable: features[draCluster], + KMMEnable: features[kmmCluster], + OpenShift: features[openshiftCluster], } if err := (&controller.ClusterPolicyReconciler{ diff --git a/config/crd/bases/intel.com_clusterpolicies.yaml b/config/crd/bases/intel.com_clusterpolicies.yaml index 870f9c5..af78116 100644 --- a/config/crd/bases/intel.com_clusterpolicies.yaml +++ b/config/crd/bases/intel.com_clusterpolicies.yaml @@ -24,6 +24,9 @@ spec: - jsonPath: .status.xpuManagerStatus name: XPU type: string + - jsonPath: .status.kmmStatus + name: KMM + type: string - jsonPath: .metadata.creationTimestamp name: Age type: date @@ -136,6 +139,147 @@ spec: minimum: 1 type: integer type: object + kernelModule: + description: |- + KernelModule configures out-of-tree kernel module loading via KMM. + When set, KMM loads the specified OOT driver module on each node. + When nil, the in-tree kernel driver is used. + properties: + firmwarePath: + description: FirmwarePath is the in-container path where firmware + files are stored. + type: string + image: + description: |- + Image is the default container image for the OOT driver module. + When KernelMappings is empty, generates a single wildcard mapping. + When KernelMappings is non-empty, serves as the KMM-level fallback + for mappings that omit ContainerImage. + type: string + inTreeModulesToRemove: + description: |- + InTreeModulesToRemove lists in-tree modules to unload before + inserting the OOT module. ModuleName is always included implicitly. + items: + type: string + type: array + kernelMappings: + description: |- + KernelMappings maps kernel version patterns to container images or + build specifications. Translates directly to KMM KernelMapping + objects. When empty, a single wildcard mapping (regexp "^.+$") is + generated from Image. + items: + description: |- + KernelMappingSpec maps a kernel version pattern to a container image + or build specification. + properties: + build: + description: |- + Build configures in-cluster building of the driver image via KMM. + When set, KMM builds the image if it doesn't exist in the registry. + properties: + buildArgs: + description: BuildArgs are key-value pairs passed to + the image builder. + items: + description: BuildArg is a key-value pair passed as + a build argument. + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + dockerfileConfigMap: + description: DockerfileConfigMap references a ConfigMap + containing the Dockerfile. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + secrets: + description: |- + Secrets are made available during the build (e.g., for private + source repos). Not for registry auth -- use pullSecret on + ClusterPolicySpec. + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + type: array + required: + - dockerfileConfigMap + type: object + containerImage: + description: |- + ContainerImage is the full image reference for this kernel version. + Required when Build is nil and no parent-level Image is set. + type: string + inTreeModulesToRemove: + description: |- + InTreeModulesToRemove overrides the parent-level list for this + specific kernel mapping. + items: + type: string + type: array + literal: + description: |- + Literal is an exact kernel version string to match. + Exactly one of Regexp or Literal must be set. + type: string + regexp: + description: |- + Regexp is a regular expression matched against node kernel versions. + Exactly one of Regexp or Literal must be set. + type: string + type: object + type: array + moduleName: + description: |- + ModuleName is the kernel module to load (e.g., "xe"). + Also used as the default InTreeModulesToRemove entry. + type: string + modulesLoadingOrder: + description: |- + ModulesLoadingOrder specifies softdep-style loading order for + multi-module drivers. First element must be ModuleName; KMM loads + in order and unloads in reverse. Must have >=2 entries if set. + items: + type: string + type: array + skipTLSVerify: + description: |- + SkipTLSVerify disables TLS certificate verification when pulling + OOT driver images. For air-gapped or internal registries. + type: boolean + required: + - moduleName + type: object kueue: description: Define Kueue queues properties: @@ -289,6 +433,8 @@ spec: items: type: string type: array + kmmStatus: + type: string xpuManagerStatus: type: string type: object diff --git a/config/deployments/deployments.go b/config/deployments/deployments.go index 9f61eb5..156d2ec 100644 --- a/config/deployments/deployments.go +++ b/config/deployments/deployments.go @@ -16,13 +16,10 @@ package deployments import ( _ "embed" - "fmt" - adreg "k8s.io/api/admissionregistration/v1" apps "k8s.io/api/apps/v1" batch "k8s.io/api/batch/v1" core "k8s.io/api/core/v1" - rbac "k8s.io/api/rbac/v1" prometheusv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" resv1 "k8s.io/api/resource/v1" @@ -31,11 +28,6 @@ import ( "sigs.k8s.io/yaml" ) -const ( - vfioExpression = `device.attributes["gpu.intel.com"].driver == 'vfio-pci'` - xeVfioExpression = `device.attributes["gpu.intel.com"].driver == 'xe-vfio-pci'` -) - // XPU Manager //go:embed xpum/xpum.yaml @@ -52,84 +44,8 @@ func XpuManagerOTelConfig() *OTelConfig { return getOTelConfig(contentXpumOTelConfig) } -// Device Plugin - -//go:embed dp/dp.yaml -var contentDPDs []byte - -func DevicePluginDaemonset() *apps.DaemonSet { - return getDaemonset(contentDPDs).DeepCopy() -} - // DRA -//go:embed dra/daemonset.yaml -var contentDRADs []byte - -func DynamicResourceAllocationDaemonset() *apps.DaemonSet { - return getDaemonset(contentDRADs).DeepCopy() -} - -//go:embed dra/clusterrole.yaml -var contentDRACR []byte - -func DynamicResourceAllocationClusterRole() *rbac.ClusterRole { - return getClusterRole(contentDRACR).DeepCopy() -} - -//go:embed dra/clusterrolebinding.yaml -var contentDRACRB []byte - -func DynamicResourceAllocationClusterRoleBinding() *rbac.ClusterRoleBinding { - return getClusterRoleBinding(contentDRACRB).DeepCopy() -} - -//go:embed dra/serviceaccount.yaml -var contentDRASA []byte - -func DynamicResourceAllocationServiceAccount() *core.ServiceAccount { - return getServiceAccount(contentDRASA).DeepCopy() -} - -//go:embed dra/device-class.yaml -var contentDRADC []byte - -func DynamicResourceAllocationDeviceClass() *resv1.DeviceClass { - return getDeviceClass(contentDRADC).DeepCopy() -} - -//go:embed dra/device-class-vfio.yaml -var contentDRADCVfio []byte - -func DynamicResourceAllocationDeviceClassVfio(limitToVfio bool) *resv1.DeviceClass { - dc := getDeviceClass(contentDRADCVfio).DeepCopy() - - // Limit VFIO device class to only VFIO-bound devices. - if limitToVfio { - dc.Spec.Selectors = append(dc.Spec.Selectors, resv1.DeviceSelector{ - CEL: &resv1.CELDeviceSelector{ - Expression: fmt.Sprintf("%s || %s", vfioExpression, xeVfioExpression), - }, - }) - } - - return dc -} - -//go:embed dra/admissionpolicy.yaml -var contentDRAAP []byte - -func DynamicResourceAllocationValidatingAdmissionPolicy() *adreg.ValidatingAdmissionPolicy { - return getAdmissionPolicy(contentDRAAP).DeepCopy() -} - -//go:embed dra/admissionpolicybinding.yaml -var contentDRAAPB []byte - -func DynamicResourceAllocationValidatingAdmissionPolicyBinding() *adreg.ValidatingAdmissionPolicyBinding { - return getAdmissionPolicyBinding(contentDRAAPB).DeepCopy() -} - //go:embed dra/monitorclaimtemplate.yaml var contentDRAMCT []byte @@ -193,72 +109,6 @@ func getService(content []byte) *core.Service { return &result } -func getServiceAccount(content []byte) *core.ServiceAccount { - var result core.ServiceAccount - - err := yaml.Unmarshal(content, &result) - if err != nil { - panic(err) - } - - return &result -} - -func getClusterRole(content []byte) *rbac.ClusterRole { - var result rbac.ClusterRole - - err := yaml.Unmarshal(content, &result) - if err != nil { - panic(err) - } - - return &result -} - -func getClusterRoleBinding(content []byte) *rbac.ClusterRoleBinding { - var result rbac.ClusterRoleBinding - - err := yaml.Unmarshal(content, &result) - if err != nil { - panic(err) - } - - return &result -} - -func getAdmissionPolicy(content []byte) *adreg.ValidatingAdmissionPolicy { - var result adreg.ValidatingAdmissionPolicy - - err := yaml.Unmarshal(content, &result) - if err != nil { - panic(err) - } - - return &result -} - -func getAdmissionPolicyBinding(content []byte) *adreg.ValidatingAdmissionPolicyBinding { - var result adreg.ValidatingAdmissionPolicyBinding - - err := yaml.Unmarshal(content, &result) - if err != nil { - panic(err) - } - - return &result -} - -func getDeviceClass(content []byte) *resv1.DeviceClass { - var result resv1.DeviceClass - - err := yaml.Unmarshal(content, &result) - if err != nil { - panic(err) - } - - return &result -} - func getResourceClaimTemplate(content []byte) *resv1.ResourceClaimTemplate { var result resv1.ResourceClaimTemplate diff --git a/config/deployments/deployments_test.go b/config/deployments/deployments_test.go index 17784cc..5825d26 100644 --- a/config/deployments/deployments_test.go +++ b/config/deployments/deployments_test.go @@ -51,69 +51,6 @@ func TestXpuManagerDaemonset(t *testing.T) { } } -func TestDevicePluginDaemonset(t *testing.T) { - ds := DevicePluginDaemonset() - if ds == nil { - t.Error("DevicePluginDaemonset returned nil") - } -} - -func TestDynamicResourceAllocationDaemonset(t *testing.T) { - ds := DynamicResourceAllocationDaemonset() - if ds == nil { - t.Error("DynamicResourceAllocationDaemonset returned nil") - } -} - -func TestDynamicResourceAllocationClusterRole(t *testing.T) { - cr := DynamicResourceAllocationClusterRole() - if cr == nil { - t.Error("DynamicResourceAllocationClusterRole returned nil") - } -} - -func TestDynamicResourceAllocationClusterRoleBinding(t *testing.T) { - crb := DynamicResourceAllocationClusterRoleBinding() - if crb == nil { - t.Error("DynamicResourceAllocationClusterRoleBinding returned nil") - } -} - -func TestDynamicResourceAllocationServiceAccount(t *testing.T) { - sa := DynamicResourceAllocationServiceAccount() - if sa == nil { - t.Error("DynamicResourceAllocationServiceAccount returned nil") - } -} - -func TestDynamicResourceAllocationDeviceClass(t *testing.T) { - dc := DynamicResourceAllocationDeviceClass() - if dc == nil { - t.Error("DynamicResourceAllocationDeviceClass returned nil") - } -} - -func TestDynamicResourceAllocationDeviceClassVfio(t *testing.T) { - dc := DynamicResourceAllocationDeviceClassVfio(false) - if dc == nil { - t.Error("DynamicResourceAllocationDeviceClassVfio returned nil") - } -} - -func TestDynamicResourceAllocationValidatingAdmissionPolicy(t *testing.T) { - ap := DynamicResourceAllocationValidatingAdmissionPolicy() - if ap == nil { - t.Error("DynamicResourceAllocationValidatingAdmissionPolicy returned nil") - } -} - -func TestDynamicResourceAllocationValidatingAdmissionPolicyBinding(t *testing.T) { - apb := DynamicResourceAllocationValidatingAdmissionPolicyBinding() - if apb == nil { - t.Error("DynamicResourceAllocationValidatingAdmissionPolicyBinding returned nil") - } -} - func TestDynamicResourceAllocationMonitorClaimTemplate(t *testing.T) { mct := DynamicResourceAllocationMonitorClaimTemplate() if mct == nil { @@ -156,33 +93,6 @@ func TestOTelConfig(t *testing.T) { } } -func TestDevicePluginDaemonset_AutomountServiceAccountToken(t *testing.T) { - ds := DevicePluginDaemonset() - if ds.Spec.Template.Spec.AutomountServiceAccountToken == nil { - t.Fatal("automountServiceAccountToken must be set (non-nil)") - } - if *ds.Spec.Template.Spec.AutomountServiceAccountToken != false { - t.Error("automountServiceAccountToken must be false") - } -} - -func TestDynamicResourceAllocationDaemonset_KubeletPluginAllowPrivilegeEscalation(t *testing.T) { - ds := DynamicResourceAllocationDaemonset() - c := findContainer(ds.Spec.Template.Spec.Containers, "kubelet-plugin") - if c == nil { - t.Fatal("kubelet-plugin container not found") - } - if c.SecurityContext == nil { - t.Fatal("SecurityContext must be set on kubelet-plugin") - } - if c.SecurityContext.AllowPrivilegeEscalation == nil { - t.Fatal("AllowPrivilegeEscalation must be set (non-nil) on kubelet-plugin") - } - if *c.SecurityContext.AllowPrivilegeEscalation != false { - t.Error("AllowPrivilegeEscalation must be false on kubelet-plugin") - } -} - func TestXpuManagerDaemonset_AutomountServiceAccountToken(t *testing.T) { ds := XpuManagerDaemonset() if ds.Spec.Template.Spec.AutomountServiceAccountToken == nil { @@ -275,85 +185,6 @@ func TestXpuFwUpdateJob_UpdaterContainerSecurityContext(t *testing.T) { } } -func TestDevicePluginDaemonset_PluginContainerSecurityContext(t *testing.T) { - ds := DevicePluginDaemonset() - c := findContainer(ds.Spec.Template.Spec.Containers, "intel-gpu-plugin") - if c == nil { - t.Fatal("intel-gpu-plugin container not found") - } - if c.SecurityContext == nil { - t.Fatal("SecurityContext must be set on intel-gpu-plugin") - } - if c.SecurityContext.AllowPrivilegeEscalation == nil { - t.Fatal("AllowPrivilegeEscalation must be set (non-nil) on intel-gpu-plugin") - } - if *c.SecurityContext.AllowPrivilegeEscalation { - t.Error("AllowPrivilegeEscalation must be false on intel-gpu-plugin") - } - if c.SecurityContext.ReadOnlyRootFilesystem == nil { - t.Fatal("ReadOnlyRootFilesystem must be set (non-nil) on intel-gpu-plugin") - } - if !*c.SecurityContext.ReadOnlyRootFilesystem { - t.Error("ReadOnlyRootFilesystem must be true on intel-gpu-plugin") - } - if c.SecurityContext.Capabilities == nil { - t.Fatal("Capabilities must be set on intel-gpu-plugin") - } - found := false - for _, cap := range c.SecurityContext.Capabilities.Drop { - if cap == allCaps { - found = true - break - } - } - if !found { - t.Error("capabilities.drop must contain ALL on intel-gpu-plugin") - } - if c.SecurityContext.SeccompProfile == nil { - t.Fatal("SeccompProfile must be set on intel-gpu-plugin") - } - if c.SecurityContext.SeccompProfile.Type != core.SeccompProfileTypeRuntimeDefault { - t.Errorf("SeccompProfile.Type: got %v, want RuntimeDefault on intel-gpu-plugin", - c.SecurityContext.SeccompProfile.Type) - } -} - -func TestDynamicResourceAllocationDaemonset_KubeletPluginFullSecurityContext(t *testing.T) { - ds := DynamicResourceAllocationDaemonset() - c := findContainer(ds.Spec.Template.Spec.Containers, "kubelet-plugin") - if c == nil { - t.Fatal("kubelet-plugin container not found") - } - if c.SecurityContext == nil { - t.Fatal("SecurityContext must be set on kubelet-plugin") - } - if c.SecurityContext.ReadOnlyRootFilesystem == nil { - t.Fatal("ReadOnlyRootFilesystem must be set (non-nil) on kubelet-plugin") - } - if !*c.SecurityContext.ReadOnlyRootFilesystem { - t.Error("ReadOnlyRootFilesystem must be true on kubelet-plugin") - } - if c.SecurityContext.Capabilities == nil { - t.Fatal("Capabilities must be set on kubelet-plugin") - } - found := false - for _, cap := range c.SecurityContext.Capabilities.Drop { - if cap == allCaps { - found = true - break - } - } - if !found { - t.Error("capabilities.drop must contain ALL on kubelet-plugin") - } - if c.SecurityContext.SeccompProfile == nil { - t.Fatal("SeccompProfile must be set on kubelet-plugin") - } - if c.SecurityContext.SeccompProfile.Type != core.SeccompProfileTypeRuntimeDefault { - t.Errorf("SeccompProfile.Type: got %v, want RuntimeDefault on kubelet-plugin", c.SecurityContext.SeccompProfile.Type) - } -} - func TestXpuManagerDaemonset_XpumdContainerSecurityContext(t *testing.T) { ds := XpuManagerDaemonset() c := findContainer(ds.Spec.Template.Spec.Containers, "xpumd") @@ -399,31 +230,3 @@ func TestXpuManagerDaemonset_XpumdContainerSecurityContext(t *testing.T) { t.Error("capabilities.add must contain SYS_ADMIN on xpumd (required for engine utilization metrics)") } } - -func TestDRAClusterRole_NoWildcardsAndNoSecrets(t *testing.T) { - cr := DynamicResourceAllocationClusterRole() - - for _, rule := range cr.Rules { - for _, apiGroup := range rule.APIGroups { - if apiGroup == "*" { - t.Errorf("DRA ClusterRole rule has wildcard apiGroup: %+v", rule) - } - } - - for _, resource := range rule.Resources { - if resource == "*" { - t.Errorf("DRA ClusterRole rule has wildcard resource: %+v", rule) - } - - if resource == "secrets" { - t.Errorf("DRA ClusterRole must not grant access to secrets: %+v", rule) - } - } - - for _, verb := range rule.Verbs { - if verb == "*" { - t.Errorf("DRA ClusterRole rule has wildcard verb: %+v", rule) - } - } - } -} diff --git a/config/deployments/dp/dp.yaml b/config/deployments/dp/dp.yaml deleted file mode 100644 index f41c46e..0000000 --- a/config/deployments/dp/dp.yaml +++ /dev/null @@ -1,80 +0,0 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - labels: - app: intel-gpu-plugin - name: intel-gpu-plugin -spec: - selector: - matchLabels: - app: intel-gpu-plugin - template: - metadata: - labels: - app: intel-gpu-plugin - spec: - automountServiceAccountToken: false - containers: - - args: - - -health-management - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: HOST_IP - valueFrom: - fieldRef: - fieldPath: status.hostIP - image: intel/intel-gpu-plugin:devel - imagePullPolicy: IfNotPresent - name: intel-gpu-plugin - resources: - limits: - cpu: 100m - memory: 90Mi - requests: - cpu: 40m - memory: 45Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - seLinuxOptions: - type: container_device_plugin_t - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /dev/dri - name: devfs - readOnly: true - - mountPath: /sys/class/drm - name: sysfsdrm - readOnly: true - - mountPath: /var/lib/kubelet/device-plugins - name: kubeletsockets - - mountPath: /var/run/cdi - name: cdipath - nodeSelector: - kubernetes.io/arch: amd64 - volumes: - - hostPath: - path: /dev/dri - name: devfs - - hostPath: - path: /sys/class/drm - name: sysfsdrm - - hostPath: - path: /var/lib/kubelet/device-plugins - name: kubeletsockets - - hostPath: - path: /var/run/cdi - type: DirectoryOrCreate - name: cdipath - updateStrategy: - rollingUpdate: - maxSurge: 0 - maxUnavailable: 1 - type: RollingUpdate diff --git a/config/deployments/dra/admissionpolicybinding.yaml b/config/deployments/dra/admissionpolicybinding.yaml deleted file mode 100644 index d7fbece..0000000 --- a/config/deployments/dra/admissionpolicybinding.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingAdmissionPolicyBinding -metadata: - name: resourceslices-policy-dra-kubelet-plugin-gpu -spec: - policyName: resourceslices-policy-dra-kubelet-plugin-gpu - validationActions: [Deny] diff --git a/config/deployments/dra/clusterrolebinding.yaml b/config/deployments/dra/clusterrolebinding.yaml deleted file mode 100644 index 145b8cc..0000000 --- a/config/deployments/dra/clusterrolebinding.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: intel-gpu-resource-driver-role-binding - namespace: intel-gpu-resource-driver -subjects: -- kind: ServiceAccount - name: intel-gpu-resource-driver-service-account - namespace: intel-gpu-resource-driver -roleRef: - kind: ClusterRole - name: intel-gpu-resource-driver-role - apiGroup: rbac.authorization.k8s.io diff --git a/config/deployments/dra/daemonset.yaml b/config/deployments/dra/daemonset.yaml deleted file mode 100644 index 89dce71..0000000 --- a/config/deployments/dra/daemonset.yaml +++ /dev/null @@ -1,90 +0,0 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: intel-gpu-resource-driver-kubelet-plugin - namespace: intel-gpu-resource-driver - labels: - app: intel-gpu-resource-driver-kubelet-plugin -spec: - selector: - matchLabels: - app: intel-gpu-resource-driver-kubelet-plugin - template: - metadata: - labels: - app: intel-gpu-resource-driver-kubelet-plugin - spec: - serviceAccount: intel-gpu-resource-driver-service-account - serviceAccountName: intel-gpu-resource-driver-service-account - containers: - - name: kubelet-plugin - image: ghcr.io/intel/intel-resource-drivers-for-kubernetes/intel-gpu-resource-driver:v0.9.0 - imagePullPolicy: IfNotPresent - command: ["/kubelet-gpu-plugin"] - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: SYSFS_ROOT - value: "/sysfs" - # Only use devdri when using fake devfs with device-faker. - # Use this to tell kubelet-plugin where the fake DRI devices nodes are. - # This will be prefix for CDI devices, runtime will try to mount devices - # with this prefix into workloads, so the path should be equal to where - # fake DRI files are in real host. - #- name: DEV_DRI_PATH - # value: "/tmp/test-0123456789/dev/dri" - volumeMounts: - - name: plugins-registry - mountPath: /var/lib/kubelet/plugins_registry - - name: plugins - mountPath: /var/lib/kubelet/plugins - - name: cdi - mountPath: /etc/cdi - - name: varruncdi - mountPath: /var/run/cdi - - name: xpumdrundir - mountPath: /run/xpumd - # when using fake sysfs - mount at the same place as on host - - name: sysfs - mountPath: "/sysfs" - # Only use devdri when using fake devfs with device-faker - #- name: devdri - # mountPath: "/tmp/test-0123456789/dev/dri" - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: [ "ALL" ] - readOnlyRootFilesystem: true - runAsUser: 0 - seccompProfile: - type: RuntimeDefault - volumes: - - name: plugins-registry - hostPath: - path: /var/lib/kubelet/plugins_registry - - name: plugins - hostPath: - path: /var/lib/kubelet/plugins - - name: cdi - hostPath: - path: /etc/cdi - - name: varruncdi - hostPath: - path: /var/run/cdi - - name: sysfs - hostPath: - path: /sys - - name: xpumdrundir - hostPath: - path: /run/xpumd - type: DirectoryOrCreate - # Only use devdri when using fake devfs with device-faker - #- name: devdri - # hostPath: - # path: /tmp/test-0123456789/dev/dri diff --git a/config/deployments/dra/device-class-vfio.yaml b/config/deployments/dra/device-class-vfio.yaml deleted file mode 100644 index 0922522..0000000 --- a/config/deployments/dra/device-class-vfio.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: resource.k8s.io/v1 -kind: DeviceClass -metadata: - name: gpu-vfio.intel.com -spec: - selectors: - - cel: - expression: device.driver == "gpu.intel.com" - # Uncomment if active binding management is disabled in kubelet-plugin, to prevent non-VFIO bound GPUs - # from being allocated to VM-based workloads. - #- cel: - # expression: device.attributes["gpu.intel.com"].driver == 'vfio-pci' || device.attributes["gpu.intel.com"].driver == 'xe-vfio-pci' diff --git a/config/deployments/dra/device-class.yaml b/config/deployments/dra/device-class.yaml deleted file mode 100644 index 64f62c0..0000000 --- a/config/deployments/dra/device-class.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: resource.k8s.io/v1 -kind: DeviceClass -metadata: - name: gpu.intel.com - -spec: - selectors: - - cel: - expression: device.driver == "gpu.intel.com" - # Available in K8s v1.34 requires feature gate enabled - # See https://github.com/kubernetes/enhancements/tree/master/keps/sig-scheduling/5004-dra-extended-resource - extendedResourceName: intel.com/gpu diff --git a/config/deployments/dra/serviceaccount.yaml b/config/deployments/dra/serviceaccount.yaml deleted file mode 100644 index 008d32f..0000000 --- a/config/deployments/dra/serviceaccount.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: intel-gpu-resource-driver-service-account - namespace: intel-gpu-resource-driver diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 201c738..857e886 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -69,6 +69,10 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: DP_SERVICE_ACCOUNT_NAME + value: intel-gpu-base-operator-intel-gpu-dp + - name: DRA_SERVICE_ACCOUNT_NAME + value: intel-gpu-base-operator-intel-gpu-dra image: ghcr.io/intel/intel-gpu-base-operator:devel imagePullPolicy: IfNotPresent name: manager diff --git a/config/rbac/dp_scc.yaml b/config/rbac/dp_scc.yaml new file mode 100644 index 0000000..ddccd03 --- /dev/null +++ b/config/rbac/dp_scc.yaml @@ -0,0 +1,35 @@ +apiVersion: security.openshift.io/v1 +kind: SecurityContextConstraints +metadata: + name: intel-gpu-dp-scc + labels: + app.kubernetes.io/name: intel-gpu-base-operator +allowPrivilegedContainer: true +allowHostDirVolumePlugin: true +allowHostIPC: false +allowHostNetwork: false +allowHostPID: false +allowHostPorts: false +allowPrivilegeEscalation: true +allowedCapabilities: null +defaultAddCapabilities: null +fsGroup: + type: RunAsAny +readOnlyRootFilesystem: false +requiredDropCapabilities: null +runAsUser: + type: RunAsAny +seLinuxContext: + type: RunAsAny +seccompProfiles: +- "*" +supplementalGroups: + type: RunAsAny +volumes: +- hostPath +- emptyDir +- projected +- secret +- configMap +users: [] +groups: [] diff --git a/config/rbac/dp_scc_role.yaml b/config/rbac/dp_scc_role.yaml new file mode 100644 index 0000000..8bc4aeb --- /dev/null +++ b/config/rbac/dp_scc_role.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: intel-gpu-dp-scc-role + labels: + app.kubernetes.io/name: intel-gpu-base-operator +rules: +- apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + # Hardcoded with kustomize namePrefix — kustomize does not transform resourceNames values. + resourceNames: ["intel-gpu-base-operator-intel-gpu-dp-scc"] + verbs: ["use"] diff --git a/config/rbac/dp_scc_rolebinding.yaml b/config/rbac/dp_scc_rolebinding.yaml new file mode 100644 index 0000000..79019f9 --- /dev/null +++ b/config/rbac/dp_scc_rolebinding.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: intel-gpu-dp-scc-binding + labels: + app.kubernetes.io/name: intel-gpu-base-operator +subjects: +- kind: ServiceAccount + name: intel-gpu-dp + namespace: system +roleRef: + kind: ClusterRole + name: intel-gpu-dp-scc-role + apiGroup: rbac.authorization.k8s.io diff --git a/config/rbac/dp_serviceaccount.yaml b/config/rbac/dp_serviceaccount.yaml new file mode 100644 index 0000000..48146aa --- /dev/null +++ b/config/rbac/dp_serviceaccount.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: intel-gpu-dp + namespace: system + labels: + app.kubernetes.io/name: intel-gpu-base-operator diff --git a/config/deployments/dra/admissionpolicy.yaml b/config/rbac/dra_admission_policy.yaml similarity index 88% rename from config/deployments/dra/admissionpolicy.yaml rename to config/rbac/dra_admission_policy.yaml index 65c5553..dd858b8 100644 --- a/config/deployments/dra/admissionpolicy.yaml +++ b/config/rbac/dra_admission_policy.yaml @@ -1,7 +1,9 @@ apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingAdmissionPolicy metadata: - name: resourceslices-policy-dra-kubelet-plugin-gpu + name: intel-gpu-dra-resourceslices + labels: + app.kubernetes.io/name: intel-gpu-base-operator spec: failurePolicy: Fail matchConstraints: @@ -13,7 +15,7 @@ spec: matchConditions: - name: isRestrictedUser expression: >- - request.userInfo.username == "system:serviceaccount:intel-gpu-base-operator:gpu-policy-sample-gpu-dra" + request.userInfo.username == "system:serviceaccount:intel-gpu-base-operator:intel-gpu-base-operator-intel-gpu-dra" variables: - name: userNodeName expression: >- diff --git a/config/rbac/dra_admission_policy_binding.yaml b/config/rbac/dra_admission_policy_binding.yaml new file mode 100644 index 0000000..fa1da0d --- /dev/null +++ b/config/rbac/dra_admission_policy_binding.yaml @@ -0,0 +1,9 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: intel-gpu-dra-resourceslices + labels: + app.kubernetes.io/name: intel-gpu-base-operator +spec: + policyName: intel-gpu-dra-resourceslices + validationActions: [Deny] diff --git a/config/deployments/dra/clusterrole.yaml b/config/rbac/dra_clusterrole.yaml similarity index 80% rename from config/deployments/dra/clusterrole.yaml rename to config/rbac/dra_clusterrole.yaml index 21d6e2b..9714dab 100644 --- a/config/deployments/dra/clusterrole.yaml +++ b/config/rbac/dra_clusterrole.yaml @@ -1,7 +1,9 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: intel-gpu-resource-driver-role + name: intel-gpu-dra + labels: + app.kubernetes.io/name: intel-gpu-base-operator rules: - apiGroups: [""] resources: ["nodes"] diff --git a/config/rbac/dra_clusterrolebinding.yaml b/config/rbac/dra_clusterrolebinding.yaml new file mode 100644 index 0000000..79b8f97 --- /dev/null +++ b/config/rbac/dra_clusterrolebinding.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: intel-gpu-dra + labels: + app.kubernetes.io/name: intel-gpu-base-operator +subjects: +- kind: ServiceAccount + name: intel-gpu-dra + namespace: system +roleRef: + kind: ClusterRole + name: intel-gpu-dra + apiGroup: rbac.authorization.k8s.io diff --git a/config/rbac/dra_scc.yaml b/config/rbac/dra_scc.yaml new file mode 100644 index 0000000..6a6bec6 --- /dev/null +++ b/config/rbac/dra_scc.yaml @@ -0,0 +1,34 @@ +apiVersion: security.openshift.io/v1 +kind: SecurityContextConstraints +metadata: + name: intel-gpu-dra-scc + labels: + app.kubernetes.io/name: intel-gpu-base-operator +allowPrivilegedContainer: true +allowHostDirVolumePlugin: true +allowHostIPC: false +allowHostNetwork: true +allowHostPID: false +allowHostPorts: false +allowPrivilegeEscalation: true +allowedCapabilities: null +defaultAddCapabilities: null +fsGroup: + type: RunAsAny +readOnlyRootFilesystem: false +requiredDropCapabilities: null +runAsUser: + type: RunAsAny +seLinuxContext: + type: RunAsAny +seccompProfiles: +- "*" +supplementalGroups: + type: RunAsAny +volumes: +- hostPath +- projected +- secret +- configMap +users: [] +groups: [] diff --git a/config/rbac/dra_scc_role.yaml b/config/rbac/dra_scc_role.yaml new file mode 100644 index 0000000..a9b6a61 --- /dev/null +++ b/config/rbac/dra_scc_role.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: intel-gpu-dra-scc-role + labels: + app.kubernetes.io/name: intel-gpu-base-operator +rules: +- apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + # Hardcoded with kustomize namePrefix — kustomize does not transform resourceNames values. + resourceNames: ["intel-gpu-base-operator-intel-gpu-dra-scc"] + verbs: ["use"] diff --git a/config/rbac/dra_scc_rolebinding.yaml b/config/rbac/dra_scc_rolebinding.yaml new file mode 100644 index 0000000..c641a94 --- /dev/null +++ b/config/rbac/dra_scc_rolebinding.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: intel-gpu-dra-scc-binding + labels: + app.kubernetes.io/name: intel-gpu-base-operator +subjects: +- kind: ServiceAccount + name: intel-gpu-dra + namespace: system +roleRef: + kind: ClusterRole + name: intel-gpu-dra-scc-role + apiGroup: rbac.authorization.k8s.io diff --git a/config/rbac/dra_serviceaccount.yaml b/config/rbac/dra_serviceaccount.yaml new file mode 100644 index 0000000..6870872 --- /dev/null +++ b/config/rbac/dra_serviceaccount.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: intel-gpu-dra + namespace: system + labels: + app.kubernetes.io/name: intel-gpu-base-operator diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index 955856d..a87f00c 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -30,4 +30,25 @@ resources: - clusterpolicy_admin_role.yaml - clusterpolicy_editor_role.yaml - clusterpolicy_viewer_role.yaml +# DP RBAC — ServiceAccount and OpenShift SCC for KMM-managed DP pods (SCC is no-op on non-OpenShift) +- dp_serviceaccount.yaml +- dp_scc.yaml +- dp_scc_role.yaml +- dp_scc_rolebinding.yaml +# Module loader RBAC — ServiceAccount and OpenShift SCC for KMM module loader worker pods +- module_loader_serviceaccount.yaml +- module_loader_scc.yaml +- module_loader_scc_role.yaml +- module_loader_scc_rolebinding.yaml +# DRA RBAC — ServiceAccount, ClusterRole, ClusterRoleBinding for KMM-managed DRA pods +- dra_serviceaccount.yaml +- dra_clusterrole.yaml +- dra_clusterrolebinding.yaml +# DRA SCC — OpenShift SecurityContextConstraints (no-op on non-OpenShift) +- dra_scc.yaml +- dra_scc_role.yaml +- dra_scc_rolebinding.yaml +# DRA admission policy — restricts ResourceSlice modifications to the DRA SA's node +- dra_admission_policy.yaml +- dra_admission_policy_binding.yaml diff --git a/config/rbac/module_loader_scc.yaml b/config/rbac/module_loader_scc.yaml new file mode 100644 index 0000000..933358e --- /dev/null +++ b/config/rbac/module_loader_scc.yaml @@ -0,0 +1,37 @@ +apiVersion: security.openshift.io/v1 +kind: SecurityContextConstraints +metadata: + name: intel-gpu-module-loader-scc + labels: + app.kubernetes.io/name: intel-gpu-base-operator +allowPrivilegedContainer: true +allowHostDirVolumePlugin: true +allowHostIPC: false +allowHostNetwork: false +allowHostPID: false +allowHostPorts: false +allowPrivilegeEscalation: true +allowedCapabilities: +- SYS_MODULE +defaultAddCapabilities: null +fsGroup: + type: RunAsAny +readOnlyRootFilesystem: false +requiredDropCapabilities: null +runAsUser: + type: RunAsAny +seLinuxContext: + type: RunAsAny +seccompProfiles: +- "*" +supplementalGroups: + type: RunAsAny +volumes: +- hostPath +- emptyDir +- projected +- secret +- configMap +- downwardAPI +users: [] +groups: [] diff --git a/config/rbac/module_loader_scc_role.yaml b/config/rbac/module_loader_scc_role.yaml new file mode 100644 index 0000000..638066c --- /dev/null +++ b/config/rbac/module_loader_scc_role.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: intel-gpu-module-loader-scc-role + labels: + app.kubernetes.io/name: intel-gpu-base-operator +rules: +- apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + # Hardcoded with kustomize namePrefix — kustomize does not transform resourceNames values. + resourceNames: ["intel-gpu-base-operator-intel-gpu-module-loader-scc"] + verbs: ["use"] diff --git a/config/rbac/module_loader_scc_rolebinding.yaml b/config/rbac/module_loader_scc_rolebinding.yaml new file mode 100644 index 0000000..e4ed2bc --- /dev/null +++ b/config/rbac/module_loader_scc_rolebinding.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: intel-gpu-module-loader-scc-binding + labels: + app.kubernetes.io/name: intel-gpu-base-operator +subjects: +- kind: ServiceAccount + name: intel-gpu-module-loader + namespace: system +roleRef: + kind: ClusterRole + name: intel-gpu-module-loader-scc-role + apiGroup: rbac.authorization.k8s.io diff --git a/config/rbac/module_loader_serviceaccount.yaml b/config/rbac/module_loader_serviceaccount.yaml new file mode 100644 index 0000000..654cd27 --- /dev/null +++ b/config/rbac/module_loader_serviceaccount.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: intel-gpu-module-loader + namespace: system + labels: + app.kubernetes.io/name: intel-gpu-base-operator diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index e691942..b68ee6b 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -78,6 +78,24 @@ rules: - get - patch - update +- apiGroups: + - kmm.sigs.x-k8s.io + resources: + - modules + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kmm.sigs.x-k8s.io + resources: + - modules/status + verbs: + - get - apiGroups: - kueue.x-k8s.io resources: diff --git a/go.mod b/go.mod index f692c28..1883686 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/distribution/reference v0.6.0 github.com/google/go-cmp v0.7.0 github.com/google/go-containerregistry v0.21.7 + github.com/kubernetes-sigs/kernel-module-management v0.0.0-20260707212627-01f137c54ff8 github.com/onsi/ginkgo/v2 v2.32.0 github.com/onsi/gomega v1.42.1 github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.92.1 @@ -98,13 +99,13 @@ require ( golang.org/x/sync v0.21.0 // indirect golang.org/x/sys v0.46.0 // indirect golang.org/x/term v0.44.0 // indirect - golang.org/x/text v0.39.0 // indirect + golang.org/x/text v0.38.0 // indirect golang.org/x/time v0.15.0 // indirect - golang.org/x/tools v0.47.0 // indirect + golang.org/x/tools v0.46.0 // indirect gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect - google.golang.org/grpc v1.82.1 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 // indirect + google.golang.org/grpc v1.80.0 // indirect google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/go.sum b/go.sum index 5332c20..002d09b 100644 --- a/go.sum +++ b/go.sum @@ -117,6 +117,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kubernetes-sigs/kernel-module-management v0.0.0-20260707212627-01f137c54ff8 h1:jTT17VOVcJPwXsSTH9f2gw6VSwnvKgXsBqWewaQYZjg= +github.com/kubernetes-sigs/kernel-module-management v0.0.0-20260707212627-01f137c54ff8/go.mod h1:tqr/yRJMo+6VzLB9B/25H+wWvB+VMr95CYHD4ObyByA= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= @@ -231,22 +233,22 @@ golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= -golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus= -golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM= +golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= +golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= -golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= -golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= +golang.org/x/tools v0.46.0 h1:7jTurBkPZu4moS/Uy4OQT1M+QBlsj3wejyZwsT8Z7rk= +golang.org/x/tools v0.46.0/go.mod h1:FrD85F8l+NWL+9XWBSyVSHO6Ne4jutsfIFba7AWQ5Ys= gomodules.xyz/jsonpatch/v2 v2.5.0 h1:JELs8RLM12qJGXU4u/TO3V25KW8GreMKl9pdkk14RM0= gomodules.xyz/jsonpatch/v2 v2.5.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 h1:yQugLulqltosq0B/f8l4w9VryjV+N/5gcW0jQ3N8Qec= -google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478/go.mod h1:C6ADNqOxbgdUUeRTU+LCHDPB9ttAMCTff6auwCVa4uc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 h1:RmoJA1ujG+/lRGNfUnOMfhCy5EipVMyvUE+KNbPbTlw= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= -google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= +google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 h1:merA0rdPeUV3YIIfHHcH4qBkiQAc1nfCKSI7lB4cV2M= +google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409/go.mod h1:fl8J1IvUjCilwZzQowmw2b7HQB2eAuYBabMXzWurF+I= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 h1:ndE4FoJqsIceKP2oYSnUZqhTdYufCYYkqwtFzfrhI7w= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= +google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/controller/clusterpolicy_controller.go b/internal/controller/clusterpolicy_controller.go index 8ecd2ee..a73041a 100644 --- a/internal/controller/clusterpolicy_controller.go +++ b/internal/controller/clusterpolicy_controller.go @@ -24,22 +24,16 @@ import ( "time" apps "k8s.io/api/apps/v1" - core "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" "k8s.io/klog/v2" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sigs.k8s.io/controller-runtime/pkg/event" - "sigs.k8s.io/controller-runtime/pkg/handler" logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/predicate" - "sigs.k8s.io/controller-runtime/pkg/reconcile" v1alpha "github.com/intel/gpu-base-operator/api/v1alpha1" + kmmv1beta1 "github.com/kubernetes-sigs/kernel-module-management/api/v1beta1" ) // ClusterPolicyReconciler reconciles a ClusterPolicy object @@ -50,12 +44,16 @@ type ClusterPolicyReconciler struct { } type ControllerOpts struct { - ReqName string - Namespace string - SecretName string - RequeueDelay time.Duration - DRAEnable bool - OpenShift bool + ReqName string + Namespace string + SecretName string + DPServiceAccountName string + DRAServiceAccountName string + ModuleLoaderServiceAccountName string + RequeueDelay time.Duration + DRAEnable bool + KMMEnable bool + OpenShift bool } type requeueReconcileErr struct { @@ -67,10 +65,9 @@ type SubControllerInterface interface { } const ( - appLabel = "app" ownerKey = "owner" - draNotEnabledMsg = "DRA is not enabled in the cluster, but ClusterPolicy requests it." + xpumdVolumeName = "runxpumd" resourceModeDRA = "dra" resourceModeDP = "dp" @@ -84,6 +81,22 @@ const ( maxKeptErrors = 10 ) +func gpuNodeSelector(cp *v1alpha.ClusterPolicy) map[string]string { + selector := map[string]string{ + "kubernetes.io/arch": "amd64", + } + + for k, v := range cp.Spec.NodeSelector { + selector[k] = v + } + + if cp.Spec.UseNFDLabeling { + selector["intel.feature.node.kubernetes.io/gpu"] = trueValue + } + + return selector +} + func addIfMissing(slice *[]string, s string) { if slices.Contains(*slice, s) { return @@ -161,13 +174,10 @@ func (r *ClusterPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Reques opts := r.Opts opts.ReqName = req.Name - subControllers := make([]SubControllerInterface, 0, 4) + subControllers := make([]SubControllerInterface, 0, 3) - // Initialize sub-controllers - subControllers = append(subControllers, &DevicePluginReconciler{Client: r.Client, Scheme: r.Scheme, Opts: opts}) + subControllers = append(subControllers, &KMMReconciler{Client: r.Client, Scheme: r.Scheme, Opts: opts}) subControllers = append(subControllers, &XpuManagerReconciler{Client: r.Client, Scheme: r.Scheme, Opts: opts}) - // Include DRA subcontroller even though cluster might not be configured to use DRA, so it can report a status correctly. - subControllers = append(subControllers, &DRAReconciler{Client: r.Client, Scheme: r.Scheme, Opts: opts}) subControllers = append(subControllers, &MiscReconciler{Client: r.Client, Scheme: r.Scheme, Opts: opts}) // Ensure finalizer is present on live (non-deleted) ClusterPolicy objects. @@ -239,72 +249,6 @@ func (r *ClusterPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Reques return ctrl.Result{}, retErr } -// draPodToClusterPolicy maps any DRA pod event to reconcile requests for all existing -// ClusterPolicy objects. This avoids relying on r.Opts.ReqName (startup config) as a -// source of truth — instead it queries the actual state of the cluster. -func (r *ClusterPolicyReconciler) draPodToClusterPolicy(ctx context.Context, _ client.Object) []reconcile.Request { - cpList := &v1alpha.ClusterPolicyList{} - if err := r.List(ctx, cpList); err != nil { - klog.Error(err, "failed to list ClusterPolicies for DRA pod event") - return nil - } - - reqs := make([]reconcile.Request, len(cpList.Items)) - for i := range cpList.Items { - reqs[i] = reconcile.Request{ - NamespacedName: types.NamespacedName{Name: cpList.Items[i].Name}, - } - } - - return reqs -} - -// draPodReadinessPredicate returns a predicate that passes only DRA pod events where -// the Ready condition has changed (or the pod was created/deleted). This avoids -// spurious reconciles while still refreshing ClusterPolicy status when a DRA pod -// health check transitions between passing and failing. -func draPodReadinessPredicate() predicate.Predicate { - isDRAPod := func(obj client.Object) bool { - return obj.GetLabels()[appLabel] == draValue - } - - isPodReady := func(pod *core.Pod) bool { - for _, c := range pod.Status.Conditions { - if c.Type == core.PodReady { - return c.Status == core.ConditionTrue - } - } - - return false - } - - return predicate.Funcs{ - CreateFunc: func(e event.CreateEvent) bool { - return isDRAPod(e.Object) - }, - UpdateFunc: func(e event.UpdateEvent) bool { - if !isDRAPod(e.ObjectNew) { - return false - } - - oldPod, ok1 := e.ObjectOld.(*core.Pod) - newPod, ok2 := e.ObjectNew.(*core.Pod) - - if !ok1 || !ok2 { - return false - } - - return isPodReady(oldPod) != isPodReady(newPod) - }, - DeleteFunc: func(e event.DeleteEvent) bool { - return isDRAPod(e.Object) - }, - GenericFunc: func(e event.GenericEvent) bool { - return false - }, - } -} - // SetupWithManager sets up the controller with the Manager. func (r *ClusterPolicyReconciler) SetupWithManager(mgr ctrl.Manager, opts ControllerOpts) error { r.Opts = opts @@ -314,14 +258,8 @@ func (r *ClusterPolicyReconciler) SetupWithManager(mgr ctrl.Manager, opts Contro Named("clusterpolicy"). Owns(&apps.DaemonSet{}) - // Only watch DRA pods when DRA is enabled in the cluster, to avoid unnecessary - // pod list/watch permissions and reconcile noise when DRA is not in use. - if opts.DRAEnable { - b = b.Watches( - &core.Pod{}, - handler.EnqueueRequestsFromMapFunc(r.draPodToClusterPolicy), - builder.WithPredicates(draPodReadinessPredicate()), - ) + if opts.KMMEnable { + b = b.Owns(&kmmv1beta1.Module{}) } return b.Complete(r) diff --git a/internal/controller/clusterpolicy_controller_test.go b/internal/controller/clusterpolicy_controller_test.go index 3a91490..daed29c 100644 --- a/internal/controller/clusterpolicy_controller_test.go +++ b/internal/controller/clusterpolicy_controller_test.go @@ -18,7 +18,6 @@ package controller import ( "context" - "fmt" "time" . "github.com/onsi/ginkgo/v2" @@ -26,12 +25,8 @@ import ( apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - "sigs.k8s.io/controller-runtime/pkg/client/interceptor" - "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/yaml" @@ -39,6 +34,7 @@ import ( v1alpha "github.com/intel/gpu-base-operator/api/v1alpha1" "github.com/intel/gpu-base-operator/config/deployments" + kmmv1beta1 "github.com/kubernetes-sigs/kernel-module-management/api/v1beta1" nfdcrd "sigs.k8s.io/node-feature-discovery/api/nfd/v1alpha1" ) @@ -96,9 +92,11 @@ var _ = Describe("ClusterPolicy Controller", func() { Client: k8sClient, Scheme: k8sClient.Scheme(), Opts: ControllerOpts{ - Namespace: defaultNamespace, - DRAEnable: true, - RequeueDelay: time.Millisecond * 50, + Namespace: defaultNamespace, + DRAEnable: true, + KMMEnable: true, + DRAServiceAccountName: "intel-gpu-dra", + RequeueDelay: time.Millisecond * 50, }, } @@ -115,22 +113,20 @@ var _ = Describe("ClusterPolicy Controller", func() { Expect(err).NotTo(HaveOccurred()) Expect(ret.RequeueAfter).To(BeZero()) + By("verifying KMM Module CR was created") + mod := &kmmv1beta1.Module{} + modKey := types.NamespacedName{Name: resourceName + "-gpu", Namespace: defaultNamespace} + Expect(k8sClient.Get(ctx, modKey, mod)).To(Succeed()) + Expect(mod.Spec.DRA).NotTo(BeNil()) + Expect(mod.Spec.DRA.Container.Image).To(Equal("dra-image:v1.2.3")) + + By("verifying XPUM DaemonSet was created") daemonSets := apps.DaemonSetList{} err = k8sClient.List(ctx, &daemonSets, client.InNamespace(defaultNamespace)) Expect(err).NotTo(HaveOccurred()) - - Expect(daemonSets.Items).To(HaveLen(2)) - - for _, ds := range daemonSets.Items { - switch ds.Name { - case "test-resource-dra-xpum-xpum-gpu-dra": - Expect(ds.Spec.Template.Spec.Containers[0].Image).To(Equal("dra-image:v1.2.3")) - case "test-resource-dra-xpum-xpum-xpu-manager": - Expect(ds.Spec.Template.Spec.Containers[0].Image).To(Equal("xpum-image:v1.2.3")) - default: - Fail("Unexpected DaemonSet found: " + ds.Name) - } - } + Expect(daemonSets.Items).To(HaveLen(1)) + Expect(daemonSets.Items[0].Name).To(Equal(resourceName + "-xpu-manager")) + Expect(daemonSets.Items[0].Spec.Template.Spec.Containers[0].Image).To(Equal("xpum-image:v1.2.3")) nfr := nfdcrd.NodeFeatureRule{} err = k8sClient.Get(ctx, types.NamespacedName{Name: "intel-gpu-devices"}, &nfr) @@ -150,15 +146,14 @@ var _ = Describe("ClusterPolicy Controller", func() { err = k8sClient.List(ctx, &daemonSets, client.InNamespace(defaultNamespace)) Expect(err).NotTo(HaveOccurred()) - - Expect(daemonSets.Items).To(HaveLen(2)) + Expect(daemonSets.Items).To(HaveLen(1)) configmaps := v1.ConfigMapList{} err = k8sClient.List(ctx, &configmaps, client.InNamespace(defaultNamespace)) Expect(err).NotTo(HaveOccurred()) Expect(configmaps.Items).To(HaveLen(1)) - Expect(configmaps.Items[0].Name).To(Equal("test-resource-dra-xpum-xpum-xpumanager-otel-config")) + Expect(configmaps.Items[0].Name).To(Equal(resourceName + "-xpumanager-otel-config")) clusterpolicy.Spec.HealthinessSpec = &v1alpha.HealthinessSpec{ CheckIntervalSeconds: 17, @@ -172,7 +167,6 @@ var _ = Describe("ClusterPolicy Controller", func() { Expect(ret.RequeueAfter).To(BeZero()) // Create XPUM config override - xpumConfig := deployments.XpuManagerOTelConfig() data, err := yaml.Marshal(xpumConfig) Expect(err).NotTo(HaveOccurred()) @@ -199,26 +193,18 @@ var _ = Describe("ClusterPolicy Controller", func() { err = k8sClient.List(ctx, &daemonSets, client.InNamespace(defaultNamespace)) Expect(err).NotTo(HaveOccurred()) + Expect(daemonSets.Items).To(HaveLen(1)) - Expect(daemonSets.Items).To(HaveLen(2)) - - for _, ds := range daemonSets.Items { - switch ds.Name { - case "test-resource-dra-xpum-xpum-gpu-dra": - case "test-resource-dra-xpum-xpum-xpu-manager": - checked := false - for _, v := range ds.Spec.Template.Spec.Volumes { - if v.Name == "config" { - Expect(v.VolumeSource.ConfigMap).NotTo(BeNil()) - Expect(v.VolumeSource.ConfigMap.Name).To(Equal("test-xpum-cm")) - checked = true - } - } - Expect(checked).To(BeTrue()) - default: - Fail("Unexpected DaemonSet found: " + ds.Name) + By("verifying XPUM config override was applied") + checked := false + for _, v := range daemonSets.Items[0].Spec.Template.Spec.Volumes { + if v.Name == "config" { + Expect(v.VolumeSource.ConfigMap).NotTo(BeNil()) + Expect(v.VolumeSource.ConfigMap.Name).To(Equal("test-xpum-cm")) + checked = true } } + Expect(checked).To(BeTrue()) err = k8sClient.Delete(ctx, clusterpolicy) Expect(err).NotTo(HaveOccurred()) @@ -240,136 +226,7 @@ var _ = Describe("ClusterPolicy Controller", func() { err = k8sClient.List(ctx, &daemonSets, client.InNamespace(defaultNamespace)) Expect(err).NotTo(HaveOccurred()) - // Both DaemonSets (DRA and XPUM) are explicitly deleted by the reconciler - // during the finalizer cleanup — we no longer rely on the K8s GC. Expect(daemonSets.Items).To(BeEmpty()) }) }) }) - -var _ = Describe("draPodReadinessPredicate", func() { - var pred = draPodReadinessPredicate() - - draLabels := map[string]string{appLabel: draValue} - - draReadyPod := func(ready bool) *v1.Pod { - status := v1.ConditionFalse - if ready { - status = v1.ConditionTrue - } - return &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{Labels: draLabels}, - Status: v1.PodStatus{ - Conditions: []v1.PodCondition{ - {Type: v1.PodReady, Status: status}, - }, - }, - } - } - - Context("CreateFunc", func() { - It("passes DRA pods", func() { - Expect(pred.Create(event.CreateEvent{Object: draReadyPod(false)})).To(BeTrue()) - }) - - It("filters out non-DRA pods", func() { - pod := &v1.Pod{} - Expect(pred.Create(event.CreateEvent{Object: pod})).To(BeFalse()) - }) - }) - - Context("DeleteFunc", func() { - It("passes DRA pods", func() { - Expect(pred.Delete(event.DeleteEvent{Object: draReadyPod(true)})).To(BeTrue()) - }) - - It("filters out non-DRA pods", func() { - Expect(pred.Delete(event.DeleteEvent{Object: &v1.Pod{}})).To(BeFalse()) - }) - }) - - Context("GenericFunc", func() { - It("always filters events", func() { - Expect(pred.Generic(event.GenericEvent{Object: draReadyPod(true)})).To(BeFalse()) - }) - }) - - Context("UpdateFunc", func() { - It("passes when DRA pod Ready condition changes from false to true", func() { - e := event.UpdateEvent{ObjectOld: draReadyPod(false), ObjectNew: draReadyPod(true)} - Expect(pred.Update(e)).To(BeTrue()) - }) - - It("passes when DRA pod Ready condition changes from true to false", func() { - e := event.UpdateEvent{ObjectOld: draReadyPod(true), ObjectNew: draReadyPod(false)} - Expect(pred.Update(e)).To(BeTrue()) - }) - - It("filters when DRA pod Ready condition is unchanged (both ready)", func() { - e := event.UpdateEvent{ObjectOld: draReadyPod(true), ObjectNew: draReadyPod(true)} - Expect(pred.Update(e)).To(BeFalse()) - }) - - It("filters when DRA pod Ready condition is unchanged (both not ready)", func() { - e := event.UpdateEvent{ObjectOld: draReadyPod(false), ObjectNew: draReadyPod(false)} - Expect(pred.Update(e)).To(BeFalse()) - }) - - It("filters non-DRA pods", func() { - nonDRA := &v1.Pod{} - e := event.UpdateEvent{ObjectOld: nonDRA, ObjectNew: nonDRA} - Expect(pred.Update(e)).To(BeFalse()) - }) - - It("filters when objects are not pods", func() { - notAPod := &v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Labels: draLabels}} - e := event.UpdateEvent{ObjectOld: notAPod, ObjectNew: notAPod} - Expect(pred.Update(e)).To(BeFalse()) - }) - }) -}) - -var _ = Describe("draPodToClusterPolicy", func() { - newScheme := func() *runtime.Scheme { - s := runtime.NewScheme() - Expect(v1alpha.AddToScheme(s)).To(Succeed()) - return s - } - - It("returns one request per existing ClusterPolicy", func() { - cp1 := &v1alpha.ClusterPolicy{ObjectMeta: metav1.ObjectMeta{Name: "policy-one"}} - cp2 := &v1alpha.ClusterPolicy{ObjectMeta: metav1.ObjectMeta{Name: "policy-two"}} - - r := &ClusterPolicyReconciler{ - Client: fake.NewClientBuilder().WithScheme(newScheme()).WithObjects(cp1, cp2).Build(), - } - - reqs := r.draPodToClusterPolicy(context.Background(), nil) - Expect(reqs).To(HaveLen(2)) - - names := []string{reqs[0].Name, reqs[1].Name} - Expect(names).To(ConsistOf("policy-one", "policy-two")) - }) - - It("returns an empty slice when no ClusterPolicies exist", func() { - r := &ClusterPolicyReconciler{ - Client: fake.NewClientBuilder().WithScheme(newScheme()).Build(), - } - - reqs := r.draPodToClusterPolicy(context.Background(), nil) - Expect(reqs).To(BeEmpty()) - }) - - It("returns nil when the List call fails", func() { - r := &ClusterPolicyReconciler{ - Client: fake.NewClientBuilder().WithScheme(newScheme()).WithInterceptorFuncs(interceptor.Funcs{ - List: func(_ context.Context, _ client.WithWatch, _ client.ObjectList, _ ...client.ListOption) error { - return fmt.Errorf("list error") - }, - }).Build(), - } - - reqs := r.draPodToClusterPolicy(context.Background(), nil) - Expect(reqs).To(BeNil()) - }) -}) diff --git a/internal/controller/deviceplugin_controller.go b/internal/controller/deviceplugin_controller.go deleted file mode 100644 index 21588f6..0000000 --- a/internal/controller/deviceplugin_controller.go +++ /dev/null @@ -1,354 +0,0 @@ -/* -Copyright 2025 Intel Corporation. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "context" - "fmt" - "slices" - "strings" - - apps "k8s.io/api/apps/v1" - core "k8s.io/api/core/v1" - "k8s.io/klog/v2" - - "k8s.io/apimachinery/pkg/runtime" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - logf "sigs.k8s.io/controller-runtime/pkg/log" - - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - v1alpha "github.com/intel/gpu-base-operator/api/v1alpha1" - "github.com/intel/gpu-base-operator/config/deployments" -) - -type DevicePluginReconciler struct { - client.Client - Scheme *runtime.Scheme - Opts ControllerOpts -} - -const ( - dpValue = "intel-gpu-plugin" - xpumdVolumeName = "runxpumd" - - dpResourcePart = "gpu-dp" -) - -func logLevelForDp(spec *v1alpha.ClusterPolicy) int32 { - logLevel := int32(0) - logLevel = max(logLevel, spec.Spec.LogLevel) - logLevel = max(logLevel, spec.Spec.DevicePluginSpec.LogLevel) - - return logLevel -} - -func hexArgStr(s []string) string { - a := []string{} - for _, str := range s { - if !strings.HasPrefix(str, "0x") { - str = "0x" + str - } - - a = append(a, str) - } - return strings.Join(a, ",") -} - -func addXpumdMounts(spec *core.PodSpec) { - for _, v := range spec.Volumes { - if v.Name == xpumdVolumeName { - return - } - } - - dirOrCreate := core.HostPathDirectoryOrCreate - - spec.Volumes = append(spec.Volumes, core.Volume{ - Name: xpumdVolumeName, - VolumeSource: core.VolumeSource{ - HostPath: &core.HostPathVolumeSource{ - Path: "/run/xpumd", - Type: &dirOrCreate, - }, - }, - }) - - spec.Containers[0].VolumeMounts = append(spec.Containers[0].VolumeMounts, - core.VolumeMount{ - Name: xpumdVolumeName, - MountPath: "/run/xpumd", - }, - ) -} - -func removeXpumdMounts(spec *core.PodSpec) { - for i, v := range spec.Volumes { - if v.Name == xpumdVolumeName { - spec.Volumes = slices.Delete(spec.Volumes, i, i+1) - break - } - } - - for i, vm := range spec.Containers[0].VolumeMounts { - if vm.Name == xpumdVolumeName { - spec.Containers[0].VolumeMounts = slices.Delete(spec.Containers[0].VolumeMounts, i, i+1) - break - } - } -} - -func dpArgs(spec *v1alpha.ClusterPolicy) []string { - args := []string{} - - dpspec := spec.Spec.DevicePluginSpec - - if spec.Spec.ResourceMonitoring { - args = append(args, - "-enable-monitoring", - "-xpumd-endpoint=/run/xpumd/intelxpuinfo.sock") - } - - logLevel := logLevelForDp(spec) - if logLevel > 0 { - args = append(args, fmt.Sprintf("-v=%d", logLevel)) - } - - if len(dpspec.ByPathMode) > 0 { - args = append(args, fmt.Sprintf("-bypath=%s", dpspec.ByPathMode)) - } - - if len(dpspec.AllowIDs) > 0 { - args = append(args, fmt.Sprintf("-allow-ids=%s", hexArgStr(dpspec.AllowIDs))) - } - - if len(dpspec.DenyIDs) > 0 { - args = append(args, fmt.Sprintf("-deny-ids=%s", hexArgStr(dpspec.DenyIDs))) - } - - return args -} - -func (r *DevicePluginReconciler) updateDaemonSetObject(ds *apps.DaemonSet, spec *v1alpha.ClusterPolicy) { - name := fmt.Sprintf("%s-device-plugin", spec.Name) - - ds.Name = name - ds.Namespace = r.Opts.Namespace - ds.Labels[ownerKey] = spec.Name - - dspec := &spec.Spec.DevicePluginSpec - - ds.Spec.Template.Spec.Containers[0].Image = dspec.PluginImage - - ds.Spec.Template.Spec.Containers[0].Args = dpArgs(spec) - - ds.Spec.Template.Spec.NodeSelector = map[string]string{ - "kubernetes.io/arch": "amd64", - } - - if len(spec.Spec.NodeSelector) > 0 { - for k, v := range spec.Spec.NodeSelector { - ds.Spec.Template.Spec.NodeSelector[k] = v - } - } - - if spec.Spec.UseNFDLabeling { - ds.Spec.Template.Spec.NodeSelector["intel.feature.node.kubernetes.io/gpu"] = trueValue - } - - if len(spec.Spec.Tolerations) > 0 { - ds.Spec.Template.Spec.Tolerations = spec.Spec.Tolerations - } else { - ds.Spec.Template.Spec.Tolerations = nil - } - - cspec := &ds.Spec.Template.Spec - - secrets := []core.LocalObjectReference{} - if r.Opts.SecretName != "" { - secrets = append(secrets, core.LocalObjectReference{Name: r.Opts.SecretName}) - } - if spec.Spec.PullSecret != nil { - secrets = append(secrets, *spec.Spec.PullSecret) - } - - if len(secrets) > 0 { - cspec.ImagePullSecrets = secrets - } else { - cspec.ImagePullSecrets = nil - } - - if spec.Spec.ResourceMonitoring { - addXpumdMounts(cspec) - } else { - removeXpumdMounts(cspec) - } - - if r.Opts.OpenShift { - _, _, _, saName := buildOpenShiftNames(spec.Name, dpResourcePart) - cspec.ServiceAccountName = saName - } -} - -func (r *DevicePluginReconciler) createOpenShiftResourcesIfNotExists(ctx context.Context, cpName string) error { - sccName, roleName, bindingName, saName := buildOpenShiftNames(cpName, dpResourcePart) - - if err := createServiceAccount(ctx, r.Client, saName, r.Opts.Namespace); err != nil { - return fmt.Errorf("failed to ensure DP ServiceAccount: %w", err) - } - - if err := ensureSCC(ctx, r.Client, buildDevicePluginSCC(sccName)); err != nil { - return fmt.Errorf("failed to ensure DP SCC: %w", err) - } - - if err := createSCCRole(ctx, r.Client, roleName, sccName); err != nil { - return fmt.Errorf("failed to ensure DP SCC ClusterRole: %w", err) - } - - if err := createSCCRoleBinding(ctx, r.Client, bindingName, roleName, saName, r.Opts.Namespace); err != nil { - return fmt.Errorf("failed to ensure DP SCC ClusterRoleBinding: %w", err) - } - - return nil -} - -func (r *DevicePluginReconciler) cleanupOpenShiftResources(ctx context.Context, cpName string) { - sccName, roleName, bindingName, saName := buildOpenShiftNames(cpName, dpResourcePart) - - deleteOpenShiftSCCResources(ctx, r.Client, sccName, roleName, bindingName, saName, r.Opts.Namespace) -} - -func (r *DevicePluginReconciler) createDaemonSet(ctx context.Context, obj client.Object) (ctrl.Result, error) { - spec := obj.(*v1alpha.ClusterPolicy) - - ds := deployments.DevicePluginDaemonset() - - r.updateDaemonSetObject(ds, spec) - - if err := ctrl.SetControllerReference(obj, ds, r.Scheme); err != nil { - klog.Error(err, "unable to set controller reference") - - return ctrl.Result{}, err - } - - if err := r.Create(ctx, ds); err != nil { - klog.Error(err, "unable to create DaemonSet") - - return ctrl.Result{}, err - } - - return ctrl.Result{}, nil -} - -func (r *DevicePluginReconciler) removeDeploymentIfExists(ctx context.Context) (ctrl.Result, error) { - klog.V(4).Info("Removing Device Plugin deployment") - - crName := r.Opts.ReqName - - if r.Opts.OpenShift { - r.cleanupOpenShiftResources(ctx, crName) - } - - dss := &apps.DaemonSetList{} - labels := client.MatchingLabels{ - appLabel: dpValue, - ownerKey: crName, - } - - if err := r.List(ctx, dss, client.InNamespace(r.Opts.Namespace), labels); err != nil { - klog.Error(err, "unable to list child DaemonSets") - - return ctrl.Result{}, err - } - - if len(dss.Items) == 0 { - klog.V(4).Info("No DevicePlugin deployment found, nothing to do") - - return ctrl.Result{}, nil - } - - if err := r.Delete(ctx, &dss.Items[0]); err != nil { - return ctrl.Result{}, err - } - - klog.V(4).Info("DevicePlugin deployment removed") - - return ctrl.Result{}, nil -} - -func (r *DevicePluginReconciler) Reconcile(ctx context.Context, cp *v1alpha.ClusterPolicy) (ctrl.Result, error) { - _ = logf.FromContext(ctx) - - if cp == nil || !cp.DeletionTimestamp.IsZero() { - return r.removeDeploymentIfExists(ctx) - } - - if cp.Spec.ResourceRegistration != "dp" { - cp.Status.DevicePluginStatus = notAvailableStatus - - return r.removeDeploymentIfExists(ctx) - } - - var olderDs apps.DaemonSetList - if err := r.List(ctx, &olderDs, client.InNamespace(r.Opts.Namespace), client.MatchingLabels{appLabel: dpValue}); err != nil { - klog.Error(err, "unable to list child DaemonSets") - - return ctrl.Result{}, err - } - - if r.Opts.OpenShift { - if err := r.createOpenShiftResourcesIfNotExists(ctx, cp.Name); err != nil { - klog.Error(err, "unable to ensure OpenShift resources for DP") - - return ctrl.Result{}, err - } - } - - if len(olderDs.Items) == 0 { - return r.createDaemonSet(ctx, cp) - } - - // Update DaemonSet - - ds := &olderDs.Items[0] - originalDs := ds.DeepCopy() - - r.updateDaemonSetObject(ds, cp) - - dsDiff := cmp.Diff(originalDs.Spec.Template.Spec, ds.Spec.Template.Spec, cmpopts.EquateEmpty()) - if len(dsDiff) > 0 { - klog.Info("DS difference", "diff", dsDiff) - - if err := r.Update(ctx, ds); err != nil { - klog.Error(err, "unable to update daemonset", "DaemonSet", ds) - - return ctrl.Result{}, err - } - } - - if err := r.List(ctx, &olderDs, client.InNamespace(r.Opts.Namespace), client.MatchingLabels{appLabel: dpValue}); err != nil { - klog.Error(err, "unable to list child DaemonSets") - - return ctrl.Result{}, err - } - - cp.Status.DevicePluginStatus = fmt.Sprintf("%d/%d", - olderDs.Items[0].Status.NumberReady, olderDs.Items[0].Status.DesiredNumberScheduled) - - return ctrl.Result{}, nil -} diff --git a/internal/controller/deviceplugin_controller_test.go b/internal/controller/deviceplugin_controller_test.go deleted file mode 100644 index 79964e4..0000000 --- a/internal/controller/deviceplugin_controller_test.go +++ /dev/null @@ -1,721 +0,0 @@ -/* -Copyright 2025 Intel Corporation. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "context" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - apps "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" - rbac "k8s.io/api/rbac/v1" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - - v1alpha "github.com/intel/gpu-base-operator/api/v1alpha1" - "github.com/intel/gpu-base-operator/config/deployments" -) - -var _ = Describe("ClusterPolicy Controller for Device Plugin", func() { - - Context("When reconciling Device Plugin and XPUM", func() { - defaultNamespace := "foobar" - const resourceName = "test-resource" - - ctx := context.Background() - - typeNamespacedName := types.NamespacedName{ - Name: resourceName, - } - clusterpolicy := &v1alpha.ClusterPolicy{} - - BeforeEach(func() { - ns := &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaultNamespace, - }, - } - - Expect(k8sClient.Create(ctx, ns)).To(Succeed()) - }) - - AfterEach(func() { - resource := &v1alpha.ClusterPolicy{} - err := k8sClient.Get(ctx, typeNamespacedName, resource) - Expect(err).NotTo(HaveOccurred()) - - By("Cleanup the specific resource instance ClusterPolicy") - Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) - }) - - It("should successfully reconcile the resource", func() { - By("creating the custom resource for the Kind ClusterPolicy") - err := k8sClient.Get(ctx, typeNamespacedName, clusterpolicy) - if err != nil && errors.IsNotFound(err) { - resource := &v1alpha.ClusterPolicy{ - ObjectMeta: metav1.ObjectMeta{ - Name: resourceName, - }, - Spec: v1alpha.ClusterPolicySpec{ - ResourceRegistration: "dp", - ResourceMonitoring: true, - UseNFDLabeling: true, - DevicePluginSpec: v1alpha.DevicePluginSpec{ - PluginImage: "intel/intel-gpu-plugin:0.32.0", - }, - XpuManagerSpec: v1alpha.XpuManagerSpec{ - Image: "intel/xpumanager:v1.2.27", - MonitoringResource: "monitoring", - }, - }, - } - Expect(k8sClient.Create(ctx, resource)).To(Succeed()) - } - - By("Reconciling the created resource") - controllerReconciler := &ClusterPolicyReconciler{ - Client: k8sClient, - Scheme: k8sClient.Scheme(), - Opts: ControllerOpts{ - Namespace: defaultNamespace, - DRAEnable: true, - }, - } - - _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ - NamespacedName: typeNamespacedName, - }) - Expect(err).NotTo(HaveOccurred()) - - By("Checking that the DaemonSet has been created") - dpDs := apps.DaemonSetList{} - err = k8sClient.List(ctx, &dpDs, client.InNamespace(defaultNamespace)) - Expect(err).NotTo(HaveOccurred()) - - Expect(dpDs.Items).To(HaveLen(2)) - Expect(dpDs.Items[0].Name).To(Equal("test-resource-device-plugin")) - Expect(dpDs.Items[0].Spec.Template.Spec.Containers[0].Args).To(ContainElement("-enable-monitoring")) - Expect(dpDs.Items[0].Spec.Template.Spec.NodeSelector).To(HaveKeyWithValue("intel.feature.node.kubernetes.io/gpu", "true")) - Expect(dpDs.Items[0].Spec.Template.Spec.NodeSelector).To(HaveKeyWithValue("kubernetes.io/arch", "amd64")) - Expect(dpDs.Items[1].Name).To(Equal("test-resource-xpu-manager")) - Expect(dpDs.Items[1].Spec.Template.Spec.NodeSelector).To(HaveKeyWithValue("intel.feature.node.kubernetes.io/gpu", "true")) - - resource := &v1alpha.ClusterPolicy{} - err = k8sClient.Get(ctx, typeNamespacedName, resource) - Expect(err).NotTo(HaveOccurred()) - - resource.Spec.UseNFDLabeling = false - resource.Spec.NodeSelector = map[string]string{"foo": "bar"} - - Expect(k8sClient.Update(ctx, resource)).To(Succeed()) - - _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ - NamespacedName: typeNamespacedName, - }) - Expect(err).NotTo(HaveOccurred()) - - err = k8sClient.List(ctx, &dpDs, client.InNamespace(defaultNamespace)) - Expect(err).NotTo(HaveOccurred()) - - Expect(dpDs.Items).To(HaveLen(2)) - Expect(dpDs.Items[0].Spec.Template.Spec.NodeSelector).To(HaveKeyWithValue("foo", "bar")) - Expect(dpDs.Items[1].Spec.Template.Spec.NodeSelector).To(HaveKeyWithValue("foo", "bar")) - - err = k8sClient.Get(ctx, typeNamespacedName, resource) - Expect(err).NotTo(HaveOccurred()) - - resource.Spec.ResourceMonitoring = false - resource.Spec.DevicePluginSpec.LogLevel = 4 - - Expect(k8sClient.Update(ctx, resource)).To(Succeed()) - - _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ - NamespacedName: typeNamespacedName, - }) - Expect(err).NotTo(HaveOccurred()) - - err = k8sClient.List(ctx, &dpDs, client.InNamespace(defaultNamespace)) - Expect(err).NotTo(HaveOccurred()) - - Expect(dpDs.Items).To(HaveLen(1)) - Expect(dpDs.Items[0].Name).To(Equal("test-resource-device-plugin")) - Expect(dpDs.Items[0].Spec.Template.Spec.Containers[0].Args).To(ContainElement("-v=4")) - Expect(dpDs.Items[0].Spec.Template.Spec.Containers[0].Args).NotTo(ContainElement("-enable-monitoring")) - }) - }) - - Context("When reconciling DP and XPUM", func() { - defaultNamespace := "foobar-xpum" - const resourceName = "test-resource-xpum" - - ctx := context.Background() - - typeNamespacedName := types.NamespacedName{ - Name: resourceName, - } - clusterpolicy := &v1alpha.ClusterPolicy{ - ObjectMeta: metav1.ObjectMeta{ - Name: resourceName, - }, - Spec: v1alpha.ClusterPolicySpec{ - ResourceRegistration: "dp", - ResourceMonitoring: true, - DevicePluginSpec: v1alpha.DevicePluginSpec{ - PluginImage: "intel/intel-gpu-plugin:0.32.0", - }, - XpuManagerSpec: v1alpha.XpuManagerSpec{ - Image: "xpum-image:v1.2.3", - LogLevel: 3, - MonitoringResource: "monitoring", - }, - }, - } - - BeforeEach(func() { - ns := &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaultNamespace, - }, - } - - Expect(k8sClient.Create(ctx, ns)).To(Succeed()) - }) - - AfterEach(func() { - resource := &v1alpha.ClusterPolicy{} - err := k8sClient.Get(ctx, typeNamespacedName, resource) - Expect(err).NotTo(HaveOccurred()) - - By("Cleanup the specific resource instance ClusterPolicy") - Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) - }) - - It("should successfully reconcile the resource", func() { - By("creating the custom resource for the Kind ClusterPolicy") - err := k8sClient.Get(ctx, typeNamespacedName, clusterpolicy) - if err != nil && errors.IsNotFound(err) { - Expect(k8sClient.Create(ctx, clusterpolicy)).To(Succeed()) - } - - By("Reconciling the created resource") - controllerReconciler := &ClusterPolicyReconciler{ - Client: k8sClient, - Scheme: k8sClient.Scheme(), - Opts: ControllerOpts{ - Namespace: defaultNamespace, - DRAEnable: true, - RequeueDelay: time.Millisecond * 50, - }, - } - - _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ - NamespacedName: typeNamespacedName, - }) - Expect(err).NotTo(HaveOccurred()) - - daemonSets := apps.DaemonSetList{} - err = k8sClient.List(ctx, &daemonSets, client.InNamespace(defaultNamespace)) - Expect(err).NotTo(HaveOccurred()) - - Expect(daemonSets.Items).To(HaveLen(2)) - - for _, ds := range daemonSets.Items { - switch ds.Name { - case "test-resource-xpum-device-plugin": - case "test-resource-xpum-xpu-manager": - default: - Fail("Unexpected DaemonSet found: " + ds.Name) - } - } - }) - }) - - Context("When reconciling from DP to DRA", func() { - defaultNamespace := "foobar-dp-to-dra" - const resourceName = "test-resource-dp-to-dra" - - ctx := context.Background() - - typeNamespacedName := types.NamespacedName{ - Name: resourceName, - } - clusterpolicy := &v1alpha.ClusterPolicy{ - ObjectMeta: metav1.ObjectMeta{ - Name: resourceName, - }, - Spec: v1alpha.ClusterPolicySpec{ - ResourceRegistration: "dp", - ResourceMonitoring: false, - DevicePluginSpec: v1alpha.DevicePluginSpec{ - PluginImage: "intel/intel-gpu-plugin:0.32.0", - }, - DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ - Image: "intel/gpu-dra-driver:1.2.3", - }, - }, - } - - BeforeEach(func() { - ns := &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaultNamespace, - }, - } - - Expect(k8sClient.Create(ctx, ns)).To(Succeed()) - }) - - AfterEach(func() { - resource := &v1alpha.ClusterPolicy{} - err := k8sClient.Get(ctx, typeNamespacedName, resource) - Expect(err).NotTo(HaveOccurred()) - - By("Cleanup the specific resource instance ClusterPolicy") - Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) - }) - - It("should successfully reconcile the resource", func() { - By("creating the custom resource for the Kind ClusterPolicy") - err := k8sClient.Get(ctx, typeNamespacedName, clusterpolicy) - if err != nil && errors.IsNotFound(err) { - Expect(k8sClient.Create(ctx, clusterpolicy)).To(Succeed()) - } - - By("Reconciling the created resource") - controllerReconciler := &ClusterPolicyReconciler{ - Client: k8sClient, - Scheme: k8sClient.Scheme(), - Opts: ControllerOpts{ - Namespace: defaultNamespace, - DRAEnable: true, - RequeueDelay: time.Millisecond * 50, - }, - } - - _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ - NamespacedName: typeNamespacedName, - }) - Expect(err).NotTo(HaveOccurred()) - - daemonSets := apps.DaemonSetList{} - err = k8sClient.List(ctx, &daemonSets, client.InNamespace(defaultNamespace)) - Expect(err).NotTo(HaveOccurred()) - - Expect(daemonSets.Items).To(HaveLen(1)) - - for _, ds := range daemonSets.Items { - switch ds.Name { - case "test-resource-dp-to-dra-device-plugin": - default: - Fail("Unexpected DaemonSet found: " + ds.Name) - } - } - - err = k8sClient.Get(ctx, typeNamespacedName, clusterpolicy) - Expect(err).NotTo(HaveOccurred()) - - clusterpolicy.Spec.ResourceRegistration = "dra" - Expect(k8sClient.Update(ctx, clusterpolicy)).To(Succeed()) - - _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ - NamespacedName: typeNamespacedName, - }) - Expect(err).NotTo(HaveOccurred()) - - err = k8sClient.List(ctx, &daemonSets, client.InNamespace(defaultNamespace)) - Expect(err).NotTo(HaveOccurred()) - - Expect(daemonSets.Items).To(HaveLen(1)) - - for _, ds := range daemonSets.Items { - switch ds.Name { - case "test-resource-dp-to-dra-gpu-dra": - default: - Fail("Unexpected DaemonSet found: " + ds.Name) - } - } - }) - }) - - Context("When reconciling Device Plugin and XPUM on OpenShift", func() { - defaultNamespace := "foobar-openshift" - const resourceName = "test-resource-ocp" - - ctx := context.Background() - - typeNamespacedName := types.NamespacedName{Name: resourceName} - - BeforeEach(func() { - Expect(k8sClient.Create(ctx, &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{Name: defaultNamespace}, - })).To(Succeed()) - }) - - AfterEach(func() { - resource := &v1alpha.ClusterPolicy{} - Expect(k8sClient.Get(ctx, typeNamespacedName, resource)).To(Succeed()) - Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) - - // Clean up cluster-scoped OpenShift resources created by the reconciler. - deleteOpenShiftSCCResources(ctx, k8sClient, - resourceName+"-gpu-dp-scc", - resourceName+"-gpu-dp-scc-role", - resourceName+"-gpu-dp-scc-binding", - resourceName+"-gpu-dp", - defaultNamespace) - deleteOpenShiftSCCResources(ctx, k8sClient, - resourceName+"-xpu-manager-scc", - resourceName+"-xpu-manager-scc-role", - resourceName+"-xpu-manager-scc-binding", - resourceName+"-xpu-manager", - defaultNamespace) - }) - - It("creates SCC resources for DP and XPUM and sets ServiceAccountName on DaemonSets", func() { - By("creating the ClusterPolicy") - Expect(k8sClient.Create(ctx, &v1alpha.ClusterPolicy{ - ObjectMeta: metav1.ObjectMeta{Name: resourceName}, - Spec: v1alpha.ClusterPolicySpec{ - ResourceRegistration: "dp", - ResourceMonitoring: true, - DevicePluginSpec: v1alpha.DevicePluginSpec{ - PluginImage: "intel/intel-gpu-plugin:test", - }, - XpuManagerSpec: v1alpha.XpuManagerSpec{ - Image: "intel/xpumanager:test", - }, - }, - })).To(Succeed()) - - reconciler := &ClusterPolicyReconciler{ - Client: k8sClient, - Scheme: k8sClient.Scheme(), - Opts: ControllerOpts{ - Namespace: defaultNamespace, - OpenShift: true, - RequeueDelay: time.Millisecond * 50, - }, - } - - By("first reconcile creates SCC resources") - _, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) - Expect(err).NotTo(HaveOccurred()) - - By("DP ServiceAccount is created") - dpSA := &v1.ServiceAccount{} - Expect(k8sClient.Get(ctx, client.ObjectKey{Name: resourceName + "-gpu-dp-sa", Namespace: defaultNamespace}, dpSA)).To(Succeed()) - - By("DP SCC is created") - dpSCC := &unstructured.Unstructured{} - dpSCC.SetAPIVersion(sccAPIVersion) - dpSCC.SetKind(sccKind) - Expect(k8sClient.Get(ctx, client.ObjectKey{Name: resourceName + "-gpu-dp-scc"}, dpSCC)).To(Succeed()) - - By("DP ClusterRole is created") - Expect(k8sClient.Get(ctx, client.ObjectKey{Name: resourceName + "-gpu-dp-scc-role"}, &rbac.ClusterRole{})).To(Succeed()) - - By("DP ClusterRoleBinding is created") - Expect(k8sClient.Get(ctx, client.ObjectKey{Name: resourceName + "-gpu-dp-scc-binding"}, &rbac.ClusterRoleBinding{})).To(Succeed()) - - By("XPUM ServiceAccount is created") - Expect(k8sClient.Get(ctx, client.ObjectKey{Name: resourceName + "-xpu-manager-sa", Namespace: defaultNamespace}, &v1.ServiceAccount{})).To(Succeed()) - - By("XPUM SCC is created") - xpumSCC := &unstructured.Unstructured{} - xpumSCC.SetAPIVersion(sccAPIVersion) - xpumSCC.SetKind(sccKind) - Expect(k8sClient.Get(ctx, client.ObjectKey{Name: resourceName + "-xpu-manager-scc"}, xpumSCC)).To(Succeed()) - - By("DaemonSets have correct ServiceAccountNames") - dsList := &apps.DaemonSetList{} - Expect(k8sClient.List(ctx, dsList, client.InNamespace(defaultNamespace))).To(Succeed()) - Expect(dsList.Items).To(HaveLen(2)) - for _, ds := range dsList.Items { - switch ds.Name { - case resourceName + "-device-plugin": - Expect(ds.Spec.Template.Spec.ServiceAccountName).To(Equal(resourceName + "-gpu-dp-sa")) - case resourceName + "-xpu-manager": - Expect(ds.Spec.Template.Spec.ServiceAccountName).To(Equal(resourceName + "-xpu-manager-sa")) - default: - Fail("Unexpected DaemonSet: " + ds.Name) - } - } - - By("second reconcile is idempotent") - _, err = reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) - Expect(err).NotTo(HaveOccurred()) - }) - }) - -}) - -var _ = Describe("Device Plugin", func() { - - Context("Arguments creation", func() { - It("with defaults", func() { - spec := &v1alpha.ClusterPolicy{ - Spec: v1alpha.ClusterPolicySpec{ - DevicePluginSpec: v1alpha.DevicePluginSpec{}, - }, - } - args := dpArgs(spec) - Expect(args).To(BeEmpty()) - }) - - It("Resource monitoring enabled", func() { - spec := &v1alpha.ClusterPolicy{ - Spec: v1alpha.ClusterPolicySpec{ - ResourceMonitoring: true, - DevicePluginSpec: v1alpha.DevicePluginSpec{}, - }, - } - args := dpArgs(spec) - Expect(args).To(ContainElement("-enable-monitoring")) - Expect(args).To(ContainElement("-xpumd-endpoint=/run/xpumd/intelxpuinfo.sock")) - }) - - It("Log level set", func() { - spec := &v1alpha.ClusterPolicy{ - Spec: v1alpha.ClusterPolicySpec{ - LogLevel: 2, - DevicePluginSpec: v1alpha.DevicePluginSpec{ - LogLevel: 3, - }, - }, - } - args := dpArgs(spec) - Expect(args).To(ContainElement("-v=3")) - }) - - It("ByPath set", func() { - spec := &v1alpha.ClusterPolicy{ - Spec: v1alpha.ClusterPolicySpec{ - DevicePluginSpec: v1alpha.DevicePluginSpec{ - ByPathMode: "all", - }, - }, - } - args := dpArgs(spec) - Expect(args).To(ContainElement("-bypath=all")) - }) - - It("Allow IDs set", func() { - spec := &v1alpha.ClusterPolicy{ - Spec: v1alpha.ClusterPolicySpec{ - DevicePluginSpec: v1alpha.DevicePluginSpec{ - AllowIDs: []string{"0xabcd", "0xdefa"}, - }, - }, - } - args := dpArgs(spec) - Expect(args).To(ContainElement("-allow-ids=0xabcd,0xdefa")) - }) - - It("Deny IDs set", func() { - spec := &v1alpha.ClusterPolicy{ - Spec: v1alpha.ClusterPolicySpec{ - DevicePluginSpec: v1alpha.DevicePluginSpec{ - DenyIDs: []string{"0x0xyz", "0x0123"}, - }, - }, - } - args := dpArgs(spec) - Expect(args).To(ContainElement("-deny-ids=0x0xyz,0x0123")) - }) - - It("All options set", func() { - spec := &v1alpha.ClusterPolicy{ - Spec: v1alpha.ClusterPolicySpec{ - ResourceMonitoring: true, - LogLevel: 1, - DevicePluginSpec: v1alpha.DevicePluginSpec{ - LogLevel: 3, - AllowIDs: []string{"0x1id1"}, - DenyIDs: []string{"0x2id2"}, - }, - }, - } - args := dpArgs(spec) - Expect(args).To(HaveLen(5)) - Expect(args).To(ContainElement("-enable-monitoring")) - Expect(args).To(ContainElement("-v=3")) - Expect(args).To(ContainElement("-xpumd-endpoint=/run/xpumd/intelxpuinfo.sock")) - Expect(args).To(ContainElement("-allow-ids=0x1id1")) - Expect(args).To(ContainElement("-deny-ids=0x2id2")) - }) - }) - - Context("DaemonSet object update", func() { - It("with tolerations and pullsecret", func() { - cp := &v1alpha.ClusterPolicy{ - Spec: v1alpha.ClusterPolicySpec{ - LogLevel: 2, - DevicePluginSpec: v1alpha.DevicePluginSpec{ - LogLevel: 3, - }, - Tolerations: []v1.Toleration{ - { - Key: "my-toleration", - Operator: v1.TolerationOpExists, - Effect: v1.TaintEffectNoExecute, - }, - }, - PullSecret: &v1.LocalObjectReference{ - Name: "my-pull-secret", - }, - }, - } - controller := &DevicePluginReconciler{} - - ds := deployments.DevicePluginDaemonset() - controller.updateDaemonSetObject(ds, cp) - - Expect(ds.Spec.Template.Spec.Tolerations[0].Key).To(Equal("my-toleration")) - Expect(ds.Spec.Template.Spec.Tolerations[0].Operator).To(Equal(v1.TolerationOpExists)) - Expect(ds.Spec.Template.Spec.Tolerations[0].Effect).To(Equal(v1.TaintEffectNoExecute)) - - Expect(ds.Spec.Template.Spec.ImagePullSecrets[0].Name).To(Equal("my-pull-secret")) - - cp.Spec.PullSecret = nil - cp.Spec.Tolerations = nil - controller.updateDaemonSetObject(ds, cp) - - Expect(ds.Spec.Template.Spec.Tolerations).To(BeEmpty()) - Expect(ds.Spec.Template.Spec.ImagePullSecrets).To(BeEmpty()) - }) - - It("preserves pod spec and container security context from YAML", func() { - cp := &v1alpha.ClusterPolicy{ - Spec: v1alpha.ClusterPolicySpec{ - DevicePluginSpec: v1alpha.DevicePluginSpec{ - PluginImage: "intel/intel-gpu-plugin:test", - }, - }, - } - controller := &DevicePluginReconciler{} - - ds := deployments.DevicePluginDaemonset() - controller.updateDaemonSetObject(ds, cp) - - By("automountServiceAccountToken is false") - Expect(ds.Spec.Template.Spec.AutomountServiceAccountToken).NotTo(BeNil()) - Expect(*ds.Spec.Template.Spec.AutomountServiceAccountToken).To(BeFalse()) - }) - - It("adds runxpumd volume and mount when ResourceMonitoring is true", func() { - cp := &v1alpha.ClusterPolicy{ - Spec: v1alpha.ClusterPolicySpec{ - ResourceMonitoring: true, - DevicePluginSpec: v1alpha.DevicePluginSpec{ - PluginImage: "intel/intel-gpu-plugin:test", - }, - }, - } - controller := &DevicePluginReconciler{} - - ds := deployments.DevicePluginDaemonset() - controller.updateDaemonSetObject(ds, cp) - - volNames := []string{} - for _, v := range ds.Spec.Template.Spec.Volumes { - volNames = append(volNames, v.Name) - } - Expect(volNames).To(ContainElement(xpumdVolumeName)) - - mountNames := []string{} - for _, vm := range ds.Spec.Template.Spec.Containers[0].VolumeMounts { - mountNames = append(mountNames, vm.Name) - } - Expect(mountNames).To(ContainElement(xpumdVolumeName)) - }) - - It("does not duplicate runxpumd volume when updateDaemonSetObject is called twice", func() { - cp := &v1alpha.ClusterPolicy{ - Spec: v1alpha.ClusterPolicySpec{ - ResourceMonitoring: true, - DevicePluginSpec: v1alpha.DevicePluginSpec{ - PluginImage: "intel/intel-gpu-plugin:test", - }, - }, - } - controller := &DevicePluginReconciler{} - - ds := deployments.DevicePluginDaemonset() - controller.updateDaemonSetObject(ds, cp) - controller.updateDaemonSetObject(ds, cp) - - count := 0 - for _, v := range ds.Spec.Template.Spec.Volumes { - if v.Name == xpumdVolumeName { - count++ - } - } - Expect(count).To(Equal(1)) - }) - - It("removes runxpumd volume and mount when ResourceMonitoring is disabled", func() { - cp := &v1alpha.ClusterPolicy{ - Spec: v1alpha.ClusterPolicySpec{ - ResourceMonitoring: true, - DevicePluginSpec: v1alpha.DevicePluginSpec{ - PluginImage: "intel/intel-gpu-plugin:test", - }, - }, - } - controller := &DevicePluginReconciler{} - - ds := deployments.DevicePluginDaemonset() - controller.updateDaemonSetObject(ds, cp) - - cp.Spec.ResourceMonitoring = false - controller.updateDaemonSetObject(ds, cp) - - for _, v := range ds.Spec.Template.Spec.Volumes { - Expect(v.Name).NotTo(Equal(xpumdVolumeName)) - } - for _, vm := range ds.Spec.Template.Spec.Containers[0].VolumeMounts { - Expect(vm.Name).NotTo(Equal(xpumdVolumeName)) - } - }) - - It("removeXpumdMounts is a no-op when runxpumd is not present", func() { - cp := &v1alpha.ClusterPolicy{ - Spec: v1alpha.ClusterPolicySpec{ - ResourceMonitoring: false, - DevicePluginSpec: v1alpha.DevicePluginSpec{ - PluginImage: "intel/intel-gpu-plugin:test", - }, - }, - } - controller := &DevicePluginReconciler{} - - ds := deployments.DevicePluginDaemonset() - initialVolCount := len(ds.Spec.Template.Spec.Volumes) - initialMountCount := len(ds.Spec.Template.Spec.Containers[0].VolumeMounts) - - Expect(func() { controller.updateDaemonSetObject(ds, cp) }).NotTo(Panic()) - Expect(ds.Spec.Template.Spec.Volumes).To(HaveLen(initialVolCount)) - Expect(ds.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(initialMountCount)) - }) - }) -}) diff --git a/internal/controller/dra_controller.go b/internal/controller/dra_controller.go deleted file mode 100644 index 1f3fee3..0000000 --- a/internal/controller/dra_controller.go +++ /dev/null @@ -1,582 +0,0 @@ -/* -Copyright 2025 Intel Corporation. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "context" - "fmt" - - adreg "k8s.io/api/admissionregistration/v1" - apps "k8s.io/api/apps/v1" - core "k8s.io/api/core/v1" - rbac "k8s.io/api/rbac/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/klog/v2" - - resv1 "k8s.io/api/resource/v1" - - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - logf "sigs.k8s.io/controller-runtime/pkg/log" - - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - v1alpha "github.com/intel/gpu-base-operator/api/v1alpha1" - "github.com/intel/gpu-base-operator/config/deployments" -) - -type DRAReconciler struct { - client.Client - Scheme *runtime.Scheme - Opts ControllerOpts -} - -const ( - draValue = "intel-gpu-resource-driver-kubelet-plugin" - healthCheckPort = 51516 - gpuDeviceClass = "gpu.intel.com" - vfioGpuDeviceClass = "gpu-vfio.intel.com" - - draResourcePart = "gpu-dra" -) - -func (r *DRAReconciler) createAll(ctx context.Context, cp *v1alpha.ClusterPolicy) error { - objects := []client.Object{} - - objName := fmt.Sprintf("%s-gpu-dra", cp.Name) - - // Service account - sa := deployments.DynamicResourceAllocationServiceAccount() - sa.Name = objName - sa.Namespace = r.Opts.Namespace - objects = append(objects, sa) - - // Cluster role - cr := deployments.DynamicResourceAllocationClusterRole() - cr.Name = objName - objects = append(objects, cr) - - // Cluster role binding - crb := deployments.DynamicResourceAllocationClusterRoleBinding() - crb.Name = objName - crb.Namespace = r.Opts.Namespace - - for i := range crb.Subjects { - crb.Subjects[i].Namespace = r.Opts.Namespace - crb.Subjects[i].Name = objName - } - crb.RoleRef.Name = objName - objects = append(objects, crb) - - // Device classes - objects = append(objects, deployments.DynamicResourceAllocationDeviceClass()) - - // Device Class for VFIO and configure it based on the ManageBinding setting in the CR. - mb := cp.Spec.DynamicResourceAllocationSpec.ManageBinding - objects = append(objects, deployments.DynamicResourceAllocationDeviceClassVfio(!mb)) - - // Validating admission policy - ap := deployments.DynamicResourceAllocationValidatingAdmissionPolicy() - ap.Name = objName - ap.Spec.MatchConditions[0].Expression = fmt.Sprintf("request.userInfo.username == \"system:serviceaccount:%s:%s\"", r.Opts.Namespace, objName) - objects = append(objects, ap) - - // Validating admission policy binding - apb := deployments.DynamicResourceAllocationValidatingAdmissionPolicyBinding() - apb.Name = objName - apb.Spec.PolicyName = apb.Name - objects = append(objects, apb) - - for _, o := range objects { - if err := r.Create(ctx, o); err != nil { - if errors.IsAlreadyExists(err) { - klog.Info("object already exists: "+o.GetName(), o) - - continue - } - - klog.Error(err, "unable to create object", "object", o) - - return err - } - } - - return r.createDaemonSet(ctx, cp) -} - -func (r *DRAReconciler) ensureVfioDeviceClass(ctx context.Context, manageBinding bool) { - existing := deployments.DynamicResourceAllocationDeviceClassVfio(!manageBinding) - - if err := r.Get(ctx, client.ObjectKey{Name: existing.Name}, existing); err != nil { - if errors.IsNotFound(err) { - klog.Info("VFIO device class not found, creating") - - if err := r.Create(ctx, existing); err != nil { - klog.Error(err, "unable to create VFIO device class") - } - - return - } - - klog.Error(err, "unable to get VFIO device class") - return - } - - desired := deployments.DynamicResourceAllocationDeviceClassVfio(!manageBinding) - - diff := cmp.Diff(existing.Spec.Selectors, desired.Spec.Selectors, cmpopts.EquateEmpty()) - if len(diff) > 0 { - klog.V(2).Info("Updating VFIO device class due to ManageBinding change", "diff", diff) - - existing.Spec.Selectors = desired.Spec.Selectors - - if err := r.Update(ctx, existing); err != nil { - klog.Error(err, "unable to update VFIO device class") - } - } -} - -func (r *DRAReconciler) deleteAll(ctx context.Context, crName string) error { - klog.Info("Deleting all DRA related objects") - - objName := fmt.Sprintf("%s-gpu-dra", crName) - - objects := []client.Object{ - &core.ServiceAccount{ - ObjectMeta: metav1.ObjectMeta{ - Name: objName, - Namespace: r.Opts.Namespace, - }, - }, - &rbac.ClusterRole{ - ObjectMeta: metav1.ObjectMeta{ - Name: objName, - }, - }, - &rbac.ClusterRoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: objName, - Namespace: r.Opts.Namespace, - }, - }, - &resv1.DeviceClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: gpuDeviceClass, - }, - }, - &resv1.DeviceClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: vfioGpuDeviceClass, - }, - }, - &adreg.ValidatingAdmissionPolicy{ - ObjectMeta: metav1.ObjectMeta{ - Name: objName, - }, - }, - &adreg.ValidatingAdmissionPolicyBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: objName, - }, - }, - &apps.DaemonSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: objName, - Namespace: r.Opts.Namespace, - }, - }, - } - - if r.Opts.OpenShift { - sccName, roleName, bindingName, _ := buildOpenShiftNames(crName, draResourcePart) - - scc := &unstructured.Unstructured{} - scc.SetAPIVersion(sccAPIVersion) - scc.SetKind(sccKind) - scc.SetName(sccName) - - objects = append(objects, - scc, - &rbac.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: roleName}}, - &rbac.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: bindingName}}, - ) - } - - for _, o := range objects { - if err := r.Delete(ctx, o); err != nil { - if errors.IsNotFound(err) { - klog.Warningf("unable to delete object, not found: %+v (%+v)", err, o) - - continue - } - - klog.Error(err, "unable to delete object", "object", o) - - return err - } - } - - klog.Info("Objects deleted") - - return nil -} - -func (r *DRAReconciler) createOpenShiftResourcesIfNotExists(ctx context.Context, crName string) error { - sccName, roleName, bindingName, _ := buildOpenShiftNames(crName, draResourcePart) - // Use the same service account name as the DaemonSet, which is based on the CR name. - saName := fmt.Sprintf("%s-gpu-dra", crName) - - if err := ensureSCC(ctx, r.Client, buildDRASCC(sccName)); err != nil { - klog.Error(err, "unable to ensure DRA SCC") - - return err - } - - if err := createSCCRole(ctx, r.Client, roleName, sccName); err != nil { - klog.Error(err, "unable to ensure DRA SCC ClusterRole") - - return err - } - - if err := createSCCRoleBinding(ctx, r.Client, bindingName, roleName, saName, r.Opts.Namespace); err != nil { - klog.Error(err, "unable to ensure DRA SCC ClusterRoleBinding") - - return err - } - - return nil -} - -func (r *DRAReconciler) anyAllocatedResourceClaims(ctx context.Context) bool { - var rcList resv1.ResourceClaimList - - klog.Info("Checking for allocated ResourceClaims that would prevent DRA removal") - - if err := r.List(ctx, &rcList); err != nil { - klog.Error(err, "unable to list ResourceClaims") - - return false - } - - klog.Infof("Found %d ResourceClaims", len(rcList.Items)) - for _, claim := range rcList.Items { - alloc := claim.Status.Allocation - - if alloc == nil { - continue - } - if len(alloc.Devices.Results) == 0 { - continue - } - - for _, dev := range alloc.Devices.Results { - if dev.Driver == gpuDeviceClass { - klog.Infof("Found allocated ResourceClaim with GPU device: %s", claim.Name) - - return true - } - } - } - - return false -} - -func addHealthCheckIfMissing(container *core.Container, port int32) { - for _, p := range container.Ports { - if p.ContainerPort == port { - // If port is already there, we assume the health check is already set up. - return - } - } - - service := "liveness" - - container.Ports = append(container.Ports, core.ContainerPort{ - Name: "healthcheck", - ContainerPort: port, - }) - - container.StartupProbe = &core.Probe{ - FailureThreshold: 60, - PeriodSeconds: 10, - TimeoutSeconds: 10, - ProbeHandler: core.ProbeHandler{ - GRPC: &core.GRPCAction{ - Port: port, - Service: &service, - }, - }, - } - - container.LivenessProbe = &core.Probe{ - FailureThreshold: 3, - PeriodSeconds: 30, - TimeoutSeconds: 10, - ProbeHandler: core.ProbeHandler{ - GRPC: &core.GRPCAction{ - Port: port, - Service: &service, - }, - }, - } -} - -func removeHealthCheckIfExists(container *core.Container) { - found := false - - for i, p := range container.Ports { - if p.Name == "healthcheck" { - container.Ports = append(container.Ports[:i], container.Ports[i+1:]...) - found = true - break - } - } - - if !found { - return - } - - container.StartupProbe = nil - container.LivenessProbe = nil -} - -func (r *DRAReconciler) updateDaemonSetObject(ds *apps.DaemonSet, spec *v1alpha.ClusterPolicy) { - name := fmt.Sprintf("%s-gpu-dra", spec.Name) - - ds.Name = name - ds.Namespace = r.Opts.Namespace - ds.Labels[ownerKey] = spec.Name - - ds.Spec.Template.Spec.ServiceAccountName = name - - dspec := spec.Spec.DynamicResourceAllocationSpec - - ds.Spec.Template.Spec.Containers[0].Image = dspec.Image - ds.Spec.Template.Spec.Containers[0].Args = r.generateArgs(spec) - - ds.Spec.Template.Spec.NodeSelector = map[string]string{ - "kubernetes.io/arch": "amd64", - } - - if len(spec.Spec.NodeSelector) > 0 { - for k, v := range spec.Spec.NodeSelector { - ds.Spec.Template.Spec.NodeSelector[k] = v - } - } - - if spec.Spec.UseNFDLabeling { - ds.Spec.Template.Spec.NodeSelector["intel.feature.node.kubernetes.io/gpu"] = trueValue - } - - if len(spec.Spec.Tolerations) > 0 { - ds.Spec.Template.Spec.Tolerations = spec.Spec.Tolerations - } else { - ds.Spec.Template.Spec.Tolerations = nil - } - - cspec := &ds.Spec.Template.Spec - - // Enable health check for the DRA Pod - if spec.Spec.DynamicResourceAllocationSpec.PodHealthCheck { - addHealthCheckIfMissing(&cspec.Containers[0], healthCheckPort) - } else { - removeHealthCheckIfExists(&cspec.Containers[0]) - } - - secrets := []core.LocalObjectReference{} - if r.Opts.SecretName != "" { - secrets = append(secrets, core.LocalObjectReference{Name: r.Opts.SecretName}) - } - if spec.Spec.PullSecret != nil { - secrets = append(secrets, *spec.Spec.PullSecret) - } - - if len(secrets) > 0 { - cspec.ImagePullSecrets = secrets - } else { - cspec.ImagePullSecrets = nil - } - - if r.Opts.OpenShift { - // On OpenShift, SELinux labels the container process as container_t which cannot - // write to host directories (e.g. /etc/cdi labeled etc_t). spc_t bypasses SELinux - // confinement for the container so it can write CDI specs to the host filesystem. - if cspec.Containers[0].SecurityContext == nil { - cspec.Containers[0].SecurityContext = &core.SecurityContext{} - } - - cspec.Containers[0].SecurityContext.SELinuxOptions = &core.SELinuxOptions{ - Type: "spc_t", - } - } -} - -func (r *DRAReconciler) createDaemonSet(ctx context.Context, spec *v1alpha.ClusterPolicy) error { - ds := deployments.DynamicResourceAllocationDaemonset() - - r.updateDaemonSetObject(ds, spec) - - if err := r.Create(ctx, ds); err != nil { - klog.Error(err, "unable to create DaemonSet") - - return err - } - - return nil -} - -func (r *DRAReconciler) removeDeploymentIfExists(ctx context.Context) (ctrl.Result, error) { - crName := r.Opts.ReqName - - dss := &apps.DaemonSetList{} - labels := client.MatchingLabels{ - appLabel: draValue, - ownerKey: crName, - } - - if err := r.List(ctx, dss, client.InNamespace(r.Opts.Namespace), labels); err != nil { - klog.Error(err, "unable to list child DaemonSets") - - return ctrl.Result{}, err - } - - if len(dss.Items) == 0 { - return ctrl.Result{}, nil - } - - // If there are any allocated ResourceClaims, removal of DRA will cause - // the Pods using them to be stuck at Terminating. - // Requeue and try again later. - if r.anyAllocatedResourceClaims(ctx) { - return ctrl.Result{RequeueAfter: r.Opts.RequeueDelay}, requeueReconcileErr{} - } - - if err := r.deleteAll(ctx, r.Opts.ReqName); err != nil { - return ctrl.Result{}, err - } - - klog.Info("DRA deployment removed") - - return ctrl.Result{}, nil -} - -func (r *DRAReconciler) generateArgs(spec *v1alpha.ClusterPolicy) []string { - targetLevel := int32(0) - - targetLevel = max(targetLevel, spec.Spec.DynamicResourceAllocationSpec.LogLevel) - targetLevel = max(targetLevel, spec.Spec.LogLevel) - - args := []string{fmt.Sprintf("-v=%d", targetLevel)} - - if spec.Spec.HealthinessSpec != nil { - args = append(args, "--health-monitoring=true") - } - - if spec.Spec.DynamicResourceAllocationSpec.PodHealthCheck { - args = append(args, fmt.Sprintf("--healthcheck-port=%d", healthCheckPort)) - } else { - args = append(args, "--healthcheck-port=-1") - } - - manageBinding := "false" - if spec.Spec.DynamicResourceAllocationSpec.ManageBinding { - manageBinding = "true" - } - - args = append(args, fmt.Sprintf("--manage-binding=%s", manageBinding)) - - return args -} - -func (r *DRAReconciler) Reconcile(ctx context.Context, cp *v1alpha.ClusterPolicy) (ctrl.Result, error) { - _ = logf.FromContext(ctx) - - // If DRA is not enable in the cluster, we shouldn't cause errors in trying to do things. - if !r.Opts.DRAEnable { - if cp != nil && cp.Spec.ResourceRegistration == resourceModeDRA { - addIfMissing(&cp.Status.Errors, draNotEnabledMsg) - } - - return ctrl.Result{}, nil - } - - if cp == nil || !cp.DeletionTimestamp.IsZero() { - return r.removeDeploymentIfExists(ctx) - } - - // DRA not selected, remove existing deployment if exists - if cp.Spec.ResourceRegistration != resourceModeDRA { - cp.Status.DRAStatus = notAvailableStatus - - return r.removeDeploymentIfExists(ctx) - } - - labels := client.MatchingLabels{appLabel: draValue} - - var olderDs apps.DaemonSetList - if err := r.List(ctx, &olderDs, client.InNamespace(r.Opts.Namespace), labels); err != nil { - klog.Error(err, "unable to list child DaemonSets") - - return ctrl.Result{}, err - } - - if r.Opts.OpenShift { - if err := r.createOpenShiftResourcesIfNotExists(ctx, cp.Name); err != nil { - klog.Error(err, "unable to ensure OpenShift resources for DRA") - - return ctrl.Result{}, err - } - } - - if len(olderDs.Items) == 0 { - return ctrl.Result{}, r.createAll(ctx, cp) - } - - // Update DaemonSet - - ds := &olderDs.Items[0] - originalDs := ds.DeepCopy() - - r.updateDaemonSetObject(ds, cp) - - dsDiff := cmp.Diff(originalDs.Spec.Template.Spec, ds.Spec.Template.Spec, cmpopts.EquateEmpty()) - if len(dsDiff) > 0 { - klog.Info("DRA difference", "diff", dsDiff) - - if err := r.Update(ctx, ds); err != nil { - klog.Error(err, "unable to update daemonset", "DaemonSet", ds) - - return ctrl.Result{}, err - } - } - - r.ensureVfioDeviceClass(ctx, cp.Spec.DynamicResourceAllocationSpec.ManageBinding) - - if err := r.List(ctx, &olderDs, client.InNamespace(r.Opts.Namespace), labels); err != nil { - klog.Error(err, "unable to list child DaemonSets") - - return ctrl.Result{}, err - } - - cp.Status.DRAStatus = fmt.Sprintf("%d/%d", - olderDs.Items[0].Status.NumberReady, olderDs.Items[0].Status.DesiredNumberScheduled) - - return ctrl.Result{}, nil -} diff --git a/internal/controller/dra_controller_test.go b/internal/controller/dra_controller_test.go deleted file mode 100644 index d34a237..0000000 --- a/internal/controller/dra_controller_test.go +++ /dev/null @@ -1,534 +0,0 @@ -/* -Copyright 2025 Intel Corporation. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "context" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - apps "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" - rbac "k8s.io/api/rbac/v1" - resv1 "k8s.io/api/resource/v1" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - - v1alpha "github.com/intel/gpu-base-operator/api/v1alpha1" - "github.com/intel/gpu-base-operator/config/deployments" -) - -var _ = Describe("ClusterPolicy Controller for DRA", func() { - - Context("When reconciling DRA", func() { - defaultNamespace := "foobar-dra" - const resourceName = "test-resource-dra" - - ctx := context.Background() - - typeNamespacedName := types.NamespacedName{ - Name: resourceName, - } - clusterpolicy := &v1alpha.ClusterPolicy{} - - BeforeEach(func() { - ns := &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaultNamespace, - }, - } - - Expect(k8sClient.Create(ctx, ns)).To(Succeed()) - }) - - AfterEach(func() { - resource := &v1alpha.ClusterPolicy{} - err := k8sClient.Get(ctx, typeNamespacedName, resource) - Expect(err).NotTo(HaveOccurred()) - - By("Cleanup the specific resource instance ClusterPolicy") - Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) - }) - - It("should successfully reconcile the resource", func() { - By("creating the custom resource for the Kind ClusterPolicy") - err := k8sClient.Get(ctx, typeNamespacedName, clusterpolicy) - if err != nil && errors.IsNotFound(err) { - resource := &v1alpha.ClusterPolicy{ - ObjectMeta: metav1.ObjectMeta{ - Name: resourceName, - }, - Spec: v1alpha.ClusterPolicySpec{ - ResourceRegistration: "dra", - DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ - Image: "dra-image:v1.2.3", - }, - }, - } - Expect(k8sClient.Create(ctx, resource)).To(Succeed()) - } - - By("Reconciling the created resource") - controllerReconciler := &ClusterPolicyReconciler{ - Client: k8sClient, - Scheme: k8sClient.Scheme(), - Opts: ControllerOpts{ - Namespace: defaultNamespace, - DRAEnable: true, - RequeueDelay: time.Millisecond * 50, - }, - } - - _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ - NamespacedName: typeNamespacedName, - }) - - Expect(err).NotTo(HaveOccurred()) - - daemonSets := apps.DaemonSetList{} - err = k8sClient.List(ctx, &daemonSets, client.InNamespace(defaultNamespace)) - Expect(err).NotTo(HaveOccurred()) - - Expect(daemonSets.Items).To(HaveLen(1)) - - for _, ds := range daemonSets.Items { - switch ds.Name { - case "test-resource-dra-gpu-dra": - Expect(ds.Spec.Template.Spec.Containers[0].Image).To(Equal("dra-image:v1.2.3")) - default: - Fail("Unexpected DaemonSet found: " + ds.Name) - } - } - }) - }) - - Context("When reconciling DRA and XPUM", func() { - defaultNamespace := "foobar-dra-xpum" - const resourceName = "test-resource-dra-xpum" - - ctx := context.Background() - - typeNamespacedName := types.NamespacedName{ - Name: resourceName, - } - clusterpolicy := &v1alpha.ClusterPolicy{} - - BeforeEach(func() { - ns := &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaultNamespace, - Labels: map[string]string{ - "resource.kubernetes.io/admin-access": "true", - }, - }, - } - - Expect(k8sClient.Create(ctx, ns)).To(Succeed()) - }) - - AfterEach(func() { - resource := &v1alpha.ClusterPolicy{} - err := k8sClient.Get(ctx, typeNamespacedName, resource) - Expect(err).NotTo(HaveOccurred()) - - By("Cleanup the specific resource instance ClusterPolicy") - Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) - }) - - It("should successfully reconcile the resource", func() { - By("creating the custom resource for the Kind ClusterPolicy") - err := k8sClient.Get(ctx, typeNamespacedName, clusterpolicy) - if err != nil && errors.IsNotFound(err) { - resource := &v1alpha.ClusterPolicy{ - ObjectMeta: metav1.ObjectMeta{ - Name: resourceName, - }, - Spec: v1alpha.ClusterPolicySpec{ - ResourceRegistration: "dra", - ResourceMonitoring: true, - UseNFDLabeling: true, - DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ - Image: "dra-image:v1.2.3", - }, - XpuManagerSpec: v1alpha.XpuManagerSpec{ - Image: "xpumanager:v3.2.1", - LogLevel: 3, - }, - }, - } - Expect(k8sClient.Create(ctx, resource)).To(Succeed()) - } - - By("Reconciling the created resource") - controllerReconciler := &ClusterPolicyReconciler{ - Client: k8sClient, - Scheme: k8sClient.Scheme(), - Opts: ControllerOpts{ - Namespace: defaultNamespace, - DRAEnable: true, - RequeueDelay: time.Millisecond * 50, - }, - } - - _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ - NamespacedName: typeNamespacedName, - }) - - Expect(err).NotTo(HaveOccurred()) - - daemonSets := apps.DaemonSetList{} - err = k8sClient.List(ctx, &daemonSets, client.InNamespace(defaultNamespace)) - Expect(err).NotTo(HaveOccurred()) - - Expect(daemonSets.Items).To(HaveLen(2)) - - for _, ds := range daemonSets.Items { - switch ds.Name { - case "test-resource-dra-xpum-gpu-dra": - Expect(ds.Spec.Template.Spec.Containers[0].Image).To(Equal("dra-image:v1.2.3")) - Expect(ds.Spec.Template.Spec.NodeSelector).To(HaveKeyWithValue("intel.feature.node.kubernetes.io/gpu", "true")) - Expect(ds.Spec.Template.Spec.NodeSelector).To(HaveKeyWithValue("kubernetes.io/arch", "amd64")) - case "test-resource-dra-xpum-xpu-manager": - Expect(ds.Spec.Template.Spec.Containers[0].Image).To(Equal("xpumanager:v3.2.1")) - Expect(ds.Spec.Template.Spec.NodeSelector).To(HaveKeyWithValue("intel.feature.node.kubernetes.io/gpu", "true")) - Expect(ds.Spec.Template.Spec.NodeSelector).To(HaveKeyWithValue("kubernetes.io/arch", "amd64")) - default: - Fail("Unexpected DaemonSet found: " + ds.Name) - } - } - - dcList := &resv1.DeviceClassList{} - err = k8sClient.List(ctx, dcList) - Expect(err).NotTo(HaveOccurred()) - Expect(dcList.Items).To(HaveLen(2)) - for _, dc := range dcList.Items { - switch dc.Name { - case "gpu.intel.com": - case "gpu-vfio.intel.com": - Expect(dc.Spec.Selectors).To(HaveLen(2)) - default: - Fail("Unexpected DeviceClass found: " + dc.Name) - } - } - - Expect(k8sClient.Get(ctx, typeNamespacedName, clusterpolicy)).To(Succeed(), "Failed to get ClusterPolicy after reconciliation") - - clusterpolicy.Spec.DynamicResourceAllocationSpec.ManageBinding = true - Expect(k8sClient.Update(ctx, clusterpolicy)).To(Succeed(), "Failed to update ClusterPolicy with ManageBinding change") - - _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ - NamespacedName: typeNamespacedName, - }) - Expect(err).NotTo(HaveOccurred()) - - err = k8sClient.List(ctx, dcList) - Expect(err).NotTo(HaveOccurred()) - Expect(dcList.Items).To(HaveLen(2)) - - for _, dc := range dcList.Items { - switch dc.Name { - case "gpu.intel.com": - case "gpu-vfio.intel.com": - Expect(dc.Spec.Selectors).To(HaveLen(1)) - default: - Fail("Unexpected DeviceClass found: " + dc.Name) - } - } - }) - }) - - Context("When reconciling DRA on OpenShift", func() { - defaultNamespace := "foobar-dra-ocp" - const resourceName = "test-resource-dra-ocp" - - ctx := context.Background() - - typeNamespacedName := types.NamespacedName{Name: resourceName} - - BeforeEach(func() { - Expect(k8sClient.Create(ctx, &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{Name: defaultNamespace}, - })).To(Succeed()) - }) - - AfterEach(func() { - resource := &v1alpha.ClusterPolicy{} - Expect(k8sClient.Get(ctx, typeNamespacedName, resource)).To(Succeed()) - Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) - - deleteOpenShiftSCCResources(ctx, k8sClient, - resourceName+"-gpu-dra-scc", - resourceName+"-gpu-dra-scc-role", - resourceName+"-gpu-dra-scc-binding", - "", "") - }) - - It("creates DRA SCC resources and cleans them up on DRA removal", func() { - By("creating the ClusterPolicy with DRA mode") - Expect(k8sClient.Create(ctx, &v1alpha.ClusterPolicy{ - ObjectMeta: metav1.ObjectMeta{Name: resourceName}, - Spec: v1alpha.ClusterPolicySpec{ - ResourceRegistration: "dra", - DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ - Image: "intel/gpu-dra-driver:test", - }, - }, - })).To(Succeed()) - - reconciler := &ClusterPolicyReconciler{ - Client: k8sClient, - Scheme: k8sClient.Scheme(), - Opts: ControllerOpts{ - Namespace: defaultNamespace, - DRAEnable: true, - OpenShift: true, - RequeueDelay: time.Millisecond * 50, - }, - } - - By("reconcile creates DRA SCC resources") - _, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) - Expect(err).NotTo(HaveOccurred()) - - By("DRA SCC is created") - draSCC := &unstructured.Unstructured{} - draSCC.SetAPIVersion(sccAPIVersion) - draSCC.SetKind(sccKind) - Expect(k8sClient.Get(ctx, client.ObjectKey{Name: resourceName + "-gpu-dra-scc"}, draSCC)).To(Succeed()) - - By("DRA SCC ClusterRole is created") - Expect(k8sClient.Get(ctx, client.ObjectKey{Name: resourceName + "-gpu-dra-scc-role"}, &rbac.ClusterRole{})).To(Succeed()) - - By("DRA SCC ClusterRoleBinding is created") - Expect(k8sClient.Get(ctx, client.ObjectKey{Name: resourceName + "-gpu-dra-scc-binding"}, &rbac.ClusterRoleBinding{})).To(Succeed()) - - By("second reconcile is idempotent") - _, err = reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) - Expect(err).NotTo(HaveOccurred()) - - By("creating a ResourceClaim exercises anyAllocatedResourceClaims loop") - resourceClaim := &resv1.ResourceClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-rc-coverage", - Namespace: defaultNamespace, - }, - Spec: resv1.ResourceClaimSpec{ - Devices: resv1.DeviceClaim{ - Requests: []resv1.DeviceRequest{ - {Name: "gpu", Exactly: &resv1.ExactDeviceRequest{DeviceClassName: gpuDeviceClass}}, - }, - }, - }, - } - Expect(k8sClient.Create(ctx, resourceClaim)).To(Succeed()) - DeferCleanup(func() { _ = k8sClient.Delete(ctx, resourceClaim) }) - - By("switching to DP mode removes DRA SCC resources") - cp := &v1alpha.ClusterPolicy{} - Expect(k8sClient.Get(ctx, typeNamespacedName, cp)).To(Succeed()) - cp.Spec.ResourceRegistration = "dp" - cp.Spec.DevicePluginSpec = v1alpha.DevicePluginSpec{ - PluginImage: "intel/intel-gpu-plugin:test", - } - Expect(k8sClient.Update(ctx, cp)).To(Succeed()) - - _, err = reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) - Expect(err).NotTo(HaveOccurred()) - - draSCCCheck := &unstructured.Unstructured{} - draSCCCheck.SetAPIVersion(sccAPIVersion) - draSCCCheck.SetKind(sccKind) - Expect(k8sClient.Get(ctx, client.ObjectKey{Name: resourceName + "-gpu-dra-scc"}, draSCCCheck)). - To(Satisfy(errors.IsNotFound)) - }) - }) - - Context("DaemonSet object update", func() { - It("with tolerations, nodeSelector and pullsecret", func() { - cp := &v1alpha.ClusterPolicy{ - Spec: v1alpha.ClusterPolicySpec{ - LogLevel: 2, - DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ - LogLevel: 3, - PodHealthCheck: true, - }, - Tolerations: []v1.Toleration{ - { - Key: "my-toleration", - Operator: v1.TolerationOpExists, - Effect: v1.TaintEffectNoExecute, - }, - }, - PullSecret: &v1.LocalObjectReference{ - Name: "my-pull-secret", - }, - NodeSelector: map[string]string{ - "foo": "bar", - }, - }, - } - controller := &DRAReconciler{} - - ds := deployments.DynamicResourceAllocationDaemonset() - controller.updateDaemonSetObject(ds, cp) - - Expect(ds.Spec.Template.Spec.NodeSelector).To(HaveKeyWithValue("foo", "bar")) - - Expect(ds.Spec.Template.Spec.Tolerations[0].Key).To(Equal("my-toleration")) - Expect(ds.Spec.Template.Spec.Tolerations[0].Operator).To(Equal(v1.TolerationOpExists)) - Expect(ds.Spec.Template.Spec.Tolerations[0].Effect).To(Equal(v1.TaintEffectNoExecute)) - - Expect(ds.Spec.Template.Spec.ImagePullSecrets[0].Name).To(Equal("my-pull-secret")) - - Expect(ds.Spec.Template.Spec.Containers[0].Args).To(ContainElement("--healthcheck-port=51516")) - Expect(ds.Spec.Template.Spec.Containers[0].Ports).To(ContainElement(v1.ContainerPort{ - Name: "healthcheck", - ContainerPort: 51516, - })) - - Expect(ds.Spec.Template.Spec.Containers[0].StartupProbe).ToNot(BeNil()) - Expect(ds.Spec.Template.Spec.Containers[0].StartupProbe.GRPC).ToNot(BeNil()) - Expect(ds.Spec.Template.Spec.Containers[0].StartupProbe.GRPC.Port).To(Equal(int32(51516))) - - Expect(ds.Spec.Template.Spec.Containers[0].LivenessProbe).ToNot(BeNil()) - Expect(ds.Spec.Template.Spec.Containers[0].LivenessProbe.GRPC).ToNot(BeNil()) - Expect(ds.Spec.Template.Spec.Containers[0].LivenessProbe.GRPC.Port).To(Equal(int32(51516))) - - // Update with no changes - controller.updateDaemonSetObject(ds, cp) - Expect(ds.Spec.Template.Spec.Containers[0].StartupProbe).ToNot(BeNil()) - Expect(ds.Spec.Template.Spec.Containers[0].LivenessProbe).ToNot(BeNil()) - - cp.Spec.PullSecret = nil - cp.Spec.Tolerations = nil - cp.Spec.DynamicResourceAllocationSpec.PodHealthCheck = false - controller.updateDaemonSetObject(ds, cp) - - Expect(ds.Spec.Template.Spec.Tolerations).To(BeEmpty()) - Expect(ds.Spec.Template.Spec.ImagePullSecrets).To(BeEmpty()) - Expect(ds.Spec.Template.Spec.Containers[0].Args).To(ContainElement("--healthcheck-port=-1")) - Expect(ds.Spec.Template.Spec.Containers[0].StartupProbe).To(BeNil()) - Expect(ds.Spec.Template.Spec.Containers[0].LivenessProbe).To(BeNil()) - }) - - It("preserves kubelet-plugin container security context from YAML", func() { - cp := &v1alpha.ClusterPolicy{ - Spec: v1alpha.ClusterPolicySpec{ - DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ - Image: "intel/gpu-dra-driver:test", - }, - }, - } - controller := &DRAReconciler{} - - ds := deployments.DynamicResourceAllocationDaemonset() - controller.updateDaemonSetObject(ds, cp) - - var kubeletPlugin *v1.Container - for i := range ds.Spec.Template.Spec.Containers { - if ds.Spec.Template.Spec.Containers[i].Name == "kubelet-plugin" { - kubeletPlugin = &ds.Spec.Template.Spec.Containers[i] - break - } - } - Expect(kubeletPlugin).NotTo(BeNil()) - Expect(kubeletPlugin.SecurityContext).NotTo(BeNil()) - Expect(kubeletPlugin.SecurityContext.AllowPrivilegeEscalation).NotTo(BeNil()) - Expect(*kubeletPlugin.SecurityContext.AllowPrivilegeEscalation).To(BeFalse()) - }) - }) - - Context("DRA arguments handling", func() { - It("with health monitoring", func() { - cp := &v1alpha.ClusterPolicy{ - Spec: v1alpha.ClusterPolicySpec{ - LogLevel: 2, - DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ - LogLevel: 3, - PodHealthCheck: false, - }, - HealthinessSpec: &v1alpha.HealthinessSpec{ - CheckIntervalSeconds: 67, - CoreTemperatureThreshold: 42, - MemoryTemperatureThreshold: 45, - }, - }, - } - controller := &DRAReconciler{} - - args := controller.generateArgs(cp) - - Expect(args).To(HaveLen(4)) - Expect(args).To(ContainElement("-v=3")) - Expect(args).To(ContainElement("--health-monitoring=true")) - Expect(args).To(ContainElement("--healthcheck-port=-1")) - Expect(args).To(ContainElement("--manage-binding=false")) - }) - - It("with device taint", func() { - cp := &v1alpha.ClusterPolicy{ - Spec: v1alpha.ClusterPolicySpec{ - LogLevel: 2, - DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ - LogLevel: 3, - PodHealthCheck: true, - }, - HealthinessSpec: &v1alpha.HealthinessSpec{ - CheckIntervalSeconds: 67, - CoreTemperatureThreshold: 42, - MemoryTemperatureThreshold: 45, - }, - }, - } - controller := &DRAReconciler{} - - args := controller.generateArgs(cp) - - Expect(args).To(HaveLen(4)) - Expect(args).To(ContainElement("-v=3")) - Expect(args).To(ContainElement("--health-monitoring=true")) - Expect(args).To(ContainElement("--healthcheck-port=51516")) - Expect(args).To(ContainElement("--manage-binding=false")) - }) - - It("with driver bind management", func() { - cp := &v1alpha.ClusterPolicy{ - Spec: v1alpha.ClusterPolicySpec{ - LogLevel: 2, - DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ - LogLevel: 3, - PodHealthCheck: true, - ManageBinding: true, - }, - }, - } - controller := &DRAReconciler{} - - args := controller.generateArgs(cp) - - Expect(args).To(HaveLen(3)) - Expect(args).To(ContainElement("-v=3")) - Expect(args).To(ContainElement("--healthcheck-port=51516")) - Expect(args).To(ContainElement("--manage-binding=true")) - }) - }) -}) diff --git a/internal/controller/kmm_controller.go b/internal/controller/kmm_controller.go new file mode 100644 index 0000000..44bd8cf --- /dev/null +++ b/internal/controller/kmm_controller.go @@ -0,0 +1,467 @@ +/* +Copyright 2026 Intel Corporation. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "fmt" + "strconv" + "strings" + + v1 "k8s.io/api/core/v1" + resourcev1 "k8s.io/api/resource/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + v1alpha "github.com/intel/gpu-base-operator/api/v1alpha1" + kmmv1beta1 "github.com/kubernetes-sigs/kernel-module-management/api/v1beta1" +) + +// +kubebuilder:rbac:groups=kmm.sigs.x-k8s.io,resources=modules,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=kmm.sigs.x-k8s.io,resources=modules/status,verbs=get + +// KMMReconciler manages a KMM Module CR that delegates DP/DRA DaemonSet lifecycle +// to the Kernel Module Management operator, replacing direct DaemonSet management. +type KMMReconciler struct { + client.Client + Scheme *runtime.Scheme + Opts ControllerOpts +} + +const ( + kmmModuleSuffix = "-gpu" + + healthCheckPort = 51516 + gpuDeviceClass = "gpu.intel.com" + vfioGpuDeviceClass = "gpu-vfio.intel.com" + + kmmNotEnabledMsg = "KMM is not installed in the cluster." +) + +var hostPathDirOrCreate = v1.HostPathDirectoryOrCreate + +func logLevelForDp(spec *v1alpha.ClusterPolicy) int32 { + return max(spec.Spec.LogLevel, spec.Spec.DevicePluginSpec.LogLevel) +} + +func hexArgStr(s []string) string { + a := make([]string, 0, len(s)) + for _, str := range s { + if !strings.HasPrefix(str, "0x") { + str = "0x" + str + } + + a = append(a, str) + } + return strings.Join(a, ",") +} + +func dpArgs(spec *v1alpha.ClusterPolicy) []string { + var args []string + + dpspec := spec.Spec.DevicePluginSpec + + if spec.Spec.ResourceMonitoring { + args = append(args, + "-enable-monitoring", + "-xpumd-endpoint=/run/xpumd/intelxpuinfo.sock") + } + + logLevel := logLevelForDp(spec) + if logLevel > 0 { + args = append(args, fmt.Sprintf("-v=%d", logLevel)) + } + + if len(dpspec.ByPathMode) > 0 { + args = append(args, fmt.Sprintf("-bypath=%s", dpspec.ByPathMode)) + } + + if len(dpspec.AllowIDs) > 0 { + args = append(args, fmt.Sprintf("-allow-ids=%s", hexArgStr(dpspec.AllowIDs))) + } + + if len(dpspec.DenyIDs) > 0 { + args = append(args, fmt.Sprintf("-deny-ids=%s", hexArgStr(dpspec.DenyIDs))) + } + + return args +} + +func kmmModuleName(cpName string) string { + return cpName + kmmModuleSuffix +} + +func (r *KMMReconciler) Reconcile(ctx context.Context, cp *v1alpha.ClusterPolicy) (ctrl.Result, error) { + moduleName := kmmModuleName(r.Opts.ReqName) + + if !r.Opts.KMMEnable { + if cp != nil { + addIfMissing(&cp.Status.Errors, kmmNotEnabledMsg) + } + + return ctrl.Result{}, nil + } + + if !r.Opts.DRAEnable && cp != nil && cp.Spec.ResourceRegistration == resourceModeDRA { + addIfMissing(&cp.Status.Errors, "DRA is not enabled in the cluster, but ClusterPolicy requests it.") + return ctrl.Result{}, nil + } + + if cp == nil || !cp.DeletionTimestamp.IsZero() { + return r.deleteModuleIfExists(ctx, moduleName) + } + + mod := &kmmv1beta1.Module{ + ObjectMeta: metav1.ObjectMeta{ + Name: moduleName, + Namespace: r.Opts.Namespace, + }, + } + + result, err := controllerutil.CreateOrPatch(ctx, r.Client, mod, func() error { + return r.setModuleDesiredState(mod, cp) + }) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to reconcile KMM Module %s for ClusterPolicy %s: %w", moduleName, cp.Name, err) + } + + klog.Infof("KMM Module %s %s", moduleName, result) + + r.updateStatus(cp, mod) + + return ctrl.Result{}, nil +} + +func (r *KMMReconciler) setModuleDesiredState(mod *kmmv1beta1.Module, cp *v1alpha.ClusterPolicy) error { + if err := ctrl.SetControllerReference(cp, mod, r.Scheme); err != nil { + return fmt.Errorf("failed to set controller reference: %w", err) + } + + mod.Spec.Selector = r.buildNodeSelector(cp) + + mod.Spec.ImageRepoSecret = cp.Spec.PullSecret + mod.Spec.Tolerations = cp.Spec.Tolerations + + r.setModuleLoader(mod, cp) + + switch cp.Spec.ResourceRegistration { + case resourceModeDRA: + r.setDRA(mod, cp) + case resourceModeDP: + r.setDevicePlugin(mod, cp) + } + + return nil +} + +func (r *KMMReconciler) buildNodeSelector(cp *v1alpha.ClusterPolicy) map[string]string { + return gpuNodeSelector(cp) +} + +func (r *KMMReconciler) setModuleLoader(mod *kmmv1beta1.Module, cp *v1alpha.ClusterPolicy) { + if cp.Spec.KernelModule == nil { + mod.Spec.ModuleLoader = nil + return + } + + km := cp.Spec.KernelModule + + container := kmmv1beta1.ModuleLoaderContainerSpec{ + Modprobe: kmmv1beta1.ModprobeSpec{ + ModuleName: km.ModuleName, + FirmwarePath: km.FirmwarePath, + ModulesLoadingOrder: km.ModulesLoadingOrder, + }, + ContainerImage: km.Image, + InTreeModulesToRemove: dedupeStrings(append([]string{km.ModuleName}, km.InTreeModulesToRemove...)), + ImagePullPolicy: v1.PullAlways, + RegistryTLS: kmmv1beta1.TLSOptions{ + InsecureSkipTLSVerify: km.SkipTLSVerify, + }, + } + + if len(km.KernelMappings) == 0 { + container.KernelMappings = []kmmv1beta1.KernelMapping{ + {Regexp: "^.+$"}, + } + } else { + mappings := make([]kmmv1beta1.KernelMapping, 0, len(km.KernelMappings)) + for _, m := range km.KernelMappings { + mapping := kmmv1beta1.KernelMapping{ + Regexp: m.Regexp, + Literal: m.Literal, + ContainerImage: m.ContainerImage, + InTreeModulesToRemove: m.InTreeModulesToRemove, + } + if m.Build != nil { + mapping.Build = convertBuildSpec(m.Build) + } + mappings = append(mappings, mapping) + } + container.KernelMappings = mappings + } + + mod.Spec.ModuleLoader = &kmmv1beta1.ModuleLoaderSpec{ + Container: container, + ServiceAccountName: r.Opts.ModuleLoaderServiceAccountName, + } +} + +func convertBuildSpec(src *v1alpha.KernelModuleBuildSpec) *kmmv1beta1.Build { + build := &kmmv1beta1.Build{ + DockerfileConfigMap: &v1.LocalObjectReference{Name: src.DockerfileConfigMap.Name}, + Secrets: src.Secrets, + } + + if len(src.BuildArgs) > 0 { + args := make([]kmmv1beta1.BuildArg, len(src.BuildArgs)) + for i, a := range src.BuildArgs { + args[i] = kmmv1beta1.BuildArg{Name: a.Name, Value: a.Value} + } + build.BuildArgs = args + } + + return build +} + +func dedupeStrings(s []string) []string { + seen := make(map[string]bool, len(s)) + result := make([]string, 0, len(s)) + + for _, v := range s { + if !seen[v] { + seen[v] = true + result = append(result, v) + } + } + + return result +} + +func (r *KMMReconciler) setDRA(mod *kmmv1beta1.Module, cp *v1alpha.ClusterPolicy) { + mod.Spec.DevicePlugin = nil + + celExpression := fmt.Sprintf("device.driver == %q", gpuDeviceClass) + + args := r.generateDRAArgs(cp) + + // KMM's DRA reconciler automatically adds: kubelet-plugins, kubelet-plugins-registry, + // cdi (/var/run/cdi) volumes/mounts and NODE_NAME, CDI_ROOT, POD_UID env vars. + // We only include what KMM doesn't provide. + mod.Spec.DRA = &kmmv1beta1.DRASpec{ + DriverName: gpuDeviceClass, + ServiceAccountName: r.Opts.DRAServiceAccountName, + Container: kmmv1beta1.CommonContainerSpec{ + Image: cp.Spec.DynamicResourceAllocationSpec.Image, + ImagePullPolicy: v1.PullIfNotPresent, + Command: []string{"/kubelet-gpu-plugin"}, + Args: args, + Env: []v1.EnvVar{ + { + Name: "POD_NAMESPACE", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, + }, + }, + { + Name: "SYSFS_ROOT", + Value: "/sysfs", + }, + }, + VolumeMounts: []v1.VolumeMount{ + {Name: "xpumdrundir", MountPath: "/var/run/xpumd"}, + {Name: "sysfs", MountPath: "/sysfs"}, + }, + }, + Volumes: draExtraVolumes(), + DeviceClasses: []kmmv1beta1.DeviceClassSpec{ + { + Name: gpuDeviceClass, + Selectors: []resourcev1.DeviceSelector{ + {CEL: &resourcev1.CELDeviceSelector{Expression: celExpression}}, + }, + }, + { + Name: vfioGpuDeviceClass, + Selectors: []resourcev1.DeviceSelector{ + {CEL: &resourcev1.CELDeviceSelector{Expression: celExpression}}, + }, + }, + }, + } +} + +func (r *KMMReconciler) generateDRAArgs(cp *v1alpha.ClusterPolicy) []string { + targetLevel := max(cp.Spec.DynamicResourceAllocationSpec.LogLevel, cp.Spec.LogLevel) + + args := []string{fmt.Sprintf("-v=%d", targetLevel)} + + if cp.Spec.HealthinessSpec != nil { + args = append(args, "--health-monitoring=true") + } + + if cp.Spec.DynamicResourceAllocationSpec.PodHealthCheck { + args = append(args, fmt.Sprintf("--healthcheck-port=%d", healthCheckPort)) + } else { + args = append(args, "--healthcheck-port=-1") + } + + args = append(args, fmt.Sprintf("--manage-binding=%s", strconv.FormatBool(cp.Spec.DynamicResourceAllocationSpec.ManageBinding))) + + return args +} + +// draExtraVolumes returns only the volumes KMM's DRA reconciler doesn't add automatically. +// KMM auto-adds: kubelet-plugins, kubelet-plugins-registry, cdi (/var/run/cdi). +func draExtraVolumes() []v1.Volume { + return []v1.Volume{ + {Name: "sysfs", VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/sys"}}}, + {Name: "xpumdrundir", VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/var/run/xpumd", Type: &hostPathDirOrCreate}}}, + } +} + +func (r *KMMReconciler) setDevicePlugin(mod *kmmv1beta1.Module, cp *v1alpha.ClusterPolicy) { + mod.Spec.DRA = nil + + mod.Spec.DevicePlugin = &kmmv1beta1.DevicePluginSpec{ + ServiceAccountName: r.Opts.DPServiceAccountName, + Container: kmmv1beta1.CommonContainerSpec{ + Image: cp.Spec.DevicePluginSpec.PluginImage, + ImagePullPolicy: v1.PullIfNotPresent, + Args: dpArgs(cp), + Env: []v1.EnvVar{ + { + Name: "NODE_NAME", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, + }, + }, + { + Name: "HOST_IP", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{FieldPath: "status.hostIP"}, + }, + }, + }, + VolumeMounts: []v1.VolumeMount{ + {Name: "devfs", MountPath: "/dev/dri", ReadOnly: true}, + {Name: "sysfsdrm", MountPath: "/sys/class/drm", ReadOnly: true}, + {Name: "cdipath", MountPath: "/var/run/cdi"}, + }, + }, + Volumes: []v1.Volume{ + {Name: "devfs", VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/dev/dri"}}}, + {Name: "sysfsdrm", VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/sys/class/drm"}}}, + {Name: "cdipath", VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/var/run/cdi", Type: &hostPathDirOrCreate}}}, + }, + } + + if cp.Spec.ResourceMonitoring { + mod.Spec.DevicePlugin.Volumes = append(mod.Spec.DevicePlugin.Volumes, v1.Volume{ + Name: xpumdVolumeName, + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/run/xpumd", + Type: &hostPathDirOrCreate, + }, + }, + }) + mod.Spec.DevicePlugin.Container.VolumeMounts = append(mod.Spec.DevicePlugin.Container.VolumeMounts, v1.VolumeMount{ + Name: xpumdVolumeName, + MountPath: "/run/xpumd", + }) + } +} + +func (r *KMMReconciler) deleteModuleIfExists(ctx context.Context, name string) (ctrl.Result, error) { + mod := &kmmv1beta1.Module{} + key := types.NamespacedName{Name: name, Namespace: r.Opts.Namespace} + + if err := r.Get(ctx, key, mod); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{}, nil + } + + return ctrl.Result{}, fmt.Errorf("failed to get KMM Module %s: %w", name, err) + } + + if mod.Spec.DRA != nil && r.anyAllocatedResourceClaims(ctx, mod.Spec.DRA.DriverName) { + return ctrl.Result{RequeueAfter: r.Opts.RequeueDelay}, requeueReconcileErr{} + } + + klog.Infof("Deleting KMM Module %s", name) + + if err := r.Delete(ctx, mod); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, fmt.Errorf("failed to delete KMM Module %s: %w", name, err) + } + + return ctrl.Result{}, nil +} + +func (r *KMMReconciler) anyAllocatedResourceClaims(ctx context.Context, driverName string) bool { + var rcList resourcev1.ResourceClaimList + + klog.Info("Checking for allocated ResourceClaims that would prevent DRA removal") + + // Fail safe: assume claims exist so we don't delete the Module while pods may still be using GPUs. + if err := r.List(ctx, &rcList); err != nil { + klog.Error(err, "unable to list ResourceClaims, assuming allocated claims exist") + return true + } + + for _, claim := range rcList.Items { + alloc := claim.Status.Allocation + if alloc == nil || len(alloc.Devices.Results) == 0 { + continue + } + + for _, dev := range alloc.Devices.Results { + if dev.Driver == driverName { + klog.Infof("Found allocated ResourceClaim with GPU device: %s", claim.Name) + return true + } + } + } + + return false +} + +func (r *KMMReconciler) updateStatus(cp *v1alpha.ClusterPolicy, mod *kmmv1beta1.Module) { + switch cp.Spec.ResourceRegistration { + case resourceModeDRA: + dsStatus := mod.Status.DRA + cp.Status.DRAStatus = fmt.Sprintf("%d/%d", dsStatus.AvailableNumber, dsStatus.DesiredNumber) + cp.Status.DevicePluginStatus = notAvailableStatus + case resourceModeDP: + dsStatus := mod.Status.DevicePlugin + cp.Status.DevicePluginStatus = fmt.Sprintf("%d/%d", dsStatus.AvailableNumber, dsStatus.DesiredNumber) + cp.Status.DRAStatus = notAvailableStatus + } + + if cp.Spec.KernelModule != nil { + mlStatus := mod.Status.ModuleLoader + cp.Status.KMMStatus = fmt.Sprintf("%d/%d", mlStatus.AvailableNumber, mlStatus.DesiredNumber) + } else { + cp.Status.KMMStatus = notAvailableStatus + } +} diff --git a/internal/controller/kmm_controller_test.go b/internal/controller/kmm_controller_test.go new file mode 100644 index 0000000..bb62ba5 --- /dev/null +++ b/internal/controller/kmm_controller_test.go @@ -0,0 +1,761 @@ +/* +Copyright 2026 Intel Corporation. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + v1alpha "github.com/intel/gpu-base-operator/api/v1alpha1" + kmmv1beta1 "github.com/kubernetes-sigs/kernel-module-management/api/v1beta1" +) + +var _ = Describe("KMM Controller", func() { + Context("When creating a KMM Module in DRA mode", func() { + const ( + namespace = "kmm-dra-create" + resourceName = "kmm-dra-create" + ) + + ctx := context.Background() + typeNamespacedName := types.NamespacedName{Name: resourceName} + + BeforeEach(func() { + Expect(k8sClient.Create(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: namespace}, + })).To(Succeed()) + }) + + AfterEach(func() { + resource := &v1alpha.ClusterPolicy{} + if err := k8sClient.Get(ctx, typeNamespacedName, resource); err == nil { + Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + } + }) + + It("should create a KMM Module CR with DRA spec", func() { + By("creating a ClusterPolicy with DRA") + cp := &v1alpha.ClusterPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: resourceName}, + Spec: v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + UseNFDLabeling: true, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + PodHealthCheck: true, + }, + }, + } + Expect(k8sClient.Create(ctx, cp)).To(Succeed()) + + By("reconciling") + reconciler := &ClusterPolicyReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + Opts: ControllerOpts{ + Namespace: namespace, + DRAEnable: true, + KMMEnable: true, + DRAServiceAccountName: "intel-gpu-dra", + }, + } + + _, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) + Expect(err).NotTo(HaveOccurred()) + + By("verifying KMM Module was created") + mod := &kmmv1beta1.Module{} + modKey := types.NamespacedName{Name: resourceName + kmmModuleSuffix, Namespace: namespace} + Expect(k8sClient.Get(ctx, modKey, mod)).To(Succeed()) + + By("verifying selector includes NFD label and arch") + Expect(mod.Spec.Selector).To(HaveKeyWithValue("intel.feature.node.kubernetes.io/gpu", "true")) + Expect(mod.Spec.Selector).To(HaveKeyWithValue("kubernetes.io/arch", "amd64")) + + By("verifying ModuleLoader is nil for in-tree driver") + Expect(mod.Spec.ModuleLoader).To(BeNil()) + + By("verifying DRA spec is set") + Expect(mod.Spec.DRA).NotTo(BeNil()) + Expect(mod.Spec.DRA.DriverName).To(Equal("gpu.intel.com")) + Expect(mod.Spec.DRA.Container.Image).To(Equal("ghcr.io/intel/gpu-dra:v0.11.0")) + Expect(mod.Spec.DRA.Container.Command).To(Equal([]string{"/kubelet-gpu-plugin"})) + + By("verifying DevicePlugin is nil") + Expect(mod.Spec.DevicePlugin).To(BeNil()) + + By("verifying DRA device classes") + Expect(mod.Spec.DRA.DeviceClasses).To(HaveLen(2)) + Expect(mod.Spec.DRA.DeviceClasses[0].Name).To(Equal("gpu.intel.com")) + Expect(mod.Spec.DRA.DeviceClasses[1].Name).To(Equal("gpu-vfio.intel.com")) + + By("verifying DRA extra volumes (KMM adds plugins/registry/cdi automatically)") + Expect(mod.Spec.DRA.Volumes).To(HaveLen(2)) + + By("verifying owner reference is set") + Expect(mod.OwnerReferences).To(HaveLen(1)) + Expect(mod.OwnerReferences[0].Name).To(Equal(resourceName)) + }) + }) + + Context("When updating a KMM Module after ClusterPolicy changes", func() { + const ( + namespace = "kmm-dra-update" + resourceName = "kmm-dra-update" + ) + + ctx := context.Background() + typeNamespacedName := types.NamespacedName{Name: resourceName} + + BeforeEach(func() { + Expect(k8sClient.Create(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: namespace}, + })).To(Succeed()) + }) + + AfterEach(func() { + resource := &v1alpha.ClusterPolicy{} + if err := k8sClient.Get(ctx, typeNamespacedName, resource); err == nil { + Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + } + }) + + It("should update the Module selector", func() { + cp := &v1alpha.ClusterPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: resourceName}, + Spec: v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + PodHealthCheck: true, + }, + }, + } + Expect(k8sClient.Create(ctx, cp)).To(Succeed()) + + reconciler := &ClusterPolicyReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + Opts: ControllerOpts{ + Namespace: namespace, + DRAEnable: true, + KMMEnable: true, + DRAServiceAccountName: "intel-gpu-dra", + }, + } + + _, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) + Expect(err).NotTo(HaveOccurred()) + + By("updating the ClusterPolicy to add NodeSelector") + Expect(k8sClient.Get(ctx, typeNamespacedName, cp)).To(Succeed()) + cp.Spec.NodeSelector = map[string]string{"zone": "gpu-pool"} + Expect(k8sClient.Update(ctx, cp)).To(Succeed()) + + _, err = reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) + Expect(err).NotTo(HaveOccurred()) + + By("verifying Module selector was updated") + mod := &kmmv1beta1.Module{} + modKey := types.NamespacedName{Name: resourceName + kmmModuleSuffix, Namespace: namespace} + Expect(k8sClient.Get(ctx, modKey, mod)).To(Succeed()) + Expect(mod.Spec.Selector).To(HaveKeyWithValue("zone", "gpu-pool")) + }) + }) + + Context("When creating a KMM Module in DP mode", func() { + const ( + namespace = "kmm-dp-create" + resourceName = "kmm-dp-create" + ) + + ctx := context.Background() + typeNamespacedName := types.NamespacedName{Name: resourceName} + + BeforeEach(func() { + Expect(k8sClient.Create(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: namespace}, + })).To(Succeed()) + }) + + AfterEach(func() { + resource := &v1alpha.ClusterPolicy{} + if err := k8sClient.Get(ctx, typeNamespacedName, resource); err == nil { + Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + } + }) + + It("should create a KMM Module CR with DevicePlugin spec", func() { + cp := &v1alpha.ClusterPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: resourceName}, + Spec: v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dp", + ResourceMonitoring: true, + DevicePluginSpec: v1alpha.DevicePluginSpec{ + PluginImage: "intel/intel-gpu-plugin:0.36.0", + }, + XpuManagerSpec: v1alpha.XpuManagerSpec{ + Image: "intel/xpumanager:v2.0.0", + MonitoringResource: "monitoring", + }, + }, + } + Expect(k8sClient.Create(ctx, cp)).To(Succeed()) + + reconciler := &ClusterPolicyReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + Opts: ControllerOpts{ + Namespace: namespace, + DRAEnable: true, + KMMEnable: true, + DRAServiceAccountName: "intel-gpu-dra", + }, + } + + _, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) + Expect(err).NotTo(HaveOccurred()) + + By("verifying KMM Module was created with DevicePlugin spec") + mod := &kmmv1beta1.Module{} + modKey := types.NamespacedName{Name: resourceName + kmmModuleSuffix, Namespace: namespace} + Expect(k8sClient.Get(ctx, modKey, mod)).To(Succeed()) + + Expect(mod.Spec.DevicePlugin).NotTo(BeNil()) + Expect(mod.Spec.DevicePlugin.Container.Image).To(Equal("intel/intel-gpu-plugin:0.36.0")) + Expect(mod.Spec.DRA).To(BeNil()) + + By("verifying DP volumes include device paths") + volNames := []string{} + for _, vol := range mod.Spec.DevicePlugin.Volumes { + volNames = append(volNames, vol.Name) + } + Expect(volNames).To(ContainElement("devfs")) + Expect(volNames).To(ContainElement("sysfsdrm")) + + By("verifying xpumd volume is present when monitoring is enabled") + Expect(volNames).To(ContainElement(xpumdVolumeName)) + }) + }) + + Context("When configuring an OOT driver via KernelModule", func() { + const ( + namespace = "kmm-oot-create" + resourceName = "kmm-oot-create" + ) + + ctx := context.Background() + typeNamespacedName := types.NamespacedName{Name: resourceName} + + BeforeEach(func() { + Expect(k8sClient.Create(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: namespace}, + })).To(Succeed()) + }) + + AfterEach(func() { + resource := &v1alpha.ClusterPolicy{} + if err := k8sClient.Get(ctx, typeNamespacedName, resource); err == nil { + Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + } + }) + + It("should configure ModuleLoader for OOT driver", func() { + cp := &v1alpha.ClusterPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: resourceName}, + Spec: v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + Image: "registry.example.com/xe-driver:1.0", + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + } + Expect(k8sClient.Create(ctx, cp)).To(Succeed()) + + reconciler := &ClusterPolicyReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + Opts: ControllerOpts{ + Namespace: namespace, + DRAEnable: true, + KMMEnable: true, + DRAServiceAccountName: "intel-gpu-dra", + }, + } + + _, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) + Expect(err).NotTo(HaveOccurred()) + + mod := &kmmv1beta1.Module{} + modKey := types.NamespacedName{Name: resourceName + kmmModuleSuffix, Namespace: namespace} + Expect(k8sClient.Get(ctx, modKey, mod)).To(Succeed()) + + Expect(mod.Spec.ModuleLoader).NotTo(BeNil()) + Expect(mod.Spec.ModuleLoader.Container.Modprobe.ModuleName).To(Equal("xe")) + Expect(mod.Spec.ModuleLoader.Container.ContainerImage).To(Equal("registry.example.com/xe-driver:1.0")) + Expect(mod.Spec.ModuleLoader.Container.InTreeModulesToRemove).To(ContainElement("xe")) + Expect(mod.Spec.ModuleLoader.Container.KernelMappings).To(HaveLen(1)) + Expect(mod.Spec.ModuleLoader.Container.KernelMappings[0].Regexp).To(Equal("^.+$")) + }) + }) + + DescribeTable("expanded KernelModule fields", func( + ns string, + cpSpec v1alpha.ClusterPolicySpec, + assertFn func(mod *kmmv1beta1.Module, cp *v1alpha.ClusterPolicy), + ) { + ctx := context.Background() + resName := ns + + Expect(k8sClient.Create(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: ns}, + })).To(Succeed()) + + cp := &v1alpha.ClusterPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: resName}, + Spec: cpSpec, + } + Expect(k8sClient.Create(ctx, cp)).To(Succeed()) + + reconciler := &ClusterPolicyReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + Opts: ControllerOpts{ + Namespace: ns, + DRAEnable: true, + KMMEnable: true, + DRAServiceAccountName: "intel-gpu-dra", + }, + } + _, err := reconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{Name: resName}, + }) + Expect(err).NotTo(HaveOccurred()) + + mod := &kmmv1beta1.Module{} + modKey := types.NamespacedName{Name: resName + kmmModuleSuffix, Namespace: ns} + Expect(k8sClient.Get(ctx, modKey, mod)).To(Succeed()) + + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: resName}, cp)).To(Succeed()) + + assertFn(mod, cp) + }, + Entry("should configure ModuleLoader with multiple kernel mappings", + "kmm-exp-multi", + v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []v1alpha.KernelMappingSpec{ + {Regexp: "^5\\.14\\.0-.*\\.el9", ContainerImage: "registry.example.com/xe-rhel9:1.0"}, + {Regexp: "^6\\.12\\..*", ContainerImage: "registry.example.com/xe-rhel10:1.0"}, + }, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + func(mod *kmmv1beta1.Module, _ *v1alpha.ClusterPolicy) { + Expect(mod.Spec.ModuleLoader.Container.KernelMappings).To(HaveLen(2)) + Expect(mod.Spec.ModuleLoader.Container.KernelMappings[0].Regexp).To(Equal("^5\\.14\\.0-.*\\.el9")) + Expect(mod.Spec.ModuleLoader.Container.KernelMappings[0].ContainerImage).To(Equal("registry.example.com/xe-rhel9:1.0")) + Expect(mod.Spec.ModuleLoader.Container.KernelMappings[1].Regexp).To(Equal("^6\\.12\\..*")) + Expect(mod.Spec.ModuleLoader.Container.KernelMappings[1].ContainerImage).To(Equal("registry.example.com/xe-rhel10:1.0")) + Expect(mod.Spec.ModuleLoader.Container.ContainerImage).To(BeEmpty()) + }, + ), + Entry("should use Image as fallback when mapping omits ContainerImage", + "kmm-exp-fallback", + v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + Image: "registry.example.com/xe-driver:1.0", + KernelMappings: []v1alpha.KernelMappingSpec{ + {Regexp: "^5\\.14\\..*"}, + }, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + func(mod *kmmv1beta1.Module, _ *v1alpha.ClusterPolicy) { + Expect(mod.Spec.ModuleLoader.Container.ContainerImage).To(Equal("registry.example.com/xe-driver:1.0")) + Expect(mod.Spec.ModuleLoader.Container.KernelMappings[0].ContainerImage).To(BeEmpty()) + }, + ), + Entry("should configure Build on kernel mapping", + "kmm-exp-build", + v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []v1alpha.KernelMappingSpec{ + { + Regexp: "^5\\.14\\..*", + Build: &v1alpha.KernelModuleBuildSpec{ + DockerfileConfigMap: v1.LocalObjectReference{Name: "xe-dockerfile"}, + BuildArgs: []v1alpha.BuildArg{{Name: "XE_TAG", Value: "v1.0"}}, + Secrets: []v1.LocalObjectReference{{Name: "private-repo"}}, + }, + }, + }, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + func(mod *kmmv1beta1.Module, _ *v1alpha.ClusterPolicy) { + kmmBuild := mod.Spec.ModuleLoader.Container.KernelMappings[0].Build + Expect(kmmBuild).NotTo(BeNil()) + Expect(kmmBuild.DockerfileConfigMap).NotTo(BeNil()) + Expect(kmmBuild.DockerfileConfigMap.Name).To(Equal("xe-dockerfile")) + Expect(kmmBuild.BuildArgs).To(HaveLen(1)) + Expect(kmmBuild.BuildArgs[0].Name).To(Equal("XE_TAG")) + Expect(kmmBuild.BuildArgs[0].Value).To(Equal("v1.0")) + Expect(kmmBuild.Secrets).To(HaveLen(1)) + Expect(kmmBuild.Secrets[0].Name).To(Equal("private-repo")) + }, + ), + Entry("should propagate FirmwarePath to ModprobeSpec", + "kmm-exp-firmware", + v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + Image: "registry.example.com/xe-driver:1.0", + FirmwarePath: "/opt/lib/firmware/xe", + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + func(mod *kmmv1beta1.Module, _ *v1alpha.ClusterPolicy) { + Expect(mod.Spec.ModuleLoader.Container.Modprobe.FirmwarePath).To(Equal("/opt/lib/firmware/xe")) + }, + ), + Entry("should propagate ModulesLoadingOrder to ModprobeSpec", + "kmm-exp-loadorder", + v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + Image: "registry.example.com/xe-driver:1.0", + ModulesLoadingOrder: []string{"xe", "drm_buddy", "drm_ttm_helper"}, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + func(mod *kmmv1beta1.Module, _ *v1alpha.ClusterPolicy) { + Expect(mod.Spec.ModuleLoader.Container.Modprobe.ModulesLoadingOrder).To(Equal([]string{"xe", "drm_buddy", "drm_ttm_helper"})) + }, + ), + Entry("should set SkipTLSVerify on RegistryTLS", + "kmm-exp-tls", + v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + Image: "registry.example.com/xe-driver:1.0", + SkipTLSVerify: true, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + func(mod *kmmv1beta1.Module, _ *v1alpha.ClusterPolicy) { + Expect(mod.Spec.ModuleLoader.Container.RegistryTLS.InsecureSkipTLSVerify).To(BeTrue()) + }, + ), + Entry("should merge InTreeModulesToRemove with ModuleName", + "kmm-exp-intree", + v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + Image: "registry.example.com/xe-driver:1.0", + InTreeModulesToRemove: []string{"i915"}, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + func(mod *kmmv1beta1.Module, _ *v1alpha.ClusterPolicy) { + Expect(mod.Spec.ModuleLoader.Container.InTreeModulesToRemove).To(ContainElement("xe")) + Expect(mod.Spec.ModuleLoader.Container.InTreeModulesToRemove).To(ContainElement("i915")) + }, + ), + Entry("should deduplicate InTreeModulesToRemove", + "kmm-exp-dedup", + v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + Image: "registry.example.com/xe-driver:1.0", + InTreeModulesToRemove: []string{"xe", "i915"}, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + func(mod *kmmv1beta1.Module, _ *v1alpha.ClusterPolicy) { + xeCount := 0 + for _, m := range mod.Spec.ModuleLoader.Container.InTreeModulesToRemove { + if m == "xe" { + xeCount++ + } + } + Expect(xeCount).To(Equal(1)) + Expect(mod.Spec.ModuleLoader.Container.InTreeModulesToRemove).To(ContainElement("i915")) + }, + ), + Entry("should set per-mapping InTreeModulesToRemove override", + "kmm-exp-permapping", + v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []v1alpha.KernelMappingSpec{ + { + Regexp: "^5\\.14\\..*", + ContainerImage: "registry.example.com/xe:1.0", + InTreeModulesToRemove: []string{"old_xe"}, + }, + }, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + func(mod *kmmv1beta1.Module, _ *v1alpha.ClusterPolicy) { + Expect(mod.Spec.ModuleLoader.Container.KernelMappings[0].InTreeModulesToRemove).To(Equal([]string{"old_xe"})) + Expect(mod.Spec.ModuleLoader.Container.InTreeModulesToRemove).To(ContainElement("xe")) + }, + ), + Entry("should set KMMStatus to N/A when KernelModule is nil", + "kmm-exp-nkmm", + v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + func(_ *kmmv1beta1.Module, cp *v1alpha.ClusterPolicy) { + Expect(cp.Status.KMMStatus).To(Equal("N/A")) + }, + ), + ) + + Context("When deleting a KMM Module", func() { + const ( + namespace = "kmm-del-create" + resourceName = "kmm-del-create" + ) + + ctx := context.Background() + typeNamespacedName := types.NamespacedName{Name: resourceName} + + BeforeEach(func() { + Expect(k8sClient.Create(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: namespace}, + })).To(Succeed()) + }) + + AfterEach(func() { + resource := &v1alpha.ClusterPolicy{} + if err := k8sClient.Get(ctx, typeNamespacedName, resource); err == nil { + Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + } + }) + + It("should delete Module when ClusterPolicy is deleted", func() { + cp := &v1alpha.ClusterPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: resourceName}, + Spec: v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + } + Expect(k8sClient.Create(ctx, cp)).To(Succeed()) + + reconciler := &ClusterPolicyReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + Opts: ControllerOpts{ + Namespace: namespace, + DRAEnable: true, + KMMEnable: true, + DRAServiceAccountName: "intel-gpu-dra", + }, + } + + _, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) + Expect(err).NotTo(HaveOccurred()) + + modKey := types.NamespacedName{Name: resourceName + kmmModuleSuffix, Namespace: namespace} + Expect(k8sClient.Get(ctx, modKey, &kmmv1beta1.Module{})).To(Succeed()) + + By("deleting the ClusterPolicy") + Expect(k8sClient.Delete(ctx, cp)).To(Succeed()) + + By("reconciling the deletion via KMM reconciler directly") + kmmReconciler := &KMMReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + Opts: ControllerOpts{ + ReqName: resourceName, + Namespace: namespace, + KMMEnable: true, + }, + } + _, err = kmmReconciler.Reconcile(ctx, nil) + Expect(err).NotTo(HaveOccurred()) + + By("verifying Module was deleted") + err = k8sClient.Get(ctx, modKey, &kmmv1beta1.Module{}) + Expect(errors.IsNotFound(err)).To(BeTrue()) + }) + }) + + Context("When ClusterPolicy has tolerations and pull secret", func() { + const ( + namespace = "kmm-opts-create" + resourceName = "kmm-opts-create" + ) + + ctx := context.Background() + typeNamespacedName := types.NamespacedName{Name: resourceName} + + BeforeEach(func() { + Expect(k8sClient.Create(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: namespace}, + })).To(Succeed()) + }) + + AfterEach(func() { + resource := &v1alpha.ClusterPolicy{} + if err := k8sClient.Get(ctx, typeNamespacedName, resource); err == nil { + Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + } + }) + + It("should propagate tolerations and pull secret to Module", func() { + cp := &v1alpha.ClusterPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: resourceName}, + Spec: v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + Tolerations: []v1.Toleration{ + { + Key: "gpu-dedicated", + Operator: v1.TolerationOpExists, + Effect: v1.TaintEffectNoSchedule, + }, + }, + PullSecret: &v1.LocalObjectReference{ + Name: "my-registry-secret", + }, + }, + } + Expect(k8sClient.Create(ctx, cp)).To(Succeed()) + + reconciler := &ClusterPolicyReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + Opts: ControllerOpts{ + Namespace: namespace, + DRAEnable: true, + KMMEnable: true, + DRAServiceAccountName: "intel-gpu-dra", + }, + } + + _, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) + Expect(err).NotTo(HaveOccurred()) + + mod := &kmmv1beta1.Module{} + modKey := types.NamespacedName{Name: resourceName + kmmModuleSuffix, Namespace: namespace} + Expect(k8sClient.Get(ctx, modKey, mod)).To(Succeed()) + + By("verifying tolerations") + Expect(mod.Spec.Tolerations).To(HaveLen(1)) + Expect(mod.Spec.Tolerations[0].Key).To(Equal("gpu-dedicated")) + + By("verifying pull secret") + Expect(mod.Spec.ImageRepoSecret).NotTo(BeNil()) + Expect(mod.Spec.ImageRepoSecret.Name).To(Equal("my-registry-secret")) + }) + }) +}) + +var _ = Describe("KMM DRA args generation", func() { + It("generates correct args with all options", func() { + cp := &v1alpha.ClusterPolicy{ + Spec: v1alpha.ClusterPolicySpec{ + LogLevel: 2, + HealthinessSpec: &v1alpha.HealthinessSpec{ + CheckIntervalSeconds: 5, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + LogLevel: 3, + PodHealthCheck: true, + ManageBinding: true, + }, + }, + } + + r := &KMMReconciler{} + args := r.generateDRAArgs(cp) + + Expect(args).To(ContainElement("-v=3")) + Expect(args).To(ContainElement("--health-monitoring=true")) + Expect(args).To(ContainElement("--healthcheck-port=51516")) + Expect(args).To(ContainElement("--manage-binding=true")) + }) + + It("disables health check port when PodHealthCheck is false", func() { + cp := &v1alpha.ClusterPolicy{ + Spec: v1alpha.ClusterPolicySpec{ + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + PodHealthCheck: false, + }, + }, + } + + r := &KMMReconciler{} + args := r.generateDRAArgs(cp) + + Expect(args).To(ContainElement("--healthcheck-port=-1")) + Expect(args).To(ContainElement("--manage-binding=false")) + }) +}) diff --git a/internal/controller/openshift.go b/internal/controller/openshift.go index 74118ed..55879d0 100644 --- a/internal/controller/openshift.go +++ b/internal/controller/openshift.go @@ -48,31 +48,6 @@ func buildSCC(name string, spec map[string]interface{}) *unstructured.Unstructur return scc } -// buildDevicePluginSCC returns the SCC for the GPU device plugin daemonset. -func buildDevicePluginSCC(name string) *unstructured.Unstructured { - return buildSCC(name, map[string]interface{}{ - "allowPrivilegedContainer": false, - "allowHostDirVolumePlugin": true, - "allowHostIPC": false, - "allowHostNetwork": false, - "allowHostPID": false, - "allowHostPorts": false, - "allowPrivilegeEscalation": false, - "allowedCapabilities": nil, - "defaultAddCapabilities": nil, - "fsGroup": map[string]interface{}{"type": "RunAsAny"}, - "readOnlyRootFilesystem": false, - "requiredDropCapabilities": []interface{}{"ALL"}, - "runAsUser": map[string]interface{}{"type": "RunAsAny"}, - "seLinuxContext": map[string]interface{}{"type": "RunAsAny"}, - "seccompProfiles": []interface{}{"*"}, - "supplementalGroups": map[string]interface{}{"type": "RunAsAny"}, - "volumes": []interface{}{"hostPath", "emptyDir"}, - "users": []interface{}{}, - "groups": []interface{}{}, - }) -} - // buildXpuManagerSCC returns the SCC for the XPU Manager daemonset. // The daemonset runs as root and requires SYS_ADMIN capability. func buildXpuManagerSCC(name string) *unstructured.Unstructured { @@ -99,32 +74,6 @@ func buildXpuManagerSCC(name string) *unstructured.Unstructured { }) } -// buildDRASCC returns the SCC for the GPU DRA driver daemonset. -// The daemonset uses hostPath volumes and mounts a service account token (projected volume). -func buildDRASCC(name string) *unstructured.Unstructured { - return buildSCC(name, map[string]interface{}{ - "allowPrivilegedContainer": false, - "allowHostDirVolumePlugin": true, - "allowHostIPC": false, - "allowHostNetwork": false, - "allowHostPID": false, - "allowHostPorts": false, - "allowPrivilegeEscalation": false, - "allowedCapabilities": nil, - "defaultAddCapabilities": nil, - "fsGroup": map[string]interface{}{"type": "RunAsAny"}, - "readOnlyRootFilesystem": false, - "requiredDropCapabilities": []interface{}{"ALL"}, - "runAsUser": map[string]interface{}{"type": "RunAsAny"}, - "seLinuxContext": map[string]interface{}{"type": "RunAsAny"}, - "seccompProfiles": []interface{}{"*"}, - "supplementalGroups": map[string]interface{}{"type": "RunAsAny"}, - "volumes": []interface{}{"hostPath", "projected"}, - "users": []interface{}{}, - "groups": []interface{}{}, - }) -} - // buildFWUpdateSCC returns the SCC for the GPU firmware update Job pods. // The updater container runs privileged as root to access GPU firmware interfaces. func buildFWUpdateSCC(name string) *unstructured.Unstructured { diff --git a/internal/controller/openshift_test.go b/internal/controller/openshift_test.go index 24b85f8..c5ff107 100644 --- a/internal/controller/openshift_test.go +++ b/internal/controller/openshift_test.go @@ -34,26 +34,6 @@ var _ = Describe("OpenShift SCC helpers", func() { ctx := context.Background() Context("SCC builder functions", func() { - It("buildDevicePluginSCC sets correct fields", func() { - scc := buildDevicePluginSCC("dp-builder-test") - - Expect(scc.GetName()).To(Equal("dp-builder-test")) - Expect(scc.GetKind()).To(Equal("SecurityContextConstraints")) - Expect(scc.GetAPIVersion()).To(Equal("security.openshift.io/v1")) - Expect(scc.Object["allowPrivilegedContainer"]).To(BeFalse()) - Expect(scc.Object["allowPrivilegeEscalation"]).To(BeFalse()) - Expect(scc.Object["allowHostDirVolumePlugin"]).To(BeTrue()) - Expect(scc.Object["allowHostNetwork"]).To(BeFalse()) - - drops, ok := scc.Object["requiredDropCapabilities"].([]interface{}) - Expect(ok).To(BeTrue()) - Expect(drops).To(ContainElement("ALL")) - - vols, ok := scc.Object["volumes"].([]interface{}) - Expect(ok).To(BeTrue()) - Expect(vols).To(ContainElements("hostPath", "emptyDir")) - }) - It("buildXpuManagerSCC sets correct fields", func() { scc := buildXpuManagerSCC("xpum-builder-test") @@ -73,22 +53,6 @@ var _ = Describe("OpenShift SCC helpers", func() { Expect(vols).To(ContainElements("hostPath", "configMap")) }) - It("buildDRASCC sets correct fields", func() { - scc := buildDRASCC("dra-builder-test") - - Expect(scc.GetName()).To(Equal("dra-builder-test")) - Expect(scc.Object["allowPrivilegedContainer"]).To(BeFalse()) - Expect(scc.Object["allowPrivilegeEscalation"]).To(BeFalse()) - - drops, ok := scc.Object["requiredDropCapabilities"].([]interface{}) - Expect(ok).To(BeTrue()) - Expect(drops).To(ContainElement("ALL")) - - vols, ok := scc.Object["volumes"].([]interface{}) - Expect(ok).To(BeTrue()) - Expect(vols).To(ContainElements("hostPath", "projected")) - }) - It("buildFWUpdateSCC sets correct fields", func() { scc := buildFWUpdateSCC("fwupdate-builder-test") @@ -109,7 +73,7 @@ var _ = Describe("OpenShift SCC helpers", func() { Context("ensureSCC", func() { It("creates SCC when not found and is idempotent", func() { name := "test-ensure-scc-create" - scc := buildDevicePluginSCC(name) + scc := buildXpuManagerSCC(name) By("first call creates the SCC") Expect(ensureSCC(ctx, k8sClient, scc)).To(Succeed()) @@ -120,7 +84,7 @@ var _ = Describe("OpenShift SCC helpers", func() { Expect(k8sClient.Get(ctx, client.ObjectKey{Name: name}, existing)).To(Succeed()) By("second call is idempotent") - Expect(ensureSCC(ctx, k8sClient, buildDevicePluginSCC(name))).To(Succeed()) + Expect(ensureSCC(ctx, k8sClient, buildXpuManagerSCC(name))).To(Succeed()) DeferCleanup(func() { scc := &unstructured.Unstructured{} @@ -212,7 +176,7 @@ var _ = Describe("OpenShift SCC helpers", func() { saName := "test-delete-sa" By("creating the resources") - Expect(ensureSCC(ctx, k8sClient, buildDevicePluginSCC(sccName))).To(Succeed()) + Expect(ensureSCC(ctx, k8sClient, buildXpuManagerSCC(sccName))).To(Succeed()) Expect(createSCCRole(ctx, k8sClient, roleName, sccName)).To(Succeed()) Expect(createSCCRoleBinding(ctx, k8sClient, bindingName, roleName, saName, deleteNs)).To(Succeed()) Expect(createServiceAccount(ctx, k8sClient, saName, deleteNs)).To(Succeed()) @@ -243,7 +207,7 @@ var _ = Describe("OpenShift SCC helpers", func() { saName := "test-skip-sa" By("creating the resources") - Expect(ensureSCC(ctx, k8sClient, buildDevicePluginSCC(sccName))).To(Succeed()) + Expect(ensureSCC(ctx, k8sClient, buildXpuManagerSCC(sccName))).To(Succeed()) Expect(createSCCRole(ctx, k8sClient, roleName, sccName)).To(Succeed()) Expect(createSCCRoleBinding(ctx, k8sClient, bindingName, roleName, saName, deleteNs)).To(Succeed()) Expect(createServiceAccount(ctx, k8sClient, saName, deleteNs)).To(Succeed()) diff --git a/internal/controller/suite_test.go b/internal/controller/suite_test.go index 22daa3c..9bf4bc6 100644 --- a/internal/controller/suite_test.go +++ b/internal/controller/suite_test.go @@ -18,8 +18,11 @@ package controller import ( "context" + "fmt" "os" + "os/exec" "path/filepath" + "strings" "testing" . "github.com/onsi/ginkgo/v2" @@ -32,6 +35,7 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" + kmmv1beta1 "github.com/kubernetes-sigs/kernel-module-management/api/v1beta1" prometheusv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" corev1 "k8s.io/api/core/v1" resv1 "k8s.io/api/resource/v1" @@ -79,6 +83,8 @@ var _ = BeforeSuite(func() { Expect(err).NotTo(HaveOccurred()) err = corev1.AddToScheme(s) Expect(err).NotTo(HaveOccurred()) + err = kmmv1beta1.AddToScheme(s) + Expect(err).NotTo(HaveOccurred()) // +kubebuilder:scaffold:scheme @@ -88,6 +94,7 @@ var _ = BeforeSuite(func() { CRDDirectoryPaths: []string{ filepath.Join("..", "..", "config", "crd", "bases"), filepath.Join("..", "..", "config", "deployments", "nfd", "crds"), + filepath.Join(goModDir("github.com/kubernetes-sigs/kernel-module-management"), "config", "crd", "bases"), "testdata", }, ErrorIfCRDPathMissing: true, @@ -127,6 +134,14 @@ var _ = AfterSuite(func() { // This function streamlines the process by finding the required binaries, similar to // setting the 'KUBEBUILDER_ASSETS' environment variable. To ensure the binaries are // properly set up, run 'make setup-envtest' beforehand. +func goModDir(module string) string { + out, err := exec.Command("go", "list", "-m", "-f", "{{.Dir}}", module).Output() + if err != nil { + panic(fmt.Sprintf("failed to resolve module directory for %s: %v", module, err)) + } + return strings.TrimSpace(string(out)) +} + func getFirstFoundEnvTestBinaryDir() string { basePath := filepath.Join("..", "..", "bin", "k8s") entries, err := os.ReadDir(basePath) diff --git a/internal/controller/xpumanager_controller.go b/internal/controller/xpumanager_controller.go index 7adbff4..f9fd3f1 100644 --- a/internal/controller/xpumanager_controller.go +++ b/internal/controller/xpumanager_controller.go @@ -199,19 +199,7 @@ func processContainerResources(ds *apps.DaemonSet, spec *v1alpha.ClusterPolicy, } func processNodeSelectors(ds *apps.DaemonSet, spec *v1alpha.ClusterPolicy) { - ds.Spec.Template.Spec.NodeSelector = map[string]string{ - "kubernetes.io/arch": "amd64", - } - - if len(spec.Spec.NodeSelector) > 0 { - for k, v := range spec.Spec.NodeSelector { - ds.Spec.Template.Spec.NodeSelector[k] = v - } - } - - if spec.Spec.UseNFDLabeling { - ds.Spec.Template.Spec.NodeSelector["intel.feature.node.kubernetes.io/gpu"] = trueValue - } + ds.Spec.Template.Spec.NodeSelector = gpuNodeSelector(spec) } // Convert the integer based log level to a string based log level for the OTel config.