Skip to content

Commit 6b8457d

Browse files
staerionstaerion
committed
allow to set resource requests and limits for deamonset through operator
1 parent 31c02b2 commit 6b8457d

6 files changed

Lines changed: 319 additions & 113 deletions

File tree

‎cli/k8s_client/types.go‎

Lines changed: 24 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import (
1616
"k8s.io/apimachinery/pkg/types"
1717
"k8s.io/apimachinery/pkg/version"
1818

19+
netappv1 "github.com/netapp/trident/operator/controllers/orchestrator/apis/netapp/v1"
1920
versionutils "github.com/netapp/trident/utils/version"
2021
)
2122

@@ -168,27 +169,29 @@ type DeploymentYAMLArguments struct {
168169
}
169170

170171
type DaemonsetYAMLArguments struct {
171-
DaemonsetName string `json:"daemonsetName"`
172-
TridentImage string `json:"tridentImage"`
173-
ImageRegistry string `json:"imageRegistry"`
174-
KubeletDir string `json:"kubeletDir"`
175-
LogFormat string `json:"logFormat"`
176-
LogLevel string `json:"logLevel"`
177-
LogWorkflows string `json:"logWorkflows"`
178-
LogLayers string `json:"logLayers"`
179-
ProbePort string `json:"probePort"`
180-
ImagePullSecrets []string `json:"imagePullSecrets"`
181-
Labels map[string]string `json:"labels"`
182-
ControllingCRDetails map[string]string `json:"controllingCRDetails"`
183-
EnableForceDetach bool `json:"enableForceDetach"`
184-
DisableAuditLog bool `json:"disableAuditLog"`
185-
Debug bool `json:"debug"`
186-
Version *versionutils.Version `json:"version"`
187-
HTTPRequestTimeout string `json:"httpRequestTimeout"`
188-
NodeSelector map[string]string `json:"nodeSelector"`
189-
Tolerations []map[string]string `json:"tolerations"`
190-
ServiceAccountName string `json:"serviceAccountName"`
191-
ImagePullPolicy string `json:"imagePullPolicy"`
172+
DaemonsetName string `json:"daemonsetName"`
173+
TridentImage string `json:"tridentImage"`
174+
ImageRegistry string `json:"imageRegistry"`
175+
KubeletDir string `json:"kubeletDir"`
176+
LogFormat string `json:"logFormat"`
177+
LogLevel string `json:"logLevel"`
178+
LogWorkflows string `json:"logWorkflows"`
179+
LogLayers string `json:"logLayers"`
180+
ProbePort string `json:"probePort"`
181+
ImagePullSecrets []string `json:"imagePullSecrets"`
182+
Labels map[string]string `json:"labels"`
183+
ControllingCRDetails map[string]string `json:"controllingCRDetails"`
184+
EnableForceDetach bool `json:"enableForceDetach"`
185+
DisableAuditLog bool `json:"disableAuditLog"`
186+
Debug bool `json:"debug"`
187+
Version *versionutils.Version `json:"version"`
188+
HTTPRequestTimeout string `json:"httpRequestTimeout"`
189+
NodeSelector map[string]string `json:"nodeSelector"`
190+
Tolerations []map[string]string `json:"tolerations"`
191+
TridentResourceSpec netappv1.ResourceSpec `json:"tridentResourceSpec"`
192+
RegistrarResourceSpec netappv1.ResourceSpec `json:"registrarSpec"`
193+
ServiceAccountName string `json:"serviceAccountName"`
194+
ImagePullPolicy string `json:"imagePullPolicy"`
192195
}
193196

194197
type TridentVersionPodYAMLArguments struct {

‎cli/k8s_client/yaml_factory.go‎

Lines changed: 67 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,11 @@ import (
99

1010
commonconfig "github.com/netapp/trident/config"
1111
. "github.com/netapp/trident/logging"
12+
netappv1 "github.com/netapp/trident/operator/controllers/orchestrator/apis/netapp/v1"
1213
"github.com/netapp/trident/utils"
14+
v1 "k8s.io/api/core/v1"
15+
"k8s.io/apimachinery/pkg/api/resource"
16+
k8syaml "sigs.k8s.io/yaml"
1317
)
1418

1519
const (
@@ -726,6 +730,25 @@ func GetCSIDaemonSetYAMLWindows(args *DaemonsetYAMLArguments) string {
726730
}
727731
}
728732

733+
defaultRegistrarResourceSpec := netappv1.ResourceSpec{Resources: v1.ResourceRequirements{
734+
Limits: v1.ResourceList{
735+
v1.ResourceMemory: resource.MustParse("100Mi"),
736+
},
737+
Requests: v1.ResourceList{
738+
v1.ResourceCPU: resource.MustParse("10m"),
739+
v1.ResourceMemory: resource.MustParse("40Mi"),
740+
},
741+
}}
742+
defaultTridentResourceSpec := netappv1.ResourceSpec{Resources: v1.ResourceRequirements{
743+
Limits: v1.ResourceList{
744+
v1.ResourceMemory: resource.MustParse("400Mi"),
745+
},
746+
Requests: v1.ResourceList{
747+
v1.ResourceCPU: resource.MustParse("10m"),
748+
v1.ResourceMemory: resource.MustParse("20Mi"),
749+
},
750+
}}
751+
729752
kubeletDir := strings.TrimRight(args.KubeletDir, "/")
730753
daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{TRIDENT_IMAGE}", args.TridentImage)
731754
daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{DAEMONSET_NAME}", args.DaemonsetName)
@@ -745,6 +768,10 @@ func GetCSIDaemonSetYAMLWindows(args *DaemonsetYAMLArguments) string {
745768
daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{IMAGE_PULL_POLICY}", args.ImagePullPolicy)
746769
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "NODE_SELECTOR", constructNodeSelector(args.NodeSelector))
747770
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "NODE_TOLERATIONS", constructTolerations(tolerations))
771+
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "REGISTRAR_RESOURCE_CLAIMS", constructResourceClaim(
772+
"DsRegistrar", args.RegistrarResourceSpec, defaultRegistrarResourceSpec))
773+
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "TRIDENT_RESOURCE_CLAIMS", constructResourceClaim(
774+
"DsTrident", args.TridentResourceSpec, defaultTridentResourceSpec))
748775
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "LABELS", constructLabels(args.Labels))
749776
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "OWNER_REF", constructOwnerRef(args.ControllingCRDetails))
750777
// Log before secrets are inserted into YAML.
@@ -799,6 +826,13 @@ func GetCSIDaemonSetYAMLLinux(args *DaemonsetYAMLArguments) string {
799826
}
800827
}
801828

829+
defaultRegistrarResourceSpec := netappv1.ResourceSpec{Resources: v1.ResourceRequirements{
830+
Requests: v1.ResourceList{},
831+
}}
832+
defaultTridentResourceSpec := netappv1.ResourceSpec{Resources: v1.ResourceRequirements{
833+
Requests: v1.ResourceList{},
834+
}}
835+
802836
kubeletDir := strings.TrimRight(args.KubeletDir, "/")
803837
daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{TRIDENT_IMAGE}", args.TridentImage)
804838
daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{DAEMONSET_NAME}", args.DaemonsetName)
@@ -819,6 +853,10 @@ func GetCSIDaemonSetYAMLLinux(args *DaemonsetYAMLArguments) string {
819853
daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{IMAGE_PULL_POLICY}", args.ImagePullPolicy)
820854
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "NODE_SELECTOR", constructNodeSelector(args.NodeSelector))
821855
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "NODE_TOLERATIONS", constructTolerations(tolerations))
856+
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "REGISTRAR_RESOURCE_CLAIMS", constructResourceClaim(
857+
"DsRegistrar", args.RegistrarResourceSpec, defaultRegistrarResourceSpec))
858+
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "TRIDENT_RESOURCE_CLAIMS", constructResourceClaim(
859+
"DsTrident", args.TridentResourceSpec, defaultTridentResourceSpec))
822860
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "LABELS", constructLabels(args.Labels))
823861
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "OWNER_REF", constructOwnerRef(args.ControllingCRDetails))
824862

@@ -907,6 +945,7 @@ spec:
907945
failureThreshold: 5
908946
initialDelaySeconds: 10
909947
periodSeconds: 10
948+
{TRIDENT_RESOURCE_CLAIMS}
910949
env:
911950
- name: KUBE_NODE_NAME
912951
valueFrom:
@@ -948,6 +987,7 @@ spec:
948987
- "--v={SIDECAR_LOG_LEVEL}"
949988
- "--csi-address=$(ADDRESS)"
950989
- "--kubelet-registration-path=$(REGISTRATION_PATH)"
990+
{REGISTRAR_RESOURCE_CLAIMS}
951991
env:
952992
- name: ADDRESS
953993
value: /plugin/csi.sock
@@ -1137,12 +1177,7 @@ spec:
11371177
mountPath: \\.\pipe\csi-proxy-filesystem-v1beta1
11381178
- name: csi-proxy-smb-pipe-v1beta1
11391179
mountPath: \\.\pipe\csi-proxy-smb-v1beta1
1140-
resources:
1141-
limits:
1142-
memory: 400Mi
1143-
requests:
1144-
cpu: 10m
1145-
memory: 20Mi
1180+
{TRIDENT_RESOURCE_CLAIMS}
11461181
- name: node-driver-registrar
11471182
image: {CSI_SIDECAR_REGISTRY}/csi-node-driver-registrar:v2.8.0
11481183
imagePullPolicy: {IMAGE_PULL_POLICY}
@@ -1174,12 +1209,7 @@ spec:
11741209
mountPath: C:\csi
11751210
- name: registration-dir
11761211
mountPath: C:\registration
1177-
resources:
1178-
limits:
1179-
memory: 200Mi
1180-
requests:
1181-
cpu: 10m
1182-
memory: 20Mi
1212+
{REGISTRAR_RESOURCE_CLAIMS}
11831213
- name: liveness-probe
11841214
volumeMounts:
11851215
- mountPath: C:\csi
@@ -1193,12 +1223,6 @@ spec:
11931223
env:
11941224
- name: CSI_ENDPOINT
11951225
value: unix:///csi/csi.sock
1196-
resources:
1197-
limits:
1198-
memory: 100Mi
1199-
requests:
1200-
cpu: 10m
1201-
memory: 40Mi
12021226
affinity:
12031227
nodeAffinity:
12041228
requiredDuringSchedulingIgnoredDuringExecution:
@@ -2509,6 +2533,31 @@ func constructNodeSelector(nodeLabels map[string]string) string {
25092533
return nodeSelector
25102534
}
25112535

2536+
func constructResourceClaim(
2537+
containerName string, resourceClaim netappv1.ResourceSpec, defaultResourceClaim netappv1.ResourceSpec,
2538+
) string {
2539+
userData, err := k8syaml.Marshal(resourceClaim)
2540+
if err != nil {
2541+
Log().WithField("container", containerName).Errorf("Cannot marshal resource claim: %s", err)
2542+
return ""
2543+
}
2544+
2545+
if err := k8syaml.Unmarshal(userData, &defaultResourceClaim); err != nil {
2546+
Log().WithField("container", containerName).Errorf(
2547+
"Cannot unmarshal provided resource claim into default resource claim: %s", err,
2548+
)
2549+
return ""
2550+
}
2551+
mergedData, err := k8syaml.Marshal(defaultResourceClaim)
2552+
2553+
if err != nil {
2554+
Log().WithField("container", containerName).Errorf("Cannot marshal resource claim: %s", err)
2555+
return ""
2556+
}
2557+
2558+
return string(mergedData)
2559+
}
2560+
25122561
func constructTolerations(tolerations []map[string]string) string {
25132562
var tolerationsString string
25142563

‎cli/k8s_client/yaml_factory_test.go‎

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,11 @@ import (
1515
pspv1beta1 "k8s.io/api/policy/v1beta1"
1616
csiv1 "k8s.io/api/storage/v1"
1717
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
18+
"k8s.io/apimachinery/pkg/api/resource"
1819
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1920

2021
"github.com/netapp/trident/config"
22+
netappv1 "github.com/netapp/trident/operator/controllers/orchestrator/apis/netapp/v1"
2123
versionutils "github.com/netapp/trident/utils/version"
2224
)
2325

@@ -652,6 +654,43 @@ func TestGetCSIDaemonSetYAMLLinux_NodeSelectors(t *testing.T) {
652654
fmt.Sprintf("expected nodeSelector in final YAML: %s", yamlData))
653655
}
654656

657+
func TestGetCSIDaemonSetYAMLLinux_ResourceClaims(t *testing.T) {
658+
daemonsetArgs := &DaemonsetYAMLArguments{
659+
RegistrarResourceSpec: netappv1.ResourceSpec{},
660+
TridentResourceSpec: netappv1.ResourceSpec{
661+
Resources: v1.ResourceRequirements{
662+
Limits: v1.ResourceList{
663+
v1.ResourceMemory: resource.MustParse("125Mi"),
664+
},
665+
Requests: v1.ResourceList{
666+
v1.ResourceCPU: resource.MustParse("50m"),
667+
v1.ResourceMemory: resource.MustParse("50Mi"),
668+
},
669+
},
670+
},
671+
}
672+
expectedRegistrarResourceClaimString := "resources: {}\n"
673+
expectedTridentResourceClaimString := `
674+
resources:
675+
limits:
676+
memory: 125Mi
677+
requests:
678+
cpu: 50m
679+
memory: 50Mi
680+
`
681+
682+
yamlData := GetCSIDaemonSetYAMLLinux(daemonsetArgs)
683+
_, err := yaml.YAMLToJSON([]byte(yamlData))
684+
if err != nil {
685+
t.Fatalf("expected valid YAML, got %s", yamlData)
686+
}
687+
assert.Contains(t, yamlData, expectedRegistrarResourceClaimString,
688+
fmt.Sprintf("expected empty registrar resource claims in final YAML: %s", yamlData))
689+
690+
assert.Contains(t, yamlData, expectedTridentResourceClaimString,
691+
fmt.Sprintf("expected trident resource claims in final YAML: %s", yamlData))
692+
}
693+
655694
func TestGetCSIDaemonSetYAMLLinux_Tolerations(t *testing.T) {
656695
daemonsetArgs := &DaemonsetYAMLArguments{
657696
Tolerations: []map[string]string{
@@ -800,6 +839,56 @@ func TestGetCSIDaemonSetYAMLWindowsImagePullPolicy(t *testing.T) {
800839
}
801840
}
802841

842+
func TestGetCSIDaemonSetYAMWindows_ResourceClaims(t *testing.T) {
843+
versions := []string{"1.26.0"}
844+
registrarResourceSpec := netappv1.ResourceSpec{}
845+
tridentResourceSpec := netappv1.ResourceSpec{
846+
Resources: v1.ResourceRequirements{
847+
Limits: v1.ResourceList{
848+
v1.ResourceMemory: resource.MustParse("125Mi"),
849+
},
850+
Requests: v1.ResourceList{
851+
v1.ResourceCPU: resource.MustParse("50m"),
852+
},
853+
},
854+
}
855+
856+
expectedRegistrarResourceClaimString := `
857+
resources:
858+
limits:
859+
memory: 100Mi
860+
requests:
861+
cpu: 10m
862+
memory: 40Mi
863+
`
864+
expectedTridentResourceClaimString := `
865+
resources:
866+
limits:
867+
memory: 125Mi
868+
requests:
869+
cpu: 50m
870+
memory: 20Mi
871+
`
872+
for _, versionString := range versions {
873+
version := versionutils.MustParseSemantic(versionString)
874+
daemonsetArgs := &DaemonsetYAMLArguments{
875+
Version: version,
876+
RegistrarResourceSpec: registrarResourceSpec,
877+
TridentResourceSpec: tridentResourceSpec,
878+
}
879+
yamlData := GetCSIDaemonSetYAMLWindows(daemonsetArgs)
880+
_, err := yaml.YAMLToJSON([]byte(yamlData))
881+
if err != nil {
882+
t.Fatalf("expected valid YAML, got %s", yamlData)
883+
}
884+
assert.Contains(t, yamlData, expectedRegistrarResourceClaimString,
885+
fmt.Sprintf("expected empty registrar resource claims in final YAML: %s", yamlData))
886+
887+
assert.Contains(t, yamlData, expectedTridentResourceClaimString,
888+
fmt.Sprintf("expected trident resource claims in final YAML: %s", yamlData))
889+
}
890+
}
891+
803892
func TestConstructNodeSelector(t *testing.T) {
804893
nodeSelMap := map[string]string{"node-label-name": "master"}
805894

@@ -809,6 +898,54 @@ func TestConstructNodeSelector(t *testing.T) {
809898
assert.Equal(t, expectedNodeSelString, result)
810899
}
811900

901+
func TestConstructResourceClaim(t *testing.T) {
902+
defaultResourceClaim := netappv1.ResourceSpec{
903+
Resources: v1.ResourceRequirements{},
904+
}
905+
resourceClaim := netappv1.ResourceSpec{
906+
Resources: v1.ResourceRequirements{
907+
Limits: v1.ResourceList{
908+
v1.ResourceCPU: resource.MustParse("100m"),
909+
v1.ResourceMemory: resource.MustParse("50Mi"),
910+
},
911+
Requests: v1.ResourceList{
912+
v1.ResourceCPU: resource.MustParse("20m"),
913+
v1.ResourceMemory: resource.MustParse("20Mi"),
914+
},
915+
},
916+
}
917+
918+
expectedResourceClaimString := `resources:
919+
limits:
920+
cpu: 100m
921+
memory: 50Mi
922+
requests:
923+
cpu: 20m
924+
memory: 20Mi
925+
`
926+
927+
result := constructResourceClaim("test", resourceClaim, defaultResourceClaim)
928+
assert.Equal(t, expectedResourceClaimString, result)
929+
930+
resourceClaim = netappv1.ResourceSpec{
931+
Resources: v1.ResourceRequirements{
932+
Limits: v1.ResourceList{
933+
v1.ResourceCPU: resource.MustParse("100m"),
934+
},
935+
},
936+
}
937+
938+
expectedResourceClaimString = "resources:\n limits:\n cpu: 100m\n"
939+
result = constructResourceClaim("test", resourceClaim, defaultResourceClaim)
940+
assert.Equal(t, expectedResourceClaimString, result)
941+
942+
resourceClaim = netappv1.ResourceSpec{}
943+
944+
expectedResourceClaimString = "resources: {}\n"
945+
result = constructResourceClaim("test", resourceClaim, defaultResourceClaim)
946+
assert.Equal(t, expectedResourceClaimString, result)
947+
}
948+
812949
func TestGetNamespaceYAML(t *testing.T) {
813950
expected := v1.Namespace{
814951
TypeMeta: metav1.TypeMeta{

0 commit comments

Comments
 (0)