Skip to content

Commit 1ef132c

Browse files
clementnussclementnuss
committed
permit specifying daemonset and deployment resources
closes NetApp#853 Signed-off-by: Clément Nussbaumer <[email protected]>
1 parent e648ed2 commit 1ef132c

File tree

7 files changed

+230
-126
lines changed

7 files changed

+230
-126
lines changed

‎cli/k8s_client/types.go

Lines changed: 58 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -143,65 +143,67 @@ type KubernetesClient interface {
143143
}
144144

145145
type DeploymentYAMLArguments struct {
146-
DeploymentName string `json:"deploymentName"`
147-
TridentImage string `json:"tridentImage"`
148-
AutosupportImage string `json:"autosupportImage"`
149-
AutosupportProxy string `json:"autosupportProxy"`
150-
AutosupportInsecure bool `json:"autosupportInsecure"`
151-
AutosupportCustomURL string `json:"autosupportCustomURL"`
152-
AutosupportSerialNumber string `json:"autosupportSerialNumber"`
153-
AutosupportHostname string `json:"autosupportHostname"`
154-
ImageRegistry string `json:"imageRegistry"`
155-
LogFormat string `json:"logFormat"`
156-
LogLevel string `json:"logLevel"`
157-
LogWorkflows string `json:"logWorkflows"`
158-
LogLayers string `json:"logLayers"`
159-
ImagePullSecrets []string `json:"imagePullSecrets"`
160-
Labels map[string]string `json:"labels"`
161-
ControllingCRDetails map[string]string `json:"controllingCRDetails"`
162-
Debug bool `json:"debug"`
163-
UseIPv6 bool `json:"useIPv6"`
164-
SilenceAutosupport bool `json:"silenceAutosupport"`
165-
Version *versionutils.Version `json:"version"`
166-
DisableAuditLog bool `json:"disableAuditLog"`
167-
HTTPRequestTimeout string `json:"httpRequestTimeout"`
168-
NodeSelector map[string]string `json:"nodeSelector"`
169-
Tolerations []map[string]string `json:"tolerations"`
170-
ServiceAccountName string `json:"serviceAccountName"`
171-
ImagePullPolicy string `json:"imagePullPolicy"`
172-
EnableForceDetach bool `json:"enableForceDetach"`
173-
ACPImage string `json:"acpImage"`
174-
EnableACP bool `json:"enableACP"`
175-
CloudProvider string `json:"cloudProvider"`
176-
IdentityLabel bool `json:"identityLabel"`
177-
K8sAPIQPS int `json:"k8sAPIQPS"`
146+
DeploymentName string `json:"deploymentName"`
147+
TridentImage string `json:"tridentImage"`
148+
AutosupportImage string `json:"autosupportImage"`
149+
AutosupportProxy string `json:"autosupportProxy"`
150+
AutosupportInsecure bool `json:"autosupportInsecure"`
151+
AutosupportCustomURL string `json:"autosupportCustomURL"`
152+
AutosupportSerialNumber string `json:"autosupportSerialNumber"`
153+
AutosupportHostname string `json:"autosupportHostname"`
154+
ImageRegistry string `json:"imageRegistry"`
155+
LogFormat string `json:"logFormat"`
156+
LogLevel string `json:"logLevel"`
157+
LogWorkflows string `json:"logWorkflows"`
158+
LogLayers string `json:"logLayers"`
159+
ImagePullSecrets []string `json:"imagePullSecrets"`
160+
Labels map[string]string `json:"labels"`
161+
ControllingCRDetails map[string]string `json:"controllingCRDetails"`
162+
Debug bool `json:"debug"`
163+
UseIPv6 bool `json:"useIPv6"`
164+
SilenceAutosupport bool `json:"silenceAutosupport"`
165+
Version *versionutils.Version `json:"version"`
166+
DisableAuditLog bool `json:"disableAuditLog"`
167+
HTTPRequestTimeout string `json:"httpRequestTimeout"`
168+
NodeSelector map[string]string `json:"nodeSelector"`
169+
Tolerations []map[string]string `json:"tolerations"`
170+
ServiceAccountName string `json:"serviceAccountName"`
171+
ImagePullPolicy string `json:"imagePullPolicy"`
172+
EnableForceDetach bool `json:"enableForceDetach"`
173+
ACPImage string `json:"acpImage"`
174+
EnableACP bool `json:"enableACP"`
175+
CloudProvider string `json:"cloudProvider"`
176+
IdentityLabel bool `json:"identityLabel"`
177+
K8sAPIQPS int `json:"k8sAPIQPS"`
178+
Resources *v1.ResourceRequirements `json:"resources"`
178179
}
179180

180181
type DaemonsetYAMLArguments struct {
181-
DaemonsetName string `json:"daemonsetName"`
182-
TridentImage string `json:"tridentImage"`
183-
ImageRegistry string `json:"imageRegistry"`
184-
KubeletDir string `json:"kubeletDir"`
185-
LogFormat string `json:"logFormat"`
186-
LogLevel string `json:"logLevel"`
187-
LogWorkflows string `json:"logWorkflows"`
188-
LogLayers string `json:"logLayers"`
189-
ProbePort string `json:"probePort"`
190-
ImagePullSecrets []string `json:"imagePullSecrets"`
191-
Labels map[string]string `json:"labels"`
192-
ControllingCRDetails map[string]string `json:"controllingCRDetails"`
193-
EnableForceDetach bool `json:"enableForceDetach"`
194-
DisableAuditLog bool `json:"disableAuditLog"`
195-
Debug bool `json:"debug"`
196-
Version *versionutils.Version `json:"version"`
197-
HTTPRequestTimeout string `json:"httpRequestTimeout"`
198-
NodeSelector map[string]string `json:"nodeSelector"`
199-
Tolerations []map[string]string `json:"tolerations"`
200-
ServiceAccountName string `json:"serviceAccountName"`
201-
ImagePullPolicy string `json:"imagePullPolicy"`
202-
ISCSISelfHealingInterval string `json:"iscsiSelfHealingInterval"`
203-
ISCSISelfHealingWaitTime string `json:"iscsiSelfHealingWaitTime"`
204-
NodePrep []string `json:"nodePrep"`
182+
DaemonsetName string `json:"daemonsetName"`
183+
TridentImage string `json:"tridentImage"`
184+
ImageRegistry string `json:"imageRegistry"`
185+
KubeletDir string `json:"kubeletDir"`
186+
LogFormat string `json:"logFormat"`
187+
LogLevel string `json:"logLevel"`
188+
LogWorkflows string `json:"logWorkflows"`
189+
LogLayers string `json:"logLayers"`
190+
ProbePort string `json:"probePort"`
191+
ImagePullSecrets []string `json:"imagePullSecrets"`
192+
Labels map[string]string `json:"labels"`
193+
ControllingCRDetails map[string]string `json:"controllingCRDetails"`
194+
EnableForceDetach bool `json:"enableForceDetach"`
195+
DisableAuditLog bool `json:"disableAuditLog"`
196+
Debug bool `json:"debug"`
197+
Version *versionutils.Version `json:"version"`
198+
HTTPRequestTimeout string `json:"httpRequestTimeout"`
199+
NodeSelector map[string]string `json:"nodeSelector"`
200+
Tolerations []map[string]string `json:"tolerations"`
201+
ServiceAccountName string `json:"serviceAccountName"`
202+
ImagePullPolicy string `json:"imagePullPolicy"`
203+
ISCSISelfHealingInterval string `json:"iscsiSelfHealingInterval"`
204+
ISCSISelfHealingWaitTime string `json:"iscsiSelfHealingWaitTime"`
205+
NodePrep []string `json:"nodePrep"`
206+
Resources *v1.ResourceRequirements `json:"resources"`
205207
}
206208

207209
type TridentVersionPodYAMLArguments struct {

‎cli/k8s_client/yaml_factory.go

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@ import (
1111
commonconfig "github.com/netapp/trident/config"
1212
. "github.com/netapp/trident/logging"
1313
"github.com/netapp/trident/utils"
14+
v1 "k8s.io/api/core/v1"
15+
"sigs.k8s.io/yaml"
1416
)
1517

1618
const (
@@ -524,6 +526,7 @@ func GetCSIDeploymentYAML(args *DeploymentYAMLArguments) string {
524526
deploymentYAML = utils.ReplaceMultilineYAMLTag(deploymentYAML, "OWNER_REF", constructOwnerRef(args.ControllingCRDetails))
525527
deploymentYAML = utils.ReplaceMultilineYAMLTag(deploymentYAML, "NODE_SELECTOR", constructNodeSelector(args.NodeSelector))
526528
deploymentYAML = utils.ReplaceMultilineYAMLTag(deploymentYAML, "NODE_TOLERATIONS", constructTolerations(args.Tolerations))
529+
deploymentYAML = utils.ReplaceMultilineYAMLTag(deploymentYAML, "RESOURCES", constructResources(args.Resources))
527530
deploymentYAML = strings.ReplaceAll(deploymentYAML, "{ENABLE_FORCE_DETACH}", strconv.FormatBool(args.EnableForceDetach))
528531
deploymentYAML = strings.ReplaceAll(deploymentYAML, "{ENABLE_ACP}", enableACP)
529532
deploymentYAML = strings.ReplaceAll(deploymentYAML, "{K8S_API_CLIENT_TRIDENT_THROTTLE}", K8sAPITridentThrottle)
@@ -594,6 +597,7 @@ spec:
594597
{ENABLE_ACP}
595598
{DEBUG}
596599
{K8S_API_CLIENT_TRIDENT_THROTTLE}
600+
{RESOURCES}
597601
livenessProbe:
598602
exec:
599603
command:
@@ -647,9 +651,7 @@ spec:
647651
{AUTOSUPPORT_HOSTNAME}
648652
{AUTOSUPPORT_DEBUG}
649653
{AUTOSUPPORT_INSECURE}
650-
resources:
651-
limits:
652-
memory: 1Gi
654+
{RESOURCES}
653655
volumeMounts:
654656
- name: asup-dir
655657
mountPath: /asup
@@ -667,6 +669,7 @@ spec:
667669
- "--retry-interval-start=8s"
668670
- "--retry-interval-max=30s"
669671
{K8S_API_CLIENT_SIDECAR_THROTTLE}
672+
{RESOURCES}
670673
env:
671674
- name: ADDRESS
672675
value: /var/lib/csi/sockets/pluginproxy/csi.sock
@@ -686,6 +689,7 @@ spec:
686689
- "--retry-interval-start=10s"
687690
- "--csi-address=$(ADDRESS)"
688691
{K8S_API_CLIENT_SIDECAR_THROTTLE}
692+
{RESOURCES}
689693
env:
690694
- name: ADDRESS
691695
value: /var/lib/csi/sockets/pluginproxy/csi.sock
@@ -700,6 +704,7 @@ spec:
700704
- "--timeout=300s"
701705
- "--csi-address=$(ADDRESS)"
702706
{K8S_API_CLIENT_SIDECAR_THROTTLE}
707+
{RESOURCES}
703708
env:
704709
- name: ADDRESS
705710
value: /var/lib/csi/sockets/pluginproxy/csi.sock
@@ -718,6 +723,7 @@ spec:
718723
- "--timeout=300s"
719724
- "--csi-address=$(ADDRESS)"
720725
{K8S_API_CLIENT_SIDECAR_THROTTLE}
726+
{RESOURCES}
721727
env:
722728
- name: ADDRESS
723729
value: /var/lib/csi/sockets/pluginproxy/csi.sock
@@ -909,6 +915,7 @@ func GetCSIDaemonSetYAMLLinux(args *DaemonsetYAMLArguments) string {
909915
daemonSetYAML = utils.ReplaceMultilineYAMLTag(daemonSetYAML, "NODE_TOLERATIONS", constructTolerations(tolerations))
910916
daemonSetYAML = utils.ReplaceMultilineYAMLTag(daemonSetYAML, "LABELS", constructLabels(args.Labels))
911917
daemonSetYAML = utils.ReplaceMultilineYAMLTag(daemonSetYAML, "OWNER_REF", constructOwnerRef(args.ControllingCRDetails))
918+
daemonSetYAML = utils.ReplaceMultilineYAMLTag(daemonSetYAML, "RESOURCES", constructResources(args.Resources))
912919

913920
// Log before secrets are inserted into YAML.
914921
Log().WithField("yaml", daemonSetYAML).Trace("CSI Daemonset Linux YAML.")
@@ -1056,6 +1063,7 @@ spec:
10561063
timeoutSeconds: 5
10571064
periodSeconds: 10
10581065
initialDelaySeconds: 15
1066+
{RESOURCES}
10591067
env:
10601068
- name: KUBE_NODE_NAME
10611069
valueFrom:
@@ -1097,6 +1105,7 @@ spec:
10971105
- "--v={SIDECAR_LOG_LEVEL}"
10981106
- "--csi-address=$(ADDRESS)"
10991107
- "--kubelet-registration-path=$(REGISTRATION_PATH)"
1108+
{RESOURCES}
11001109
env:
11011110
- name: ADDRESS
11021111
value: /plugin/csi.sock
@@ -2762,6 +2771,16 @@ func constructOwnerRef(ownerRef map[string]string) string {
27622771
return ownerRefData
27632772
}
27642773

2774+
func constructResources(res *v1.ResourceRequirements) string {
2775+
r := struct {
2776+
Resources *v1.ResourceRequirements `json:"resources"`
2777+
}{
2778+
Resources: res,
2779+
}
2780+
bytes, _ := yaml.Marshal(r)
2781+
return string(bytes)
2782+
}
2783+
27652784
func constructImagePullSecrets(imagePullSecrets []string) string {
27662785
var imagePullSecretsData string
27672786
if len(imagePullSecrets) > 0 {

‎cli/k8s_client/yaml_factory_test.go

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import (
1616
pspv1beta1 "k8s.io/api/policy/v1beta1"
1717
csiv1 "k8s.io/api/storage/v1"
1818
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
19+
"k8s.io/apimachinery/pkg/api/resource"
1920
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2021

2122
"github.com/netapp/trident/config"
@@ -514,6 +515,30 @@ func TestGetCSIDeploymentYAML_K8sAPIQPS(t *testing.T) {
514515
}
515516
}
516517

518+
func TestGetCSIDeploymentYAML_Resources(t *testing.T) {
519+
args := &DeploymentYAMLArguments{Resources: &v1.ResourceRequirements{
520+
Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("114m"), v1.ResourceMemory: resource.MustParse("256Mi")},
521+
Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("200m"), v1.ResourceMemory: resource.MustParse("512Mi")},
522+
}}
523+
524+
expectedResourcesString := `
525+
resources:
526+
limits:
527+
cpu: 114m
528+
memory: 256Mi
529+
requests:
530+
cpu: 200m
531+
memory: 512Mi
532+
`
533+
yamlData := GetCSIDeploymentYAML(args)
534+
_, err := yaml.YAMLToJSON([]byte(yamlData))
535+
if err != nil {
536+
t.Fatalf("expected valid YAML, got %s", yamlData)
537+
}
538+
assert.Contains(t, yamlData, expectedResourcesString,
539+
fmt.Sprintf("expected resources in final YAML: %s", yamlData))
540+
}
541+
517542
func TestGetCSIDaemonSetYAMLLinux(t *testing.T) {
518543
versions := []string{"1.21.0", "1.23.0", "1.25.0"}
519544

@@ -859,6 +884,30 @@ func TestGetCSIDaemonSetYAMLLinux_Tolerations(t *testing.T) {
859884
fmt.Sprintf("expected default tolerations to not appear in final YAML: %s", yamlData))
860885
}
861886

887+
func TestGetCSIDaemonSetYAMLLinux_Resources(t *testing.T) {
888+
args := &DaemonsetYAMLArguments{Resources: &v1.ResourceRequirements{
889+
Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("114m"), v1.ResourceMemory: resource.MustParse("256Mi")},
890+
Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("200m"), v1.ResourceMemory: resource.MustParse("512Mi")},
891+
}}
892+
893+
expectedResourcesString := `
894+
resources:
895+
limits:
896+
cpu: 114m
897+
memory: 256Mi
898+
requests:
899+
cpu: 200m
900+
memory: 512Mi
901+
`
902+
yamlData := GetCSIDaemonSetYAMLLinux(args)
903+
_, err := yaml.YAMLToJSON([]byte(yamlData))
904+
if err != nil {
905+
t.Fatalf("expected valid YAML, got %s", yamlData)
906+
}
907+
assert.Contains(t, yamlData, expectedResourcesString,
908+
fmt.Sprintf("expected resources in final YAML: %s", yamlData))
909+
}
910+
862911
func TestGetCSIDaemonSetYAMLWindows(t *testing.T) {
863912
versions := []string{"1.21.0", "1.23.0", "1.25.0"}
864913

‎helm/trident-operator/templates/tridentorchestrator.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,10 @@ spec:
1717
{{- if .Values.tridentDebug }}
1818
debug: {{ .Values.tridentDebug }}
1919
{{- end }}
20+
daemonsetResources:
21+
{{- toYaml .Values.tridentDaemonsetResources | nindent 4 }}
22+
deploymentResources:
23+
{{- toYaml .Values.tridentDeploymentResources | nindent 4 }}
2024
{{- if .Values.tridentLogLevel }}
2125
logLevel: {{ .Values.tridentLogLevel }}
2226
{{- end }}

‎helm/trident-operator/values.yaml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,18 @@ tridentDisableAuditLog: true
112112
# tridentDebug allows the log level of Trident to be set to debug
113113
tridentDebug: false
114114

115+
# tridentDaemonsetResources permits specifying the resources for all containers of the trident-node daemonset. example:
116+
# tridentDaemonsetResources:
117+
# requests:
118+
# cpu: 250m
119+
# limits:
120+
# cpu: 100m
121+
# memory: 1Gi
122+
tridentDaemonsetResources: {}
123+
124+
# tridentDeploymentResources permits specifying the resources for all containers of the trident-controller deployment.
125+
tridentDeploymentResources:
126+
115127
# tridentLogWorkflows allows specific Trident workflows to be enabled for trace logging or log suppression.
116128
tridentLogWorkflows: ""
117129

‎operator/controllers/orchestrator/installer/installer.go

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,9 @@ var (
103103

104104
nodePrep []string
105105

106+
daemonsetResources *v1.ResourceRequirements
107+
deploymentResources *v1.ResourceRequirements
108+
106109
CRDnames = []string{
107110
ActionMirrorUpdateCRDName,
108111
ActionSnapshotRestoreCRDName,
@@ -391,6 +394,14 @@ func (i *Installer) setInstallationParams(
391394

392395
nodePrep = validation.FormatProtocols(cr.Spec.NodePrep)
393396

397+
if cr.Spec.DaemonsetResources != nil {
398+
daemonsetResources = cr.Spec.DaemonsetResources
399+
}
400+
401+
if cr.Spec.DeploymentResources != nil {
402+
deploymentResources = cr.Spec.DeploymentResources
403+
}
404+
394405
if cr.Spec.ProbePort != nil {
395406
probePort = strconv.FormatInt(*cr.Spec.ProbePort, 10)
396407
}
@@ -1526,6 +1537,7 @@ func (i *Installer) createOrPatchTridentDeployment(
15261537
EnableACP: enableACP,
15271538
IdentityLabel: identityLabel,
15281539
K8sAPIQPS: k8sAPIQPS,
1540+
Resources: deploymentResources,
15291541
}
15301542

15311543
newDeploymentYAML := k8sclient.GetCSIDeploymentYAML(deploymentArgs)
@@ -1606,6 +1618,7 @@ func (i *Installer) createOrPatchTridentDaemonSet(
16061618
ISCSISelfHealingInterval: iscsiSelfHealingInterval,
16071619
ISCSISelfHealingWaitTime: iscsiSelfHealingWaitTime,
16081620
NodePrep: nodePrep,
1621+
Resources: daemonsetResources,
16091622
}
16101623

16111624
var newDaemonSetYAML string

0 commit comments

Comments
 (0)