From 63e8d996af35f797806db199728554ff13bc3fd9 Mon Sep 17 00:00:00 2001 From: Saba Touheed Mujawar Date: Wed, 8 Jan 2025 07:28:12 -0500 Subject: [PATCH] kubeadm: create platform pods with zero CPU resources This specifies zero CPU resources when creating the manifests for the static platform pods, as a workaround for the lack of separate resource tracking for platform resources. This specifies zero CPU and Memory resources for the coredns deployment. manifests.go is the main source file for this, not sure if the coredns.yaml are used but they are updated to be consistent. This specifies CPU limit of 1 for kube-apiserver pod so that it is treated as a burstable QoS. This gives a boost of cgroup CPUShares since the burstable cgroup parent has significantly more CPUShares than best-effort on typical systems. This improves kube-apiserver API responsiveness. This increases kube-apiserver Readiness probe periodSeconds to 10 based on WRS/SS joint recommendation for minimum probe settings. This reduces likelihood of kube-apiserver probe failure and subsequent pod-restart under servere load. This also reduces CPU demand. Signed-off-by: Daniel Safta Signed-off-by: Saba Touheed Mujawar Signed-off-by: Boovan Rajendran Signed-off-by: Jim Gauld Signed-off-by: Saba Touheed Mujawar --- cluster/addons/dns/coredns/coredns.yaml.base | 4 ++-- cluster/addons/dns/coredns/coredns.yaml.in | 4 ++-- cluster/addons/dns/coredns/coredns.yaml.sed | 4 ++-- cmd/kubeadm/app/phases/addons/dns/dns_test.go | 8 ++++---- cmd/kubeadm/app/phases/addons/dns/manifests.go | 4 ++-- .../app/phases/controlplane/manifests.go | 10 ++++++---- cmd/kubeadm/app/util/staticpod/utils.go | 17 ++++++++++++++++- 7 files changed, 34 insertions(+), 17 deletions(-) diff --git a/cluster/addons/dns/coredns/coredns.yaml.base b/cluster/addons/dns/coredns/coredns.yaml.base index 3d438dce445..41088f063eb 100644 --- a/cluster/addons/dns/coredns/coredns.yaml.base +++ b/cluster/addons/dns/coredns/coredns.yaml.base @@ -139,8 +139,8 @@ spec: limits: memory: __DNS__MEMORY__LIMIT__ requests: - cpu: 100m - memory: 70Mi + cpu: 0 + memory: 0 args: [ "-conf", "/etc/coredns/Corefile" ] volumeMounts: - name: config-volume diff --git a/cluster/addons/dns/coredns/coredns.yaml.in b/cluster/addons/dns/coredns/coredns.yaml.in index 419acb0e966..906d6d28890 100644 --- a/cluster/addons/dns/coredns/coredns.yaml.in +++ b/cluster/addons/dns/coredns/coredns.yaml.in @@ -139,8 +139,8 @@ spec: limits: memory: 'dns_memory_limit' requests: - cpu: 100m - memory: 70Mi + cpu: 0 + memory: 0 args: [ "-conf", "/etc/coredns/Corefile" ] volumeMounts: - name: config-volume diff --git a/cluster/addons/dns/coredns/coredns.yaml.sed b/cluster/addons/dns/coredns/coredns.yaml.sed index a35df71454f..af0fae57dbd 100644 --- a/cluster/addons/dns/coredns/coredns.yaml.sed +++ b/cluster/addons/dns/coredns/coredns.yaml.sed @@ -139,8 +139,8 @@ spec: limits: memory: $DNS_MEMORY_LIMIT requests: - cpu: 100m - memory: 70Mi + cpu: 0 + memory: 0 args: [ "-conf", "/etc/coredns/Corefile" ] volumeMounts: - name: config-volume diff --git a/cmd/kubeadm/app/phases/addons/dns/dns_test.go b/cmd/kubeadm/app/phases/addons/dns/dns_test.go index 0b6c12eb41b..4f2022d7105 100644 --- a/cmd/kubeadm/app/phases/addons/dns/dns_test.go +++ b/cmd/kubeadm/app/phases/addons/dns/dns_test.go @@ -705,8 +705,8 @@ spec: limits: memory: 170Mi requests: - cpu: 100m - memory: 70Mi + cpu: 0 + memory: 0 args: [ "-conf", "/etc/coredns/Corefile" ] volumeMounts: - name: config-volume @@ -970,8 +970,8 @@ spec: limits: memory: 170Mi requests: - cpu: 100m - memory: 70Mi + cpu: 0 + memory: 0 args: [ "-conf", "/etc/coredns/Corefile" ] volumeMounts: - name: config-volume diff --git a/cmd/kubeadm/app/phases/addons/dns/manifests.go b/cmd/kubeadm/app/phases/addons/dns/manifests.go index 905a2e050e6..2a2212d5d37 100644 --- a/cmd/kubeadm/app/phases/addons/dns/manifests.go +++ b/cmd/kubeadm/app/phases/addons/dns/manifests.go @@ -104,8 +104,8 @@ spec: limits: memory: 170Mi requests: - cpu: 100m - memory: 70Mi + cpu: 0 + memory: 0 args: [ "-conf", "/etc/coredns/Corefile" ] volumeMounts: - name: config-volume diff --git a/cmd/kubeadm/app/phases/controlplane/manifests.go b/cmd/kubeadm/app/phases/controlplane/manifests.go index 11b93e083db..3458efe3a03 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests.go @@ -67,8 +67,10 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/livez", endpoint.BindPort, v1.URISchemeHTTPS), ReadinessProbe: staticpodutil.ReadinessProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/readyz", endpoint.BindPort, v1.URISchemeHTTPS), StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/livez", endpoint.BindPort, v1.URISchemeHTTPS, componentHealthCheckTimeout), - Resources: staticpodutil.ComponentResources("250m"), - Env: kubeadmutil.MergeKubeadmEnvVars(proxyEnvs, cfg.APIServer.ExtraEnvs), + // WRS: Increase kube-apiserver cgroup CPUShares to improve API responsiveness; + // achieved by setting CPU Limits to make it burstable QoS. + Resources: staticpodutil.ComponentLimitResources("0", "1"), + Env: kubeadmutil.MergeKubeadmEnvVars(proxyEnvs, cfg.APIServer.ExtraEnvs), }, mounts.GetVolumes(kubeadmconstants.KubeAPIServer), map[string]string{kubeadmconstants.KubeAPIServerAdvertiseAddressEndpointAnnotationKey: endpoint.String()}), kubeadmconstants.KubeControllerManager: staticpodutil.ComponentPod(v1.Container{ @@ -79,7 +81,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeControllerManager)), LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetControllerManagerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeControllerManagerPort, v1.URISchemeHTTPS), StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetControllerManagerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeControllerManagerPort, v1.URISchemeHTTPS, componentHealthCheckTimeout), - Resources: staticpodutil.ComponentResources("200m"), + Resources: staticpodutil.ComponentResources("0"), Env: kubeadmutil.MergeKubeadmEnvVars(proxyEnvs, cfg.ControllerManager.ExtraEnvs), }, mounts.GetVolumes(kubeadmconstants.KubeControllerManager), nil), kubeadmconstants.KubeScheduler: staticpodutil.ComponentPod(v1.Container{ @@ -90,7 +92,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeScheduler)), LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetSchedulerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeSchedulerPort, v1.URISchemeHTTPS), StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetSchedulerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeSchedulerPort, v1.URISchemeHTTPS, componentHealthCheckTimeout), - Resources: staticpodutil.ComponentResources("100m"), + Resources: staticpodutil.ComponentResources("0"), Env: kubeadmutil.MergeKubeadmEnvVars(proxyEnvs, cfg.Scheduler.ExtraEnvs), }, mounts.GetVolumes(kubeadmconstants.KubeScheduler), nil), } diff --git a/cmd/kubeadm/app/util/staticpod/utils.go b/cmd/kubeadm/app/util/staticpod/utils.go index fcd28eff4d0..c87d6c954eb 100644 --- a/cmd/kubeadm/app/util/staticpod/utils.go +++ b/cmd/kubeadm/app/util/staticpod/utils.go @@ -99,6 +99,18 @@ func ComponentResources(cpu string) v1.ResourceRequirements { } } +// ComponentLimitResources returns the v1.ResourceRequirements object needed for allocating a specified amount of the CPU with Limits +func ComponentLimitResources(cpu string, lcpu string) v1.ResourceRequirements { + return v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse(cpu), + }, + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse(lcpu), + }, + } +} + // NewVolume creates a v1.Volume with a hostPath mount to the specified location func NewVolume(name, path string, pathType *v1.HostPathType) v1.Volume { return v1.Volume{ @@ -255,7 +267,10 @@ func LivenessProbe(host, path string, port int32, scheme v1.URIScheme) *v1.Probe func ReadinessProbe(host, path string, port int32, scheme v1.URIScheme) *v1.Probe { // sets initialDelaySeconds as '0' because we don't want to delay user infrastructure checks // looking for "ready" status on kubeadm static Pods - return createHTTPProbe(host, path, port, scheme, 0, 15, 3, 1) + // WRS/SS joint recommendation: All pods probes should have following minimum probe + // settings unless required by the service (initialDelaySecond 0, periodSeconds 10, + // timeoutSeconds 5, successThreshold 1, failureThreshold 3) + return createHTTPProbe(host, path, port, scheme, 0, 15, 3, 10) } // StartupProbe creates a Probe object with a HTTPGet handler -- 2.25.1