starlingx/integ
Code Issues Proposed changes

Merge "Remove obsolete versions of kubernetes 1.24-1.28 and update cni plugins"

Browse Source
This commit is contained in:
Zuul 2025-04-28 15:20:41 +00:00 committed by Gerrit Code Review
commit e2992798b5
182 changed files with 17 additions and 59664 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
debian_iso_image.inc
View File

@ -195,26 +195,6 @@ lvm2
python3-keyrings.alt
#kubernetes-x (-master, -misc, -unit-test used only for build)
#kubernetes-1.24.4
kubernetes-1.24.4-client
kubernetes-1.24.4-kubeadm
kubernetes-1.24.4-node
#kubernetes-1.25.3
kubernetes-1.25.3-client
kubernetes-1.25.3-kubeadm
kubernetes-1.25.3-node
#kubernetes-1.26.1
kubernetes-1.26.1-client
kubernetes-1.26.1-kubeadm
kubernetes-1.26.1-node
#kubernetes-1.27.5
kubernetes-1.27.5-client
kubernetes-1.27.5-kubeadm
kubernetes-1.27.5-node
#kubernetes-1.28.4
kubernetes-1.28.4-client
kubernetes-1.28.4-kubeadm
kubernetes-1.28.4-node
#kubernetes-1.29.2
kubernetes-1.29.2-client
kubernetes-1.29.2-kubeadm

5
debian_pkg_dirs
View File

@ -72,11 +72,6 @@ kubernetes/flux2-charts
kubernetes/helm
kubernetes/k8s-cni-cache-cleanup
kubernetes/k8s-pod-recovery
kubernetes/kubernetes-1.24.4
kubernetes/kubernetes-1.25.3
kubernetes/kubernetes-1.26.1
kubernetes/kubernetes-1.27.5
kubernetes/kubernetes-1.28.4
kubernetes/kubernetes-1.29.2
kubernetes/kubernetes-1.30.6
kubernetes/kubernetes-1.31.5

7
kubernetes/cni/bond-cni/debian/deb_folder/changelog
View File

@ -1,3 +1,10 @@
bond-cni (1.0-91b412e3eb8c5dfc315d8c73fb386d005e44d5f3) unstable; urgency=medium
* Updated golang to 1.21
-- Jim Gauld <james.gauld@windriver.com> Mon, 21 Apr 2025 10:26:40 +0000
bond-cni (1.0-91b412e3eb8c5dfc315d8c73fb386d005e44d5f3) unstable; urgency=medium
* Updated using latest commit

2
kubernetes/cni/bond-cni/debian/deb_folder/control
View File

@ -6,7 +6,7 @@ Standards-Version: 4.6.0
Homepage: https://www.starlingx.io
Build-Depends: debhelper-compat (= 13),
dh-golang,
golang-1.18,
golang-1.21,
golang-github-appc-cni-dev,
golang-github-containernetworking-plugins-dev,
golang-github-vishvananda-netlink-dev,

2
kubernetes/cni/bond-cni/debian/deb_folder/rules
View File

@ -1,6 +1,6 @@
#!/usr/bin/make -f
export PATH := /usr/lib/go-1.18/bin:$(PATH)
export PATH := /usr/lib/go-1.21/bin:$(PATH)
export DH_VERBOSE = 1
export DH_GOLANG_GO_GENERATE := 1
export INSTALL_DIR := $(CURDIR)/debian/bond-cni/

6
kubernetes/cni/plugins/debian/deb_folder/changelog
View File

@ -1,3 +1,9 @@
golang-github-containernetworking-plugins (1.5.0) unstable; urgency=medium
* Updated golang to 1.21
-- Jim Gauld <james.gauld@windriver.com> Mon, 21 Apr 2025 10:26:40 +0000
golang-github-containernetworking-plugins (1.5.0) unstable; urgency=medium
* Updated for stx debian packaging

2
kubernetes/cni/plugins/debian/deb_folder/control
View File

@ -5,7 +5,7 @@ Standards-Version: 4.6.1
Maintainer: StarlingX Developers <starlingx-discuss@lists.starlingx.io>
Build-Depends: debhelper-compat (= 13),
dh-golang,
golang-1.18,
golang-1.21,
golang-dbus-dev (>= 5.0.2~),
golang-github-alexflint-go-filemutex-dev,
golang-github-appc-cni-dev (>= 1.0.1~),

2
kubernetes/cni/plugins/debian/deb_folder/rules
View File

@ -1,6 +1,6 @@
#!/usr/bin/make -f
export PATH := /usr/lib/go-1.18/bin:$(PATH)
export PATH := /usr/lib/go-1.21/bin:$(PATH)
export DH_VERBOSE := 1
export DH_GOLANG_GO_GENERATE := 1
export INSTALL_DIR := $(CURDIR)/debian/plugins

259
kubernetes/kubernetes-1.24.4/debian/deb_folder/changelog
View File

@ -1,259 +0,0 @@
kubernetes-1.24.4 (1.24.4-1) unstable; urgency=medium
* Updated for stx debian packaging
-- James Gauld james.gauld@windriver.com Wed, 14 Sep 2022 11:30:00 +0000
kubernetes-1.23.1 (1.23.1-1) unstable; urgency=medium
* Updated for stx debian packaging
-- Kaustubh Dhokte <kaustubh.dhokte@opendev.org> Tue, 22 Feb 2022 20:57:45 +0000
kubernetes (1.21.8-1) unstable; urgency=medium
* Updated for stx debian packaging
-- Mihnea Saracin mihnea.saracin@opendev.org Fri, 29 Oct 2021 12:51:12 +0000
kubernetes (1.20.5+really1.20.2-1.1) unstable; urgency=medium
* Non-maintainer upload.
* Build using golang-go (Closes: #1000980)
-- Florian Ernst <florian@debian.org> Sat, 02 Apr 2022 16:49:13 +0200
kubernetes (1.20.5+really1.20.2-1) unstable; urgency=medium
* This is actually still 1.20.2
* Only contains the client, for the server binaries see unstable/fasttrack
-- Janos Lenart <ocsi@debian.org> Sun, 13 Jun 2021 07:08:18 +0000
kubernetes (1.20.2-1) unstable; urgency=medium
* New upstream release: 1.20.2
-- Janos Lenart <ocsi@debian.org> Thu, 14 Jan 2021 10:55:09 +0000
kubernetes (1.20.0-1) unstable; urgency=medium
* New upstream release: 1.20.0
* Fixes CVE-2020-8565
-- Janos Lenart <ocsi@debian.org> Wed, 09 Dec 2020 12:33:59 +0000
kubernetes (1.19.4-2) unstable; urgency=medium
* Updated README.Debian
-- Janos Lenart <ocsi@debian.org> Sat, 21 Nov 2020 14:06:21 +0000
kubernetes (1.19.4-1) unstable; urgency=medium
* New upstream release: 1.19.4
-- Janos Lenart <ocsi@debian.org> Tue, 17 Nov 2020 09:30:59 +0000
kubernetes (1.19.3-1) unstable; urgency=medium
* New upstream release: 1.19.3
* Building with golang-1.15
* Fixes CVE-2020-8564, CVE-2020-8566
-- Janos Lenart <ocsi@debian.org> Wed, 21 Oct 2020 10:38:41 +0100
kubernetes (1.18.6-1) unstable; urgency=medium
* New upstream release: 1.18.6
* (An earlier version, 1.17.4-1 fixes CVE-2019-9946)
-- Janos Lenart <ocsi@debian.org> Thu, 16 Jul 2020 10:08:46 +0100
kubernetes (1.18.5-1) unstable; urgency=medium
* New upstream release: 1.18.5
* Fixes CVE-2020-8557, CVE-2020-8558, CVE-2020-8559
-- Janos Lenart <ocsi@debian.org> Wed, 15 Jul 2020 17:19:40 +0100
kubernetes (1.18.3-1) unstable; urgency=medium
* New upstream release: 1.18.3
* Improved build reproducibility
-- Janos Lenart <ocsi@debian.org> Tue, 02 Jun 2020 11:18:12 +0000
kubernetes (1.18.2-3) unstable; urgency=medium
* Bumped Standards-Version
* Improved build reproducibility
-- Janos Lenart <ocsi@debian.org> Fri, 15 May 2020 13:17:53 +0000
kubernetes (1.18.2-2) unstable; urgency=medium
* Added i386 back
-- Janos Lenart <ocsi@debian.org> Sun, 03 May 2020 21:13:17 +0000
kubernetes (1.18.2-1) unstable; urgency=medium
* New upstream release: 1.18.2
-- Janos Lenart <ocsi@debian.org> Sun, 03 May 2020 19:25:37 +0000
kubernetes (1.18.0-1) unstable; urgency=medium
* New upstream release: 1.18.0
-- Janos Lenart <ocsi@debian.org> Sat, 28 Mar 2020 12:58:42 +0000
kubernetes (1.17.4-1) unstable; urgency=high
* New maintainer (Closes: #886739)
* New upstream release: 1.17.4 (Closes: #887741)
* New Debian packaging from scratch. See README.Debian
* kubernetes-node
- Moved docker from Depends into Recommends as kubelet can also work with
rkt, cri-o, etc. (Closes: #872690)
- Not shipping systemd units for kubelet and kube-proxy for now
* kubernetes-master
- Moved etcd from Depends into Recommends as apiserver can also connect to
a remote etcd/cluster.
- Not shipping systemd units for kube-apiserver, kube-schedules and
kube-controller-manager for now
-- Janos Lenart <ocsi@debian.org> Sun, 15 Mar 2020 21:46:45 +0000
kubernetes (1.7.16+dfsg-1) unstable; urgency=medium
[ Michael Stapelberg ]
* Switch to XS-Go-Import-Path
[ Dmitry Smirnov ]
* Resurrected "mergo.patch" that has been mistakenly removed
(Closes: #878254).
* Re-enabled safeguard test for the above problem.
* New upstream release:
+ CVE-2017-1002101 (Closes: #892801)
+ CVE-2017-1002102 (Closes: #894051)
* Updated Vcs URLs for Salsa.
* Standards-Version: 4.1.4
* Build-Depends:
- golang-go
+ golang-any
+ golang-github-appc-cni-dev
+ golang-github-armon-circbuf-dev
+ golang-github-azure-azure-sdk-for-go-dev
+ golang-github-dgrijalva-jwt-go-v3-dev
+ golang-github-docker-distribution-dev
+ golang-github-docker-docker-dev
+ golang-github-emicklei-go-restful-swagger12-dev
+ golang-github-gogo-protobuf-dev
+ golang-github-gorilla-websocket-dev
+ golang-github-grpc-ecosystem-go-grpc-prometheus-dev
+ golang-github-karlseguin-ccache-dev
- golang-github-opencontainers-runc-dev
+ golang-github-opencontainers-docker-runc-dev
+ golang-github-pmezard-go-difflib-dev
+ golang-golang-x-time-dev
+ golang-golang-x-tools-dev
+ golang-google-grpc-dev
+ golang-gopkg-warnings.v0-dev
+ golang-goprotobuf-dev
-- Dmitry Smirnov <onlyjob@debian.org> Sun, 06 May 2018 16:20:21 +1000
kubernetes (1.7.7+dfsg-3) unstable; urgency=medium
* kubernetes-master should depend on etcd (Closes: #855218).
-- Andrew Shadura <andrewsh@debian.org> Sun, 22 Oct 2017 19:40:46 +0100
kubernetes (1.7.7+dfsg-2) unstable; urgency=medium
* Use CURDIR, not PWD, unbreaks the build at buildds.
-- Andrew Shadura <andrewsh@debian.org> Fri, 06 Oct 2017 19:25:45 +0200
kubernetes (1.7.7+dfsg-1) unstable; urgency=medium
[ Tim Potter ]
* Open work for new release
* Remove unused Files-Excluded entries from d/copyright
* Remove Skydns B-D as no longer used
* Don't build on ppc64 or ppc64le architectures
[ Andrew Shadura ]
* New upstream release.
* Refresh patches.
* Update build dependencies.
* Symlink vendor packages to the build directory.
-- Andrew Shadura <andrewsh@debian.org> Fri, 06 Oct 2017 18:54:06 +0200
kubernetes (1.5.5+dfsg-2) unstable; urgency=medium
* Team upload.
* Don't build on ppc64le due to Go linker problems. See GitHub issue
https://github.com/golang/go/issues/15823.
* Don't build on ppc64 as it's not supported by upstream at the
moment. (Closes: #860505)
-- Tim Potter <tpot@hpe.com> Sat, 03 Jun 2017 08:00:51 +1000
kubernetes (1.5.5+dfsg-1) unstable; urgency=low
[ Dmitry Smirnov ]
* Switch to bundled "rkt".
* rules: remove "-p" option from build and test overrides.
* control: drop obsolete "golang-clockwork-dev" alternative.
* New patch to disable test failing on [armel].
* Upload to unstable.
[ Tim Potter ]
* New upstream version. [March 2017]
* Big updates to d/rules and d/copyright to update to upstream
changes made since the 1.2.x release.
* Refresh patches to bring up to date with upstream changes since
1.2.x.
* control: add lsb-base as dependency for sysvinit scripts.
* Suppress spelling-error-in-binary Lintian messages.
-- Tim Potter <tpot@hpe.com> Thu, 13 Apr 2017 16:45:57 +1000
kubernetes (1.2.5+dfsg-1) experimental; urgency=medium
* New upstream release [June 2016].
* Switch to private "github.com/golang/glog" due to log noise.
* Disabled failing tests; no longer ignore failures in tests.
* Build/test using 2 cores only.
* New patch to update appc/cni name space (fixes FTBFS).
* Removed obsolete "spf13-cobra.patch".
-- Dmitry Smirnov <onlyjob@debian.org> Sun, 03 Jul 2016 04:12:28 +1000
kubernetes (1.2.4+dfsg-2) experimental; urgency=medium
* Added new patch to fix incompatibility with "imdario/mergo" v0.2.2
(Closes: #825753).
Thanks, Florian Ernst.
* Enable tests but ignore failures for now.
-- Dmitry Smirnov <onlyjob@debian.org> Fri, 17 Jun 2016 01:41:38 +1000
kubernetes (1.2.4+dfsg-1) experimental; urgency=medium
* New upstream release [May 2016].
* New patch to print output of "uname -m" on unsupported architectures.
* New "docker.patch" to fix potential FTBFS.
+ Build-Depends += "golang-github-docker-distribution-dev".
-- Dmitry Smirnov <onlyjob@debian.org> Wed, 15 Jun 2016 21:03:01 +1000
kubernetes (1.2.3+dfsg-1) experimental; urgency=low
* Initial release (Closes: #795652).
-- Dmitry Smirnov <onlyjob@debian.org> Mon, 25 Apr 2016 22:40:12 +1000

93
kubernetes/kubernetes-1.24.4/debian/deb_folder/control
View File

@ -1,93 +0,0 @@
Source: kubernetes-1.24.4
Section: admin
Priority: optional
Maintainer: StarlingX Developers <StarlingX-discuss@lists.StarlingX.io>
Build-Depends: debhelper-compat (= 13),
build-essential,
bash-completion,
jq,
rsync,
go-bindata,
go-md2man,
golang-1.18
Standards-Version: 4.4.1
Homepage: http://kubernetes.io/
XS-Build-Size: 15GB
Package: kubernetes-1.24.4-client
Provides: kubernetes-utils
Architecture: amd64
Depends: ${misc:Depends}, ${shlibs:Depends}
Recommends: ${misc:Recommends}
Built-Using: ${misc:Built-Using}
Description: Kubernetes Command Line Tool
The Kubernetes command line tool for interacting with the Kubernetes API.
Package: kubernetes-1.24.4-master
Architecture: amd64
Depends: ${misc:Depends}, ${shlibs:Depends},
adduser,
lsb-base,
etcd
Recommends: ${misc:Recommends}, kubernetes-1.24.4-client
Built-Using: ${misc:Built-Using}
Description: Kubernetes services for master host
Container Cluster Manager from Google. Kubernetes is an open source system
for managing containerized applications across multiple hosts, providing
basic mechanisms for deployment, maintenance, and scaling of applications.
.
Linux kernel version 3.8 or above is required for proper operation of the
daemon process, and that any lower versions may have subtle and/or glaring
issues.
.
This package provides "kube-apiserver", "kube-controller-manager" and
"kube-scheduler" daemons.
Package: kubernetes-1.24.4-node
Provides: cadvisor
Architecture: amd64
Depends: ${misc:Depends}, ${shlibs:Depends},
adduser,
conntrack,
conntrackd,
docker.io,
lsb-base,
socat,
Recommends: ${misc:Recommends}, kubernetes-1.24.4-client
Built-Using: ${misc:Built-Using}
Description: Kubernetes services for node host
Container Cluster Manager from Google. Kubernetes is an open source system
for managing containerized applications across multiple hosts, providing
basic mechanisms for deployment, maintenance, and scaling of applications.
.
Linux kernel version 3.8 or above is required for proper operation of the
daemon process, and that any lower versions may have subtle and/or glaring
issues.
Package: kubernetes-1.24.4-kubeadm
Architecture: amd64
Depends: ${misc:Depends}, containernetworking-plugins
Recommends: ${misc:Recommends}, kubernetes-1.24.4-client
Built-Using: ${misc:Built-Using}
Description: Kubernetes Cluster Bootstrapping Tool
The Kubernetes command line tool for bootstrapping a Kubernetes cluster.
Package: kubernetes-1.24.4-misc
Architecture: amd64
Depends: ${misc:Depends}, ${shlibs:Depends}
Recommends: ${misc:Recommends}
Built-Using: ${misc:Built-Using}
Description: dummy package
Kubernetes dummy package for misc stuff we don't want to install in production.
Package: kubernetes-1.24.4-unit-test
Architecture: amd64
Depends: ${misc:Depends}, ${shlibs:Depends},
hostname,
rsync,
etcd (>= 2.0.9),
network-manager,
Recommends: ${misc:Recommends}
Built-Using: ${misc:Built-Using}
Description: Kubernetes unit test
Kubernetes unit-test framework.

1820
kubernetes/kubernetes-1.24.4/debian/deb_folder/copyright
View File

File diff suppressed because it is too large Load Diff

17
kubernetes/kubernetes-1.24.4/debian/deb_folder/kubeadm.conf
View File

@ -1,17 +0,0 @@
# Note: This dropin only works with kubeadm and kubelet v1.11+
[Service]
Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf"
Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml"
# This is a file that "kubeadm init" and "kubeadm join" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically
EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env
# This is a file that the user can use for overrides of the kubelet args as a last resort. Preferably, the user should use
# the .NodeRegistration.KubeletExtraArgs object in the configuration files instead. KUBELET_EXTRA_ARGS should be sourced from this file.
EnvironmentFile=-/etc/default/kubelet
ExecStart=
ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS
ExecStartPre=-/usr/local/sbin/sanitize_kubelet_reserved_cpus.sh /etc/default/kubelet
ExecStartPre=-/usr/bin/kubelet-cgroup-setup.sh
ExecStartPost=/bin/bash -c 'echo $MAINPID > /var/run/kubelet.pid;'
ExecStopPost=/bin/rm -f /var/run/kubelet.pid
StartLimitInterval=0
RestartSec=10

2
kubernetes/kubernetes-1.24.4/debian/deb_folder/kubernetes-1.24.4-client.install
View File

@ -1,2 +0,0 @@
usr/local/kubernetes/1.24.4/stage2/usr/bin/kubectl
usr/local/kubernetes/1.24.4/stage2/usr/share/bash-completion/completions/kubectl

9
kubernetes/kubernetes-1.24.4/debian/deb_folder/kubernetes-1.24.4-client.lintian-overrides
View File

@ -1,9 +0,0 @@
## Generated man pages: TODO
manpage-has-bad-whatis-entry usr/share/man/*
manpage-has-errors-from-man usr/share/man/man1/*
## Bash-completion script does not have to be executable:
script-not-executable usr/share/bash-completion/completions/kubectl
## Override annoying/useless messages
kubernetes-client: spelling-error-in-binary

1
kubernetes/kubernetes-1.24.4/debian/deb_folder/kubernetes-1.24.4-kubeadm.dirs
View File

@ -1 +0,0 @@
usr/local/kubernetes/1.24.4/stage2/etc/systemd/system/kubelet.service.d/

2
kubernetes/kubernetes-1.24.4/debian/deb_folder/kubernetes-1.24.4-kubeadm.install
View File

@ -1,2 +0,0 @@
usr/local/kubernetes/1.24.4/stage1/usr/bin/kubeadm
usr/local/kubernetes/1.24.4/stage2/etc/systemd/system/kubelet.service.d/kubeadm.conf

5
kubernetes/kubernetes-1.24.4/debian/deb_folder/kubernetes-1.24.4-master.dirs
View File

@ -1,5 +0,0 @@
etc/kubernetes-1.24.4
etc/kubernetes-1.24.4/addons
etc/kubernetes-1.24.4/addons/volumesnapshots
etc/kubernetes-1.24.4/addons/volumesnapshots/crd
etc/kubernetes-1.24.4/addons/volumesnapshots/volume-snapshot-controller

8
kubernetes/kubernetes-1.24.4/debian/deb_folder/kubernetes-1.24.4-master.install
View File

@ -1,8 +0,0 @@
usr/bin/kube-apiserver
usr/bin/kube-controller-manager
usr/bin/kube-scheduler
etc/kubernetes-1.24.4/addons/volumesnapshots/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml
etc/kubernetes-1.24.4/addons/volumesnapshots/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml
etc/kubernetes-1.24.4/addons/volumesnapshots/crd/snapshot.storage.k8s.io_volumesnapshots.yaml
etc/kubernetes-1.24.4/addons/volumesnapshots/volume-snapshot-controller/volume-snapshot-controller-deployment.yaml
etc/kubernetes-1.24.4/addons/volumesnapshots/volume-snapshot-controller/rbac-volume-snapshot-controller.yaml

7
kubernetes/kubernetes-1.24.4/debian/deb_folder/kubernetes-1.24.4-master.lintian-overrides
View File

@ -1,7 +0,0 @@
## No manual page for hyperkube
kubernetes-master: binary-without-manpage usr/bin/hyperkube
## Override annoying/useless messages
kubernetes-master: spelling-error-in-binary
kubernetes-master: manpage-has-errors-from-man usr/share/man/man1/*
kubernetes-master: manpage-has-bad-whatis-entry usr/share/man/man1/*

3
kubernetes/kubernetes-1.24.4/debian/deb_folder/kubernetes-1.24.4-misc.docs
View File

@ -1,3 +0,0 @@
src/k8s.io/kubernetes/README.md
src/k8s.io/kubernetes/SUPPORT.md
src/k8s.io/kubernetes/_output/NOTICE

1
kubernetes/kubernetes-1.24.4/debian/deb_folder/kubernetes-1.24.4-misc.install
View File

@ -1 +0,0 @@
usr/bin/kube-proxy

10
kubernetes/kubernetes-1.24.4/debian/deb_folder/kubernetes-1.24.4-misc.manpages
View File

@ -1,10 +0,0 @@
# kubernetes-client
src/k8s.io/kubernetes/_output/man/kubeadm*
src/k8s.io/kubernetes/_output/man/kubectl*
# kubernetes-master
src/k8s.io/kubernetes/_output/man/kube-apiserver*
src/k8s.io/kubernetes/_output/man/kube-scheduler*
src/k8s.io/kubernetes/_output/man/kube-controller-manager*
# kubernetes-node
src/k8s.io/kubernetes/_output/man/kubelet*
src/k8s.io/kubernetes/_output/man/kube-proxy*

1
kubernetes/kubernetes-1.24.4/debian/deb_folder/kubernetes-1.24.4-node.install
View File

@ -1 +0,0 @@
usr/local/kubernetes/1.24.4/stage2/usr/bin/kubelet

4
kubernetes/kubernetes-1.24.4/debian/deb_folder/kubernetes-1.24.4-node.lintian-overrides
View File

@ -1,4 +0,0 @@
## Override annoying/useless messages
kubernetes-node: spelling-error-in-binary
kubernetes-node: manpage-has-errors-from-man usr/share/man/man1/*
kubernetes-node: manpage-has-bad-whatis-entry usr/share/man/man1/*

1
kubernetes/kubernetes-1.24.4/debian/deb_folder/kubernetes-1.24.4-unit-test.install
View File

@ -1 +0,0 @@
var/lib/kubernetes-unit-test/

407
kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/Identify-platform-pods-based-on-pod-or-namespace-labels.patch
View File

@ -1,407 +0,0 @@
From 5e86e6319f5e6d0f0a56863a141238b9c4721ceb Mon Sep 17 00:00:00 2001
From: Boovan Rajendran <boovan.rajendran@windriver.com>
Date: Mon, 6 May 2024 02:29:04 -0400
Subject: [PATCH] Identify platform pods based on pod or namespace labels
Pods with labeled with 'app.starlingx.io/component=platform'
are identified as 'platform'. These have isolated cpu affinity
cpuset when cpu-manager 'static' policy is configured.
For k8s 1.24.4 to identify the pod as 'platform', keep existing
hardcoded namespace list to support the application that
have not upgraded yet, from old versions, and also to support
the new application, which has pod/namespace labels.
Signed-off-by: Boovan Rajendran <boovan.rajendran@windriver.com>
---
pkg/kubelet/cm/cpumanager/policy_static.go | 81 +++++++++-
.../cm/cpumanager/policy_static_test.go | 147 ++++++++++++++++++
.../cm/cpumanager/topology_hints_test.go | 3 +
3 files changed, 229 insertions(+), 2 deletions(-)
diff --git a/pkg/kubelet/cm/cpumanager/policy_static.go b/pkg/kubelet/cm/cpumanager/policy_static.go
index 94f18152d5b..286e983ec32 100644
--- a/pkg/kubelet/cm/cpumanager/policy_static.go
+++ b/pkg/kubelet/cm/cpumanager/policy_static.go
@@ -17,11 +17,17 @@ limitations under the License.
package cpumanager
import (
+ "context"
"fmt"
"strconv"
+ k8sclient "k8s.io/client-go/kubernetes"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ restclient "k8s.io/client-go/rest"
v1 "k8s.io/api/core/v1"
+ "k8s.io/client-go/tools/clientcmd"
"k8s.io/klog/v2"
+ "k8s.io/kubernetes/cmd/kubeadm/app/constants"
v1qos "k8s.io/kubernetes/pkg/apis/core/v1/helper/qos"
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state"
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
@@ -41,6 +47,23 @@ const (
ErrorSMTAlignment = "SMTAlignmentError"
)
+// Declared as variables so that they can easily more
+// overridden during testing
+type getPodNamespace func(string) (*v1.Namespace, error)
+type buildFromConfigFlag func(masterUrl string, kubeconfigPath string) (*restclient.Config, error)
+type isKubeInfraFunc func(pod *v1.Pod) bool
+
+var varGetNamespaceObject getPodNamespace
+var varBuildConfigFromFlags buildFromConfigFlag
+var varIsKubeInfra isKubeInfraFunc
+
+func init() {
+ varIsKubeInfra = isKubeInfra
+ varGetNamespaceObject = getPodNamespaceObject
+ varBuildConfigFromFlags = clientcmd.BuildConfigFromFlags
+}
+
+
// SMTAlignmentError represents an error due to SMT alignment
type SMTAlignmentError struct {
RequestedCPUs int
@@ -286,7 +309,7 @@ func (p *staticPolicy) updateCPUsToReuse(pod *v1.Pod, container *v1.Container, c
func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Container) error {
// Process infra pods before guaranteed pods
- if isKubeInfra(pod) {
+ if varIsKubeInfra(pod) {
// Container belongs in reserved pool.
// We don't want to fall through to the p.guaranteedCPUs() clause below so return either nil or error.
if _, ok := s.GetCPUSet(string(pod.UID), container.Name); ok {
@@ -451,7 +474,7 @@ func (p *staticPolicy) guaranteedCPUs(pod *v1.Pod, container *v1.Container) int
return 0
}
// Infrastructure pods use reserved CPUs even if they're in the Guaranteed QoS class
- if isKubeInfra(pod) {
+ if varIsKubeInfra(pod) {
return 0
}
// Safe downcast to do for all systems with < 2.1 billion CPUs.
@@ -668,14 +691,68 @@ func (p *staticPolicy) generateCPUTopologyHints(availableCPUs cpuset.CPUSet, reu
return hints
}
+func getPodNamespaceObject(podNamespaceName string) (*v1.Namespace, error) {
+
+ kubeConfigPath := constants.GetKubeletKubeConfigPath()
+ cfg, err := varBuildConfigFromFlags("", kubeConfigPath)
+ if err != nil {
+ klog.Error("Failed to build client config from ", kubeConfigPath, err.Error())
+ return nil, err
+ }
+
+ clientset, err := k8sclient.NewForConfig(cfg)
+ if err != nil {
+ klog.Error("Failed to get clientset for KUBECONFIG ", kubeConfigPath, err.Error())
+ return nil, err
+ }
+
+ namespaceObj, err := clientset.CoreV1().Namespaces().Get(context.TODO(), podNamespaceName, metav1.GetOptions{})
+ if err != nil {
+ klog.Error("Error getting namespace object:", err.Error())
+ return nil, err
+ }
+
+ return namespaceObj, nil
+
+}
+
// check if a given pod is in a platform infrastructure namespace
+// or check if a given pod is labelled as platform pod or is in
+// a namespace labelled as a platform namespace
func isKubeInfra(pod *v1.Pod) bool {
+ podName := pod.GetName()
+ podNamespaceName := pod.GetNamespace()
for _, namespace := range infraNamespaces {
if namespace == pod.Namespace {
+ klog.Infof("Pod %s has %s namespace. Treating as platform pod.", podName , podNamespaceName)
return true
}
}
+
+ klog.InfoS("Checking pod ", podName , " for label 'app.starlingx.io/component=platform'.")
+ podLabels := pod.GetLabels()
+ val, ok := podLabels["app.starlingx.io/component"]
+ if (ok && val == "platform") {
+ klog.InfoS("Pod ", podName, " has 'app.starlingx.io/component=platform' label. Treating as platform pod.")
+ return true
+ }
+
+ klog.V(4).InfoS("Pod ", podName, " does not have 'app.starlingx.io/component=platform' label. Checking its namespace information...")
+ namespaceObj, err := varGetNamespaceObject(podNamespaceName)
+ if err != nil {
+ return false
+ }
+
+ namespaceLabels := namespaceObj.GetLabels()
+ val, ok = namespaceLabels["app.starlingx.io/component"]
+ if ok && val == "platform" {
+ klog.InfoS("For pod: ", podName, ", its Namespace ", podNamespaceName, " has 'app.starlingx.io/component=platform' label. Treating as platform pod.")
+ return true
+ }
+
+ klog.InfoS("Neither pod ", podName, " nor its namespace ", podNamespaceName, " has 'app.starlingx.io/component=platform' label. Not treating as platform pod.")
return false
+
}
// get the isolated CPUs (if any) from the devices associated with a specific container
diff --git a/pkg/kubelet/cm/cpumanager/policy_static_test.go b/pkg/kubelet/cm/cpumanager/policy_static_test.go
index d0308556c6d..95ed4122aeb 100644
--- a/pkg/kubelet/cm/cpumanager/policy_static_test.go
+++ b/pkg/kubelet/cm/cpumanager/policy_static_test.go
@@ -17,10 +17,13 @@ limitations under the License.
package cpumanager
import (
+ "errors"
"fmt"
"reflect"
"testing"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ restclient "k8s.io/client-go/rest"
v1 "k8s.io/api/core/v1"
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state"
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
@@ -823,6 +826,7 @@ type staticPolicyTestWithResvList struct {
stAssignments state.ContainerCPUAssignments
stDefaultCPUSet cpuset.CPUSet
pod *v1.Pod
+ isKubeInfraPodfunc isKubeInfraFunc
expErr error
expNewErr error
expCPUAlloc bool
@@ -894,6 +898,14 @@ func TestStaticPolicyStartWithResvList(t *testing.T) {
}
}
+func fakeIsKubeInfraTrue(pod *v1.Pod) bool {
+ return true
+}
+
+func fakeIsKubeInfraFalse(pod *v1.Pod) bool {
+ return false
+}
+
func TestStaticPolicyAddWithResvList(t *testing.T) {
infraPod := makePod("fakePod", "fakeContainer2", "200m", "200m")
infraPod.Namespace = "kube-system"
@@ -907,6 +919,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
stAssignments: state.ContainerCPUAssignments{},
stDefaultCPUSet: cpuset.NewCPUSet(1, 2, 3, 4, 5, 6, 7),
pod: makePod("fakePod", "fakeContainer2", "8000m", "8000m"),
+ isKubeInfraPodfunc: fakeIsKubeInfraFalse,
expErr: fmt.Errorf("not enough cpus available to satisfy request"),
expCPUAlloc: false,
expCSet: cpuset.NewCPUSet(),
@@ -920,6 +933,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
stAssignments: state.ContainerCPUAssignments{},
stDefaultCPUSet: cpuset.NewCPUSet(2, 3, 4, 5, 6, 7),
pod: makePod("fakePod", "fakeContainer2", "1000m", "1000m"),
+ isKubeInfraPodfunc: fakeIsKubeInfraFalse,
expErr: nil,
expCPUAlloc: true,
expCSet: cpuset.NewCPUSet(4), // expect sibling of partial core
@@ -937,6 +951,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
},
stDefaultCPUSet: cpuset.NewCPUSet(0, 1, 4, 5),
pod: makePod("fakePod", "fakeContainer3", "2000m", "2000m"),
+ isKubeInfraPodfunc: fakeIsKubeInfraFalse,
expErr: nil,
expCPUAlloc: true,
expCSet: cpuset.NewCPUSet(4, 5),
@@ -954,6 +969,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
},
stDefaultCPUSet: cpuset.NewCPUSet(4, 5),
pod: infraPod,
+ isKubeInfraPodfunc: fakeIsKubeInfraTrue,
expErr: nil,
expCPUAlloc: true,
expCSet: cpuset.NewCPUSet(0, 1),
@@ -971,6 +987,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
},
stDefaultCPUSet: cpuset.NewCPUSet(4, 5),
pod: infraPod,
+ isKubeInfraPodfunc: fakeIsKubeInfraTrue,
expErr: nil,
expCPUAlloc: true,
expCSet: cpuset.NewCPUSet(0),
@@ -987,6 +1004,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
defaultCPUSet: testCase.stDefaultCPUSet,
}
+ varIsKubeInfra = testCase.isKubeInfraPodfunc
container := &testCase.pod.Spec.Containers[0]
err := policy.Allocate(st, testCase.pod, container)
if !reflect.DeepEqual(err, testCase.expErr) {
@@ -1106,3 +1124,132 @@ func TestStaticPolicyOptions(t *testing.T) {
})
}
}
+
+func makePodWithLabels(podLabels map[string]string) *v1.Pod {
+ return &v1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-pod",
+ Namespace: "test-namespace",
+ Labels: podLabels,
+ },
+ }
+}
+
+func fakeBuildConfigFromFlags(masterUrl string, kubeconfigPath string) (*restclient.Config, error) {
+
+ return &restclient.Config{}, nil
+}
+
+func fakeBuildConfigFromFlagsError(masterUrl string, kubeconfigPath string) (*restclient.Config, error) {
+
+ errString := fmt.Sprintf("%s file not found", kubeconfigPath)
+ return nil, errors.New(errString)
+
+}
+
+func getFakeInfraPodNamespace(_ string) (*v1.Namespace, error) {
+
+ return &v1.Namespace{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-namespace",
+ Labels: map[string]string{
+ "app.starlingx.io/component": "platform",
+ },
+ }}, nil
+}
+
+func getFakeNonInfraPodNamespace(_ string) (*v1.Namespace, error) {
+
+ return &v1.Namespace{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-namespace",
+ Labels: map[string]string{
+ "fake": "label",
+ }}}, nil
+
+}
+
+type kubeInfraPodTestCase struct {
+ description string
+ pod *v1.Pod
+ namespaceFunc getPodNamespace
+ expectedValue bool
+}
+
+func TestKubeInfraPod(t *testing.T) {
+ testCases := []kubeInfraPodTestCase{
+ {
+ description: "Pod with platform label and namespace with platform label",
+ pod: makePodWithLabels(map[string]string{
+ "app.starlingx.io/component": "platform",
+ }),
+ namespaceFunc: getFakeInfraPodNamespace,
+ expectedValue: true,
+ },
+ {
+ description: "Pod with platform label and namespace without platform label",
+ pod: makePodWithLabels(map[string]string{
+ "app.starlingx.io/component": "platform",
+ }),
+ namespaceFunc: getFakeNonInfraPodNamespace,
+ expectedValue: true,
+
+ },
+ {
+ description: "Pod without platform label and namespace with platform label",
+ pod: makePodWithLabels(map[string]string{
+ "test": "label",
+ }),
+ namespaceFunc: getFakeInfraPodNamespace,
+ expectedValue: true,
+ },
+ {
+ description: "Pod without platform label and namespace without platform label",
+ pod: makePodWithLabels(map[string]string{
+ "test": "namespace",
+ }),
+ namespaceFunc: getFakeNonInfraPodNamespace,
+ expectedValue: false,
+ },
+
+ }
+
+ for _, testCase := range testCases {
+ t.Run(testCase.description, func(t *testing.T) {
+
+ varGetNamespaceObject = testCase.namespaceFunc
+ varBuildConfigFromFlags = fakeBuildConfigFromFlags
+ gotValue := isKubeInfra(testCase.pod)
+
+ if gotValue != testCase.expectedValue {
+ t.Errorf("StaticPolicy isKubeInfraPod() error %v. expected value %v actual value %v",
+ testCase.description, testCase.expectedValue, gotValue)
+ } else {
+ fmt.Printf("StaticPolicy isKubeInfraPod() test successful. : %v ", testCase.description)
+ }
+
+ })
+ }
+
+ test := kubeInfraPodTestCase{
+ description: "Failure reading kubeconfig file",
+ pod: makePodWithLabels(map[string]string{
+ "test": "namespace",
+ }),
+ namespaceFunc: getFakeNonInfraPodNamespace,
+ expectedValue: false,
+ }
+
+ varGetNamespaceObject = getPodNamespaceObject
+ varBuildConfigFromFlags = fakeBuildConfigFromFlagsError
+
+ gotValue := isKubeInfra(test.pod)
+
+ if gotValue != test.expectedValue {
+ t.Errorf("StaticPolicy isKubeInfraPod() error %v. expected value %v actual value %v",
+ test.description, test.expectedValue, gotValue)
+ } else {
+ fmt.Printf("StaticPolicy isKubeInfraPod() test successful. : %v ", test.description)
+ }
+
+}
diff --git a/pkg/kubelet/cm/cpumanager/topology_hints_test.go b/pkg/kubelet/cm/cpumanager/topology_hints_test.go
index 9b8abe77488..b5a0ca41ab1 100644
--- a/pkg/kubelet/cm/cpumanager/topology_hints_test.go
+++ b/pkg/kubelet/cm/cpumanager/topology_hints_test.go
@@ -142,6 +142,7 @@ func TestPodGuaranteedCPUs(t *testing.T) {
expectedCPU: 6,
},
}
+ varIsKubeInfra = fakeIsKubeInfraFalse
for _, tc := range tcases {
requestedCPU := p.podGuaranteedCPUs(tc.pod)
@@ -184,6 +185,7 @@ func TestGetTopologyHints(t *testing.T) {
sourcesReady: &sourcesReadyStub{},
}
+ varIsKubeInfra = fakeIsKubeInfraFalse
hints := m.GetTopologyHints(&tc.pod, &tc.container)[string(v1.ResourceCPU)]
if len(tc.expectedHints) == 0 && len(hints) == 0 {
continue
@@ -237,6 +239,7 @@ func TestGetPodTopologyHints(t *testing.T) {
sourcesReady: &sourcesReadyStub{},
}
+ varIsKubeInfra = fakeIsKubeInfraFalse
podHints := m.GetPodTopologyHints(&tc.pod)[string(v1.ResourceCPU)]
if len(tc.expectedHints) == 0 && len(podHints) == 0 {
continue
--
2.25.1

32
kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/Revert-e2e-framework-test_context-add-control-plane-.patch
View File

@ -1,32 +0,0 @@
From a218316fb35a2ec67b7c8ad0fc2e8df537ee3434 Mon Sep 17 00:00:00 2001
From: Jim Gauld <James.Gauld@windriver.com>
Date: Wed, 12 Oct 2022 13:57:45 -0400
Subject: [PATCH 1/5] Revert "e2e/framework/test_context: add "control-plane"
to non-blocking-taints"
This reverts commit 8641897057431d6c89a716d86c997f29049df0f7.
---
test/e2e/framework/test_context.go | 7 +------
1 file changed, 1 insertion(+), 6 deletions(-)
diff --git a/test/e2e/framework/test_context.go b/test/e2e/framework/test_context.go
index b4c4743b6ea..8bc5827d89a 100644
--- a/test/e2e/framework/test_context.go
+++ b/test/e2e/framework/test_context.go
@@ -311,12 +311,7 @@ func RegisterCommonFlags(flags *flag.FlagSet) {
flags.StringVar(&TestContext.SystemdServices, "systemd-services", "docker", "The comma separated list of systemd services the framework will dump logs for.")
flags.BoolVar(&TestContext.DumpSystemdJournal, "dump-systemd-journal", false, "Whether to dump the full systemd journal.")
flags.StringVar(&TestContext.ImageServiceEndpoint, "image-service-endpoint", "", "The image service endpoint of cluster VM instances.")
- // TODO: remove the node-role.kubernetes.io/master taint in 1.25 or later.
- // The change will likely require an action for some users that do not
- // use k8s originated tools like kubeadm or kOps for creating clusters
- // and taint their control plane nodes with "master", expecting the test
- // suite to work with this legacy non-blocking taint.
- flags.StringVar(&TestContext.NonblockingTaints, "non-blocking-taints", `node-role.kubernetes.io/control-plane,node-role.kubernetes.io/master`, "Nodes with taints in this comma-delimited list will not block the test framework from starting tests. The default taint 'node-role.kubernetes.io/master' is DEPRECATED and will be removed from the list in a future release.")
+ flags.StringVar(&TestContext.NonblockingTaints, "non-blocking-taints", `node-role.kubernetes.io/master`, "Nodes with taints in this comma-delimited list will not block the test framework from starting tests.")
flags.BoolVar(&TestContext.ListImages, "list-images", false, "If true, will show list of images used for runnning tests.")
flags.BoolVar(&TestContext.ListConformanceTests, "list-conformance-tests", false, "If true, will show list of conformance tests.")
--
2.25.1

86
kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/Revert-kubeadm-apply-the-new-control-plane-taint-dur.patch
View File

@ -1,86 +0,0 @@
From 2c9b24f312305a29de2c861a35c3ec18b4ad3994 Mon Sep 17 00:00:00 2001
From: Jim Gauld <James.Gauld@windriver.com>
Date: Wed, 12 Oct 2022 13:59:48 -0400
Subject: [PATCH 2/5] Revert "kubeadm: apply the new "control-plane" taint
during upgrade"
This reverts commit db6061f5a6e6eb0002d572fa5ab1142e9b60bc1b.
---
cmd/kubeadm/app/cmd/upgrade/apply.go | 9 -----
cmd/kubeadm/app/phases/upgrade/postupgrade.go | 40 -------------------
2 files changed, 49 deletions(-)
diff --git a/cmd/kubeadm/app/cmd/upgrade/apply.go b/cmd/kubeadm/app/cmd/upgrade/apply.go
index 042f10796b6..3644146d804 100644
--- a/cmd/kubeadm/app/cmd/upgrade/apply.go
+++ b/cmd/kubeadm/app/cmd/upgrade/apply.go
@@ -163,15 +163,6 @@ func runApply(flags *applyFlags, args []string) error {
return err
}
- // TODO: https://github.com/kubernetes/kubeadm/issues/2200
- fmt.Printf("[upgrade/postupgrade] Adding the new taint %s to all control plane Nodes. "+
- "After this step both taints %s and %s should be present on control plane Nodes.\n",
- kubeadmconstants.ControlPlaneTaint.String(), kubeadmconstants.ControlPlaneTaint.String(),
- kubeadmconstants.OldControlPlaneTaint.String())
- if err := upgrade.AddNewControlPlaneTaint(client); err != nil {
- return err
- }
-
// Upgrade RBAC rules and addons.
klog.V(1).Infoln("[upgrade/postupgrade] upgrading RBAC rules and addons")
if err := upgrade.PerformPostUpgradeTasks(client, cfg, flags.dryRun); err != nil {
diff --git a/cmd/kubeadm/app/phases/upgrade/postupgrade.go b/cmd/kubeadm/app/phases/upgrade/postupgrade.go
index 55828597310..fa215bfedf8 100644
--- a/cmd/kubeadm/app/phases/upgrade/postupgrade.go
+++ b/cmd/kubeadm/app/phases/upgrade/postupgrade.go
@@ -238,46 +238,6 @@ func RemoveOldControlPlaneLabel(client clientset.Interface) error {
return nil
}
-// AddNewControlPlaneTaint finds all nodes with the new "control-plane" node-role label
-// and adds the new "control-plane" taint to them.
-// TODO: https://github.com/kubernetes/kubeadm/issues/2200
-func AddNewControlPlaneTaint(client clientset.Interface) error {
- selectorControlPlane := labels.SelectorFromSet(labels.Set(map[string]string{
- kubeadmconstants.LabelNodeRoleControlPlane: "",
- }))
- nodes, err := client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{
- LabelSelector: selectorControlPlane.String(),
- })
- if err != nil {
- return errors.Wrapf(err, "could not list nodes labeled with %q", kubeadmconstants.LabelNodeRoleControlPlane)
- }
-
- for _, n := range nodes.Items {
- // Check if the node has the old / new taints
- hasOldTaint := false
- hasNewTaint := false
- for _, t := range n.Spec.Taints {
- switch t.String() {
- case kubeadmconstants.OldControlPlaneTaint.String():
- hasOldTaint = true
- case kubeadmconstants.ControlPlaneTaint.String():
- hasNewTaint = true
- }
- }
- // If the old taint is present and the new taint is missing, patch the node with the new taint.
- // When the old taint is missing, assume the user has manually untainted the node and take no action.
- if !hasNewTaint && hasOldTaint {
- err = apiclient.PatchNode(client, n.Name, func(n *v1.Node) {
- n.Spec.Taints = append(n.Spec.Taints, kubeadmconstants.ControlPlaneTaint)
- })
- if err != nil {
- return err
- }
- }
- }
- return nil
-}
-
// UpdateKubeletDynamicEnvFileWithURLScheme reads the kubelet dynamic environment file
// from disk, ensure that the CRI endpoint flag has a scheme prefix and writes it
// back to disk.
--
2.25.1

144
kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/Revert-kubeadm-apply-the-new-control-plane-taint-on-.patch
View File

@ -1,144 +0,0 @@
From 7d898d974f2353b5984deb4ad42347726205696a Mon Sep 17 00:00:00 2001
From: Jim Gauld <James.Gauld@windriver.com>
Date: Wed, 12 Oct 2022 14:00:19 -0400
Subject: [PATCH 3/5] Revert "kubeadm: apply the new "control-plane" taint on
CP nodes"
This reverts commit 370031cadac6240e49e7b30a644d19735b7d3338.
---
cmd/kubeadm/app/apis/kubeadm/types.go | 6 +++---
cmd/kubeadm/app/apis/kubeadm/v1beta2/doc.go | 2 +-
cmd/kubeadm/app/apis/kubeadm/v1beta2/types.go | 6 +++---
cmd/kubeadm/app/apis/kubeadm/v1beta3/doc.go | 2 +-
cmd/kubeadm/app/apis/kubeadm/v1beta3/types.go | 6 +++---
cmd/kubeadm/app/util/config/initconfiguration.go | 2 +-
cmd/kubeadm/app/util/config/initconfiguration_test.go | 8 ++++----
7 files changed, 16 insertions(+), 16 deletions(-)
diff --git a/cmd/kubeadm/app/apis/kubeadm/types.go b/cmd/kubeadm/app/apis/kubeadm/types.go
index d49256908e7..55d1fd9a06f 100644
--- a/cmd/kubeadm/app/apis/kubeadm/types.go
+++ b/cmd/kubeadm/app/apis/kubeadm/types.go
@@ -218,9 +218,9 @@ type NodeRegistrationOptions struct {
// CRISocket is used to retrieve container runtime info. This information will be annotated to the Node API object, for later re-use
CRISocket string
- // Taints specifies the taints the Node API object should be registered with. If this field is unset, i.e. nil,
- // it will be defaulted with a control-plane taint for control-plane nodes. If you don't want to taint your control-plane
- // node, set this field to an empty slice, i.e. `taints: []` in the YAML file. This field is solely used for Node registration.
+ // Taints specifies the taints the Node API object should be registered with. If this field is unset, i.e. nil, in the `kubeadm init` process
+ // it will be defaulted to []v1.Taint{'node-role.kubernetes.io/master=""'}. If you don't want to taint your control-plane node, set this field to an
+ // empty slice, i.e. `taints: []` in the YAML file. This field is solely used for Node registration.
Taints []v1.Taint
// KubeletExtraArgs passes through extra arguments to the kubelet. The arguments here are passed to the kubelet command line via the environment file
diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta2/doc.go b/cmd/kubeadm/app/apis/kubeadm/v1beta2/doc.go
index 64ae2bb8f7c..24e58868dbe 100644
--- a/cmd/kubeadm/app/apis/kubeadm/v1beta2/doc.go
+++ b/cmd/kubeadm/app/apis/kubeadm/v1beta2/doc.go
@@ -172,7 +172,7 @@ limitations under the License.
// criSocket: "unix:///var/run/containerd/containerd.sock"
// taints:
// - key: "kubeadmNode"
-// value: "someValue"
+// value: "master"
// effect: "NoSchedule"
// kubeletExtraArgs:
// v: 4
diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta2/types.go b/cmd/kubeadm/app/apis/kubeadm/v1beta2/types.go
index a5cf40c513a..30037e30d40 100644
--- a/cmd/kubeadm/app/apis/kubeadm/v1beta2/types.go
+++ b/cmd/kubeadm/app/apis/kubeadm/v1beta2/types.go
@@ -201,9 +201,9 @@ type NodeRegistrationOptions struct {
// CRISocket is used to retrieve container runtime info. This information will be annotated to the Node API object, for later re-use
CRISocket string `json:"criSocket,omitempty"`
- // Taints specifies the taints the Node API object should be registered with. If this field is unset, i.e. nil,
- // it will be defaulted with a control-plane taint for control-plane nodes. If you don't want to taint your control-plane
- // node, set this field to an empty slice, i.e. `taints: []` in the YAML file. This field is solely used for Node registration.
+ // Taints specifies the taints the Node API object should be registered with. If this field is unset, i.e. nil, in the `kubeadm init` process
+ // it will be defaulted to []v1.Taint{'node-role.kubernetes.io/master=""'}. If you don't want to taint your control-plane node, set this field to an
+ // empty slice, i.e. `taints: []` in the YAML file. This field is solely used for Node registration.
Taints []v1.Taint `json:"taints"`
// KubeletExtraArgs passes through extra arguments to the kubelet. The arguments here are passed to the kubelet command line via the environment file
diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta3/doc.go b/cmd/kubeadm/app/apis/kubeadm/v1beta3/doc.go
index 8353359f91a..f7f34a7cffa 100644
--- a/cmd/kubeadm/app/apis/kubeadm/v1beta3/doc.go
+++ b/cmd/kubeadm/app/apis/kubeadm/v1beta3/doc.go
@@ -176,7 +176,7 @@ limitations under the License.
// criSocket: "unix:///var/run/containerd/containerd.sock"
// taints:
// - key: "kubeadmNode"
-// value: "someValue"
+// value: "master"
// effect: "NoSchedule"
// kubeletExtraArgs:
// v: 4
diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta3/types.go b/cmd/kubeadm/app/apis/kubeadm/v1beta3/types.go
index 5a5151bf64b..82ae10cc271 100644
--- a/cmd/kubeadm/app/apis/kubeadm/v1beta3/types.go
+++ b/cmd/kubeadm/app/apis/kubeadm/v1beta3/types.go
@@ -215,9 +215,9 @@ type NodeRegistrationOptions struct {
// +optional
CRISocket string `json:"criSocket,omitempty"`
- // Taints specifies the taints the Node API object should be registered with. If this field is unset, i.e. nil,
- // it will be defaulted with a control-plane taint for control-plane nodes. If you don't want to taint your control-plane
- // node, set this field to an empty slice, i.e. `taints: []` in the YAML file. This field is solely used for Node registration.
+ // Taints specifies the taints the Node API object should be registered with. If this field is unset, i.e. nil, in the `kubeadm init` process
+ // it will be defaulted to []v1.Taint{'node-role.kubernetes.io/master=""'}. If you don't want to taint your control-plane node, set this field to an
+ // empty slice, i.e. `taints: []` in the YAML file. This field is solely used for Node registration.
Taints []corev1.Taint `json:"taints"`
// KubeletExtraArgs passes through extra arguments to the kubelet. The arguments here are passed to the kubelet command line via the environment file
diff --git a/cmd/kubeadm/app/util/config/initconfiguration.go b/cmd/kubeadm/app/util/config/initconfiguration.go
index 7d1bb67fc02..71c3c514bf0 100644
--- a/cmd/kubeadm/app/util/config/initconfiguration.go
+++ b/cmd/kubeadm/app/util/config/initconfiguration.go
@@ -106,7 +106,7 @@ func SetNodeRegistrationDynamicDefaults(cfg *kubeadmapi.NodeRegistrationOptions,
// Only if the slice is nil, we should append the control-plane taint. This allows the user to specify an empty slice for no default control-plane taint
if controlPlaneTaint && cfg.Taints == nil {
// TODO: https://github.com/kubernetes/kubeadm/issues/2200
- cfg.Taints = []v1.Taint{kubeadmconstants.OldControlPlaneTaint, kubeadmconstants.ControlPlaneTaint}
+ cfg.Taints = []v1.Taint{kubeadmconstants.OldControlPlaneTaint}
}
if cfg.CRISocket == "" {
diff --git a/cmd/kubeadm/app/util/config/initconfiguration_test.go b/cmd/kubeadm/app/util/config/initconfiguration_test.go
index 074a1d821f4..93d7817a232 100644
--- a/cmd/kubeadm/app/util/config/initconfiguration_test.go
+++ b/cmd/kubeadm/app/util/config/initconfiguration_test.go
@@ -115,17 +115,17 @@ func TestDefaultTaintsMarshaling(t *testing.T) {
expectedTaintCnt int
}{
{
- desc: "Uninitialized nodeRegistration field produces expected taints",
+ desc: "Uninitialized nodeRegistration field produces a single taint (the master one)",
cfg: kubeadmapiv1.InitConfiguration{
TypeMeta: metav1.TypeMeta{
APIVersion: kubeadmapiv1.SchemeGroupVersion.String(),
Kind: constants.InitConfigurationKind,
},
},
- expectedTaintCnt: 2,
+ expectedTaintCnt: 1,
},
{
- desc: "Uninitialized taints field produces expected taints",
+ desc: "Uninitialized taints field produces a single taint (the master one)",
cfg: kubeadmapiv1.InitConfiguration{
TypeMeta: metav1.TypeMeta{
APIVersion: kubeadmapiv1.SchemeGroupVersion.String(),
@@ -133,7 +133,7 @@ func TestDefaultTaintsMarshaling(t *testing.T) {
},
NodeRegistration: kubeadmapiv1.NodeRegistrationOptions{},
},
- expectedTaintCnt: 2,
+ expectedTaintCnt: 1,
},
{
desc: "Forsing taints to an empty slice produces no taints",
--
2.25.1

119
kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/Revert-kubeadm-delete-the-old-master-label-during-up.patch
View File

@ -1,119 +0,0 @@
From 42c1abc4763c795b0c9bfb65b1ceba1fd2fa3427 Mon Sep 17 00:00:00 2001
From: Jim Gauld <James.Gauld@windriver.com>
Date: Wed, 12 Oct 2022 14:00:47 -0400
Subject: [PATCH 5/5] Revert "kubeadm: delete the old "master" label during
upgrade"
This reverts commit c0871b4433783a30c97e204b2011cf17d0457a62.
---
cmd/kubeadm/app/cmd/join.go | 2 +-
cmd/kubeadm/app/cmd/upgrade/apply.go | 7 +++----
cmd/kubeadm/app/phases/upgrade/health.go | 21 +++++++++++++++++--
cmd/kubeadm/app/phases/upgrade/postupgrade.go | 10 ++++++---
4 files changed, 30 insertions(+), 10 deletions(-)
diff --git a/cmd/kubeadm/app/cmd/join.go b/cmd/kubeadm/app/cmd/join.go
index fa0b07e3631..934a0951ff3 100644
--- a/cmd/kubeadm/app/cmd/join.go
+++ b/cmd/kubeadm/app/cmd/join.go
@@ -64,7 +64,7 @@ var (
* Certificate signing request was sent to apiserver and approval was received.
* The Kubelet was informed of the new secure connection details.
- * Control plane label and taint were applied to the new node.
+ * Control plane (master) label and taint were applied to the new node.
* The Kubernetes control plane instances scaled up.
{{.etcdMessage}}
diff --git a/cmd/kubeadm/app/cmd/upgrade/apply.go b/cmd/kubeadm/app/cmd/upgrade/apply.go
index 3644146d804..d49859aab9d 100644
--- a/cmd/kubeadm/app/cmd/upgrade/apply.go
+++ b/cmd/kubeadm/app/cmd/upgrade/apply.go
@@ -156,10 +156,9 @@ func runApply(flags *applyFlags, args []string) error {
}
// TODO: https://github.com/kubernetes/kubeadm/issues/2200
- fmt.Printf("[upgrade/postupgrade] Removing the deprecated label %s='' from all control plane Nodes. "+
- "After this step only the label %s='' will be present on control plane Nodes.\n",
- kubeadmconstants.LabelNodeRoleOldControlPlane, kubeadmconstants.LabelNodeRoleControlPlane)
- if err := upgrade.RemoveOldControlPlaneLabel(client); err != nil {
+ fmt.Printf("[upgrade/postupgrade] Applying label %s='' to Nodes with label %s='' (deprecated)\n",
+ kubeadmconstants.LabelNodeRoleControlPlane, kubeadmconstants.LabelNodeRoleOldControlPlane)
+ if err := upgrade.LabelOldControlPlaneNodes(client); err != nil {
return err
}
diff --git a/cmd/kubeadm/app/phases/upgrade/health.go b/cmd/kubeadm/app/phases/upgrade/health.go
index 55acdf865a6..b14bc6f3bbd 100644
--- a/cmd/kubeadm/app/phases/upgrade/health.go
+++ b/cmd/kubeadm/app/phases/upgrade/health.go
@@ -212,17 +212,34 @@ func deleteHealthCheckJob(client clientset.Interface, ns, jobName string) error
// controlPlaneNodesReady checks whether all control-plane Nodes in the cluster are in the Running state
func controlPlaneNodesReady(client clientset.Interface, _ *kubeadmapi.ClusterConfiguration) error {
+ // list nodes labeled with a "master" node-role
+ selectorOldControlPlane := labels.SelectorFromSet(labels.Set(map[string]string{
+ constants.LabelNodeRoleOldControlPlane: "",
+ }))
+ nodesWithOldLabel, err := client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{
+ LabelSelector: selectorOldControlPlane.String(),
+ })
+ if err != nil {
+ return errors.Wrapf(err, "could not list nodes labeled with %q", constants.LabelNodeRoleOldControlPlane)
+ }
+
+ // list nodes labeled with a "control-plane" node-role
selectorControlPlane := labels.SelectorFromSet(labels.Set(map[string]string{
constants.LabelNodeRoleControlPlane: "",
}))
- nodes, err := client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{
+ nodesControlPlane, err := client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{
LabelSelector: selectorControlPlane.String(),
})
if err != nil {
return errors.Wrapf(err, "could not list nodes labeled with %q", constants.LabelNodeRoleControlPlane)
}
- notReadyControlPlanes := getNotReadyNodes(nodes.Items)
+ nodes := append(nodesWithOldLabel.Items, nodesControlPlane.Items...)
+ if len(nodes) == 0 {
+ return errors.New("failed to find any nodes with a control-plane role")
+ }
+
+ notReadyControlPlanes := getNotReadyNodes(nodes)
if len(notReadyControlPlanes) != 0 {
return errors.Errorf("there are NotReady control-planes in the cluster: %v", notReadyControlPlanes)
}
diff --git a/cmd/kubeadm/app/phases/upgrade/postupgrade.go b/cmd/kubeadm/app/phases/upgrade/postupgrade.go
index fa215bfedf8..36e884195bc 100644
--- a/cmd/kubeadm/app/phases/upgrade/postupgrade.go
+++ b/cmd/kubeadm/app/phases/upgrade/postupgrade.go
@@ -214,9 +214,10 @@ func rollbackFiles(files map[string]string, originalErr error) error {
return errors.Errorf("couldn't move these files: %v. Got errors: %v", files, errorsutil.NewAggregate(errs))
}
-// RemoveOldControlPlaneLabel finds all nodes with the legacy node-role label and removes it
+// LabelOldControlPlaneNodes finds all nodes with the legacy node-role label and also applies
+// the "control-plane" node-role label to them.
// TODO: https://github.com/kubernetes/kubeadm/issues/2200
-func RemoveOldControlPlaneLabel(client clientset.Interface) error {
+func LabelOldControlPlaneNodes(client clientset.Interface) error {
selectorOldControlPlane := labels.SelectorFromSet(labels.Set(map[string]string{
kubeadmconstants.LabelNodeRoleOldControlPlane: "",
}))
@@ -228,8 +229,11 @@ func RemoveOldControlPlaneLabel(client clientset.Interface) error {
}
for _, n := range nodesWithOldLabel.Items {
+ if _, hasNewLabel := n.ObjectMeta.Labels[kubeadmconstants.LabelNodeRoleControlPlane]; hasNewLabel {
+ continue
+ }
err = apiclient.PatchNode(client, n.Name, func(n *v1.Node) {
- delete(n.ObjectMeta.Labels, kubeadmconstants.LabelNodeRoleOldControlPlane)
+ n.ObjectMeta.Labels[kubeadmconstants.LabelNodeRoleControlPlane] = ""
})
if err != nil {
return err
--
2.25.1

123
kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/Revert-kubeadm-only-apply-the-new-control-plane-labe.patch
View File

@ -1,123 +0,0 @@
From 0119bd093db1d72ebf4fa00c4869979da2f9e2a1 Mon Sep 17 00:00:00 2001
From: Jim Gauld <James.Gauld@windriver.com>
Date: Wed, 12 Oct 2022 14:00:33 -0400
Subject: [PATCH 4/5] Revert "kubeadm: only apply the new "control-plane" label
during init/join"
This reverts commit a3d5e5598290df09f4ffd5cf6813653a346c8f4c.
---
.../phases/markcontrolplane/markcontrolplane.go | 14 +++++++++++---
.../markcontrolplane/markcontrolplane_test.go | 9 ++++++---
cmd/kubeadm/app/util/staticpod/utils.go | 7 +++++++
3 files changed, 24 insertions(+), 6 deletions(-)
diff --git a/cmd/kubeadm/app/phases/markcontrolplane/markcontrolplane.go b/cmd/kubeadm/app/phases/markcontrolplane/markcontrolplane.go
index dd4c89eca6d..c68f80f7fd6 100644
--- a/cmd/kubeadm/app/phases/markcontrolplane/markcontrolplane.go
+++ b/cmd/kubeadm/app/phases/markcontrolplane/markcontrolplane.go
@@ -19,23 +19,31 @@ package markcontrolplane
import (
"fmt"
- v1 "k8s.io/api/core/v1"
+ "k8s.io/api/core/v1"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
)
-// labelsToAdd holds a list of labels that are applied on kubeadm managed control plane nodes
var labelsToAdd = []string{
+ // TODO: remove this label:
+ // https://github.com/kubernetes/kubeadm/issues/2200
+ constants.LabelNodeRoleOldControlPlane,
constants.LabelNodeRoleControlPlane,
constants.LabelExcludeFromExternalLB,
}
// MarkControlPlane taints the control-plane and sets the control-plane label
func MarkControlPlane(client clientset.Interface, controlPlaneName string, taints []v1.Taint) error {
+ // TODO: remove this "deprecated" amend and pass "labelsToAdd" directly:
+ // https://github.com/kubernetes/kubeadm/issues/2200
+ labels := make([]string, len(labelsToAdd))
+ copy(labels, labelsToAdd)
+ labels[0] = constants.LabelNodeRoleOldControlPlane + "(deprecated)"
+
fmt.Printf("[mark-control-plane] Marking the node %s as control-plane by adding the labels: %v\n",
- controlPlaneName, labelsToAdd)
+ controlPlaneName, labels)
if len(taints) > 0 {
taintStrs := []string{}
diff --git a/cmd/kubeadm/app/phases/markcontrolplane/markcontrolplane_test.go b/cmd/kubeadm/app/phases/markcontrolplane/markcontrolplane_test.go
index b171072f80a..cb9275124ab 100644
--- a/cmd/kubeadm/app/phases/markcontrolplane/markcontrolplane_test.go
+++ b/cmd/kubeadm/app/phases/markcontrolplane/markcontrolplane_test.go
@@ -49,25 +49,26 @@ func TestMarkControlPlane(t *testing.T) {
existingLabels: []string{""},
existingTaints: nil,
newTaints: []v1.Taint{kubeadmconstants.OldControlPlaneTaint},
- expectedPatch: `{"metadata":{"labels":{"node-role.kubernetes.io/control-plane":"","node.kubernetes.io/exclude-from-external-load-balancers":""}},"spec":{"taints":[{"effect":"NoSchedule","key":"node-role.kubernetes.io/master"}]}}`,
+ expectedPatch: `{"metadata":{"labels":{"node-role.kubernetes.io/control-plane":"","node-role.kubernetes.io/master":"","node.kubernetes.io/exclude-from-external-load-balancers":""}},"spec":{"taints":[{"effect":"NoSchedule","key":"node-role.kubernetes.io/master"}]}}`,
},
{
name: "control-plane label and taint missing but taint not wanted",
existingLabels: []string{""},
existingTaints: nil,
newTaints: nil,
- expectedPatch: `{"metadata":{"labels":{"node-role.kubernetes.io/control-plane":"","node.kubernetes.io/exclude-from-external-load-balancers":""}}}`,
+ expectedPatch: `{"metadata":{"labels":{"node-role.kubernetes.io/control-plane":"","node-role.kubernetes.io/master":"","node.kubernetes.io/exclude-from-external-load-balancers":""}}}`,
},
{
name: "control-plane label missing",
existingLabels: []string{""},
existingTaints: []v1.Taint{kubeadmconstants.OldControlPlaneTaint},
newTaints: []v1.Taint{kubeadmconstants.OldControlPlaneTaint},
- expectedPatch: `{"metadata":{"labels":{"node-role.kubernetes.io/control-plane":"","node.kubernetes.io/exclude-from-external-load-balancers":""}}}`,
+ expectedPatch: `{"metadata":{"labels":{"node-role.kubernetes.io/control-plane":"","node-role.kubernetes.io/master":"","node.kubernetes.io/exclude-from-external-load-balancers":""}}}`,
},
{
name: "control-plane taint missing",
existingLabels: []string{
+ kubeadmconstants.LabelNodeRoleOldControlPlane,
kubeadmconstants.LabelNodeRoleControlPlane,
kubeadmconstants.LabelExcludeFromExternalLB,
},
@@ -78,6 +79,7 @@ func TestMarkControlPlane(t *testing.T) {
{
name: "nothing missing",
existingLabels: []string{
+ kubeadmconstants.LabelNodeRoleOldControlPlane,
kubeadmconstants.LabelNodeRoleControlPlane,
kubeadmconstants.LabelExcludeFromExternalLB,
},
@@ -88,6 +90,7 @@ func TestMarkControlPlane(t *testing.T) {
{
name: "has taint and no new taints wanted",
existingLabels: []string{
+ kubeadmconstants.LabelNodeRoleOldControlPlane,
kubeadmconstants.LabelNodeRoleControlPlane,
kubeadmconstants.LabelExcludeFromExternalLB,
},
diff --git a/cmd/kubeadm/app/util/staticpod/utils.go b/cmd/kubeadm/app/util/staticpod/utils.go
index 75efc4f0f98..f0d8fed0d46 100644
--- a/cmd/kubeadm/app/util/staticpod/utils.go
+++ b/cmd/kubeadm/app/util/staticpod/utils.go
@@ -287,6 +287,13 @@ func createHTTPProbe(host, path string, port int, scheme v1.URIScheme, initialDe
// GetAPIServerProbeAddress returns the probe address for the API server
func GetAPIServerProbeAddress(endpoint *kubeadmapi.APIEndpoint) string {
+ // In the case of a self-hosted deployment, the initial host on which kubeadm --init is run,
+ // will generate a DaemonSet with a nodeSelector such that all nodes with the label
+ // node-role.kubernetes.io/master='' will have the API server deployed to it. Since the init
+ // is run only once on an initial host, the API advertise address will be invalid for any
+ // future hosts that do not have the same address. Furthermore, since liveness and readiness
+ // probes do not support the Downward API we cannot dynamically set the advertise address to
+ // the node's IP. The only option then is to use localhost.
if endpoint != nil && endpoint.AdvertiseAddress != "" {
return getProbeAddress(endpoint.AdvertiseAddress)
}
--
2.25.1

113
kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/Revert-use-subpath-for-coredns-only-for-default-repo.patch
View File

@ -1,113 +0,0 @@
From f9dd597f4e8c8c66f08d661efcbd29479e4e069d Mon Sep 17 00:00:00 2001
From: Gleb Aronsky <gleb.aronsky@windriver.com>
Date: Tue, 25 Jan 2022 13:56:30 -0500
Subject: [PATCH] Revert "use subpath for coredns only for default repository"
This reverts commit 38a41e1557649a7cc763bf737779db9aa03ec75e.
Co-authored-by: Jim Gauld <james.gauld@windriver.com>
Signed-off-by: Gleb Aronsky <gleb.aronsky@windriver.com>
diff --git a/cmd/kubeadm/app/constants/constants.go b/cmd/kubeadm/app/constants/constants.go
index c2b8f6e64be..b00ccea315e 100644
--- a/cmd/kubeadm/app/constants/constants.go
+++ b/cmd/kubeadm/app/constants/constants.go
@@ -337,7 +337,7 @@ const (
CoreDNSDeploymentName = "coredns"
// CoreDNSImageName specifies the name of the image for CoreDNS add-on
- CoreDNSImageName = "coredns"
+ CoreDNSImageName = "coredns/coredns"
// CoreDNSVersion is the version of CoreDNS to be deployed if it is used
CoreDNSVersion = "v1.8.6"
diff --git a/cmd/kubeadm/app/images/images.go b/cmd/kubeadm/app/images/images.go
index ee55eb6c995..bdb61caa373 100644
--- a/cmd/kubeadm/app/images/images.go
+++ b/cmd/kubeadm/app/images/images.go
@@ -22,7 +22,6 @@ import (
"k8s.io/klog/v2"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
- kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
)
@@ -48,10 +47,6 @@ func GetDNSImage(cfg *kubeadmapi.ClusterConfiguration) string {
if cfg.DNS.ImageRepository != "" {
dnsImageRepository = cfg.DNS.ImageRepository
}
- // Handle the renaming of the official image from "k8s.gcr.io/coredns" to "k8s.gcr.io/coredns/coredns
- if dnsImageRepository == kubeadmapiv1beta2.DefaultImageRepository {
- dnsImageRepository = fmt.Sprintf("%s/coredns", dnsImageRepository)
- }
// DNS uses an imageTag that corresponds to the DNS version matching the Kubernetes version
dnsImageTag := constants.CoreDNSVersion
diff --git a/cmd/kubeadm/app/images/images_test.go b/cmd/kubeadm/app/images/images_test.go
index 2b8affce236..91cd4294351 100644
--- a/cmd/kubeadm/app/images/images_test.go
+++ b/cmd/kubeadm/app/images/images_test.go
@@ -22,7 +22,6 @@ import (
"testing"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
- kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
)
@@ -227,51 +226,4 @@ func TestGetAllImages(t *testing.T) {
}
}
-func TestGetDNSImage(t *testing.T) {
- var tests = []struct {
- expected string
- cfg *kubeadmapi.ClusterConfiguration
- }{
- {
- expected: "foo.io/coredns:v1.8.6",
- cfg: &kubeadmapi.ClusterConfiguration{
- ImageRepository: "foo.io",
- DNS: kubeadmapi.DNS{
- Type: kubeadmapi.CoreDNS,
- },
- },
- },
- {
- expected: kubeadmapiv1beta2.DefaultImageRepository + "/coredns/coredns:v1.8.6",
- cfg: &kubeadmapi.ClusterConfiguration{
- ImageRepository: kubeadmapiv1beta2.DefaultImageRepository,
- DNS: kubeadmapi.DNS{
- Type: kubeadmapi.CoreDNS,
- },
- },
- },
- {
- expected: "foo.io/coredns/coredns:v1.8.6",
- cfg: &kubeadmapi.ClusterConfiguration{
- ImageRepository: "foo.io",
- DNS: kubeadmapi.DNS{
- Type: kubeadmapi.CoreDNS,
- ImageMeta: kubeadmapi.ImageMeta{
- ImageRepository: "foo.io/coredns",
- },
- },
- },
- },
- }
-
- for _, test := range tests {
- actual := GetDNSImage(test.cfg)
- if actual != test.expected {
- t.Errorf(
- "failed to GetDNSImage:\n\texpected: %s\n\t actual: %s",
- test.expected,
- actual,
- )
- }
- }
}
--
2.25.1

85
kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/cpumanager-policy-static-test-refactor.patch
View File

@ -1,85 +0,0 @@
From 9ede7db445ff799d78fae4c20d9558962573ede7 Mon Sep 17 00:00:00 2001
From: Sachin Gopala Krishna <saching.krishna@windriver.com>
Date: Tue, 11 Oct 2022 09:09:00 -0400
Subject: [PATCH] cpumanager policy static test refactor
This refactors the tests which were breaking due to changes in isolated
CPUs and reserved CPUs.
Signed-off-by: Sachin Gopala Krishna <saching.krishna@windriver.com>
---
pkg/kubelet/cm/cpumanager/cpu_manager_test.go | 11 ++++++++---
pkg/kubelet/cm/cpumanager/policy_static_test.go | 5 +++--
2 files changed, 11 insertions(+), 5 deletions(-)
diff --git a/pkg/kubelet/cm/cpumanager/cpu_manager_test.go b/pkg/kubelet/cm/cpumanager/cpu_manager_test.go
index 31e4d0585fb..87c4ae036ca 100644
--- a/pkg/kubelet/cm/cpumanager/cpu_manager_test.go
+++ b/pkg/kubelet/cm/cpumanager/cpu_manager_test.go
@@ -715,6 +715,8 @@ func TestCPUManagerRemove(t *testing.T) {
}
func TestReconcileState(t *testing.T) {
+ testExcl := false
+ testDM, _ := devicemanager.NewManagerStub()
testPolicy, _ := NewStaticPolicy(
&topology.CPUTopology{
NumCPUs: 8,
@@ -733,8 +735,11 @@ func TestReconcileState(t *testing.T) {
},
0,
cpuset.NewCPUSet(),
+ cpuset.NewCPUSet(),
topologymanager.NewFakeManager(),
- nil)
+ nil,
+ testDM,
+ testExcl)
testCases := []struct {
description string
@@ -1369,8 +1374,8 @@ func TestCPUManagerHandlePolicyOptions(t *testing.T) {
t.Errorf("cannot create state file: %s", err.Error())
}
defer os.RemoveAll(sDir)
-
- _, err = NewManager(testCase.cpuPolicyName, testCase.cpuPolicyOptions, 5*time.Second, machineInfo, cpuset.NewCPUSet(), nodeAllocatableReservation, sDir, topologymanager.NewFakeManager())
+ testDM, err := devicemanager.NewManagerStub()
+ _, err = NewManager(testCase.cpuPolicyName, testCase.cpuPolicyOptions, 5*time.Second, machineInfo, cpuset.NewCPUSet(), nodeAllocatableReservation, sDir, topologymanager.NewFakeManager(), testDM)
if err == nil {
t.Errorf("Expected error, but NewManager succeeded")
}
diff --git a/pkg/kubelet/cm/cpumanager/policy_static_test.go b/pkg/kubelet/cm/cpumanager/policy_static_test.go
index 39aaaf95b28..d0308556c6d 100644
--- a/pkg/kubelet/cm/cpumanager/policy_static_test.go
+++ b/pkg/kubelet/cm/cpumanager/policy_static_test.go
@@ -202,7 +202,6 @@ func TestStaticPolicyAdd(t *testing.T) {
largeTopoCPUSet := largeTopoBuilder.Result()
largeTopoSock0CPUSet := largeTopoSock0Builder.Result()
largeTopoSock1CPUSet := largeTopoSock1Builder.Result()
- testDM, _ := devicemanager.NewManagerStub()
// these are the cases which must behave the same regardless the policy options.
// So we will permutate the options to ensure this holds true.
optionsInsensitiveTestCases := []staticPolicyTest{
@@ -576,6 +575,8 @@ func runStaticPolicyTestCase(t *testing.T, testCase staticPolicyTest) {
}
func TestStaticPolicyReuseCPUs(t *testing.T) {
+ excludeReserved := false
+ testDM, _ := devicemanager.NewManagerStub()
testCases := []struct {
staticPolicyTest
expCSetAfterAlloc cpuset.CPUSet
@@ -692,7 +693,7 @@ func TestStaticPolicyRemove(t *testing.T) {
}
for _, testCase := range testCases {
- policy, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil)
+ policy, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, cpuset.NewCPUSet(), cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil, testDM, excludeReserved)
st := &mockState{
assignments: testCase.stAssignments,
--
2.25.1

169
kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/kubeadm-create-platform-pods-with-zero-CPU-resources.patch
View File

@ -1,169 +0,0 @@
From 099b906fe716e4606bbb405632e0a80503d4a708 Mon Sep 17 00:00:00 2001
From: Chris Friesen <chris.friesen@windriver.com>
Date: Fri, 3 Sep 2021 18:05:15 -0400
Subject: [PATCH] kubeadm: create platform pods with zero CPU resources
This specifies zero CPU resources when creating the manifests
for the static platform pods, as a workaround for the lack of
separate resource tracking for platform resources.
This specifies zero CPU and Memory resources for the coredns
deployment. manifests.go is the main source file for this,
not sure if the coredns.yaml are used but they are updated to
be consistent.
This specifies CPU limit of 1 for kube-apiserver pod so that it is
treated as a burstable QoS. This gives a boost of cgroup CPUShares
since the burstable cgroup parent has significantly more CPUShares
than best-effort on typical systems. This improves kube-apiserver
API responsiveness.
This increases kube-apiserver Readiness probe periodSeconds to 10
based on WRS/SS joint recommendation for minimum probe settings.
This reduces likelihood of kube-apiserver probe failure and
subsequent pod-restart under servere load. This also reduces CPU
demand.
Signed-off-by: Daniel Safta <daniel.safta@windriver.com>
Signed-off-by: Boovan Rajendran <boovan.rajendran@windriver.com>
Signed-off-by: Jim Gauld <James.Gauld@windriver.com>
---
cluster/addons/dns/coredns/coredns.yaml.base | 4 ++--
cluster/addons/dns/coredns/coredns.yaml.in | 4 ++--
cluster/addons/dns/coredns/coredns.yaml.sed | 4 ++--
cmd/kubeadm/app/phases/addons/dns/manifests.go | 4 ++--
.../app/phases/controlplane/manifests.go | 8 +++++---
cmd/kubeadm/app/util/staticpod/utils.go | 17 ++++++++++++++++-
6 files changed, 29 insertions(+), 12 deletions(-)
diff --git a/cluster/addons/dns/coredns/coredns.yaml.base b/cluster/addons/dns/coredns/coredns.yaml.base
index dea3749f217..908fd2b246a 100644
--- a/cluster/addons/dns/coredns/coredns.yaml.base
+++ b/cluster/addons/dns/coredns/coredns.yaml.base
@@ -145,8 +145,8 @@ spec:
limits:
memory: __DNS__MEMORY__LIMIT__
requests:
- cpu: 100m
- memory: 70Mi
+ cpu: 0
+ memory: 0
args: [ "-conf", "/etc/coredns/Corefile" ]
volumeMounts:
- name: config-volume
diff --git a/cluster/addons/dns/coredns/coredns.yaml.in b/cluster/addons/dns/coredns/coredns.yaml.in
index 808dd54b545..1bc9c632a3a 100644
--- a/cluster/addons/dns/coredns/coredns.yaml.in
+++ b/cluster/addons/dns/coredns/coredns.yaml.in
@@ -145,8 +145,8 @@ spec:
limits:
memory: 'dns_memory_limit'
requests:
- cpu: 100m
- memory: 70Mi
+ cpu: 0
+ memory: 0
args: [ "-conf", "/etc/coredns/Corefile" ]
volumeMounts:
- name: config-volume
diff --git a/cluster/addons/dns/coredns/coredns.yaml.sed b/cluster/addons/dns/coredns/coredns.yaml.sed
index 8788f41d31c..5be046ad844 100644
--- a/cluster/addons/dns/coredns/coredns.yaml.sed
+++ b/cluster/addons/dns/coredns/coredns.yaml.sed
@@ -145,8 +145,8 @@ spec:
limits:
memory: $DNS_MEMORY_LIMIT
requests:
- cpu: 100m
- memory: 70Mi
+ cpu: 0
+ memory: 0
args: [ "-conf", "/etc/coredns/Corefile" ]
volumeMounts:
- name: config-volume
diff --git a/cmd/kubeadm/app/phases/addons/dns/manifests.go b/cmd/kubeadm/app/phases/addons/dns/manifests.go
index 97c7f8b3e60..87df378cfc0 100644
--- a/cmd/kubeadm/app/phases/addons/dns/manifests.go
+++ b/cmd/kubeadm/app/phases/addons/dns/manifests.go
@@ -95,8 +95,8 @@ spec:
limits:
memory: 170Mi
requests:
- cpu: 100m
- memory: 70Mi
+ cpu: 0
+ memory: 0
args: [ "-conf", "/etc/coredns/Corefile" ]
volumeMounts:
- name: config-volume
diff --git a/cmd/kubeadm/app/phases/controlplane/manifests.go b/cmd/kubeadm/app/phases/controlplane/manifests.go
index 8cd0bf577cd..03e5739f5dc 100644
--- a/cmd/kubeadm/app/phases/controlplane/manifests.go
+++ b/cmd/kubeadm/app/phases/controlplane/manifests.go
@@ -63,7 +63,9 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap
LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/livez", int(endpoint.BindPort), v1.URISchemeHTTPS),
ReadinessProbe: staticpodutil.ReadinessProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/readyz", int(endpoint.BindPort), v1.URISchemeHTTPS),
StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/livez", int(endpoint.BindPort), v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane),
- Resources: staticpodutil.ComponentResources("250m"),
+ // WRS: Increase kube-apiserver cgroup CPUShares to improve API responsiveness;
+ // achieved by setting CPU Limits to make it burstable QoS.
+ Resources: staticpodutil.ComponentLimitResources("0", "1"),
Env: kubeadmutil.GetProxyEnvVars(),
}, mounts.GetVolumes(kubeadmconstants.KubeAPIServer),
map[string]string{kubeadmconstants.KubeAPIServerAdvertiseAddressEndpointAnnotationKey: endpoint.String()}),
@@ -75,7 +77,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap
VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeControllerManager)),
LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetControllerManagerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeControllerManagerPort, v1.URISchemeHTTPS),
StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetControllerManagerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeControllerManagerPort, v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane),
- Resources: staticpodutil.ComponentResources("200m"),
+ Resources: staticpodutil.ComponentResources("0"),
Env: kubeadmutil.GetProxyEnvVars(),
}, mounts.GetVolumes(kubeadmconstants.KubeControllerManager), nil),
kubeadmconstants.KubeScheduler: staticpodutil.ComponentPod(v1.Container{
@@ -86,7 +88,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap
VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeScheduler)),
LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetSchedulerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeSchedulerPort, v1.URISchemeHTTPS),
StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetSchedulerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeSchedulerPort, v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane),
- Resources: staticpodutil.ComponentResources("100m"),
+ Resources: staticpodutil.ComponentResources("0"),
Env: kubeadmutil.GetProxyEnvVars(),
}, mounts.GetVolumes(kubeadmconstants.KubeScheduler), nil),
}
diff --git a/cmd/kubeadm/app/util/staticpod/utils.go b/cmd/kubeadm/app/util/staticpod/utils.go
index 75efc4f0f98..46fa7a006ba 100644
--- a/cmd/kubeadm/app/util/staticpod/utils.go
+++ b/cmd/kubeadm/app/util/staticpod/utils.go
@@ -92,6 +92,18 @@ func ComponentResources(cpu string) v1.ResourceRequirements {
}
}
+// ComponentLimitResources returns the v1.ResourceRequirements object needed for allocating a specified amount of the CPU with Limits
+func ComponentLimitResources(cpu string, lcpu string) v1.ResourceRequirements {
+ return v1.ResourceRequirements{
+ Requests: v1.ResourceList{
+ v1.ResourceCPU: resource.MustParse(cpu),
+ },
+ Limits: v1.ResourceList{
+ v1.ResourceCPU: resource.MustParse(lcpu),
+ },
+ }
+}
+
// NewVolume creates a v1.Volume with a hostPath mount to the specified location
func NewVolume(name, path string, pathType *v1.HostPathType) v1.Volume {
return v1.Volume{
@@ -252,7 +264,10 @@ func LivenessProbe(host, path string, port int, scheme v1.URIScheme) *v1.Probe {
func ReadinessProbe(host, path string, port int, scheme v1.URIScheme) *v1.Probe {
// sets initialDelaySeconds as '0' because we don't want to delay user infrastructure checks
// looking for "ready" status on kubeadm static Pods
- return createHTTPProbe(host, path, port, scheme, 0, 15, 3, 1)
+ // WRS/SS joint recommendation: All pods probes should have following minimum probe
+ // settings unless required by the service (initialDelaySecond 0, periodSeconds 10,
+ // timeoutSeconds 5, successThreshold 1, failureThreshold 3)
+ return createHTTPProbe(host, path, port, scheme, 0, 15, 3, 10)
}
// StartupProbe creates a Probe object with a HTTPGet handler
--
2.25.1

67
kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/kubeadm-readiness-probe-timeout-core-dns.patch
View File

@ -1,67 +0,0 @@
From 583589c3574ffd6e0376579316b30d2a2dcf82f8 Mon Sep 17 00:00:00 2001
From: Ferdinando Terada <Ferdinando.GodoyTerada@windriver.com>
Date: Mon, 23 Dec 2024 17:53:09 -0300
Subject: [PATCH] Adjust timeout for coredns readinessProbe
The timeout value for the readinessProbe of CoreDNS was increased.
This adjustment was necessary to avoid issues during stress testing,
ensuring that the component can properly handle high-load situations
and prevent premature failure in readiness checks.
---
cluster/addons/dns/coredns/coredns.yaml.base | 1 +
cluster/addons/dns/coredns/coredns.yaml.in | 1 +
cluster/addons/dns/coredns/coredns.yaml.sed | 1 +
cmd/kubeadm/app/phases/addons/dns/manifests.go | 1 +
4 files changed, 4 insertions(+)
diff --git a/cluster/addons/dns/coredns/coredns.yaml.base b/cluster/addons/dns/coredns/coredns.yaml.base
index 3a0fd7adb72..c8289f7c136 100644
--- a/cluster/addons/dns/coredns/coredns.yaml.base
+++ b/cluster/addons/dns/coredns/coredns.yaml.base
@@ -170,6 +170,7 @@ spec:
path: /ready
port: 8181
scheme: HTTP
+ timeoutSeconds: 5
securityContext:
allowPrivilegeEscalation: false
capabilities:
diff --git a/cluster/addons/dns/coredns/coredns.yaml.in b/cluster/addons/dns/coredns/coredns.yaml.in
index 74b59584bc7..974c8337031 100644
--- a/cluster/addons/dns/coredns/coredns.yaml.in
+++ b/cluster/addons/dns/coredns/coredns.yaml.in
@@ -170,6 +170,7 @@ spec:
path: /ready
port: 8181
scheme: HTTP
+ timeoutSeconds: 5
securityContext:
allowPrivilegeEscalation: false
capabilities:
diff --git a/cluster/addons/dns/coredns/coredns.yaml.sed b/cluster/addons/dns/coredns/coredns.yaml.sed
index 61afbecd9da..563a8980e07 100644
--- a/cluster/addons/dns/coredns/coredns.yaml.sed
+++ b/cluster/addons/dns/coredns/coredns.yaml.sed
@@ -170,6 +170,7 @@ spec:
path: /ready
port: 8181
scheme: HTTP
+ timeoutSeconds: 5
securityContext:
allowPrivilegeEscalation: false
capabilities:
diff --git a/cmd/kubeadm/app/phases/addons/dns/manifests.go b/cmd/kubeadm/app/phases/addons/dns/manifests.go
index 2a2212d5d37..c0be57357e4 100644
--- a/cmd/kubeadm/app/phases/addons/dns/manifests.go
+++ b/cmd/kubeadm/app/phases/addons/dns/manifests.go
@@ -135,6 +135,7 @@ spec:
path: /ready
port: 8181
scheme: HTTP
+ timeoutSeconds: 5
securityContext:
allowPrivilegeEscalation: false
capabilities:
--
2.34.1

33
kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/kubeadm-reduce-UpgradeManifestTimeout.patch
View File

@ -1,33 +0,0 @@
From 52519a7549da1466d486c19dc5430850bcfdefee Mon Sep 17 00:00:00 2001
From: Ramesh Kumar Sivanandam <rameshkumar.sivanandam@windriver.com>
Date: Fri, 15 Mar 2024 03:28:13 -0400
Subject: [PATCH] kubeadm: reduce UpgradeManifestTimeout
This modifies kubeadm UpgradeManifestTimeout from 5 minutes default
to 3 minutes to reduce the unnecessary delay in retries during
kubeadm-upgrade-apply failures.
The typical control-plane upgrade of static pods is 75 to 85 seconds,
so 3 minutes gives adequate buffer to complete the operation.
Signed-off-by: Ramesh Kumar Sivanandam <rameshkumar.sivanandam@windriver.com>
---
cmd/kubeadm/app/phases/upgrade/staticpods.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/cmd/kubeadm/app/phases/upgrade/staticpods.go b/cmd/kubeadm/app/phases/upgrade/staticpods.go
index 524a624c34f..37562342368 100644
--- a/cmd/kubeadm/app/phases/upgrade/staticpods.go
+++ b/cmd/kubeadm/app/phases/upgrade/staticpods.go
@@ -46,7 +46,7 @@ import (
const (
// UpgradeManifestTimeout is timeout of upgrading the static pod manifest
- UpgradeManifestTimeout = 5 * time.Minute
+ UpgradeManifestTimeout = 3 * time.Minute
)
// StaticPodPathManager is responsible for tracking the directories used in the static pod upgrade transition
--
2.25.1

30
kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/kubelet-CFS-quota-throttling-for-non-integer-cpulimit.patch
View File

@ -1,30 +0,0 @@
From 2e957044cbcde858abb9c46d177d5cf4ae1407df Mon Sep 17 00:00:00 2001
From: Boovan Rajendran <boovan.rajendran@windriver.com>
Date: Wed, 30 Nov 2022 04:17:19 -0500
Subject: [PATCH] kubelet CFS quota throttling for non integer cpulimit
Signed-off-by: Boovan Rajendran <boovan.rajendran@windriver.com>
---
pkg/kubelet/cm/internal_container_lifecycle_linux.go | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/pkg/kubelet/cm/internal_container_lifecycle_linux.go b/pkg/kubelet/cm/internal_container_lifecycle_linux.go
index 75406dd8564..05366ab6fcb 100644
--- a/pkg/kubelet/cm/internal_container_lifecycle_linux.go
+++ b/pkg/kubelet/cm/internal_container_lifecycle_linux.go
@@ -39,7 +39,11 @@ func (i *internalContainerLifecycleImpl) PreCreateContainer(pod *v1.Pod, contain
// Disable cgroup CFS throttle at the container level.
// /sys/fs/cgroup/cpu/k8s-infra/kubepods/<pod>/<container>/cpu.cfs_quota_us
// /sys/fs/cgroup/cpu/k8s-infra/kubepods/<pod>/<container>/cpu.cfs_period_us
- if i.cpuManager.GetCPUPolicy() == "static" && v1qos.GetPodQOS(pod) == v1.PodQOSGuaranteed {
+ // We can only set CpuQuota to -1 if we're allocating the entire CPU.
+ // For fractional CPUs the CpuQuota is needed to enforce the limit.
+ cpuQuantity := container.Resources.Requests[v1.ResourceCPU]
+ fractionalCpuQuantity := cpuQuantity.MilliValue()%1000
+ if i.cpuManager.GetCPUPolicy() == "static" && v1qos.GetPodQOS(pod) == v1.PodQOSGuaranteed && fractionalCpuQuantity == 0 {
containerConfig.Linux.Resources.CpuPeriod = int64(100000)
containerConfig.Linux.Resources.CpuQuota = int64(-1)
}
--
2.25.1

255
kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/kubelet-cpumanager-disable-CFS-quota-throttling.patch
View File

@ -1,255 +0,0 @@
From 95e547b2d3d0af6b0f2083c064bcbdbe39716250 Mon Sep 17 00:00:00 2001
From: Sachin Gopala Krishna <saching.krishna@windriver.com>
Date: Mon, 3 Oct 2022 19:19:48 -0400
Subject: [PATCH] kubelet cpumanager disable CFS quota throttling
This disables CFS CPU quota to avoid performance degradation due to
Linux kernel CFS quota implementation. Note that 4.18 kernel attempts
to solve the CFS throttling problem, but there are reports that it is
not completely effective.
This disables CFS quota throttling for Guaranteed pods for both
parent and container cgroups by writing -1 to cgroup cpu.cfs_quota_us.
Disabling has a dramatic latency improvement for HTTP response times.
This patch is refactored in 1.22.5 due to new internal_container_lifecycle
framework. We leverage the same mechanism to set Linux resources as:
cpu manager: specify the container CPU set during the creation
Co-authored-by: Jim Gauld <james.gauld@windriver.com>
Signed-off-by: Sachin Gopala Krishna <saching.krishna@windriver.com>
---
pkg/kubelet/cm/cpumanager/cpu_manager.go | 7 +++
pkg/kubelet/cm/cpumanager/fake_cpu_manager.go | 10 ++++-
pkg/kubelet/cm/helpers_linux.go | 10 +++++
pkg/kubelet/cm/helpers_linux_test.go | 43 ++++++++++---------
.../cm/internal_container_lifecycle_linux.go | 9 ++++
5 files changed, 57 insertions(+), 22 deletions(-)
diff --git a/pkg/kubelet/cm/cpumanager/cpu_manager.go b/pkg/kubelet/cm/cpumanager/cpu_manager.go
index dde49b6ec8c..df431b06601 100644
--- a/pkg/kubelet/cm/cpumanager/cpu_manager.go
+++ b/pkg/kubelet/cm/cpumanager/cpu_manager.go
@@ -72,6 +72,9 @@ type Manager interface {
// State returns a read-only interface to the internal CPU manager state.
State() state.Reader
+ // GetCPUPolicy returns the assigned CPU manager policy
+ GetCPUPolicy() string
+
// GetTopologyHints implements the topologymanager.HintProvider Interface
// and is consulted to achieve NUMA aware resource alignment among this
// and other resource controllers.
@@ -314,6 +317,10 @@ func (m *manager) State() state.Reader {
return m.state
}
+func (m *manager) GetCPUPolicy() string {
+ return m.policy.Name()
+}
+
func (m *manager) GetTopologyHints(pod *v1.Pod, container *v1.Container) map[string][]topologymanager.TopologyHint {
// The pod is during the admission phase. We need to save the pod to avoid it
// being cleaned before the admission ended
diff --git a/pkg/kubelet/cm/cpumanager/fake_cpu_manager.go b/pkg/kubelet/cm/cpumanager/fake_cpu_manager.go
index 93369705135..2e277da9c84 100644
--- a/pkg/kubelet/cm/cpumanager/fake_cpu_manager.go
+++ b/pkg/kubelet/cm/cpumanager/fake_cpu_manager.go
@@ -28,7 +28,8 @@ import (
)
type fakeManager struct {
- state state.State
+ policy Policy
+ state state.State
}
func (m *fakeManager) Start(activePods ActivePodsFunc, sourcesReady config.SourcesReady, podStatusProvider status.PodStatusProvider, containerRuntime runtimeService, initialContainers containermap.ContainerMap) error {
@@ -70,6 +71,10 @@ func (m *fakeManager) State() state.Reader {
return m.state
}
+func (m *fakeManager) GetCPUPolicy() string {
+ return m.policy.Name()
+}
+
func (m *fakeManager) GetExclusiveCPUs(podUID, containerName string) cpuset.CPUSet {
klog.InfoS("GetExclusiveCPUs", "podUID", podUID, "containerName", containerName)
return cpuset.CPUSet{}
@@ -88,6 +93,7 @@ func (m *fakeManager) GetCPUAffinity(podUID, containerName string) cpuset.CPUSet
// NewFakeManager creates empty/fake cpu manager
func NewFakeManager() Manager {
return &fakeManager{
- state: state.NewMemoryState(),
+ policy: &nonePolicy{},
+ state: state.NewMemoryState(),
}
}
diff --git a/pkg/kubelet/cm/helpers_linux.go b/pkg/kubelet/cm/helpers_linux.go
index 25ff3f13b82..e9ea6bab8dc 100644
--- a/pkg/kubelet/cm/helpers_linux.go
+++ b/pkg/kubelet/cm/helpers_linux.go
@@ -182,6 +182,16 @@ func ResourceConfigForPod(pod *v1.Pod, enforceCPULimits bool, cpuPeriod uint64,
// build the result
result := &ResourceConfig{}
if qosClass == v1.PodQOSGuaranteed {
+ // Disable CFS CPU quota to avoid performance degradation due to
+ // Linux kernel CFS throttle implementation.
+ // NOTE: 4.18 kernel attempts to solve CFS throttling problem,
+ // but there are reports that it is not completely effective.
+ // This will configure cgroup CFS parameters at pod level:
+ // /sys/fs/cgroup/cpu/k8s-infra/kubepods/<pod>/cpu.cfs_quota_us
+ // /sys/fs/cgroup/cpu/k8s-infra/kubepods/<pod>/cpu.cfs_period_us
+ cpuQuota = int64(-1)
+ cpuPeriod = uint64(100000)
+
result.CpuShares = &cpuShares
result.CpuQuota = &cpuQuota
result.CpuPeriod = &cpuPeriod
diff --git a/pkg/kubelet/cm/helpers_linux_test.go b/pkg/kubelet/cm/helpers_linux_test.go
index 101b21e682a..9b98fb7e1c1 100644
--- a/pkg/kubelet/cm/helpers_linux_test.go
+++ b/pkg/kubelet/cm/helpers_linux_test.go
@@ -64,8 +64,9 @@ func TestResourceConfigForPod(t *testing.T) {
burstablePartialShares := MilliCPUToShares(200)
burstableQuota := MilliCPUToQuota(200, int64(defaultQuotaPeriod))
guaranteedShares := MilliCPUToShares(100)
- guaranteedQuota := MilliCPUToQuota(100, int64(defaultQuotaPeriod))
- guaranteedTunedQuota := MilliCPUToQuota(100, int64(tunedQuotaPeriod))
+ guaranteedQuotaPeriod := uint64(100000)
+ guaranteedQuota := int64(-1)
+ guaranteedTunedQuota := int64(-1)
memoryQuantity = resource.MustParse("100Mi")
cpuNoLimit := int64(-1)
guaranteedMemory := memoryQuantity.Value()
@@ -204,8 +205,8 @@ func TestResourceConfigForPod(t *testing.T) {
},
},
enforceCPULimits: true,
- quotaPeriod: defaultQuotaPeriod,
- expected: &ResourceConfig{CpuShares: &guaranteedShares, CpuQuota: &guaranteedQuota, CpuPeriod: &defaultQuotaPeriod, Memory: &guaranteedMemory},
+ quotaPeriod: guaranteedQuotaPeriod,
+ expected: &ResourceConfig{CpuShares: &guaranteedShares, CpuQuota: &guaranteedQuota, CpuPeriod: &guaranteedQuotaPeriod, Memory: &guaranteedMemory},
},
"guaranteed-no-cpu-enforcement": {
pod: &v1.Pod{
@@ -218,8 +219,8 @@ func TestResourceConfigForPod(t *testing.T) {
},
},
enforceCPULimits: false,
- quotaPeriod: defaultQuotaPeriod,
- expected: &ResourceConfig{CpuShares: &guaranteedShares, CpuQuota: &cpuNoLimit, CpuPeriod: &defaultQuotaPeriod, Memory: &guaranteedMemory},
+ quotaPeriod: guaranteedQuotaPeriod,
+ expected: &ResourceConfig{CpuShares: &guaranteedShares, CpuQuota: &cpuNoLimit, CpuPeriod: &guaranteedQuotaPeriod, Memory: &guaranteedMemory},
},
"guaranteed-with-tuned-quota": {
pod: &v1.Pod{
@@ -232,8 +233,8 @@ func TestResourceConfigForPod(t *testing.T) {
},
},
enforceCPULimits: true,
- quotaPeriod: tunedQuotaPeriod,
- expected: &ResourceConfig{CpuShares: &guaranteedShares, CpuQuota: &guaranteedTunedQuota, CpuPeriod: &tunedQuotaPeriod, Memory: &guaranteedMemory},
+ quotaPeriod: guaranteedQuotaPeriod,
+ expected: &ResourceConfig{CpuShares: &guaranteedShares, CpuQuota: &guaranteedTunedQuota, CpuPeriod: &guaranteedQuotaPeriod, Memory: &guaranteedMemory},
},
"guaranteed-no-cpu-enforcement-with-tuned-quota": {
pod: &v1.Pod{
@@ -246,8 +247,8 @@ func TestResourceConfigForPod(t *testing.T) {
},
},
enforceCPULimits: false,
- quotaPeriod: tunedQuotaPeriod,
- expected: &ResourceConfig{CpuShares: &guaranteedShares, CpuQuota: &cpuNoLimit, CpuPeriod: &tunedQuotaPeriod, Memory: &guaranteedMemory},
+ quotaPeriod: guaranteedQuotaPeriod,
+ expected: &ResourceConfig{CpuShares: &guaranteedShares, CpuQuota: &cpuNoLimit, CpuPeriod: &guaranteedQuotaPeriod, Memory: &guaranteedMemory},
},
"burstable-partial-limits-with-init-containers": {
pod: &v1.Pod{
@@ -309,8 +310,10 @@ func TestResourceConfigForPodWithCustomCPUCFSQuotaPeriod(t *testing.T) {
burstablePartialShares := MilliCPUToShares(200)
burstableQuota := MilliCPUToQuota(200, int64(defaultQuotaPeriod))
guaranteedShares := MilliCPUToShares(100)
- guaranteedQuota := MilliCPUToQuota(100, int64(defaultQuotaPeriod))
- guaranteedTunedQuota := MilliCPUToQuota(100, int64(tunedQuotaPeriod))
+ guaranteedQuotaPeriod := uint64(100000)
+ guaranteedQuota := int64(-1)
+ guaranteedTunedQuota := int64(-1)
+
memoryQuantity = resource.MustParse("100Mi")
cpuNoLimit := int64(-1)
guaranteedMemory := memoryQuantity.Value()
@@ -449,8 +452,8 @@ func TestResourceConfigForPodWithCustomCPUCFSQuotaPeriod(t *testing.T) {
},
},
enforceCPULimits: true,
- quotaPeriod: defaultQuotaPeriod,
- expected: &ResourceConfig{CpuShares: &guaranteedShares, CpuQuota: &guaranteedQuota, CpuPeriod: &defaultQuotaPeriod, Memory: &guaranteedMemory},
+ quotaPeriod: guaranteedQuotaPeriod,
+ expected: &ResourceConfig{CpuShares: &guaranteedShares, CpuQuota: &guaranteedQuota, CpuPeriod: &guaranteedQuotaPeriod, Memory: &guaranteedMemory},
},
"guaranteed-no-cpu-enforcement": {
pod: &v1.Pod{
@@ -463,8 +466,8 @@ func TestResourceConfigForPodWithCustomCPUCFSQuotaPeriod(t *testing.T) {
},
},
enforceCPULimits: false,
- quotaPeriod: defaultQuotaPeriod,
- expected: &ResourceConfig{CpuShares: &guaranteedShares, CpuQuota: &cpuNoLimit, CpuPeriod: &defaultQuotaPeriod, Memory: &guaranteedMemory},
+ quotaPeriod: guaranteedQuotaPeriod,
+ expected: &ResourceConfig{CpuShares: &guaranteedShares, CpuQuota: &cpuNoLimit, CpuPeriod: &guaranteedQuotaPeriod, Memory: &guaranteedMemory},
},
"guaranteed-with-tuned-quota": {
pod: &v1.Pod{
@@ -477,8 +480,8 @@ func TestResourceConfigForPodWithCustomCPUCFSQuotaPeriod(t *testing.T) {
},
},
enforceCPULimits: true,
- quotaPeriod: tunedQuotaPeriod,
- expected: &ResourceConfig{CpuShares: &guaranteedShares, CpuQuota: &guaranteedTunedQuota, CpuPeriod: &tunedQuotaPeriod, Memory: &guaranteedMemory},
+ quotaPeriod: guaranteedQuotaPeriod,
+ expected: &ResourceConfig{CpuShares: &guaranteedShares, CpuQuota: &guaranteedTunedQuota, CpuPeriod: &guaranteedQuotaPeriod, Memory: &guaranteedMemory},
},
"guaranteed-no-cpu-enforcement-with-tuned-quota": {
pod: &v1.Pod{
@@ -491,8 +494,8 @@ func TestResourceConfigForPodWithCustomCPUCFSQuotaPeriod(t *testing.T) {
},
},
enforceCPULimits: false,
- quotaPeriod: tunedQuotaPeriod,
- expected: &ResourceConfig{CpuShares: &guaranteedShares, CpuQuota: &cpuNoLimit, CpuPeriod: &tunedQuotaPeriod, Memory: &guaranteedMemory},
+ quotaPeriod: guaranteedQuotaPeriod,
+ expected: &ResourceConfig{CpuShares: &guaranteedShares, CpuQuota: &cpuNoLimit, CpuPeriod: &guaranteedQuotaPeriod, Memory: &guaranteedMemory},
},
}
diff --git a/pkg/kubelet/cm/internal_container_lifecycle_linux.go b/pkg/kubelet/cm/internal_container_lifecycle_linux.go
index cb7c0cfa543..75406dd8564 100644
--- a/pkg/kubelet/cm/internal_container_lifecycle_linux.go
+++ b/pkg/kubelet/cm/internal_container_lifecycle_linux.go
@@ -25,6 +25,7 @@ import (
"k8s.io/api/core/v1"
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
+ v1qos "k8s.io/kubernetes/pkg/apis/core/v1/helper/qos"
)
func (i *internalContainerLifecycleImpl) PreCreateContainer(pod *v1.Pod, container *v1.Container, containerConfig *runtimeapi.ContainerConfig) error {
@@ -35,6 +36,14 @@ func (i *internalContainerLifecycleImpl) PreCreateContainer(pod *v1.Pod, contain
}
}
+ // Disable cgroup CFS throttle at the container level.
+ // /sys/fs/cgroup/cpu/k8s-infra/kubepods/<pod>/<container>/cpu.cfs_quota_us
+ // /sys/fs/cgroup/cpu/k8s-infra/kubepods/<pod>/<container>/cpu.cfs_period_us
+ if i.cpuManager.GetCPUPolicy() == "static" && v1qos.GetPodQOS(pod) == v1.PodQOSGuaranteed {
+ containerConfig.Linux.Resources.CpuPeriod = int64(100000)
+ containerConfig.Linux.Resources.CpuQuota = int64(-1)
+ }
+
if i.memoryManager != nil {
numaNodes := i.memoryManager.GetMemoryNUMANodes(pod, container)
if numaNodes.Len() > 0 {
--
2.25.1

155
kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/kubelet-cpumanager-infra-pods-use-system-reserved-CP.patch
View File

@ -1,155 +0,0 @@
From 42ecf1cfaee7f9a5f16643465a964cfda25efaec Mon Sep 17 00:00:00 2001
From: Gleb Aronsky <gleb.aronsky@windriver.com>
Date: Tue, 15 Aug 2023 22:13:29 -0300
Subject: [PATCH] kubelet cpumanager infra pods use system reserved CPUs
This assigns system infrastructure pods to the "reserved" cpuset
to isolate them from the shared pool of CPUs.
Infrastructure pods include any pods that belong to the kube-system,
armada, cert-manager, vault, platform-deployment-manager, portieris,
notification, flux-helm, metrics-server, node-feature-discovery,
intel-power, power-metrics or sriov-fec-system namespaces.
The implementation is a bit simplistic, it is assumed that the
"reserved" cpuset is large enough to handle all infrastructure pods
CPU allocations.
This also prevents infrastucture pods from using Guaranteed resources.
Co-authored-by: Jim Gauld <james.gauld@windriver.com>
Signed-off-by: Gleb Aronsky <gleb.aronsky@windriver.com>
Signed-off-by: Thiago Miranda <ThiagoOliveira.Miranda@windriver.com>
Signed-off-by: Kaustubh Dhokte <kaustubh.dhokte@windriver.com>
Signed-off-by: Marcos Silva <Marcos.PauloOliveiraSilva@windriver.com>
Signed-off-by: Alyson Deives Pereira <alyson.deivespereira@windriver.com>
Signed-off-by: Marcos Silva <Marcos.PauloOliveiraSilva@windriver.com>
---
pkg/kubelet/cm/cpumanager/policy_static.go | 47 +++++++++++++++++--
.../cm/cpumanager/policy_static_test.go | 19 +++++++-
2 files changed, 61 insertions(+), 5 deletions(-)
diff --git a/pkg/kubelet/cm/cpumanager/policy_static.go b/pkg/kubelet/cm/cpumanager/policy_static.go
index 09e0fc0ea0e..216b6ce9bf8 100644
--- a/pkg/kubelet/cm/cpumanager/policy_static.go
+++ b/pkg/kubelet/cm/cpumanager/policy_static.go
@@ -53,6 +53,11 @@ func (e SMTAlignmentError) Type() string {
return ErrorSMTAlignment
}
+// Define namespaces used by platform infrastructure pods
+var infraNamespaces = [...]string{
+ "kube-system", "armada", "cert-manager", "platform-deployment-manager", "portieris", "vault", "notification", "flux-helm", "metrics-server", "node-feature-discovery", "intel-power", "power-metrics", "sriov-fec-system",
+}
+
// staticPolicy is a CPU manager policy that does not change CPU
// assignments for exclusively pinned guaranteed containers after the main
// container process starts.
@@ -121,10 +126,11 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv
klog.InfoS("Static policy created with configuration", "options", opts)
policy := &staticPolicy{
- topology: topology,
- affinity: affinity,
- cpusToReuse: make(map[string]cpuset.CPUSet),
- options: opts,
+ topology: topology,
+ affinity: affinity,
+ excludeReserved: excludeReserved,
+ cpusToReuse: make(map[string]cpuset.CPUSet),
+ options: opts,
}
allCPUs := topology.CPUDetails.CPUs()
@@ -263,6 +269,25 @@ func (p *staticPolicy) updateCPUsToReuse(pod *v1.Pod, container *v1.Container, c
}
func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Container) error {
+ // Process infra pods before guaranteed pods
+ if isKubeInfra(pod) {
+ // Container belongs in reserved pool.
+ // We don't want to fall through to the p.guaranteedCPUs() clause below so return either nil or error.
+ if _, ok := s.GetCPUSet(string(pod.UID), container.Name); ok {
+ klog.Infof("[cpumanager] static policy: reserved container already present in state, skipping (namespace: %s, pod UID: %s, pod: %s, container: %s)", pod.Namespace, string(pod.UID), pod.Name, container.Name)
+ return nil
+ }
+
+ cpuset := p.reserved
+ if cpuset.IsEmpty() {
+ // If this happens then someone messed up.
+ return fmt.Errorf("[cpumanager] static policy: reserved container unable to allocate cpus (namespace: %s, pod UID: %s, pod: %s, container: %s); cpuset=%v, reserved:%v", pod.Namespace, string(pod.UID), pod.Name, container.Name, cpuset, p.reserved)
+ }
+ s.SetCPUSet(string(pod.UID), container.Name, cpuset)
+ klog.Infof("[cpumanager] static policy: reserved: AddContainer (namespace: %s, pod UID: %s, pod: %s, container: %s); cpuset=%v", pod.Namespace, string(pod.UID), pod.Name, container.Name, cpuset)
+ return nil
+ }
+
if numCPUs := p.guaranteedCPUs(pod, container); numCPUs != 0 {
klog.InfoS("Static policy: Allocate", "pod", klog.KObj(pod), "containerName", container.Name)
// container belongs in an exclusively allocated pool
@@ -382,6 +407,10 @@ func (p *staticPolicy) guaranteedCPUs(pod *v1.Pod, container *v1.Container) int
if cpuQuantity.Value()*1000 != cpuQuantity.MilliValue() {
return 0
}
+ // Infrastructure pods use reserved CPUs even if they're in the Guaranteed QoS class
+ if isKubeInfra(pod) {
+ return 0
+ }
// Safe downcast to do for all systems with < 2.1 billion CPUs.
// Per the language spec, `int` is guaranteed to be at least 32 bits wide.
// https://golang.org/ref/spec#Numeric_types
@@ -595,3 +624,13 @@ func (p *staticPolicy) generateCPUTopologyHints(availableCPUs cpuset.CPUSet, reu
return hints
}
+
+// check if a given pod is in a platform infrastructure namespace
+func isKubeInfra(pod *v1.Pod) bool {
+ for _, namespace := range infraNamespaces {
+ if namespace == pod.Namespace {
+ return true
+ }
+ }
+ return false
+}
diff --git a/pkg/kubelet/cm/cpumanager/policy_static_test.go b/pkg/kubelet/cm/cpumanager/policy_static_test.go
index edfb40d880e..7938f787a57 100644
--- a/pkg/kubelet/cm/cpumanager/policy_static_test.go
+++ b/pkg/kubelet/cm/cpumanager/policy_static_test.go
@@ -886,7 +886,8 @@ func TestStaticPolicyStartWithResvList(t *testing.T) {
}
func TestStaticPolicyAddWithResvList(t *testing.T) {
-
+ infraPod := makePod("fakePod", "fakeContainer2", "200m", "200m")
+ infraPod.Namespace = "kube-system"
testCases := []staticPolicyTestWithResvList{
{
description: "GuPodSingleCore, SingleSocketHT, ExpectError",
@@ -928,6 +929,22 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
expCPUAlloc: true,
expCSet: cpuset.NewCPUSet(4, 5),
},
+ {
+ description: "InfraPod, SingleSocketHT, ExpectAllocReserved",
+ topo: topoSingleSocketHT,
+ numReservedCPUs: 2,
+ reserved: cpuset.NewCPUSet(0, 1),
+ stAssignments: state.ContainerCPUAssignments{
+ "fakePod": map[string]cpuset.CPUSet{
+ "fakeContainer100": cpuset.NewCPUSet(2, 3, 6, 7),
+ },
+ },
+ stDefaultCPUSet: cpuset.NewCPUSet(4, 5),
+ pod: infraPod,
+ expErr: nil,
+ expCPUAlloc: true,
+ expCSet: cpuset.NewCPUSet(0, 1),
+ },
}
testExcl := true
--
2.25.1

562
kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/kubelet-cpumanager-introduce-concept-of-isolated-CPU.patch
View File

@ -1,562 +0,0 @@
From ae7fc7b39bfde784340068b388a13a28b4e76398 Mon Sep 17 00:00:00 2001
From: Gleb Aronsky <gleb.aronsky@windriver.com>
Date: Tue, 25 Jan 2022 13:27:25 -0500
Subject: [PATCH] kubelet cpumanager introduce concept of isolated CPUs
This introduces the concept of "isolated CPUs", which are CPUs that
have been isolated at the kernel level via the "isolcpus" kernel boot
parameter.
When starting the kubelet process, two separate sets of reserved CPUs
may be specified. With this change CPUs reserved via
'--system-reserved=cpu' will be used for infrastructure pods while the
isolated CPUs should be reserved via '--kube-reserved=cpu' to cause
kubelet to skip over them for "normal" CPU resource tracking. The
kubelet code will double-check that the specified isolated CPUs match
what the kernel exposes in "/sys/devices/system/cpu/isolated".
A plugin (outside the scope of this commit) will expose the isolated
CPUs to kubelet via the device plugin API.
If a pod specifies some number of "isolcpus" resources, the device
manager will allocate them. In this code we check whether such
resources have been allocated, and if so we set the container cpuset to
the isolated CPUs. This does mean that it really only makes sense to
specify "isolcpus" resources for best-effort or burstable pods, not for
guaranteed ones since that would throw off the accounting code. In
order to ensure the accounting still works as designed, if "isolcpus"
are specified for guaranteed pods, the affinity will be set to the
non-isolated CPUs.
This patch was refactored in 1.21.3 due to upstream API change
node: podresources: make GetDevices() consistent
(commit ad68f9588c72d6477b5a290c548a9031063ac659).
The routine podIsolCPUs() was refactored in 1.21.3 since the API
p.deviceManager.GetDevices() is returning multiple devices with
a device per cpu. The resultant cpuset needs to be the aggregate.
The routine NewStaticPolicy was refactored in 1.22.5, adding a new argument
in its signature: cpuPolicyOptions map[string]string. This change is implies
shifting the new arguments(deviceManager, excludeReserved) with one position
to the right.
Co-authored-by: Jim Gauld <james.gauld@windriver.com>
Co-authored-by: Chris Friesen <chris.friesen@windriver.com>
Signed-off-by: Gleb Aronsky <gleb.aronsky@windriver.com>
---
pkg/kubelet/cm/container_manager_linux.go | 1 +
pkg/kubelet/cm/cpumanager/cpu_manager.go | 35 +++++++-
pkg/kubelet/cm/cpumanager/cpu_manager_test.go | 14 +++-
pkg/kubelet/cm/cpumanager/policy_static.go | 83 +++++++++++++++++--
.../cm/cpumanager/policy_static_test.go | 50 ++++++++---
5 files changed, 164 insertions(+), 19 deletions(-)
diff --git a/pkg/kubelet/cm/container_manager_linux.go b/pkg/kubelet/cm/container_manager_linux.go
index 0f09f3eb331..770922ca55d 100644
--- a/pkg/kubelet/cm/container_manager_linux.go
+++ b/pkg/kubelet/cm/container_manager_linux.go
@@ -321,6 +321,7 @@ func NewContainerManager(mountUtil mount.Interface, cadvisorInterface cadvisor.I
cm.GetNodeAllocatableReservation(),
nodeConfig.KubeletRootDir,
cm.topologyManager,
+ cm.deviceManager,
)
if err != nil {
klog.ErrorS(err, "Failed to initialize cpu manager")
diff --git a/pkg/kubelet/cm/cpumanager/cpu_manager.go b/pkg/kubelet/cm/cpumanager/cpu_manager.go
index 884c7323a79..ea466dbcd37 100644
--- a/pkg/kubelet/cm/cpumanager/cpu_manager.go
+++ b/pkg/kubelet/cm/cpumanager/cpu_manager.go
@@ -18,7 +18,9 @@ package cpumanager
import (
"fmt"
+ "io/ioutil"
"math"
+ "strings"
"sync"
"time"
@@ -32,6 +34,7 @@ import (
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state"
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
"k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
+ "k8s.io/kubernetes/pkg/kubelet/cm/devicemanager"
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager"
"k8s.io/kubernetes/pkg/kubelet/config"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
@@ -50,6 +53,25 @@ type policyName string
// cpuManagerStateFileName is the file name where cpu manager stores its state
const cpuManagerStateFileName = "cpu_manager_state"
+// get the system-level isolated CPUs
+func getIsolcpus() cpuset.CPUSet {
+ dat, err := ioutil.ReadFile("/sys/devices/system/cpu/isolated")
+ if err != nil {
+ klog.Errorf("[cpumanager] unable to read sysfs isolcpus subdir")
+ return cpuset.NewCPUSet()
+ }
+
+ // The isolated cpus string ends in a newline
+ cpustring := strings.TrimSuffix(string(dat), "\n")
+ cset, err := cpuset.Parse(cpustring)
+ if err != nil {
+ klog.Errorf("[cpumanager] unable to parse sysfs isolcpus string to cpuset")
+ return cpuset.NewCPUSet()
+ }
+
+ return cset
+}
+
// Manager interface provides methods for Kubelet to manage pod cpus.
type Manager interface {
// Start is called during Kubelet initialization.
@@ -153,7 +175,8 @@ func (s *sourcesReadyStub) AddSource(source string) {}
func (s *sourcesReadyStub) AllReady() bool { return true }
// NewManager creates new cpu manager based on provided policy
-func NewManager(cpuPolicyName string, cpuPolicyOptions map[string]string, reconcilePeriod time.Duration, machineInfo *cadvisorapi.MachineInfo, specificCPUs cpuset.CPUSet, nodeAllocatableReservation v1.ResourceList, stateFileDirectory string, affinity topologymanager.Store) (Manager, error) {
+func NewManager(cpuPolicyName string, cpuPolicyOptions map[string]string, reconcilePeriod time.Duration, machineInfo *cadvisorapi.MachineInfo, specificCPUs cpuset.CPUSet, nodeAllocatableReservation v1.ResourceList, stateFileDirectory string, affinity topologymanager.Store, deviceManager devicemanager.Manager) (Manager, error) {
+
var topo *topology.CPUTopology
var policy Policy
var err error
@@ -194,7 +217,15 @@ func NewManager(cpuPolicyName string, cpuPolicyOptions map[string]string, reconc
// NOTE: Set excludeReserved unconditionally to exclude reserved CPUs from default cpuset.
// This variable is primarily to make testing easier.
excludeReserved := true
- policy, err = NewStaticPolicy(topo, numReservedCPUs, specificCPUs, affinity, cpuPolicyOptions, excludeReserved)
+
+ // isolCPUs is the set of kernel-isolated CPUs. They should be a subset of specificCPUs or
+ // of the CPUs that NewStaticPolicy() will pick if numReservedCPUs is set. It's only in the
+ // argument list here for ease of testing, it's really internal to the policy.
+ isolCPUs := getIsolcpus()
+ policy, err = NewStaticPolicy(topo, numReservedCPUs, specificCPUs, isolCPUs, affinity, cpuPolicyOptions, deviceManager, excludeReserved)
+ if err != nil {
+ return nil, fmt.Errorf("new static policy error: %v", err)
+ }
if err != nil {
return nil, fmt.Errorf("new static policy error: %w", err)
diff --git a/pkg/kubelet/cm/cpumanager/cpu_manager_test.go b/pkg/kubelet/cm/cpumanager/cpu_manager_test.go
index 2c8349662c4..31e4d0585fb 100644
--- a/pkg/kubelet/cm/cpumanager/cpu_manager_test.go
+++ b/pkg/kubelet/cm/cpumanager/cpu_manager_test.go
@@ -37,6 +37,7 @@ import (
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state"
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
"k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
+ "k8s.io/kubernetes/pkg/kubelet/cm/devicemanager"
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager"
)
@@ -215,6 +216,7 @@ func makeMultiContainerPod(initCPUs, appCPUs []struct{ request, limit string })
}
func TestCPUManagerAdd(t *testing.T) {
+ testDM, _ := devicemanager.NewManagerStub()
testExcl := false
testPolicy, _ := NewStaticPolicy(
&topology.CPUTopology{
@@ -230,8 +232,10 @@ func TestCPUManagerAdd(t *testing.T) {
},
0,
cpuset.NewCPUSet(),
+ cpuset.NewCPUSet(),
topologymanager.NewFakeManager(),
nil,
+ testDM,
testExcl)
testCases := []struct {
description string
@@ -482,8 +486,9 @@ func TestCPUManagerAddWithInitContainers(t *testing.T) {
}
testExcl := false
+ testDM, _ := devicemanager.NewManagerStub()
for _, testCase := range testCases {
- policy, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil, testExcl)
+ policy, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, cpuset.NewCPUSet(), cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil, testDM, testExcl)
mockState := &mockState{
assignments: testCase.stAssignments,
@@ -638,7 +643,9 @@ func TestCPUManagerGenerate(t *testing.T) {
}
defer os.RemoveAll(sDir)
- mgr, err := NewManager(testCase.cpuPolicyName, nil, 5*time.Second, machineInfo, cpuset.NewCPUSet(), testCase.nodeAllocatableReservation, sDir, topologymanager.NewFakeManager())
+ testDM, err := devicemanager.NewManagerStub()
+ mgr, err := NewManager(testCase.cpuPolicyName, nil, 5*time.Second, machineInfo, cpuset.NewCPUSet(), testCase.nodeAllocatableReservation, sDir, topologymanager.NewFakeManager(), testDM)
+
if testCase.expectedError != nil {
if !strings.Contains(err.Error(), testCase.expectedError.Error()) {
t.Errorf("Unexpected error message. Have: %s wants %s", err.Error(), testCase.expectedError.Error())
@@ -1232,6 +1239,7 @@ func TestReconcileState(t *testing.T) {
// the following tests are with --reserved-cpus configured
func TestCPUManagerAddWithResvList(t *testing.T) {
testExcl := false
+ testDM, _ := devicemanager.NewManagerStub()
testPolicy, _ := NewStaticPolicy(
&topology.CPUTopology{
NumCPUs: 4,
@@ -1246,8 +1254,10 @@ func TestCPUManagerAddWithResvList(t *testing.T) {
},
1,
cpuset.NewCPUSet(0),
+ cpuset.NewCPUSet(),
topologymanager.NewFakeManager(),
nil,
+ testDM,
testExcl)
testCases := []struct {
description string
diff --git a/pkg/kubelet/cm/cpumanager/policy_static.go b/pkg/kubelet/cm/cpumanager/policy_static.go
index a3c93a896df..d6fe69e7165 100644
--- a/pkg/kubelet/cm/cpumanager/policy_static.go
+++ b/pkg/kubelet/cm/cpumanager/policy_static.go
@@ -18,6 +18,7 @@ package cpumanager
import (
"fmt"
+ "strconv"
v1 "k8s.io/api/core/v1"
"k8s.io/klog/v2"
@@ -25,6 +26,7 @@ import (
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state"
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
"k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
+ "k8s.io/kubernetes/pkg/kubelet/cm/devicemanager"
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager"
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask"
)
@@ -101,6 +103,10 @@ type staticPolicy struct {
topology *topology.CPUTopology
// set of CPUs that is not available for exclusive assignment
reserved cpuset.CPUSet
+ // subset of reserved CPUs with isolcpus attribute
+ isolcpus cpuset.CPUSet
+ // parent containerManager, used to get device list
+ deviceManager devicemanager.Manager
// If true, default CPUSet should exclude reserved CPUs
excludeReserved bool
// topology manager reference to get container Topology affinity
@@ -117,7 +123,8 @@ var _ Policy = &staticPolicy{}
// NewStaticPolicy returns a CPU manager policy that does not change CPU
// assignments for exclusively pinned guaranteed containers after the main
// container process starts.
-func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reservedCPUs cpuset.CPUSet, affinity topologymanager.Store, cpuPolicyOptions map[string]string, excludeReserved bool) (Policy, error) {
+func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reservedCPUs cpuset.CPUSet, isolCPUs cpuset.CPUSet, affinity topologymanager.Store, cpuPolicyOptions map[string]string, deviceManager devicemanager.Manager, excludeReserved bool) (Policy, error) {
+
opts, err := NewStaticPolicyOptions(cpuPolicyOptions)
if err != nil {
return nil, err
@@ -128,6 +135,8 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv
policy := &staticPolicy{
topology: topology,
affinity: affinity,
+ isolcpus: isolCPUs,
+ deviceManager: deviceManager,
excludeReserved: excludeReserved,
cpusToReuse: make(map[string]cpuset.CPUSet),
options: opts,
@@ -154,6 +163,12 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv
klog.InfoS("Reserved CPUs not available for exclusive assignment", "reservedSize", reserved.Size(), "reserved", reserved)
policy.reserved = reserved
+ if !isolCPUs.IsSubsetOf(reserved) {
+ klog.Errorf("[cpumanager] isolCPUs %v is not a subset of reserved %v", isolCPUs, reserved)
+ reserved = reserved.Union(isolCPUs)
+ klog.Warningf("[cpumanager] mismatch isolCPUs %v, force reserved %v", isolCPUs, reserved)
+ }
+
return policy, nil
}
@@ -187,8 +202,9 @@ func (p *staticPolicy) validateState(s state.State) error {
} else {
s.SetDefaultCPUSet(allCPUs)
}
- klog.Infof("[cpumanager] static policy: CPUSet: allCPUs:%v, reserved:%v, default:%v\n",
- allCPUs, p.reserved, s.GetDefaultCPUSet())
+ klog.Infof("[cpumanager] static policy: CPUSet: allCPUs:%v, reserved:%v, isolcpus:%v, default:%v\n",
+ allCPUs, p.reserved, p.isolcpus, s.GetDefaultCPUSet())
+
return nil
}
@@ -278,10 +294,11 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai
return nil
}
- cpuset := p.reserved
+ cpuset := p.reserved.Clone().Difference(p.isolcpus)
if cpuset.IsEmpty() {
// If this happens then someone messed up.
- return fmt.Errorf("[cpumanager] static policy: reserved container unable to allocate cpus (namespace: %s, pod UID: %s, pod: %s, container: %s); cpuset=%v, reserved:%v", pod.Namespace, string(pod.UID), pod.Name, container.Name, cpuset, p.reserved)
+ return fmt.Errorf("[cpumanager] static policy: reserved container unable to allocate cpus (namespace: %s, pod UID: %s, pod: %s, container: %s); cpuset=%v, reserved:%v, isolcpus:%v", pod.Namespace, string(pod.UID), pod.Name, container.Name, cpuset, p.reserved, p.isolcpus)
+
}
s.SetCPUSet(string(pod.UID), container.Name, cpuset)
klog.Infof("[cpumanager] static policy: reserved: AddContainer (namespace: %s, pod UID: %s, pod: %s, container: %s); cpuset=%v", pod.Namespace, string(pod.UID), pod.Name, container.Name, cpuset)
@@ -325,8 +342,34 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai
}
s.SetCPUSet(string(pod.UID), container.Name, cpuset)
p.updateCPUsToReuse(pod, container, cpuset)
+ klog.Infof("[cpumanager] guaranteed: AddContainer "+
+ "(namespace: %s, pod UID: %s, pod: %s, container: %s); numCPUS=%d, cpuset=%v",
+ pod.Namespace, string(pod.UID), pod.Name, container.Name, numCPUs, cpuset)
+ return nil
+ }
+ if isolcpus := p.podIsolCPUs(pod, container); isolcpus.Size() > 0 {
+ // container has requested isolated CPUs
+ if set, ok := s.GetCPUSet(string(pod.UID), container.Name); ok {
+ if set.Equals(isolcpus) {
+ klog.Infof("[cpumanager] isolcpus container already present in state, skipping (namespace: %s, pod UID: %s, pod: %s, container: %s)",
+ pod.Namespace, string(pod.UID), pod.Name, container.Name)
+ return nil
+ } else {
+ klog.Infof("[cpumanager] isolcpus container state has cpus %v, should be %v (namespace: %s, pod UID: %s, pod: %s, container: %s)",
+ isolcpus, set, pod.Namespace, string(pod.UID), pod.Name, container.Name)
+ }
+ }
+ // Note that we do not do anything about init containers here.
+ // It looks like devices are allocated per-pod based on effective requests/limits
+ // and extra devices from initContainers are not freed up when the regular containers start.
+ // TODO: confirm this is still true for 1.20
+ s.SetCPUSet(string(pod.UID), container.Name, isolcpus)
+ klog.Infof("[cpumanager] isolcpus: AddContainer (namespace: %s, pod UID: %s, pod: %s, container: %s); cpuset=%v",
+ pod.Namespace, string(pod.UID), pod.Name, container.Name, isolcpus)
+ return nil
}
+
// container belongs in the shared pool (nothing to do; use default cpuset)
return nil
}
@@ -634,3 +677,33 @@ func isKubeInfra(pod *v1.Pod) bool {
}
return false
}
+
+// get the isolated CPUs (if any) from the devices associated with a specific container
+func (p *staticPolicy) podIsolCPUs(pod *v1.Pod, container *v1.Container) cpuset.CPUSet {
+ // NOTE: This is required for TestStaticPolicyAdd() since makePod() does
+ // not create UID. We also need a way to properly stub devicemanager.
+ if len(string(pod.UID)) == 0 {
+ return cpuset.NewCPUSet()
+ }
+ resContDevices := p.deviceManager.GetDevices(string(pod.UID), container.Name)
+ cpuSet := cpuset.NewCPUSet()
+ for resourceName, resourceDevs := range resContDevices {
+ // this resource name needs to match the isolcpus device plugin
+ if resourceName == "windriver.com/isolcpus" {
+ for devID, _ := range resourceDevs {
+ cpuStrList := []string{devID}
+ if len(cpuStrList) > 0 {
+ // loop over the list of strings, convert each one to int, add to cpuset
+ for _, cpuStr := range cpuStrList {
+ cpu, err := strconv.Atoi(cpuStr)
+ if err != nil {
+ panic(err)
+ }
+ cpuSet = cpuSet.Union(cpuset.NewCPUSet(cpu))
+ }
+ }
+ }
+ }
+ }
+ return cpuSet
+}
diff --git a/pkg/kubelet/cm/cpumanager/policy_static_test.go b/pkg/kubelet/cm/cpumanager/policy_static_test.go
index d4b4b790210..ecd3e9598d0 100644
--- a/pkg/kubelet/cm/cpumanager/policy_static_test.go
+++ b/pkg/kubelet/cm/cpumanager/policy_static_test.go
@@ -25,6 +25,7 @@ import (
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state"
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
"k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
+ "k8s.io/kubernetes/pkg/kubelet/cm/devicemanager"
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager"
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask"
)
@@ -65,8 +66,9 @@ func (spt staticPolicyTest) PseudoClone() staticPolicyTest {
}
func TestStaticPolicyName(t *testing.T) {
+ testDM, _ := devicemanager.NewManagerStub()
testExcl := false
- policy, _ := NewStaticPolicy(topoSingleSocketHT, 1, cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil, testExcl)
+ policy, _ := NewStaticPolicy(topoSingleSocketHT, 1, cpuset.NewCPUSet(), cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil, testDM, testExcl)
policyName := policy.Name()
if policyName != "static" {
@@ -76,6 +78,7 @@ func TestStaticPolicyName(t *testing.T) {
}
func TestStaticPolicyStart(t *testing.T) {
+ testDM, _ := devicemanager.NewManagerStub()
testCases := []staticPolicyTest{
{
description: "non-corrupted state",
@@ -151,7 +154,7 @@ func TestStaticPolicyStart(t *testing.T) {
}
for _, testCase := range testCases {
t.Run(testCase.description, func(t *testing.T) {
- p, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil, testCase.excludeReserved)
+ p, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, cpuset.NewCPUSet(), cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil, testDM, testCase.excludeReserved)
policy := p.(*staticPolicy)
st := &mockState{
@@ -199,7 +202,7 @@ func TestStaticPolicyAdd(t *testing.T) {
largeTopoCPUSet := largeTopoBuilder.Result()
largeTopoSock0CPUSet := largeTopoSock0Builder.Result()
largeTopoSock1CPUSet := largeTopoSock1Builder.Result()
-
+ testDM, _ := devicemanager.NewManagerStub()
// these are the cases which must behave the same regardless the policy options.
// So we will permutate the options to ensure this holds true.
optionsInsensitiveTestCases := []staticPolicyTest{
@@ -529,8 +532,9 @@ func TestStaticPolicyAdd(t *testing.T) {
}
func runStaticPolicyTestCase(t *testing.T, testCase staticPolicyTest) {
+ testDM, _ := devicemanager.NewManagerStub()
testExcl := false
- policy, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, cpuset.NewCPUSet(), topologymanager.NewFakeManager(), testCase.options, testExcl)
+ policy, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, cpuset.NewCPUSet(), cpuset.NewCPUSet(), topologymanager.NewFakeManager(), testCase.options, testDM, testExcl)
st := &mockState{
assignments: testCase.stAssignments,
@@ -596,7 +600,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) {
}
for _, testCase := range testCases {
- policy, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil, testCase.excludeReserved)
+ policy, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, cpuset.NewCPUSet(), cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil, testDM, excludeReserved)
st := &mockState{
assignments: testCase.stAssignments,
@@ -629,6 +633,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) {
func TestStaticPolicyRemove(t *testing.T) {
excludeReserved := false
+ testDM, _ := devicemanager.NewManagerStub()
testCases := []staticPolicyTest{
{
description: "SingleSocketHT, DeAllocOneContainer",
@@ -710,6 +715,7 @@ func TestStaticPolicyRemove(t *testing.T) {
func TestTopologyAwareAllocateCPUs(t *testing.T) {
excludeReserved := false
+ testDM, _ := devicemanager.NewManagerStub()
testCases := []struct {
description string
topo *topology.CPUTopology
@@ -778,7 +784,8 @@ func TestTopologyAwareAllocateCPUs(t *testing.T) {
},
}
for _, tc := range testCases {
- p, _ := NewStaticPolicy(tc.topo, 0, cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil, excludeReserved)
+ p, _ := NewStaticPolicy(tc.topo, 0, cpuset.NewCPUSet(), cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil, testDM, excludeReserved)
+
policy := p.(*staticPolicy)
st := &mockState{
assignments: tc.stAssignments,
@@ -811,6 +818,7 @@ type staticPolicyTestWithResvList struct {
topo *topology.CPUTopology
numReservedCPUs int
reserved cpuset.CPUSet
+ isolcpus cpuset.CPUSet
stAssignments state.ContainerCPUAssignments
stDefaultCPUSet cpuset.CPUSet
pod *v1.Pod
@@ -821,6 +829,8 @@ type staticPolicyTestWithResvList struct {
}
func TestStaticPolicyStartWithResvList(t *testing.T) {
+ testDM, _ := devicemanager.NewManagerStub()
+ testExcl := false
testCases := []staticPolicyTestWithResvList{
{
description: "empty cpuset",
@@ -850,11 +860,9 @@ func TestStaticPolicyStartWithResvList(t *testing.T) {
expNewErr: fmt.Errorf("[cpumanager] unable to reserve the required amount of CPUs (size of 0-1 did not equal 1)"),
},
}
- testExcl := false
for _, testCase := range testCases {
t.Run(testCase.description, func(t *testing.T) {
- p, err := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, testCase.reserved, topologymanager.NewFakeManager(), nil, testExcl)
-
+ p, err := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, testCase.reserved, cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil, testDM, testExcl)
if !reflect.DeepEqual(err, testCase.expNewErr) {
t.Errorf("StaticPolicy Start() error (%v). expected error: %v but got: %v",
testCase.description, testCase.expNewErr, err)
@@ -894,6 +902,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
topo: topoSingleSocketHT,
numReservedCPUs: 1,
reserved: cpuset.NewCPUSet(0),
+ isolcpus: cpuset.NewCPUSet(),
stAssignments: state.ContainerCPUAssignments{},
stDefaultCPUSet: cpuset.NewCPUSet(1, 2, 3, 4, 5, 6, 7),
pod: makePod("fakePod", "fakeContainer2", "8000m", "8000m"),
@@ -906,6 +915,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
topo: topoSingleSocketHT,
numReservedCPUs: 2,
reserved: cpuset.NewCPUSet(0, 1),
+ isolcpus: cpuset.NewCPUSet(),
stAssignments: state.ContainerCPUAssignments{},
stDefaultCPUSet: cpuset.NewCPUSet(2, 3, 4, 5, 6, 7),
pod: makePod("fakePod", "fakeContainer2", "1000m", "1000m"),
@@ -918,6 +928,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
topo: topoSingleSocketHT,
numReservedCPUs: 2,
reserved: cpuset.NewCPUSet(0, 1),
+ isolcpus: cpuset.NewCPUSet(),
stAssignments: state.ContainerCPUAssignments{
"fakePod": map[string]cpuset.CPUSet{
"fakeContainer100": cpuset.NewCPUSet(2, 3, 6, 7),
@@ -934,6 +945,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
topo: topoSingleSocketHT,
numReservedCPUs: 2,
reserved: cpuset.NewCPUSet(0, 1),
+ isolcpus: cpuset.NewCPUSet(),
stAssignments: state.ContainerCPUAssignments{
"fakePod": map[string]cpuset.CPUSet{
"fakeContainer100": cpuset.NewCPUSet(2, 3, 6, 7),
@@ -945,11 +957,29 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
expCPUAlloc: true,
expCSet: cpuset.NewCPUSet(0, 1),
},
+ {
+ description: "InfraPod, SingleSocketHT, Isolcpus, ExpectAllocReserved",
+ topo: topoSingleSocketHT,
+ numReservedCPUs: 2,
+ reserved: cpuset.NewCPUSet(0, 1),
+ isolcpus: cpuset.NewCPUSet(1),
+ stAssignments: state.ContainerCPUAssignments{
+ "fakePod": map[string]cpuset.CPUSet{
+ "fakeContainer100": cpuset.NewCPUSet(2, 3, 6, 7),
+ },
+ },
+ stDefaultCPUSet: cpuset.NewCPUSet(4, 5),
+ pod: infraPod,
+ expErr: nil,
+ expCPUAlloc: true,
+ expCSet: cpuset.NewCPUSet(0),
+ },
}
testExcl := true
+ testDM, _ := devicemanager.NewManagerStub()
for _, testCase := range testCases {
- policy, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, testCase.reserved, topologymanager.NewFakeManager(), nil, testExcl)
+ policy, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, testCase.reserved, testCase.isolcpus, topologymanager.NewFakeManager(), nil, testDM, testExcl)
st := &mockState{
assignments: testCase.stAssignments,
--
2.25.1

303
kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/kubelet-cpumanager-keep-normal-containers-off-reserv.patch
View File

@ -1,303 +0,0 @@
From c827ea3b075774c9c72c33d38c973d05276cb7ac Mon Sep 17 00:00:00 2001
From: Sachin Gopala Krishna <saching.krishna@windriver.com>
Date: Mon, 3 Oct 2022 19:22:14 -0400
Subject: [PATCH] kubelet cpumanager keep normal containers off reserved CPUs
When starting the kubelet process, two separate sets of reserved CPUs
may be specified. With this change CPUs reserved via
'--system-reserved=cpu'
or '--kube-reserved=cpu' will be ignored by kubernetes itself. A small
tweak to the default CPU affinity ensures that "normal" Kubernetes
pods won't run on the reserved CPUs.
Co-authored-by: Jim Gauld <james.gauld@windriver.com>
Signed-off-by: Sachin Gopala Krishna <saching.krishna@windriver.com>
---
pkg/kubelet/cm/cpumanager/cpu_manager.go | 6 +++-
pkg/kubelet/cm/cpumanager/cpu_manager_test.go | 11 ++++--
pkg/kubelet/cm/cpumanager/policy_static.go | 29 +++++++++++----
.../cm/cpumanager/policy_static_test.go | 36 ++++++++++++++-----
4 files changed, 63 insertions(+), 19 deletions(-)
diff --git a/pkg/kubelet/cm/cpumanager/cpu_manager.go b/pkg/kubelet/cm/cpumanager/cpu_manager.go
index df431b06601..884c7323a79 100644
--- a/pkg/kubelet/cm/cpumanager/cpu_manager.go
+++ b/pkg/kubelet/cm/cpumanager/cpu_manager.go
@@ -191,7 +191,11 @@ func NewManager(cpuPolicyName string, cpuPolicyOptions map[string]string, reconc
// exclusively allocated.
reservedCPUsFloat := float64(reservedCPUs.MilliValue()) / 1000
numReservedCPUs := int(math.Ceil(reservedCPUsFloat))
- policy, err = NewStaticPolicy(topo, numReservedCPUs, specificCPUs, affinity, cpuPolicyOptions)
+ // NOTE: Set excludeReserved unconditionally to exclude reserved CPUs from default cpuset.
+ // This variable is primarily to make testing easier.
+ excludeReserved := true
+ policy, err = NewStaticPolicy(topo, numReservedCPUs, specificCPUs, affinity, cpuPolicyOptions, excludeReserved)
+
if err != nil {
return nil, fmt.Errorf("new static policy error: %w", err)
}
diff --git a/pkg/kubelet/cm/cpumanager/cpu_manager_test.go b/pkg/kubelet/cm/cpumanager/cpu_manager_test.go
index 9b3e24fc3b2..2c8349662c4 100644
--- a/pkg/kubelet/cm/cpumanager/cpu_manager_test.go
+++ b/pkg/kubelet/cm/cpumanager/cpu_manager_test.go
@@ -215,6 +215,7 @@ func makeMultiContainerPod(initCPUs, appCPUs []struct{ request, limit string })
}
func TestCPUManagerAdd(t *testing.T) {
+ testExcl := false
testPolicy, _ := NewStaticPolicy(
&topology.CPUTopology{
NumCPUs: 4,
@@ -230,7 +231,8 @@ func TestCPUManagerAdd(t *testing.T) {
0,
cpuset.NewCPUSet(),
topologymanager.NewFakeManager(),
- nil)
+ nil,
+ testExcl)
testCases := []struct {
description string
updateErr error
@@ -479,8 +481,9 @@ func TestCPUManagerAddWithInitContainers(t *testing.T) {
},
}
+ testExcl := false
for _, testCase := range testCases {
- policy, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil)
+ policy, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil, testExcl)
mockState := &mockState{
assignments: testCase.stAssignments,
@@ -1228,6 +1231,7 @@ func TestReconcileState(t *testing.T) {
// above test cases are without kubelet --reserved-cpus cmd option
// the following tests are with --reserved-cpus configured
func TestCPUManagerAddWithResvList(t *testing.T) {
+ testExcl := false
testPolicy, _ := NewStaticPolicy(
&topology.CPUTopology{
NumCPUs: 4,
@@ -1243,7 +1247,8 @@ func TestCPUManagerAddWithResvList(t *testing.T) {
1,
cpuset.NewCPUSet(0),
topologymanager.NewFakeManager(),
- nil)
+ nil,
+ testExcl)
testCases := []struct {
description string
updateErr error
diff --git a/pkg/kubelet/cm/cpumanager/policy_static.go b/pkg/kubelet/cm/cpumanager/policy_static.go
index a872b389c46..09e0fc0ea0e 100644
--- a/pkg/kubelet/cm/cpumanager/policy_static.go
+++ b/pkg/kubelet/cm/cpumanager/policy_static.go
@@ -96,6 +96,8 @@ type staticPolicy struct {
topology *topology.CPUTopology
// set of CPUs that is not available for exclusive assignment
reserved cpuset.CPUSet
+ // If true, default CPUSet should exclude reserved CPUs
+ excludeReserved bool
// topology manager reference to get container Topology affinity
affinity topologymanager.Store
// set of CPUs to reuse across allocations in a pod
@@ -110,7 +112,7 @@ var _ Policy = &staticPolicy{}
// NewStaticPolicy returns a CPU manager policy that does not change CPU
// assignments for exclusively pinned guaranteed containers after the main
// container process starts.
-func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reservedCPUs cpuset.CPUSet, affinity topologymanager.Store, cpuPolicyOptions map[string]string) (Policy, error) {
+func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reservedCPUs cpuset.CPUSet, affinity topologymanager.Store, cpuPolicyOptions map[string]string, excludeReserved bool) (Policy, error) {
opts, err := NewStaticPolicyOptions(cpuPolicyOptions)
if err != nil {
return nil, err
@@ -172,7 +174,15 @@ func (p *staticPolicy) validateState(s state.State) error {
}
// state is empty initialize
allCPUs := p.topology.CPUDetails.CPUs()
- s.SetDefaultCPUSet(allCPUs)
+ if p.excludeReserved {
+ // Exclude reserved CPUs from the default CPUSet to keep containers off them
+ // unless explicitly affined.
+ s.SetDefaultCPUSet(allCPUs.Difference(p.reserved))
+ } else {
+ s.SetDefaultCPUSet(allCPUs)
+ }
+ klog.Infof("[cpumanager] static policy: CPUSet: allCPUs:%v, reserved:%v, default:%v\n",
+ allCPUs, p.reserved, s.GetDefaultCPUSet())
return nil
}
@@ -180,11 +190,12 @@ func (p *staticPolicy) validateState(s state.State) error {
// 1. Check if the reserved cpuset is not part of default cpuset because:
// - kube/system reserved have changed (increased) - may lead to some containers not being able to start
// - user tampered with file
- if !p.reserved.Intersection(tmpDefaultCPUset).Equals(p.reserved) {
- return fmt.Errorf("not all reserved cpus: \"%s\" are present in defaultCpuSet: \"%s\"",
- p.reserved.String(), tmpDefaultCPUset.String())
+ if !p.excludeReserved {
+ if !p.reserved.Intersection(tmpDefaultCPUset).Equals(p.reserved) {
+ return fmt.Errorf("not all reserved cpus: \"%s\" are present in defaultCpuSet: \"%s\"",
+ p.reserved.String(), tmpDefaultCPUset.String())
+ }
}
-
// 2. Check if state for static policy is consistent
for pod := range tmpAssignments {
for container, cset := range tmpAssignments[pod] {
@@ -211,6 +222,9 @@ func (p *staticPolicy) validateState(s state.State) error {
}
}
totalKnownCPUs = totalKnownCPUs.UnionAll(tmpCPUSets)
+ if p.excludeReserved {
+ totalKnownCPUs = totalKnownCPUs.Union(p.reserved)
+ }
if !totalKnownCPUs.Equals(p.topology.CPUDetails.CPUs()) {
return fmt.Errorf("current set of available CPUs \"%s\" doesn't match with CPUs in state \"%s\"",
p.topology.CPUDetails.CPUs().String(), totalKnownCPUs.String())
@@ -310,6 +324,9 @@ func (p *staticPolicy) RemoveContainer(s state.State, podUID string, containerNa
cpusInUse := getAssignedCPUsOfSiblings(s, podUID, containerName)
if toRelease, ok := s.GetCPUSet(podUID, containerName); ok {
s.Delete(podUID, containerName)
+ if p.excludeReserved {
+ toRelease = toRelease.Difference(p.reserved)
+ }
// Mutate the shared pool, adding released cpus.
toRelease = toRelease.Difference(cpusInUse)
s.SetDefaultCPUSet(s.GetDefaultCPUSet().Union(toRelease))
diff --git a/pkg/kubelet/cm/cpumanager/policy_static_test.go b/pkg/kubelet/cm/cpumanager/policy_static_test.go
index 4e3255fff01..edfb40d880e 100644
--- a/pkg/kubelet/cm/cpumanager/policy_static_test.go
+++ b/pkg/kubelet/cm/cpumanager/policy_static_test.go
@@ -33,6 +33,7 @@ type staticPolicyTest struct {
description string
topo *topology.CPUTopology
numReservedCPUs int
+ excludeReserved bool
podUID string
options map[string]string
containerName string
@@ -64,7 +65,8 @@ func (spt staticPolicyTest) PseudoClone() staticPolicyTest {
}
func TestStaticPolicyName(t *testing.T) {
- policy, _ := NewStaticPolicy(topoSingleSocketHT, 1, cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil)
+ testExcl := false
+ policy, _ := NewStaticPolicy(topoSingleSocketHT, 1, cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil, testExcl)
policyName := policy.Name()
if policyName != "static" {
@@ -94,6 +96,15 @@ func TestStaticPolicyStart(t *testing.T) {
stDefaultCPUSet: cpuset.NewCPUSet(),
expCSet: cpuset.NewCPUSet(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11),
},
+ {
+ description: "empty cpuset exclude reserved",
+ topo: topoDualSocketHT,
+ numReservedCPUs: 2,
+ excludeReserved: true,
+ stAssignments: state.ContainerCPUAssignments{},
+ stDefaultCPUSet: cpuset.NewCPUSet(),
+ expCSet: cpuset.NewCPUSet(1, 2, 3, 4, 5, 7, 8, 9, 10, 11),
+ },
{
description: "reserved cores 0 & 6 are not present in available cpuset",
topo: topoDualSocketHT,
@@ -140,7 +151,8 @@ func TestStaticPolicyStart(t *testing.T) {
}
for _, testCase := range testCases {
t.Run(testCase.description, func(t *testing.T) {
- p, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil)
+ p, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil, testCase.excludeReserved)
+
policy := p.(*staticPolicy)
st := &mockState{
assignments: testCase.stAssignments,
@@ -517,7 +529,8 @@ func TestStaticPolicyAdd(t *testing.T) {
}
func runStaticPolicyTestCase(t *testing.T, testCase staticPolicyTest) {
- policy, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, cpuset.NewCPUSet(), topologymanager.NewFakeManager(), testCase.options)
+ testExcl := false
+ policy, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, cpuset.NewCPUSet(), topologymanager.NewFakeManager(), testCase.options, testExcl)
st := &mockState{
assignments: testCase.stAssignments,
@@ -583,7 +596,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) {
}
for _, testCase := range testCases {
- policy, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil)
+ policy, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil, testCase.excludeReserved)
st := &mockState{
assignments: testCase.stAssignments,
@@ -615,6 +628,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) {
}
func TestStaticPolicyRemove(t *testing.T) {
+ excludeReserved := false
testCases := []staticPolicyTest{
{
description: "SingleSocketHT, DeAllocOneContainer",
@@ -695,6 +709,7 @@ func TestStaticPolicyRemove(t *testing.T) {
}
func TestTopologyAwareAllocateCPUs(t *testing.T) {
+ excludeReserved := false
testCases := []struct {
description string
topo *topology.CPUTopology
@@ -763,7 +778,7 @@ func TestTopologyAwareAllocateCPUs(t *testing.T) {
},
}
for _, tc := range testCases {
- p, _ := NewStaticPolicy(tc.topo, 0, cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil)
+ p, _ := NewStaticPolicy(tc.topo, 0, cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil, excludeReserved)
policy := p.(*staticPolicy)
st := &mockState{
assignments: tc.stAssignments,
@@ -835,9 +850,11 @@ func TestStaticPolicyStartWithResvList(t *testing.T) {
expNewErr: fmt.Errorf("[cpumanager] unable to reserve the required amount of CPUs (size of 0-1 did not equal 1)"),
},
}
+ testExcl := false
for _, testCase := range testCases {
t.Run(testCase.description, func(t *testing.T) {
- p, err := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, testCase.reserved, topologymanager.NewFakeManager(), nil)
+ p, err := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, testCase.reserved, topologymanager.NewFakeManager(), nil, testExcl)
+
if !reflect.DeepEqual(err, testCase.expNewErr) {
t.Errorf("StaticPolicy Start() error (%v). expected error: %v but got: %v",
testCase.description, testCase.expNewErr, err)
@@ -877,7 +894,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
numReservedCPUs: 1,
reserved: cpuset.NewCPUSet(0),
stAssignments: state.ContainerCPUAssignments{},
- stDefaultCPUSet: cpuset.NewCPUSet(0, 1, 2, 3, 4, 5, 6, 7),
+ stDefaultCPUSet: cpuset.NewCPUSet(1, 2, 3, 4, 5, 6, 7),
pod: makePod("fakePod", "fakeContainer2", "8000m", "8000m"),
expErr: fmt.Errorf("not enough cpus available to satisfy request"),
expCPUAlloc: false,
@@ -889,7 +906,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
numReservedCPUs: 2,
reserved: cpuset.NewCPUSet(0, 1),
stAssignments: state.ContainerCPUAssignments{},
- stDefaultCPUSet: cpuset.NewCPUSet(0, 1, 2, 3, 4, 5, 6, 7),
+ stDefaultCPUSet: cpuset.NewCPUSet(2, 3, 4, 5, 6, 7),
pod: makePod("fakePod", "fakeContainer2", "1000m", "1000m"),
expErr: nil,
expCPUAlloc: true,
@@ -913,8 +930,9 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
},
}
+ testExcl := true
for _, testCase := range testCases {
- policy, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, testCase.reserved, topologymanager.NewFakeManager(), nil)
+ policy, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, testCase.reserved, topologymanager.NewFakeManager(), nil, testExcl)
st := &mockState{
assignments: testCase.stAssignments,
--
2.25.1

50
kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/kubelet-sort-isolcpus-allocation-when-SMT-enabled.patch
View File

@ -1,50 +0,0 @@
From ba9ab333c8b7dca5252e604837914293dc232732 Mon Sep 17 00:00:00 2001
From: Jim Gauld <James.Gauld@windriver.com>
Date: Fri, 11 Feb 2022 11:06:35 -0500
Subject: [PATCH] kubelet: sort isolcpus allocation when SMT enabled
The existing device manager code returns CPUs as devices in unsorted
order. This numerically sorts isolcpus allocations when SMT/HT is
enabled on the host. This logs SMT pairs, singletons, and algorithm
order details to make the algorithm understandable.
Signed-off-by: Jim Gauld <James.Gauld@windriver.com>
---
pkg/kubelet/cm/devicemanager/manager.go | 13 ++++++++++++-
1 file changed, 12 insertions(+), 1 deletion(-)
diff --git a/pkg/kubelet/cm/devicemanager/manager.go b/pkg/kubelet/cm/devicemanager/manager.go
index 609da8ed86b..a4b247714f7 100644
--- a/pkg/kubelet/cm/devicemanager/manager.go
+++ b/pkg/kubelet/cm/devicemanager/manager.go
@@ -686,7 +686,16 @@ func order_devices_by_sibling(devices sets.String, needed int) ([]string, error)
return cpu_lst[0]
}
}
+ //Make post-analysis of selection algorithm obvious by numerical sorting
+ //the available isolated cpu_id.
+ cpu_ids := make([]int, 0, int(devices.Len()))
for cpu_id := range devices {
+ cpu_id_, _ := strconv.Atoi(cpu_id)
+ cpu_ids = append(cpu_ids, cpu_id_)
+ }
+ sort.Ints(cpu_ids)
+ for _, _cpu_id := range cpu_ids {
+ cpu_id := strconv.Itoa(_cpu_id)
// If we've already found cpu_id as a sibling, skip it.
if _, ok := _iterated_cpu[cpu_id]; ok {
continue
@@ -728,7 +737,9 @@ func order_devices_by_sibling(devices sets.String, needed int) ([]string, error)
}
}
}
- //klog.Infof("needed=%d ordered_cpu_list=%v", needed, dev_lst)
+ //This algorithm will get some attention. Show minimal details.
+ klog.Infof("order_devices_by_sibling: needed=%d, smtpairs=%v, singletons=%v, order=%v",
+ needed, sibling_lst, single_lst, dev_lst)
return dev_lst, nil
}
func smt_enabled() bool {
--
2.25.1

151
kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/kubernetes-make-isolcpus-allocation-SMT-aware.patch
View File

@ -1,151 +0,0 @@
From 95b7b6e1ddb25511c67a3d4018f62df1e76ee7bc Mon Sep 17 00:00:00 2001
From: Tao Wang <tao.wang@windriver.com>
Date: Tue, 25 Jan 2022 19:25:45 -0500
Subject: [PATCH] kubernetes: make isolcpus allocation SMT-aware
Enhance isolcpus support in Kubernetes to allocate isolated SMT
siblings to the same container when SMT/HT is enabled on the host.
As it stands, the device manager code in Kubernetes is not SMT-aware
(since normally it doesn't deal with CPUs). However, StarlingX
exposes isolated CPUs as devices and if possible we want to allocate
all SMT siblings from a CPU core to the same container in order to
minimize cross- container interference due to resource contention
within the CPU core.
The solution is basically to take the list of isolated CPUs and
re-order it so that the SMT siblings are next to each other. That
way the existing resource selection code will allocate the siblings
together. As an optimization, if it is known that an odd number
of isolated CPUs are desired, a singleton SMT sibling will be
inserted into the list to avoid breaking up sibling pairs.
Signed-off-by: Tao Wang <tao.wang@windriver.com>
---
pkg/kubelet/cm/devicemanager/manager.go | 84 ++++++++++++++++++++++++-
1 file changed, 83 insertions(+), 1 deletion(-)
diff --git a/pkg/kubelet/cm/devicemanager/manager.go b/pkg/kubelet/cm/devicemanager/manager.go
index 60de14a9..609da8ed 100644
--- a/pkg/kubelet/cm/devicemanager/manager.go
+++ b/pkg/kubelet/cm/devicemanager/manager.go
@@ -19,11 +19,14 @@ package devicemanager
import (
"context"
"fmt"
+ "io/ioutil"
"net"
"os"
"path/filepath"
"runtime"
"sort"
+ "strconv"
+ "strings"
"sync"
"time"
@@ -41,6 +44,7 @@ import (
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/kubelet/checkpointmanager"
"k8s.io/kubernetes/pkg/kubelet/checkpointmanager/errors"
+ "k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
"k8s.io/kubernetes/pkg/kubelet/cm/devicemanager/checkpoint"
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager"
"k8s.io/kubernetes/pkg/kubelet/config"
@@ -667,6 +671,75 @@ func (m *ManagerImpl) UpdateAllocatedDevices() {
m.allocatedDevices = m.podDevices.devices()
}
+//Given a list of isolated CPUs in 'devices', and the number of desired CPUs in 'needed',
+//return an ordered list of isolated CPUs such that the first 'needed' CPUs in the list
+//contain as many hyperthread sibling pairs as possible.
+func order_devices_by_sibling(devices sets.String, needed int) ([]string, error) {
+ var dev_lst []string
+ var single_lst []string
+ sibling_lst := make([]string, 0, int(devices.Len()))
+ _iterated_cpu := make(map[string]string)
+ get_sibling := func(cpu string, cpu_lst []string) string {
+ if cpu_lst[0] == cpu {
+ return cpu_lst[1]
+ } else {
+ return cpu_lst[0]
+ }
+ }
+ for cpu_id := range devices {
+ // If we've already found cpu_id as a sibling, skip it.
+ if _, ok := _iterated_cpu[cpu_id]; ok {
+ continue
+ }
+ devPath := fmt.Sprintf("/sys/devices/system/cpu/cpu%s/topology/thread_siblings_list", cpu_id)
+ dat, err := ioutil.ReadFile(devPath)
+ if err != nil {
+ return dev_lst, fmt.Errorf("Can't read cpu[%s] thread_siblings_list", cpu_id)
+ }
+ cpustring := strings.TrimSuffix(string(dat), "\n")
+ cpu_pair_set, err := cpuset.Parse(cpustring)
+ if err != nil {
+ return dev_lst, fmt.Errorf("Unable to parse thread_siblings_list[%s] string to cpuset", cpustring)
+ }
+ var cpu_pair_lst []string
+ for _, v := range cpu_pair_set.ToSlice() {
+ cpu_pair_lst = append(cpu_pair_lst, strconv.Itoa(v))
+ }
+ sibling_cpu_id := get_sibling(cpu_id, cpu_pair_lst)
+ if _, ok := devices[sibling_cpu_id]; ok {
+ sibling_lst = append(sibling_lst, cpu_id, sibling_cpu_id)
+ _iterated_cpu[sibling_cpu_id] = ""
+ } else {
+ single_lst = append(single_lst, cpu_id)
+ }
+ _iterated_cpu[cpu_id] = ""
+ }
+ if needed%2 == 0 {
+ dev_lst = append(sibling_lst, single_lst...)
+ } else {
+ if len(single_lst) > 1 {
+ _tmp_list := append(sibling_lst, single_lst[1:]...)
+ dev_lst = append(single_lst[0:1], _tmp_list...)
+ } else {
+ if len(single_lst) == 0 {
+ dev_lst = sibling_lst
+ } else {
+ dev_lst = append(single_lst, sibling_lst...)
+ }
+ }
+ }
+ //klog.Infof("needed=%d ordered_cpu_list=%v", needed, dev_lst)
+ return dev_lst, nil
+}
+func smt_enabled() bool {
+ dat, _ := ioutil.ReadFile("/sys/devices/system/cpu/smt/active")
+ state := strings.TrimSuffix(string(dat), "\n")
+ if state == "0" {
+ return false
+ }
+ return true
+}
+
// Returns list of device Ids we need to allocate with Allocate rpc call.
// Returns empty list in case we don't need to issue the Allocate rpc call.
func (m *ManagerImpl) devicesToAllocate(podUID, contName, resource string, required int, reusableDevices sets.String) (sets.String, error) {
@@ -702,7 +775,16 @@ func (m *ManagerImpl) devicesToAllocate(podUID, contName, resource string, requi
// Create a closure to help with device allocation
// Returns 'true' once no more devices need to be allocated.
allocateRemainingFrom := func(devices sets.String) bool {
- for device := range devices.Difference(allocated) {
+ availableDevices := devices.Difference(allocated).List()
+ // If we're dealing with isolcpus and SMT is enabled, reorder to group SMT siblings together.
+ if resource == "windriver.com/isolcpus" && len(devices) > 0 && smt_enabled() {
+ var err error
+ availableDevices, err = order_devices_by_sibling(devices.Difference(allocated), needed)
+ if err != nil {
+ klog.Errorf("error in order_devices_by_sibling: %v", err)
+ }
+ }
+ for _, device := range availableDevices {
m.allocatedDevices[resource].Insert(device)
allocated.Insert(device)
needed--
--
2.22.5

23798
kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/kubernetes-pull-117892-vendor-bump-runc-to-1.1.6.patch
View File

File diff suppressed because it is too large Load Diff

19
kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/series
View File

@ -1,19 +0,0 @@
Revert-e2e-framework-test_context-add-control-plane-.patch
Revert-kubeadm-apply-the-new-control-plane-taint-dur.patch
Revert-kubeadm-apply-the-new-control-plane-taint-on-.patch
Revert-kubeadm-only-apply-the-new-control-plane-labe.patch
Revert-kubeadm-delete-the-old-master-label-during-up.patch
kubeadm-create-platform-pods-with-zero-CPU-resources.patch
Revert-use-subpath-for-coredns-only-for-default-repo.patch
kubernetes-make-isolcpus-allocation-SMT-aware.patch
kubelet-sort-isolcpus-allocation-when-SMT-enabled.patch
kubelet-cpumanager-disable-CFS-quota-throttling.patch
kubelet-cpumanager-keep-normal-containers-off-reserv.patch
kubelet-cpumanager-infra-pods-use-system-reserved-CP.patch
kubelet-cpumanager-introduce-concept-of-isolated-CPU.patch
cpumanager-policy-static-test-refactor.patch
kubelet-CFS-quota-throttling-for-non-integer-cpulimit.patch
kubeadm-reduce-UpgradeManifestTimeout.patch
Identify-platform-pods-based-on-pod-or-namespace-labels.patch
kubeadm-readiness-probe-timeout-core-dns.patch
kubernetes-pull-117892-vendor-bump-runc-to-1.1.6.patch

116
kubernetes/kubernetes-1.24.4/debian/deb_folder/rules
View File

@ -1,116 +0,0 @@
#!/usr/bin/make -f
#
# Copyright (c) 2022 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
# This debian/rules file is based on:
# https://packages.debian.org/source/bookworm/kubernetes
# http://deb.debian.org/debian/pool/main/k/kubernetes/kubernetes_1.20.5+really1.20.2-1.1.debian.tar.xz
# Customizations support kubernetes upgrades:
# - specific directory locations with kubernetes version, upgrades stage,
# and version specific golang compiler
# - build output not required on the production host is moved to
# kubernetes-misc package
kube_version := 1.24.4
kube_git_version := v${kube_version}
name := kubernetes-${kube_version}
go_version := 1.18.5
_stage1 := /usr/local/kubernetes/${kube_version}/stage1
_stage2 := /usr/local/kubernetes/${kube_version}/stage2
_bindir := /usr/bin
kube_dir := src/k8s.io/kubernetes
output_dir := ${kube_dir}/_output
output_bindir := ${output_dir}/bin
output_mandir := ${output_dir}/man
DEBIAN_DESTDIR := $(CURDIR)/debian/tmp
export DH_VERBOSE = 1
export PATH := /usr/lib/go-1.18/bin:$(PATH)
export KUBE_GIT_TREE_STATE="clean"
export KUBE_GIT_COMMIT=${kube_version}
export KUBE_GIT_VERSION=${kube_git_version}
export KUBE_EXTRA_GOPATH=$(pwd)/Godeps/_workspace
export PBR_VERSION=${kube_git_version}
bins = kube-proxy kube-apiserver kube-controller-manager kubelet kubeadm kube-scheduler kubectl
%:
dh $@ --with=bash-completion --builddirectory=src --without=build-stamp
override_dh_auto_build:
# we support multiple go compilers; indicate the version we are using
go version
which go
mkdir -pv ${kube_dir}
mv -v $$(ls | grep -v "^src$$" | grep -v "^debian$$") ${kube_dir}/.
cd ${kube_dir} && make WHAT="$(addprefix cmd/,$(bins) genman)"
# manpages
mkdir -p ${output_mandir}
echo $(bins) | xargs --max-args=1 ${output_bindir}/genman ${output_mandir}
# NOTICE files
find ${kube_dir}/vendor -name '*NOTICE*' -print0 | xargs -0 head -n1000 > ${output_dir}/NOTICE
override_dh_install:
# kube_version stage1
install -m 755 -d ${DEBIAN_DESTDIR}${_stage1}${_bindir}
install -p -m 755 -t ${DEBIAN_DESTDIR}${_stage1}${_bindir} ${output_bindir}/kubeadm
# kube_version stage2
install -m 755 -d ${DEBIAN_DESTDIR}${_stage2}${_bindir}
install -d -m 0755 ${DEBIAN_DESTDIR}${_stage2}/etc/systemd/system/kubelet.service.d
install -p -m 0644 -t ${DEBIAN_DESTDIR}${_stage2}/etc/systemd/system/kubelet.service.d debian/kubeadm.conf
install -p -m 750 -t ${DEBIAN_DESTDIR}${_stage2}${_bindir} ${output_bindir}/kubelet
install -p -m 755 -t ${DEBIAN_DESTDIR}${_stage2}${_bindir} ${output_bindir}/kubectl
# bash completions
install -d -m 0755 ${DEBIAN_DESTDIR}${_stage2}/usr/share/bash-completion/completions/
${DEBIAN_DESTDIR}${_stage2}${_bindir}/kubectl completion bash > ${DEBIAN_DESTDIR}${_stage2}/usr/share/bash-completion/completions/kubectl
# remaining are not kube_version staged, i.e., kubernetes-master, kubernetes-misc
install -m 755 -d ${DEBIAN_DESTDIR}${_bindir}
install -p -m 750 -t ${DEBIAN_DESTDIR}${_bindir} ${output_bindir}/kube-apiserver
install -p -m 750 -t ${DEBIAN_DESTDIR}${_bindir} ${output_bindir}/kube-controller-manager
install -p -m 750 -t ${DEBIAN_DESTDIR}${_bindir} ${output_bindir}/kube-scheduler
install -p -m 750 -t ${DEBIAN_DESTDIR}${_bindir} ${output_bindir}/kube-proxy
# specific cluster addons for optional use
install -d -m 0755 ${DEBIAN_DESTDIR}/etc/${name}/addons
# Addon: volumesnapshots
install -d -m 0755 ${DEBIAN_DESTDIR}/etc/${name}/addons/volumesnapshots
install -d -m 0755 ${DEBIAN_DESTDIR}/etc/${name}/addons/volumesnapshots/crd
install -m 0644 -t ${DEBIAN_DESTDIR}/etc/${name}/addons/volumesnapshots/crd ${kube_dir}/cluster/addons/volumesnapshots/crd/*
install -d -m 0755 ${DEBIAN_DESTDIR}/etc/${name}/addons/volumesnapshots/volume-snapshot-controller
install -m 0644 -t ${DEBIAN_DESTDIR}/etc/${name}/addons/volumesnapshots/volume-snapshot-controller ${kube_dir}/cluster/addons/volumesnapshots/volume-snapshot-controller/*
# unit-test
# - everything from the root directory is needed
# - unit-tests needs source code
# - integration tests needs docs and other files
# - test-cmd.sh atm needs cluster, examples and other
install -d -m 0755 ${DEBIAN_DESTDIR}/var/lib/kubernetes-unit-test/
cp -a src ${DEBIAN_DESTDIR}/var/lib/kubernetes-unit-test/
# remove generated output, i.e., binaries, go cache, man pages, violations report
rm -rf ${DEBIAN_DESTDIR}/var/lib/kubernetes-unit-test/${output_dir}
dh_install
override_dh_auto_test:
${kube_dir}/hack/test-cmd.sh
${kube_dir}/hack/benchmark-go.sh
${kube_dir}/hack/test-go.sh
${kube_dir}/hack/test-integration.sh --use_go_build
override_dh_fixperms:
dh_fixperms -Xkube-apiserver -Xkubeadm -Xkubeadm.conf \
-Xkubelet-cgroup-setup.sh -Xkube-apiserver \
-Xkube-controller-manager -Xkube-scheduler \
-Xkube-proxy -Xkubelet -Xkubectl
override_dh_usrlocal:

1
kubernetes/kubernetes-1.24.4/debian/deb_folder/source/format
View File

@ -1 +0,0 @@
3.0 (quilt)

12
kubernetes/kubernetes-1.24.4/debian/meta_data.yaml
View File

@ -1,12 +0,0 @@
---
debver: 1.24.4
dl_path:
name: kubernetes-1.24.4.tar.gz
url: https://github.com/kubernetes/kubernetes/archive/refs/tags/v1.24.4.tar.gz
md5sum: 4798c96475ce89c6354317f7b4ec08ca
sha256sum: 16e7112d8efa46c0a36976b001efe335eea4b9e1dd721824c9c2c064ae7f6bbe
revision:
dist: $STX_DIST
GITREVCOUNT:
BASE_SRCREV: 6d85d048d08bcc17bb37e9af154bfbb5115ab7f7
SRC_DIR: ${MY_REPO}/stx/integ/kubernetes/kubernetes-1.24.4

265
kubernetes/kubernetes-1.25.3/debian/deb_folder/changelog
View File

@ -1,265 +0,0 @@
kubernetes-1.25.3 (1.25.3-1) unstable; urgency=medium
* Updated for stx debian packaging
-- Ramesh Kumar Sivanandam rameshkumar.sivanandam@windriver.com Thu, 14 Nov 2022 14:30:00 +0000
kubernetes-1.24.4 (1.24.4-1) unstable; urgency=medium
* Updated for stx debian packaging
-- James Gauld james.gauld@windriver.com Wed, 14 Sep 2022 11:30:00 +0000
kubernetes-1.23.1 (1.23.1-1) unstable; urgency=medium
* Updated for stx debian packaging
-- Kaustubh Dhokte <kaustubh.dhokte@opendev.org> Tue, 22 Feb 2022 20:57:45 +0000
kubernetes (1.21.8-1) unstable; urgency=medium
* Updated for stx debian packaging
-- Mihnea Saracin mihnea.saracin@opendev.org Fri, 29 Oct 2021 12:51:12 +0000
kubernetes (1.20.5+really1.20.2-1.1) unstable; urgency=medium
* Non-maintainer upload.
* Build using golang-go (Closes: #1000980)
-- Florian Ernst <florian@debian.org> Sat, 02 Apr 2022 16:49:13 +0200
kubernetes (1.20.5+really1.20.2-1) unstable; urgency=medium
* This is actually still 1.20.2
* Only contains the client, for the server binaries see unstable/fasttrack
-- Janos Lenart <ocsi@debian.org> Sun, 13 Jun 2021 07:08:18 +0000
kubernetes (1.20.2-1) unstable; urgency=medium
* New upstream release: 1.20.2
-- Janos Lenart <ocsi@debian.org> Thu, 14 Jan 2021 10:55:09 +0000
kubernetes (1.20.0-1) unstable; urgency=medium
* New upstream release: 1.20.0
* Fixes CVE-2020-8565
-- Janos Lenart <ocsi@debian.org> Wed, 09 Dec 2020 12:33:59 +0000
kubernetes (1.19.4-2) unstable; urgency=medium
* Updated README.Debian
-- Janos Lenart <ocsi@debian.org> Sat, 21 Nov 2020 14:06:21 +0000
kubernetes (1.19.4-1) unstable; urgency=medium
* New upstream release: 1.19.4
-- Janos Lenart <ocsi@debian.org> Tue, 17 Nov 2020 09:30:59 +0000
kubernetes (1.19.3-1) unstable; urgency=medium
* New upstream release: 1.19.3
* Building with golang-1.15
* Fixes CVE-2020-8564, CVE-2020-8566
-- Janos Lenart <ocsi@debian.org> Wed, 21 Oct 2020 10:38:41 +0100
kubernetes (1.18.6-1) unstable; urgency=medium
* New upstream release: 1.18.6
* (An earlier version, 1.17.4-1 fixes CVE-2019-9946)
-- Janos Lenart <ocsi@debian.org> Thu, 16 Jul 2020 10:08:46 +0100
kubernetes (1.18.5-1) unstable; urgency=medium
* New upstream release: 1.18.5
* Fixes CVE-2020-8557, CVE-2020-8558, CVE-2020-8559
-- Janos Lenart <ocsi@debian.org> Wed, 15 Jul 2020 17:19:40 +0100
kubernetes (1.18.3-1) unstable; urgency=medium
* New upstream release: 1.18.3
* Improved build reproducibility
-- Janos Lenart <ocsi@debian.org> Tue, 02 Jun 2020 11:18:12 +0000
kubernetes (1.18.2-3) unstable; urgency=medium
* Bumped Standards-Version
* Improved build reproducibility
-- Janos Lenart <ocsi@debian.org> Fri, 15 May 2020 13:17:53 +0000
kubernetes (1.18.2-2) unstable; urgency=medium
* Added i386 back
-- Janos Lenart <ocsi@debian.org> Sun, 03 May 2020 21:13:17 +0000
kubernetes (1.18.2-1) unstable; urgency=medium
* New upstream release: 1.18.2
-- Janos Lenart <ocsi@debian.org> Sun, 03 May 2020 19:25:37 +0000
kubernetes (1.18.0-1) unstable; urgency=medium
* New upstream release: 1.18.0
-- Janos Lenart <ocsi@debian.org> Sat, 28 Mar 2020 12:58:42 +0000
kubernetes (1.17.4-1) unstable; urgency=high
* New maintainer (Closes: #886739)
* New upstream release: 1.17.4 (Closes: #887741)
* New Debian packaging from scratch. See README.Debian
* kubernetes-node
- Moved docker from Depends into Recommends as kubelet can also work with
rkt, cri-o, etc. (Closes: #872690)
- Not shipping systemd units for kubelet and kube-proxy for now
* kubernetes-master
- Moved etcd from Depends into Recommends as apiserver can also connect to
a remote etcd/cluster.
- Not shipping systemd units for kube-apiserver, kube-schedules and
kube-controller-manager for now
-- Janos Lenart <ocsi@debian.org> Sun, 15 Mar 2020 21:46:45 +0000
kubernetes (1.7.16+dfsg-1) unstable; urgency=medium
[ Michael Stapelberg ]
* Switch to XS-Go-Import-Path
[ Dmitry Smirnov ]
* Resurrected "mergo.patch" that has been mistakenly removed
(Closes: #878254).
* Re-enabled safeguard test for the above problem.
* New upstream release:
+ CVE-2017-1002101 (Closes: #892801)
+ CVE-2017-1002102 (Closes: #894051)
* Updated Vcs URLs for Salsa.
* Standards-Version: 4.1.4
* Build-Depends:
- golang-go
+ golang-any
+ golang-github-appc-cni-dev
+ golang-github-armon-circbuf-dev
+ golang-github-azure-azure-sdk-for-go-dev
+ golang-github-dgrijalva-jwt-go-v3-dev
+ golang-github-docker-distribution-dev
+ golang-github-docker-docker-dev
+ golang-github-emicklei-go-restful-swagger12-dev
+ golang-github-gogo-protobuf-dev
+ golang-github-gorilla-websocket-dev
+ golang-github-grpc-ecosystem-go-grpc-prometheus-dev
+ golang-github-karlseguin-ccache-dev
- golang-github-opencontainers-runc-dev
+ golang-github-opencontainers-docker-runc-dev
+ golang-github-pmezard-go-difflib-dev
+ golang-golang-x-time-dev
+ golang-golang-x-tools-dev
+ golang-google-grpc-dev
+ golang-gopkg-warnings.v0-dev
+ golang-goprotobuf-dev
-- Dmitry Smirnov <onlyjob@debian.org> Sun, 06 May 2018 16:20:21 +1000
kubernetes (1.7.7+dfsg-3) unstable; urgency=medium
* kubernetes-master should depend on etcd (Closes: #855218).
-- Andrew Shadura <andrewsh@debian.org> Sun, 22 Oct 2017 19:40:46 +0100
kubernetes (1.7.7+dfsg-2) unstable; urgency=medium
* Use CURDIR, not PWD, unbreaks the build at buildds.
-- Andrew Shadura <andrewsh@debian.org> Fri, 06 Oct 2017 19:25:45 +0200
kubernetes (1.7.7+dfsg-1) unstable; urgency=medium
[ Tim Potter ]
* Open work for new release
* Remove unused Files-Excluded entries from d/copyright
* Remove Skydns B-D as no longer used
* Don't build on ppc64 or ppc64le architectures
[ Andrew Shadura ]
* New upstream release.
* Refresh patches.
* Update build dependencies.
* Symlink vendor packages to the build directory.
-- Andrew Shadura <andrewsh@debian.org> Fri, 06 Oct 2017 18:54:06 +0200
kubernetes (1.5.5+dfsg-2) unstable; urgency=medium
* Team upload.
* Don't build on ppc64le due to Go linker problems. See GitHub issue
https://github.com/golang/go/issues/15823.
* Don't build on ppc64 as it's not supported by upstream at the
moment. (Closes: #860505)
-- Tim Potter <tpot@hpe.com> Sat, 03 Jun 2017 08:00:51 +1000
kubernetes (1.5.5+dfsg-1) unstable; urgency=low
[ Dmitry Smirnov ]
* Switch to bundled "rkt".
* rules: remove "-p" option from build and test overrides.
* control: drop obsolete "golang-clockwork-dev" alternative.
* New patch to disable test failing on [armel].
* Upload to unstable.
[ Tim Potter ]
* New upstream version. [March 2017]
* Big updates to d/rules and d/copyright to update to upstream
changes made since the 1.2.x release.
* Refresh patches to bring up to date with upstream changes since
1.2.x.
* control: add lsb-base as dependency for sysvinit scripts.
* Suppress spelling-error-in-binary Lintian messages.
-- Tim Potter <tpot@hpe.com> Thu, 13 Apr 2017 16:45:57 +1000
kubernetes (1.2.5+dfsg-1) experimental; urgency=medium
* New upstream release [June 2016].
* Switch to private "github.com/golang/glog" due to log noise.
* Disabled failing tests; no longer ignore failures in tests.
* Build/test using 2 cores only.
* New patch to update appc/cni name space (fixes FTBFS).
* Removed obsolete "spf13-cobra.patch".
-- Dmitry Smirnov <onlyjob@debian.org> Sun, 03 Jul 2016 04:12:28 +1000
kubernetes (1.2.4+dfsg-2) experimental; urgency=medium
* Added new patch to fix incompatibility with "imdario/mergo" v0.2.2
(Closes: #825753).
Thanks, Florian Ernst.
* Enable tests but ignore failures for now.
-- Dmitry Smirnov <onlyjob@debian.org> Fri, 17 Jun 2016 01:41:38 +1000
kubernetes (1.2.4+dfsg-1) experimental; urgency=medium
* New upstream release [May 2016].
* New patch to print output of "uname -m" on unsupported architectures.
* New "docker.patch" to fix potential FTBFS.
+ Build-Depends += "golang-github-docker-distribution-dev".
-- Dmitry Smirnov <onlyjob@debian.org> Wed, 15 Jun 2016 21:03:01 +1000
kubernetes (1.2.3+dfsg-1) experimental; urgency=low
* Initial release (Closes: #795652).
-- Dmitry Smirnov <onlyjob@debian.org> Mon, 25 Apr 2016 22:40:12 +1000

93
kubernetes/kubernetes-1.25.3/debian/deb_folder/control
View File

@ -1,93 +0,0 @@
Source: kubernetes-1.25.3
Section: admin
Priority: optional
Maintainer: StarlingX Developers <StarlingX-discuss@lists.StarlingX.io>
Build-Depends: debhelper-compat (= 13),
build-essential,
bash-completion,
jq,
rsync,
go-bindata,
go-md2man,
golang-1.19
Standards-Version: 4.4.1
Homepage: http://kubernetes.io/
XS-Build-Size: 15GB
Package: kubernetes-1.25.3-client
Provides: kubernetes-utils
Architecture: amd64
Depends: ${misc:Depends}, ${shlibs:Depends}
Recommends: ${misc:Recommends}
Built-Using: ${misc:Built-Using}
Description: Kubernetes Command Line Tool
The Kubernetes command line tool for interacting with the Kubernetes API.
Package: kubernetes-1.25.3-master
Architecture: amd64
Depends: ${misc:Depends}, ${shlibs:Depends},
adduser,
lsb-base,
etcd
Recommends: ${misc:Recommends}, kubernetes-1.25.3-client
Built-Using: ${misc:Built-Using}
Description: Kubernetes services for master host
Container Cluster Manager from Google. Kubernetes is an open source system
for managing containerized applications across multiple hosts, providing
basic mechanisms for deployment, maintenance, and scaling of applications.
.
Linux kernel version 3.8 or above is required for proper operation of the
daemon process, and that any lower versions may have subtle and/or glaring
issues.
.
This package provides "kube-apiserver", "kube-controller-manager" and
"kube-scheduler" daemons.
Package: kubernetes-1.25.3-node
Provides: cadvisor
Architecture: amd64
Depends: ${misc:Depends}, ${shlibs:Depends},
adduser,
conntrack,
conntrackd,
docker.io,
lsb-base,
socat,
Recommends: ${misc:Recommends}, kubernetes-1.25.3-client
Built-Using: ${misc:Built-Using}
Description: Kubernetes services for node host
Container Cluster Manager from Google. Kubernetes is an open source system
for managing containerized applications across multiple hosts, providing
basic mechanisms for deployment, maintenance, and scaling of applications.
.
Linux kernel version 3.8 or above is required for proper operation of the
daemon process, and that any lower versions may have subtle and/or glaring
issues.
Package: kubernetes-1.25.3-kubeadm
Architecture: amd64
Depends: ${misc:Depends}, containernetworking-plugins
Recommends: ${misc:Recommends}, kubernetes-1.25.3-client
Built-Using: ${misc:Built-Using}
Description: Kubernetes Cluster Bootstrapping Tool
The Kubernetes command line tool for bootstrapping a Kubernetes cluster.
Package: kubernetes-1.25.3-misc
Architecture: amd64
Depends: ${misc:Depends}, ${shlibs:Depends}
Recommends: ${misc:Recommends}
Built-Using: ${misc:Built-Using}
Description: dummy package
Kubernetes dummy package for misc stuff we don't want to install in production.
Package: kubernetes-1.25.3-unit-test
Architecture: amd64
Depends: ${misc:Depends}, ${shlibs:Depends},
hostname,
rsync,
etcd (>= 2.0.9),
network-manager,
Recommends: ${misc:Recommends}
Built-Using: ${misc:Built-Using}
Description: Kubernetes unit test
Kubernetes unit-test framework.

1820
kubernetes/kubernetes-1.25.3/debian/deb_folder/copyright
View File

File diff suppressed because it is too large Load Diff

17
kubernetes/kubernetes-1.25.3/debian/deb_folder/kubeadm.conf
View File

@ -1,17 +0,0 @@
# Note: This dropin only works with kubeadm and kubelet v1.11+
[Service]
Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf"
Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml"
# This is a file that "kubeadm init" and "kubeadm join" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically
EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env
# This is a file that the user can use for overrides of the kubelet args as a last resort. Preferably, the user should use
# the .NodeRegistration.KubeletExtraArgs object in the configuration files instead. KUBELET_EXTRA_ARGS should be sourced from this file.
EnvironmentFile=-/etc/default/kubelet
ExecStart=
ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS
ExecStartPre=-/usr/local/sbin/sanitize_kubelet_reserved_cpus.sh /etc/default/kubelet
ExecStartPre=-/usr/bin/kubelet-cgroup-setup.sh
ExecStartPost=/bin/bash -c 'echo $MAINPID > /var/run/kubelet.pid;'
ExecStopPost=/bin/rm -f /var/run/kubelet.pid
StartLimitInterval=0
RestartSec=10

2
kubernetes/kubernetes-1.25.3/debian/deb_folder/kubernetes-1.25.3-client.install
View File

@ -1,2 +0,0 @@
usr/local/kubernetes/1.25.3/stage2/usr/bin/kubectl
usr/local/kubernetes/1.25.3/stage2/usr/share/bash-completion/completions/kubectl

9
kubernetes/kubernetes-1.25.3/debian/deb_folder/kubernetes-1.25.3-client.lintian-overrides
View File

@ -1,9 +0,0 @@
## Generated man pages: TODO
manpage-has-bad-whatis-entry usr/share/man/*
manpage-has-errors-from-man usr/share/man/man1/*
## Bash-completion script does not have to be executable:
script-not-executable usr/share/bash-completion/completions/kubectl
## Override annoying/useless messages
kubernetes-client: spelling-error-in-binary

1
kubernetes/kubernetes-1.25.3/debian/deb_folder/kubernetes-1.25.3-kubeadm.dirs
View File

@ -1 +0,0 @@
usr/local/kubernetes/1.25.3/stage2/etc/systemd/system/kubelet.service.d/

2
kubernetes/kubernetes-1.25.3/debian/deb_folder/kubernetes-1.25.3-kubeadm.install
View File

@ -1,2 +0,0 @@
usr/local/kubernetes/1.25.3/stage1/usr/bin/kubeadm
usr/local/kubernetes/1.25.3/stage2/etc/systemd/system/kubelet.service.d/kubeadm.conf

5
kubernetes/kubernetes-1.25.3/debian/deb_folder/kubernetes-1.25.3-master.dirs
View File

@ -1,5 +0,0 @@
etc/kubernetes-1.25.3
etc/kubernetes-1.25.3/addons
etc/kubernetes-1.25.3/addons/volumesnapshots
etc/kubernetes-1.25.3/addons/volumesnapshots/crd
etc/kubernetes-1.25.3/addons/volumesnapshots/volume-snapshot-controller

8
kubernetes/kubernetes-1.25.3/debian/deb_folder/kubernetes-1.25.3-master.install
View File

@ -1,8 +0,0 @@
usr/bin/kube-apiserver
usr/bin/kube-controller-manager
usr/bin/kube-scheduler
etc/kubernetes-1.25.3/addons/volumesnapshots/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml
etc/kubernetes-1.25.3/addons/volumesnapshots/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml
etc/kubernetes-1.25.3/addons/volumesnapshots/crd/snapshot.storage.k8s.io_volumesnapshots.yaml
etc/kubernetes-1.25.3/addons/volumesnapshots/volume-snapshot-controller/volume-snapshot-controller-deployment.yaml
etc/kubernetes-1.25.3/addons/volumesnapshots/volume-snapshot-controller/rbac-volume-snapshot-controller.yaml

7
kubernetes/kubernetes-1.25.3/debian/deb_folder/kubernetes-1.25.3-master.lintian-overrides
View File

@ -1,7 +0,0 @@
## No manual page for hyperkube
kubernetes-master: binary-without-manpage usr/bin/hyperkube
## Override annoying/useless messages
kubernetes-master: spelling-error-in-binary
kubernetes-master: manpage-has-errors-from-man usr/share/man/man1/*
kubernetes-master: manpage-has-bad-whatis-entry usr/share/man/man1/*

3
kubernetes/kubernetes-1.25.3/debian/deb_folder/kubernetes-1.25.3-misc.docs
View File

@ -1,3 +0,0 @@
src/k8s.io/kubernetes/README.md
src/k8s.io/kubernetes/SUPPORT.md
src/k8s.io/kubernetes/_output/NOTICE

1
kubernetes/kubernetes-1.25.3/debian/deb_folder/kubernetes-1.25.3-misc.install
View File

@ -1 +0,0 @@
usr/bin/kube-proxy

10
kubernetes/kubernetes-1.25.3/debian/deb_folder/kubernetes-1.25.3-misc.manpages
View File

@ -1,10 +0,0 @@
# kubernetes-client
src/k8s.io/kubernetes/_output/man/kubeadm*
src/k8s.io/kubernetes/_output/man/kubectl*
# kubernetes-master
src/k8s.io/kubernetes/_output/man/kube-apiserver*
src/k8s.io/kubernetes/_output/man/kube-scheduler*
src/k8s.io/kubernetes/_output/man/kube-controller-manager*
# kubernetes-node
src/k8s.io/kubernetes/_output/man/kubelet*
src/k8s.io/kubernetes/_output/man/kube-proxy*

1
kubernetes/kubernetes-1.25.3/debian/deb_folder/kubernetes-1.25.3-node.install
View File

@ -1 +0,0 @@
usr/local/kubernetes/1.25.3/stage2/usr/bin/kubelet

4
kubernetes/kubernetes-1.25.3/debian/deb_folder/kubernetes-1.25.3-node.lintian-overrides
View File

@ -1,4 +0,0 @@
## Override annoying/useless messages
kubernetes-node: spelling-error-in-binary
kubernetes-node: manpage-has-errors-from-man usr/share/man/man1/*
kubernetes-node: manpage-has-bad-whatis-entry usr/share/man/man1/*

1
kubernetes/kubernetes-1.25.3/debian/deb_folder/kubernetes-1.25.3-unit-test.install
View File

@ -1 +0,0 @@
var/lib/kubernetes-unit-test/

427
kubernetes/kubernetes-1.25.3/debian/deb_folder/patches/Identify-platform-pods-based-on-pod-or-namespace-labels.patch
View File

@ -1,427 +0,0 @@
From cacb801636c985c4519cb7798d3963872368875c Mon Sep 17 00:00:00 2001
From: Boovan Rajendran <boovan.rajendran@windriver.com>
Date: Thu, 28 Mar 2024 06:13:17 -0400
Subject: [PATCH] Identify platform pods based on pod or namespace labels
Pods with namespace 'kube-system', or labeled with
'app.starlingx.io/component=platform' are identified as 'platform'.
These have isolated cpu affinity cpuset when cpu-manager 'static'
policy is configured.
Signed-off-by: Boovan Rajendran <boovan.rajendran@windriver.com>
---
pkg/kubelet/cm/cpumanager/policy_static.go | 95 +++++++++--
.../cm/cpumanager/policy_static_test.go | 147 ++++++++++++++++++
.../cm/cpumanager/topology_hints_test.go | 4 +
3 files changed, 234 insertions(+), 12 deletions(-)
diff --git a/pkg/kubelet/cm/cpumanager/policy_static.go b/pkg/kubelet/cm/cpumanager/policy_static.go
index 34fdaaf0db1..3fda96098c0 100644
--- a/pkg/kubelet/cm/cpumanager/policy_static.go
+++ b/pkg/kubelet/cm/cpumanager/policy_static.go
@@ -17,11 +17,17 @@ limitations under the License.
package cpumanager
import (
+ "context"
"fmt"
"strconv"
+ k8sclient "k8s.io/client-go/kubernetes"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ restclient "k8s.io/client-go/rest"
v1 "k8s.io/api/core/v1"
+ "k8s.io/client-go/tools/clientcmd"
"k8s.io/klog/v2"
+ "k8s.io/kubernetes/cmd/kubeadm/app/constants"
v1qos "k8s.io/kubernetes/pkg/apis/core/v1/helper/qos"
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state"
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
@@ -41,6 +47,23 @@ const (
ErrorSMTAlignment = "SMTAlignmentError"
)
+// Declared as variables so that they can easily more
+// overridden during testing
+type getPodNamespace func(string) (*v1.Namespace, error)
+type buildFromConfigFlag func(masterUrl string, kubeconfigPath string) (*restclient.Config, error)
+type isKubeInfraFunc func(pod *v1.Pod) bool
+
+var varGetNamespaceObject getPodNamespace
+var varBuildConfigFromFlags buildFromConfigFlag
+var varIsKubeInfra isKubeInfraFunc
+
+func init() {
+ varIsKubeInfra = isKubeInfra
+ varGetNamespaceObject = getPodNamespaceObject
+ varBuildConfigFromFlags = clientcmd.BuildConfigFromFlags
+}
+
+
// SMTAlignmentError represents an error due to SMT alignment
type SMTAlignmentError struct {
RequestedCPUs int
@@ -55,11 +78,6 @@ func (e SMTAlignmentError) Type() string {
return ErrorSMTAlignment
}
-// Define namespaces used by platform infrastructure pods
-var infraNamespaces = [...]string{
- "kube-system", "armada", "cert-manager", "platform-deployment-manager", "portieris", "vault", "notification", "flux-helm", "metrics-server", "node-feature-discovery", "intel-power", "power-metrics", "sriov-fec-system",
-}
-
// staticPolicy is a CPU manager policy that does not change CPU
// assignments for exclusively pinned guaranteed containers after the main
// container process starts.
@@ -290,7 +308,7 @@ func (p *staticPolicy) updateCPUsToReuse(pod *v1.Pod, container *v1.Container, c
func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Container) error {
// Process infra pods before guaranteed pods
- if isKubeInfra(pod) {
+ if varIsKubeInfra(pod) {
// Container belongs in reserved pool.
// We don't want to fall through to the p.guaranteedCPUs() clause below so return either nil or error.
if _, ok := s.GetCPUSet(string(pod.UID), container.Name); ok {
@@ -452,7 +470,7 @@ func (p *staticPolicy) guaranteedCPUs(pod *v1.Pod, container *v1.Container) int
return 0
}
// Infrastructure pods use reserved CPUs even if they're in the Guaranteed QoS class
- if isKubeInfra(pod) {
+ if varIsKubeInfra(pod) {
return 0
}
// Safe downcast to do for all systems with < 2.1 billion CPUs.
@@ -673,14 +691,67 @@ func (p *staticPolicy) generateCPUTopologyHints(availableCPUs cpuset.CPUSet, reu
return hints
}
-// check if a given pod is in a platform infrastructure namespace
+func getPodNamespaceObject(podNamespaceName string) (*v1.Namespace, error) {
+
+ kubeConfigPath := constants.GetKubeletKubeConfigPath()
+ cfg, err := varBuildConfigFromFlags("", kubeConfigPath)
+ if err != nil {
+ klog.Error("Failed to build client config from ", kubeConfigPath, err.Error())
+ return nil, err
+ }
+
+ clientset, err := k8sclient.NewForConfig(cfg)
+ if err != nil {
+ klog.Error("Failed to get clientset for KUBECONFIG ", kubeConfigPath, err.Error())
+ return nil, err
+ }
+
+ namespaceObj, err := clientset.CoreV1().Namespaces().Get(context.TODO(), podNamespaceName, metav1.GetOptions{})
+ if err != nil {
+ klog.Error("Error getting namespace object:", err.Error())
+ return nil, err
+ }
+
+ return namespaceObj, nil
+
+}
+
+// check if a given pod is labelled as platform pod or
+// is in a namespace labelled as a platform namespace
func isKubeInfra(pod *v1.Pod) bool {
- for _, namespace := range infraNamespaces {
- if namespace == pod.Namespace {
- return true
- }
+
+ podName := pod.GetName()
+ podNamespaceName := pod.GetNamespace()
+
+ if podNamespaceName == "kube-system" {
+ klog.Infof("Pod %s has %s namespace. Treating as platform pod.", podName , podNamespaceName)
+ return true
+ }
+
+ klog.InfoS("Checking pod ", podName , " for label 'app.starlingx.io/component=platform'.")
+ podLabels := pod.GetLabels()
+ val, ok := podLabels["app.starlingx.io/component"]
+ if (ok && val == "platform") {
+ klog.InfoS("Pod ", podName, " has 'app.starlingx.io/component=platform' label. Treating as platform pod.")
+ return true
+ }
+
+ klog.V(4).InfoS("Pod ", pod.GetName(), " does not have 'app.starlingx.io/component=platform' label. Checking its namespace information...")
+ namespaceObj, err := varGetNamespaceObject(podNamespaceName)
+ if err != nil {
+ return false
}
+
+ namespaceLabels := namespaceObj.GetLabels()
+ val, ok = namespaceLabels["app.starlingx.io/component"]
+ if ok && val == "platform" {
+ klog.InfoS("For pod: ", podName, ", its Namespace ", podNamespaceName, " has 'app.starlingx.io/component=platform' label. Treating as platform pod.")
+ return true
+ }
+
+ klog.InfoS("Neither pod ", podName, " nor its namespace ", podNamespaceName, " has 'app.starlingx.io/component=platform' label. Not treating as platform pod.")
return false
+
}
// get the isolated CPUs (if any) from the devices associated with a specific container
diff --git a/pkg/kubelet/cm/cpumanager/policy_static_test.go b/pkg/kubelet/cm/cpumanager/policy_static_test.go
index 1c43df3b85f..892723d3b07 100644
--- a/pkg/kubelet/cm/cpumanager/policy_static_test.go
+++ b/pkg/kubelet/cm/cpumanager/policy_static_test.go
@@ -17,10 +17,13 @@ limitations under the License.
package cpumanager
import (
+ "errors"
"fmt"
"reflect"
"testing"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ restclient "k8s.io/client-go/rest"
v1 "k8s.io/api/core/v1"
utilfeature "k8s.io/apiserver/pkg/util/feature"
featuregatetesting "k8s.io/component-base/featuregate/testing"
@@ -877,6 +880,7 @@ type staticPolicyTestWithResvList struct {
stAssignments state.ContainerCPUAssignments
stDefaultCPUSet cpuset.CPUSet
pod *v1.Pod
+ isKubeInfraPodfunc isKubeInfraFunc
expErr error
expNewErr error
expCPUAlloc bool
@@ -949,6 +953,14 @@ func TestStaticPolicyStartWithResvList(t *testing.T) {
}
}
+func fakeIsKubeInfraTrue(pod *v1.Pod) bool {
+ return true
+}
+
+func fakeIsKubeInfraFalse(pod *v1.Pod) bool {
+ return false
+}
+
func TestStaticPolicyAddWithResvList(t *testing.T) {
infraPod := makePod("fakePod", "fakeContainer2", "200m", "200m")
infraPod.Namespace = "kube-system"
@@ -962,6 +974,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
stAssignments: state.ContainerCPUAssignments{},
stDefaultCPUSet: cpuset.NewCPUSet(1, 2, 3, 4, 5, 6, 7),
pod: makePod("fakePod", "fakeContainer2", "8000m", "8000m"),
+ isKubeInfraPodfunc: fakeIsKubeInfraFalse,
expErr: fmt.Errorf("not enough cpus available to satisfy request"),
expCPUAlloc: false,
expCSet: cpuset.NewCPUSet(),
@@ -975,6 +988,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
stAssignments: state.ContainerCPUAssignments{},
stDefaultCPUSet: cpuset.NewCPUSet(2, 3, 4, 5, 6, 7),
pod: makePod("fakePod", "fakeContainer2", "1000m", "1000m"),
+ isKubeInfraPodfunc: fakeIsKubeInfraFalse,
expErr: nil,
expCPUAlloc: true,
expCSet: cpuset.NewCPUSet(4), // expect sibling of partial core
@@ -992,6 +1006,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
},
stDefaultCPUSet: cpuset.NewCPUSet(0, 1, 4, 5),
pod: makePod("fakePod", "fakeContainer3", "2000m", "2000m"),
+ isKubeInfraPodfunc: fakeIsKubeInfraFalse,
expErr: nil,
expCPUAlloc: true,
expCSet: cpuset.NewCPUSet(4, 5),
@@ -1009,6 +1024,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
},
stDefaultCPUSet: cpuset.NewCPUSet(4, 5),
pod: infraPod,
+ isKubeInfraPodfunc: fakeIsKubeInfraTrue,
expErr: nil,
expCPUAlloc: true,
expCSet: cpuset.NewCPUSet(0, 1),
@@ -1026,6 +1042,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
},
stDefaultCPUSet: cpuset.NewCPUSet(4, 5),
pod: infraPod,
+ isKubeInfraPodfunc: fakeIsKubeInfraTrue,
expErr: nil,
expCPUAlloc: true,
expCSet: cpuset.NewCPUSet(0),
@@ -1042,6 +1059,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
defaultCPUSet: testCase.stDefaultCPUSet,
}
+ varIsKubeInfra = testCase.isKubeInfraPodfunc
container := &testCase.pod.Spec.Containers[0]
err := policy.Allocate(st, testCase.pod, container)
if !reflect.DeepEqual(err, testCase.expErr) {
@@ -1161,3 +1179,132 @@ func TestStaticPolicyOptions(t *testing.T) {
})
}
}
+
+func makePodWithLabels(podLabels map[string]string) *v1.Pod {
+ return &v1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-pod",
+ Namespace: "test-namespace",
+ Labels: podLabels,
+ },
+ }
+}
+
+func fakeBuildConfigFromFlags(masterUrl string, kubeconfigPath string) (*restclient.Config, error) {
+
+ return &restclient.Config{}, nil
+}
+
+func fakeBuildConfigFromFlagsError(masterUrl string, kubeconfigPath string) (*restclient.Config, error) {
+
+ errString := fmt.Sprintf("%s file not found", kubeconfigPath)
+ return nil, errors.New(errString)
+
+}
+
+func getFakeInfraPodNamespace(_ string) (*v1.Namespace, error) {
+
+ return &v1.Namespace{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-namespace",
+ Labels: map[string]string{
+ "app.starlingx.io/component": "platform",
+ },
+ }}, nil
+}
+
+func getFakeNonInfraPodNamespace(_ string) (*v1.Namespace, error) {
+
+ return &v1.Namespace{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-namespace",
+ Labels: map[string]string{
+ "fake": "label",
+ }}}, nil
+
+}
+
+type kubeInfraPodTestCase struct {
+ description string
+ pod *v1.Pod
+ namespaceFunc getPodNamespace
+ expectedValue bool
+}
+
+func TestKubeInfraPod(t *testing.T) {
+ testCases := []kubeInfraPodTestCase{
+ {
+ description: "Pod with platform label and namespace with platform label",
+ pod: makePodWithLabels(map[string]string{
+ "app.starlingx.io/component": "platform",
+ }),
+ namespaceFunc: getFakeInfraPodNamespace,
+ expectedValue: true,
+ },
+ {
+ description: "Pod with platform label and namespace without platform label",
+ pod: makePodWithLabels(map[string]string{
+ "app.starlingx.io/component": "platform",
+ }),
+ namespaceFunc: getFakeNonInfraPodNamespace,
+ expectedValue: true,
+
+ },
+ {
+ description: "Pod without platform label and namespace with platform label",
+ pod: makePodWithLabels(map[string]string{
+ "test": "label",
+ }),
+ namespaceFunc: getFakeInfraPodNamespace,
+ expectedValue: true,
+ },
+ {
+ description: "Pod without platform label and namespace without platform label",
+ pod: makePodWithLabels(map[string]string{
+ "test": "namespace",
+ }),
+ namespaceFunc: getFakeNonInfraPodNamespace,
+ expectedValue: false,
+ },
+
+ }
+
+ for _, testCase := range testCases {
+ t.Run(testCase.description, func(t *testing.T) {
+
+ varGetNamespaceObject = testCase.namespaceFunc
+ varBuildConfigFromFlags = fakeBuildConfigFromFlags
+ gotValue := isKubeInfra(testCase.pod)
+
+ if gotValue != testCase.expectedValue {
+ t.Errorf("StaticPolicy isKubeInfraPod() error %v. expected value %v actual value %v",
+ testCase.description, testCase.expectedValue, gotValue)
+ } else {
+ fmt.Printf("StaticPolicy isKubeInfraPod() test successful. : %v ", testCase.description)
+ }
+
+ })
+ }
+
+ test := kubeInfraPodTestCase{
+ description: "Failure reading kubeconfig file",
+ pod: makePodWithLabels(map[string]string{
+ "test": "namespace",
+ }),
+ namespaceFunc: getFakeNonInfraPodNamespace,
+ expectedValue: false,
+ }
+
+ varGetNamespaceObject = getPodNamespaceObject
+ varBuildConfigFromFlags = fakeBuildConfigFromFlagsError
+
+ gotValue := isKubeInfra(test.pod)
+
+ if gotValue != test.expectedValue {
+ t.Errorf("StaticPolicy isKubeInfraPod() error %v. expected value %v actual value %v",
+ test.description, test.expectedValue, gotValue)
+ } else {
+ fmt.Printf("StaticPolicy isKubeInfraPod() test successful. : %v ", test.description)
+ }
+
+}
diff --git a/pkg/kubelet/cm/cpumanager/topology_hints_test.go b/pkg/kubelet/cm/cpumanager/topology_hints_test.go
index 02d064e71e3..d84b8240c30 100644
--- a/pkg/kubelet/cm/cpumanager/topology_hints_test.go
+++ b/pkg/kubelet/cm/cpumanager/topology_hints_test.go
@@ -145,6 +145,7 @@ func TestPodGuaranteedCPUs(t *testing.T) {
expectedCPU: 6,
},
}
+ varIsKubeInfra = fakeIsKubeInfraFalse
for _, tc := range tcases {
requestedCPU := p.podGuaranteedCPUs(tc.pod)
@@ -187,6 +188,7 @@ func TestGetTopologyHints(t *testing.T) {
sourcesReady: &sourcesReadyStub{},
}
+ varIsKubeInfra = fakeIsKubeInfraFalse
hints := m.GetTopologyHints(&tc.pod, &tc.container)[string(v1.ResourceCPU)]
if len(tc.expectedHints) == 0 && len(hints) == 0 {
continue
@@ -240,6 +242,7 @@ func TestGetPodTopologyHints(t *testing.T) {
sourcesReady: &sourcesReadyStub{},
}
+ varIsKubeInfra = fakeIsKubeInfraFalse
podHints := m.GetPodTopologyHints(&tc.pod)[string(v1.ResourceCPU)]
if len(tc.expectedHints) == 0 && len(podHints) == 0 {
continue
@@ -423,6 +426,7 @@ func TestGetPodTopologyHintsWithPolicyOptions(t *testing.T) {
sourcesReady: &sourcesReadyStub{},
}
+ varIsKubeInfra = fakeIsKubeInfraFalse
podHints := m.GetPodTopologyHints(&testCase.pod)[string(v1.ResourceCPU)]
sort.SliceStable(podHints, func(i, j int) bool {
return podHints[i].LessThan(podHints[j])
--
2.25.1

96
kubernetes/kubernetes-1.25.3/debian/deb_folder/patches/Revert-kubeadm-cleanup-the-master-taint-on-CP-nodes-.patch
View File

@ -1,96 +0,0 @@
From 094f22ddb7c82523c716b6866c7f9135da6119a6 Mon Sep 17 00:00:00 2001
From: Boovan Rajendran <boovan.rajendran@windriver.com>
Date: Wed, 15 Feb 2023 03:28:16 -0500
Subject: [PATCH] Revert "kubeadm: cleanup the "master" taint on CP nodes
during upgrade"
This partially reverts commit ddd046f3dd88186cbc83b57e83144db96eae4af4.
Signed-off-by: Boovan Rajendran <boovan.rajendran@windriver.com>
---
cmd/kubeadm/app/cmd/upgrade/apply.go | 17 ++++++++
cmd/kubeadm/app/phases/upgrade/postupgrade.go | 40 +++++++++++++++++++
2 files changed, 57 insertions(+)
diff --git a/cmd/kubeadm/app/cmd/upgrade/apply.go b/cmd/kubeadm/app/cmd/upgrade/apply.go
index 4687c5bcdd2..03c1bf19f14 100644
--- a/cmd/kubeadm/app/cmd/upgrade/apply.go
+++ b/cmd/kubeadm/app/cmd/upgrade/apply.go
@@ -158,6 +158,23 @@ func runApply(flags *applyFlags, args []string) error {
return errors.Wrap(err, "[upgrade/apply] FATAL")
}
+ // TODO: https://github.com/kubernetes/kubeadm/issues/2200
+ fmt.Printf("[upgrade/postupgrade] Removing the deprecated label %s='' from all control plane Nodes. "+
+ "After this step only the label %s='' will be present on control plane Nodes.\n",
+ kubeadmconstants.LabelNodeRoleOldControlPlane, kubeadmconstants.LabelNodeRoleControlPlane)
+ if err := upgrade.RemoveOldControlPlaneLabel(client); err != nil {
+ return err
+ }
+
+ // TODO: https://github.com/kubernetes/kubeadm/issues/2200
+ fmt.Printf("[upgrade/postupgrade] Adding the new taint %s to all control plane Nodes. "+
+ "After this step both taints %s and %s should be present on control plane Nodes.\n",
+ kubeadmconstants.ControlPlaneTaint.String(), kubeadmconstants.ControlPlaneTaint.String(),
+ kubeadmconstants.OldControlPlaneTaint.String())
+ if err := upgrade.AddNewControlPlaneTaint(client); err != nil {
+ return err
+ }
+
// Clean this up in 1.26
// TODO: https://github.com/kubernetes/kubeadm/issues/2200
fmt.Printf("[upgrade/postupgrade] Removing the old taint %s from all control plane Nodes. "+
diff --git a/cmd/kubeadm/app/phases/upgrade/postupgrade.go b/cmd/kubeadm/app/phases/upgrade/postupgrade.go
index eb67ccffaf8..95752b609b7 100644
--- a/cmd/kubeadm/app/phases/upgrade/postupgrade.go
+++ b/cmd/kubeadm/app/phases/upgrade/postupgrade.go
@@ -232,6 +232,46 @@ func RemoveOldControlPlaneLabel(client clientset.Interface) error {
return nil
}
+// AddNewControlPlaneTaint finds all nodes with the new "control-plane" node-role label
+// and adds the new "control-plane" taint to them.
+// TODO: https://github.com/kubernetes/kubeadm/issues/2200
+func AddNewControlPlaneTaint(client clientset.Interface) error {
+ selectorControlPlane := labels.SelectorFromSet(labels.Set(map[string]string{
+ kubeadmconstants.LabelNodeRoleControlPlane: "",
+ }))
+ nodes, err := client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{
+ LabelSelector: selectorControlPlane.String(),
+ })
+ if err != nil {
+ return errors.Wrapf(err, "could not list nodes labeled with %q", kubeadmconstants.LabelNodeRoleControlPlane)
+ }
+
+ for _, n := range nodes.Items {
+ // Check if the node has the old / new taints
+ hasOldTaint := false
+ hasNewTaint := false
+ for _, t := range n.Spec.Taints {
+ switch t.String() {
+ case kubeadmconstants.OldControlPlaneTaint.String():
+ hasOldTaint = true
+ case kubeadmconstants.ControlPlaneTaint.String():
+ hasNewTaint = true
+ }
+ }
+ // If the old taint is present and the new taint is missing, patch the node with the new taint.
+ // When the old taint is missing, assume the user has manually untainted the node and take no action.
+ if !hasNewTaint && hasOldTaint {
+ err = apiclient.PatchNode(client, n.Name, func(n *v1.Node) {
+ n.Spec.Taints = append(n.Spec.Taints, kubeadmconstants.ControlPlaneTaint)
+ })
+ if err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+}
+
// RemoveOldControlPlaneTaint finds all nodes with the new "control-plane" node-role label
// and removes the old "control-plane" taint to them.
// TODO: https://github.com/kubernetes/kubeadm/issues/2200
--
2.25.1

50
kubernetes/kubernetes-1.25.3/debian/deb_folder/patches/Revert-kubeadm-remove-RemoveOldControlPlaneLabel.patch
View File

@ -1,50 +0,0 @@
From dbf1b405fd31c548992fb73bafcb44c8ffe208ee Mon Sep 17 00:00:00 2001
From: Boovan Rajendran <boovan.rajendran@windriver.com>
Date: Wed, 15 Feb 2023 02:47:26 -0500
Subject: [PATCH] Revert "kubeadm: remove RemoveOldControlPlaneLabel"
This reverts commit 80e5bcae9b885179446fa27fa3b4da0992c1b8a8.
Signed-off-by: Boovan Rajendran <boovan.rajendran@windriver.com>
---
cmd/kubeadm/app/phases/upgrade/postupgrade.go | 24 +++++++++++++++++++
1 file changed, 24 insertions(+)
diff --git a/cmd/kubeadm/app/phases/upgrade/postupgrade.go b/cmd/kubeadm/app/phases/upgrade/postupgrade.go
index d6a5394ccde..eb67ccffaf8 100644
--- a/cmd/kubeadm/app/phases/upgrade/postupgrade.go
+++ b/cmd/kubeadm/app/phases/upgrade/postupgrade.go
@@ -208,6 +208,30 @@ func rollbackFiles(files map[string]string, originalErr error) error {
return errors.Errorf("couldn't move these files: %v. Got errors: %v", files, errorsutil.NewAggregate(errs))
}
+// RemoveOldControlPlaneLabel finds all nodes with the legacy node-role label and removes it
+// TODO: https://github.com/kubernetes/kubeadm/issues/2200
+func RemoveOldControlPlaneLabel(client clientset.Interface) error {
+ selectorOldControlPlane := labels.SelectorFromSet(labels.Set(map[string]string{
+ kubeadmconstants.LabelNodeRoleOldControlPlane: "",
+ }))
+ nodesWithOldLabel, err := client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{
+ LabelSelector: selectorOldControlPlane.String(),
+ })
+ if err != nil {
+ return errors.Wrapf(err, "could not list nodes labeled with %q", kubeadmconstants.LabelNodeRoleOldControlPlane)
+ }
+
+ for _, n := range nodesWithOldLabel.Items {
+ err = apiclient.PatchNode(client, n.Name, func(n *v1.Node) {
+ delete(n.ObjectMeta.Labels, kubeadmconstants.LabelNodeRoleOldControlPlane)
+ })
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
// RemoveOldControlPlaneTaint finds all nodes with the new "control-plane" node-role label
// and removes the old "control-plane" taint to them.
// TODO: https://github.com/kubernetes/kubeadm/issues/2200
--
2.25.1

169
kubernetes/kubernetes-1.25.3/debian/deb_folder/patches/kubeadm-create-platform-pods-with-zero-CPU-resources.patch
View File

@ -1,169 +0,0 @@
From 0dc5e95fb37f7df6a3edb7cfed312554fcca8270 Mon Sep 17 00:00:00 2001
From: Chris Friesen <chris.friesen@windriver.com>
Date: Fri, 3 Sep 2021 18:05:15 -0400
Subject: [PATCH] kubeadm: create platform pods with zero CPU resources
This specifies zero CPU resources when creating the manifests
for the static platform pods, as a workaround for the lack of
separate resource tracking for platform resources.
This specifies zero CPU and Memory resources for the coredns
deployment. manifests.go is the main source file for this,
not sure if the coredns.yaml are used but they are updated to
be consistent.
This specifies CPU limit of 1 for kube-apiserver pod so that it is
treated as a burstable QoS. This gives a boost of cgroup CPUShares
since the burstable cgroup parent has significantly more CPUShares
than best-effort on typical systems. This improves kube-apiserver
API responsiveness.
This increases kube-apiserver Readiness probe periodSeconds to 10
based on WRS/SS joint recommendation for minimum probe settings.
This reduces likelihood of kube-apiserver probe failure and
subsequent pod-restart under servere load. This also reduces CPU
demand.
Signed-off-by: Daniel Safta <daniel.safta@windriver.com>
Signed-off-by: Boovan Rajendran <boovan.rajendran@windriver.com>
Signed-off-by: Jim Gauld <James.Gauld@windriver.com>
---
cluster/addons/dns/coredns/coredns.yaml.base | 4 ++--
cluster/addons/dns/coredns/coredns.yaml.in | 4 ++--
cluster/addons/dns/coredns/coredns.yaml.sed | 4 ++--
cmd/kubeadm/app/phases/addons/dns/manifests.go | 4 ++--
.../app/phases/controlplane/manifests.go | 8 +++++---
cmd/kubeadm/app/util/staticpod/utils.go | 17 ++++++++++++++++-
6 files changed, 29 insertions(+), 12 deletions(-)
diff --git a/cluster/addons/dns/coredns/coredns.yaml.base b/cluster/addons/dns/coredns/coredns.yaml.base
index e03559423e6..49e88afc976 100644
--- a/cluster/addons/dns/coredns/coredns.yaml.base
+++ b/cluster/addons/dns/coredns/coredns.yaml.base
@@ -145,8 +145,8 @@ spec:
limits:
memory: __DNS__MEMORY__LIMIT__
requests:
- cpu: 100m
- memory: 70Mi
+ cpu: 0
+ memory: 0
args: [ "-conf", "/etc/coredns/Corefile" ]
volumeMounts:
- name: config-volume
diff --git a/cluster/addons/dns/coredns/coredns.yaml.in b/cluster/addons/dns/coredns/coredns.yaml.in
index 9b241370bea..78a23317b56 100644
--- a/cluster/addons/dns/coredns/coredns.yaml.in
+++ b/cluster/addons/dns/coredns/coredns.yaml.in
@@ -145,8 +145,8 @@ spec:
limits:
memory: 'dns_memory_limit'
requests:
- cpu: 100m
- memory: 70Mi
+ cpu: 0
+ memory: 0
args: [ "-conf", "/etc/coredns/Corefile" ]
volumeMounts:
- name: config-volume
diff --git a/cluster/addons/dns/coredns/coredns.yaml.sed b/cluster/addons/dns/coredns/coredns.yaml.sed
index 561fdf9aea8..536513d1e9d 100644
--- a/cluster/addons/dns/coredns/coredns.yaml.sed
+++ b/cluster/addons/dns/coredns/coredns.yaml.sed
@@ -145,8 +145,8 @@ spec:
limits:
memory: $DNS_MEMORY_LIMIT
requests:
- cpu: 100m
- memory: 70Mi
+ cpu: 0
+ memory: 0
args: [ "-conf", "/etc/coredns/Corefile" ]
volumeMounts:
- name: config-volume
diff --git a/cmd/kubeadm/app/phases/addons/dns/manifests.go b/cmd/kubeadm/app/phases/addons/dns/manifests.go
index bb5455a15ff..24b7112ae81 100644
--- a/cmd/kubeadm/app/phases/addons/dns/manifests.go
+++ b/cmd/kubeadm/app/phases/addons/dns/manifests.go
@@ -106,8 +106,8 @@ spec:
limits:
memory: 170Mi
requests:
- cpu: 100m
- memory: 70Mi
+ cpu: 0
+ memory: 0
args: [ "-conf", "/etc/coredns/Corefile" ]
volumeMounts:
- name: config-volume
diff --git a/cmd/kubeadm/app/phases/controlplane/manifests.go b/cmd/kubeadm/app/phases/controlplane/manifests.go
index 8cd0bf577cd..03e5739f5dc 100644
--- a/cmd/kubeadm/app/phases/controlplane/manifests.go
+++ b/cmd/kubeadm/app/phases/controlplane/manifests.go
@@ -63,7 +63,9 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap
LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/livez", int(endpoint.BindPort), v1.URISchemeHTTPS),
ReadinessProbe: staticpodutil.ReadinessProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/readyz", int(endpoint.BindPort), v1.URISchemeHTTPS),
StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/livez", int(endpoint.BindPort), v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane),
- Resources: staticpodutil.ComponentResources("250m"),
+ // WRS: Increase kube-apiserver cgroup CPUShares to improve API responsiveness;
+ // achieved by setting CPU Limits to make it burstable QoS.
+ Resources: staticpodutil.ComponentLimitResources("0", "1"),
Env: kubeadmutil.GetProxyEnvVars(),
}, mounts.GetVolumes(kubeadmconstants.KubeAPIServer),
map[string]string{kubeadmconstants.KubeAPIServerAdvertiseAddressEndpointAnnotationKey: endpoint.String()}),
@@ -75,7 +77,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap
VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeControllerManager)),
LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetControllerManagerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeControllerManagerPort, v1.URISchemeHTTPS),
StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetControllerManagerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeControllerManagerPort, v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane),
- Resources: staticpodutil.ComponentResources("200m"),
+ Resources: staticpodutil.ComponentResources("0"),
Env: kubeadmutil.GetProxyEnvVars(),
}, mounts.GetVolumes(kubeadmconstants.KubeControllerManager), nil),
kubeadmconstants.KubeScheduler: staticpodutil.ComponentPod(v1.Container{
@@ -86,7 +88,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap
VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeScheduler)),
LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetSchedulerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeSchedulerPort, v1.URISchemeHTTPS),
StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetSchedulerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeSchedulerPort, v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane),
- Resources: staticpodutil.ComponentResources("100m"),
+ Resources: staticpodutil.ComponentResources("0"),
Env: kubeadmutil.GetProxyEnvVars(),
}, mounts.GetVolumes(kubeadmconstants.KubeScheduler), nil),
}
diff --git a/cmd/kubeadm/app/util/staticpod/utils.go b/cmd/kubeadm/app/util/staticpod/utils.go
index 56a3f25b72a..4ea5e4635df 100644
--- a/cmd/kubeadm/app/util/staticpod/utils.go
+++ b/cmd/kubeadm/app/util/staticpod/utils.go
@@ -92,6 +92,18 @@ func ComponentResources(cpu string) v1.ResourceRequirements {
}
}
+// ComponentLimitResources returns the v1.ResourceRequirements object needed for allocating a specified amount of the CPU with Limits
+func ComponentLimitResources(cpu string, lcpu string) v1.ResourceRequirements {
+ return v1.ResourceRequirements{
+ Requests: v1.ResourceList{
+ v1.ResourceCPU: resource.MustParse(cpu),
+ },
+ Limits: v1.ResourceList{
+ v1.ResourceCPU: resource.MustParse(lcpu),
+ },
+ }
+}
+
// NewVolume creates a v1.Volume with a hostPath mount to the specified location
func NewVolume(name, path string, pathType *v1.HostPathType) v1.Volume {
return v1.Volume{
@@ -245,7 +257,10 @@ func LivenessProbe(host, path string, port int, scheme v1.URIScheme) *v1.Probe {
func ReadinessProbe(host, path string, port int, scheme v1.URIScheme) *v1.Probe {
// sets initialDelaySeconds as '0' because we don't want to delay user infrastructure checks
// looking for "ready" status on kubeadm static Pods
- return createHTTPProbe(host, path, port, scheme, 0, 15, 3, 1)
+ // WRS/SS joint recommendation: All pods probes should have following minimum probe
+ // settings unless required by the service (initialDelaySecond 0, periodSeconds 10,
+ // timeoutSeconds 5, successThreshold 1, failureThreshold 3)
+ return createHTTPProbe(host, path, port, scheme, 0, 15, 3, 10)
}
// StartupProbe creates a Probe object with a HTTPGet handler
--
2.25.1

67
kubernetes/kubernetes-1.25.3/debian/deb_folder/patches/kubeadm-readiness-probe-timeout-core-dns.patch
View File

@ -1,67 +0,0 @@
From 583589c3574ffd6e0376579316b30d2a2dcf82f8 Mon Sep 17 00:00:00 2001
From: Ferdinando Terada <Ferdinando.GodoyTerada@windriver.com>
Date: Mon, 23 Dec 2024 17:53:09 -0300
Subject: [PATCH] Adjust timeout for coredns readinessProbe
The timeout value for the readinessProbe of CoreDNS was increased.
This adjustment was necessary to avoid issues during stress testing,
ensuring that the component can properly handle high-load situations
and prevent premature failure in readiness checks.
---
cluster/addons/dns/coredns/coredns.yaml.base | 1 +
cluster/addons/dns/coredns/coredns.yaml.in | 1 +
cluster/addons/dns/coredns/coredns.yaml.sed | 1 +
cmd/kubeadm/app/phases/addons/dns/manifests.go | 1 +
4 files changed, 4 insertions(+)
diff --git a/cluster/addons/dns/coredns/coredns.yaml.base b/cluster/addons/dns/coredns/coredns.yaml.base
index 3a0fd7adb72..c8289f7c136 100644
--- a/cluster/addons/dns/coredns/coredns.yaml.base
+++ b/cluster/addons/dns/coredns/coredns.yaml.base
@@ -170,6 +170,7 @@ spec:
path: /ready
port: 8181
scheme: HTTP
+ timeoutSeconds: 5
securityContext:
allowPrivilegeEscalation: false
capabilities:
diff --git a/cluster/addons/dns/coredns/coredns.yaml.in b/cluster/addons/dns/coredns/coredns.yaml.in
index 74b59584bc7..974c8337031 100644
--- a/cluster/addons/dns/coredns/coredns.yaml.in
+++ b/cluster/addons/dns/coredns/coredns.yaml.in
@@ -170,6 +170,7 @@ spec:
path: /ready
port: 8181
scheme: HTTP
+ timeoutSeconds: 5
securityContext:
allowPrivilegeEscalation: false
capabilities:
diff --git a/cluster/addons/dns/coredns/coredns.yaml.sed b/cluster/addons/dns/coredns/coredns.yaml.sed
index 61afbecd9da..563a8980e07 100644
--- a/cluster/addons/dns/coredns/coredns.yaml.sed
+++ b/cluster/addons/dns/coredns/coredns.yaml.sed
@@ -170,6 +170,7 @@ spec:
path: /ready
port: 8181
scheme: HTTP
+ timeoutSeconds: 5
securityContext:
allowPrivilegeEscalation: false
capabilities:
diff --git a/cmd/kubeadm/app/phases/addons/dns/manifests.go b/cmd/kubeadm/app/phases/addons/dns/manifests.go
index 2a2212d5d37..c0be57357e4 100644
--- a/cmd/kubeadm/app/phases/addons/dns/manifests.go
+++ b/cmd/kubeadm/app/phases/addons/dns/manifests.go
@@ -135,6 +135,7 @@ spec:
path: /ready
port: 8181
scheme: HTTP
+ timeoutSeconds: 5
securityContext:
allowPrivilegeEscalation: false
capabilities:
--
2.34.1

33
kubernetes/kubernetes-1.25.3/debian/deb_folder/patches/kubeadm-reduce-UpgradeManifestTimeout.patch
View File

@ -1,33 +0,0 @@
From e03a40ad73d4c0e01b52c9df85d2cd735d75d403 Mon Sep 17 00:00:00 2001
From: Ramesh Kumar Sivanandam <rameshkumar.sivanandam@windriver.com>
Date: Fri, 15 Mar 2024 03:41:15 -0400
Subject: [PATCH] kubeadm: reduce UpgradeManifestTimeout
This modifies kubeadm UpgradeManifestTimeout from 5 minutes default
to 3 minutes to reduce the unnecessary delay in retries during
kubeadm-upgrade-apply failures.
The typical control-plane upgrade of static pods is 75 to 85 seconds,
so 3 minutes gives adequate buffer to complete the operation.
Signed-off-by: Ramesh Kumar Sivanandam <rameshkumar.sivanandam@windriver.com>
---
cmd/kubeadm/app/phases/upgrade/staticpods.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/cmd/kubeadm/app/phases/upgrade/staticpods.go b/cmd/kubeadm/app/phases/upgrade/staticpods.go
index 524a624c34f..37562342368 100644
--- a/cmd/kubeadm/app/phases/upgrade/staticpods.go
+++ b/cmd/kubeadm/app/phases/upgrade/staticpods.go
@@ -46,7 +46,7 @@ import (
const (
// UpgradeManifestTimeout is timeout of upgrading the static pod manifest
- UpgradeManifestTimeout = 5 * time.Minute
+ UpgradeManifestTimeout = 3 * time.Minute
)
// StaticPodPathManager is responsible for tracking the directories used in the static pod upgrade transition
--
2.25.1

30
kubernetes/kubernetes-1.25.3/debian/deb_folder/patches/kubelet-CFS-quota-throttling-for-non-integer-cpulimit.patch
View File

@ -1,30 +0,0 @@
From 2e957044cbcde858abb9c46d177d5cf4ae1407df Mon Sep 17 00:00:00 2001
From: Boovan Rajendran <boovan.rajendran@windriver.com>
Date: Wed, 30 Nov 2022 04:17:19 -0500
Subject: [PATCH] kubelet CFS quota throttling for non integer cpulimit
Signed-off-by: Boovan Rajendran <boovan.rajendran@windriver.com>
---
pkg/kubelet/cm/internal_container_lifecycle_linux.go | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/pkg/kubelet/cm/internal_container_lifecycle_linux.go b/pkg/kubelet/cm/internal_container_lifecycle_linux.go
index 75406dd8564..05366ab6fcb 100644
--- a/pkg/kubelet/cm/internal_container_lifecycle_linux.go
+++ b/pkg/kubelet/cm/internal_container_lifecycle_linux.go
@@ -39,7 +39,11 @@ func (i *internalContainerLifecycleImpl) PreCreateContainer(pod *v1.Pod, contain
// Disable cgroup CFS throttle at the container level.
// /sys/fs/cgroup/cpu/k8s-infra/kubepods/<pod>/<container>/cpu.cfs_quota_us
// /sys/fs/cgroup/cpu/k8s-infra/kubepods/<pod>/<container>/cpu.cfs_period_us
- if i.cpuManager.GetCPUPolicy() == "static" && v1qos.GetPodQOS(pod) == v1.PodQOSGuaranteed {
+ // We can only set CpuQuota to -1 if we're allocating the entire CPU.
+ // For fractional CPUs the CpuQuota is needed to enforce the limit.
+ cpuQuantity := container.Resources.Requests[v1.ResourceCPU]
+ fractionalCpuQuantity := cpuQuantity.MilliValue()%1000
+ if i.cpuManager.GetCPUPolicy() == "static" && v1qos.GetPodQOS(pod) == v1.PodQOSGuaranteed && fractionalCpuQuantity == 0 {
containerConfig.Linux.Resources.CpuPeriod = int64(100000)
containerConfig.Linux.Resources.CpuQuota = int64(-1)
}
--
2.25.1

255
kubernetes/kubernetes-1.25.3/debian/deb_folder/patches/kubelet-cpumanager-disable-CFS-quota-throttling.patch
View File

@ -1,255 +0,0 @@
From 95e547b2d3d0af6b0f2083c064bcbdbe39716250 Mon Sep 17 00:00:00 2001
From: Sachin Gopala Krishna <saching.krishna@windriver.com>
Date: Mon, 3 Oct 2022 19:19:48 -0400
Subject: [PATCH] kubelet cpumanager disable CFS quota throttling
This disables CFS CPU quota to avoid performance degradation due to
Linux kernel CFS quota implementation. Note that 4.18 kernel attempts
to solve the CFS throttling problem, but there are reports that it is
not completely effective.
This disables CFS quota throttling for Guaranteed pods for both
parent and container cgroups by writing -1 to cgroup cpu.cfs_quota_us.
Disabling has a dramatic latency improvement for HTTP response times.
This patch is refactored in 1.22.5 due to new internal_container_lifecycle
framework. We leverage the same mechanism to set Linux resources as:
cpu manager: specify the container CPU set during the creation
Co-authored-by: Jim Gauld <james.gauld@windriver.com>
Signed-off-by: Sachin Gopala Krishna <saching.krishna@windriver.com>
---
pkg/kubelet/cm/cpumanager/cpu_manager.go | 7 +++
pkg/kubelet/cm/cpumanager/fake_cpu_manager.go | 10 ++++-
pkg/kubelet/cm/helpers_linux.go | 10 +++++
pkg/kubelet/cm/helpers_linux_test.go | 43 ++++++++++---------
.../cm/internal_container_lifecycle_linux.go | 9 ++++
5 files changed, 57 insertions(+), 22 deletions(-)
diff --git a/pkg/kubelet/cm/cpumanager/cpu_manager.go b/pkg/kubelet/cm/cpumanager/cpu_manager.go
index dde49b6ec8c..df431b06601 100644
--- a/pkg/kubelet/cm/cpumanager/cpu_manager.go
+++ b/pkg/kubelet/cm/cpumanager/cpu_manager.go
@@ -72,6 +72,9 @@ type Manager interface {
// State returns a read-only interface to the internal CPU manager state.
State() state.Reader
+ // GetCPUPolicy returns the assigned CPU manager policy
+ GetCPUPolicy() string
+
// GetTopologyHints implements the topologymanager.HintProvider Interface
// and is consulted to achieve NUMA aware resource alignment among this
// and other resource controllers.
@@ -314,6 +317,10 @@ func (m *manager) State() state.Reader {
return m.state
}
+func (m *manager) GetCPUPolicy() string {
+ return m.policy.Name()
+}
+
func (m *manager) GetTopologyHints(pod *v1.Pod, container *v1.Container) map[string][]topologymanager.TopologyHint {
// The pod is during the admission phase. We need to save the pod to avoid it
// being cleaned before the admission ended
diff --git a/pkg/kubelet/cm/cpumanager/fake_cpu_manager.go b/pkg/kubelet/cm/cpumanager/fake_cpu_manager.go
index 93369705135..2e277da9c84 100644
--- a/pkg/kubelet/cm/cpumanager/fake_cpu_manager.go
+++ b/pkg/kubelet/cm/cpumanager/fake_cpu_manager.go
@@ -28,7 +28,8 @@ import (
)
type fakeManager struct {
- state state.State
+ policy Policy
+ state state.State
}
func (m *fakeManager) Start(activePods ActivePodsFunc, sourcesReady config.SourcesReady, podStatusProvider status.PodStatusProvider, containerRuntime runtimeService, initialContainers containermap.ContainerMap) error {
@@ -70,6 +71,10 @@ func (m *fakeManager) State() state.Reader {
return m.state
}
+func (m *fakeManager) GetCPUPolicy() string {
+ return m.policy.Name()
+}
+
func (m *fakeManager) GetExclusiveCPUs(podUID, containerName string) cpuset.CPUSet {
klog.InfoS("GetExclusiveCPUs", "podUID", podUID, "containerName", containerName)
return cpuset.CPUSet{}
@@ -88,6 +93,7 @@ func (m *fakeManager) GetCPUAffinity(podUID, containerName string) cpuset.CPUSet
// NewFakeManager creates empty/fake cpu manager
func NewFakeManager() Manager {
return &fakeManager{
- state: state.NewMemoryState(),
+ policy: &nonePolicy{},
+ state: state.NewMemoryState(),
}
}
diff --git a/pkg/kubelet/cm/helpers_linux.go b/pkg/kubelet/cm/helpers_linux.go
index 25ff3f13b82..e9ea6bab8dc 100644
--- a/pkg/kubelet/cm/helpers_linux.go
+++ b/pkg/kubelet/cm/helpers_linux.go
@@ -182,6 +182,16 @@ func ResourceConfigForPod(pod *v1.Pod, enforceCPULimits bool, cpuPeriod uint64,
// build the result
result := &ResourceConfig{}
if qosClass == v1.PodQOSGuaranteed {
+ // Disable CFS CPU quota to avoid performance degradation due to
+ // Linux kernel CFS throttle implementation.
+ // NOTE: 4.18 kernel attempts to solve CFS throttling problem,
+ // but there are reports that it is not completely effective.
+ // This will configure cgroup CFS parameters at pod level:
+ // /sys/fs/cgroup/cpu/k8s-infra/kubepods/<pod>/cpu.cfs_quota_us
+ // /sys/fs/cgroup/cpu/k8s-infra/kubepods/<pod>/cpu.cfs_period_us
+ cpuQuota = int64(-1)
+ cpuPeriod = uint64(100000)
+
result.CpuShares = &cpuShares
result.CpuQuota = &cpuQuota
result.CpuPeriod = &cpuPeriod
diff --git a/pkg/kubelet/cm/helpers_linux_test.go b/pkg/kubelet/cm/helpers_linux_test.go
index 101b21e682a..9b98fb7e1c1 100644
--- a/pkg/kubelet/cm/helpers_linux_test.go
+++ b/pkg/kubelet/cm/helpers_linux_test.go
@@ -64,8 +64,9 @@ func TestResourceConfigForPod(t *testing.T) {
burstablePartialShares := MilliCPUToShares(200)
burstableQuota := MilliCPUToQuota(200, int64(defaultQuotaPeriod))
guaranteedShares := MilliCPUToShares(100)
- guaranteedQuota := MilliCPUToQuota(100, int64(defaultQuotaPeriod))
- guaranteedTunedQuota := MilliCPUToQuota(100, int64(tunedQuotaPeriod))
+ guaranteedQuotaPeriod := uint64(100000)
+ guaranteedQuota := int64(-1)
+ guaranteedTunedQuota := int64(-1)
memoryQuantity = resource.MustParse("100Mi")
cpuNoLimit := int64(-1)
guaranteedMemory := memoryQuantity.Value()
@@ -204,8 +205,8 @@ func TestResourceConfigForPod(t *testing.T) {
},
},
enforceCPULimits: true,
- quotaPeriod: defaultQuotaPeriod,
- expected: &ResourceConfig{CpuShares: &guaranteedShares, CpuQuota: &guaranteedQuota, CpuPeriod: &defaultQuotaPeriod, Memory: &guaranteedMemory},
+ quotaPeriod: guaranteedQuotaPeriod,
+ expected: &ResourceConfig{CpuShares: &guaranteedShares, CpuQuota: &guaranteedQuota, CpuPeriod: &guaranteedQuotaPeriod, Memory: &guaranteedMemory},
},
"guaranteed-no-cpu-enforcement": {
pod: &v1.Pod{
@@ -218,8 +219,8 @@ func TestResourceConfigForPod(t *testing.T) {
},
},
enforceCPULimits: false,
- quotaPeriod: defaultQuotaPeriod,
- expected: &ResourceConfig{CpuShares: &guaranteedShares, CpuQuota: &cpuNoLimit, CpuPeriod: &defaultQuotaPeriod, Memory: &guaranteedMemory},
+ quotaPeriod: guaranteedQuotaPeriod,
+ expected: &ResourceConfig{CpuShares: &guaranteedShares, CpuQuota: &cpuNoLimit, CpuPeriod: &guaranteedQuotaPeriod, Memory: &guaranteedMemory},
},
"guaranteed-with-tuned-quota": {
pod: &v1.Pod{
@@ -232,8 +233,8 @@ func TestResourceConfigForPod(t *testing.T) {
},
},
enforceCPULimits: true,
- quotaPeriod: tunedQuotaPeriod,
- expected: &ResourceConfig{CpuShares: &guaranteedShares, CpuQuota: &guaranteedTunedQuota, CpuPeriod: &tunedQuotaPeriod, Memory: &guaranteedMemory},
+ quotaPeriod: guaranteedQuotaPeriod,
+ expected: &ResourceConfig{CpuShares: &guaranteedShares, CpuQuota: &guaranteedTunedQuota, CpuPeriod: &guaranteedQuotaPeriod, Memory: &guaranteedMemory},
},
"guaranteed-no-cpu-enforcement-with-tuned-quota": {
pod: &v1.Pod{
@@ -246,8 +247,8 @@ func TestResourceConfigForPod(t *testing.T) {
},
},
enforceCPULimits: false,
- quotaPeriod: tunedQuotaPeriod,
- expected: &ResourceConfig{CpuShares: &guaranteedShares, CpuQuota: &cpuNoLimit, CpuPeriod: &tunedQuotaPeriod, Memory: &guaranteedMemory},
+ quotaPeriod: guaranteedQuotaPeriod,
+ expected: &ResourceConfig{CpuShares: &guaranteedShares, CpuQuota: &cpuNoLimit, CpuPeriod: &guaranteedQuotaPeriod, Memory: &guaranteedMemory},
},
"burstable-partial-limits-with-init-containers": {
pod: &v1.Pod{
@@ -309,8 +310,10 @@ func TestResourceConfigForPodWithCustomCPUCFSQuotaPeriod(t *testing.T) {
burstablePartialShares := MilliCPUToShares(200)
burstableQuota := MilliCPUToQuota(200, int64(defaultQuotaPeriod))
guaranteedShares := MilliCPUToShares(100)
- guaranteedQuota := MilliCPUToQuota(100, int64(defaultQuotaPeriod))
- guaranteedTunedQuota := MilliCPUToQuota(100, int64(tunedQuotaPeriod))
+ guaranteedQuotaPeriod := uint64(100000)
+ guaranteedQuota := int64(-1)
+ guaranteedTunedQuota := int64(-1)
+
memoryQuantity = resource.MustParse("100Mi")
cpuNoLimit := int64(-1)
guaranteedMemory := memoryQuantity.Value()
@@ -449,8 +452,8 @@ func TestResourceConfigForPodWithCustomCPUCFSQuotaPeriod(t *testing.T) {
},
},
enforceCPULimits: true,
- quotaPeriod: defaultQuotaPeriod,
- expected: &ResourceConfig{CpuShares: &guaranteedShares, CpuQuota: &guaranteedQuota, CpuPeriod: &defaultQuotaPeriod, Memory: &guaranteedMemory},
+ quotaPeriod: guaranteedQuotaPeriod,
+ expected: &ResourceConfig{CpuShares: &guaranteedShares, CpuQuota: &guaranteedQuota, CpuPeriod: &guaranteedQuotaPeriod, Memory: &guaranteedMemory},
},
"guaranteed-no-cpu-enforcement": {
pod: &v1.Pod{
@@ -463,8 +466,8 @@ func TestResourceConfigForPodWithCustomCPUCFSQuotaPeriod(t *testing.T) {
},
},
enforceCPULimits: false,
- quotaPeriod: defaultQuotaPeriod,
- expected: &ResourceConfig{CpuShares: &guaranteedShares, CpuQuota: &cpuNoLimit, CpuPeriod: &defaultQuotaPeriod, Memory: &guaranteedMemory},
+ quotaPeriod: guaranteedQuotaPeriod,
+ expected: &ResourceConfig{CpuShares: &guaranteedShares, CpuQuota: &cpuNoLimit, CpuPeriod: &guaranteedQuotaPeriod, Memory: &guaranteedMemory},
},
"guaranteed-with-tuned-quota": {
pod: &v1.Pod{
@@ -477,8 +480,8 @@ func TestResourceConfigForPodWithCustomCPUCFSQuotaPeriod(t *testing.T) {
},
},
enforceCPULimits: true,
- quotaPeriod: tunedQuotaPeriod,
- expected: &ResourceConfig{CpuShares: &guaranteedShares, CpuQuota: &guaranteedTunedQuota, CpuPeriod: &tunedQuotaPeriod, Memory: &guaranteedMemory},
+ quotaPeriod: guaranteedQuotaPeriod,
+ expected: &ResourceConfig{CpuShares: &guaranteedShares, CpuQuota: &guaranteedTunedQuota, CpuPeriod: &guaranteedQuotaPeriod, Memory: &guaranteedMemory},
},
"guaranteed-no-cpu-enforcement-with-tuned-quota": {
pod: &v1.Pod{
@@ -491,8 +494,8 @@ func TestResourceConfigForPodWithCustomCPUCFSQuotaPeriod(t *testing.T) {
},
},
enforceCPULimits: false,
- quotaPeriod: tunedQuotaPeriod,
- expected: &ResourceConfig{CpuShares: &guaranteedShares, CpuQuota: &cpuNoLimit, CpuPeriod: &tunedQuotaPeriod, Memory: &guaranteedMemory},
+ quotaPeriod: guaranteedQuotaPeriod,
+ expected: &ResourceConfig{CpuShares: &guaranteedShares, CpuQuota: &cpuNoLimit, CpuPeriod: &guaranteedQuotaPeriod, Memory: &guaranteedMemory},
},
}
diff --git a/pkg/kubelet/cm/internal_container_lifecycle_linux.go b/pkg/kubelet/cm/internal_container_lifecycle_linux.go
index cb7c0cfa543..75406dd8564 100644
--- a/pkg/kubelet/cm/internal_container_lifecycle_linux.go
+++ b/pkg/kubelet/cm/internal_container_lifecycle_linux.go
@@ -25,6 +25,7 @@ import (
"k8s.io/api/core/v1"
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
+ v1qos "k8s.io/kubernetes/pkg/apis/core/v1/helper/qos"
)
func (i *internalContainerLifecycleImpl) PreCreateContainer(pod *v1.Pod, container *v1.Container, containerConfig *runtimeapi.ContainerConfig) error {
@@ -35,6 +36,14 @@ func (i *internalContainerLifecycleImpl) PreCreateContainer(pod *v1.Pod, contain
}
}
+ // Disable cgroup CFS throttle at the container level.
+ // /sys/fs/cgroup/cpu/k8s-infra/kubepods/<pod>/<container>/cpu.cfs_quota_us
+ // /sys/fs/cgroup/cpu/k8s-infra/kubepods/<pod>/<container>/cpu.cfs_period_us
+ if i.cpuManager.GetCPUPolicy() == "static" && v1qos.GetPodQOS(pod) == v1.PodQOSGuaranteed {
+ containerConfig.Linux.Resources.CpuPeriod = int64(100000)
+ containerConfig.Linux.Resources.CpuQuota = int64(-1)
+ }
+
if i.memoryManager != nil {
numaNodes := i.memoryManager.GetMemoryNUMANodes(pod, container)
if numaNodes.Len() > 0 {
--
2.25.1

143
kubernetes/kubernetes-1.25.3/debian/deb_folder/patches/kubelet-cpumanager-infra-pods-use-system-reserved-CP.patch
View File

@ -1,143 +0,0 @@
From 727313d6272fe7905a88fc403b27f4a6febd1369 Mon Sep 17 00:00:00 2001
From: Ramesh Kumar Sivanandam <rameshkumar.sivanandam@windriver.com>
Date: Tue, 15 Aug 2023 22:57:13 -0300
Subject: [PATCH] kubelet cpumanager infra pods use system reserved CPUs
This assigns system infrastructure pods to the "reserved" cpuset
to isolate them from the shared pool of CPUs.
Infrastructure pods include any pods that belong to the kube-system,
armada, cert-manager, vault, platform-deployment-manager, portieris,
notification, flux-helm, metrics-server, node-feature-discovery,
intel-power, power-metrics or sriov-fec-system namespaces.
The implementation is a bit simplistic, it is assumed that the
"reserved" cpuset is large enough to handle all infrastructure pods
CPU allocations.
This also prevents infrastucture pods from using Guaranteed resources.
Co-authored-by: Jim Gauld <james.gauld@windriver.com>
Signed-off-by: Gleb Aronsky <gleb.aronsky@windriver.com>
Signed-off-by: Thiago Miranda <ThiagoOliveira.Miranda@windriver.com>
Signed-off-by: Kaustubh Dhokte <kaustubh.dhokte@windriver.com>
Signed-off-by: Ramesh Kumar Sivanandam <rameshkumar.sivanandam@windriver.com>
Signed-off-by: Marcos Silva <Marcos.PauloOliveiraSilva@windriver.com>
Signed-off-by: Alyson Deives Pereira <alyson.deivespereira@windriver.com>
Signed-off-by: Marcos Silva <Marcos.PauloOliveiraSilva@windriver.com>
---
pkg/kubelet/cm/cpumanager/policy_static.go | 38 +++++++++++++++++++
.../cm/cpumanager/policy_static_test.go | 19 +++++++++-
2 files changed, 56 insertions(+), 1 deletion(-)
diff --git a/pkg/kubelet/cm/cpumanager/policy_static.go b/pkg/kubelet/cm/cpumanager/policy_static.go
index 14a6e7af9be..4fc96303622 100644
--- a/pkg/kubelet/cm/cpumanager/policy_static.go
+++ b/pkg/kubelet/cm/cpumanager/policy_static.go
@@ -53,6 +53,11 @@ func (e SMTAlignmentError) Type() string {
return ErrorSMTAlignment
}
+// Define namespaces used by platform infrastructure pods
+var infraNamespaces = [...]string{
+ "kube-system", "armada", "cert-manager", "platform-deployment-manager", "portieris", "vault", "notification", "flux-helm", "metrics-server", "node-feature-discovery", "intel-power", "power-metrics", "sriov-fec-system",
+}
+
// staticPolicy is a CPU manager policy that does not change CPU
// assignments for exclusively pinned guaranteed containers after the main
// container process starts.
@@ -268,6 +273,25 @@ func (p *staticPolicy) updateCPUsToReuse(pod *v1.Pod, container *v1.Container, c
}
func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Container) error {
+ // Process infra pods before guaranteed pods
+ if isKubeInfra(pod) {
+ // Container belongs in reserved pool.
+ // We don't want to fall through to the p.guaranteedCPUs() clause below so return either nil or error.
+ if _, ok := s.GetCPUSet(string(pod.UID), container.Name); ok {
+ klog.Infof("[cpumanager] static policy: reserved container already present in state, skipping (namespace: %s, pod UID: %s, pod: %s, container: %s)", pod.Namespace, string(pod.UID), pod.Name, container.Name)
+ return nil
+ }
+
+ cpuset := p.reserved
+ if cpuset.IsEmpty() {
+ // If this happens then someone messed up.
+ return fmt.Errorf("[cpumanager] static policy: reserved container unable to allocate cpus (namespace: %s, pod UID: %s, pod: %s, container: %s); cpuset=%v, reserved:%v", pod.Namespace, string(pod.UID), pod.Name, container.Name, cpuset, p.reserved)
+ }
+ s.SetCPUSet(string(pod.UID), container.Name, cpuset)
+ klog.Infof("[cpumanager] static policy: reserved: AddContainer (namespace: %s, pod UID: %s, pod: %s, container: %s); cpuset=%v", pod.Namespace, string(pod.UID), pod.Name, container.Name, cpuset)
+ return nil
+ }
+
if numCPUs := p.guaranteedCPUs(pod, container); numCPUs != 0 {
klog.InfoS("Static policy: Allocate", "pod", klog.KObj(pod), "containerName", container.Name)
// container belongs in an exclusively allocated pool
@@ -384,6 +408,10 @@ func (p *staticPolicy) guaranteedCPUs(pod *v1.Pod, container *v1.Container) int
if cpuQuantity.Value()*1000 != cpuQuantity.MilliValue() {
return 0
}
+ // Infrastructure pods use reserved CPUs even if they're in the Guaranteed QoS class
+ if isKubeInfra(pod) {
+ return 0
+ }
// Safe downcast to do for all systems with < 2.1 billion CPUs.
// Per the language spec, `int` is guaranteed to be at least 32 bits wide.
// https://golang.org/ref/spec#Numeric_types
@@ -602,6 +630,16 @@ func (p *staticPolicy) generateCPUTopologyHints(availableCPUs cpuset.CPUSet, reu
return hints
}
+// check if a given pod is in a platform infrastructure namespace
+func isKubeInfra(pod *v1.Pod) bool {
+ for _, namespace := range infraNamespaces {
+ if namespace == pod.Namespace {
+ return true
+ }
+ }
+ return false
+}
+
// isHintSocketAligned function return true if numa nodes in hint are socket aligned.
func (p *staticPolicy) isHintSocketAligned(hint topologymanager.TopologyHint, minAffinitySize int) bool {
numaNodesBitMask := hint.NUMANodeAffinity.GetBits()
diff --git a/pkg/kubelet/cm/cpumanager/policy_static_test.go b/pkg/kubelet/cm/cpumanager/policy_static_test.go
index 80a0c5a9e70..414e5ce144c 100644
--- a/pkg/kubelet/cm/cpumanager/policy_static_test.go
+++ b/pkg/kubelet/cm/cpumanager/policy_static_test.go
@@ -939,7 +939,8 @@ func TestStaticPolicyStartWithResvList(t *testing.T) {
}
func TestStaticPolicyAddWithResvList(t *testing.T) {
-
+ infraPod := makePod("fakePod", "fakeContainer2", "200m", "200m")
+ infraPod.Namespace = "kube-system"
testCases := []staticPolicyTestWithResvList{
{
description: "GuPodSingleCore, SingleSocketHT, ExpectError",
@@ -981,6 +982,22 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
expCPUAlloc: true,
expCSet: cpuset.NewCPUSet(4, 5),
},
+ {
+ description: "InfraPod, SingleSocketHT, ExpectAllocReserved",
+ topo: topoSingleSocketHT,
+ numReservedCPUs: 2,
+ reserved: cpuset.NewCPUSet(0, 1),
+ stAssignments: state.ContainerCPUAssignments{
+ "fakePod": map[string]cpuset.CPUSet{
+ "fakeContainer100": cpuset.NewCPUSet(2, 3, 6, 7),
+ },
+ },
+ stDefaultCPUSet: cpuset.NewCPUSet(4, 5),
+ pod: infraPod,
+ expErr: nil,
+ expCPUAlloc: true,
+ expCSet: cpuset.NewCPUSet(0, 1),
+ },
}
testExcl := true
--
2.25.1

613
kubernetes/kubernetes-1.25.3/debian/deb_folder/patches/kubelet-cpumanager-introduce-concept-of-isolated-CPU.patch
View File

@ -1,613 +0,0 @@
From e8608eb7c300b6e0503885a4848fafc75f20d909 Mon Sep 17 00:00:00 2001
From: Ramesh Kumar Sivanandam <rameshkumar.sivanandam@windriver.com>
Date: Mon, 7 Nov 2022 13:33:03 -0500
Subject: [PATCH] kubelet cpumanager introduce concept of isolated CPUs
This introduces the concept of "isolated CPUs", which are CPUs that
have been isolated at the kernel level via the "isolcpus" kernel boot
parameter.
When starting the kubelet process, two separate sets of reserved CPUs
may be specified. With this change CPUs reserved via
'--system-reserved=cpu' will be used for infrastructure pods while the
isolated CPUs should be reserved via '--kube-reserved=cpu' to cause
kubelet to skip over them for "normal" CPU resource tracking. The
kubelet code will double-check that the specified isolated CPUs match
what the kernel exposes in "/sys/devices/system/cpu/isolated".
A plugin (outside the scope of this commit) will expose the isolated
CPUs to kubelet via the device plugin API.
If a pod specifies some number of "isolcpus" resources, the device
manager will allocate them. In this code we check whether such
resources have been allocated, and if so we set the container cpuset to
the isolated CPUs. This does mean that it really only makes sense to
specify "isolcpus" resources for best-effort or burstable pods, not for
guaranteed ones since that would throw off the accounting code. In
order to ensure the accounting still works as designed, if "isolcpus"
are specified for guaranteed pods, the affinity will be set to the
non-isolated CPUs.
This patch was refactored in 1.21.3 due to upstream API change
node: podresources: make GetDevices() consistent
(commit ad68f9588c72d6477b5a290c548a9031063ac659).
The routine podIsolCPUs() was refactored in 1.21.3 since the API
p.deviceManager.GetDevices() is returning multiple devices with
a device per cpu. The resultant cpuset needs to be the aggregate.
The routine NewStaticPolicy was refactored in 1.22.5, adding a new argument
in its signature: cpuPolicyOptions map[string]string. This change is implies
shifting the new arguments(deviceManager, excludeReserved) with one position
to the right.
Co-authored-by: Jim Gauld <james.gauld@windriver.com>
Co-authored-by: Chris Friesen <chris.friesen@windriver.com>
Signed-off-by: Gleb Aronsky <gleb.aronsky@windriver.com>
Signed-off-by: Ramesh Kumar Sivanandam <rameshkumar.sivanandam@windriver.com>
---
pkg/kubelet/cm/container_manager_linux.go | 1 +
pkg/kubelet/cm/cpumanager/cpu_manager.go | 35 +++++++-
pkg/kubelet/cm/cpumanager/cpu_manager_test.go | 20 ++++-
pkg/kubelet/cm/cpumanager/policy_static.go | 83 +++++++++++++++++--
.../cm/cpumanager/policy_static_test.go | 53 +++++++++---
5 files changed, 172 insertions(+), 20 deletions(-)
diff --git a/pkg/kubelet/cm/container_manager_linux.go b/pkg/kubelet/cm/container_manager_linux.go
index 44c8cda6c40..a3f92b23c69 100644
--- a/pkg/kubelet/cm/container_manager_linux.go
+++ b/pkg/kubelet/cm/container_manager_linux.go
@@ -321,6 +321,7 @@ func NewContainerManager(mountUtil mount.Interface, cadvisorInterface cadvisor.I
cm.GetNodeAllocatableReservation(),
nodeConfig.KubeletRootDir,
cm.topologyManager,
+ cm.deviceManager,
)
if err != nil {
klog.ErrorS(err, "Failed to initialize cpu manager")
diff --git a/pkg/kubelet/cm/cpumanager/cpu_manager.go b/pkg/kubelet/cm/cpumanager/cpu_manager.go
index 00c034bc753..5a5c4ad9f6f 100644
--- a/pkg/kubelet/cm/cpumanager/cpu_manager.go
+++ b/pkg/kubelet/cm/cpumanager/cpu_manager.go
@@ -18,7 +18,9 @@ package cpumanager
import (
"fmt"
+ "io/ioutil"
"math"
+ "strings"
"sync"
"time"
@@ -32,6 +34,7 @@ import (
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state"
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
"k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
+ "k8s.io/kubernetes/pkg/kubelet/cm/devicemanager"
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager"
"k8s.io/kubernetes/pkg/kubelet/config"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
@@ -50,6 +53,25 @@ type policyName string
// cpuManagerStateFileName is the file name where cpu manager stores its state
const cpuManagerStateFileName = "cpu_manager_state"
+// get the system-level isolated CPUs
+func getIsolcpus() cpuset.CPUSet {
+ dat, err := ioutil.ReadFile("/sys/devices/system/cpu/isolated")
+ if err != nil {
+ klog.Errorf("[cpumanager] unable to read sysfs isolcpus subdir")
+ return cpuset.NewCPUSet()
+ }
+
+ // The isolated cpus string ends in a newline
+ cpustring := strings.TrimSuffix(string(dat), "\n")
+ cset, err := cpuset.Parse(cpustring)
+ if err != nil {
+ klog.Errorf("[cpumanager] unable to parse sysfs isolcpus string to cpuset")
+ return cpuset.NewCPUSet()
+ }
+
+ return cset
+}
+
// Manager interface provides methods for Kubelet to manage pod cpus.
type Manager interface {
// Start is called during Kubelet initialization.
@@ -153,7 +175,8 @@ func (s *sourcesReadyStub) AddSource(source string) {}
func (s *sourcesReadyStub) AllReady() bool { return true }
// NewManager creates new cpu manager based on provided policy
-func NewManager(cpuPolicyName string, cpuPolicyOptions map[string]string, reconcilePeriod time.Duration, machineInfo *cadvisorapi.MachineInfo, specificCPUs cpuset.CPUSet, nodeAllocatableReservation v1.ResourceList, stateFileDirectory string, affinity topologymanager.Store) (Manager, error) {
+func NewManager(cpuPolicyName string, cpuPolicyOptions map[string]string, reconcilePeriod time.Duration, machineInfo *cadvisorapi.MachineInfo, specificCPUs cpuset.CPUSet, nodeAllocatableReservation v1.ResourceList, stateFileDirectory string, affinity topologymanager.Store, deviceManager devicemanager.Manager) (Manager, error) {
+
var topo *topology.CPUTopology
var policy Policy
var err error
@@ -194,7 +217,15 @@ func NewManager(cpuPolicyName string, cpuPolicyOptions map[string]string, reconc
// NOTE: Set excludeReserved unconditionally to exclude reserved CPUs from default cpuset.
// This variable is primarily to make testing easier.
excludeReserved := true
- policy, err = NewStaticPolicy(topo, numReservedCPUs, specificCPUs, affinity, cpuPolicyOptions, excludeReserved)
+
+ // isolCPUs is the set of kernel-isolated CPUs. They should be a subset of specificCPUs or
+ // of the CPUs that NewStaticPolicy() will pick if numReservedCPUs is set. It's only in the
+ // argument list here for ease of testing, it's really internal to the policy.
+ isolCPUs := getIsolcpus()
+ policy, err = NewStaticPolicy(topo, numReservedCPUs, specificCPUs, isolCPUs, affinity, cpuPolicyOptions, deviceManager, excludeReserved)
+ if err != nil {
+ return nil, fmt.Errorf("new static policy error: %v", err)
+ }
if err != nil {
return nil, fmt.Errorf("new static policy error: %w", err)
diff --git a/pkg/kubelet/cm/cpumanager/cpu_manager_test.go b/pkg/kubelet/cm/cpumanager/cpu_manager_test.go
index d553b182e0b..57f3f9a1c97 100644
--- a/pkg/kubelet/cm/cpumanager/cpu_manager_test.go
+++ b/pkg/kubelet/cm/cpumanager/cpu_manager_test.go
@@ -37,6 +37,7 @@ import (
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state"
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
"k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
+ "k8s.io/kubernetes/pkg/kubelet/cm/devicemanager"
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager"
)
@@ -215,6 +216,7 @@ func makeMultiContainerPod(initCPUs, appCPUs []struct{ request, limit string })
}
func TestCPUManagerAdd(t *testing.T) {
+ testDM, _ := devicemanager.NewManagerStub()
testExcl := false
testPolicy, _ := NewStaticPolicy(
&topology.CPUTopology{
@@ -230,8 +232,10 @@ func TestCPUManagerAdd(t *testing.T) {
},
0,
cpuset.NewCPUSet(),
+ cpuset.NewCPUSet(),
topologymanager.NewFakeManager(),
nil,
+ testDM,
testExcl)
testCases := []struct {
description string
@@ -482,8 +486,9 @@ func TestCPUManagerAddWithInitContainers(t *testing.T) {
}
testExcl := false
+ testDM, _ := devicemanager.NewManagerStub()
for _, testCase := range testCases {
- policy, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil, testExcl)
+ policy, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, cpuset.NewCPUSet(), cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil, testDM, testExcl)
mockState := &mockState{
assignments: testCase.stAssignments,
@@ -638,7 +643,9 @@ func TestCPUManagerGenerate(t *testing.T) {
}
defer os.RemoveAll(sDir)
- mgr, err := NewManager(testCase.cpuPolicyName, nil, 5*time.Second, machineInfo, cpuset.NewCPUSet(), testCase.nodeAllocatableReservation, sDir, topologymanager.NewFakeManager())
+ testDM, err := devicemanager.NewManagerStub()
+ mgr, err := NewManager(testCase.cpuPolicyName, nil, 5*time.Second, machineInfo, cpuset.NewCPUSet(), testCase.nodeAllocatableReservation, sDir, topologymanager.NewFakeManager(), testDM)
+
if testCase.expectedError != nil {
if !strings.Contains(err.Error(), testCase.expectedError.Error()) {
t.Errorf("Unexpected error message. Have: %s wants %s", err.Error(), testCase.expectedError.Error())
@@ -709,6 +716,7 @@ func TestCPUManagerRemove(t *testing.T) {
func TestReconcileState(t *testing.T) {
testExcl := false
+ testDM, _ := devicemanager.NewManagerStub()
testPolicy, _ := NewStaticPolicy(
&topology.CPUTopology{
NumCPUs: 8,
@@ -727,8 +735,10 @@ func TestReconcileState(t *testing.T) {
},
0,
cpuset.NewCPUSet(),
+ cpuset.NewCPUSet(),
topologymanager.NewFakeManager(),
nil,
+ testDM,
testExcl)
testCases := []struct {
@@ -1234,6 +1244,7 @@ func TestReconcileState(t *testing.T) {
// the following tests are with --reserved-cpus configured
func TestCPUManagerAddWithResvList(t *testing.T) {
testExcl := false
+ testDM, _ := devicemanager.NewManagerStub()
testPolicy, _ := NewStaticPolicy(
&topology.CPUTopology{
NumCPUs: 4,
@@ -1248,8 +1259,10 @@ func TestCPUManagerAddWithResvList(t *testing.T) {
},
1,
cpuset.NewCPUSet(0),
+ cpuset.NewCPUSet(),
topologymanager.NewFakeManager(),
nil,
+ testDM,
testExcl)
testCases := []struct {
description string
@@ -1362,7 +1375,8 @@ func TestCPUManagerHandlePolicyOptions(t *testing.T) {
}
defer os.RemoveAll(sDir)
- _, err = NewManager(testCase.cpuPolicyName, testCase.cpuPolicyOptions, 5*time.Second, machineInfo, cpuset.NewCPUSet(), nodeAllocatableReservation, sDir, topologymanager.NewFakeManager())
+ testDM, err := devicemanager.NewManagerStub()
+ _, err = NewManager(testCase.cpuPolicyName, testCase.cpuPolicyOptions, 5*time.Second, machineInfo, cpuset.NewCPUSet(), nodeAllocatableReservation, sDir, topologymanager.NewFakeManager(), testDM)
if err == nil {
t.Errorf("Expected error, but NewManager succeeded")
}
diff --git a/pkg/kubelet/cm/cpumanager/policy_static.go b/pkg/kubelet/cm/cpumanager/policy_static.go
index 341e9f3dffe..802e289bfaf 100644
--- a/pkg/kubelet/cm/cpumanager/policy_static.go
+++ b/pkg/kubelet/cm/cpumanager/policy_static.go
@@ -18,6 +18,7 @@ package cpumanager
import (
"fmt"
+ "strconv"
v1 "k8s.io/api/core/v1"
"k8s.io/klog/v2"
@@ -25,6 +26,7 @@ import (
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state"
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
"k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
+ "k8s.io/kubernetes/pkg/kubelet/cm/devicemanager"
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager"
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask"
)
@@ -101,6 +103,10 @@ type staticPolicy struct {
topology *topology.CPUTopology
// set of CPUs that is not available for exclusive assignment
reserved cpuset.CPUSet
+ // subset of reserved CPUs with isolcpus attribute
+ isolcpus cpuset.CPUSet
+ // parent containerManager, used to get device list
+ deviceManager devicemanager.Manager
// If true, default CPUSet should exclude reserved CPUs
excludeReserved bool
// topology manager reference to get container Topology affinity
@@ -117,7 +123,8 @@ var _ Policy = &staticPolicy{}
// NewStaticPolicy returns a CPU manager policy that does not change CPU
// assignments for exclusively pinned guaranteed containers after the main
// container process starts.
-func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reservedCPUs cpuset.CPUSet, affinity topologymanager.Store, cpuPolicyOptions map[string]string, excludeReserved bool) (Policy, error) {
+func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reservedCPUs cpuset.CPUSet, isolCPUs cpuset.CPUSet, affinity topologymanager.Store, cpuPolicyOptions map[string]string, deviceManager devicemanager.Manager, excludeReserved bool) (Policy, error) {
+
opts, err := NewStaticPolicyOptions(cpuPolicyOptions)
if err != nil {
return nil, err
@@ -132,6 +139,8 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv
policy := &staticPolicy{
topology: topology,
affinity: affinity,
+ isolcpus: isolCPUs,
+ deviceManager: deviceManager,
excludeReserved: excludeReserved,
cpusToReuse: make(map[string]cpuset.CPUSet),
options: opts,
@@ -158,6 +167,12 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv
klog.InfoS("Reserved CPUs not available for exclusive assignment", "reservedSize", reserved.Size(), "reserved", reserved)
policy.reserved = reserved
+ if !isolCPUs.IsSubsetOf(reserved) {
+ klog.Errorf("[cpumanager] isolCPUs %v is not a subset of reserved %v", isolCPUs, reserved)
+ reserved = reserved.Union(isolCPUs)
+ klog.Warningf("[cpumanager] mismatch isolCPUs %v, force reserved %v", isolCPUs, reserved)
+ }
+
return policy, nil
}
@@ -191,8 +206,9 @@ func (p *staticPolicy) validateState(s state.State) error {
} else {
s.SetDefaultCPUSet(allCPUs)
}
- klog.Infof("[cpumanager] static policy: CPUSet: allCPUs:%v, reserved:%v, default:%v\n",
- allCPUs, p.reserved, s.GetDefaultCPUSet())
+ klog.Infof("[cpumanager] static policy: CPUSet: allCPUs:%v, reserved:%v, isolcpus:%v, default:%v\n",
+ allCPUs, p.reserved, p.isolcpus, s.GetDefaultCPUSet())
+
return nil
}
@@ -282,10 +298,11 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai
return nil
}
- cpuset := p.reserved
+ cpuset := p.reserved.Clone().Difference(p.isolcpus)
if cpuset.IsEmpty() {
// If this happens then someone messed up.
- return fmt.Errorf("[cpumanager] static policy: reserved container unable to allocate cpus (namespace: %s, pod UID: %s, pod: %s, container: %s); cpuset=%v, reserved:%v", pod.Namespace, string(pod.UID), pod.Name, container.Name, cpuset, p.reserved)
+ return fmt.Errorf("[cpumanager] static policy: reserved container unable to allocate cpus (namespace: %s, pod UID: %s, pod: %s, container: %s); cpuset=%v, reserved:%v, isolcpus:%v", pod.Namespace, string(pod.UID), pod.Name, container.Name, cpuset, p.reserved, p.isolcpus)
+
}
s.SetCPUSet(string(pod.UID), container.Name, cpuset)
klog.Infof("[cpumanager] static policy: reserved: AddContainer (namespace: %s, pod UID: %s, pod: %s, container: %s); cpuset=%v", pod.Namespace, string(pod.UID), pod.Name, container.Name, cpuset)
@@ -329,8 +346,34 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai
}
s.SetCPUSet(string(pod.UID), container.Name, cpuset)
p.updateCPUsToReuse(pod, container, cpuset)
+ klog.Infof("[cpumanager] guaranteed: AddContainer "+
+ "(namespace: %s, pod UID: %s, pod: %s, container: %s); numCPUS=%d, cpuset=%v",
+ pod.Namespace, string(pod.UID), pod.Name, container.Name, numCPUs, cpuset)
+ return nil
+ }
+ if isolcpus := p.podIsolCPUs(pod, container); isolcpus.Size() > 0 {
+ // container has requested isolated CPUs
+ if set, ok := s.GetCPUSet(string(pod.UID), container.Name); ok {
+ if set.Equals(isolcpus) {
+ klog.Infof("[cpumanager] isolcpus container already present in state, skipping (namespace: %s, pod UID: %s, pod: %s, container: %s)",
+ pod.Namespace, string(pod.UID), pod.Name, container.Name)
+ return nil
+ } else {
+ klog.Infof("[cpumanager] isolcpus container state has cpus %v, should be %v (namespace: %s, pod UID: %s, pod: %s, container: %s)",
+ isolcpus, set, pod.Namespace, string(pod.UID), pod.Name, container.Name)
+ }
+ }
+ // Note that we do not do anything about init containers here.
+ // It looks like devices are allocated per-pod based on effective requests/limits
+ // and extra devices from initContainers are not freed up when the regular containers start.
+ // TODO: confirm this is still true for 1.20
+ s.SetCPUSet(string(pod.UID), container.Name, isolcpus)
+ klog.Infof("[cpumanager] isolcpus: AddContainer (namespace: %s, pod UID: %s, pod: %s, container: %s); cpuset=%v",
+ pod.Namespace, string(pod.UID), pod.Name, container.Name, isolcpus)
+ return nil
}
+
// container belongs in the shared pool (nothing to do; use default cpuset)
return nil
}
@@ -640,6 +683,36 @@ func isKubeInfra(pod *v1.Pod) bool {
return false
}
+// get the isolated CPUs (if any) from the devices associated with a specific container
+func (p *staticPolicy) podIsolCPUs(pod *v1.Pod, container *v1.Container) cpuset.CPUSet {
+ // NOTE: This is required for TestStaticPolicyAdd() since makePod() does
+ // not create UID. We also need a way to properly stub devicemanager.
+ if len(string(pod.UID)) == 0 {
+ return cpuset.NewCPUSet()
+ }
+ resContDevices := p.deviceManager.GetDevices(string(pod.UID), container.Name)
+ cpuSet := cpuset.NewCPUSet()
+ for resourceName, resourceDevs := range resContDevices {
+ // this resource name needs to match the isolcpus device plugin
+ if resourceName == "windriver.com/isolcpus" {
+ for devID, _ := range resourceDevs {
+ cpuStrList := []string{devID}
+ if len(cpuStrList) > 0 {
+ // loop over the list of strings, convert each one to int, add to cpuset
+ for _, cpuStr := range cpuStrList {
+ cpu, err := strconv.Atoi(cpuStr)
+ if err != nil {
+ panic(err)
+ }
+ cpuSet = cpuSet.Union(cpuset.NewCPUSet(cpu))
+ }
+ }
+ }
+ }
+ }
+ return cpuSet
+}
+
// isHintSocketAligned function return true if numa nodes in hint are socket aligned.
func (p *staticPolicy) isHintSocketAligned(hint topologymanager.TopologyHint, minAffinitySize int) bool {
numaNodesBitMask := hint.NUMANodeAffinity.GetBits()
diff --git a/pkg/kubelet/cm/cpumanager/policy_static_test.go b/pkg/kubelet/cm/cpumanager/policy_static_test.go
index 414e5ce144c..1c43df3b85f 100644
--- a/pkg/kubelet/cm/cpumanager/policy_static_test.go
+++ b/pkg/kubelet/cm/cpumanager/policy_static_test.go
@@ -28,6 +28,7 @@ import (
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state"
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
"k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
+ "k8s.io/kubernetes/pkg/kubelet/cm/devicemanager"
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager"
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask"
)
@@ -69,8 +70,9 @@ func (spt staticPolicyTest) PseudoClone() staticPolicyTest {
}
func TestStaticPolicyName(t *testing.T) {
+ testDM, _ := devicemanager.NewManagerStub()
testExcl := false
- policy, _ := NewStaticPolicy(topoSingleSocketHT, 1, cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil, testExcl)
+ policy, _ := NewStaticPolicy(topoSingleSocketHT, 1, cpuset.NewCPUSet(), cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil, testDM, testExcl)
policyName := policy.Name()
if policyName != "static" {
@@ -80,6 +82,7 @@ func TestStaticPolicyName(t *testing.T) {
}
func TestStaticPolicyStart(t *testing.T) {
+ testDM, _ := devicemanager.NewManagerStub()
testCases := []staticPolicyTest{
{
description: "non-corrupted state",
@@ -155,7 +158,7 @@ func TestStaticPolicyStart(t *testing.T) {
}
for _, testCase := range testCases {
t.Run(testCase.description, func(t *testing.T) {
- p, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil, testCase.excludeReserved)
+ p, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, cpuset.NewCPUSet(), cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil, testDM, testCase.excludeReserved)
policy := p.(*staticPolicy)
st := &mockState{
@@ -203,7 +206,6 @@ func TestStaticPolicyAdd(t *testing.T) {
largeTopoCPUSet := largeTopoBuilder.Result()
largeTopoSock0CPUSet := largeTopoSock0Builder.Result()
largeTopoSock1CPUSet := largeTopoSock1Builder.Result()
-
// these are the cases which must behave the same regardless the policy options.
// So we will permutate the options to ensure this holds true.
@@ -577,8 +579,10 @@ func runStaticPolicyTestCase(t *testing.T, testCase staticPolicyTest) {
if testCase.topologyHint != nil {
tm = topologymanager.NewFakeManagerWithHint(testCase.topologyHint)
}
+ testDM, _ := devicemanager.NewManagerStub()
testExcl := false
- policy, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, cpuset.NewCPUSet(), tm, testCase.options, testExcl)
+ policy, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, cpuset.NewCPUSet(), cpuset.NewCPUSet(), tm, testCase.options, testDM, testExcl)
+
st := &mockState{
assignments: testCase.stAssignments,
@@ -625,6 +629,8 @@ func runStaticPolicyTestCaseWithFeatureGate(t *testing.T, testCase staticPolicyT
}
func TestStaticPolicyReuseCPUs(t *testing.T) {
+ excludeReserved := false
+ testDM, _ := devicemanager.NewManagerStub()
testCases := []struct {
staticPolicyTest
expCSetAfterAlloc cpuset.CPUSet
@@ -649,7 +655,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) {
}
for _, testCase := range testCases {
- policy, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil, testCase.excludeReserved)
+ policy, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, cpuset.NewCPUSet(), cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil, testDM, excludeReserved)
st := &mockState{
assignments: testCase.stAssignments,
@@ -682,6 +688,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) {
func TestStaticPolicyRemove(t *testing.T) {
excludeReserved := false
+ testDM, _ := devicemanager.NewManagerStub()
testCases := []staticPolicyTest{
{
description: "SingleSocketHT, DeAllocOneContainer",
@@ -740,7 +747,7 @@ func TestStaticPolicyRemove(t *testing.T) {
}
for _, testCase := range testCases {
- policy, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil, excludeReserved)
+ policy, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, cpuset.NewCPUSet(), cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil, testDM, excludeReserved)
st := &mockState{
assignments: testCase.stAssignments,
@@ -763,6 +770,7 @@ func TestStaticPolicyRemove(t *testing.T) {
func TestTopologyAwareAllocateCPUs(t *testing.T) {
excludeReserved := false
+ testDM, _ := devicemanager.NewManagerStub()
testCases := []struct {
description string
topo *topology.CPUTopology
@@ -831,7 +839,8 @@ func TestTopologyAwareAllocateCPUs(t *testing.T) {
},
}
for _, tc := range testCases {
- p, _ := NewStaticPolicy(tc.topo, 0, cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil, excludeReserved)
+ p, _ := NewStaticPolicy(tc.topo, 0, cpuset.NewCPUSet(), cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil, testDM, excludeReserved)
+
policy := p.(*staticPolicy)
st := &mockState{
assignments: tc.stAssignments,
@@ -864,6 +873,7 @@ type staticPolicyTestWithResvList struct {
topo *topology.CPUTopology
numReservedCPUs int
reserved cpuset.CPUSet
+ isolcpus cpuset.CPUSet
stAssignments state.ContainerCPUAssignments
stDefaultCPUSet cpuset.CPUSet
pod *v1.Pod
@@ -874,6 +884,8 @@ type staticPolicyTestWithResvList struct {
}
func TestStaticPolicyStartWithResvList(t *testing.T) {
+ testDM, _ := devicemanager.NewManagerStub()
+ testExcl := false
testCases := []staticPolicyTestWithResvList{
{
description: "empty cpuset",
@@ -903,11 +915,10 @@ func TestStaticPolicyStartWithResvList(t *testing.T) {
expNewErr: fmt.Errorf("[cpumanager] unable to reserve the required amount of CPUs (size of 0-1 did not equal 1)"),
},
}
- testExcl := false
for _, testCase := range testCases {
t.Run(testCase.description, func(t *testing.T) {
- p, err := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, testCase.reserved, topologymanager.NewFakeManager(), nil, testExcl)
+ p, err := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, testCase.reserved, cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil, testDM, testExcl)
if !reflect.DeepEqual(err, testCase.expNewErr) {
t.Errorf("StaticPolicy Start() error (%v). expected error: %v but got: %v",
testCase.description, testCase.expNewErr, err)
@@ -947,6 +958,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
topo: topoSingleSocketHT,
numReservedCPUs: 1,
reserved: cpuset.NewCPUSet(0),
+ isolcpus: cpuset.NewCPUSet(),
stAssignments: state.ContainerCPUAssignments{},
stDefaultCPUSet: cpuset.NewCPUSet(1, 2, 3, 4, 5, 6, 7),
pod: makePod("fakePod", "fakeContainer2", "8000m", "8000m"),
@@ -959,6 +971,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
topo: topoSingleSocketHT,
numReservedCPUs: 2,
reserved: cpuset.NewCPUSet(0, 1),
+ isolcpus: cpuset.NewCPUSet(),
stAssignments: state.ContainerCPUAssignments{},
stDefaultCPUSet: cpuset.NewCPUSet(2, 3, 4, 5, 6, 7),
pod: makePod("fakePod", "fakeContainer2", "1000m", "1000m"),
@@ -971,6 +984,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
topo: topoSingleSocketHT,
numReservedCPUs: 2,
reserved: cpuset.NewCPUSet(0, 1),
+ isolcpus: cpuset.NewCPUSet(),
stAssignments: state.ContainerCPUAssignments{
"fakePod": map[string]cpuset.CPUSet{
"fakeContainer100": cpuset.NewCPUSet(2, 3, 6, 7),
@@ -987,6 +1001,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
topo: topoSingleSocketHT,
numReservedCPUs: 2,
reserved: cpuset.NewCPUSet(0, 1),
+ isolcpus: cpuset.NewCPUSet(),
stAssignments: state.ContainerCPUAssignments{
"fakePod": map[string]cpuset.CPUSet{
"fakeContainer100": cpuset.NewCPUSet(2, 3, 6, 7),
@@ -998,11 +1013,29 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
expCPUAlloc: true,
expCSet: cpuset.NewCPUSet(0, 1),
},
+ {
+ description: "InfraPod, SingleSocketHT, Isolcpus, ExpectAllocReserved",
+ topo: topoSingleSocketHT,
+ numReservedCPUs: 2,
+ reserved: cpuset.NewCPUSet(0, 1),
+ isolcpus: cpuset.NewCPUSet(1),
+ stAssignments: state.ContainerCPUAssignments{
+ "fakePod": map[string]cpuset.CPUSet{
+ "fakeContainer100": cpuset.NewCPUSet(2, 3, 6, 7),
+ },
+ },
+ stDefaultCPUSet: cpuset.NewCPUSet(4, 5),
+ pod: infraPod,
+ expErr: nil,
+ expCPUAlloc: true,
+ expCSet: cpuset.NewCPUSet(0),
+ },
}
testExcl := true
+ testDM, _ := devicemanager.NewManagerStub()
for _, testCase := range testCases {
- policy, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, testCase.reserved, topologymanager.NewFakeManager(), nil, testExcl)
+ policy, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, testCase.reserved, testCase.isolcpus, topologymanager.NewFakeManager(), nil, testDM, testExcl)
st := &mockState{
assignments: testCase.stAssignments,
--
2.25.1

338
kubernetes/kubernetes-1.25.3/debian/deb_folder/patches/kubelet-cpumanager-keep-normal-containers-off-reserv.patch
View File

@ -1,338 +0,0 @@
From a793b871526189690d624379ce4e4e9ec834ccf6 Mon Sep 17 00:00:00 2001
From: Ramesh Kumar Sivanandam <rameshkumar.sivanandam@windriver.com>
Date: Mon, 7 Nov 2022 08:43:43 -0500
Subject: [PATCH] kubelet cpumanager keep normal containers off reserved CPUs
When starting the kubelet process, two separate sets of reserved CPUs
may be specified. With this change CPUs reserved via
'--system-reserved=cpu'
or '--kube-reserved=cpu' will be ignored by kubernetes itself. A small
tweak to the default CPU affinity ensures that "normal" Kubernetes
pods won't run on the reserved CPUs.
Co-authored-by: Jim Gauld <james.gauld@windriver.com>
Signed-off-by: Sachin Gopala Krishna <saching.krishna@windriver.com>
Signed-off-by: Ramesh Kumar Sivanandam <rameshkumar.sivanandam@windriver.com>
---
pkg/kubelet/cm/cpumanager/cpu_manager.go | 6 ++-
pkg/kubelet/cm/cpumanager/cpu_manager_test.go | 15 ++++++--
pkg/kubelet/cm/cpumanager/policy_static.go | 30 ++++++++++++---
.../cm/cpumanager/policy_static_test.go | 38 ++++++++++++++-----
4 files changed, 68 insertions(+), 21 deletions(-)
diff --git a/pkg/kubelet/cm/cpumanager/cpu_manager.go b/pkg/kubelet/cm/cpumanager/cpu_manager.go
index 32eabc1340e..00c034bc753 100644
--- a/pkg/kubelet/cm/cpumanager/cpu_manager.go
+++ b/pkg/kubelet/cm/cpumanager/cpu_manager.go
@@ -191,7 +191,11 @@ func NewManager(cpuPolicyName string, cpuPolicyOptions map[string]string, reconc
// exclusively allocated.
reservedCPUsFloat := float64(reservedCPUs.MilliValue()) / 1000
numReservedCPUs := int(math.Ceil(reservedCPUsFloat))
- policy, err = NewStaticPolicy(topo, numReservedCPUs, specificCPUs, affinity, cpuPolicyOptions)
+ // NOTE: Set excludeReserved unconditionally to exclude reserved CPUs from default cpuset.
+ // This variable is primarily to make testing easier.
+ excludeReserved := true
+ policy, err = NewStaticPolicy(topo, numReservedCPUs, specificCPUs, affinity, cpuPolicyOptions, excludeReserved)
+
if err != nil {
return nil, fmt.Errorf("new static policy error: %w", err)
}
diff --git a/pkg/kubelet/cm/cpumanager/cpu_manager_test.go b/pkg/kubelet/cm/cpumanager/cpu_manager_test.go
index 834d099ac3e..d553b182e0b 100644
--- a/pkg/kubelet/cm/cpumanager/cpu_manager_test.go
+++ b/pkg/kubelet/cm/cpumanager/cpu_manager_test.go
@@ -215,6 +215,7 @@ func makeMultiContainerPod(initCPUs, appCPUs []struct{ request, limit string })
}
func TestCPUManagerAdd(t *testing.T) {
+ testExcl := false
testPolicy, _ := NewStaticPolicy(
&topology.CPUTopology{
NumCPUs: 4,
@@ -230,7 +231,8 @@ func TestCPUManagerAdd(t *testing.T) {
0,
cpuset.NewCPUSet(),
topologymanager.NewFakeManager(),
- nil)
+ nil,
+ testExcl)
testCases := []struct {
description string
updateErr error
@@ -479,8 +481,9 @@ func TestCPUManagerAddWithInitContainers(t *testing.T) {
},
}
+ testExcl := false
for _, testCase := range testCases {
- policy, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil)
+ policy, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil, testExcl)
mockState := &mockState{
assignments: testCase.stAssignments,
@@ -705,6 +708,7 @@ func TestCPUManagerRemove(t *testing.T) {
}
func TestReconcileState(t *testing.T) {
+ testExcl := false
testPolicy, _ := NewStaticPolicy(
&topology.CPUTopology{
NumCPUs: 8,
@@ -724,7 +728,8 @@ func TestReconcileState(t *testing.T) {
0,
cpuset.NewCPUSet(),
topologymanager.NewFakeManager(),
- nil)
+ nil,
+ testExcl)
testCases := []struct {
description string
@@ -1228,6 +1233,7 @@ func TestReconcileState(t *testing.T) {
// above test cases are without kubelet --reserved-cpus cmd option
// the following tests are with --reserved-cpus configured
func TestCPUManagerAddWithResvList(t *testing.T) {
+ testExcl := false
testPolicy, _ := NewStaticPolicy(
&topology.CPUTopology{
NumCPUs: 4,
@@ -1243,7 +1249,8 @@ func TestCPUManagerAddWithResvList(t *testing.T) {
1,
cpuset.NewCPUSet(0),
topologymanager.NewFakeManager(),
- nil)
+ nil,
+ testExcl)
testCases := []struct {
description string
updateErr error
diff --git a/pkg/kubelet/cm/cpumanager/policy_static.go b/pkg/kubelet/cm/cpumanager/policy_static.go
index ff9e6a20e14..14a6e7af9be 100644
--- a/pkg/kubelet/cm/cpumanager/policy_static.go
+++ b/pkg/kubelet/cm/cpumanager/policy_static.go
@@ -96,6 +96,8 @@ type staticPolicy struct {
topology *topology.CPUTopology
// set of CPUs that is not available for exclusive assignment
reserved cpuset.CPUSet
+ // If true, default CPUSet should exclude reserved CPUs
+ excludeReserved bool
// topology manager reference to get container Topology affinity
affinity topologymanager.Store
// set of CPUs to reuse across allocations in a pod
@@ -110,7 +112,7 @@ var _ Policy = &staticPolicy{}
// NewStaticPolicy returns a CPU manager policy that does not change CPU
// assignments for exclusively pinned guaranteed containers after the main
// container process starts.
-func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reservedCPUs cpuset.CPUSet, affinity topologymanager.Store, cpuPolicyOptions map[string]string) (Policy, error) {
+func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reservedCPUs cpuset.CPUSet, affinity topologymanager.Store, cpuPolicyOptions map[string]string, excludeReserved bool) (Policy, error) {
opts, err := NewStaticPolicyOptions(cpuPolicyOptions)
if err != nil {
return nil, err
@@ -125,6 +127,7 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv
policy := &staticPolicy{
topology: topology,
affinity: affinity,
+ excludeReserved: excludeReserved,
cpusToReuse: make(map[string]cpuset.CPUSet),
options: opts,
}
@@ -176,7 +179,15 @@ func (p *staticPolicy) validateState(s state.State) error {
}
// state is empty initialize
allCPUs := p.topology.CPUDetails.CPUs()
- s.SetDefaultCPUSet(allCPUs)
+ if p.excludeReserved {
+ // Exclude reserved CPUs from the default CPUSet to keep containers off them
+ // unless explicitly affined.
+ s.SetDefaultCPUSet(allCPUs.Difference(p.reserved))
+ } else {
+ s.SetDefaultCPUSet(allCPUs)
+ }
+ klog.Infof("[cpumanager] static policy: CPUSet: allCPUs:%v, reserved:%v, default:%v\n",
+ allCPUs, p.reserved, s.GetDefaultCPUSet())
return nil
}
@@ -184,11 +195,12 @@ func (p *staticPolicy) validateState(s state.State) error {
// 1. Check if the reserved cpuset is not part of default cpuset because:
// - kube/system reserved have changed (increased) - may lead to some containers not being able to start
// - user tampered with file
- if !p.reserved.Intersection(tmpDefaultCPUset).Equals(p.reserved) {
- return fmt.Errorf("not all reserved cpus: \"%s\" are present in defaultCpuSet: \"%s\"",
- p.reserved.String(), tmpDefaultCPUset.String())
+ if !p.excludeReserved {
+ if !p.reserved.Intersection(tmpDefaultCPUset).Equals(p.reserved) {
+ return fmt.Errorf("not all reserved cpus: \"%s\" are present in defaultCpuSet: \"%s\"",
+ p.reserved.String(), tmpDefaultCPUset.String())
+ }
}
-
// 2. Check if state for static policy is consistent
for pod := range tmpAssignments {
for container, cset := range tmpAssignments[pod] {
@@ -215,6 +227,9 @@ func (p *staticPolicy) validateState(s state.State) error {
}
}
totalKnownCPUs = totalKnownCPUs.UnionAll(tmpCPUSets)
+ if p.excludeReserved {
+ totalKnownCPUs = totalKnownCPUs.Union(p.reserved)
+ }
if !totalKnownCPUs.Equals(p.topology.CPUDetails.CPUs()) {
return fmt.Errorf("current set of available CPUs \"%s\" doesn't match with CPUs in state \"%s\"",
p.topology.CPUDetails.CPUs().String(), totalKnownCPUs.String())
@@ -314,6 +329,9 @@ func (p *staticPolicy) RemoveContainer(s state.State, podUID string, containerNa
cpusInUse := getAssignedCPUsOfSiblings(s, podUID, containerName)
if toRelease, ok := s.GetCPUSet(podUID, containerName); ok {
s.Delete(podUID, containerName)
+ if p.excludeReserved {
+ toRelease = toRelease.Difference(p.reserved)
+ }
// Mutate the shared pool, adding released cpus.
toRelease = toRelease.Difference(cpusInUse)
s.SetDefaultCPUSet(s.GetDefaultCPUSet().Union(toRelease))
diff --git a/pkg/kubelet/cm/cpumanager/policy_static_test.go b/pkg/kubelet/cm/cpumanager/policy_static_test.go
index 4c10af065a4..80a0c5a9e70 100644
--- a/pkg/kubelet/cm/cpumanager/policy_static_test.go
+++ b/pkg/kubelet/cm/cpumanager/policy_static_test.go
@@ -36,6 +36,7 @@ type staticPolicyTest struct {
description string
topo *topology.CPUTopology
numReservedCPUs int
+ excludeReserved bool
podUID string
options map[string]string
containerName string
@@ -68,7 +69,8 @@ func (spt staticPolicyTest) PseudoClone() staticPolicyTest {
}
func TestStaticPolicyName(t *testing.T) {
- policy, _ := NewStaticPolicy(topoSingleSocketHT, 1, cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil)
+ testExcl := false
+ policy, _ := NewStaticPolicy(topoSingleSocketHT, 1, cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil, testExcl)
policyName := policy.Name()
if policyName != "static" {
@@ -98,6 +100,15 @@ func TestStaticPolicyStart(t *testing.T) {
stDefaultCPUSet: cpuset.NewCPUSet(),
expCSet: cpuset.NewCPUSet(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11),
},
+ {
+ description: "empty cpuset exclude reserved",
+ topo: topoDualSocketHT,
+ numReservedCPUs: 2,
+ excludeReserved: true,
+ stAssignments: state.ContainerCPUAssignments{},
+ stDefaultCPUSet: cpuset.NewCPUSet(),
+ expCSet: cpuset.NewCPUSet(1, 2, 3, 4, 5, 7, 8, 9, 10, 11),
+ },
{
description: "reserved cores 0 & 6 are not present in available cpuset",
topo: topoDualSocketHT,
@@ -144,7 +155,8 @@ func TestStaticPolicyStart(t *testing.T) {
}
for _, testCase := range testCases {
t.Run(testCase.description, func(t *testing.T) {
- p, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil)
+ p, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil, testCase.excludeReserved)
+
policy := p.(*staticPolicy)
st := &mockState{
assignments: testCase.stAssignments,
@@ -565,7 +577,8 @@ func runStaticPolicyTestCase(t *testing.T, testCase staticPolicyTest) {
if testCase.topologyHint != nil {
tm = topologymanager.NewFakeManagerWithHint(testCase.topologyHint)
}
- policy, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, cpuset.NewCPUSet(), tm, testCase.options)
+ testExcl := false
+ policy, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, cpuset.NewCPUSet(), tm, testCase.options, testExcl)
st := &mockState{
assignments: testCase.stAssignments,
@@ -636,7 +649,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) {
}
for _, testCase := range testCases {
- policy, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil)
+ policy, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil, testCase.excludeReserved)
st := &mockState{
assignments: testCase.stAssignments,
@@ -668,6 +681,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) {
}
func TestStaticPolicyRemove(t *testing.T) {
+ excludeReserved := false
testCases := []staticPolicyTest{
{
description: "SingleSocketHT, DeAllocOneContainer",
@@ -726,7 +740,7 @@ func TestStaticPolicyRemove(t *testing.T) {
}
for _, testCase := range testCases {
- policy, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil)
+ policy, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil, excludeReserved)
st := &mockState{
assignments: testCase.stAssignments,
@@ -748,6 +762,7 @@ func TestStaticPolicyRemove(t *testing.T) {
}
func TestTopologyAwareAllocateCPUs(t *testing.T) {
+ excludeReserved := false
testCases := []struct {
description string
topo *topology.CPUTopology
@@ -816,7 +831,7 @@ func TestTopologyAwareAllocateCPUs(t *testing.T) {
},
}
for _, tc := range testCases {
- p, _ := NewStaticPolicy(tc.topo, 0, cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil)
+ p, _ := NewStaticPolicy(tc.topo, 0, cpuset.NewCPUSet(), topologymanager.NewFakeManager(), nil, excludeReserved)
policy := p.(*staticPolicy)
st := &mockState{
assignments: tc.stAssignments,
@@ -888,9 +903,11 @@ func TestStaticPolicyStartWithResvList(t *testing.T) {
expNewErr: fmt.Errorf("[cpumanager] unable to reserve the required amount of CPUs (size of 0-1 did not equal 1)"),
},
}
+ testExcl := false
for _, testCase := range testCases {
t.Run(testCase.description, func(t *testing.T) {
- p, err := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, testCase.reserved, topologymanager.NewFakeManager(), nil)
+ p, err := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, testCase.reserved, topologymanager.NewFakeManager(), nil, testExcl)
+
if !reflect.DeepEqual(err, testCase.expNewErr) {
t.Errorf("StaticPolicy Start() error (%v). expected error: %v but got: %v",
testCase.description, testCase.expNewErr, err)
@@ -930,7 +947,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
numReservedCPUs: 1,
reserved: cpuset.NewCPUSet(0),
stAssignments: state.ContainerCPUAssignments{},
- stDefaultCPUSet: cpuset.NewCPUSet(0, 1, 2, 3, 4, 5, 6, 7),
+ stDefaultCPUSet: cpuset.NewCPUSet(1, 2, 3, 4, 5, 6, 7),
pod: makePod("fakePod", "fakeContainer2", "8000m", "8000m"),
expErr: fmt.Errorf("not enough cpus available to satisfy request"),
expCPUAlloc: false,
@@ -942,7 +959,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
numReservedCPUs: 2,
reserved: cpuset.NewCPUSet(0, 1),
stAssignments: state.ContainerCPUAssignments{},
- stDefaultCPUSet: cpuset.NewCPUSet(0, 1, 2, 3, 4, 5, 6, 7),
+ stDefaultCPUSet: cpuset.NewCPUSet(2, 3, 4, 5, 6, 7),
pod: makePod("fakePod", "fakeContainer2", "1000m", "1000m"),
expErr: nil,
expCPUAlloc: true,
@@ -966,8 +983,9 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
},
}
+ testExcl := true
for _, testCase := range testCases {
- policy, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, testCase.reserved, topologymanager.NewFakeManager(), nil)
+ policy, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, testCase.reserved, topologymanager.NewFakeManager(), nil, testExcl)
st := &mockState{
assignments: testCase.stAssignments,
--
2.25.1

50
kubernetes/kubernetes-1.25.3/debian/deb_folder/patches/kubelet-sort-isolcpus-allocation-when-SMT-enabled.patch
View File

@ -1,50 +0,0 @@
From ba9ab333c8b7dca5252e604837914293dc232732 Mon Sep 17 00:00:00 2001
From: Jim Gauld <James.Gauld@windriver.com>
Date: Fri, 11 Feb 2022 11:06:35 -0500
Subject: [PATCH] kubelet: sort isolcpus allocation when SMT enabled
The existing device manager code returns CPUs as devices in unsorted
order. This numerically sorts isolcpus allocations when SMT/HT is
enabled on the host. This logs SMT pairs, singletons, and algorithm
order details to make the algorithm understandable.
Signed-off-by: Jim Gauld <James.Gauld@windriver.com>
---
pkg/kubelet/cm/devicemanager/manager.go | 13 ++++++++++++-
1 file changed, 12 insertions(+), 1 deletion(-)
diff --git a/pkg/kubelet/cm/devicemanager/manager.go b/pkg/kubelet/cm/devicemanager/manager.go
index 609da8ed86b..a4b247714f7 100644
--- a/pkg/kubelet/cm/devicemanager/manager.go
+++ b/pkg/kubelet/cm/devicemanager/manager.go
@@ -686,7 +686,16 @@ func order_devices_by_sibling(devices sets.String, needed int) ([]string, error)
return cpu_lst[0]
}
}
+ //Make post-analysis of selection algorithm obvious by numerical sorting
+ //the available isolated cpu_id.
+ cpu_ids := make([]int, 0, int(devices.Len()))
for cpu_id := range devices {
+ cpu_id_, _ := strconv.Atoi(cpu_id)
+ cpu_ids = append(cpu_ids, cpu_id_)
+ }
+ sort.Ints(cpu_ids)
+ for _, _cpu_id := range cpu_ids {
+ cpu_id := strconv.Itoa(_cpu_id)
// If we've already found cpu_id as a sibling, skip it.
if _, ok := _iterated_cpu[cpu_id]; ok {
continue
@@ -728,7 +737,9 @@ func order_devices_by_sibling(devices sets.String, needed int) ([]string, error)
}
}
}
- //klog.Infof("needed=%d ordered_cpu_list=%v", needed, dev_lst)
+ //This algorithm will get some attention. Show minimal details.
+ klog.Infof("order_devices_by_sibling: needed=%d, smtpairs=%v, singletons=%v, order=%v",
+ needed, sibling_lst, single_lst, dev_lst)
return dev_lst, nil
}
func smt_enabled() bool {
--
2.25.1

151
kubernetes/kubernetes-1.25.3/debian/deb_folder/patches/kubernetes-make-isolcpus-allocation-SMT-aware.patch
View File

@ -1,151 +0,0 @@
From c34527067c49c13741484e6977fff9640d9b6efe Mon Sep 17 00:00:00 2001
From: Ramesh Kumar Sivanandam <rameshkumar.sivanandam@windriver.com>
Date: Tue, 25 Oct 2022 11:13:35 -0400
Subject: [PATCH] kubernetes: make isolcpus allocation SMT-aware
Enhance isolcpus support in Kubernetes to allocate isolated SMT
siblings to the same container when SMT/HT is enabled on the host.
As it stands, the device manager code in Kubernetes is not SMT-aware
(since normally it doesn't deal with CPUs). However, StarlingX
exposes isolated CPUs as devices and if possible we want to allocate
all SMT siblings from a CPU core to the same container in order to
minimize cross- container interference due to resource contention
within the CPU core.
The solution is basically to take the list of isolated CPUs and
re-order it so that the SMT siblings are next to each other. That
way the existing resource selection code will allocate the siblings
together. As an optimization, if it is known that an odd number
of isolated CPUs are desired, a singleton SMT sibling will be
inserted into the list to avoid breaking up sibling pairs.
Signed-off-by: Tao Wang <tao.wang@windriver.com>
Signed-off-by: Ramesh Kumar Sivanandam <rameshkumar.sivanandam@windriver.com>
---
pkg/kubelet/cm/devicemanager/manager.go | 84 ++++++++++++++++++++++++-
1 file changed, 83 insertions(+), 1 deletion(-)
diff --git a/pkg/kubelet/cm/devicemanager/manager.go b/pkg/kubelet/cm/devicemanager/manager.go
index 7b0283d93e4..1fdc2018ab5 100644
--- a/pkg/kubelet/cm/devicemanager/manager.go
+++ b/pkg/kubelet/cm/devicemanager/manager.go
@@ -19,10 +19,13 @@ package devicemanager
import (
"context"
"fmt"
+ "io/ioutil"
"os"
"path/filepath"
"runtime"
"sort"
+ "strconv"
+ "strings"
"sync"
"time"
@@ -38,6 +41,7 @@ import (
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/kubelet/checkpointmanager"
"k8s.io/kubernetes/pkg/kubelet/checkpointmanager/errors"
+ "k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
"k8s.io/kubernetes/pkg/kubelet/cm/devicemanager/checkpoint"
plugin "k8s.io/kubernetes/pkg/kubelet/cm/devicemanager/plugin/v1beta1"
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager"
@@ -518,6 +522,75 @@ func (m *ManagerImpl) UpdateAllocatedDevices() {
m.allocatedDevices = m.podDevices.devices()
}
+//Given a list of isolated CPUs in 'devices', and the number of desired CPUs in 'needed',
+//return an ordered list of isolated CPUs such that the first 'needed' CPUs in the list
+//contain as many hyperthread sibling pairs as possible.
+func order_devices_by_sibling(devices sets.String, needed int) ([]string, error) {
+ var dev_lst []string
+ var single_lst []string
+ sibling_lst := make([]string, 0, int(devices.Len()))
+ _iterated_cpu := make(map[string]string)
+ get_sibling := func(cpu string, cpu_lst []string) string {
+ if cpu_lst[0] == cpu {
+ return cpu_lst[1]
+ } else {
+ return cpu_lst[0]
+ }
+ }
+ for cpu_id := range devices {
+ // If we've already found cpu_id as a sibling, skip it.
+ if _, ok := _iterated_cpu[cpu_id]; ok {
+ continue
+ }
+ devPath := fmt.Sprintf("/sys/devices/system/cpu/cpu%s/topology/thread_siblings_list", cpu_id)
+ dat, err := ioutil.ReadFile(devPath)
+ if err != nil {
+ return dev_lst, fmt.Errorf("Can't read cpu[%s] thread_siblings_list", cpu_id)
+ }
+ cpustring := strings.TrimSuffix(string(dat), "\n")
+ cpu_pair_set, err := cpuset.Parse(cpustring)
+ if err != nil {
+ return dev_lst, fmt.Errorf("Unable to parse thread_siblings_list[%s] string to cpuset", cpustring)
+ }
+ var cpu_pair_lst []string
+ for _, v := range cpu_pair_set.ToSlice() {
+ cpu_pair_lst = append(cpu_pair_lst, strconv.Itoa(v))
+ }
+ sibling_cpu_id := get_sibling(cpu_id, cpu_pair_lst)
+ if _, ok := devices[sibling_cpu_id]; ok {
+ sibling_lst = append(sibling_lst, cpu_id, sibling_cpu_id)
+ _iterated_cpu[sibling_cpu_id] = ""
+ } else {
+ single_lst = append(single_lst, cpu_id)
+ }
+ _iterated_cpu[cpu_id] = ""
+ }
+ if needed%2 == 0 {
+ dev_lst = append(sibling_lst, single_lst...)
+ } else {
+ if len(single_lst) > 1 {
+ _tmp_list := append(sibling_lst, single_lst[1:]...)
+ dev_lst = append(single_lst[0:1], _tmp_list...)
+ } else {
+ if len(single_lst) == 0 {
+ dev_lst = sibling_lst
+ } else {
+ dev_lst = append(single_lst, sibling_lst...)
+ }
+ }
+ }
+ //klog.Infof("needed=%d ordered_cpu_list=%v", needed, dev_lst)
+ return dev_lst, nil
+}
+func smt_enabled() bool {
+ dat, _ := ioutil.ReadFile("/sys/devices/system/cpu/smt/active")
+ state := strings.TrimSuffix(string(dat), "\n")
+ if state == "0" {
+ return false
+ }
+ return true
+}
+
// Returns list of device Ids we need to allocate with Allocate rpc call.
// Returns empty list in case we don't need to issue the Allocate rpc call.
func (m *ManagerImpl) devicesToAllocate(podUID, contName, resource string, required int, reusableDevices sets.String) (sets.String, error) {
@@ -553,7 +626,16 @@ func (m *ManagerImpl) devicesToAllocate(podUID, contName, resource string, requi
// Create a closure to help with device allocation
// Returns 'true' once no more devices need to be allocated.
allocateRemainingFrom := func(devices sets.String) bool {
- for device := range devices.Difference(allocated) {
+ availableDevices := devices.Difference(allocated).List()
+ // If we're dealing with isolcpus and SMT is enabled, reorder to group SMT siblings together.
+ if resource == "windriver.com/isolcpus" && len(devices) > 0 && smt_enabled() {
+ var err error
+ availableDevices, err = order_devices_by_sibling(devices.Difference(allocated), needed)
+ if err != nil {
+ klog.Errorf("error in order_devices_by_sibling: %v", err)
+ }
+ }
+ for _, device := range availableDevices {
m.allocatedDevices[resource].Insert(device)
allocated.Insert(device)
needed--
--
2.25.1

11178
kubernetes/kubernetes-1.25.3/debian/deb_folder/patches/kubernetes-pull-117682-vendor-bump-runc-to-1.1.6.patch
View File

File diff suppressed because it is too large Load Diff

14
kubernetes/kubernetes-1.25.3/debian/deb_folder/patches/series
View File

@ -1,14 +0,0 @@
kubeadm-create-platform-pods-with-zero-CPU-resources.patch
kubernetes-make-isolcpus-allocation-SMT-aware.patch
kubelet-sort-isolcpus-allocation-when-SMT-enabled.patch
kubelet-cpumanager-disable-CFS-quota-throttling.patch
kubelet-cpumanager-keep-normal-containers-off-reserv.patch
kubelet-cpumanager-infra-pods-use-system-reserved-CP.patch
kubelet-cpumanager-introduce-concept-of-isolated-CPU.patch
kubelet-CFS-quota-throttling-for-non-integer-cpulimit.patch
Revert-kubeadm-remove-RemoveOldControlPlaneLabel.patch
Revert-kubeadm-cleanup-the-master-taint-on-CP-nodes-.patch
kubeadm-reduce-UpgradeManifestTimeout.patch
Identify-platform-pods-based-on-pod-or-namespace-labels.patch
kubeadm-readiness-probe-timeout-core-dns.patch
kubernetes-pull-117682-vendor-bump-runc-to-1.1.6.patch

116
kubernetes/kubernetes-1.25.3/debian/deb_folder/rules
View File

@ -1,116 +0,0 @@
#!/usr/bin/make -f
#
# Copyright (c) 2022 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
# This debian/rules file is based on:
# https://packages.debian.org/source/bookworm/kubernetes
# http://deb.debian.org/debian/pool/main/k/kubernetes/kubernetes_1.20.5+really1.20.2-1.1.debian.tar.xz
# Customizations support kubernetes upgrades:
# - specific directory locations with kubernetes version, upgrades stage,
# and version specific golang compiler
# - build output not required on the production host is moved to
# kubernetes-misc package
kube_version := 1.25.3
kube_git_version := v${kube_version}
name := kubernetes-${kube_version}
go_version := 1.19.2
_stage1 := /usr/local/kubernetes/${kube_version}/stage1
_stage2 := /usr/local/kubernetes/${kube_version}/stage2
_bindir := /usr/bin
kube_dir := src/k8s.io/kubernetes
output_dir := ${kube_dir}/_output
output_bindir := ${output_dir}/bin
output_mandir := ${output_dir}/man
DEBIAN_DESTDIR := $(CURDIR)/debian/tmp
export DH_VERBOSE = 1
export PATH := /usr/lib/go-1.19/bin:$(PATH)
export KUBE_GIT_TREE_STATE="clean"
export KUBE_GIT_COMMIT=${kube_version}
export KUBE_GIT_VERSION=${kube_git_version}
export KUBE_EXTRA_GOPATH=$(pwd)/Godeps/_workspace
export PBR_VERSION=${kube_git_version}
bins = kube-proxy kube-apiserver kube-controller-manager kubelet kubeadm kube-scheduler kubectl
%:
dh $@ --with=bash-completion --builddirectory=src --without=build-stamp
override_dh_auto_build:
# we support multiple go compilers; indicate the version we are using
go version
which go
mkdir -pv ${kube_dir}
mv -v $$(ls | grep -v "^src$$" | grep -v "^debian$$") ${kube_dir}/.
cd ${kube_dir} && make WHAT="$(addprefix cmd/,$(bins) genman)"
# manpages
mkdir -p ${output_mandir}
echo $(bins) | xargs --max-args=1 ${output_bindir}/genman ${output_mandir}
# NOTICE files
find ${kube_dir}/vendor -name '*NOTICE*' -print0 | xargs -0 head -n1000 > ${output_dir}/NOTICE
override_dh_install:
# kube_version stage1
install -m 755 -d ${DEBIAN_DESTDIR}${_stage1}${_bindir}
install -p -m 755 -t ${DEBIAN_DESTDIR}${_stage1}${_bindir} ${output_bindir}/kubeadm
# kube_version stage2
install -m 755 -d ${DEBIAN_DESTDIR}${_stage2}${_bindir}
install -d -m 0755 ${DEBIAN_DESTDIR}${_stage2}/etc/systemd/system/kubelet.service.d
install -p -m 0644 -t ${DEBIAN_DESTDIR}${_stage2}/etc/systemd/system/kubelet.service.d debian/kubeadm.conf
install -p -m 750 -t ${DEBIAN_DESTDIR}${_stage2}${_bindir} ${output_bindir}/kubelet
install -p -m 755 -t ${DEBIAN_DESTDIR}${_stage2}${_bindir} ${output_bindir}/kubectl
# bash completions
install -d -m 0755 ${DEBIAN_DESTDIR}${_stage2}/usr/share/bash-completion/completions/
${DEBIAN_DESTDIR}${_stage2}${_bindir}/kubectl completion bash > ${DEBIAN_DESTDIR}${_stage2}/usr/share/bash-completion/completions/kubectl
# remaining are not kube_version staged, i.e., kubernetes-master, kubernetes-misc
install -m 755 -d ${DEBIAN_DESTDIR}${_bindir}
install -p -m 750 -t ${DEBIAN_DESTDIR}${_bindir} ${output_bindir}/kube-apiserver
install -p -m 750 -t ${DEBIAN_DESTDIR}${_bindir} ${output_bindir}/kube-controller-manager
install -p -m 750 -t ${DEBIAN_DESTDIR}${_bindir} ${output_bindir}/kube-scheduler
install -p -m 750 -t ${DEBIAN_DESTDIR}${_bindir} ${output_bindir}/kube-proxy
# specific cluster addons for optional use
install -d -m 0755 ${DEBIAN_DESTDIR}/etc/${name}/addons
# Addon: volumesnapshots
install -d -m 0755 ${DEBIAN_DESTDIR}/etc/${name}/addons/volumesnapshots
install -d -m 0755 ${DEBIAN_DESTDIR}/etc/${name}/addons/volumesnapshots/crd
install -m 0644 -t ${DEBIAN_DESTDIR}/etc/${name}/addons/volumesnapshots/crd ${kube_dir}/cluster/addons/volumesnapshots/crd/*
install -d -m 0755 ${DEBIAN_DESTDIR}/etc/${name}/addons/volumesnapshots/volume-snapshot-controller
install -m 0644 -t ${DEBIAN_DESTDIR}/etc/${name}/addons/volumesnapshots/volume-snapshot-controller ${kube_dir}/cluster/addons/volumesnapshots/volume-snapshot-controller/*
# unit-test
# - everything from the root directory is needed
# - unit-tests needs source code
# - integration tests needs docs and other files
# - test-cmd.sh atm needs cluster, examples and other
install -d -m 0755 ${DEBIAN_DESTDIR}/var/lib/kubernetes-unit-test/
cp -a src ${DEBIAN_DESTDIR}/var/lib/kubernetes-unit-test/
# remove generated output, i.e., binaries, go cache, man pages, violations report
rm -rf ${DEBIAN_DESTDIR}/var/lib/kubernetes-unit-test/${output_dir}
dh_install
override_dh_auto_test:
${kube_dir}/hack/test-cmd.sh
${kube_dir}/hack/benchmark-go.sh
${kube_dir}/hack/test-go.sh
${kube_dir}/hack/test-integration.sh --use_go_build
override_dh_fixperms:
dh_fixperms -Xkube-apiserver -Xkubeadm -Xkubeadm.conf \
-Xkubelet-cgroup-setup.sh -Xkube-apiserver \
-Xkube-controller-manager -Xkube-scheduler \
-Xkube-proxy -Xkubelet -Xkubectl
override_dh_usrlocal:

1
kubernetes/kubernetes-1.25.3/debian/deb_folder/source/format
View File

@ -1 +0,0 @@
3.0 (quilt)

12
kubernetes/kubernetes-1.25.3/debian/meta_data.yaml
View File

@ -1,12 +0,0 @@
---
debver: 1.25.3
dl_path:
name: kubernetes-1.25.3.tar.gz
url: https://github.com/kubernetes/kubernetes/archive/refs/tags/v1.25.3.tar.gz
md5sum: 595ce52826e10827086a1c0f1c0dcef2
sha256sum: 957428759c846ed6a71453ce39b9bd7a91501209c222dcae5d73348b90d68ced
revision:
dist: $STX_DIST
GITREVCOUNT:
BASE_SRCREV: aacf601fd5983a950a1fe205cac2490b9cb0a5ad
SRC_DIR: ${MY_REPO}/stx/integ/kubernetes/kubernetes-1.25.3

271
kubernetes/kubernetes-1.26.1/debian/deb_folder/changelog
View File

@ -1,271 +0,0 @@
kubernetes-1.26.1 (1.26.1-1) unstable; urgency=medium
* Updated for stx debian packaging
-- Saba Touheed Mujawar sabatouheed.mujawar@windriver.com Thu, 16 Feb 2023 18:38:00 +0000
kubernetes-1.25.3 (1.25.3-1) unstable; urgency=medium
* Updated for stx debian packaging
-- Ramesh Kumar Sivanandam rameshkumar.sivanandam@windriver.com Thu, 14 Nov 2022 14:30:00 +0000
kubernetes-1.24.4 (1.24.4-1) unstable; urgency=medium
* Updated for stx debian packaging
-- James Gauld james.gauld@windriver.com Wed, 14 Sep 2022 11:30:00 +0000
kubernetes-1.23.1 (1.23.1-1) unstable; urgency=medium
* Updated for stx debian packaging
-- Kaustubh Dhokte <kaustubh.dhokte@opendev.org> Tue, 22 Feb 2022 20:57:45 +0000
kubernetes (1.21.8-1) unstable; urgency=medium
* Updated for stx debian packaging
-- Mihnea Saracin mihnea.saracin@opendev.org Fri, 29 Oct 2021 12:51:12 +0000
kubernetes (1.20.5+really1.20.2-1.1) unstable; urgency=medium
* Non-maintainer upload.
* Build using golang-go (Closes: #1000980)
-- Florian Ernst <florian@debian.org> Sat, 02 Apr 2022 16:49:13 +0200
kubernetes (1.20.5+really1.20.2-1) unstable; urgency=medium
* This is actually still 1.20.2
* Only contains the client, for the server binaries see unstable/fasttrack
-- Janos Lenart <ocsi@debian.org> Sun, 13 Jun 2021 07:08:18 +0000
kubernetes (1.20.2-1) unstable; urgency=medium
* New upstream release: 1.20.2
-- Janos Lenart <ocsi@debian.org> Thu, 14 Jan 2021 10:55:09 +0000
kubernetes (1.20.0-1) unstable; urgency=medium
* New upstream release: 1.20.0
* Fixes CVE-2020-8565
-- Janos Lenart <ocsi@debian.org> Wed, 09 Dec 2020 12:33:59 +0000
kubernetes (1.19.4-2) unstable; urgency=medium
* Updated README.Debian
-- Janos Lenart <ocsi@debian.org> Sat, 21 Nov 2020 14:06:21 +0000
kubernetes (1.19.4-1) unstable; urgency=medium
* New upstream release: 1.19.4
-- Janos Lenart <ocsi@debian.org> Tue, 17 Nov 2020 09:30:59 +0000
kubernetes (1.19.3-1) unstable; urgency=medium
* New upstream release: 1.19.3
* Building with golang-1.15
* Fixes CVE-2020-8564, CVE-2020-8566
-- Janos Lenart <ocsi@debian.org> Wed, 21 Oct 2020 10:38:41 +0100
kubernetes (1.18.6-1) unstable; urgency=medium
* New upstream release: 1.18.6
* (An earlier version, 1.17.4-1 fixes CVE-2019-9946)
-- Janos Lenart <ocsi@debian.org> Thu, 16 Jul 2020 10:08:46 +0100
kubernetes (1.18.5-1) unstable; urgency=medium
* New upstream release: 1.18.5
* Fixes CVE-2020-8557, CVE-2020-8558, CVE-2020-8559
-- Janos Lenart <ocsi@debian.org> Wed, 15 Jul 2020 17:19:40 +0100
kubernetes (1.18.3-1) unstable; urgency=medium
* New upstream release: 1.18.3
* Improved build reproducibility
-- Janos Lenart <ocsi@debian.org> Tue, 02 Jun 2020 11:18:12 +0000
kubernetes (1.18.2-3) unstable; urgency=medium
* Bumped Standards-Version
* Improved build reproducibility
-- Janos Lenart <ocsi@debian.org> Fri, 15 May 2020 13:17:53 +0000
kubernetes (1.18.2-2) unstable; urgency=medium
* Added i386 back
-- Janos Lenart <ocsi@debian.org> Sun, 03 May 2020 21:13:17 +0000
kubernetes (1.18.2-1) unstable; urgency=medium
* New upstream release: 1.18.2
-- Janos Lenart <ocsi@debian.org> Sun, 03 May 2020 19:25:37 +0000
kubernetes (1.18.0-1) unstable; urgency=medium
* New upstream release: 1.18.0
-- Janos Lenart <ocsi@debian.org> Sat, 28 Mar 2020 12:58:42 +0000
kubernetes (1.17.4-1) unstable; urgency=high
* New maintainer (Closes: #886739)
* New upstream release: 1.17.4 (Closes: #887741)
* New Debian packaging from scratch. See README.Debian
* kubernetes-node
- Moved docker from Depends into Recommends as kubelet can also work with
rkt, cri-o, etc. (Closes: #872690)
- Not shipping systemd units for kubelet and kube-proxy for now
* kubernetes-master
- Moved etcd from Depends into Recommends as apiserver can also connect to
a remote etcd/cluster.
- Not shipping systemd units for kube-apiserver, kube-schedules and
kube-controller-manager for now
-- Janos Lenart <ocsi@debian.org> Sun, 15 Mar 2020 21:46:45 +0000
kubernetes (1.7.16+dfsg-1) unstable; urgency=medium
[ Michael Stapelberg ]
* Switch to XS-Go-Import-Path
[ Dmitry Smirnov ]
* Resurrected "mergo.patch" that has been mistakenly removed
(Closes: #878254).
* Re-enabled safeguard test for the above problem.
* New upstream release:
+ CVE-2017-1002101 (Closes: #892801)
+ CVE-2017-1002102 (Closes: #894051)
* Updated Vcs URLs for Salsa.
* Standards-Version: 4.1.4
* Build-Depends:
- golang-go
+ golang-any
+ golang-github-appc-cni-dev
+ golang-github-armon-circbuf-dev
+ golang-github-azure-azure-sdk-for-go-dev
+ golang-github-dgrijalva-jwt-go-v3-dev
+ golang-github-docker-distribution-dev
+ golang-github-docker-docker-dev
+ golang-github-emicklei-go-restful-swagger12-dev
+ golang-github-gogo-protobuf-dev
+ golang-github-gorilla-websocket-dev
+ golang-github-grpc-ecosystem-go-grpc-prometheus-dev
+ golang-github-karlseguin-ccache-dev
- golang-github-opencontainers-runc-dev
+ golang-github-opencontainers-docker-runc-dev
+ golang-github-pmezard-go-difflib-dev
+ golang-golang-x-time-dev
+ golang-golang-x-tools-dev
+ golang-google-grpc-dev
+ golang-gopkg-warnings.v0-dev
+ golang-goprotobuf-dev
-- Dmitry Smirnov <onlyjob@debian.org> Sun, 06 May 2018 16:20:21 +1000
kubernetes (1.7.7+dfsg-3) unstable; urgency=medium
* kubernetes-master should depend on etcd (Closes: #855218).
-- Andrew Shadura <andrewsh@debian.org> Sun, 22 Oct 2017 19:40:46 +0100
kubernetes (1.7.7+dfsg-2) unstable; urgency=medium
* Use CURDIR, not PWD, unbreaks the build at buildds.
-- Andrew Shadura <andrewsh@debian.org> Fri, 06 Oct 2017 19:25:45 +0200
kubernetes (1.7.7+dfsg-1) unstable; urgency=medium
[ Tim Potter ]
* Open work for new release
* Remove unused Files-Excluded entries from d/copyright
* Remove Skydns B-D as no longer used
* Don't build on ppc64 or ppc64le architectures
[ Andrew Shadura ]
* New upstream release.
* Refresh patches.
* Update build dependencies.
* Symlink vendor packages to the build directory.
-- Andrew Shadura <andrewsh@debian.org> Fri, 06 Oct 2017 18:54:06 +0200
kubernetes (1.5.5+dfsg-2) unstable; urgency=medium
* Team upload.
* Don't build on ppc64le due to Go linker problems. See GitHub issue
https://github.com/golang/go/issues/15823.
* Don't build on ppc64 as it's not supported by upstream at the
moment. (Closes: #860505)
-- Tim Potter <tpot@hpe.com> Sat, 03 Jun 2017 08:00:51 +1000
kubernetes (1.5.5+dfsg-1) unstable; urgency=low
[ Dmitry Smirnov ]
* Switch to bundled "rkt".
* rules: remove "-p" option from build and test overrides.
* control: drop obsolete "golang-clockwork-dev" alternative.
* New patch to disable test failing on [armel].
* Upload to unstable.
[ Tim Potter ]
* New upstream version. [March 2017]
* Big updates to d/rules and d/copyright to update to upstream
changes made since the 1.2.x release.
* Refresh patches to bring up to date with upstream changes since
1.2.x.
* control: add lsb-base as dependency for sysvinit scripts.
* Suppress spelling-error-in-binary Lintian messages.
-- Tim Potter <tpot@hpe.com> Thu, 13 Apr 2017 16:45:57 +1000
kubernetes (1.2.5+dfsg-1) experimental; urgency=medium
* New upstream release [June 2016].
* Switch to private "github.com/golang/glog" due to log noise.
* Disabled failing tests; no longer ignore failures in tests.
* Build/test using 2 cores only.
* New patch to update appc/cni name space (fixes FTBFS).
* Removed obsolete "spf13-cobra.patch".
-- Dmitry Smirnov <onlyjob@debian.org> Sun, 03 Jul 2016 04:12:28 +1000
kubernetes (1.2.4+dfsg-2) experimental; urgency=medium
* Added new patch to fix incompatibility with "imdario/mergo" v0.2.2
(Closes: #825753).
Thanks, Florian Ernst.
* Enable tests but ignore failures for now.
-- Dmitry Smirnov <onlyjob@debian.org> Fri, 17 Jun 2016 01:41:38 +1000
kubernetes (1.2.4+dfsg-1) experimental; urgency=medium
* New upstream release [May 2016].
* New patch to print output of "uname -m" on unsupported architectures.
* New "docker.patch" to fix potential FTBFS.
+ Build-Depends += "golang-github-docker-distribution-dev".
-- Dmitry Smirnov <onlyjob@debian.org> Wed, 15 Jun 2016 21:03:01 +1000
kubernetes (1.2.3+dfsg-1) experimental; urgency=low
* Initial release (Closes: #795652).
-- Dmitry Smirnov <onlyjob@debian.org> Mon, 25 Apr 2016 22:40:12 +1000

93
kubernetes/kubernetes-1.26.1/debian/deb_folder/control
View File

@ -1,93 +0,0 @@
Source: kubernetes-1.26.1
Section: admin
Priority: optional
Maintainer: StarlingX Developers <StarlingX-discuss@lists.StarlingX.io>
Build-Depends: debhelper-compat (= 13),
build-essential,
bash-completion,
jq,
rsync,
go-bindata,
go-md2man,
golang-1.19
Standards-Version: 4.4.1
Homepage: http://kubernetes.io/
XS-Build-Size: 15GB
Package: kubernetes-1.26.1-client
Provides: kubernetes-utils
Architecture: amd64
Depends: ${misc:Depends}, ${shlibs:Depends}
Recommends: ${misc:Recommends}
Built-Using: ${misc:Built-Using}
Description: Kubernetes Command Line Tool
The Kubernetes command line tool for interacting with the Kubernetes API.
Package: kubernetes-1.26.1-master
Architecture: amd64
Depends: ${misc:Depends}, ${shlibs:Depends},
adduser,
lsb-base,
etcd
Recommends: ${misc:Recommends}, kubernetes-1.26.1-client
Built-Using: ${misc:Built-Using}
Description: Kubernetes services for master host
Container Cluster Manager from Google. Kubernetes is an open source system
for managing containerized applications across multiple hosts, providing
basic mechanisms for deployment, maintenance, and scaling of applications.
.
Linux kernel version 3.8 or above is required for proper operation of the
daemon process, and that any lower versions may have subtle and/or glaring
issues.
.
This package provides "kube-apiserver", "kube-controller-manager" and
"kube-scheduler" daemons.
Package: kubernetes-1.26.1-node
Provides: cadvisor
Architecture: amd64
Depends: ${misc:Depends}, ${shlibs:Depends},
adduser,
conntrack,
conntrackd,
docker.io,
lsb-base,
socat,
Recommends: ${misc:Recommends}, kubernetes-1.26.1-client
Built-Using: ${misc:Built-Using}
Description: Kubernetes services for node host
Container Cluster Manager from Google. Kubernetes is an open source system
for managing containerized applications across multiple hosts, providing
basic mechanisms for deployment, maintenance, and scaling of applications.
.
Linux kernel version 3.8 or above is required for proper operation of the
daemon process, and that any lower versions may have subtle and/or glaring
issues.
Package: kubernetes-1.26.1-kubeadm
Architecture: amd64
Depends: ${misc:Depends}, containernetworking-plugins
Recommends: ${misc:Recommends}, kubernetes-1.26.1-client
Built-Using: ${misc:Built-Using}
Description: Kubernetes Cluster Bootstrapping Tool
The Kubernetes command line tool for bootstrapping a Kubernetes cluster.
Package: kubernetes-1.26.1-misc
Architecture: amd64
Depends: ${misc:Depends}, ${shlibs:Depends}
Recommends: ${misc:Recommends}
Built-Using: ${misc:Built-Using}
Description: dummy package
Kubernetes dummy package for misc stuff we don't want to install in production.
Package: kubernetes-1.26.1-unit-test
Architecture: amd64
Depends: ${misc:Depends}, ${shlibs:Depends},
hostname,
rsync,
etcd (>= 2.0.9),
network-manager,
Recommends: ${misc:Recommends}
Built-Using: ${misc:Built-Using}
Description: Kubernetes unit test
Kubernetes unit-test framework.

1820
kubernetes/kubernetes-1.26.1/debian/deb_folder/copyright
View File

File diff suppressed because it is too large Load Diff

17
kubernetes/kubernetes-1.26.1/debian/deb_folder/kubeadm.conf
View File

@ -1,17 +0,0 @@
# Note: This dropin only works with kubeadm and kubelet v1.11+
[Service]
Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf"
Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml"
# This is a file that "kubeadm init" and "kubeadm join" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically
EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env
# This is a file that the user can use for overrides of the kubelet args as a last resort. Preferably, the user should use
# the .NodeRegistration.KubeletExtraArgs object in the configuration files instead. KUBELET_EXTRA_ARGS should be sourced from this file.
EnvironmentFile=-/etc/default/kubelet
ExecStart=
ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS
ExecStartPre=-/usr/local/sbin/sanitize_kubelet_reserved_cpus.sh /etc/default/kubelet
ExecStartPre=-/usr/bin/kubelet-cgroup-setup.sh
ExecStartPost=/bin/bash -c 'echo $MAINPID > /var/run/kubelet.pid;'
ExecStopPost=/bin/rm -f /var/run/kubelet.pid
StartLimitInterval=0
RestartSec=10

2
kubernetes/kubernetes-1.26.1/debian/deb_folder/kubernetes-1.26.1-client.install
View File

@ -1,2 +0,0 @@
usr/local/kubernetes/1.26.1/stage2/usr/bin/kubectl
usr/local/kubernetes/1.26.1/stage2/usr/share/bash-completion/completions/kubectl

9
kubernetes/kubernetes-1.26.1/debian/deb_folder/kubernetes-1.26.1-client.lintian-overrides
View File

@ -1,9 +0,0 @@
## Generated man pages: TODO
manpage-has-bad-whatis-entry usr/share/man/*
manpage-has-errors-from-man usr/share/man/man1/*
## Bash-completion script does not have to be executable:
script-not-executable usr/share/bash-completion/completions/kubectl
## Override annoying/useless messages
kubernetes-client: spelling-error-in-binary

1
kubernetes/kubernetes-1.26.1/debian/deb_folder/kubernetes-1.26.1-kubeadm.dirs
View File

@ -1 +0,0 @@
usr/local/kubernetes/1.26.1/stage2/etc/systemd/system/kubelet.service.d/

2
kubernetes/kubernetes-1.26.1/debian/deb_folder/kubernetes-1.26.1-kubeadm.install
View File

@ -1,2 +0,0 @@
usr/local/kubernetes/1.26.1/stage1/usr/bin/kubeadm
usr/local/kubernetes/1.26.1/stage2/etc/systemd/system/kubelet.service.d/kubeadm.conf

5
kubernetes/kubernetes-1.26.1/debian/deb_folder/kubernetes-1.26.1-master.dirs
View File

@ -1,5 +0,0 @@
etc/kubernetes-1.26.1
etc/kubernetes-1.26.1/addons
etc/kubernetes-1.26.1/addons/volumesnapshots
etc/kubernetes-1.26.1/addons/volumesnapshots/crd
etc/kubernetes-1.26.1/addons/volumesnapshots/volume-snapshot-controller

8
kubernetes/kubernetes-1.26.1/debian/deb_folder/kubernetes-1.26.1-master.install
View File

@ -1,8 +0,0 @@
usr/bin/kube-apiserver
usr/bin/kube-controller-manager
usr/bin/kube-scheduler
etc/kubernetes-1.26.1/addons/volumesnapshots/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml
etc/kubernetes-1.26.1/addons/volumesnapshots/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml
etc/kubernetes-1.26.1/addons/volumesnapshots/crd/snapshot.storage.k8s.io_volumesnapshots.yaml
etc/kubernetes-1.26.1/addons/volumesnapshots/volume-snapshot-controller/volume-snapshot-controller-deployment.yaml
etc/kubernetes-1.26.1/addons/volumesnapshots/volume-snapshot-controller/rbac-volume-snapshot-controller.yaml

7
kubernetes/kubernetes-1.26.1/debian/deb_folder/kubernetes-1.26.1-master.lintian-overrides
View File

@ -1,7 +0,0 @@
## No manual page for hyperkube
kubernetes-master: binary-without-manpage usr/bin/hyperkube
## Override annoying/useless messages
kubernetes-master: spelling-error-in-binary
kubernetes-master: manpage-has-errors-from-man usr/share/man/man1/*
kubernetes-master: manpage-has-bad-whatis-entry usr/share/man/man1/*

3
kubernetes/kubernetes-1.26.1/debian/deb_folder/kubernetes-1.26.1-misc.docs
View File

@ -1,3 +0,0 @@
src/k8s.io/kubernetes/README.md
src/k8s.io/kubernetes/SUPPORT.md
src/k8s.io/kubernetes/_output/NOTICE

1
kubernetes/kubernetes-1.26.1/debian/deb_folder/kubernetes-1.26.1-misc.install
View File

@ -1 +0,0 @@
usr/bin/kube-proxy

10
kubernetes/kubernetes-1.26.1/debian/deb_folder/kubernetes-1.26.1-misc.manpages
View File

@ -1,10 +0,0 @@
# kubernetes-client
src/k8s.io/kubernetes/_output/man/kubeadm*
src/k8s.io/kubernetes/_output/man/kubectl*
# kubernetes-master
src/k8s.io/kubernetes/_output/man/kube-apiserver*
src/k8s.io/kubernetes/_output/man/kube-scheduler*
src/k8s.io/kubernetes/_output/man/kube-controller-manager*
# kubernetes-node
src/k8s.io/kubernetes/_output/man/kubelet*
src/k8s.io/kubernetes/_output/man/kube-proxy*

1
kubernetes/kubernetes-1.26.1/debian/deb_folder/kubernetes-1.26.1-node.install
View File

@ -1 +0,0 @@
usr/local/kubernetes/1.26.1/stage2/usr/bin/kubelet

4
kubernetes/kubernetes-1.26.1/debian/deb_folder/kubernetes-1.26.1-node.lintian-overrides
View File

@ -1,4 +0,0 @@
## Override annoying/useless messages
kubernetes-node: spelling-error-in-binary
kubernetes-node: manpage-has-errors-from-man usr/share/man/man1/*
kubernetes-node: manpage-has-bad-whatis-entry usr/share/man/man1/*

1
kubernetes/kubernetes-1.26.1/debian/deb_folder/kubernetes-1.26.1-unit-test.install
View File

@ -1 +0,0 @@
var/lib/kubernetes-unit-test/

Some files were not shown because too many files have changed in this diff Show More