Adjusting daemonset anchor readiness check

To avoid pods cycling too quickly by checking if manifest was created by daemonset and the component on the same host is ready Change-Id: I7f9b35e222ef5934fca71f30fdf9941caa60ccd7
2023-03-30 13:32:26 -04:00 · 2023-03-30 13:32:26 -04:00 · 5f62088d01
commit 5f62088d01
parent 2125b61b57
10 changed files with 148 additions and 55 deletions
--- a/charts/controller_manager/Chart.yaml
+++ b/charts/controller_manager/Chart.yaml
@ -15,4 +15,4 @@
 apiVersion: v1
 description: A chart for Kubernetes controller-manager
 name: controller_manager
-version: 0.1.1
+version: 0.1.2
--- a/charts/controller_manager/templates/bin/_anchor.tpl
+++ b/charts/controller_manager/templates/bin/_anchor.tpl
@ -15,24 +15,60 @@

 set -xu

-compare_copy_files() {
+snapshot_files() {
+    SNAPSHOT_DIR=${1}
+    {{ range $dest, $source := .Values.anchor.files_to_copy }}
+    mkdir -p $(dirname "${SNAPSHOT_DIR}{{ $dest }}")
+    cp "{{ $source }}" "${SNAPSHOT_DIR}{{ $dest }}"
+    {{- end }}
+    {{ range $key, $val := .Values.conf }}
+    {{- if $val.file }}
+    cp "/tmp/etc/{{ $val.file }}" "${SNAPSHOT_DIR}/etc/kubernetes/controller-manager/{{ $val.file }}"
+    {{- end }}
+    {{- end }}
+    # annotate the static manifest with the name of the creating anchor pod
+    sed -i "/created-by: /s/ANCHOR_POD/${POD_NAME}/" "${SNAPSHOT_DIR}{{ .Values.anchor.kubelet.manifest_path }}/kubernetes-controller-manager.yaml"
+}

-    {{range .Values.anchor.files_to_copy}}
-    if [ ! -e /host{{ .dest }} ] || ! cmp -s {{ .source }} /host{{ .dest }}; then
-        mkdir -p $(dirname /host{{ .dest }})
-        cp {{ .source }} /host{{ .dest }}
-        chmod go-rwx /host{{ .dest }}
+compare_copy_files() {
+    SNAPSHOT_DIR=${1}
+    {{ range $dest, $source := .Values.anchor.files_to_copy }}
+    SRC="${SNAPSHOT_DIR}{{ $dest }}"
+    DEST="/host{{ $dest }}"
+    if [ ! -e "${DEST}" ] || ! cmp -s "${SRC}" "${DEST}"; then
+        mkdir -p $(dirname "${DEST}")
+        cp "${SRC}" "${DEST}"
+        chmod go-rwx "${DEST}"
    fi
-    {{end}}
+    {{- end}}
+    {{ range $key, $val := .Values.conf }}
+    {{- if $val.file }}
+    SRC="${SNAPSHOT_DIR}/etc/kubernetes/controller-manager/{{ $val.file }}"
+    DEST="/host/etc/kubernetes/controller-manager/{{ $val.file }}"
+    if [ ! -e "${DEST}" ] || ! cmp -s "${SRC}" "${DEST}"; then
+        mkdir -p $(dirname "${DEST}")
+        cp "${SRC}" "${DEST}"
+        chmod go-rwx "${DEST}"
+    fi
+    {{- end }}
+    {{- end }}
 }

 cleanup() {
-
-    {{range .Values.anchor.files_to_copy}}
-    rm -f /host{{ .dest }}
-    {{end}}
+    {{- range $dest, $source := .Values.anchor.files_to_copy }}
+    rm -f "/host{{ $dest }}"
+    {{- end }}
+    {{  range $key, $val := .Values.conf }}
+    {{- if $val.file }}
+    rm -f "/host/etc/kubernetes/controller-manager/{{ $val.file }}"
+    {{- end }}
+    {{- end }}
 }

+SNAPSHOT_DIR=$(mktemp -d)
+
+snapshot_files "${SNAPSHOT_DIR}"
+
 while true; do

    if [ -e /tmp/stop ]; then
@ -45,7 +81,7 @@ while true; do

    # Compare and replace files on Genesis host if needed
    # Copy files to other master nodes
-    compare_copy_files
+    compare_copy_files "${SNAPSHOT_DIR}"

    sleep {{ .Values.anchor.period }}
 done
--- a/charts/controller_manager/templates/daemonset.yaml
+++ b/charts/controller_manager/templates/daemonset.yaml
@ -64,6 +64,10 @@ spec:
            value: /host{{ .Values.anchor.kubelet.manifest_path }}/kubernetes-controller-manager.yaml
          - name: ETC_PATH
            value: /host{{ .Values.controller_manager.host_etc_path }}
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
 {{ tuple $envAll $envAll.Values.pod.resources.anchor_pod | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
 {{ dict "envAll" $envAll "application" "kubernetes" "container" "anchor" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
          command:
@ -73,13 +77,20 @@ spec:
              exec:
                command:
                  - /tmp/bin/pre_stop
-
          readinessProbe:
-            httpGet:
-              host: 127.0.0.1
-              path: /healthz
-              port: {{ .Values.network.kubernetes_controller_manager.port }}
-              scheme: HTTPS
+            exec:
+              command:
+              - /bin/bash
+              - -c
+              - |-
+                grep -q "created-by: ${POD_NAME}" "${MANIFEST_PATH}" || exit 1
+                [ "$(curl -k -s -S -o /dev/null \
+                    --cert "/host{{ .Values.controller_manager.host_etc_path }}/controller-manager.pem" \
+                    --key "/host{{ .Values.controller_manager.host_etc_path }}/controller-manager-key.pem" \
+                    --cacert "/host{{ .Values.controller_manager.host_etc_path }}/cluster-ca.pem" \
+                    "https://localhost:{{ .Values.network.kubernetes_controller_manager.port }}/healthz" \
+                    -w "%{http_code}")" = "200" ]
+                exit $?
            initialDelaySeconds: 10
            periodSeconds: 5
            timeoutSeconds: 5
--- a/charts/controller_manager/templates/etc/_kubernetes-controller-manager.yaml.tpl
+++ b/charts/controller_manager/templates/etc/_kubernetes-controller-manager.yaml.tpl
@ -23,6 +23,7 @@ metadata:
    {{ .Values.service.name }}-service: enabled
 {{ tuple $envAll "kubernetes" "controller-manager" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
  annotations:
+    created-by: ANCHOR_POD
    {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
 {{ dict "envAll" $envAll "podName" "controller-manager" "containerNames" (list "controller-manager") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 4 }}
 spec:
--- a/charts/controller_manager/values.yaml
+++ b/charts/controller_manager/values.yaml
@ -32,18 +32,12 @@ anchor:
    manifest_path: /etc/kubernetes/manifests
  period: 15
  files_to_copy:
-    - source: /configmap/cluster-ca.pem
-      dest: /etc/kubernetes/controller-manager/cluster-ca.pem
-    - source: /configmap/controller-manager.pem
-      dest: /etc/kubernetes/controller-manager/controller-manager.pem
-    - source: /configmap/kubeconfig.yaml
-      dest: /etc/kubernetes/controller-manager/kubeconfig.yaml
-    - source: /secret/controller-manager-key.pem
-      dest: /etc/kubernetes/controller-manager/controller-manager-key.pem
-    - source: /secret/service-account.priv
-      dest: /etc/kubernetes/controller-manager/service-account.priv
-    - source: /configmap/kubernetes-controller-manager.yaml
-      dest: /etc/kubernetes/manifests/kubernetes-controller-manager.yaml
+    /etc/kubernetes/controller-manager/cluster-ca.pem: /configmap/cluster-ca.pem
+    /etc/kubernetes/controller-manager/controller-manager.pem: /configmap/controller-manager.pem
+    /etc/kubernetes/controller-manager/kubeconfig.yaml: /configmap/kubeconfig.yaml
+    /etc/kubernetes/controller-manager/controller-manager-key.pem: /secret/controller-manager-key.pem
+    /etc/kubernetes/controller-manager/service-account.priv: /secret/service-account.priv
+    /etc/kubernetes/manifests/kubernetes-controller-manager.yaml: /configmap/kubernetes-controller-manager.yaml

 controller_manager:
  host_etc_path: /etc/kubernetes/controller-manager
--- a/charts/scheduler/Chart.yaml
+++ b/charts/scheduler/Chart.yaml
@ -1,4 +1,4 @@
 apiVersion: v1
 description: A chart for Kubernetes scheduler.
 name: scheduler
-version: 0.1.1
+version: 0.1.2
--- a/charts/scheduler/templates/bin/_anchor.tpl
+++ b/charts/scheduler/templates/bin/_anchor.tpl
@ -17,22 +17,60 @@

 set -xu

+snapshot_files() {
+    SNAPSHOT_DIR=${1}
+    {{ range $dest, $source := .Values.anchor.files_to_copy }}
+    mkdir -p $(dirname "${SNAPSHOT_DIR}{{ $dest }}")
+    cp "{{ $source }}" "${SNAPSHOT_DIR}{{ $dest }}"
+    {{- end }}
+    {{ range $key, $val := .Values.conf }}
+    {{- if $val.file }}
+    cp "/tmp/etc/{{ $val.file }}" "${SNAPSHOT_DIR}/etc/kubernetes/scheduler/{{ $val.file }}"
+    {{- end }}
+    {{- end }}
+    # annotate the static manifest with the name of the creating anchor pod
+    sed -i "/created-by: /s/ANCHOR_POD/${POD_NAME}/" "${SNAPSHOT_DIR}{{ .Values.anchor.kubelet.manifest_path }}/kubernetes-scheduler.yaml"
+}
+
 compare_copy_files() {
-    {{- range .Values.anchor.files_to_copy }}
-    if [ ! -e /host{{ .dest }} ] || ! cmp -s {{ .source }} /host{{ .dest }}; then
-        mkdir -p $(dirname /host{{ .dest }})
-        cp {{ .source }} /host{{ .dest }}
-        chmod go-rwx /host{{ .dest }}
+    SNAPSHOT_DIR=${1}
+    {{ range $dest, $source := .Values.anchor.files_to_copy }}
+    SRC="${SNAPSHOT_DIR}{{ $dest }}"
+    DEST="/host{{ $dest }}"
+    if [ ! -e "${DEST}" ] || ! cmp -s "${SRC}" "${DEST}"; then
+        mkdir -p $(dirname "${DEST}")
+        cp "${SRC}" "${DEST}"
+        chmod go-rwx "${DEST}"
    fi
+    {{- end}}
+    {{ range $key, $val := .Values.conf }}
+    {{- if $val.file }}
+    SRC="${SNAPSHOT_DIR}/etc/kubernetes/scheduler/{{ $val.file }}"
+    DEST="/host/etc/kubernetes/scheduler/{{ $val.file }}"
+    if [ ! -e "${DEST}" ] || ! cmp -s "${SRC}" "${DEST}"; then
+        mkdir -p $(dirname "${DEST}")
+        cp "${SRC}" "${DEST}"
+        chmod go-rwx "${DEST}"
+    fi
+    {{- end }}
    {{- end }}
 }

 cleanup() {
-    {{- range .Values.anchor.files_to_copy }}
-    rm -f /host{{ .dest }}
+    {{- range $dest, $source := .Values.anchor.files_to_copy }}
+    rm -f "/host{{ $dest }}"
+    {{- end }}
+    {{  range $key, $val := .Values.conf }}
+    {{- if $val.file }}
+    rm -f "/host/etc/kubernetes/scheduler/{{ $val.file }}"
+    {{- end }}
    {{- end }}
 }

+SNAPSHOT_DIR=$(mktemp -d)
+
+snapshot_files "${SNAPSHOT_DIR}"
+
 while true; do
    if [ -e /tmp/stop ]; then
        echo Stopping
@ -44,7 +82,7 @@ while true; do

    # Compare and replace files on Genesis host if needed
    # Copy files to other master nodes
-    compare_copy_files
+    compare_copy_files "${SNAPSHOT_DIR}"

    sleep {{ .Values.anchor.period }}
 done
--- a/charts/scheduler/templates/etc/_kubernetes-scheduler.yaml.tpl
+++ b/charts/scheduler/templates/etc/_kubernetes-scheduler.yaml.tpl
@ -25,6 +25,7 @@ metadata:
    {{ .Values.service.name }}-service: enabled
 {{ tuple $envAll "kubernetes" "scheduler" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
  annotations:
+    created-by: ANCHOR_POD
    {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
 {{ dict "envAll" $envAll "podName" "scheduler" "containerNames" (list "scheduler") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 4 }}
 spec:
--- a/charts/scheduler/templates/sched-anchor.yaml
+++ b/charts/scheduler/templates/sched-anchor.yaml
@ -56,6 +56,15 @@ spec:
        - name: anchor
          image: {{ .Values.images.tags.anchor }}
          imagePullPolicy: {{ .Values.images.pull_policy }}
+          env:
+          - name: MANIFEST_PATH
+            value: /host{{ .Values.anchor.kubelet.manifest_path }}/kubernetes-scheduler.yaml
+          - name: ETC_PATH
+            value: /host{{ .Values.scheduler.host_etc_path }}
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
 {{ tuple $envAll $envAll.Values.pod.resources.anchor_daemonset | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
 {{ dict "envAll" $envAll "application" "scheduler" "container" "anchor" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
          command:
@ -67,11 +76,19 @@ spec:
                  - /tmp/bin/pre_stop

          readinessProbe:
-            httpGet:
-              host: 127.0.0.1
-              path: /healthz
-              port: {{ .Values.network.kubernetes_scheduler.port }}
-              scheme: HTTPS
+            exec:
+              command:
+              - /bin/bash
+              - -c
+              - |-
+                grep -q "created-by: ${POD_NAME}" "${MANIFEST_PATH}" || exit 1
+                [ "$(curl -k -s -S -o /dev/null \
+                    --cert "/host{{ .Values.scheduler.host_etc_path }}/scheduler.pem" \
+                    --key "/host{{ .Values.scheduler.host_etc_path }}/scheduler-key.pem" \
+                    --cacert "/host{{ .Values.scheduler.host_etc_path }}/cluster-ca.pem" \
+                    "https://localhost:{{ .Values.network.kubernetes_scheduler.port }}/healthz" \
+                    -w "%{http_code}")" = "200" ]
+                exit $?
            initialDelaySeconds: 10
            periodSeconds: 5
            timeoutSeconds: 5
--- a/charts/scheduler/values.yaml
+++ b/charts/scheduler/values.yaml
@ -8,16 +8,11 @@ anchor:
  period: 15
  termination_grace_period: 3600
  files_to_copy:
-    - source: /configmap/cluster-ca.pem
-      dest: /etc/kubernetes/scheduler/cluster-ca.pem
-    - source: /configmap/scheduler.pem
-      dest: /etc/kubernetes/scheduler/scheduler.pem
-    - source: /configmap/kubeconfig.yaml
-      dest: /etc/kubernetes/scheduler/kubeconfig.yaml
-    - source: /secret/scheduler-key.pem
-      dest: /etc/kubernetes/scheduler/scheduler-key.pem
-    - source: /configmap/kubernetes-scheduler.yaml
-      dest: /etc/kubernetes/manifests/kubernetes-scheduler.yaml
+    /etc/kubernetes/scheduler/cluster-ca.pem: /configmap/cluster-ca.pem
+    /etc/kubernetes/scheduler/scheduler.pem: /configmap/scheduler.pem
+    /etc/kubernetes/scheduler/kubeconfig.yaml: /configmap/kubeconfig.yaml
+    /etc/kubernetes/scheduler/scheduler-key.pem: /secret/scheduler-key.pem
+    /etc/kubernetes/manifests/kubernetes-scheduler.yaml: /configmap/kubernetes-scheduler.yaml

 labels:
  scheduler: