Use HAProxy for apiserver discovery

This removes the reliance on coredns for APIserver discovery, allowing
a simpler configuration that is compatible with corednx 1.0.x

Change-Id: Ia3b7b5627c16ec47af6b0d6d5e8dee2674e9b1ee

charts/coredns/templates/bin/_anchor.tpl

@@ -1,130 +0,0 @@
-#!/bin/sh
-
-{{- $envAll := . }}
-
-set -x
-
-export MANIFEST_PATH=/host{{ .Values.anchor.kubelet.manifest_path }}/{{ .Values.service.name }}.yaml
-export ETC_PATH=/host{{ .Values.coredns.host_etc_path }}
-TOKEN_PATH=/var/run/secrets/kubernetes.io/serviceaccount/token
-CA_CERT_PATH=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-
-copy_etc_files() {
-    mkdir -p $ETC_PATH/zones
-    cp /configmap/* /secret/* $ETC_PATH
-    create_corefile
-}
-
-create_corefile() {
-    cat <<EOCOREFILE > $ETC_PATH/Corefile
-promenade {
-    file /etc/coredns/zones/promenade
-    loadbalance
-    errors stdout
-    log stdout
-}
-
-. {
-    kubernetes{{- range .Values.coredns.kubernetes_zones }} {{ . -}}{{- end }} {
-        endpoint https://{{ .Values.network.kubernetes_netloc }}
-        tls /etc/coredns/coredns.pem /etc/coredns/coredns-key.pem /etc/coredns/cluster-ca.pem
-
-        pods insecure
-    }
-    {{- if .Values.coredns.upstream_nameservers }}
-    {{ range .Values.coredns.upstream_nameservers }}
-    proxy . {{ . }}
-    {{- end }}
-    {{- end }}
-
-    loadbalance
-    cache {{ .Values.coredns.cache.ttl }}
-
-    errors stdout
-    log stdout
-}
-EOCOREFILE
-}
-
-create_manifest() {
-    mkdir -p $(dirname $MANIFEST_PATH)
-# XXX liveness/readiness probes
-    cat <<EODOC > $MANIFEST_PATH
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: {{ .Values.service.name }}
-  namespace: {{ .Release.Namespace }}
-  labels:
-    {{ .Values.service.name }}-service: enabled
-    anchor-managed: enabled
-spec:
-  hostNetwork: true
-  containers:
-    - name: coredns
-      image: {{ .Values.images.coredns }}
-      command:
-        - /coredns
-        - -conf
-        - /etc/coredns/Corefile
-      volumeMounts:
-        - name: etc
-          mountPath: /etc/coredns
-  volumes:
-    - name: etc
-      hostPath:
-        path: {{ .Values.coredns.host_etc_path }}
-EODOC
-}
-
-update_managed_zones() {
-{{- range .Values.coredns.zones }}
-
-FILENAME="$ETC_PATH/zones/{{ .name }}"
-NEXT_FILENAME="${FILENAME}-next"
-SUCCESS=1
-NOW=$(date +%s)
-
-# Add Header
-cat <<EOBIND > $NEXT_FILENAME
-\$ORIGIN {{ .name }}.
-{{ .name }}. IN SOA @ root $NOW 3h 15m 1w 1d
-
-EOBIND
-{{ range .services }}
-# Don't accidentally log service account token
-set +x
-SERVICE_IPS=$(kubectl \
-    --server https://{{ $envAll.Values.network.kubernetes_netloc }} \
-    --certificate-authority $CA_CERT_PATH \
-    --token $(cat $TOKEN_PATH) \
-    -n {{ .service.namespace }} \
-        get ep {{ .service.name }} \
-            -o 'jsonpath={.subsets[*].addresses[*].ip}')
-set -x
-if [ "x$SERVICE_IPS" != "x" ]; then
-    for IP in $SERVICE_IPS; do
-        echo {{ .bind_name }} IN A $IP >> $NEXT_FILENAME
-    done
-else
-    echo Failed to upate zone file for {{ .name }}
-    SUCCESS=0
-fi
-{{- end }}
-
-if [ $SUCCESS = 1 ]; then
-    echo Replacing zone file $FILENAME
-    mv $NEXT_FILENAME $FILENAME
-fi
-{{- end }}
-}
-
-copy_etc_files
-create_manifest
-
-while true; do
-    update_managed_zones
-
-    sleep {{ .Values.anchor.period }}
-done

charts/coredns/templates/configmap-bin.yaml

@@ -1,8 +0,0 @@
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: {{ .Values.service.name }}-bin
-data:
-  anchor: |+
-{{ tuple "bin/_anchor.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}

charts/coredns/templates/configmap-etc.yaml

@@ -4,5 +4,4 @@ kind: ConfigMap
 metadata:
   name: {{ .Values.service.name }}-etc
 data:
-  cluster-ca.pem: {{ .Values.tls.ca | quote }}
+  Corefile: {{ .Values.conf.coredns.corefile | quote }}
-  coredns.pem: {{ .Values.tls.cert | quote }}

charts/coredns/templates/daemonset.yaml

@@ -1,83 +0,0 @@
----
-apiVersion: "extensions/v1beta1"
-kind: DaemonSet
-metadata:
-  name: {{ .Values.service.name }}-anchor
-  labels:
-    application: coredns
-    component: coredns-anchor
-spec:
-  selector:
-    matchLabels:
-      {{ .Values.service.name | quote }}: anchor
-  updateStrategy:
-    rollingUpdate:
-      maxUnavailable: 1
-  template:
-    metadata:
-      labels:
-        {{ .Values.service.name | quote }}: anchor
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
-        configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
-        configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }}
-    spec:
-      hostNetwork: true
-      {{- if .Values.node_selector.key }}
-      nodeSelector:
-        {{ .Values.node_selector.key }}: {{ .Values.node_selector.value }}
-      {{- end }}
-      tolerations:
-        - key: node-role.kubernetes.io/master
-          effect: NoSchedule
-        - key: CriticalAddonsOnly
-          operator: Exists
-      serviceAccountName: {{ .Values.service.name }}
-      containers:
-        - name: anchor
-          image: {{ .Values.images.anchor }}
-          command:
-            - /tmp/bin/anchor
-#        livenessProbe:
-#          httpGet:
-#            path: /healthz
-#            port: 10249
-#          initialDelaySeconds: 15
-#          periodSeconds: 15
-#          failureThreshold: 3
-#        readinessProbe:
-#          exec:
-#            command:
-#            - sh
-#            - -c
-#            - |-
-#              set -ex
-#
-#              iptables-save | grep 'default/kubernetes:https'
-#          initialDelaySeconds: 15
-#          periodSeconds: 15
-          volumeMounts:
-            - name: bin
-              mountPath: /tmp/bin
-            - name: etc
-              mountPath: /configmap
-            - name: host
-              mountPath: /host
-            - name: secret
-              mountPath: /secret
-      volumes:
-        - name: bin
-          configMap:
-            name: {{ .Values.service.name }}-bin
-            defaultMode: 0555
-        - name: etc
-          configMap:
-            name: {{ .Values.service.name }}-etc
-            defaultMode: 0444
-        - name: host
-          hostPath:
-            path: /
-        - name: secret
-          secret:
-            secretName: {{ .Values.service.name }}
-            defaultMode: 0444

charts/coredns/templates/deployment.yaml

@@ -0,0 +1,85 @@
+{{/*
+Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: coredns
+  labels:
+    {{ .Values.service.name }}: enabled
+    kubernetes.io/name: "CoreDNS"
+spec:
+  replicas: 2
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 1
+  selector:
+    matchLabels:
+      {{ .Values.service.name }}: enabled
+  template:
+    metadata:
+      labels:
+        {{ .Values.service.name }}: enabled
+    spec:
+      serviceAccountName: coredns
+      tolerations:
+        - key: "CriticalAddonsOnly"
+          operator: "Exists"
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: {{ .Values.service.name }}
+                  operator: In
+                  values:
+                  - enabled
+              topologyKey: kubernetes.io/hostname
+      containers:
+      - name: coredns
+        image: {{ .Values.images.tags.coredns | quote }}
+        imagePullPolicy: {{ .Values.images.pull_policy | quote }}
+        args: [ "-conf", "/etc/coredns/Corefile" ]
+        volumeMounts:
+        - name: config-volume
+          mountPath: /etc/coredns
+        ports:
+        - containerPort: 53
+          name: dns
+          protocol: UDP
+        - containerPort: 53
+          name: dns-tcp
+          protocol: TCP
+        livenessProbe:
+          httpGet:
+            path: /health
+            port: 8080
+            scheme: HTTP
+          initialDelaySeconds: 60
+          timeoutSeconds: 5
+          successThreshold: 1
+          failureThreshold: 5
+      dnsPolicy: Default
+      volumes:
+        - name: config-volume
+          configMap:
+            name: {{ .Values.service.name }}-etc
+            items:
+            - key: Corefile
+              path: Corefile

charts/coredns/templates/rbac.yaml

@@ -3,7 +3,7 @@ apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: coredns
-  namespace: kube-system
+  namespace: {{ .Release.Namespace }}
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
@@ -17,7 +17,7 @@ subjects:
     apiGroup: rbac.authorization.k8s.io
   - kind: ServiceAccount
     name: coredns
-    namespace: kube-system
+    namespace: {{ .Release.Namespace }}
 roleRef:
   kind: ClusterRole
   name: system:coredns

charts/coredns/templates/secret.yaml

@@ -1,8 +0,0 @@
----
-apiVersion: v1
-kind: Secret
-metadata:
-  name: {{ .Values.service.name }}
-type: Opaque
-data:
-  coredns-key.pem: {{ .Values.tls.key | b64enc }}

charts/coredns/templates/service.yaml

@@ -3,9 +3,12 @@ apiVersion: v1
 kind: Service
 metadata:
   name: {{ .Values.service.name }}
+  labels:
+    kubernetes.io/cluster-service: "true"
+    kubernetes.io/name: "CoreDNS"
 spec:
   selector:
-    {{ .Values.service.name }}-service: enabled
+    {{ .Values.service.name }}: enabled
   clusterIP: {{ .Values.service.ip }}
   ports:
     - name: dns

charts/coredns/values.yaml

@@ -1,38 +1,26 @@
-anchor:
+conf:
-  kubelet:
+  coredns:
-    manifest_path: /etc/kubernetes/manifests
+    corefile: |
-  node_selector: {}
+      .:53 {
-  period: 30
+          errors
-
+          health
-tls:
+          autopath @kubernetes
-  ca: placeholder
+          kubernetes cluster.local 10.96.0.0/16 10.97.0.0/16 {
-  cert: placeholder
+            pods insecure
-  key: placeholder
+            fallthrough in-addr.arpa ip6.arpa
-
+            upstream 8.8.8.8
-coredns:
+            upstream 8.8.4.4
-  kubernetes_zones:
+          }
-    - cluster.local
+          prometheus :9153
-  cache:
+          proxy . 8.8.8.8
-    ttl: 60
+          proxy . 8.8.4.4
-  host_etc_path: /etc/coredns
+          cache 30
-  upstream_nameservers:
+      }
-    - 8.8.8.8
-    - 8.8.4.4
-
-  zones:
-    - name: promenade
-      services:
-        - bind_name: apiserver.kubernetes
-          service:
-            name: kubernetes
-            namespace: default
 
 images:
-  anchor: gcr.io/google_containers/hyperkube-amd64:v1.8.6
+  tags:
-  coredns: coredns/coredns:0.9.9
+    coredns: coredns/coredns:1.0.5
-
+  pull_policy: "IfNotPresent"
-network:
-  kubernetes_netloc: 10.96.0.1
 
 node_selector: {}
 

charts/haproxy/Chart.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+description: A chart for using HAProxy for Kubernetes API server discovery
+name: haproxy
+version: 0.1.0

charts/haproxy/requirements.yaml

@@ -0,0 +1,4 @@
+dependencies:
+  - name: helm-toolkit
+    repository: http://localhost:8879/charts
+    version: 0.1.0

charts/haproxy/templates/bin/_anchor.tpl

@@ -0,0 +1,129 @@
+#!/bin/sh
+{{/*
+Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- $envAll := . }}
+
+set -x
+
+compare_copy_files() {
+    {{- range .Values.conf.anchor.files_to_copy }}
+    if [ ! -e /host{{ .dest }} ] || ! cmp -s {{ .source }} /host{{ .dest }}; then
+        mkdir -p $(dirname /host{{ .dest }})
+        cp {{ .source }} /host{{ .dest }}
+    fi
+    {{- end }}
+}
+
+install_config() {
+    SUCCESS=1
+    # Inject global and default config
+    mkdir -p $(dirname "$HAPROXY_CONF")
+    cp "$HAPROXY_HEADER" "$NEXT_HAPROXY_CONF"
+
+    {{- range $namespace, $services := $envAll.Values.conf.anchor.services }}
+    {{- range $service, $svc_data := $services }}
+    echo Constructing config for namespace=\"{{ $namespace }}\" service=\"{{ $service }}\"
+
+    # NOTE(mark-burnett): Don't accidentally log service account token.
+    set +x
+    SERVICE_IPS=$(kubectl \
+        --server "$KUBE_URL" \
+        --certificate-authority "$KUBE_CA" \
+        --token $(cat "$KUBE_TOKEN") \
+        --namespace {{ $namespace }} \
+            get endpoints {{ $service }} \
+                -o 'jsonpath={.subsets[0].addresses[*].ip}')
+    DEST_PORT=$(kubectl \
+        --server "$KUBE_URL" \
+        --certificate-authority "$KUBE_CA" \
+        --token $(cat "$KUBE_TOKEN") \
+        --namespace {{ $namespace }} \
+            get endpoints {{ $service }} \
+                -o 'jsonpath={.subsets[0].ports[0].port}')
+    set -x
+
+    if [ "x$SERVICE_IPS" != "x" ]; then
+        if [ "x$DEST_PORT" != "x" ]; then
+            IDENTIFIER=$(echo "{{ $namespace }}-{{ $service }}")
+            # Add frontend config
+            echo >> "$NEXT_HAPROXY_CONF"
+            echo "frontend ${IDENTIFIER}-fe" >> "$NEXT_HAPROXY_CONF"
+            {{- range $envAll.Values.conf.haproxy.conf_parts.frontend }}
+            echo "  {{ . }}" >> "$NEXT_HAPROXY_CONF"
+            {{- end }}
+            {{- range $svc_data.conf_parts.frontend }}
+            echo "  {{ . }}" >> "$NEXT_HAPROXY_CONF"
+            {{- end }}
+            echo "  default_backend ${IDENTIFIER}-be" >> "$NEXT_HAPROXY_CONF"
+
+            # Add backend config
+            echo >> "$NEXT_HAPROXY_CONF"
+            echo "backend ${IDENTIFIER}-be" >> "$NEXT_HAPROXY_CONF"
+            {{- range $envAll.Values.conf.haproxy.conf_parts.backend }}
+            echo "  {{ . }}" >> "$NEXT_HAPROXY_CONF"
+            {{- end }}
+            {{- range $svc_data.conf_parts.backend }}
+            echo "  {{ . }}" >> "$NEXT_HAPROXY_CONF"
+            {{- end }}
+
+            for IP in $SERVICE_IPS; do
+                echo "  server s$IP $IP:$DEST_PORT" {{ $svc_data.server_opts | quote }} >> "$NEXT_HAPROXY_CONF"
+            done
+        else
+            echo Failed to get destination port for service.
+            SUCCESS=0
+        fi
+    else
+        echo Failed to get endpoint IPs for service.
+        SUCCESS=0
+    fi
+    {{- end }}
+    {{- end }}
+
+    if [ $SUCCESS = 1 ]; then
+        mkdir -p $(dirname "$HAPROXY_CONF")
+        if ! cmp -s "$HAPROXY_CONF" "$NEXT_HAPROXY_CONF"; then
+            echo Replacing HAProxy config file "$HAPROXY_CONF" with:
+            cat "$NEXT_HAPROXY_CONF"
+            echo
+            mv "$NEXT_HAPROXY_CONF" "$HAPROXY_CONF"
+        else
+            echo HAProxy config file unchanged.
+        fi
+    fi
+}
+
+cleanup() {
+    {{- range .Values.conf.anchor.files_to_copy }}
+    rm -f /host{{ .dest }}
+    {{- end }}
+    rm -f "$HAPROXY_CONF" "$NEXT_HAPROXY_CONF"
+}
+
+while true; do
+    if [ -e /tmp/stop ]; then
+        echo Stopping
+        cleanup
+        break
+    fi
+
+    install_config
+
+    compare_copy_files
+
+    sleep {{ .Values.conf.anchor.period }}
+done

charts/haproxy/templates/bin/_pre_stop.tpl

@@ -0,0 +1,21 @@
+#!/bin/sh
+{{/*
+Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+set -x
+
+touch /tmp/stop
+sleep {{ .Values.conf.anchor.period }}

charts/haproxy/templates/configmap-bin.yaml

@@ -0,0 +1,25 @@
+{{/*
+Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: haproxy-bin
+data:
+  anchor.sh: |
+{{ tuple "bin/_anchor.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  pre_stop.sh: |
+{{ tuple "bin/_pre_stop.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}

charts/haproxy/templates/configmap-etc.yaml

@@ -0,0 +1,35 @@
+{{/*
+Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: haproxy-etc
+data:
+  haproxy.cfg.header: |
+    global
+    {{- range .Values.conf.haproxy.conf_parts.global }}
+      {{ . }}
+    {{- end }}
+
+    stats socket /tmp/haproxy.sock mode 700 level admin expose-fd listeners
+
+    defaults
+    {{- range .Values.conf.haproxy.conf_parts.defaults }}
+      {{ . }}
+    {{- end }}
+  haproxy.yaml: |
+{{ tuple "etc/_haproxy.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}

charts/haproxy/templates/daemonset.yaml

@@ -0,0 +1,81 @@
+{{/*
+Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- $envAll := . }}
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: haproxy-anchor
+spec:
+{{ tuple $envAll "haproxy_anchor" | include "helm-toolkit.snippets.kubernetes_upgrades_daemonset" | indent 2 }}
+  template:
+    metadata:
+      labels:
+{{ tuple $envAll "kubernetes" "haproxy-anchor" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+    spec:
+      tolerations:
+        - key: node-role.kubernetes.io/master
+          effect: NoSchedule
+        - key: CriticalAddonsOnly
+          operator: Exists
+      terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.haproxy_anchor.timeout }}
+      serviceAccountName: haproxy-anchor
+      containers:
+        - name: anchor
+          image: {{ .Values.images.tags.anchor }}
+          imagePullPolicy: {{ .Values.images.pull_policy }}
+          env:
+            - name: HAPROXY_HEADER
+              value: /tmp/etc/haproxy.cfg.header
+            - name: HAPROXY_CONF
+              value: /host{{ .Values.conf.haproxy.host_config_dir }}/haproxy.cfg
+            - name: NEXT_HAPROXY_CONF
+              value: "$(HAPROXY_CONF)-next"
+            - name: HAPROXY_SERVER_OPTS
+              value: {{ .Values.conf.haproxy.server_opts | quote }}
+            - name: KUBE_CA
+              value: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+            - name: KUBE_TOKEN
+              value: /var/run/secrets/kubernetes.io/serviceaccount/token
+            - name: KUBE_URL
+              value: {{ .Values.conf.anchor.kubernetes_url | quote }}
+          command:
+            - /tmp/bin/anchor.sh
+          lifecycle:
+            preStop:
+              exec:
+                command:
+                  - /tmp/bin/pre_stop.sh
+          volumeMounts:
+            - name: haproxy-bin
+              mountPath: /tmp/bin
+            - name: haproxy-etc
+              mountPath: /tmp/etc
+            - name: host
+              mountPath: /host
+      volumes:
+        - name: haproxy-bin
+          configMap:
+            name: haproxy-bin
+            defaultMode: 0555
+        - name: haproxy-etc
+          configMap:
+            name: haproxy-etc
+            defaultMode: 0444
+        - name: host
+          hostPath:
+            path: /

charts/haproxy/templates/etc/_haproxy.yaml.tpl

@@ -0,0 +1,88 @@
+{{/*
+Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: haproxy
+  namespace: {{ .Release.Namespace }}
+spec:
+  hostNetwork: true
+  containers:
+    - name: haproxy
+      image: {{ .Values.images.tags.haproxy }}
+      imagePullPolicy: {{ .Values.images.pull_policy }}
+      hostNetwork: true
+      env:
+        - name: HAPROXY_CONF
+          value: {{ .Values.conf.haproxy.container_config_dir }}/haproxy.cfg
+        - name: LIVE_HAPROXY_CONF
+          value: /tmp/live_haproxy.cfg
+      command:
+        - /bin/sh
+        - -c
+        - |
+            set -eux
+
+            while [ ! -s "$HAPROXY_CONF" ]; do
+                echo Waiting for "HAPROXY_CONF"
+                sleep 1
+            done
+            echo vvv Starting with initial config vvv
+            cat "$HAPROXY_CONF"
+            echo
+            cp "$HAPROXY_CONF" "$LIVE_HAPROXY_CONF"
+
+            # NOTE(mark-burnett): sleep for clearer log output
+            sleep 1
+
+            haproxy -D -f "$LIVE_HAPROXY_CONF" -p /tmp/haproxy.pid
+
+            echo HAProxy started, monitoring for config changes..
+
+            set +x
+            while true; do
+                if ! cmp -s "$HAPROXY_CONF" "$LIVE_HAPROXY_CONF"; then
+                    echo vvv Replacing old config vvv
+                    cat "$LIVE_HAPROXY_CONF"
+                    echo
+
+                    echo vvv With new config vvv
+                    cat "$HAPROXY_CONF"
+                    echo
+
+                    cat "$HAPROXY_CONF" > "$LIVE_HAPROXY_CONF"
+
+                    # NOTE(mark-burnett): sleep for clearer log output
+                    sleep 1
+
+                    set -x
+                    haproxy -D -f "$LIVE_HAPROXY_CONF" -p /tmp/haproxy.pid \
+                        -x /tmp/haproxy.sock \
+                        -sf $(cat /tmp/haproxy.pid)
+                    set +x
+                fi
+                sleep {{ .Values.conf.haproxy.period }}
+            done
+
+      volumeMounts:
+        - name: etc
+          mountPath: {{ .Values.conf.haproxy.container_config_dir }}
+          readOnly: True
+  volumes:
+    - name: etc
+      hostPath:
+        path: {{ .Values.conf.haproxy.host_config_dir }}

charts/haproxy/templates/rbac.yaml

@@ -0,0 +1,52 @@
+{{/*
+Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- $envAll := . }}
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: haproxy-anchor
+{{- range $namespace, $services := $envAll.Values.conf.anchor.services }}
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: haproxy-anchor
+  namespace: {{ $namespace }}
+rules:
+  - apiGroups: [""]
+    resources:
+      - endpoints
+    verbs:
+      - get
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: haproxy-anchor
+  namespace: {{ $namespace }}
+subjects:
+  - kind: ServiceAccount
+    name: haproxy-anchor
+    namespace: {{ $envAll.Release.Namespace }}
+    apiGroup: ""
+roleRef:
+  kind: Role
+  name: haproxy-anchor
+  apiGroup: rbac.authorization.k8s.io
+{{- end }}

charts/haproxy/values.yaml

@@ -0,0 +1,88 @@
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+conf:
+  anchor:
+    files_to_copy:
+      - source: /tmp/etc/haproxy.yaml
+        dest: /etc/kubernetes/manifests/haproxy.yaml
+    period: 30
+    kubernetes_url: https://kubernetes.default:443
+    services:
+      default:
+        kubernetes:
+          server_opts: "check"
+          conf_parts:
+            frontend:
+              - mode tcp
+              - option tcpka
+              - bind *:6553
+            backend:
+              - mode tcp
+              - option tcpka
+      kube-system:
+        kubernetes-etcd:
+          server_opts: "check"
+          conf_parts:
+            frontend:
+              - mode tcp
+              - option tcpka
+              - bind *:2378
+            backend:
+              - mode tcp
+              - option tcpka
+
+  haproxy:
+    container_config_dir: /usr/local/etc/haproxy
+    host_config_dir: /etc/promenade/haproxy
+    period: 5
+    conf_parts:
+      global:
+        - maxconn 10240
+      defaults:
+        - timeout connect 5000ms
+        - timeout client 24h
+        - timeout server 24h
+      frontend: []
+      backend: []
+
+images:
+  tags:
+    anchor: gcr.io/google_containers/hyperkube-amd64:v1.8.6
+    haproxy: haproxy:1.8.3
+  pull_policy: "IfNotPresent"
+
+pod:
+  lifecycle:
+    upgrades:
+      daemonsets:
+        pod_replacement_strategy: RollingUpdate
+        haproxy_anchor:
+          enabled: false
+          min_ready_seconds: 0
+          max_unavailable: 1
+    termination_grace_period:
+      haproxy_anchor:
+        timeout: 3600
+  resources:
+    enabled: false
+    haproxy_anchor:
+      requests:
+        memory: "128Mi"
+        cpu: "100m"
+      limits:
+        memory: "256Mi"
+        cpu: "2000m"
+
+release_group: null

charts/promenade/templates/deployment-api.yaml

@@ -46,11 +46,11 @@ spec:
           command:
             - /opt/promenade/entrypoint.sh
             - server
-          {{- if .Values.pod.env.promenade_api }}
+          {{- if $envAll.Values.pod.env.promenade_api }}
           env:
-            {{- range .Values.pod.env.promenade_api }}
+            {{- range $envAll.Values.pod.env.promenade_api }}
-            - name: {{ .name }}
+            - name: {{ .name | quote }}
-              value: {{ .value }}
+              value: {{ .value | quote }}
             {{- end }}
           {{- end }}
           ports:

docs/source/configuration/host-system.rst

@@ -20,7 +20,7 @@ Sample Document
           tar_path: kubernetes/node/bin/kubelet
           mode: 0555
       images:
-        coredns: coredns/coredns:011
+        haproxy: haproxy:1.8.3
         helm:
           helm: lachlanevenson/k8s-helm:v2.7.2
         kubernetes:
@@ -91,13 +91,15 @@ Core Images
 
 These images are used for essential functionality:
 
-``coredns``
+``haproxy``
-    coredns_ is configured and used for Kubernetes API discovery during
+    HAProxy_ is configured and used for Kubernetes API discovery during
     bootstrapping.
 
 ``kubectl``
     Used for label application and validation tasks during bootstrapping.
 
+.. _HAProxy: https://www.haproxy.org/
+
 
 Convenience Images
 ^^^^^^^^^^^^^^^^^^
@@ -105,9 +107,6 @@ Convenience Images
 The ``helm`` image is available for convenience.
 
 
-.. _coredns: https://github.com/coredns/coredns
-
-
 Packages
 --------
 

docs/source/configuration/kubernetes-network.rst

@@ -30,12 +30,15 @@ Sample Document
           - 8.8.4.4
 
       kubernetes:
+        apiserver_port: 6443
+        haproxy_port: 6553
         pod_cidr: 10.97.0.0/16
         service_cidr: 10.96.0.0/16
         service_ip: 10.96.0.1
 
       etcd:
-        service_ip: 10.96.0.2
+        container_port: 2379
+        haproxy_port: 2378
 
       hosts_entries:
         - ip: 192.168.77.1
@@ -72,6 +75,13 @@ Kubernetes
 
 The ``kubernetes`` key contains:
 
+``apiserver_port``
+    The port that the Kubernetes API server process will listen on on hosts where it runs.
+
+``haproxy_port``
+    The port that HAProxy will listen on on each host.  This port will be used
+    by the ``kubelet`` and ``kube-proxy`` to find API servers in the cluster.
+
 ``pod_cidr``
     The CIDR from which the Kubernetes Controller Manager assigns pod IPs.
 

entrypoint.sh

@@ -4,14 +4,25 @@ set -ex
 PORT=${PORT:-9000}
 UWSGI_TIMEOUT=${UWSGI_TIMEOUT:-300}
 
+PROMENADE_THREADS=${PROMENADE_THREADS:-1}
+PROMENADE_WORKERS=${PROMENADE_WORKERS:-4}
+
 if [ "$1" = 'server' ]; then
     exec uwsgi \
-        --http :${PORT} \
+        --http ":${PORT}" \
-        --http-timeout ${UWSGI_TIMEOUT} \
+        --http-timeout "${UWSGI_TIMEOUT}" \
-        -z ${UWSGI_TIMEOUT} \
+        --harakiri "${UWSGI_TIMEOUT}" \
+        --socket-timeout "${UWSGI_TIMEOUT}" \
+        --harakiri-verbose \
+        --lazy-apps \
+        --master \
+        --thunder-lock \
+        --die-on-term \
+        -z "${UWSGI_TIMEOUT}" \
         --paste config:/etc/promenade/api-paste.ini \
-        --enable-threads -L \
+        --enable-threads \
-        --workers 4
+        --threads "${PROMENADE_THREADS}" \
+        --workers "${PROMENADE_WORKERS}"
 fi
 
 exec ${@}

examples/basic/HostSystem.yaml

@@ -30,7 +30,7 @@ data:
             rotate 1
         }
   images:
-    coredns: coredns/coredns:0.9.9
+    haproxy: haproxy:1.8.3
     helm:
       helm: lachlanevenson/k8s-helm:v2.7.2
     kubernetes:

examples/basic/KubernetesNetwork.yaml

@@ -21,12 +21,15 @@ data:
       - 8.8.4.4
 
   kubernetes:
+    apiserver_port: 6443
+    haproxy_port: 6553
     pod_cidr: 10.97.0.0/16
     service_cidr: 10.96.0.0/16
     service_ip: 10.96.0.1
 
   etcd:
-    service_ip: 10.96.0.2
+    container_port: 2379
+    haproxy_port: 2378
 
   hosts_entries:
     - ip: 192.168.77.1

examples/basic/armada-resources.yaml

@@ -69,6 +69,7 @@ metadata:
 data:
   description: Kubernetes components
   chart_group:
+    - haproxy
     - kubernetes-etcd
     - kubernetes-apiserver
     - kubernetes-controller-manager
@@ -119,6 +120,7 @@ metadata:
   layeringDefinition:
     abstract: false
     layer: site
+  storagePolicy: cleartext
   substitutions:
     -
       src:
@@ -160,7 +162,7 @@ data:
       tags:
         proxy: gcr.io/google_containers/hyperkube-amd64:v1.8.6
     network:
-      kubernetes_netloc: apiserver.kubernetes.promenade:6443
+      kubernetes_netloc: 127.0.0.1:6553
   source:
     type: local
     location: /etc/genesis/armada/assets/charts
@@ -175,6 +177,7 @@ metadata:
   layeringDefinition:
     abstract: false
     layer: site
+  storagePolicy: cleartext
   substitutions:
     -
       src:
@@ -419,6 +422,7 @@ metadata:
   layeringDefinition:
     abstract: false
     layer: site
+  storagePolicy: cleartext
   substitutions:
     -
       src:
@@ -482,28 +486,6 @@ metadata:
   layeringDefinition:
     abstract: false
     layer: site
-  substitutions:
-    -
-      src:
-        schema: deckhand/CertificateAuthority/v1
-        name: kubernetes
-        path: .
-      dest:
-        path: '.values.tls.ca'
-    -
-      src:
-        schema: deckhand/Certificate/v1
-        name: coredns
-        path: .
-      dest:
-        path: '.values.tls.cert'
-    -
-      src:
-        schema: deckhand/CertificateKey/v1
-        name: coredns
-        path: .
-      dest:
-        path: '.values.tls.key'
 data:
   chart_name: coredns
   release: coredns
@@ -514,39 +496,9 @@ data:
   upgrade:
     no_hooks: true
   values:
-    coredns:
-      kubernetes_zones:
-        - cluster.local
-        - 10.96.0.0/16
-        - 10.97.0.0/16
-      upstream_nameservers:
-        - 8.8.8.8
-        - 8.8.4.4
-
-      zones:
-        - name: promenade
-          services:
-            - bind_name: apiserver.kubernetes
-              service:
-                name: kubernetes-apiserver
-                namespace: kube-system
-            - bind_name: etcd.kubernetes
-              service:
-                name: kubernetes-etcd
-                namespace: kube-system
-            - bind_name: etcd.calico
-              service:
-                name: calico-etcd
-                namespace: kube-system
     images:
-      anchor: gcr.io/google_containers/hyperkube-amd64:v1.8.6
+      tags:
-      coredns: coredns/coredns:0.9.9
+        coredns: coredns/coredns:1.0.5
-    tls:
-      ca: placeholder
-      cert: placeholder
-      key: placeholder
-    network:
-      kubernetes_netloc: apiserver.kubernetes.promenade:6443
   source:
     type: local
     location: /etc/genesis/armada/assets/charts
@@ -555,6 +507,62 @@ data:
     - helm-toolkit
 ---
 schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: haproxy
+  layeringDefinition:
+    abstract: false
+    layer: site
+data:
+  chart_name: haproxy
+  release: haproxy
+  namespace: kube-system
+  timeout: 600
+  wait:
+    timeout: 600
+  upgrade:
+    no_hooks: true
+  values:
+    conf:
+      anchor:
+        kubernetes_url: https://kubernetes.default:443
+        services:
+          default:
+            kubernetes:
+              server_opts: "check"
+              conf_parts:
+                frontend:
+                  - mode tcp
+                  - option tcpka
+                  - bind *:6553
+                backend:
+                  - mode tcp
+                  - option tcpka
+          kube-system:
+            kubernetes-etcd:
+              server_opts: "check"
+              conf_parts:
+                frontend:
+                  - mode tcp
+                  - option tcpka
+                  - bind *:2378
+                backend:
+                  - mode tcp
+                  - option tcpka
+
+    images:
+      tags:
+        anchor: gcr.io/google_containers/hyperkube-amd64:v1.8.6
+        haproxy: haproxy:1.8.3
+
+  source:
+    type: local
+    location: /etc/genesis/armada/assets/charts
+    subpath: haproxy
+  dependencies:
+    - helm-toolkit
+---
+schema: armada/Chart/v1
 metadata:
   schema: metadata/Document/v1
   name: kubernetes-apiserver
@@ -626,7 +634,7 @@ data:
   values:
     apiserver:
       etcd:
-        endpoints: https://etcd.kubernetes.promenade:2379
+        endpoints: https://127.0.0.1:2378
     images:
       tags:
         anchor: gcr.io/google_containers/hyperkube-amd64:v1.8.6
@@ -716,7 +724,7 @@ data:
         cert: placeholder
         key: placeholder
     network:
-      kubernetes_netloc: apiserver.kubernetes.promenade:6443
+      kubernetes_netloc: 127.0.0.1:6553
       pod_cidr: 10.97.0.0/16
       service_cidr: 10.96.0.0/16
 
@@ -775,7 +783,7 @@ data:
         key: placeholder
 
     network:
-      kubernetes_netloc: apiserver.kubernetes.promenade:6443
+      kubernetes_netloc: 127.0.0.1:6553
 
     images:
       tags:
@@ -955,7 +963,7 @@ data:
     no_hooks: true
   values:
     anchor:
-      etcdctl_endpoint: 10.96.0.2
+      etcdctl_endpoint: kubernetes-etcd.kube-system.svc.cluster.local
     labels:
       anchor:
         node_selector_key: kubernetes-etcd
@@ -1012,7 +1020,6 @@ data:
             key: placeholder
     service:
       name: kubernetes-etcd
-      ip: 10.96.0.2
     network:
       service_client:
         name: service_client
@@ -1045,6 +1052,11 @@ data:
   wait:
     timeout: 600
   values:
+    pod:
+      env:
+        promenade_api:
+          - name: PROMENADE_DEBUG
+            value: '1'
     conf:
       paste:
         app:promenade-api:

examples/complete/HostSystem.yaml

@@ -31,7 +31,7 @@ data:
             rotate 1
         }
   images:
-    coredns: coredns/coredns:0.9.9
+    haproxy: haproxy:1.8.3
     helm:
       helm: lachlanevenson/k8s-helm:v2.7.2
     kubernetes:

examples/complete/KubernetesNetwork.yaml

@@ -20,12 +20,15 @@ data:
       - 8.8.4.4
 
   kubernetes:
+    apiserver_port: 6443
+    haproxy_port: 6553
     pod_cidr: 10.97.0.0/16
     service_cidr: 10.96.0.0/16
     service_ip: 10.96.0.1
 
   etcd:
-    service_ip: 10.96.0.2
+    container_port: 2379
+    haproxy_port: 2378
 
   hosts_entries:
     - ip: 192.168.77.1

examples/complete/armada-resources.yaml

@@ -86,6 +86,7 @@ metadata:
 data:
   description: Kubernetes components
   chart_group:
+    - haproxy
     - kubernetes-etcd
     - kubernetes-apiserver
     - kubernetes-controller-manager
@@ -204,7 +205,7 @@ data:
       tags:
         proxy: gcr.io/google_containers/hyperkube-amd64:v1.8.6
     network:
-      kubernetes_netloc: apiserver.kubernetes.promenade:6443
+      kubernetes_netloc: 127.0.0.1:6553
   source:
     type: local
     location: /etc/genesis/armada/assets/charts
@@ -528,29 +529,6 @@ metadata:
   layeringDefinition:
     abstract: false
     layer: site
-  storagePolicy: cleartext
-  substitutions:
-    -
-      src:
-        schema: deckhand/CertificateAuthority/v1
-        name: kubernetes
-        path: .
-      dest:
-        path: '.values.tls.ca'
-    -
-      src:
-        schema: deckhand/Certificate/v1
-        name: coredns
-        path: .
-      dest:
-        path: '.values.tls.cert'
-    -
-      src:
-        schema: deckhand/CertificateKey/v1
-        name: coredns
-        path: .
-      dest:
-        path: '.values.tls.key'
 data:
   chart_name: coredns
   release: coredns
@@ -561,39 +539,9 @@ data:
   upgrade:
     no_hooks: true
   values:
-    coredns:
-      kubernetes_zones:
-        - cluster.local
-        - 10.96.0.0/16
-        - 10.97.0.0/16
-      upstream_nameservers:
-        - 8.8.8.8
-        - 8.8.4.4
-
-      zones:
-        - name: promenade
-          services:
-            - bind_name: apiserver.kubernetes
-              service:
-                name: kubernetes-apiserver
-                namespace: kube-system
-            - bind_name: etcd.kubernetes
-              service:
-                name: kubernetes-etcd
-                namespace: kube-system
-            - bind_name: etcd.calico
-              service:
-                name: calico-etcd
-                namespace: kube-system
     images:
-      anchor: gcr.io/google_containers/hyperkube-amd64:v1.8.6
+      tags:
-      coredns: coredns/coredns:0.9.9
+        coredns: coredns/coredns:1.0.5
-    tls:
-      ca: placeholder
-      cert: placeholder
-      key: placeholder
-    network:
-      kubernetes_netloc: apiserver.kubernetes.promenade:6443
   source:
     type: local
     location: /etc/genesis/armada/assets/charts
@@ -602,6 +550,62 @@ data:
     - helm-toolkit
 ---
 schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: haproxy
+  layeringDefinition:
+    abstract: false
+    layer: site
+data:
+  chart_name: haproxy
+  release: haproxy
+  namespace: kube-system
+  timeout: 600
+  wait:
+    timeout: 600
+  upgrade:
+    no_hooks: true
+  values:
+    conf:
+      anchor:
+        kubernetes_url: https://kubernetes.default:443
+        services:
+          default:
+            kubernetes:
+              server_opts: "check"
+              conf_parts:
+                frontend:
+                  - mode tcp
+                  - option tcpka
+                  - bind *:6553
+                backend:
+                  - mode tcp
+                  - option tcpka
+          kube-system:
+            kubernetes-etcd:
+              server_opts: "check"
+              conf_parts:
+                frontend:
+                  - mode tcp
+                  - option tcpka
+                  - bind *:2378
+                backend:
+                  - mode tcp
+                  - option tcpka
+
+    images:
+      tags:
+        anchor: gcr.io/google_containers/hyperkube-amd64:v1.8.6
+        haproxy: haproxy:1.8.3
+
+  source:
+    type: local
+    location: /etc/genesis/armada/assets/charts
+    subpath: haproxy
+  dependencies:
+    - helm-toolkit
+---
+schema: armada/Chart/v1
 metadata:
   schema: metadata/Document/v1
   name: kubernetes-apiserver
@@ -673,7 +677,7 @@ data:
   values:
     apiserver:
       etcd:
-        endpoints: https://etcd.kubernetes.promenade:2379
+        endpoints: https://127.0.0.1:2378
     images:
       tags:
         anchor: gcr.io/google_containers/hyperkube-amd64:v1.8.6
@@ -763,7 +767,7 @@ data:
         cert: placeholder
         key: placeholder
     network:
-      kubernetes_netloc: apiserver.kubernetes.promenade:6443
+      kubernetes_netloc: 127.0.0.1:6553
       pod_cidr: 10.97.0.0/16
       service_cidr: 10.96.0.0/16
 
@@ -822,7 +826,7 @@ data:
         key: placeholder
 
     network:
-      kubernetes_netloc: apiserver.kubernetes.promenade:6443
+      kubernetes_netloc: 127.0.0.1:6553
 
     images:
       tags:
@@ -1002,7 +1006,7 @@ data:
     no_hooks: true
   values:
     anchor:
-      etcdctl_endpoint: 10.96.0.2
+      etcdctl_endpoint: kubernetes-etcd.kube-system.svc.cluster.local
     labels:
       anchor:
         node_selector_key: kubernetes-etcd
@@ -1059,7 +1063,6 @@ data:
             key: placeholder
     service:
       name: kubernetes-etcd
-      ip: 10.96.0.2
     network:
       service_client:
         name: service_client
@@ -1756,6 +1759,11 @@ data:
   wait:
     timeout: 600
   values:
+    pod:
+      env:
+        promenade_api:
+          - name: PROMENADE_DEBUG
+            value: '1'
     images:
       tags:
         dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.2.1

promenade/generator.py

@@ -33,7 +33,7 @@ class Generator:
             ca='kubernetes',
             cn='apiserver',
             hosts=self._service_dns('kubernetes', 'default') +
-            ['localhost', '127.0.0.1', 'apiserver.kubernetes.promenade'] +
+            ['localhost', '127.0.0.1'] +
             [self.config['KubernetesNetwork:kubernetes.service_ip']])
         self.gen(
             'certificate',
@@ -75,25 +75,19 @@ class Generator:
             cn='armada',
             groups=['system:masters'])
 
-        # Certificates for coredns
-        self.gen('certificate', 'coredns', ca='kubernetes', cn='coredns')
-
         # Certificates for Kubernetes's etcd servers
         self.gen_etcd_certificates(
             ca='kubernetes-etcd',
             genesis=True,
             service_name='kubernetes-etcd',
-            service_namespace='kube-system',
+            service_namespace='kube-system')
-            service_ip=self.config['KubernetesNetwork:etcd.service_ip'],
-            additional_hosts=['etcd.kubernetes.promenade'])
 
         # Certificates for Calico's etcd servers
         self.gen_etcd_certificates(
             ca='calico-etcd',
             service_name='calico-etcd',
             service_namespace='kube-system',
-            service_ip=self.calico_etcd_service_ip,
+            service_ip=self.calico_etcd_service_ip)
-            additional_hosts=['etcd.calico.promenade'])
 
         # Certificates for Calico node
         self.gen(

promenade/schemas/HostSystem.yaml

@@ -71,8 +71,11 @@ data:
     images:
       type: object
       properties:
+        # NOTE(mark-burnett): No longer used.
         coredns:
           $ref: '#/definitions/image'
+        haproxy:
+          $ref: '#/definitions/image'
         helm:
           type: object
           properties:
@@ -90,7 +93,7 @@ data:
             - kubectl
           additionalProperties: false
       required:
-        - coredns
+        - haproxy
         - helm
         - kubernetes
       additionalProperties: false

promenade/schemas/KubernetesNetwork.yaml

@@ -52,10 +52,16 @@ data:
     etcd:
       type: object
       properties:
+        container_port:
+          type: integer
+        haproxy_port:
+          type: integer
+        # NOTE(mark-burnett): No longer used.
         service_ip:
           $ref: '#/definitions/ip_address'
       required:
-        - service_ip
+        - container_port
+        - haproxy_port
       additionalProperties: false
 
     kubernetes:
@@ -67,10 +73,16 @@ data:
           $ref: '#/definitions/ip_address'
         service_cidr:
           $ref: '#/definitions/cidr'
+        apiserver_port:
+          type: integer
+        haproxy_port:
+          type: integer
       required:
         - pod_cidr
         - service_cidr
         - service_ip
+        - apiserver_port
+        - haproxy_port
       additionalProperties: false
     hosts_entries:
       type: array

promenade/templates/roles/common/etc/coredns/Corefile

@@ -1,13 +0,0 @@
-promenade {
-    file /etc/coredns/zones/promenade
-    errors stdout
-    log stdout
-}
-
-. {
-    {%- if config['KubernetesNetwork:dns.upstream_servers'] is defined %}
-    proxy .  {%- for server in config['KubernetesNetwork:dns.upstream_servers'] %} {{ server -}}{%- endfor %}
-    {%- endif %}
-    errors stdout
-    log stdout
-}

promenade/templates/roles/common/etc/coredns/zones/promenade

@@ -1,11 +0,0 @@
-$ORIGIN promenade.
-
-promenade. IN SOA @ root {{ now }} 3h 15m 1w 1d
-
-{%- if config['KubernetesNode:join_ip'] is defined %}
-apiserver.kubernetes IN A {{ config['KubernetesNode:join_ip'] }}
-{%- else %}
-apiserver.kubernetes IN A 127.0.0.1
-{%- endif %}
-
-etcd.kubernetes IN A 127.0.0.1

promenade/templates/roles/common/etc/kubernetes/admin/kubeconfig.yaml

@@ -2,7 +2,7 @@
 apiVersion: v1
 clusters:
 - cluster:
-    server: https://apiserver.kubernetes.promenade:6443
+    server: https://127.0.0.1:6553
     certificate-authority: pki/cluster-ca.pem
   name: kubernetes
 contexts:

promenade/templates/roles/common/etc/kubernetes/kubeconfig

@@ -2,7 +2,7 @@
 apiVersion: v1
 clusters:
 - cluster:
-    server: https://apiserver.kubernetes.promenade:6443
+    server: https://127.0.0.1:6553
     certificate-authority: pki/cluster-ca.pem
   name: kubernetes
 contexts:

promenade/templates/roles/common/etc/kubernetes/manifests/coredns.yaml

@@ -1,30 +0,0 @@
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: coredns
-  namespace: kube-system
-  labels:
-    # NOTE(mark-burnett): This is a host initialization Pod and should not be
-    # included in the Kubernetes DNS service, so ensure it does not get
-    # selected.
-    coredns-service: disabled
-  annotations:
-    scheduler.alpha.kubernetes.io/critical-pod: ''
-spec:
-  hostNetwork: true
-  containers:
-    - name: coredns
-      image: {{ config['HostSystem:images.coredns'] }}
-      command:
-        - /coredns
-        - -conf
-        - /etc/coredns/Corefile
-      volumeMounts:
-        - name: config
-          mountPath: /etc/coredns
-          readOnly: true
-  volumes:
-    - name: config
-      hostPath:
-        path: /etc/coredns

promenade/templates/roles/common/etc/kubernetes/manifests/haproxy.yaml

@@ -0,0 +1,39 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: haproxy
+  namespace: kube-system
+  annotations:
+    scheduler.alpha.kubernetes.io/critical-pod: ''
+spec:
+  hostNetwork: true
+  containers:
+    - name: haproxy
+      image: {{ config['HostSystem:images.haproxy'] }}
+      imagePullPolicy: IfNotPresent
+      hostNetwork: true
+      env:
+        - name: HAPROXY_CONF
+          value: /usr/local/etc/haproxy/haproxy.cfg
+      command:
+        - /bin/sh
+        - -c
+        - |
+            set -eux
+
+            while [ ! -s "$HAPROXY_CONF" ]; do
+                echo Waiting for "HAPROXY_CONF"
+                sleep 1
+            done
+
+            haproxy -f "$HAPROXY_CONF"
+
+      volumeMounts:
+        - name: etc
+          mountPath: /usr/local/etc/haproxy
+          readOnly: true
+  volumes:
+    - name: etc
+      hostPath:
+        path: /etc/promenade/haproxy

promenade/templates/roles/common/etc/promenade/haproxy/haproxy.cfg

@@ -0,0 +1,43 @@
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+global
+  maxconn 10240
+
+defaults
+  mode tcp
+  option tcpka
+  timeout connect 5000ms
+  timeout client 24h
+  timeout server 24h
+
+frontend default-kubernetes-fe
+  bind *:{{ config['KubernetesNetwork:kubernetes.haproxy_port'] }}
+  default_backend default-kubernetes-be
+
+{% set ip = config.get_first('KubernetesNode:join_ip', 'Genesis:ip') -%}
+
+backend default-kubernetes-be
+  option tcp-check
+  {%- set port = config['KubernetesNetwork:kubernetes.apiserver_port'] %}
+  server s{{ ip }} {{ ip }}:{{ port }} check port {{ port }}
+
+frontend kube-system-kubernetes-etcd-fe
+  bind *:{{ config['KubernetesNetwork:etcd.haproxy_port'] }}
+  default_backend kube-system-kubernetes-etcd-be
+
+backend kube-system-kubernetes-etcd-be
+  option tcp-check
+  {%- set port = config['KubernetesNetwork:etcd.container_port'] %}
+  server s{{ ip }} {{ ip }}:{{ port }} check port {{ port }}

promenade/templates/roles/common/etc/resolv.conf

@@ -1,6 +1,6 @@
 options timeout:1 attempts:1
 
-nameserver 127.0.0.1
+nameserver 10.96.0.10
 {% for server in config['KubernetesNetwork:dns.upstream_servers'] | default([]) %}
 nameserver {{ server }}
 {%- endfor %}

promenade/templates/roles/genesis/etc/genesis/controller-manager/kubeconfig.yaml

@@ -2,7 +2,7 @@
 apiVersion: v1
 clusters:
 - cluster:
-    server: https://apiserver.kubernetes.promenade:6443
+    server: https://127.0.0.1:6553
     certificate-authority: pki/cluster-ca.pem
   name: kubernetes
 contexts:

promenade/templates/roles/genesis/etc/genesis/scheduler/kubeconfig.yaml

@@ -2,7 +2,7 @@
 apiVersion: v1
 clusters:
 - cluster:
-    server: https://apiserver.kubernetes.promenade:6443
+    server: https://127.0.0.1:6553
     certificate-authority: pki/cluster-ca.pem
   name: kubernetes
 contexts:

tools/g2/lib/registry.sh

@@ -7,7 +7,7 @@ registry_down() {
 }
 
 registry_list_images() {
-    FILES=($(echo "$(config_configuration)" | xargs -n 1 -I DIRNAME find DIRNAME -type f -name '*.yaml'))
+    FILES=($(config_configuration | xargs -n 1 -I DIRNAME find DIRNAME -type f -name '*.yaml'))
 
     HOSTNAME_REGEX='[a-zA-Z0-9][a-zA-Z0-9_-]{0,62}'
     DOMAIN_NAME_REGEX="${HOSTNAME_REGEX}(\.${HOSTNAME_REGEX})*"

tools/g2/lib/validate.sh

@@ -12,6 +12,8 @@ validate_etcd_membership() {
     shift 2
     EXPECTED_MEMBERS="${*}"
 
+    # NOTE(mark-burnett): Wait a moment for disks in test environment to settle.
+    sleep 10
     log Validating "${CLUSTER}" etcd membership via "${VM}"
     FOUND_MEMBERS=$(etcdctl_member_list "${CLUSTER}" "${VM}" | tr '\n' ' ' | sed 's/ $//')
 

tools/g2/lib/virsh.sh

@@ -129,7 +129,7 @@ vm_create() {
         --cpu host \
         --graphics vnc,listen=0.0.0.0 \
         --noautoconsole \
-        --network network=promenade \
+        --network "network=promenade,model=virtio" \
         --vcpus "$(config_vm_vcpus)" \
         --memory "$(config_vm_memory)" \
         --import \

tools/gate/config-templates/bootstrap-armada-config.yaml

@@ -82,6 +82,7 @@ metadata:
 data:
   description: Kubernetes components
   chart_group:
+    - haproxy
     - kubernetes-etcd
     - kubernetes-apiserver
     - kubernetes-controller-manager
@@ -123,23 +124,23 @@ metadata:
       src:
         schema: deckhand/CertificateAuthority/v1
         name: kubernetes
-        path: $
+        path: .
       dest:
-        path: '$.values.secrets.tls.ca'
+        path: '.values.secrets.tls.ca'
     -
       src:
         schema: deckhand/Certificate/v1
         name: proxy
-        path: $
+        path: .
       dest:
-        path: '$.values.secrets.tls.cert'
+        path: '.values.secrets.tls.cert'
     -
       src:
         schema: deckhand/CertificateKey/v1
         name: proxy
-        path: $
+        path: .
       dest:
-        path: '$.values.secrets.tls.key'
+        path: '.values.secrets.tls.key'
 data:
   chart_name: proxy
   release: kubernetes-proxy
@@ -157,7 +158,7 @@ data:
       tags:
         proxy: ${IMAGE_HYPERKUBE}
     network:
-      kubernetes_netloc: apiserver.kubernetes.promenade:6443
+      kubernetes_netloc: 127.0.0.1:6553
   source:
     type: local
     location: /etc/genesis/armada/assets/charts
@@ -178,147 +179,147 @@ metadata:
       src:
         schema: deckhand/CertificateAuthority/v1
         name: calico-etcd
-        path: $
+        path: .
       dest:
-        path: '$.values.secrets.tls.client.ca'
+        path: '.values.secrets.tls.client.ca'
     -
       src:
         schema: deckhand/CertificateAuthority/v1
         name: calico-etcd-peer
-        path: $
+        path: .
       dest:
-        path: '$.values.secrets.tls.peer.ca'
+        path: '.values.secrets.tls.peer.ca'
 
     -
       src:
         schema: deckhand/Certificate/v1
         name: calico-etcd-anchor
-        path: $
+        path: .
       dest:
-        path: '$.values.secrets.anchor.tls.cert'
+        path: '.values.secrets.anchor.tls.cert'
     -
       src:
         schema: deckhand/CertificateKey/v1
         name: calico-etcd-anchor
-        path: $
+        path: .
       dest:
-        path: '$.values.secrets.anchor.tls.key'
+        path: '.values.secrets.anchor.tls.key'
 
     -
       src:
         schema: deckhand/Certificate/v1
         name: calico-etcd-${GENESIS_HOSTNAME}
-        path: $
+        path: .
       dest:
-        path: '$.values.nodes[0].tls.client.cert'
+        path: '.values.nodes[0].tls.client.cert'
     -
       src:
         schema: deckhand/CertificateKey/v1
         name: calico-etcd-${GENESIS_HOSTNAME}
-        path: $
+        path: .
       dest:
-        path: '$.values.nodes[0].tls.client.key'
+        path: '.values.nodes[0].tls.client.key'
     -
       src:
         schema: deckhand/Certificate/v1
         name: calico-etcd-${GENESIS_HOSTNAME}-peer
-        path: $
+        path: .
       dest:
-        path: '$.values.nodes[0].tls.peer.cert'
+        path: '.values.nodes[0].tls.peer.cert'
     -
       src:
         schema: deckhand/CertificateKey/v1
         name: calico-etcd-${GENESIS_HOSTNAME}-peer
-        path: $
+        path: .
       dest:
-        path: '$.values.nodes[0].tls.peer.key'
+        path: '.values.nodes[0].tls.peer.key'
 
     -
       src:
         schema: deckhand/Certificate/v1
         name: calico-etcd-${MASTER1_HOSTNAME}
-        path: $
+        path: .
       dest:
-        path: '$.values.nodes[1].tls.client.cert'
+        path: '.values.nodes[1].tls.client.cert'
     -
       src:
         schema: deckhand/CertificateKey/v1
         name: calico-etcd-${MASTER1_HOSTNAME}
-        path: $
+        path: .
       dest:
-        path: '$.values.nodes[1].tls.client.key'
+        path: '.values.nodes[1].tls.client.key'
     -
       src:
         schema: deckhand/Certificate/v1
         name: calico-etcd-${MASTER1_HOSTNAME}-peer
-        path: $
+        path: .
       dest:
-        path: '$.values.nodes[1].tls.peer.cert'
+        path: '.values.nodes[1].tls.peer.cert'
     -
       src:
         schema: deckhand/CertificateKey/v1
         name: calico-etcd-${MASTER1_HOSTNAME}-peer
-        path: $
+        path: .
       dest:
-        path: '$.values.nodes[1].tls.peer.key'
+        path: '.values.nodes[1].tls.peer.key'
 
     -
       src:
         schema: deckhand/Certificate/v1
         name: calico-etcd-${MASTER2_HOSTNAME}
-        path: $
+        path: .
       dest:
-        path: '$.values.nodes[2].tls.client.cert'
+        path: '.values.nodes[2].tls.client.cert'
     -
       src:
         schema: deckhand/CertificateKey/v1
         name: calico-etcd-${MASTER2_HOSTNAME}
-        path: $
+        path: .
       dest:
-        path: '$.values.nodes[2].tls.client.key'
+        path: '.values.nodes[2].tls.client.key'
     -
       src:
         schema: deckhand/Certificate/v1
         name: calico-etcd-${MASTER2_HOSTNAME}-peer
-        path: $
+        path: .
       dest:
-        path: '$.values.nodes[2].tls.peer.cert'
+        path: '.values.nodes[2].tls.peer.cert'
     -
       src:
         schema: deckhand/CertificateKey/v1
         name: calico-etcd-${MASTER2_HOSTNAME}-peer
-        path: $
+        path: .
       dest:
-        path: '$.values.nodes[2].tls.peer.key'
+        path: '.values.nodes[2].tls.peer.key'
 
     -
       src:
         schema: deckhand/Certificate/v1
         name: calico-etcd-${WORKER_HOSTNAME}
-        path: $
+        path: .
       dest:
-        path: '$.values.nodes[3].tls.client.cert'
+        path: '.values.nodes[3].tls.client.cert'
     -
       src:
         schema: deckhand/CertificateKey/v1
         name: calico-etcd-${WORKER_HOSTNAME}
-        path: $
+        path: .
       dest:
-        path: '$.values.nodes[3].tls.client.key'
+        path: '.values.nodes[3].tls.client.key'
     -
       src:
         schema: deckhand/Certificate/v1
         name: calico-etcd-${WORKER_HOSTNAME}-peer
-        path: $
+        path: .
       dest:
-        path: '$.values.nodes[3].tls.peer.cert'
+        path: '.values.nodes[3].tls.peer.cert'
     -
       src:
         schema: deckhand/CertificateKey/v1
         name: calico-etcd-${WORKER_HOSTNAME}-peer
-        path: $
+        path: .
       dest:
-        path: '$.values.nodes[3].tls.peer.key'
+        path: '.values.nodes[3].tls.peer.key'
 
 data:
   chart_name: etcd
@@ -421,23 +422,23 @@ metadata:
       src:
         schema: deckhand/CertificateAuthority/v1
         name: calico-etcd
-        path: $
+        path: .
       dest:
-        path: '$.values.etcd.tls.ca'
+        path: '.values.etcd.tls.ca'
     -
       src:
         schema: deckhand/Certificate/v1
         name: calico-node
-        path: $
+        path: .
       dest:
-        path: '$.values.etcd.tls.cert'
+        path: '.values.etcd.tls.cert'
     -
       src:
         schema: deckhand/CertificateKey/v1
         name: calico-node
-        path: $
+        path: .
       dest:
-        path: '$.values.etcd.tls.key'
+        path: '.values.etcd.tls.key'
 data:
   chart_name: calico
   release: calico
@@ -479,28 +480,6 @@ metadata:
     abstract: false
     layer: site
   storagePolicy: cleartext
-  substitutions:
-    -
-      src:
-        schema: deckhand/CertificateAuthority/v1
-        name: kubernetes
-        path: $
-      dest:
-        path: '$.values.tls.ca'
-    -
-      src:
-        schema: deckhand/Certificate/v1
-        name: coredns
-        path: $
-      dest:
-        path: '$.values.tls.cert'
-    -
-      src:
-        schema: deckhand/CertificateKey/v1
-        name: coredns
-        path: $
-      dest:
-        path: '$.values.tls.key'
 data:
   chart_name: coredns
   release: coredns
@@ -509,39 +488,9 @@ data:
   upgrade:
     no_hooks: true
   values:
-    coredns:
-      kubernetes_zones:
-        - cluster.local
-        - 10.96.0.0/16
-        - 10.97.0.0/16
-      upstream_nameservers:
-        - 8.8.8.8
-        - 8.8.4.4
-
-      zones:
-        - name: promenade
-          services:
-            - bind_name: apiserver.kubernetes
-              service:
-                name: kubernetes-apiserver
-                namespace: kube-system
-            - bind_name: etcd.kubernetes
-              service:
-                name: kubernetes-etcd
-                namespace: kube-system
-            - bind_name: etcd.calico
-              service:
-                name: calico-etcd
-                namespace: kube-system
     images:
-      anchor: ${IMAGE_HYPERKUBE}
+      tags:
-      coredns: ${IMAGE_COREDNS}
+        coredns: ${IMAGE_COREDNS}
-    tls:
-      ca: placeholder
-      cert: placeholder
-      key: placeholder
-    network:
-      kubernetes_netloc: apiserver.kubernetes.promenade:6443
   source:
     type: local
     location: /etc/genesis/armada/assets/charts
@@ -550,6 +499,62 @@ data:
     - helm-toolkit
 ---
 schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: haproxy
+  layeringDefinition:
+    abstract: false
+    layer: site
+data:
+  chart_name: haproxy
+  release: haproxy
+  namespace: kube-system
+  timeout: 600
+  wait:
+    timeout: 600
+  upgrade:
+    no_hooks: true
+  values:
+    conf:
+      anchor:
+        kubernetes_url: https://kubernetes.default:443
+        services:
+          default:
+            kubernetes:
+              server_opts: "check"
+              conf_parts:
+                frontend:
+                  - mode tcp
+                  - option tcpka
+                  - bind *:6553
+                backend:
+                  - mode tcp
+                  - option tcpka
+          kube-system:
+            kubernetes-etcd:
+              server_opts: "check"
+              conf_parts:
+                frontend:
+                  - mode tcp
+                  - option tcpka
+                  - bind *:2378
+                backend:
+                  - mode tcp
+                  - option tcpka
+
+    images:
+      tags:
+        anchor: ${IMAGE_HYPERKUBE}
+        haproxy: ${IMAGE_HAPROXY}
+
+  source:
+    type: local
+    location: /etc/genesis/armada/assets/charts
+    subpath: haproxy
+  dependencies:
+    - helm-toolkit
+---
+schema: armada/Chart/v1
 metadata:
   schema: metadata/Document/v1
   name: kubernetes-apiserver
@@ -562,52 +567,52 @@ metadata:
       src:
         schema: deckhand/CertificateAuthority/v1
         name: kubernetes
-        path: $
+        path: .
       dest:
-        path: $.values.secrets.tls.ca
+        path: .values.secrets.tls.ca
 
     -
       src:
         schema: deckhand/Certificate/v1
         name: apiserver
-        path: $
+        path: .
       dest:
-        path: $.values.secrets.tls.cert
+        path: .values.secrets.tls.cert
     -
       src:
         schema: deckhand/CertificateKey/v1
         name: apiserver
-        path: $
+        path: .
       dest:
-        path: $.values.secrets.tls.key
+        path: .values.secrets.tls.key
     -
       src:
         schema: deckhand/CertificateAuthority/v1
         name: kubernetes-etcd
-        path: $
+        path: .
       dest:
-        path: $.values.secrets.etcd.tls.ca
+        path: .values.secrets.etcd.tls.ca
     -
       src:
         schema: deckhand/Certificate/v1
         name: apiserver-etcd
-        path: $
+        path: .
       dest:
-        path: $.values.secrets.etcd.tls.cert
+        path: .values.secrets.etcd.tls.cert
     -
       src:
         schema: deckhand/CertificateKey/v1
         name: apiserver-etcd
-        path: $
+        path: .
       dest:
-        path: $.values.secrets.etcd.tls.key
+        path: .values.secrets.etcd.tls.key
     -
       src:
         schema: deckhand/PublicKey/v1
         name: service-account
-        path: $
+        path: .
       dest:
-        path: $.values.secrets.service_account.public_key
+        path: .values.secrets.service_account.public_key
 
 data:
   chart_name: apiserver
@@ -619,7 +624,7 @@ data:
   values:
     apiserver:
       etcd:
-        endpoints: https://etcd.kubernetes.promenade:2379
+        endpoints: https://127.0.0.1:2378
     images:
       tags:
         anchor: ${IMAGE_HYPERKUBE}
@@ -661,31 +666,31 @@ metadata:
       src:
         schema: deckhand/CertificateAuthority/v1
         name: kubernetes
-        path: $
+        path: .
       dest:
-        path: $.values.secrets.tls.ca
+        path: .values.secrets.tls.ca
 
     -
       src:
         schema: deckhand/Certificate/v1
         name: controller-manager
-        path: $
+        path: .
       dest:
-        path: $.values.secrets.tls.cert
+        path: .values.secrets.tls.cert
     -
       src:
         schema: deckhand/CertificateKey/v1
         name: controller-manager
-        path: $
+        path: .
       dest:
-        path: $.values.secrets.tls.key
+        path: .values.secrets.tls.key
     -
       src:
         schema: deckhand/PrivateKey/v1
         name: service-account
-        path: $
+        path: .
       dest:
-        path: $.values.secrets.service_account.private_key
+        path: .values.secrets.service_account.private_key
 
 data:
   chart_name: controller_manager
@@ -707,7 +712,7 @@ data:
         cert: placeholder
         key: placeholder
     network:
-      kubernetes_netloc: apiserver.kubernetes.promenade:6443
+      kubernetes_netloc: 127.0.0.1:6553
       pod_cidr: 10.97.0.0/16
       service_cidr: 10.96.0.0/16
 
@@ -731,23 +736,23 @@ metadata:
       src:
         schema: deckhand/CertificateAuthority/v1
         name: kubernetes
-        path: $
+        path: .
       dest:
-        path: $.values.secrets.tls.ca
+        path: .values.secrets.tls.ca
     -
       src:
         schema: deckhand/Certificate/v1
         name: scheduler
-        path: $
+        path: .
       dest:
-        path: $.values.secrets.tls.cert
+        path: .values.secrets.tls.cert
     -
       src:
         schema: deckhand/CertificateKey/v1
         name: scheduler
-        path: $
+        path: .
       dest:
-        path: $.values.secrets.tls.key
+        path: .values.secrets.tls.key
 
 data:
   chart_name: scheduler
@@ -764,7 +769,7 @@ data:
         key: placeholder
 
     network:
-      kubernetes_netloc: apiserver.kubernetes.promenade:6443
+      kubernetes_netloc: 127.0.0.1:6553
 
     images:
       tags:
@@ -791,118 +796,118 @@ metadata:
       src:
         schema: deckhand/CertificateAuthority/v1
         name: kubernetes-etcd
-        path: $
+        path: .
       dest:
-        path: '$.values.secrets.tls.client.ca'
+        path: '.values.secrets.tls.client.ca'
     -
       src:
         schema: deckhand/CertificateAuthority/v1
         name: kubernetes-etcd-peer
-        path: $
+        path: .
       dest:
-        path: '$.values.secrets.tls.peer.ca'
+        path: '.values.secrets.tls.peer.ca'
 
     -
       src:
         schema: deckhand/Certificate/v1
         name: kubernetes-etcd-anchor
-        path: $
+        path: .
       dest:
-        path: '$.values.secrets.anchor.tls.cert'
+        path: '.values.secrets.anchor.tls.cert'
     -
       src:
         schema: deckhand/CertificateKey/v1
         name: kubernetes-etcd-anchor
-        path: $
+        path: .
       dest:
-        path: '$.values.secrets.anchor.tls.key'
+        path: '.values.secrets.anchor.tls.key'
 
     -
       src:
         schema: deckhand/Certificate/v1
         name: kubernetes-etcd-${GENESIS_HOSTNAME}
-        path: $
+        path: .
       dest:
-        path: '$.values.nodes[0].tls.client.cert'
+        path: '.values.nodes[0].tls.client.cert'
     -
       src:
         schema: deckhand/CertificateKey/v1
         name: kubernetes-etcd-${GENESIS_HOSTNAME}
-        path: $
+        path: .
       dest:
-        path: '$.values.nodes[0].tls.client.key'
+        path: '.values.nodes[0].tls.client.key'
     -
       src:
         schema: deckhand/Certificate/v1
         name: kubernetes-etcd-${GENESIS_HOSTNAME}-peer
-        path: $
+        path: .
       dest:
-        path: '$.values.nodes[0].tls.peer.cert'
+        path: '.values.nodes[0].tls.peer.cert'
     -
       src:
         schema: deckhand/CertificateKey/v1
         name: kubernetes-etcd-${GENESIS_HOSTNAME}-peer
-        path: $
+        path: .
       dest:
-        path: '$.values.nodes[0].tls.peer.key'
+        path: '.values.nodes[0].tls.peer.key'
 
     -
       src:
         schema: deckhand/Certificate/v1
         name: kubernetes-etcd-${MASTER1_HOSTNAME}
-        path: $
+        path: .
       dest:
-        path: '$.values.nodes[1].tls.client.cert'
+        path: '.values.nodes[1].tls.client.cert'
     -
       src:
         schema: deckhand/CertificateKey/v1
         name: kubernetes-etcd-${MASTER1_HOSTNAME}
-        path: $
+        path: .
       dest:
-        path: '$.values.nodes[1].tls.client.key'
+        path: '.values.nodes[1].tls.client.key'
     -
       src:
         schema: deckhand/Certificate/v1
         name: kubernetes-etcd-${MASTER1_HOSTNAME}-peer
-        path: $
+        path: .
       dest:
-        path: '$.values.nodes[1].tls.peer.cert'
+        path: '.values.nodes[1].tls.peer.cert'
     -
       src:
         schema: deckhand/CertificateKey/v1
         name: kubernetes-etcd-${MASTER1_HOSTNAME}-peer
-        path: $
+        path: .
       dest:
-        path: '$.values.nodes[1].tls.peer.key'
+        path: '.values.nodes[1].tls.peer.key'
 
     -
       src:
         schema: deckhand/Certificate/v1
         name: kubernetes-etcd-${MASTER2_HOSTNAME}
-        path: $
+        path: .
       dest:
-        path: '$.values.nodes[2].tls.client.cert'
+        path: '.values.nodes[2].tls.client.cert'
     -
       src:
         schema: deckhand/CertificateKey/v1
         name: kubernetes-etcd-${MASTER2_HOSTNAME}
-        path: $
+        path: .
       dest:
-        path: '$.values.nodes[2].tls.client.key'
+        path: '.values.nodes[2].tls.client.key'
     -
       src:
         schema: deckhand/Certificate/v1
         name: kubernetes-etcd-${MASTER2_HOSTNAME}-peer
-        path: $
+        path: .
       dest:
-        path: '$.values.nodes[2].tls.peer.cert'
+        path: '.values.nodes[2].tls.peer.cert'
     -
       src:
         schema: deckhand/CertificateKey/v1
         name: kubernetes-etcd-${MASTER2_HOSTNAME}-peer
-        path: $
+        path: .
       dest:
-        path: '$.values.nodes[2].tls.peer.key'
+        path: '.values.nodes[2].tls.peer.key'
 
 data:
   chart_name: etcd
@@ -913,7 +918,7 @@ data:
     no_hooks: true
   values:
     anchor:
-      etcdctl_endpoint: 10.96.0.2
+      etcdctl_endpoint: kubernetes-etcd
     labels:
       anchor:
         node_selector_key: kubernetes-etcd
@@ -962,7 +967,6 @@ data:
             key: placeholder
     service:
       name: kubernetes-etcd
-      ip: 10.96.0.2
     network:
       service_client:
         name: service_client

tools/gate/config-templates/site-config.yaml

@@ -21,12 +21,15 @@ data:
       - 8.8.4.4
 
   kubernetes:
+    apiserver_port: 6443
+    haproxy_port: 6553
     pod_cidr: 10.97.0.0/16
     service_cidr: 10.96.0.0/16
     service_ip: 10.96.0.1
 
   etcd:
-    service_ip: 10.96.0.2
+    container_port: 2379
+    haproxy_port: 2378
 ---
 schema: promenade/Docker/v1
 metadata:
@@ -60,7 +63,7 @@ data:
       tar_path: kubernetes/node/bin/kubelet
       mode: 0555
   images:
-    coredns: ${IMAGE_COREDNS}
+    haproxy: ${IMAGE_HAPROXY}
     helm:
       helm: ${IMAGE_HELM}
     kubernetes:

tools/gate/default-config-env

@@ -3,9 +3,10 @@ IMAGE_CALICO_CNI=quay.io/calico/cni:v1.11.2
 IMAGE_CALICO_CTL=quay.io/calico/ctl:v1.6.3
 IMAGE_CALICO_KUBE_CONTROLLERS=quay.io/calico/kube-controllers:v1.0.2
 IMAGE_CALICO_NODE=quay.io/calico/node:v2.6.5
-IMAGE_COREDNS=coredns/coredns:0.9.9
+IMAGE_COREDNS=coredns/coredns:1.0.5
 IMAGE_DEP_CHECK=quay.io/stackanetes/kubernetes-entrypoint:v0.2.1
 IMAGE_ETCD=quay.io/coreos/etcd:v3.2.14
+IMAGE_HAPROXY=haproxy:1.8.3
 IMAGE_HELM=lachlanevenson/k8s-helm:v2.7.2
 IMAGE_HYPERKUBE=gcr.io/google_containers/hyperkube-amd64:v1.8.6
 IMAGE_TILLER=gcr.io/kubernetes-helm/tiller:v2.7.2