From c681e2197dcbdba5747b68a102281156935fc523 Mon Sep 17 00:00:00 2001
From: Stan Lagun <slagun@mirantis.com>
Date: Mon, 20 Apr 2015 19:54:59 +0300
Subject: [PATCH] Update for changes in k8s 0.15

* Migrated to v1beta3 API
* cAdvisor is now enabled by default
* command line daemon settings updated
* HAproxy template updated for multi-port services
* Better reporting

Change-Id: Iefcbe8073fb35503e2fe2ee5c6bf4610561c3cb6
Closes-Bug: #1446306
Closes-Bug: #1445541
---
 .../package/Classes/KubernetesCluster.yaml    | 403 +++++++++---------
 .../package/Classes/KubernetesMasterNode.yaml |   4 +
 .../package/Classes/KubernetesMinionNode.yaml |  56 ++-
 .../Resources/KubeMinionSetup.template        |   4 +-
 .../Resources/KubeRegisterNode.template       |  14 +-
 .../package/Resources/UpdateService.template  |   4 +-
 .../Resources/scripts/cadvisor.manifest       |  32 --
 .../scripts/default_scripts/kube-proxy        |   3 +-
 .../package/Resources/scripts/getServiceIp.sh |   2 +-
 .../package/Resources/scripts/haproxy.tmpl    |  17 +-
 .../Resources/scripts/kube-add-node.sh        |   9 +-
 .../Resources/scripts/minion-kube-setup.sh    |  11 -
 .../Resources/scripts/minion-node.json        |  14 -
 .../Resources/scripts/updateService.sh        |   5 +-
 .../KubernetesCluster/package/UI/ui.yaml      |   8 +-
 .../package/Classes/KubernetesPod.yaml        |  65 +--
 16 files changed, 299 insertions(+), 352 deletions(-)
 delete mode 100644 Kubernetes/KubernetesCluster/package/Resources/scripts/cadvisor.manifest
 delete mode 100644 Kubernetes/KubernetesCluster/package/Resources/scripts/minion-node.json

diff --git a/Kubernetes/KubernetesCluster/package/Classes/KubernetesCluster.yaml b/Kubernetes/KubernetesCluster/package/Classes/KubernetesCluster.yaml
index bb157b1..944d7fb 100644
--- a/Kubernetes/KubernetesCluster/package/Classes/KubernetesCluster.yaml
+++ b/Kubernetes/KubernetesCluster/package/Classes/KubernetesCluster.yaml
@@ -91,23 +91,30 @@ Methods:
           - $._environment.securityGroupManager.addGroupIngress($securityGroupIngress)
           - $.setAttr(deployed, true)
 
-      - $._environment.reporter.report($this, 'Setting up Kubernetes cluster')
-      - Parallel:
-          - Do: $.masterNode.deployInstance()
-          - Do: $.minionNodes.take($.nodeCount).pselect($.deployInstance())
-          - Do: $.gatewayNodes.take($.gatewayCount).pselect($.deployInstance())
+      - $prevNodeCount: $.getAttr(lastNodeCount, 0)
+      - $prevGatewayCount: $.getAttr(lastGatewayCount, 0)
 
-      - $.masterNode.setupEtcd()
-      - $.minionNodes.take($.nodeCount).select($.setupEtcd())
-      - $.gatewayNodes.take($.gatewayCount).select($.setupEtcd())
+      - If: $prevNodeCount != $.nodeCount or $prevGatewayCount != $.gatewayCount
+        Then:
+          - $._environment.reporter.report($this, 'Setting up Kubernetes cluster')
+          - Parallel:
+              - Do: $.masterNode.deployInstance()
+              - Do: $.minionNodes.take($.nodeCount).pselect($.deployInstance())
+              - Do: $.gatewayNodes.take($.gatewayCount).pselect($.deployInstance())
 
-      - $.masterNode.setupNode()
-      - Parallel:
-          - Do: $.minionNodes.take($.nodeCount).pselect($.setupNode())
-          - Do: $.minionNodes.skip($.nodeCount).pselect($.removeFromCluster())
-          - Do: $.gatewayNodes.take($.gatewayCount).pselect($.setupNode())
-      - $._updateEndpoints()
-      - $._environment.reporter.report($this, 'Kubernetes cluster is up and running')
+          - $.masterNode.setupEtcd()
+          - $.minionNodes.take($.nodeCount).select($.setupEtcd())
+          - $.gatewayNodes.take($.gatewayCount).select($.setupEtcd())
+
+          - $.masterNode.setupNode()
+          - Parallel:
+              - Do: $.minionNodes.take($.nodeCount).pselect($.setupNode())
+              - Do: $.minionNodes.skip($.nodeCount).pselect($.removeFromCluster())
+              - Do: $.gatewayNodes.take($.gatewayCount).pselect($.setupNode())
+          - $._updateServicePublicIps()
+          - $.setAttr(lastNodeCount, $.nodeCount)
+          - $.setAttr(lastGatewayCount, $.gatewayCount)
+          - $._environment.reporter.report($this, 'Kubernetes cluster is up and running')
 
 
   getIp:
@@ -172,7 +179,7 @@ Methods:
       - $.masterNode.instance.agent.call($template, $resources)
 
   
-  createServices:
+  createService:
     Arguments:
       - applicationName:
           Contract: $.string().notNull()
@@ -182,197 +189,199 @@ Methods:
       - podId:
           Contract: $.string().notNull()
     Body:
-      - $resources: new(sys:Resources)
-
-      - $applicationServices: {}
+      - $currentEndpoints: $.serviceEndpoints.where($.applicationName = $applicationName and $.podId = $podId and $.scope = internal)
+      - $serviceName: format('svc-{0}', randomName())
+      - $endpointMap: {}
       - For: endpoint
-        In: $.serviceEndpoints.where($.applicationName = $applicationName).where($.serviceName != null)
+        In: $currentEndpoints
         Do:
           - $serviceName: $endpoint.serviceName
-          - $applicationServices[$serviceName]: $endpoint
+          - $key: format('{0}-{1}', $endpoint.containerPort, $endpoint.protocol)
+          - $endpointMap[$key]: $endpoint
 
-      - $.serviceEndpoints: $.serviceEndpoints.where($.applicationName != $applicationName)
+      - $serviceChanged: len(list($applicationPorts.where($.scope != host))) != len($currentEndpoints)
 
-      - $servicesUsed: []
+      - $servicePorts: []
       - For: applicationPort
-        In: $applicationPorts
+        In: $applicationPorts.where($.scope != host)
         Do:
-          - If: $applicationPort.scope != host
+          - $key: format('{0}-{1}', $applicationPort.port, $applicationPort.protocol)
+          - $endpoint: $endpointMap.get($key)
+          - If: $endpoint != null
             Then:
-              - $serviceName: null
-              - $reuseEndpoint: null
-              - For: service
-                In: $applicationServices.keys()
-                Do:
-                  - $endpoint: $applicationServices.get($service)
-                  - If: $endpoint.containerPort = $applicationPort.port and $endpoint.protocol = $applicationPort.protocol
-                    Then:
-                      - $serviceName: $service
-                      - $reuseEndpoint: $endpoint
-                      - Break:
+              - $record:
+                  - assignedPort: $endpoint.port
+                    applicationPort: $applicationPort
+            Else:
+              - $port: $._findUnusedPort($applicationPort.port, $applicationPort.protocol)
+              - $record:
+                  - assignedPort: $port
+                    applicationPort: $applicationPort
+              - $serviceChanged: true
 
-              - If: $serviceName = null
-                Then:
-                  - $serviceName: format('svc-{0}', randomName())
-                  - $servicePort: $._findUnusedPort($applicationPort.port, $applicationPort.protocol)
-                  - $serviceIp: $._createService($podId, $serviceName, $servicePort, $applicationPort)
-                Else:
-                  - $servicesUsed: $servicesUsed + list($serviceName)
-                  - $servicePort: $reuseEndpoint.port
-                  - $serviceIp: $._updateService($podId, $reuseEndpoint)
+              - $securityGroupIngress:
+                  - ToPort: $port
+                    FromPort: $port
+                    IpProtocol: toLower($applicationPort.protocol)
+                    External: $applicationPort.scope = public
+              - $._environment.securityGroupManager.addGroupIngress($securityGroupIngress)
 
-              - $newEndpoint:
-                  port: $servicePort
-                  address: $serviceIp
-                  scope: internal
-                  portScope: $applicationPort.scope
-                  applicationName: $applicationName
-                  containerPort: $applicationPort.port
-                  protocol: $applicationPort.protocol
-                  podId: $podId
-                  serviceName: $serviceName
-              - $.serviceEndpoints: $.serviceEndpoints + list($newEndpoint)
+          - $servicePorts: $servicePorts + $record
 
-              - If: $applicationPort.scope in list(public, cloud)
-                Then:
-                  - If: $.gatewayCount > 0
-                    Then:
-                      - $nodes: $.gatewayNodes.take($.gatewayCount)
-                    Else:
-                      - $nodes: $.minionNodes.take($.nodeCount)
+      - If: $serviceChanged
+        Then:
+          - $serviceIp: $._createOrUpdateService(
+                name => $serviceName,
+                ports => $servicePorts,
+                podId => $podId,
+                isNew => len($currentEndpoints) = 0
+              )
+          - $._updateEndpoints(
+                ports => $servicePorts,
+                applicationName => $applicationName,
+                podId => $podId,
+                serviceName => $serviceName,
+                serviceIp => $serviceIp
+              )
+          - $._environment.stack.push()
 
-                  - For: t
-                    In: $nodes
-                    Do:
-                      - $newEndpoint.address: $t.getIp()
-                      - $newEndpoint.scope: cloud
-                      - $.serviceEndpoints: $.serviceEndpoints + list($newEndpoint)
 
-                      - If: $t.instance.floatingIpAddress != null and $applicationPort.scope = public
-                        Then:
-                        - $newEndpoint.address: $t.instance.floatingIpAddress
-                        - $newEndpoint.scope: public
-                        - $.serviceEndpoints: $.serviceEndpoints + list($newEndpoint)
+  _createOrUpdateService:
+    Arguments:
+      - name:
+          Contract: $.string().notNull()
+      - ports:
+          Contract:
+            - assignedPort: $.int().notNull()
+              applicationPort: $.class(docker:ApplicationPort).notNull()
+      - podId:
+          Contract: $.string().notNull()
+      - isNew:
+          Contract: $.bool().notNull()
+
+    Body:
+      - $serviceDefinition:
+          apiVersion: v1beta3
+          kind: Service
+          metadata:
+            labels:
+              name: $name
+            name: $name
+          spec:
+            ports: $ports.select(dict(
+                port => $.assignedPort,
+                targetPort => $.applicationPort.port,
+                protocol => $.applicationPort.protocol,
+                name => str($.assignedPort)
+              ))
+            selector:
+              id: $podId
+
+      - If: $.gatewayCount = 0
+        Then:
+          - $serviceDefinition.spec.publicIPs: $.minionNodes.take($.nodeCount).select($.getIp())
+
+      - $resources: new(sys:Resources)
+      - $template: $resources.yaml('UpdateService.template').bind(dict(
+            serviceDefinition => $serviceDefinition,
+            isNew => $isNew
+          ))
+      - Return: $.masterNode.instance.agent.call($template, $resources)
+
+
+  _updateEndpoints:
+    Arguments:
+      - ports:
+          Contract:
+            - assignedPort: $.int().notNull()
+              applicationPort: $.class(docker:ApplicationPort).notNull()
+      - applicationName:
+          Contract: $.string().notNull()
+      - podId:
+          Contract: $.string().notNull()
+      - serviceName:
+          Contract: $.string().notNull()
+      - serviceIp:
+          Contract: $.string().notNull()
+    Body:
+      - $.serviceEndpoints: $.serviceEndpoints.where($.applicationName != $applicationName or $.podId != $podId)
+      - For: port
+        In: $ports
+        Do:
           - $newEndpoint:
-              port: $applicationPort.port
+              port: $port.assignedPort
+              address: $serviceIp
+              scope: internal
+              portScope: $port.applicationPort.scope
+              applicationName: $applicationName
+              containerPort: $port.applicationPort.port
+              protocol: $port.applicationPort.protocol
+              podId: $podId
+              serviceName: $serviceName
+
+          - $.serviceEndpoints: $.serviceEndpoints + list($newEndpoint)
+          - If: $port.applicationPort.scope in list(public, cloud)
+            Then:
+              - If: $.gatewayCount > 0
+                Then:
+                  - $nodes: $.gatewayNodes.take($.gatewayCount)
+                Else:
+                  - $nodes: $.minionNodes.take($.nodeCount)
+
+              - For: t
+                In: $nodes
+                Do:
+                  - $newEndpoint.address: $t.getIp()
+                  - $newEndpoint.scope: cloud
+                  - $.serviceEndpoints: $.serviceEndpoints + list($newEndpoint)
+
+                  - If: $t.instance.floatingIpAddress != null and $port.applicationPort.scope = public
+                    Then:
+                    - $newEndpoint.address: $t.instance.floatingIpAddress
+                    - $newEndpoint.scope: public
+                    - $.serviceEndpoints: $.serviceEndpoints + list($newEndpoint)
+          - $newEndpoint:
+              port: $port.applicationPort.port
               address: '127.0.0.1'
               scope: host
-              portScope: $applicationPort.scope
-              containerPort: $applicationPort.port
-              protocol: $applicationPort.protocol
+              portScope: $port.applicationPort.scope
+              containerPort: $port.applicationPort.port
+              protocol: $port.applicationPort.protocol
               applicationName: $applicationName
               podId: $podId
               serviceName: null
           - $.serviceEndpoints: $.serviceEndpoints + list($newEndpoint)
 
-      - For: service
-        In: $applicationServices.keys()
-        Do:
-          - If: not $service in $servicesUsed
-            Then:
-              - $._deleteService($service)
-      - $._environment.stack.push()
 
-
-  _createService:
-    Arguments:
-      - podId:
-          Contract: $.string().notNull()
-      - serviceName:
-          Contract: $.string().notNull()
-      - servicePort:
-          Contract: $.int().notNull()
-      - applicationPort:
-          Contract: $.class(docker:ApplicationPort)
+  _updateServicePublicIps:
     Body:
-      - $resources: new(sys:Resources)
-      - $serviceDefinition: $._buildServiceDefinition(
-            $serviceName,
-            $servicePort,
-            $applicationPort.protocol,
-            $applicationPort.port,
-            $podId,
-            $.gatewayCount = 0
-          )
-      - $template: $resources.yaml('UpdateService.template').bind(dict(
-            serviceDefinition => $serviceDefinition,
-            isNew => true
-          ))
-      - $securityGroupIngress:
-          - ToPort: $servicePort
-            FromPort: $servicePort
-            IpProtocol: toLower($applicationPort.protocol)
-            External: $applicationPort.scope = public
-
-      - $._environment.securityGroupManager.addGroupIngress($securityGroupIngress)
-
-      - Return: $.masterNode.instance.agent.call($template, $resources)
-
-
-  _updateService:
-    Arguments:
-      - podId:
-          Contract: $.string().notNull()
-      - endpoint:
-          Contract:
-            port: $.int().notNull().check($ > 0)
-            address: $.string().notNull()
-            scope: $.string().notNull().check($ in list(public, cloud, internal, host))
-            containerPort: $.int().notNull().check($ > 0)
-            protocol: $.string().notNull().check($ in list(TCP, UDP))
-            applicationName: $.string().notNull()
-            podId: $.string().notNull()
-            serviceName: $.string()
-
-    Body:
-      - $resources: new(sys:Resources)
-      - $prevNodeCount: $.getAttr(lastNodeCount, 0-1) # 0-1 instead of -1 because YAQL 0.2 doesn't understand unary operators
-      - $prevGatewayCount: $.getAttr(lastGatewayCount, 0-1)
+      - $prevNodeCount: $.getAttr(lastNodeCount, 0)
+      - $prevGatewayCount: $.getAttr(lastGatewayCount, 0)
       - $gatewayModeChanged: $prevGatewayCount != $.gatewayCount and $prevGatewayCount * $.gatewayCount = 0
-
-      - $serviceChanged: $endpoint.podId != $podId or
-          $endpoint.portScope in list(public, cloud) and (
-          $gatewayModeChanged or $.gatewayCount = 0 and $prevNodeCount != $.nodeCount)
-      - If: $serviceChanged
+      - If: $prevGatewayCount > 0 and $.gatewayCount > 0
         Then:
-          - $serviceDefinition: $._buildServiceDefinition(
-                $endpoint.serviceName,
-                $endpoint.port,
-                $endpoint.protocol,
-                $endpoint.containerPort,
-                $podId,
-                $.gatewayCount = 0
-              )
-          - $template: $resources.yaml('UpdateService.template').bind(dict(
-                serviceDefinition => $serviceDefinition,
-                isNew => false
-              ))
-          - $serviceIp: $.masterNode.instance.agent.call($template, $resources)
-        Else:
-          - $serviceIp: $endpoint.address
-      - Return: $serviceIp
-
-
-  _updateEndpoints:
-    Body:
+          - Return:
+      - If: $prevGatewayCount = 0 and $.gatewayCount = 0 and $prevNodeCount = $.nodeCount
+        Then:
+          - Return:
+      - $serviceNameMap: {}
       - For: endpoint
         In: $.serviceEndpoints
         Do:
-          - $._updateService($endpoint.podId, $endpoint)
-      -  $.setAttr(lastNodeCount, $.nodeCount)
-      -  $.setAttr(lastGatewayCount, $.gatewayCount)
-
-
-  _deleteService:
-    Arguments:
-      serviceName:
-        Contract: $.string().notNull()
-    Body:
-      - $resources: new(sys:Resources)
-      - $template: $resources.yaml('DeleteService.template').bind(dict(
-            serviceId => $service
-          ))
-      - $.masterNode.instance.agent.call($template, $resources)
+          - $serviceName: $endpoint.serviceName
+          - If: $serviceName != null
+            Then:
+              - $serviceNameMap[$serviceName]: true
+      - $uniqueServiceNames: $serviceNameMap.keys()
+      - If: len($uniqueServiceNames) > 0
+        Then:
+          - $publicIPs: $.minionNodes.take($.nodeCount).select($.getIp())
+          - $resources: new(sys:Resources)
+          - $template: $resources.yaml('PatchServices.template').bind(dict(
+                services => $uniqueServiceNames,
+                publicIPs => $publicIPs
+              ))
+          - $.masterNode.instance.agent.call($template, $resources)
 
 
   deleteService:
@@ -382,7 +391,11 @@ Methods:
       - podId:
           Contract: $.string().notNull()
     Body:
-      - $._deleteService($applicationName, $podId)
+      - $resources: new(sys:Resources)
+      - $template: $resources.yaml('DeleteService.template').bind(dict(
+            serviceId => $service
+          ))
+      - $.masterNode.instance.agent.call($template, $resources)
 
 
   _findUnusedPort:
@@ -413,36 +426,6 @@ Methods:
       - Return: len(list($.serviceEndpoints.where($.port = $port).where($.protocol = $protocol))) = 0
 
 
-  _buildServiceDefinition:
-    Arguments:
-      - serviceName:
-          Contract: $.string().notNull()
-      - servicePort:
-          Contract: $.int().notNull()
-      - protocol:
-          Contract: $.string().notNull()
-      - containerPort:
-          Contract: $.int().notNull()
-      - podId:
-          Contract: $.string().notNull()
-      - withNodeIps:
-          Contract: $.bool().notNull()
-    Body:
-      - $result:
-          id: $serviceName
-          kind: Service
-          apiVersion: v1beta1
-          port: $servicePort
-          containerPort: $containerPort
-          protocol: $protocol
-          selector:
-            id: $podId
-      - If: $withNodeIps
-        Then:
-          - $result.publicIPs: $.minionNodes.take($.nodeCount).select($.getIp())
-      - Return: $result
-
-
   scaleNodesUp:
     Usage: Action
     Body:
diff --git a/Kubernetes/KubernetesCluster/package/Classes/KubernetesMasterNode.yaml b/Kubernetes/KubernetesCluster/package/Classes/KubernetesMasterNode.yaml
index 02c802b..284a90d 100644
--- a/Kubernetes/KubernetesCluster/package/Classes/KubernetesMasterNode.yaml
+++ b/Kubernetes/KubernetesCluster/package/Classes/KubernetesMasterNode.yaml
@@ -54,3 +54,7 @@ Methods:
               ))
           - $.instance.agent.call($template, $resources)
           - $.setAttr(nodeConfigured, true)
+          - $msg: 'Kubernetes API is now available at http://{0}:8080'
+          - $ip: coalesce($.instance.floatingIpAddress, $.getIp())
+          - $._environment.reporter.report($this, $msg.format($ip))
+
diff --git a/Kubernetes/KubernetesCluster/package/Classes/KubernetesMinionNode.yaml b/Kubernetes/KubernetesCluster/package/Classes/KubernetesMinionNode.yaml
index 70dc628..2659d4a 100644
--- a/Kubernetes/KubernetesCluster/package/Classes/KubernetesMinionNode.yaml
+++ b/Kubernetes/KubernetesCluster/package/Classes/KubernetesMinionNode.yaml
@@ -9,7 +9,7 @@ Name: KubernetesMinionNode
 Extends: KubernetesNode
 
 Properties:
-  enableMonitoring:
+  exposeCAdvisor:
     Contract: $.bool().notNull()
     Default: false
 
@@ -25,7 +25,7 @@ Methods:
     Body:
       - If: not $.getAttr(instanceDeployed, false)
         Then:
-          - $._environment.reporter.report($this, 'Creating Kubernetes Minion')
+          - $._environment.reporter.report($this, 'Creating Kubernetes Node {0}'.format($.instance.name))
           - $.super($.deployInstance())
           - $.setAttr(instanceDeployed, true)
 
@@ -60,32 +60,52 @@ Methods:
           - $template: $resources.yaml('SetupFlannelNode.template')
           - $.instance.agent.call($template, $resources)
 
-          - If: $.enableMonitoring
-            Then:
-              - $._environment.reporter.report($this, 'Adding access to cAdvisor')
-              - $securityGroupIngress:
-                  - ToPort: 4194
-                    FromPort: 4194
-                    IpProtocol: tcp
-                    External: true
-              - $._environment.securityGroupManager.addGroupIngress($securityGroupIngress)
+          - $securityGroupIngress:
+              - ToPort: 4194
+                FromPort: 4194
+                IpProtocol: tcp
+                External: $.exposeCAdvisor
+          - $._environment.securityGroupManager.addGroupIngress($securityGroupIngress)
 
           - $._environment.reporter.report($, 'Setup Kubernetes Minion on {0}'.format($.instance.name))
           - $template: $resources.yaml('KubeMinionSetup.template').bind(dict(
                 name => $.instance.name,
                 ip => $.getIp(),
                 masterIp => $._cluster.masterNode.getIp(),
-                enableMonitoring => $.enableMonitoring,
                 dockerRegistry => $._cluster.dockerRegistry
               ))
           - $.instance.agent.call($template, $resources)
-
-          - $template: $resources.yaml('KubeRegisterNode.template').bind(dict(
-                name => $.instance.name,
-                nodeId => $.getIp()
-              ))
-          - $._cluster.masterNode.instance.agent.call($template, $resources)
+          - $._registerNode()
           - $.setAttr(nodeConfigured, true)
+          - $msg: 'cAdvisor monitoring for Node {0} is now available at http://{1}:4194'
+          - $ip: $.getIp()
+          - If: $.exposeCAdvisor
+            Then:
+              - $ip: coalesce($.instance.floatingIpAddress, $.getIp())
+          - $._environment.reporter.report($this, $msg.format($.instance.name, $ip))
+
+
+  _registerNode:
+    Body:
+      - $nodeDefinition:
+          kind: Node
+          apiVersion: v1beta3
+          metadata:
+            name: $.getIp()
+            labels:
+              name: $.instance.name
+          spec:
+            externalID: $.id()
+            status:
+              capacity:
+                cpu: 200
+                memory: 4145438720
+
+      - $resources: new(sys:Resources)
+      - $template: $resources.yaml('KubeRegisterNode.template').bind(dict(
+            nodeDefinition => $nodeDefinition
+          ))
+      - $._cluster.masterNode.instance.agent.call($template, $resources)
 
 
   removeFromCluster:
diff --git a/Kubernetes/KubernetesCluster/package/Resources/KubeMinionSetup.template b/Kubernetes/KubernetesCluster/package/Resources/KubeMinionSetup.template
index 0dc037f..de0ff56 100644
--- a/Kubernetes/KubernetesCluster/package/Resources/KubeMinionSetup.template
+++ b/Kubernetes/KubernetesCluster/package/Resources/KubeMinionSetup.template
@@ -6,13 +6,12 @@ Parameters:
   name: $name
   ip: $ip
   masterIp: $masterIp
-  enableMonitoring: $enableMonitoring
   dockerRegistry: $dockerRegistry
 
 Body: |
   if args.dockerRegistry:
     setupRegistry(args.dockerRegistry)
-  setup('{0} {1} {2} {3}'.format(args.name, args.ip, args.masterIp, args.enableMonitoring))
+  setup('{0} {1} {2}'.format(args.name, args.ip, args.masterIp))
 
 Scripts:
   setup:
@@ -26,7 +25,6 @@ Scripts:
       - init_conf/kube-proxy.conf
       - initd_scripts/kubelet
       - initd_scripts/kube-proxy
-      - cadvisor.manifest
 
     Options:
       captureStdout: true
diff --git a/Kubernetes/KubernetesCluster/package/Resources/KubeRegisterNode.template b/Kubernetes/KubernetesCluster/package/Resources/KubeRegisterNode.template
index b9fbdf6..8992dcf 100644
--- a/Kubernetes/KubernetesCluster/package/Resources/KubeRegisterNode.template
+++ b/Kubernetes/KubernetesCluster/package/Resources/KubeRegisterNode.template
@@ -3,19 +3,23 @@ Version: 1.0.0
 Name: Register Kubernetes Node
 
 Parameters:
-  name: $name
-  nodeId: $nodeId
+  nodeDefinition: $nodeDefinition
 
 Body: |
-  return register('{0} {1}'.format(args.name, args.nodeId)).stdout
+  import json
+  import uuid
+  fileName = '/var/run/murano-kubernetes/' + str(uuid.uuid4()) + '.json'
+  with open(fileName, 'w') as f:
+    json.dump(args.nodeDefinition, f)
+
+  return register(fileName).stdout
 
 Scripts:
   register:
     Type: Application
     Version: 1.0.0
     EntryPoint: kube-add-node.sh
-    Files:
-      - minion-node.json
+    Files: []
 
     Options:
       captureStdout: true
diff --git a/Kubernetes/KubernetesCluster/package/Resources/UpdateService.template b/Kubernetes/KubernetesCluster/package/Resources/UpdateService.template
index 705fa75..1e583af 100644
--- a/Kubernetes/KubernetesCluster/package/Resources/UpdateService.template
+++ b/Kubernetes/KubernetesCluster/package/Resources/UpdateService.template
@@ -13,8 +13,8 @@ Body: |
   with open(fileName, 'w') as f:
     json.dump(args.serviceDefinition, f)
 
-  updateService('{0} {1} {2} {3}'.format(args.isNew, args.serviceDefinition['id'], args.serviceDefinition['kind'], fileName))
-  return getServiceIp(args.serviceDefinition['id']).stdout
+  updateService('{0} {1} {2}'.format(args.isNew, args.serviceDefinition['metadata']['name'], fileName))
+  return getServiceIp(args.serviceDefinition['metadata']['name']).stdout
 
 Scripts:
   updateService:
diff --git a/Kubernetes/KubernetesCluster/package/Resources/scripts/cadvisor.manifest b/Kubernetes/KubernetesCluster/package/Resources/scripts/cadvisor.manifest
deleted file mode 100644
index 18827e1..0000000
--- a/Kubernetes/KubernetesCluster/package/Resources/scripts/cadvisor.manifest
+++ /dev/null
@@ -1,32 +0,0 @@
-version: v1beta2
-id: cadvisor-agent
-containers:
-  - name: cadvisor
-    image: google/cadvisor:latest
-    ports:
-      - name: http
-        containerPort: 8080
-        hostPort: 4194
-    volumeMounts:
-      - name: varrun
-        mountPath: /var/run
-        readOnly: false
-      - name: varlibdocker
-        mountPath: /var/lib/docker
-        readOnly: true
-      - name: cgroups
-        mountPath: /sys/fs/cgroup
-        readOnly: true
-volumes:
-  - name: varrun
-    source:
-      hostDir:
-        path: /var/run
-  - name: varlibdocker
-    source:
-      hostDir:
-        path: /var/lib/docker
-  - name: cgroups
-    source:
-      hostDir:
-        path: /sys/fs/cgroup
diff --git a/Kubernetes/KubernetesCluster/package/Resources/scripts/default_scripts/kube-proxy b/Kubernetes/KubernetesCluster/package/Resources/scripts/default_scripts/kube-proxy
index 37a786f..fba60b9 100644
--- a/Kubernetes/KubernetesCluster/package/Resources/scripts/default_scripts/kube-proxy
+++ b/Kubernetes/KubernetesCluster/package/Resources/scripts/default_scripts/kube-proxy
@@ -4,7 +4,6 @@
 # KUBE_PROXY="/opt/bin/kube-proxy"
 
 # Use KUBE_PROXY_OPTS to modify the start/restart options
-KUBE_PROXY_OPTS="--etcd_servers=http://127.0.0.1:4001 \
-  --logtostderr=false --master=http://%%MASTER_IP%%:8080 --log_dir=/var/log/kubernetes"
+KUBE_PROXY_OPTS="--logtostderr=false --master=http://%%MASTER_IP%%:8080 --log_dir=/var/log/kubernetes"
 
 # Add more envionrment settings used by kube-apiserver here
\ No newline at end of file
diff --git a/Kubernetes/KubernetesCluster/package/Resources/scripts/getServiceIp.sh b/Kubernetes/KubernetesCluster/package/Resources/scripts/getServiceIp.sh
index 2be4daf..8b28cc5 100644
--- a/Kubernetes/KubernetesCluster/package/Resources/scripts/getServiceIp.sh
+++ b/Kubernetes/KubernetesCluster/package/Resources/scripts/getServiceIp.sh
@@ -1,3 +1,3 @@
 #!/bin/bash
 
-/opt/bin/kubectl get service $1 -t '{{.portalIP}}' -o template
\ No newline at end of file
+/opt/bin/kubectl get service $1 -t '{{.spec.portalIP}}' -o template
\ No newline at end of file
diff --git a/Kubernetes/KubernetesCluster/package/Resources/scripts/haproxy.tmpl b/Kubernetes/KubernetesCluster/package/Resources/scripts/haproxy.tmpl
index 87e8983..8d4b544 100644
--- a/Kubernetes/KubernetesCluster/package/Resources/scripts/haproxy.tmpl
+++ b/Kubernetes/KubernetesCluster/package/Resources/scripts/haproxy.tmpl
@@ -5,15 +5,16 @@ defaults
   contimeout 5000
   clitimeout 50000
   srvtimeout 50000
-
 {{range $svc := ls "/registry/services/endpoints/default"}}
-{{$se := printf "/registry/services/endpoints/default/%s" $svc }}{{$ss := printf "/registry/services/specs/default/%s" $svc }}
-{{$seKey := get $se}}{{$ssKey := get $ss}}{{$seJson := json $seKey.Value}}{{$ssJson := json $ssKey.Value}}{{$baseSvc := base $svc}}
-{{if and (ne "kubernetes" $baseSvc) (ne "kubernetes-ro" $baseSvc)}}
-listen {{$baseSvc}} 0.0.0.0:{{$ssJson.port}}
+{{$se := printf "/registry/services/endpoints/default/%s" $svc }}
+{{$ss := printf "/registry/services/specs/default/%s" $svc }}
+{{$seKey := get $se}}{{$ssKey := get $ss}}{{$seJson := json $seKey.Value}}
+{{$ssJson := json $ssKey.Value}}{{$baseSvc := base $svc}}
+{{if and (ne "kubernetes" $baseSvc) (ne "kubernetes-ro" $baseSvc)}}{{range $port := $ssJson.spec.ports}}
+listen {{$baseSvc}}-{{$port.port}} 0.0.0.0:{{$port.port}}
     mode tcp
     balance leastconn
-{{range $index, $endpoint := $seJson.endpoints}}
-    server svr{{$index}} {{$endpoint}}{{end}}
+{{range $subset := $seJson.subsets}}{{range $index, $endpoint := $subset.addresses}}
+    server svr{{$index}} {{$endpoint.IP}}:{{$port.targetPort}}{{end}}{{end}}
 
-{{end}}{{end}}
+{{end}}{{end}}{{end}}
diff --git a/Kubernetes/KubernetesCluster/package/Resources/scripts/kube-add-node.sh b/Kubernetes/KubernetesCluster/package/Resources/scripts/kube-add-node.sh
index 6925867..eb4516a 100644
--- a/Kubernetes/KubernetesCluster/package/Resources/scripts/kube-add-node.sh
+++ b/Kubernetes/KubernetesCluster/package/Resources/scripts/kube-add-node.sh
@@ -1,10 +1,5 @@
 #!/bin/bash
 
-# $1 - NAME
-# $2 - IP
-#
+# $1 - file path
 
-sed -i.bkp "s/%%NAME%%/$1/g" minion-node.json
-sed -i.bkp "s/%%IP%%/$2/g" minion-node.json
-
-/opt/bin/kubectl create -f minion-node.json
+/opt/bin/kubectl create -f $1
diff --git a/Kubernetes/KubernetesCluster/package/Resources/scripts/minion-kube-setup.sh b/Kubernetes/KubernetesCluster/package/Resources/scripts/minion-kube-setup.sh
index e09af20..1617961 100644
--- a/Kubernetes/KubernetesCluster/package/Resources/scripts/minion-kube-setup.sh
+++ b/Kubernetes/KubernetesCluster/package/Resources/scripts/minion-kube-setup.sh
@@ -3,7 +3,6 @@
 # $1 - NAME
 # $2 - IP
 # $3 - MASTER_IP
-# $4 - IS_CA_ENABLED
 
 mkdir /var/log/kubernetes
 mkdir -p /var/run/murano-kubernetes
@@ -22,16 +21,6 @@ cp initd_scripts/kube-proxy /etc/init.d/
 cp -f default_scripts/kube-proxy /etc/default
 cp -f default_scripts/kubelet /etc/default/
 
-if [ "$4" == "True" ]; then
-   #Create directory for manifests used by kubelet
-   mkdir /etc/kubernetes
-   mkdir /etc/kubernetes/manifests
-   cp -f cadvisor.manifest /etc/kubernetes/manifests
-   #Add path to kubelet parameters
-   sed -i 's/kubernetes"/kubernetes \\/g' /etc/default/kubelet
-   sed -i '/--log_dir*/a --config=/etc/kubernetes/manifests"' /etc/default/kubelet
-fi
-
 service kubelet start
 service kube-proxy start
 
diff --git a/Kubernetes/KubernetesCluster/package/Resources/scripts/minion-node.json b/Kubernetes/KubernetesCluster/package/Resources/scripts/minion-node.json
deleted file mode 100644
index 1f8fce9..0000000
--- a/Kubernetes/KubernetesCluster/package/Resources/scripts/minion-node.json
+++ /dev/null
@@ -1,14 +0,0 @@
-{
-  "id": "%%IP%%",
-  "kind": "Minion",
-  "apiVersion": "v1beta1",
-  "resources": {
-    "capacity": {
-      "cpu": 200,
-      "memory": 4145438720
-    }
-  },
-  "labels": {
-    "name": "%%NAME%%"
-  }
-}
\ No newline at end of file
diff --git a/Kubernetes/KubernetesCluster/package/Resources/scripts/updateService.sh b/Kubernetes/KubernetesCluster/package/Resources/scripts/updateService.sh
index 5361ddd..848a51e 100644
--- a/Kubernetes/KubernetesCluster/package/Resources/scripts/updateService.sh
+++ b/Kubernetes/KubernetesCluster/package/Resources/scripts/updateService.sh
@@ -5,10 +5,9 @@
 DEFINITION_DIR=/var/run/murano-kubernetes
 mkdir -p $DEFINITION_DIR
 serviceId=$2
-kind=$3
-fileName=$4
+fileName=$3
 
-echo "$serviceId $kind $fileName" >> $DEFINITION_DIR/elements.list
+echo "$serviceId Service $fileName" >> $DEFINITION_DIR/elements.list
 
 if [ "$1" == "True" ]; then
   echo "Creating a new Service" >> /tmp/murano-kube.log
diff --git a/Kubernetes/KubernetesCluster/package/UI/ui.yaml b/Kubernetes/KubernetesCluster/package/UI/ui.yaml
index ad0694e..fb6aa85 100644
--- a/Kubernetes/KubernetesCluster/package/UI/ui.yaml
+++ b/Kubernetes/KubernetesCluster/package/UI/ui.yaml
@@ -24,7 +24,7 @@ Templates:
       image: 'ubuntu14.04-x64-kubernetes'
       assignFloatingIp: $.appConfiguration.assignFloatingIP
       keyname: $.instanceConfiguration.keyPair
-    enableMonitoring: $.appConfiguration.enableMonitoring
+    exposeCAdvisor: $.appConfiguration.exposeCAdvisor
 
 
   gatewayNode:
@@ -97,13 +97,13 @@ Forms:
           errorMessages:
             invalid: Just letters, numbers, underscores, sharps and hyphens are allowed.
           label: Kubernetes node hostname pattern
-        - name: enableMonitoring
+        - name: exposeCAdvisor
           type: boolean
           initial: true
           required: false
-          label: Enable cAdvisor monitoring
+          label: Expose cAdvisor UI
           description: >-
-            Enable cAdvisor monitoring
+            Opens external access to cAdvisor interface
         - name: gatewayCount
           type: integer
           label: Initial/current number of gateway nodes
diff --git a/Kubernetes/KubernetesPod/package/Classes/KubernetesPod.yaml b/Kubernetes/KubernetesPod/package/Classes/KubernetesPod.yaml
index 9865593..ddb0aff 100644
--- a/Kubernetes/KubernetesPod/package/Classes/KubernetesPod.yaml
+++ b/Kubernetes/KubernetesPod/package/Classes/KubernetesPod.yaml
@@ -35,16 +35,14 @@ Methods:
       - If: $podDefinition = null
         Then:
           - $podDefinition:
-              id: $podName
+              apiVersion: v1beta3
               kind: Pod
-              apiVersion: v1beta1
-              desiredState:
-                manifest:
-                  version: v1beta1
-                  id: $podName
-                  containers: []
-                  volumes: []
-              labels: $._getPodLabels($podName)
+              metadata:
+                name: $podName
+                labels: $._getPodLabels($podName)
+              spec:
+                containers: []
+                volumes: []
 
           - $.setAttr(lastPodDeployed, $podDefinition)
       - $._podDefinition: $podDefinition
@@ -81,23 +79,23 @@ Methods:
       - $containerDef:
           name: toLower($container.name)
           image: $container.image
-          command: $container.commands
+          args: $container.commands
           ports: $container.ports.select($this._getPortDefinition($))
           volumeMounts: $container.volumes.keys().select(dict(name => $this._generateVolumeName($container.volumes.get($)), mountPath => $))
           env: $container.env.keys().select(dict(key => $, value => $container.env.get($)))
 
-      - $newVolumes: $container.volumes.values().where(not $this._generateVolumeName($) in $this._podDefinition.desiredState.volumes.name).
+      - $newVolumes: $container.volumes.values().where(not $this._generateVolumeName($) in $this._podDefinition.spec.volumes.name).
           select($this._buildVolumeEntry($))
 
       - $diff:
-          desiredState:
-            manifest:
-              containers: [$containerDef]
-              volumes: $newVolumes
+      - $diff:
+          spec:
+            containers: [$containerDef]
+            volumes: $newVolumes
       - $._podDefinition: $._podDefinition.mergeWith($diff)
       - $.deploy()
       - $._environment.reporter.report($, 'Creating services for Pod {0}'.format($.name))
-      - $.kubernetesCluster.createServices(
+      - $.kubernetesCluster.createService(
             applicationName => $container.name,
             applicationPorts => $container.ports,
             podId => $podName)
@@ -153,14 +151,14 @@ Methods:
       - name:
           Contract: $.string().notNull()
     Body:
-      - $lenBefore: len($._podDefinition.desiredState.manifest.containers) + len($._podDefinition.desiredState.manifest.volumes)
-      - $newContainers: $._podDefinition.desiredState.manifest.containers.where($.name != $name)
-      - $newVolumes: $._podDefinition.desiredState.manifest.volumes.where(
-            $.name in $._podDefinition.desiredState.manifest.containers.volumeMounts.name)
+      - $lenBefore: len($._podDefinition.spec.containers) + len($._podDefinition.spec.volumes)
+      - $newContainers: $._podDefinition.spec.containers.where($.name != $name)
+      - $newVolumes: $._podDefinition.spec.volumes.where(
+            $.name in $._podDefinition.spec.containers.volumeMounts.name)
       - If: len($newContainers) + len($newVolumes) != $lenBefore
         Then:
-          - $._podDefinition.desiredState.manifest.containers: $newContainers
-          - $._podDefinition.desiredState.manifest.volumes: $newVolumes
+          - $._podDefinition.spec.containers: $newContainers
+          - $._podDefinition.spec.volumes: $newVolumes
 
 
   deleteContainer:
@@ -190,7 +188,7 @@ Methods:
 
       - $podDefinition: $._podDefinition
       - $replicas: $.replicas
-      - If: len($podDefinition.desiredState.manifest.containers) = 0
+      - If: len($podDefinition.spec.containers) = 0
         Then:
           - $replicas: 0
       - $.setAttr(lastReplicas, $replicas)
@@ -205,10 +203,10 @@ Methods:
           - If: $replicas = 0 and $prevReplicas > 0
             Then:
               - $.kubernetesCluster.deleteReplicationController($._getReplicationControllerId())
-          - If: $prevPod != $podDefinition
+          - If: $prevPod != $podDefinition and len($prevPod.spec.containers) > 0
             Then:
               - $.kubernetesCluster.deletePods(dict(id => $._getPodName()))
-          - If: $.replicas = 0 and len($podDefinition.desiredState.manifest.containers) > 0
+          - If: $.replicas = 0 and len($podDefinition.spec.containers) > 0
             Then:
               - $.kubernetesCluster.createPod(definition => $podDefinition, isNew => true)
 
@@ -222,16 +220,19 @@ Methods:
           Contract: {}
     Body:
       Return:
-        id: $._getReplicationControllerId()
+        apiVersion: v1beta3
         kind: ReplicationController
-        apiVersion: v1beta1
-        desiredState:
+        metadata:
+          name: $._getReplicationControllerId()
+          labels: $podDefinition.metadata.labels
+        spec:
           replicas: $.replicas
-          replicaSelector:
+          selector:
             id: $._getPodName()
-          podTemplate:
-            desiredState: $podDefinition.desiredState
-            labels: $podDefinition.labels
+          template:
+            metadata:
+              labels: $podDefinition.metadata.labels
+            spec: $podDefinition.spec
 
 
   _getReplicationControllerId: