a88a5cf15a
Puts into place the DeploymentConfiguration yaml that provides the options that should be configured by the site design to the deployment (and update) workflows. This change additionally refactors reused parts to common modules as related to info passing (xcom) Change-Id: Ib6470899b204dbc18d2a9a2e4f95540b3b0032b0
333 lines
12 KiB
Python
333 lines
12 KiB
Python
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import json
|
|
import logging
|
|
import os
|
|
import requests
|
|
from urllib.parse import urlparse
|
|
|
|
from airflow.models import BaseOperator
|
|
from airflow.utils.decorators import apply_defaults
|
|
from airflow.plugins_manager import AirflowPlugin
|
|
from airflow.exceptions import AirflowException
|
|
|
|
import armada.common.client as client
|
|
import armada.common.session as session
|
|
from get_k8s_pod_port_ip import get_pod_port_ip
|
|
from service_endpoint import ucp_service_endpoint
|
|
from service_token import shipyard_service_token
|
|
from xcom_puller import XcomPuller
|
|
|
|
|
|
class ArmadaOperator(BaseOperator):
|
|
"""
|
|
Supports interaction with Armada
|
|
:param action: Task to perform
|
|
:param main_dag_name: Parent Dag
|
|
:param shipyard_conf: Location of shipyard.conf
|
|
:param sub_dag_name: Child Dag
|
|
|
|
The Drydock operator assumes that prior steps have set xcoms for
|
|
the action and the deployment configuration
|
|
"""
|
|
|
|
@apply_defaults
|
|
def __init__(self,
|
|
action=None,
|
|
main_dag_name=None,
|
|
shipyard_conf=None,
|
|
svc_token=None,
|
|
sub_dag_name=None,
|
|
xcom_push=True,
|
|
*args, **kwargs):
|
|
|
|
super(ArmadaOperator, self).__init__(*args, **kwargs)
|
|
self.action = action
|
|
self.main_dag_name = main_dag_name
|
|
self.shipyard_conf = shipyard_conf
|
|
self.svc_token = svc_token
|
|
self.sub_dag_name = sub_dag_name
|
|
self.xcom_push_flag = xcom_push
|
|
|
|
def execute(self, context):
|
|
# Initialize Variables
|
|
armada_client = None
|
|
design_ref = None
|
|
|
|
# Define task_instance
|
|
task_instance = context['task_instance']
|
|
|
|
# Set up and retrieve values from xcom
|
|
self.xcom_puller = XcomPuller(self.main_dag_name, task_instance)
|
|
self.action_info = self.xcom_puller.get_action_info()
|
|
|
|
# Logs uuid of action performed by the Operator
|
|
logging.info("Armada Operator for action %s", self.action_info['id'])
|
|
|
|
# Retrieve Deckhand Design Reference
|
|
design_ref = self.get_deckhand_design_ref(context)
|
|
|
|
if design_ref:
|
|
logging.info("Design YAMLs will be retrieved from %s",
|
|
design_ref)
|
|
else:
|
|
raise AirflowException("Unable to Retrieve Design Reference!")
|
|
|
|
# Validate Site Design
|
|
if self.action == 'validate_site_design':
|
|
# Initialize variable
|
|
armada_svc_endpoint = None
|
|
site_design_validity = 'invalid'
|
|
|
|
# Retrieve Endpoint Information
|
|
svc_type = 'armada'
|
|
armada_svc_endpoint = ucp_service_endpoint(self,
|
|
svc_type=svc_type)
|
|
|
|
site_design_validity = self.armada_validate_site_design(
|
|
armada_svc_endpoint, design_ref)
|
|
|
|
if site_design_validity == 'valid':
|
|
logging.info("Site Design has been successfully validated")
|
|
else:
|
|
raise AirflowException("Site Design Validation Failed!")
|
|
|
|
return site_design_validity
|
|
|
|
# Set up target manifest (only if not doing validate)
|
|
self.dc = self.xcom_puller.get_deployment_configuration()
|
|
self.target_manifest = self.dc['armada.manifest']
|
|
|
|
# Create Armada Client
|
|
# Retrieve Endpoint Information
|
|
svc_type = 'armada'
|
|
armada_svc_endpoint = ucp_service_endpoint(self,
|
|
svc_type=svc_type)
|
|
logging.info("Armada endpoint is %s", armada_svc_endpoint)
|
|
|
|
# Set up Armada Client
|
|
armada_client = self.armada_session_client(armada_svc_endpoint)
|
|
|
|
# Retrieve Tiller Information and assign to context 'query'
|
|
context['query'] = self.get_tiller_info(context)
|
|
|
|
# Armada API Call
|
|
# Armada Status
|
|
if self.action == 'armada_status':
|
|
self.get_armada_status(context, armada_client)
|
|
|
|
# Armada Apply
|
|
elif self.action == 'armada_apply':
|
|
self.armada_apply(context, armada_client, design_ref,
|
|
self.target_manifest)
|
|
|
|
# Armada Get Releases
|
|
elif self.action == 'armada_get_releases':
|
|
self.armada_get_releases(context, armada_client)
|
|
|
|
else:
|
|
logging.info('No Action to Perform')
|
|
|
|
@shipyard_service_token
|
|
def armada_session_client(self, armada_svc_endpoint):
|
|
# Initialize Variables
|
|
armada_url = None
|
|
a_session = None
|
|
a_client = None
|
|
|
|
# Parse Armada Service Endpoint
|
|
armada_url = urlparse(armada_svc_endpoint)
|
|
|
|
# Build a ArmadaSession with credentials and target host
|
|
# information.
|
|
logging.info("Build Armada Session")
|
|
a_session = session.ArmadaSession(host=armada_url.hostname,
|
|
port=armada_url.port,
|
|
scheme='http',
|
|
token=self.svc_token,
|
|
marker=None)
|
|
|
|
# Raise Exception if we are not able to get armada session
|
|
if a_session:
|
|
logging.info("Successfully Set Up Armada Session")
|
|
else:
|
|
raise AirflowException("Failed to set up Armada Session!")
|
|
|
|
# Use session to build a ArmadaClient to make one or more
|
|
# API calls. The ArmadaSession will care for TCP connection
|
|
# pooling and header management
|
|
logging.info("Create Armada Client")
|
|
a_client = client.ArmadaClient(a_session)
|
|
|
|
# Raise Exception if we are not able to build armada client
|
|
if a_client:
|
|
logging.info("Successfully Set Up Armada client")
|
|
else:
|
|
raise AirflowException("Failed to set up Armada client!")
|
|
|
|
# Return Armada client for XCOM Usage
|
|
return a_client
|
|
|
|
@get_pod_port_ip('tiller', namespace='kube-system')
|
|
def get_tiller_info(self, context, *args, **kwargs):
|
|
# Initialize Variable
|
|
query = {}
|
|
|
|
# Get IP and port information of Pods from context
|
|
k8s_pods_ip_port = context['pods_ip_port']
|
|
|
|
# Assign value to the 'query' dictionary so that we can pass
|
|
# it via the Armada Client
|
|
query['tiller_host'] = k8s_pods_ip_port['tiller'].get('ip')
|
|
query['tiller_port'] = k8s_pods_ip_port['tiller'].get('port')
|
|
|
|
return query
|
|
|
|
def get_armada_status(self, context, armada_client):
|
|
# Check State of Tiller
|
|
armada_status = armada_client.get_status(context['query'])
|
|
|
|
# Tiller State will return boolean value, i.e. True/False
|
|
# Raise Exception if Tiller is in a bad state
|
|
if armada_status['tiller']['state']:
|
|
logging.info("Tiller is in running state")
|
|
logging.info("Tiller version is %s",
|
|
armada_status['tiller']['version'])
|
|
else:
|
|
raise AirflowException("Please check Tiller!")
|
|
|
|
def armada_apply(self, context, armada_client, design_ref,
|
|
target_manifest):
|
|
'''Run Armada Apply
|
|
'''
|
|
# Initialize Variables
|
|
armada_manifest = None
|
|
armada_ref = design_ref
|
|
armada_post_apply = {}
|
|
override_values = []
|
|
chart_set = []
|
|
# enhance the context's query entity with target_manifest
|
|
query = context.get('query', {})
|
|
query['target_manifest'] = target_manifest
|
|
|
|
# Execute Armada Apply to install the helm charts in sequence
|
|
logging.info("Armada Apply")
|
|
armada_post_apply = armada_client.post_apply(manifest=armada_manifest,
|
|
manifest_ref=armada_ref,
|
|
values=override_values,
|
|
set=chart_set,
|
|
query=query)
|
|
|
|
# We will expect Armada to return the releases that it is
|
|
# deploying. Note that if we try and deploy the same release
|
|
# twice, we will end up with empty response as nothing has
|
|
# changed.
|
|
if armada_post_apply['message']['install']:
|
|
logging.info("Armada Apply Successfully Executed")
|
|
logging.info(armada_post_apply)
|
|
else:
|
|
logging.warning("No new changes/updates were detected")
|
|
logging.info(armada_post_apply)
|
|
|
|
def armada_get_releases(self, context, armada_client):
|
|
# Initialize Variables
|
|
armada_releases = {}
|
|
deckhand_svc_endpoint = None
|
|
|
|
# Retrieve Armada Releases after deployment
|
|
logging.info("Retrieving Armada Releases after deployment..")
|
|
armada_releases = armada_client.get_releases(context['query'])
|
|
|
|
if armada_releases:
|
|
logging.info("Retrieved current Armada Releases")
|
|
logging.info(armada_releases)
|
|
else:
|
|
raise AirflowException("Failed to retrieve Armada Releases")
|
|
|
|
def get_deckhand_design_ref(self, context):
|
|
|
|
# Retrieve DeckHand Endpoint Information
|
|
svc_type = 'deckhand'
|
|
deckhand_svc_endpoint = ucp_service_endpoint(self,
|
|
svc_type=svc_type)
|
|
logging.info("Deckhand endpoint is %s", deckhand_svc_endpoint)
|
|
|
|
# Retrieve revision_id from xcom
|
|
committed_revision_id = self.xcom_puller.get_design_version()
|
|
|
|
# Form Design Reference Path that we will use to retrieve
|
|
# the Design YAMLs
|
|
deckhand_path = "deckhand+" + deckhand_svc_endpoint
|
|
deckhand_design_ref = os.path.join(deckhand_path,
|
|
"revisions",
|
|
str(committed_revision_id),
|
|
"rendered-documents")
|
|
|
|
return deckhand_design_ref
|
|
|
|
@shipyard_service_token
|
|
def armada_validate_site_design(self, armada_svc_endpoint, design_ref):
|
|
|
|
# Form Validation Endpoint
|
|
validation_endpoint = os.path.join(armada_svc_endpoint,
|
|
'validatedesign')
|
|
|
|
logging.info("Validation Endpoint is %s", validation_endpoint)
|
|
|
|
# Define Headers and Payload
|
|
headers = {
|
|
'Content-Type': 'application/json',
|
|
'X-Auth-Token': self.svc_token
|
|
}
|
|
|
|
payload = {
|
|
'rel': "design",
|
|
'href': design_ref,
|
|
'type': "application/x-yaml"
|
|
}
|
|
|
|
# Requests Armada to validate site design
|
|
logging.info("Waiting for Armada to validate site design...")
|
|
|
|
try:
|
|
design_validate_response = requests.post(validation_endpoint,
|
|
headers=headers,
|
|
data=json.dumps(payload))
|
|
except requests.exceptions.RequestException as e:
|
|
raise AirflowException(e)
|
|
|
|
# Convert response to string
|
|
validate_site_design = design_validate_response.text
|
|
|
|
# Print response
|
|
logging.info("Retrieving Armada validate site design response...")
|
|
|
|
try:
|
|
validate_site_design_dict = json.loads(validate_site_design)
|
|
logging.info(validate_site_design_dict)
|
|
except json.JSONDecodeError as e:
|
|
raise AirflowException(e)
|
|
|
|
# Check if site design is valid
|
|
if validate_site_design_dict.get('status') == 'Success':
|
|
return 'valid'
|
|
else:
|
|
return 'invalid'
|
|
|
|
|
|
class ArmadaOperatorPlugin(AirflowPlugin):
|
|
name = 'armada_operator_plugin'
|
|
operators = [ArmadaOperator]
|