airship/shipyard
Code Issues Proposed changes

Align Operators with UcpBaseOperator

Browse Source 
All UCP Operators will inherit from the UcpBaseOperator [0]

This patch set will align the rest of the Operators, i.e. Armada,
Deckhand and Promenade Operators with the UcpBaseOperator

It also updates the name of the shipyard container to be
'shipyard-api' instead of 'shipyard'

[0] https://review.gerrithub.io/#/c/407736/

Change-Id: I516590c492e9bb5554161119dade278d74197374
This commit is contained in:
Anthony Lin 2018-04-18 07:15:26 +00:00
parent 91b60ac595
commit 47cd7a25f4
17 changed files with 153 additions and 184 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
charts/shipyard/templates/deployment-shipyard.yaml
View File

@ -24,7 +24,7 @@
apiVersion: apps/v1beta1 apiVersion: apps/v1beta1
kind: Deployment kind: Deployment
metadata: metadata:
name: shipyard name: shipyard-api
spec: spec:
replicas: {{ .Values.pod.replicas.shipyard.api }} replicas: {{ .Values.pod.replicas.shipyard.api }}
{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }} {{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }}
@ -44,7 +44,7 @@ spec:
initContainers: initContainers:
{{ tuple $envAll $dependencies $mounts_shipyard_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} {{ tuple $envAll $dependencies $mounts_shipyard_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
containers: containers:
- name: shipyard - name: shipyard-api
env: env:
- name: 'SHIPYARD_API_WORKERS' - name: 'SHIPYARD_API_WORKERS'
value: {{ .Values.conf.uwsgi.workers | quote }} value: {{ .Values.conf.uwsgi.workers | quote }}

62
shipyard_airflow/plugins/armada_base_operator.py
View File

@ -11,13 +11,11 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import logging import logging
import os import os
from urllib.parse import urlparse from urllib.parse import urlparse
from airflow.exceptions import AirflowException from airflow.exceptions import AirflowException
from airflow.models import BaseOperator
from airflow.plugins_manager import AirflowPlugin from airflow.plugins_manager import AirflowPlugin
from airflow.utils.decorators import apply_defaults from airflow.utils.decorators import apply_defaults
@ -26,11 +24,13 @@ import armada.common.session as session
from get_k8s_pod_port_ip import get_pod_port_ip from get_k8s_pod_port_ip import get_pod_port_ip
from service_endpoint import ucp_service_endpoint from service_endpoint import ucp_service_endpoint
from service_token import shipyard_service_token from service_token import shipyard_service_token
from xcom_puller import XcomPuller from ucp_base_operator import UcpBaseOperator
from xcom_pusher import XcomPusher from xcom_pusher import XcomPusher
LOG = logging.getLogger(__name__)
class ArmadaBaseOperator(BaseOperator):
class ArmadaBaseOperator(UcpBaseOperator):
"""Armada Base Operator """Armada Base Operator
@ -44,66 +44,42 @@ class ArmadaBaseOperator(BaseOperator):
def __init__(self, def __init__(self,
armada_svc_type='armada', armada_svc_type='armada',
deckhand_svc_type='deckhand', deckhand_svc_type='deckhand',
main_dag_name=None,
query={}, query={},
shipyard_conf=None,
sub_dag_name=None,
svc_session=None, svc_session=None,
svc_token=None, svc_token=None,
xcom_push=True,
*args, **kwargs): *args, **kwargs):
"""Initialization of ArmadaBaseOperator object. """Initialization of ArmadaBaseOperator object.
:param armada_svc_type: Armada Service Type :param armada_svc_type: Armada Service Type
:param deckhand_svc_type: Deckhand Service Type :param deckhand_svc_type: Deckhand Service Type
:param main_dag_name: Parent Dag
:param query: A dictionary containing explicit query string parameters :param query: A dictionary containing explicit query string parameters
:param shipyard_conf: Location of shipyard.conf
:param sub_dag_name: Child Dag
:param svc_session: Keystone Session :param svc_session: Keystone Session
:param svc_token: Keystone Token :param svc_token: Keystone Token
:param xcom_push: xcom usage
The Armada operator assumes that prior steps have set xcoms for The Armada operator assumes that prior steps have set xcoms for
the action and the deployment configuration the action and the deployment configuration
""" """
super(ArmadaBaseOperator, self).__init__(*args, **kwargs) super(ArmadaBaseOperator,
self).__init__(
pod_selector_pattern=[{'pod_pattern': 'armada-api',
'container': 'armada-api'}],
*args, **kwargs)
self.armada_svc_type = armada_svc_type self.armada_svc_type = armada_svc_type
self.deckhand_svc_type = deckhand_svc_type self.deckhand_svc_type = deckhand_svc_type
self.main_dag_name = main_dag_name
self.query = query self.query = query
self.shipyard_conf = shipyard_conf
self.sub_dag_name = sub_dag_name
self.svc_session = svc_session self.svc_session = svc_session
self.svc_token = svc_token self.svc_token = svc_token
self.xcom_push_flag = xcom_push
def execute(self, context):
# Execute armada base function
self.armada_base(context)
# Exeute child function
self.do_execute()
@shipyard_service_token @shipyard_service_token
def armada_base(self, context): def run_base(self, context):
# Define task_instance
self.task_instance = context['task_instance']
# Set up and retrieve values from xcom
self.xcom_puller = XcomPuller(self.main_dag_name, self.task_instance)
self.action_info = self.xcom_puller.get_action_info()
self.dc = self.xcom_puller.get_deployment_configuration()
# Set up xcom_pusher to push values to xcom # Set up xcom_pusher to push values to xcom
self.xcom_pusher = XcomPusher(self.task_instance) self.xcom_pusher = XcomPusher(self.task_instance)
# Logs uuid of action performed by the Operator # Logs uuid of action performed by the Operator
logging.info("Armada Operator for action %s", self.action_info['id']) LOG.info("Armada Operator for action %s", self.action_info['id'])
# Retrieve Endpoint Information # Retrieve Endpoint Information
armada_svc_endpoint = ucp_service_endpoint( armada_svc_endpoint = ucp_service_endpoint(
@ -128,14 +104,14 @@ class ArmadaBaseOperator(BaseOperator):
@staticmethod @staticmethod
def _init_armada_client(armada_svc_endpoint, svc_token): def _init_armada_client(armada_svc_endpoint, svc_token):
logging.info("Armada endpoint is %s", armada_svc_endpoint) LOG.info("Armada endpoint is %s", armada_svc_endpoint)
# Parse Armada Service Endpoint # Parse Armada Service Endpoint
armada_url = urlparse(armada_svc_endpoint) armada_url = urlparse(armada_svc_endpoint)
# Build a ArmadaSession with credentials and target host # Build a ArmadaSession with credentials and target host
# information. # information.
logging.info("Build Armada Session") LOG.info("Build Armada Session")
a_session = session.ArmadaSession(host=armada_url.hostname, a_session = session.ArmadaSession(host=armada_url.hostname,
port=armada_url.port, port=armada_url.port,
scheme='http', scheme='http',
@ -144,18 +120,18 @@ class ArmadaBaseOperator(BaseOperator):
# Raise Exception if we are not able to set up the session # Raise Exception if we are not able to set up the session
if a_session: if a_session:
logging.info("Successfully Set Up Armada Session") LOG.info("Successfully Set Up Armada Session")
else: else:
raise AirflowException("Failed to set up Armada Session!") raise AirflowException("Failed to set up Armada Session!")
# Use the ArmadaSession to build a ArmadaClient that can # Use the ArmadaSession to build a ArmadaClient that can
# be used to make one or more API calls # be used to make one or more API calls
logging.info("Create Armada Client") LOG.info("Create Armada Client")
_armada_client = client.ArmadaClient(a_session) _armada_client = client.ArmadaClient(a_session)
# Raise Exception if we are not able to build armada client # Raise Exception if we are not able to build armada client
if _armada_client: if _armada_client:
logging.info("Successfully Set Up Armada client") LOG.info("Successfully Set Up Armada client")
return _armada_client return _armada_client
else: else:
@ -165,7 +141,7 @@ class ArmadaBaseOperator(BaseOperator):
def _init_deckhand_design_ref(deckhand_svc_endpoint, def _init_deckhand_design_ref(deckhand_svc_endpoint,
committed_revision_id): committed_revision_id):
logging.info("Deckhand endpoint is %s", deckhand_svc_endpoint) LOG.info("Deckhand endpoint is %s", deckhand_svc_endpoint)
# Form DeckHand Design Reference Path # Form DeckHand Design Reference Path
# This URL will be used to retrieve the Site Design YAMLs # This URL will be used to retrieve the Site Design YAMLs
@ -176,8 +152,8 @@ class ArmadaBaseOperator(BaseOperator):
"rendered-documents") "rendered-documents")
if _deckhand_design_ref: if _deckhand_design_ref:
logging.info("Design YAMLs will be retrieved from %s", LOG.info("Design YAMLs will be retrieved from %s",
_deckhand_design_ref) _deckhand_design_ref)
return _deckhand_design_ref return _deckhand_design_ref
else: else:

12
shipyard_airflow/plugins/armada_get_releases.py
View File

@ -11,7 +11,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import logging import logging
from airflow.exceptions import AirflowException from airflow.exceptions import AirflowException
@ -20,6 +19,8 @@ from airflow.plugins_manager import AirflowPlugin
from armada_base_operator import ArmadaBaseOperator from armada_base_operator import ArmadaBaseOperator
from armada.exceptions import api_exceptions as errors from armada.exceptions import api_exceptions as errors
LOG = logging.getLogger(__name__)
class ArmadaGetReleasesOperator(ArmadaBaseOperator): class ArmadaGetReleasesOperator(ArmadaBaseOperator):
@ -39,7 +40,7 @@ class ArmadaGetReleasesOperator(ArmadaBaseOperator):
timeout = self.dc['armada.get_releases_timeout'] timeout = self.dc['armada.get_releases_timeout']
# Retrieve Armada Releases after deployment # Retrieve Armada Releases after deployment
logging.info("Retrieving Helm charts releases after deployment..") LOG.info("Retrieving Helm charts releases after deployment..")
try: try:
armada_get_releases = self.armada_client.get_releases( armada_get_releases = self.armada_client.get_releases(
@ -50,9 +51,12 @@ class ArmadaGetReleasesOperator(ArmadaBaseOperator):
raise AirflowException(client_error) raise AirflowException(client_error)
if armada_get_releases: if armada_get_releases:
logging.info("Successfully retrieved Helm charts releases") LOG.info("Successfully retrieved Helm charts releases")
logging.info(armada_get_releases) LOG.info(armada_get_releases)
else: else:
# Dump logs from Armada API pods
self.get_k8s_logs()
raise AirflowException("Failed to retrieve Helm charts releases!") raise AirflowException("Failed to retrieve Helm charts releases!")

12
shipyard_airflow/plugins/armada_get_status.py
View File

@ -11,7 +11,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import logging import logging
from airflow.exceptions import AirflowException from airflow.exceptions import AirflowException
@ -20,6 +19,8 @@ from airflow.plugins_manager import AirflowPlugin
from armada_base_operator import ArmadaBaseOperator from armada_base_operator import ArmadaBaseOperator
from armada.exceptions import api_exceptions as errors from armada.exceptions import api_exceptions as errors
LOG = logging.getLogger(__name__)
class ArmadaGetStatusOperator(ArmadaBaseOperator): class ArmadaGetStatusOperator(ArmadaBaseOperator):
@ -51,11 +52,14 @@ class ArmadaGetStatusOperator(ArmadaBaseOperator):
# Tiller State will return boolean value, i.e. True/False # Tiller State will return boolean value, i.e. True/False
# Raise Exception if Tiller is unhealthy # Raise Exception if Tiller is unhealthy
if armada_get_status['tiller']['state']: if armada_get_status['tiller']['state']:
logging.info("Tiller is in running state") LOG.info("Tiller is in running state")
logging.info("Tiller version is %s", LOG.info("Tiller version is %s",
armada_get_status['tiller']['version']) armada_get_status['tiller']['version'])
else: else:
# Dump logs from Armada API pods
self.get_k8s_logs()
raise AirflowException("Please check Tiller!") raise AirflowException("Please check Tiller!")

20
shipyard_airflow/plugins/armada_post_apply.py
View File

@ -11,7 +11,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import logging import logging
from airflow.exceptions import AirflowException from airflow.exceptions import AirflowException
@ -20,6 +19,8 @@ from airflow.plugins_manager import AirflowPlugin
from armada_base_operator import ArmadaBaseOperator from armada_base_operator import ArmadaBaseOperator
from armada.exceptions import api_exceptions as errors from armada.exceptions import api_exceptions as errors
LOG = logging.getLogger(__name__)
class ArmadaPostApplyOperator(ArmadaBaseOperator): class ArmadaPostApplyOperator(ArmadaBaseOperator):
@ -52,7 +53,7 @@ class ArmadaPostApplyOperator(ArmadaBaseOperator):
timeout = self.dc['armada.post_apply_timeout'] timeout = self.dc['armada.post_apply_timeout']
# Execute Armada Apply to install the helm charts in sequence # Execute Armada Apply to install the helm charts in sequence
logging.info("Armada Apply") LOG.info("Armada Apply")
try: try:
armada_post_apply = self.armada_client.post_apply( armada_post_apply = self.armada_client.post_apply(
@ -64,6 +65,9 @@ class ArmadaPostApplyOperator(ArmadaBaseOperator):
timeout=timeout) timeout=timeout)
except errors.ClientError as client_error: except errors.ClientError as client_error:
# Dump logs from Armada API pods
self.get_k8s_logs()
raise AirflowException(client_error) raise AirflowException(client_error)
# if this is a retry, assume that the airflow worker needs to be # if this is a retry, assume that the airflow worker needs to be
@ -74,7 +78,7 @@ class ArmadaPostApplyOperator(ArmadaBaseOperator):
# needs to. Problem with xcom is that it is cleared for the task # needs to. Problem with xcom is that it is cleared for the task
# on retry, which means we can't use it as a flag reliably. # on retry, which means we can't use it as a flag reliably.
if self.task_instance.try_number > 1: if self.task_instance.try_number > 1:
logging.info( LOG.info(
"Airflow Worker will be upgraded because retry may obfuscate " "Airflow Worker will be upgraded because retry may obfuscate "
"an upgrade of shipyard/airflow." "an upgrade of shipyard/airflow."
) )
@ -87,7 +91,7 @@ class ArmadaPostApplyOperator(ArmadaBaseOperator):
# part of the name of the Shipyard Helm Chart. # part of the name of the Shipyard Helm Chart.
for i in armada_post_apply['message']['upgrade']: for i in armada_post_apply['message']['upgrade']:
if 'shipyard' in i: if 'shipyard' in i:
logging.info( LOG.info(
"Shipyard was upgraded. Airflow worker must be " "Shipyard was upgraded. Airflow worker must be "
"restarted to reflect any workflow changes." "restarted to reflect any workflow changes."
) )
@ -110,11 +114,11 @@ class ArmadaPostApplyOperator(ArmadaBaseOperator):
# changed. # changed.
if (armada_post_apply['message']['install'] or if (armada_post_apply['message']['install'] or
armada_post_apply['message']['upgrade']): armada_post_apply['message']['upgrade']):
logging.info("Successfully Executed Armada Apply") LOG.info("Successfully Executed Armada Apply")
logging.info(armada_post_apply) LOG.info(armada_post_apply)
else: else:
logging.warning("No new changes/updates were detected!") LOG.warning("No new changes/updates were detected!")
logging.info(armada_post_apply) LOG.info(armada_post_apply)
class ArmadaPostApplyOperatorPlugin(AirflowPlugin): class ArmadaPostApplyOperatorPlugin(AirflowPlugin):

17
shipyard_airflow/plugins/armada_validate_design.py
View File

@ -11,7 +11,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import logging import logging
from airflow.plugins_manager import AirflowPlugin from airflow.plugins_manager import AirflowPlugin
@ -20,6 +19,8 @@ from airflow.exceptions import AirflowException
from armada_base_operator import ArmadaBaseOperator from armada_base_operator import ArmadaBaseOperator
from armada.exceptions import api_exceptions as errors from armada.exceptions import api_exceptions as errors
LOG = logging.getLogger(__name__)
class ArmadaValidateDesignOperator(ArmadaBaseOperator): class ArmadaValidateDesignOperator(ArmadaBaseOperator):
@ -33,7 +34,7 @@ class ArmadaValidateDesignOperator(ArmadaBaseOperator):
def do_execute(self): def do_execute(self):
# Requests Armada to validate site design # Requests Armada to validate site design
logging.info("Waiting for Armada to validate site design...") LOG.info("Waiting for Armada to validate site design...")
# Retrieve read timeout # Retrieve read timeout
timeout = self.dc['armada.validate_design_timeout'] timeout = self.dc['armada.validate_design_timeout']
@ -45,18 +46,24 @@ class ArmadaValidateDesignOperator(ArmadaBaseOperator):
timeout=timeout) timeout=timeout)
except errors.ClientError as client_error: except errors.ClientError as client_error:
# Dump logs from Armada API pods
self.get_k8s_logs()
raise AirflowException(client_error) raise AirflowException(client_error)
# Print results # Print results
logging.info("Retrieving Armada validate site design response...") LOG.info("Retrieving Armada validate site design response...")
logging.info(post_validate) LOG.info(post_validate)
# Check if site design is valid # Check if site design is valid
status = str(post_validate.get('status', 'unspecified')) status = str(post_validate.get('status', 'unspecified'))
if status.lower() == 'success': if status.lower() == 'success':
logging.info("Site Design has been successfully validated") LOG.info("Site Design has been successfully validated")
else: else:
# Dump logs from Armada API pods
self.get_k8s_logs()
raise AirflowException("Site Design Validation Failed " raise AirflowException("Site Design Validation Failed "
"with status: {}!".format(status)) "with status: {}!".format(status))

60
shipyard_airflow/plugins/deckhand_base_operator.py
View File

@ -11,11 +11,9 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import configparser import configparser
import logging import logging
from airflow.models import BaseOperator
from airflow.utils.decorators import apply_defaults from airflow.utils.decorators import apply_defaults
from airflow.plugins_manager import AirflowPlugin from airflow.plugins_manager import AirflowPlugin
from airflow.exceptions import AirflowException from airflow.exceptions import AirflowException
@ -23,10 +21,12 @@ from airflow.exceptions import AirflowException
from deckhand.client import client as deckhand_client from deckhand.client import client as deckhand_client
from service_endpoint import ucp_service_endpoint from service_endpoint import ucp_service_endpoint
from service_token import shipyard_service_token from service_token import shipyard_service_token
from xcom_puller import XcomPuller from ucp_base_operator import UcpBaseOperator
LOG = logging.getLogger(__name__)
class DeckhandBaseOperator(BaseOperator): class DeckhandBaseOperator(UcpBaseOperator):
"""Deckhand Base Operator """Deckhand Base Operator
@ -43,14 +43,10 @@ class DeckhandBaseOperator(BaseOperator):
deckhand_client_read_timeout=None, deckhand_client_read_timeout=None,
deckhand_svc_endpoint=None, deckhand_svc_endpoint=None,
deckhand_svc_type='deckhand', deckhand_svc_type='deckhand',
main_dag_name=None,
revision_id=None, revision_id=None,
shipyard_conf=None,
sub_dag_name=None,
svc_session=None, svc_session=None,
svc_token=None, svc_token=None,
validation_read_timeout=None, validation_read_timeout=None,
xcom_push=True,
*args, **kwargs): *args, **kwargs):
"""Initialization of DeckhandBaseOperator object. """Initialization of DeckhandBaseOperator object.
@ -59,47 +55,30 @@ class DeckhandBaseOperator(BaseOperator):
:param deckhand_client_read_timeout: Deckhand client connect timeout :param deckhand_client_read_timeout: Deckhand client connect timeout
:param deckhand_svc_endpoint: Deckhand Service Endpoint :param deckhand_svc_endpoint: Deckhand Service Endpoint
:param deckhand_svc_type: Deckhand Service Type :param deckhand_svc_type: Deckhand Service Type
:param main_dag_name: Parent Dag
:param revision_id: Target revision for workflow :param revision_id: Target revision for workflow
:param shipyard_conf: Path of shipyard.conf
:param sub_dag_name: Child Dag
:param svc_session: Keystone Session :param svc_session: Keystone Session
:param svc_token: Keystone Token :param svc_token: Keystone Token
:param validation_read_timeout: Deckhand validation timeout :param validation_read_timeout: Deckhand validation timeout
:param xcom_push: xcom usage
""" """
super(DeckhandBaseOperator, self).__init__(*args, **kwargs) super(DeckhandBaseOperator,
self).__init__(
pod_selector_pattern=[{'pod_pattern': 'deckhand-api',
'container': 'deckhand-api'}],
*args, **kwargs)
self.committed_ver = committed_ver self.committed_ver = committed_ver
self.deckhandclient = deckhandclient self.deckhandclient = deckhandclient
self.deckhand_client_read_timeout = deckhand_client_read_timeout self.deckhand_client_read_timeout = deckhand_client_read_timeout
self.deckhand_svc_endpoint = deckhand_svc_endpoint self.deckhand_svc_endpoint = deckhand_svc_endpoint
self.deckhand_svc_type = deckhand_svc_type self.deckhand_svc_type = deckhand_svc_type
self.main_dag_name = main_dag_name
self.revision_id = revision_id self.revision_id = revision_id
self.shipyard_conf = shipyard_conf
self.sub_dag_name = sub_dag_name
self.svc_session = svc_session self.svc_session = svc_session
self.svc_token = svc_token self.svc_token = svc_token
self.validation_read_timeout = validation_read_timeout self.validation_read_timeout = validation_read_timeout
self.xcom_push_flag = xcom_push
def execute(self, context):
# Execute deckhand base function
self.deckhand_base(context)
# Exeute child function
self.do_execute()
# Push last committed version to xcom for the
# 'get_design_version' subdag
if self.sub_dag_name == 'get_design_version':
return self.committed_ver
@shipyard_service_token @shipyard_service_token
def deckhand_base(self, context): def run_base(self, context):
# Read and parse shiyard.conf # Read and parse shiyard.conf
config = configparser.ConfigParser() config = configparser.ConfigParser()
@ -112,26 +91,19 @@ class DeckhandBaseOperator(BaseOperator):
self.validation_read_timeout = int(config.get( self.validation_read_timeout = int(config.get(
'requests_config', 'validation_read_timeout')) 'requests_config', 'validation_read_timeout'))
# Define task_instance
task_instance = context['task_instance']
# Set up and retrieve values from xcom
self.xcom_puller = XcomPuller(self.main_dag_name, task_instance)
self.action_info = self.xcom_puller.get_action_info()
# Logs uuid of Shipyard action # Logs uuid of Shipyard action
logging.info("Executing Shipyard Action %s", LOG.info("Executing Shipyard Action %s",
self.action_info['id']) self.action_info['id'])
# Retrieve Endpoint Information # Retrieve Endpoint Information
self.deckhand_svc_endpoint = ucp_service_endpoint( self.deckhand_svc_endpoint = ucp_service_endpoint(
self, svc_type=self.deckhand_svc_type) self, svc_type=self.deckhand_svc_type)
logging.info("Deckhand endpoint is %s", LOG.info("Deckhand endpoint is %s",
self.deckhand_svc_endpoint) self.deckhand_svc_endpoint)
# Set up DeckHand Client # Set up DeckHand Client
logging.info("Setting up DeckHand Client...") LOG.info("Setting up DeckHand Client...")
# NOTE: The communication between the Airflow workers # NOTE: The communication between the Airflow workers
# and Deckhand happens via the 'internal' endpoint. # and Deckhand happens via the 'internal' endpoint.
@ -155,7 +127,7 @@ class DeckhandBaseOperator(BaseOperator):
self.revision_id = self.xcom_puller.get_design_version() self.revision_id = self.xcom_puller.get_design_version()
if self.revision_id: if self.revision_id:
logging.info("Revision ID is %d", self.revision_id) LOG.info("Revision ID is %d", self.revision_id)
else: else:
raise AirflowException('Failed to retrieve Revision ID!') raise AirflowException('Failed to retrieve Revision ID!')

19
shipyard_airflow/plugins/deckhand_get_design.py
View File

@ -11,7 +11,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import logging import logging
import os import os
import requests import requests
@ -22,6 +21,8 @@ from airflow.exceptions import AirflowException
from deckhand_base_operator import DeckhandBaseOperator from deckhand_base_operator import DeckhandBaseOperator
LOG = logging.getLogger(__name__)
class DeckhandGetDesignOperator(DeckhandBaseOperator): class DeckhandGetDesignOperator(DeckhandBaseOperator):
@ -42,7 +43,7 @@ class DeckhandGetDesignOperator(DeckhandBaseOperator):
'revisions') 'revisions')
# Retrieve Revision # Retrieve Revision
logging.info("Retrieving revisions information...") LOG.info("Retrieving revisions information...")
try: try:
query_params = {'tag': 'committed', 'sort': 'id', 'order': 'desc'} query_params = {'tag': 'committed', 'sort': 'id', 'order': 'desc'}
@ -53,12 +54,15 @@ class DeckhandGetDesignOperator(DeckhandBaseOperator):
timeout=self.deckhand_client_read_timeout).text) timeout=self.deckhand_client_read_timeout).text)
except requests.exceptions.RequestException as e: except requests.exceptions.RequestException as e:
# Dump logs from Deckhand pods
self.get_k8s_logs()
raise AirflowException(e) raise AirflowException(e)
# Print info about revisions from DeckHand # Print info about revisions from DeckHand
logging.info("Revisions response: %s", revisions) LOG.info("Revisions response: %s", revisions)
logging.info("The number of committed revisions is %s", LOG.info("The number of committed revisions is %s",
revisions['count']) revisions['count'])
# Search for the latest committed version and save it as xcom. # Search for the latest committed version and save it as xcom.
# Since the order : desc paramater above, this is index 0 if there # Since the order : desc paramater above, this is index 0 if there
@ -66,10 +70,13 @@ class DeckhandGetDesignOperator(DeckhandBaseOperator):
revision_list = revisions.get('results', []) revision_list = revisions.get('results', [])
if revision_list: if revision_list:
self.committed_ver = revision_list[0].get('id') self.committed_ver = revision_list[0].get('id')
logging.info("Latest committed revision is %d", self.committed_ver) LOG.info("Latest committed revision is %d", self.committed_ver)
# Error if we cannot resolve the committed version to use. # Error if we cannot resolve the committed version to use.
if not self.committed_ver: if not self.committed_ver:
# Dump logs from Deckhand pods
self.get_k8s_logs()
raise AirflowException("No committed revision found in Deckhand!") raise AirflowException("No committed revision found in Deckhand!")

13
shipyard_airflow/plugins/deckhand_retrieve_rendered_doc.py
View File

@ -11,7 +11,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import logging import logging
from airflow.plugins_manager import AirflowPlugin from airflow.plugins_manager import AirflowPlugin
@ -19,6 +18,8 @@ from airflow.exceptions import AirflowException
from deckhand_base_operator import DeckhandBaseOperator from deckhand_base_operator import DeckhandBaseOperator
LOG = logging.getLogger(__name__)
class DeckhandRetrieveRenderedDocOperator(DeckhandBaseOperator): class DeckhandRetrieveRenderedDocOperator(DeckhandBaseOperator):
@ -30,17 +31,19 @@ class DeckhandRetrieveRenderedDocOperator(DeckhandBaseOperator):
def do_execute(self): def do_execute(self):
logging.info("Retrieving Rendered Document...") LOG.info("Retrieving Rendered Document...")
# Retrieve Rendered Document # Retrieve Rendered Document
try: try:
rendered_doc = self.deckhandclient.revisions.documents( self.deckhandclient.revisions.documents(
self.revision_id, rendered=True) self.revision_id, rendered=True)
logging.info("Successfully Retrieved Rendered Document") LOG.info("Successfully Retrieved Rendered Document")
logging.info(rendered_doc)
except: except:
# Dump logs from Deckhand pods
self.get_k8s_logs()
raise AirflowException("Failed to Retrieve Rendered Document!") raise AirflowException("Failed to Retrieve Rendered Document!")

15
shipyard_airflow/plugins/deckhand_validate_site.py
View File

@ -11,7 +11,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import logging import logging
import os import os
import requests import requests
@ -22,6 +21,8 @@ from airflow.exceptions import AirflowException
from deckhand_base_operator import DeckhandBaseOperator from deckhand_base_operator import DeckhandBaseOperator
LOG = logging.getLogger(__name__)
class DeckhandValidateSiteDesignOperator(DeckhandBaseOperator): class DeckhandValidateSiteDesignOperator(DeckhandBaseOperator):
@ -43,7 +44,7 @@ class DeckhandValidateSiteDesignOperator(DeckhandBaseOperator):
str(self.revision_id), str(self.revision_id),
'validations') 'validations')
# Retrieve Validation list # Retrieve Validation list
logging.info("Retrieving validation list...") LOG.info("Retrieving validation list...")
try: try:
retrieved_list = yaml.safe_load( retrieved_list = yaml.safe_load(
@ -52,14 +53,20 @@ class DeckhandValidateSiteDesignOperator(DeckhandBaseOperator):
timeout=self.validation_read_timeout).text) timeout=self.validation_read_timeout).text)
except requests.exceptions.RequestException as e: except requests.exceptions.RequestException as e:
# Dump logs from Deckhand pods
self.get_k8s_logs()
raise AirflowException(e) raise AirflowException(e)
if (any([str(v.get('status', 'unspecified')).lower() == 'failure' if (any([str(v.get('status', 'unspecified')).lower() == 'failure'
for v in retrieved_list.get('results', [])])): for v in retrieved_list.get('results', [])])):
# Dump logs from Deckhand pods
self.get_k8s_logs()
raise AirflowException("DeckHand Site Design Validation Failed!") raise AirflowException("DeckHand Site Design Validation Failed!")
else: else:
logging.info("Revision %d has been successfully validated", LOG.info("Revision %d has been successfully validated",
self.revision_id) self.revision_id)
class DeckhandValidateSiteDesignOperatorPlugin(AirflowPlugin): class DeckhandValidateSiteDesignOperatorPlugin(AirflowPlugin):

51
shipyard_airflow/plugins/promenade_base_operator.py
View File

@ -11,20 +11,20 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import logging import logging
from airflow.models import BaseOperator
from airflow.utils.decorators import apply_defaults from airflow.utils.decorators import apply_defaults
from airflow.plugins_manager import AirflowPlugin from airflow.plugins_manager import AirflowPlugin
from airflow.exceptions import AirflowException from airflow.exceptions import AirflowException
from service_endpoint import ucp_service_endpoint from service_endpoint import ucp_service_endpoint
from service_token import shipyard_service_token from service_token import shipyard_service_token
from xcom_puller import XcomPuller from ucp_base_operator import UcpBaseOperator
LOG = logging.getLogger(__name__)
class PromenadeBaseOperator(BaseOperator): class PromenadeBaseOperator(UcpBaseOperator):
"""Promenade Base Operator """Promenade Base Operator
@ -35,59 +35,36 @@ class PromenadeBaseOperator(BaseOperator):
@apply_defaults @apply_defaults
def __init__(self, def __init__(self,
main_dag_name=None,
promenade_svc_endpoint=None, promenade_svc_endpoint=None,
promenade_svc_type='kubernetesprovisioner', promenade_svc_type='kubernetesprovisioner',
redeploy_server=None, redeploy_server=None,
shipyard_conf=None,
sub_dag_name=None,
svc_token=None, svc_token=None,
xcom_push=True,
*args, **kwargs): *args, **kwargs):
"""Initialization of PromenadeBaseOperator object. """Initialization of PromenadeBaseOperator object.
:param main_dag_name: Parent Dag
:param promenade_svc_endpoint: Promenade Service Endpoint :param promenade_svc_endpoint: Promenade Service Endpoint
:param promenade_svc_type: Promenade Service Type :param promenade_svc_type: Promenade Service Type
:param redeploy_server: Server to be redeployed :param redeploy_server: Server to be redeployed
:param shipyard_conf: Path of shipyard.conf
:param sub_dag_name: Child Dag
:param svc_token: Keystone Token :param svc_token: Keystone Token
:param xcom_push: xcom usage
The Drydock operator assumes that prior steps have set xcoms for The Drydock operator assumes that prior steps have set xcoms for
the action and the deployment configuration the action and the deployment configuration
""" """
super(PromenadeBaseOperator, self).__init__(*args, super(PromenadeBaseOperator,
**kwargs) self).__init__(
self.main_dag_name = main_dag_name pod_selector_pattern=[{'pod_pattern': 'promenade-api',
'container': 'promenade-api'}],
*args, **kwargs)
self.promenade_svc_endpoint = promenade_svc_endpoint self.promenade_svc_endpoint = promenade_svc_endpoint
self.promenade_svc_type = promenade_svc_type self.promenade_svc_type = promenade_svc_type
self.redeploy_server = redeploy_server self.redeploy_server = redeploy_server
self.shipyard_conf = shipyard_conf
self.sub_dag_name = sub_dag_name
self.svc_token = svc_token self.svc_token = svc_token
self.xcom_push_flag = xcom_push
def execute(self, context):
# Execute promenade base function
self.promenade_base(context)
# Exeute child function
self.do_execute()
@shipyard_service_token @shipyard_service_token
def promenade_base(self, context): def run_base(self, context):
# Define task_instance
task_instance = context['task_instance']
# Set up and retrieve values from xcom
self.xcom_puller = XcomPuller(self.main_dag_name, task_instance)
self.action_info = self.xcom_puller.get_action_info()
self.dc = self.xcom_puller.get_deployment_configuration()
# Logs uuid of Shipyard action # Logs uuid of Shipyard action
logging.info("Executing Shipyard Action %s", self.action_info['id']) LOG.info("Executing Shipyard Action %s", self.action_info['id'])
# Retrieve information of the server that we want to redeploy # Retrieve information of the server that we want to redeploy
# if user executes the 'redeploy_server' dag # if user executes the 'redeploy_server' dag
@ -96,8 +73,7 @@ class PromenadeBaseOperator(BaseOperator):
'server-name') 'server-name')
if self.redeploy_server: if self.redeploy_server:
logging.info("Server to be redeployed is %s", LOG.info("Server to be redeployed is %s", self.redeploy_server)
self.redeploy_server)
else: else:
raise AirflowException('%s was unable to retrieve the ' raise AirflowException('%s was unable to retrieve the '
'server to be redeployed.' 'server to be redeployed.'
@ -107,8 +83,7 @@ class PromenadeBaseOperator(BaseOperator):
self.promenade_svc_endpoint = ucp_service_endpoint( self.promenade_svc_endpoint = ucp_service_endpoint(
self, svc_type=self.promenade_svc_type) self, svc_type=self.promenade_svc_type)
logging.info("Promenade endpoint is %s", LOG.info("Promenade endpoint is %s", self.promenade_svc_endpoint)
self.promenade_svc_endpoint)
class PromenadeBaseOperatorPlugin(AirflowPlugin): class PromenadeBaseOperatorPlugin(AirflowPlugin):

7
shipyard_airflow/plugins/promenade_check_etcd.py
View File

@ -11,7 +11,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import logging import logging
import time import time
@ -20,6 +19,8 @@ from airflow.exceptions import AirflowException
from promenade_base_operator import PromenadeBaseOperator from promenade_base_operator import PromenadeBaseOperator
LOG = logging.getLogger(__name__)
class PromenadeCheckEtcdOperator(PromenadeBaseOperator): class PromenadeCheckEtcdOperator(PromenadeBaseOperator):
@ -37,13 +38,13 @@ class PromenadeCheckEtcdOperator(PromenadeBaseOperator):
# TODO(bryan-strassner) use: # TODO(bryan-strassner) use:
# self.dc['kubernetes_provisioner.etcd_ready_timeout'] # self.dc['kubernetes_provisioner.etcd_ready_timeout']
# self.dc['kubernetes_provisioner.remove_etcd_timeout'] # self.dc['kubernetes_provisioner.remove_etcd_timeout']
logging.info("Performing health check on etcd...") LOG.info("Performing health check on etcd...")
time.sleep(5) time.sleep(5)
check_etcd = True check_etcd = True
if check_etcd: if check_etcd:
logging.info("The etcd cluster is healthy and ready") LOG.info("The etcd cluster is healthy and ready")
else: else:
raise AirflowException('Please check the state of etcd!') raise AirflowException('Please check the state of etcd!')

9
shipyard_airflow/plugins/promenade_clear_labels.py
View File

@ -11,7 +11,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import logging import logging
import time import time
@ -20,6 +19,8 @@ from airflow.exceptions import AirflowException
from promenade_base_operator import PromenadeBaseOperator from promenade_base_operator import PromenadeBaseOperator
LOG = logging.getLogger(__name__)
class PromenadeClearLabelsOperator(PromenadeBaseOperator): class PromenadeClearLabelsOperator(PromenadeBaseOperator):
@ -37,14 +38,14 @@ class PromenadeClearLabelsOperator(PromenadeBaseOperator):
# TODO(bryan-strassner) use: # TODO(bryan-strassner) use:
# self.dc['kubernetes_provisioner.clear_labels_timeout'] # self.dc['kubernetes_provisioner.clear_labels_timeout']
logging.info("Removing labels on node...") LOG.info("Removing labels on node...")
time.sleep(5) time.sleep(5)
labels_removed = True labels_removed = True
if labels_removed: if labels_removed:
logging.info("Successfully removed labels on %s", LOG.info("Successfully removed labels on %s",
self.redeploy_server) self.redeploy_server)
else: else:
raise AirflowException('Failed to remove labels on %s!', raise AirflowException('Failed to remove labels on %s!',
self.redeploy_server) self.redeploy_server)

9
shipyard_airflow/plugins/promenade_decommission_node.py
View File

@ -11,7 +11,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import logging import logging
import time import time
@ -20,6 +19,8 @@ from airflow.exceptions import AirflowException
from promenade_base_operator import PromenadeBaseOperator from promenade_base_operator import PromenadeBaseOperator
LOG = logging.getLogger(__name__)
class PromenadeDecommissionNodeOperator(PromenadeBaseOperator): class PromenadeDecommissionNodeOperator(PromenadeBaseOperator):
@ -33,14 +34,14 @@ class PromenadeDecommissionNodeOperator(PromenadeBaseOperator):
def do_execute(self): def do_execute(self):
# Placeholder function. Updates will be made when the Promenade # Placeholder function. Updates will be made when the Promenade
# API is ready for consumption. # API is ready for consumption.
logging.info("Decommissioning node from Kubernetes cluster...") LOG.info("Decommissioning node from Kubernetes cluster...")
time.sleep(5) time.sleep(5)
decommission_node = True decommission_node = True
if decommission_node: if decommission_node:
logging.info("Succesfully decommissioned node %s", LOG.info("Succesfully decommissioned node %s",
self.redeploy_server) self.redeploy_server)
else: else:
raise AirflowException('Failed to decommission node %s!', raise AirflowException('Failed to decommission node %s!',
self.redeploy_server) self.redeploy_server)

9
shipyard_airflow/plugins/promenade_drain_node.py
View File

@ -11,7 +11,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import logging import logging
import time import time
@ -20,6 +19,8 @@ from airflow.exceptions import AirflowException
from promenade_base_operator import PromenadeBaseOperator from promenade_base_operator import PromenadeBaseOperator
LOG = logging.getLogger(__name__)
class PromenadeDrainNodeOperator(PromenadeBaseOperator): class PromenadeDrainNodeOperator(PromenadeBaseOperator):
@ -40,14 +41,14 @@ class PromenadeDrainNodeOperator(PromenadeBaseOperator):
# self.dc['kubernetes_provisioner.drain_timeout'] # self.dc['kubernetes_provisioner.drain_timeout']
# self.dc['kubernetes_provisioner.drain_grace_period'] # self.dc['kubernetes_provisioner.drain_grace_period']
logging.info("Draining node...") LOG.info("Draining node...")
time.sleep(5) time.sleep(5)
node_drained = True node_drained = True
if node_drained: if node_drained:
logging.info("Node %s has been successfully drained", LOG.info("Node %s has been successfully drained",
self.redeploy_server) self.redeploy_server)
else: else:
raise AirflowException('Failed to drain %s!', raise AirflowException('Failed to drain %s!',
self.redeploy_server) self.redeploy_server)

9
shipyard_airflow/plugins/promenade_shutdown_kubelet.py
View File

@ -11,7 +11,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import logging import logging
import time import time
@ -20,6 +19,8 @@ from airflow.exceptions import AirflowException
from promenade_base_operator import PromenadeBaseOperator from promenade_base_operator import PromenadeBaseOperator
LOG = logging.getLogger(__name__)
class PromenadeShutdownKubeletOperator(PromenadeBaseOperator): class PromenadeShutdownKubeletOperator(PromenadeBaseOperator):
@ -33,14 +34,14 @@ class PromenadeShutdownKubeletOperator(PromenadeBaseOperator):
def do_execute(self): def do_execute(self):
# Placeholder function. Updates will be made when the Promenade # Placeholder function. Updates will be made when the Promenade
# API is ready for consumption. # API is ready for consumption.
logging.info("Shutting down kubelet on node...") LOG.info("Shutting down kubelet on node...")
time.sleep(5) time.sleep(5)
shutdown_kubelet = True shutdown_kubelet = True
if shutdown_kubelet: if shutdown_kubelet:
logging.info("Successfully shut down kubelet on %s", LOG.info("Successfully shut down kubelet on %s",
self.redeploy_server) self.redeploy_server)
else: else:
raise AirflowException('Failed to shut down kubelet on %s!', raise AirflowException('Failed to shut down kubelet on %s!',
self.redeploy_server) self.redeploy_server)

9
shipyard_airflow/plugins/ucp_base_operator.py
View File

@ -86,6 +86,11 @@ class UcpBaseOperator(BaseOperator):
# Exeute child function # Exeute child function
self.do_execute() self.do_execute()
# Push last committed version to xcom for the
# 'get_design_version' subdag
if self.sub_dag_name == 'get_design_version':
return self.committed_ver
def ucp_base(self, context): def ucp_base(self, context):
LOG.info("Running UCP Base Operator...") LOG.info("Running UCP Base Operator...")
@ -98,10 +103,10 @@ class UcpBaseOperator(BaseOperator):
self.ucp_namespace = config.get('k8s_logs', 'ucp_namespace') self.ucp_namespace = config.get('k8s_logs', 'ucp_namespace')
# Define task_instance # Define task_instance
task_instance = context['task_instance'] self.task_instance = context['task_instance']
# Set up and retrieve values from xcom # Set up and retrieve values from xcom
self.xcom_puller = XcomPuller(self.main_dag_name, task_instance) self.xcom_puller = XcomPuller(self.main_dag_name, self.task_instance)
self.action_info = self.xcom_puller.get_action_info() self.action_info = self.xcom_puller.get_action_info()
self.dc = self.xcom_puller.get_deployment_configuration() self.dc = self.xcom_puller.get_deployment_configuration()