airship/shipyard
Code Issues Proposed changes

Update DryDock Operator

Browse Source 
It seems that there is a newly introduced bug in DryDock where
the result of a completed task is being reported as 'incomplete'.
This P.S is meant to update the DryDock Operator.

The DryDock Operator will be updated again when the bug is fixed in
DryDock.

The bug in DryDock is currently tracked by [0]

We will also add additional logic to check that all the nodes have
joined the cluster before declaring the 'deploy_node' task as completed

[0]: https://github.com/att-comdev/drydock/issues/73

Change-Id: I661995621520fa3d621623da41d984159e251edb
This commit is contained in:
Anthony Lin 2017-11-30 18:04:34 +00:00
parent 2c7cf10515
commit a61e324f10
1 changed files with 22 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
shipyard_airflow/plugins/drydock_operators.py
View File

@ -25,6 +25,7 @@ from airflow.utils.decorators import apply_defaults
import drydock_provisioner.drydock_client.client as client
import drydock_provisioner.drydock_client.session as session
from check_k8s_node_status import check_node_status
from service_endpoint import ucp_service_endpoint
from service_token import shipyard_service_token
@ -164,6 +165,11 @@ class DryDockOperator(BaseOperator):
self.drydock_action(drydock_client, context, self.action,
query_interval, task_timeout)
# Check that cluster join process is completed before declaring
# deploy_node as 'completed'. Set time out to 30 minutes and set
# polling interval to 30 seconds.
check_node_status(1800, 30)
# Do not perform any action
else:
logging.info('No Action to Perform')
@ -257,13 +263,16 @@ class DryDockOperator(BaseOperator):
# Query task status
for i in range(0, end_range + 1):
try:
# Retrieve current task state
task_state = drydock_client.get_task(task_id=task_id)
task_status = task_state.get('status')
task_results = task_state.get('result')['status']
task_result = task_state.get('result')['status']
logging.info("Current status of task id %s is %s",
task_id, task_status)
except:
logging.info("Unable to retrieve task state. Retrying...")
# Raise Time Out Exception
if task_status == 'running' and i == end_range:
@ -272,17 +281,18 @@ class DryDockOperator(BaseOperator):
# Exit 'for' loop if the task is in 'complete' or 'terminated'
# state
if task_status in ['complete', 'terminated']:
logging.info('Task result is %s', task_result)
break
else:
time.sleep(int(interval))
# Get final task state
# Get final task result
# NOTE: There is a known bug in Drydock where the task result
# for a successfully completed task can either be 'success' or
# 'partial success'. This will be fixed in Drydock in the near
# future. Updates will be made to the Drydock Operator once the
# bug is fixed.
if task_results in ['success', 'partial_success']:
# for a successfully completed task can either be 'success',
# 'partial success' or 'incomplete'. This will be fixed in Drydock
# in the near future. Updates will be made to the Drydock Operator
# once the bug is fixed.
if task_result in ['success', 'partial_success', 'incomplete']:
logging.info('Task id %s has been successfully completed',
self.task_id)
else: