3ac47328c3
The purpose of the document staging api is to provide a client of shipyard to ingest and manipulate documents within the UCP platform In support of the need to reach out to other services, this change also introduces the use of keystone for service discovery and connection to other UCP services. Change-Id: I6fa113f8786cad2884c0b791788a4ef40cd1a6b6
610 lines
23 KiB
Python
610 lines
23 KiB
Python
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
"""
|
|
Configdocs helper primarily masquerades as a layer in front of
|
|
Deckhand, providing a representation of a buffer and a committed
|
|
bucket for Shipyard
|
|
"""
|
|
import enum
|
|
import json
|
|
import logging
|
|
import threading
|
|
|
|
import falcon
|
|
from oslo_config import cfg
|
|
import requests
|
|
|
|
from shipyard_airflow.control.deckhand_client import (
|
|
DeckhandClient,
|
|
DeckhandPaths,
|
|
DeckhandRejectedInputError,
|
|
DeckhandResponseError,
|
|
DocumentExistsElsewhereError,
|
|
NoRevisionsExistError
|
|
)
|
|
from shipyard_airflow.control.service_endpoints import (
|
|
Endpoints,
|
|
get_endpoint,
|
|
get_token
|
|
)
|
|
from shipyard_airflow.errors import ApiError, AppError
|
|
|
|
CONF = cfg.CONF
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
# keys for the revision dict, and consistency of strings for committed
|
|
# and buffer.
|
|
COMMITTED = 'committed'
|
|
BUFFER = 'buffer'
|
|
LATEST = 'latest'
|
|
REVISION_COUNT = 'count'
|
|
|
|
# string for rollback_commit consistency
|
|
ROLLBACK_COMMIT = 'rollback_commit'
|
|
|
|
|
|
class BufferMode(enum.Enum):
|
|
"""
|
|
Enumeration of the valid values for BufferMode
|
|
"""
|
|
REJECTONCONTENTS = 'rejectoncontents'
|
|
APPEND = 'append'
|
|
REPLACE = 'replace'
|
|
|
|
|
|
class ConfigdocsHelper(object):
|
|
"""
|
|
ConfigdocsHelper provides a layer to represent the buffer and committed
|
|
versions of design documents.
|
|
A new configdocs_helper is intended to be used for each invocation of the
|
|
service.
|
|
"""
|
|
|
|
def __init__(self, context_marker):
|
|
"""
|
|
Sets up this Configdocs helper with the supplied
|
|
context marker
|
|
"""
|
|
self.deckhand = DeckhandClient(context_marker)
|
|
self.context_marker = context_marker
|
|
# The revision_dict indicates the revisions that are
|
|
# associated with the buffered and committed doc sets. There
|
|
# is a risk of this being out of sync if there is high volume
|
|
# of commits and adds of docs going on in parallel, but not
|
|
# really much more than if this data is used in subsequent
|
|
# statements following a call within a method.
|
|
self.revision_dict = None
|
|
|
|
@staticmethod
|
|
def get_buffer_mode(buffermode_string):
|
|
"""
|
|
Checks the buffer mode for valid values.
|
|
"""
|
|
if buffermode_string:
|
|
try:
|
|
buffer_mode = BufferMode(buffermode_string.lower())
|
|
return buffer_mode
|
|
except ValueError:
|
|
return None
|
|
return BufferMode.REJECTONCONTENTS
|
|
|
|
def is_buffer_empty(self):
|
|
""" Check if the buffer is empty. """
|
|
return self._get_buffer_revision() is None
|
|
|
|
def is_collection_in_buffer(self, collection_id):
|
|
"""
|
|
Returns if the collection is represented in the buffer
|
|
"""
|
|
if self.is_buffer_empty():
|
|
return False
|
|
|
|
# If there is no committed revision, then it's 0.
|
|
# new revision is ok because we just checked for buffer emptiness
|
|
old_revision_id = self._get_committed_rev_id()
|
|
if old_revision_id is None:
|
|
old_revision_id = 0
|
|
try:
|
|
diff = self.deckhand.get_diff(
|
|
old_revision_id=old_revision_id,
|
|
new_revision_id=self._get_buffer_rev_id()
|
|
)
|
|
# the collection is in the buffer if it's not unmodified
|
|
return diff.get(collection_id, 'unmodified') != 'unmodified'
|
|
except DeckhandResponseError as drex:
|
|
raise AppError(
|
|
title='Unable to retrieve revisions',
|
|
description=(
|
|
'Deckhand has responded unexpectedly: {}:{}'.format(
|
|
drex.status_code,
|
|
drex.response_message
|
|
)
|
|
),
|
|
status=falcon.HTTP_500,
|
|
retry=False,
|
|
)
|
|
|
|
def is_buffer_valid_for_bucket(self, collection_id, buffermode):
|
|
"""
|
|
Indicates if the buffer as it currently is, may be written to
|
|
for the specified collection, based on the buffermode.
|
|
"""
|
|
# can always write if buffer is empty.
|
|
if self.is_buffer_empty():
|
|
return True
|
|
# from here down, the buffer is NOT empty.
|
|
# determine steps by the buffermode
|
|
if buffermode == BufferMode.REJECTONCONTENTS:
|
|
return False
|
|
elif buffermode == BufferMode.APPEND:
|
|
return not self.is_collection_in_buffer(collection_id)
|
|
# replace the buffer with last commit.
|
|
elif buffermode == BufferMode.REPLACE:
|
|
committed_rev_id = None
|
|
committed_rev = self._get_committed_revision()
|
|
if committed_rev:
|
|
committed_rev_id = committed_rev['id']
|
|
if committed_rev_id is None:
|
|
# TODO (bryan-strassner) use rollback to 0 if/when that
|
|
# is implemented in deckhand.
|
|
# reset_to_empty has side effect of deleting history
|
|
# from deckhand although it is only the corner case
|
|
# where no commit has ever been done.
|
|
self.deckhand.reset_to_empty()
|
|
else:
|
|
self.deckhand.rollback(committed_rev_id)
|
|
return True
|
|
|
|
def _get_revision_dict(self):
|
|
"""
|
|
Returns a dictionary with values representing the revisions in
|
|
Deckhand that Shipyard cares about - committed, buffer,
|
|
and latest, as well as a count of revisions
|
|
Committed and buffer are revisions associated with the
|
|
shipyard tags. If either of those are not present in deckhand,
|
|
returns None for the value.
|
|
Latest holds the revision information for the newest revision.
|
|
"""
|
|
# return the cached instance version of the revision dict.
|
|
if self.revision_dict is not None:
|
|
return self.revision_dict
|
|
# or generate one for the cache
|
|
committed_revision = None
|
|
buffer_revision = None
|
|
latest_revision = None
|
|
revision_count = 0
|
|
try:
|
|
revisions = self.deckhand.get_revision_list()
|
|
revision_count = len(revisions)
|
|
if revisions:
|
|
latest_revision = revisions[-1]
|
|
for revision in reversed(revisions):
|
|
tags = revision.get('tags', [])
|
|
if COMMITTED in tags or ROLLBACK_COMMIT in tags:
|
|
committed_revision = revision
|
|
break
|
|
else:
|
|
# there are buffer revisions, only grab it on
|
|
# the first pass through
|
|
# if the first revision is committed, or if there
|
|
# are no revsisions, buffer revsision stays None
|
|
if buffer_revision is None:
|
|
buffer_revision = revision
|
|
except NoRevisionsExistError:
|
|
# the values of None/None/None/0 are fine
|
|
pass
|
|
except DeckhandResponseError as drex:
|
|
raise AppError(
|
|
title='Unable to retrieve revisions',
|
|
description=(
|
|
'Deckhand has responded unexpectedly: {}:{}'.format(
|
|
drex.status_code,
|
|
drex.response_message
|
|
)
|
|
),
|
|
status=falcon.HTTP_500,
|
|
retry=False,
|
|
)
|
|
self.revision_dict = {
|
|
COMMITTED: committed_revision,
|
|
BUFFER: buffer_revision,
|
|
LATEST: latest_revision,
|
|
REVISION_COUNT: revision_count
|
|
}
|
|
return self.revision_dict
|
|
|
|
def _get_buffer_revision(self):
|
|
# convenience helper to drill down to Buffer revision
|
|
return self._get_revision_dict().get(BUFFER)
|
|
|
|
def _get_buffer_rev_id(self):
|
|
# convenience helper to drill down to Buffer revision id
|
|
buf_rev = self._get_revision_dict().get(BUFFER)
|
|
return buf_rev['id'] if buf_rev else None
|
|
|
|
def _get_latest_revision(self):
|
|
# convenience helper to drill down to latest revision
|
|
return self._get_revision_dict().get(LATEST)
|
|
|
|
def _get_latest_rev_id(self):
|
|
# convenience helper to drill down to latest revision id
|
|
latest_rev = self._get_revision_dict().get(LATEST)
|
|
return latest_rev['id'] if latest_rev else None
|
|
|
|
def _get_committed_revision(self):
|
|
# convenience helper to drill down to committed revision
|
|
return self._get_revision_dict().get(COMMITTED)
|
|
|
|
def _get_committed_rev_id(self):
|
|
# convenience helper to drill down to committed revision id
|
|
committed_rev = self._get_revision_dict().get(COMMITTED)
|
|
return committed_rev['id'] if committed_rev else None
|
|
|
|
def get_collection_docs(self, version, collection_id):
|
|
"""
|
|
Returns the requested collection of docs based on the version
|
|
specifier. Since the default is the buffer, only return
|
|
committed if explicitly stated. No need to further check the
|
|
parameter for validity here.
|
|
"""
|
|
LOG.info('Retrieving collection %s from %s', collection_id, version)
|
|
if version == COMMITTED:
|
|
return self._get_committed_docs(collection_id)
|
|
return self._get_doc_from_buffer(collection_id)
|
|
|
|
def _get_doc_from_buffer(self, collection_id):
|
|
"""
|
|
Returns the collection if it exists in the buffer.
|
|
If the buffer contains the collection, the latest
|
|
representation is what we want.
|
|
"""
|
|
# Need to guard with this check for buffer to ensure
|
|
# that the collection is not just carried through unmodified
|
|
# into the buffer, and is actually represented.
|
|
if self.is_collection_in_buffer(collection_id):
|
|
# prior check for collection in buffer means the buffer
|
|
# revision exists
|
|
buffer_id = self._get_buffer_rev_id()
|
|
return self.deckhand.get_docs_from_revision(
|
|
revision_id=buffer_id,
|
|
bucket_id=collection_id
|
|
)
|
|
raise ApiError(
|
|
title='No documents to retrieve',
|
|
description=('The Shipyard buffer is empty or does not contain '
|
|
'this collection'),
|
|
status=falcon.HTTP_404,
|
|
retry=False,
|
|
)
|
|
|
|
def _get_committed_docs(self, collection_id):
|
|
"""
|
|
Returns the collection if it exists as committed.
|
|
"""
|
|
committed_id = self._get_committed_rev_id()
|
|
if committed_id:
|
|
return self.deckhand.get_docs_from_revision(
|
|
revision_id=committed_id,
|
|
bucket_id=collection_id
|
|
)
|
|
# if there is no committed...
|
|
raise ApiError(
|
|
title='No documents to retrieve',
|
|
description='There is no committed version of this collection',
|
|
status=falcon.HTTP_404,
|
|
retry=False,
|
|
)
|
|
|
|
def get_rendered_configdocs(self, version=BUFFER):
|
|
"""
|
|
Returns the rendered configuration documents for the specified
|
|
revision (by name BUFFER, COMMITTED)
|
|
"""
|
|
revision_dict = self._get_revision_dict()
|
|
if version in (BUFFER, COMMITTED):
|
|
if revision_dict.get(version):
|
|
revision_id = revision_dict.get(version).get('id')
|
|
return self.deckhand.get_rendered_docs_from_revision(
|
|
revision_id=revision_id
|
|
)
|
|
else:
|
|
raise ApiError(
|
|
title='This revision does not exist',
|
|
description='{} version does not exist'.format(version),
|
|
status=falcon.HTTP_404,
|
|
retry=False,
|
|
)
|
|
|
|
def get_validations_for_buffer(self):
|
|
"""
|
|
Convenience method to do validations for buffer version.
|
|
"""
|
|
buffer_rev_id = self._get_buffer_rev_id()
|
|
if buffer_rev_id:
|
|
return self.get_validations_for_revision(buffer_rev_id)
|
|
raise AppError(
|
|
title='Unable to start validation of buffer',
|
|
description=('Buffer revision id could not be determined from'
|
|
'Deckhand'),
|
|
status=falcon.HTTP_500,
|
|
retry=False,
|
|
)
|
|
|
|
@staticmethod
|
|
def _get_design_reference(revision_id):
|
|
# Constructs the design reference as json for use by other components
|
|
design_reference = {
|
|
"rel": "design",
|
|
"href": "deckhand+{}/rendered-documents".format(
|
|
DeckhandPaths.RENDERED_REVISION_DOCS.value.format(revision_id)
|
|
),
|
|
"type": "application/x-yaml"
|
|
}
|
|
return json.dumps(design_reference)
|
|
|
|
@staticmethod
|
|
def _get_validation_endpoints():
|
|
# returns the list of validation endpoint supported
|
|
val_ep = '{}/validatedesign'
|
|
return [
|
|
{'name': 'Drydock',
|
|
'url': val_ep.format(get_endpoint(Endpoints.DRYDOCK))},
|
|
{'name': 'Armada',
|
|
'url': val_ep.format(get_endpoint(Endpoints.ARMADA))},
|
|
]
|
|
|
|
@staticmethod
|
|
def _get_validation_threads(validation_endpoints,
|
|
revision_id,
|
|
context_marker):
|
|
# create a list of validation threads from the endpoints
|
|
validation_threads = []
|
|
for endpoint in validation_endpoints:
|
|
# create a holder for things we need back from the threads
|
|
response = {'response': None}
|
|
exception = {'exception': None}
|
|
validation_threads.append(
|
|
{
|
|
'thread': threading.Thread(
|
|
target=ConfigdocsHelper._get_validations_for_component,
|
|
args=(
|
|
endpoint['url'],
|
|
ConfigdocsHelper._get_design_reference(
|
|
revision_id
|
|
),
|
|
response,
|
|
exception,
|
|
context_marker
|
|
)
|
|
),
|
|
'name': endpoint['name'],
|
|
'url': endpoint['url'],
|
|
'response': response,
|
|
'exception': exception
|
|
}
|
|
)
|
|
return validation_threads
|
|
|
|
@staticmethod
|
|
def _get_validations_for_component(url,
|
|
design_reference,
|
|
response,
|
|
exception,
|
|
context_marker):
|
|
# Invoke the POST for validation
|
|
try:
|
|
headers = {
|
|
'X-Context-Marker': context_marker,
|
|
'X-Auth-Token': get_token(),
|
|
'content-type': 'application/x-yaml'
|
|
}
|
|
|
|
http_resp = requests.post(url,
|
|
headers=headers,
|
|
data=design_reference,
|
|
timeout=(5, 30))
|
|
http_resp.raise_for_status()
|
|
raw_response = http_resp.decode('utf-8')
|
|
response_dict = json.loads(raw_response)
|
|
response['response'] = response_dict
|
|
except Exception as ex:
|
|
# catch anything exceptional as a failure to validate
|
|
LOG.error(ex)
|
|
exception['exception'] = ex
|
|
|
|
def get_validations_for_revision(self, revision_id):
|
|
"""
|
|
Use the endpoints for each of the UCP components to validate the
|
|
version indicated. Uses:
|
|
https://github.com/att-comdev/ucp-integration/blob/master/docs/api-conventions.md#post-v10validatedesign
|
|
format.
|
|
"""
|
|
resp_msgs = []
|
|
|
|
validation_threads = ConfigdocsHelper._get_validation_threads(
|
|
ConfigdocsHelper._get_validation_endpoints(),
|
|
revision_id,
|
|
self.context_marker
|
|
)
|
|
# trigger each validation in parallel
|
|
for validation_thread in validation_threads:
|
|
if validation_thread.get('thread'):
|
|
validation_thread.get('thread').start()
|
|
# wait for all validations to return
|
|
for validation_thread in validation_threads:
|
|
validation_thread.get('thread').join()
|
|
|
|
# check on the response, extract the validations
|
|
error_count = 0
|
|
for validation_thread in validation_threads:
|
|
val_response = validation_thread.get('response')['response']
|
|
if (not val_response or
|
|
validation_thread.get('exception')['exception']):
|
|
# exception was raised, or no body was returned.
|
|
raise AppError(
|
|
title='Unable to properly validate configdocs',
|
|
description=(
|
|
'Invocation of validation by {} has failed'.format(
|
|
validation_thread.get('name')
|
|
)
|
|
),
|
|
status=falcon.HTTP_500,
|
|
retry=False,
|
|
)
|
|
if not val_response:
|
|
raise AppError(
|
|
title='An invalid response was returned by validation',
|
|
description='No valid response status from {}'.format(
|
|
validation_thread.get('name')),
|
|
status=falcon.HTTP_500,
|
|
retry=False,
|
|
)
|
|
# invalid status needs collection of messages
|
|
# valid status means that it passed. No messages to collect
|
|
msg_list = val_response.get('details').get('messageList')
|
|
for msg in msg_list:
|
|
if msg.get('error'):
|
|
error_count = error_count + 1
|
|
resp_msgs.append(
|
|
{
|
|
'name': validation_thread.get('name'),
|
|
'message': msg.get('message'),
|
|
'error': True
|
|
}
|
|
)
|
|
else:
|
|
resp_msgs.append(
|
|
{
|
|
'name': validation_thread.get('name'),
|
|
'message': msg.get('message'),
|
|
'error': False
|
|
}
|
|
)
|
|
# Deckhand does it differently. Incorporate those validation
|
|
# failures
|
|
dh_validations = self._get_deckhand_validations(revision_id)
|
|
error_count += len(dh_validations)
|
|
resp_msgs.extend(dh_validations)
|
|
# return the formatted status response
|
|
return ConfigdocsHelper._format_validations_to_status(
|
|
resp_msgs,
|
|
error_count
|
|
)
|
|
|
|
def get_deckhand_validation_status(self, revision_id):
|
|
"""
|
|
Returns the status object representing the deckhand validation
|
|
results
|
|
"""
|
|
dh_validations = self._get_deckhand_validations(revision_id)
|
|
error_count = len(dh_validations)
|
|
return ConfigdocsHelper._format_validations_to_status(
|
|
dh_validations,
|
|
error_count
|
|
)
|
|
|
|
def _get_deckhand_validations(self, revision_id):
|
|
# Returns any validations that deckhand has on hand for this
|
|
# revision.
|
|
resp_msgs = []
|
|
deckhand_val = self.deckhand.get_all_revision_validations(revision_id)
|
|
if deckhand_val.get('results'):
|
|
for dh_result in deckhand_val.get('results'):
|
|
if dh_result.get('errors'):
|
|
for error in dh_result.get('errors'):
|
|
resp_msgs.append(
|
|
{
|
|
'name': dh_result.get('name'),
|
|
'message': error.get('message'),
|
|
'error': True
|
|
}
|
|
)
|
|
return resp_msgs
|
|
|
|
@staticmethod
|
|
def _format_validations_to_status(val_msgs, error_count):
|
|
# Using a list of validation messages and an error count,
|
|
# formulates and returns a status response dict
|
|
|
|
status = 'Valid'
|
|
message = 'Validations succeeded'
|
|
code = falcon.HTTP_200
|
|
if error_count > 0:
|
|
status = 'Invalid'
|
|
message = 'Validations failed'
|
|
code = falcon.HTTP_400
|
|
|
|
return {
|
|
"kind": "Status",
|
|
"apiVersion": "v1",
|
|
"metadata": {},
|
|
"status": status,
|
|
"message": message,
|
|
"reason": "Validation",
|
|
"details": {
|
|
"errorCount": error_count,
|
|
"messageList": val_msgs,
|
|
},
|
|
"code": code
|
|
}
|
|
|
|
def tag_buffer(self, tag):
|
|
"""
|
|
Convenience method to tag the buffer version.
|
|
"""
|
|
buffer_rev_id = self._get_buffer_rev_id()
|
|
if buffer_rev_id is None:
|
|
raise AppError(
|
|
title='Unable to tag buffer as {}'.format(tag),
|
|
description=('Buffer revision id could not be determined from'
|
|
'Deckhand'),
|
|
status=falcon.HTTP_500,
|
|
retry=False,
|
|
)
|
|
self.tag_revision(buffer_rev_id, tag)
|
|
|
|
def tag_revision(self, revision_id, tag):
|
|
"""
|
|
Tags the specified revision with the specified tag
|
|
"""
|
|
self.deckhand.tag_revision(revision_id=revision_id, tag=tag)
|
|
|
|
def add_collection(self, collection_id, document_string):
|
|
"""
|
|
Triggers a call to Deckhand to add a collection(bucket)
|
|
Documents are assumed to be a string input, not a
|
|
collection.
|
|
Returns the id of the buffer version.
|
|
"""
|
|
try:
|
|
self.deckhand.put_bucket(collection_id, document_string)
|
|
except DocumentExistsElsewhereError as deee:
|
|
LOG.info('Deckhand has rejected this input because an included '
|
|
'document exists in another bucket already')
|
|
raise ApiError(
|
|
title='Documents may not exist in more than one collection',
|
|
description=deee.response_message,
|
|
status=falcon.HTTP_409
|
|
)
|
|
except DeckhandRejectedInputError as drie:
|
|
LOG.info('Deckhand has rejected this input because: %s',
|
|
drie.response_message)
|
|
raise ApiError(
|
|
title="Document(s) invalid syntax or otherwise unsuitable",
|
|
description=drie.response_message,
|
|
)
|
|
# reset the revision dict so it regenerates.
|
|
self.revision_dict = None
|
|
return self._get_buffer_rev_id()
|