From dacedae17b71acd249dc01ad0540e58136245fe1 Mon Sep 17 00:00:00 2001 From: Sergiy Markin Date: Mon, 16 Dec 2024 21:03:09 +0000 Subject: [PATCH] Airflow and Openstack modules upgrade This PS upgrades Airflow to 2.10.4 and Openstack modules to 2024.1 Caracal versions Change-Id: I577520cddaa73f9e1441922ccd67f75a5f1c3908 --- .zuul.yaml | 2 +- ChangeLog | 5 + charts/deckhand/Chart.yaml | 2 +- deckhand/common/utils.py | 13 +++ deckhand/control/buckets.py | 5 +- deckhand/control/revision_deepdiffing.py | 3 +- deckhand/control/revision_diffing.py | 4 +- deckhand/control/revision_documents.py | 6 +- deckhand/control/revision_tags.py | 8 +- deckhand/control/revisions.py | 5 +- deckhand/control/rollback.py | 4 +- deckhand/control/validations.py | 8 +- deckhand/control/versions.py | 4 +- deckhand/errors.py | 4 +- requirements-direct.txt | 46 ++++----- requirements-frozen.txt | 115 +++++++++++------------ tools/whitespace-linter.sh | 1 + tox.ini | 4 +- 18 files changed, 137 insertions(+), 102 deletions(-) diff --git a/.zuul.yaml b/.zuul.yaml index 216956ce..30bcbd35 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -313,7 +313,7 @@ commit: true static: - latest - - airflow_2.10.2 + - airflow_2.10.4 - job: name: deckhand-docker-tag-ubuntu_jammy diff --git a/ChangeLog b/ChangeLog index fa54fb58..c405470f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,6 +1,11 @@ CHANGES ======= +* Airflow and Openstack modules upgrade +* Remove ubuntu\_focal as image alias +* Bump up docker Pypi module version +* Airflow 2.10.2 fixes +* Airflow 2.10.2 + ubuntu\_jammy * Update deploy-env parameters * Kubeadm based Airskiff gate * Fix deckhand-api dependences diff --git a/charts/deckhand/Chart.yaml b/charts/deckhand/Chart.yaml index ad0a0871..ca8bef6f 100644 --- a/charts/deckhand/Chart.yaml +++ b/charts/deckhand/Chart.yaml @@ -15,7 +15,7 @@ apiVersion: v1 description: A Helm chart for Deckhand name: deckhand -version: 0.2.2 +version: 0.2.3 appVersion: 1.1.0 keywords: - deckhand diff --git a/deckhand/common/utils.py b/deckhand/common/utils.py index 4854584e..77633f9e 100644 --- a/deckhand/common/utils.py +++ b/deckhand/common/utils.py @@ -17,6 +17,7 @@ import copy import re import six import string +import yaml from beaker.cache import CacheManager from beaker.util import parse_cache_config_options @@ -41,6 +42,18 @@ _CACHE = CacheManager(**parse_cache_config_options(_CACHE_OPTS)) _ARRAY_RE = re.compile(r'.*\[\d+\].*') +def safe_yaml_dump(data): + """ + Automatically handle the difference between + yaml.safe_dump and yaml.safe_dump_all + to avoid errors when dumping YAML. + """ + if isinstance(data, (list, tuple)): # Multiple documents + return yaml.safe_dump_all(data) + else: # Single document + return yaml.safe_dump(data) + + def to_camel_case(s): """Convert string to camel case.""" return (s[0].lower() + string.capwords(s, sep='_') diff --git a/deckhand/control/buckets.py b/deckhand/control/buckets.py index 17279ca3..2fcd6e99 100644 --- a/deckhand/control/buckets.py +++ b/deckhand/control/buckets.py @@ -13,9 +13,11 @@ # limitations under the License. import falcon + from oslo_log import log as logging from oslo_utils import excutils +from deckhand.common import utils from deckhand.common import document as document_wrapper from deckhand.control import base as api_base from deckhand.control.views import document as document_view @@ -65,7 +67,8 @@ class BucketsResource(api_base.BaseResource): created_documents = self._create_revision_documents( bucket_name, documents) - resp.text = self.view_builder.list(created_documents) + resp.text = utils.safe_yaml_dump( + self.view_builder.list(created_documents)) resp.status = falcon.HTTP_200 def _encrypt_secret_documents(self, documents): diff --git a/deckhand/control/revision_deepdiffing.py b/deckhand/control/revision_deepdiffing.py index 8e4824d0..a815e9de 100644 --- a/deckhand/control/revision_deepdiffing.py +++ b/deckhand/control/revision_deepdiffing.py @@ -18,6 +18,7 @@ import six from oslo_log import log as logging from oslo_utils import excutils +from deckhand.common import utils from deckhand.control import base as api_base from deckhand.engine.revision_diff import revision_diff from deckhand import errors @@ -51,4 +52,4 @@ class RevisionDeepDiffingResource(api_base.BaseResource): LOG.exception(message) resp.status = falcon.HTTP_200 - resp.text = resp_body + resp.text = utils.safe_yaml_dump(resp_body) diff --git a/deckhand/control/revision_diffing.py b/deckhand/control/revision_diffing.py index e25685d1..06c1124a 100644 --- a/deckhand/control/revision_diffing.py +++ b/deckhand/control/revision_diffing.py @@ -13,10 +13,12 @@ # limitations under the License. import falcon + import six from oslo_log import log as logging from oslo_utils import excutils +from deckhand.common import utils from deckhand.control import base as api_base from deckhand.engine.revision_diff import revision_diff from deckhand import errors @@ -50,4 +52,4 @@ class RevisionDiffingResource(api_base.BaseResource): LOG.exception(message) resp.status = falcon.HTTP_200 - resp.text = resp_body + resp.text = utils.safe_yaml_dump(resp_body) diff --git a/deckhand/control/revision_documents.py b/deckhand/control/revision_documents.py index e08fe233..99a90da1 100644 --- a/deckhand/control/revision_documents.py +++ b/deckhand/control/revision_documents.py @@ -13,6 +13,7 @@ # limitations under the License. import falcon + from oslo_log import log as logging import six @@ -81,7 +82,7 @@ class RevisionDocumentsResource(api_base.BaseResource): documents = documents[:limit] resp.status = falcon.HTTP_200 - resp.text = self.view_builder.list(documents) + resp.text = utils.safe_yaml_dump(self.view_builder.list(documents)) class RenderedDocumentsResource(api_base.BaseResource): @@ -156,4 +157,5 @@ class RenderedDocumentsResource(api_base.BaseResource): rendered_documents = rendered_documents[:limit] resp.status = falcon.HTTP_200 - resp.text = self.view_builder.list(rendered_documents) + resp.text = utils.safe_yaml_dump( + self.view_builder.list(rendered_documents)) diff --git a/deckhand/control/revision_tags.py b/deckhand/control/revision_tags.py index 3a159923..7e5615ef 100644 --- a/deckhand/control/revision_tags.py +++ b/deckhand/control/revision_tags.py @@ -13,9 +13,11 @@ # limitations under the License. import falcon + from oslo_log import log as logging from oslo_utils import excutils +from deckhand.common import utils from deckhand.control import base as api_base from deckhand.control.views import revision_tag as revision_tag_view from deckhand.db.sqlalchemy import api as db_api @@ -44,7 +46,7 @@ class RevisionTagsResource(api_base.BaseResource): resp_body = revision_tag_view.ViewBuilder().show(resp_tag) resp.status = falcon.HTTP_201 - resp.text = resp_body + resp.text = utils.safe_yaml_dump(resp_body) def on_get(self, req, resp, revision_id, tag=None): """Show tag details or list all tags for a revision.""" @@ -66,7 +68,7 @@ class RevisionTagsResource(api_base.BaseResource): resp_body = revision_tag_view.ViewBuilder().show(resp_tag) resp.status = falcon.HTTP_200 - resp.text = resp_body + resp.text = utils.safe_yaml_dump(resp_body) @policy.authorize('deckhand:list_tags') def _list_all_tags(self, req, resp, revision_id): @@ -80,7 +82,7 @@ class RevisionTagsResource(api_base.BaseResource): resp_body = revision_tag_view.ViewBuilder().list(resp_tags) resp.status = falcon.HTTP_200 - resp.text = resp_body + resp.text = utils.safe_yaml_dump(resp_body) def on_delete(self, req, resp, revision_id, tag=None): """Deletes a single tag or deletes all tags for a revision.""" diff --git a/deckhand/control/revisions.py b/deckhand/control/revisions.py index 702cc434..df99337d 100644 --- a/deckhand/control/revisions.py +++ b/deckhand/control/revisions.py @@ -13,6 +13,7 @@ # limitations under the License. import falcon + from oslo_log import log as logging from oslo_utils import excutils @@ -61,7 +62,7 @@ class RevisionsResource(api_base.BaseResource): revision_resp = self.view_builder.show(revision) resp.status = falcon.HTTP_200 - resp.text = revision_resp + resp.text = utils.safe_yaml_dump(revision_resp) @policy.authorize('deckhand:list_revisions') @common.sanitize_params(['tag', 'order', 'sort']) @@ -74,7 +75,7 @@ class RevisionsResource(api_base.BaseResource): revisions = utils.multisort(revisions, sort_by, order_by) resp.status = falcon.HTTP_200 - resp.text = self.view_builder.list(revisions) + resp.text = utils.safe_yaml_dump(self.view_builder.list(revisions)) def _delete_all_barbican_secrets(self): filters = {'metadata.storagePolicy': 'encrypted'} diff --git a/deckhand/control/rollback.py b/deckhand/control/rollback.py index b1408cd2..d9b24f69 100644 --- a/deckhand/control/rollback.py +++ b/deckhand/control/rollback.py @@ -13,9 +13,11 @@ # limitations under the License. import falcon + from oslo_log import log as logging from oslo_utils import excutils +from deckhand.common import utils from deckhand.control import base as api_base from deckhand.control.views import revision as revision_view from deckhand.db.sqlalchemy import api as db_api @@ -55,4 +57,4 @@ class RollbackResource(api_base.BaseResource): revision_resp = self.view_builder.show(rollback_revision) resp.status = falcon.HTTP_201 - resp.text = revision_resp + resp.text = utils.safe_yaml_dump(revision_resp) diff --git a/deckhand/control/validations.py b/deckhand/control/validations.py index 6d6a4891..d5c96456 100644 --- a/deckhand/control/validations.py +++ b/deckhand/control/validations.py @@ -13,9 +13,11 @@ # limitations under the License. import falcon + from oslo_log import log as logging from oslo_utils import excutils +from deckhand.common import utils from deckhand.control import base as api_base from deckhand.control.views import validation as validation_view from deckhand.db.sqlalchemy import api as db_api @@ -50,7 +52,7 @@ class ValidationsResource(api_base.BaseResource): LOG.exception(message) resp.status = falcon.HTTP_201 - resp.text = self.view_builder.show(resp_body) + resp.text = utils.safe_yaml_dump(self.view_builder.show(resp_body)) def on_get(self, req, resp, revision_id, validation_name=None, entry_id=None): @@ -64,7 +66,7 @@ class ValidationsResource(api_base.BaseResource): resp_body = self._list_all_validations(req, resp, revision_id) resp.status = falcon.HTTP_200 - resp.text = resp_body + resp.text = utils.safe_yaml_dump(resp_body) @policy.authorize('deckhand:show_validation') def _show_validation_entry(self, req, resp, revision_id, validation_name, @@ -128,4 +130,4 @@ class ValidationsDetailsResource(api_base.BaseResource): raise falcon.HTTPNotFound(description=e.format_message()) resp.status = falcon.HTTP_200 - resp.text = self.view_builder.detail(entries) + resp.text = utils.safe_yaml_dump(self.view_builder.detail(entries)) diff --git a/deckhand/control/versions.py b/deckhand/control/versions.py index da3db46e..30986074 100644 --- a/deckhand/control/versions.py +++ b/deckhand/control/versions.py @@ -14,6 +14,7 @@ import falcon +from deckhand.common import utils from deckhand.control import base as api_base @@ -26,10 +27,11 @@ class VersionsResource(api_base.BaseResource): no_authentication_methods = ['GET'] def on_get(self, req, resp): - resp.text = { + resp_body = { 'v1.0': { 'path': '/api/v1.0', 'status': 'stable' } } + resp.text = utils.safe_yaml_dump(resp_body) resp.status = falcon.HTTP_200 diff --git a/deckhand/errors.py b/deckhand/errors.py index a9e0bbdd..7bbfb7ee 100644 --- a/deckhand/errors.py +++ b/deckhand/errors.py @@ -15,7 +15,7 @@ import falcon from oslo_log import log as logging import six -import yaml +from deckhand.common import utils LOG = logging.getLogger(__name__) @@ -102,7 +102,7 @@ def format_error_resp(req, } resp.status = status_code - resp.text = yaml.safe_dump(error_response) + resp.text = utils.safe_yaml_dump(error_response) def default_exception_handler(req, resp, ex, params): diff --git a/requirements-direct.txt b/requirements-direct.txt index ba96fe6b..524b8cc0 100644 --- a/requirements-direct.txt +++ b/requirements-direct.txt @@ -18,29 +18,29 @@ psycopg2-binary uwsgi xattr==0.10.1 -# Openstack Antelope 2023.1 -# https://releases.openstack.org/antelope/index.html -barbican==16.0.0 +# Openstack Caracal 2024.1 +# https://releases.openstack.org/caracal/index.html +barbican==18.0.0 -python-barbicanclient==5.5.0 -python-keystoneclient==5.1.0 +python-barbicanclient==5.7.0 +python-keystoneclient==5.4.0 -keystoneauth1==5.1.2 -keystonemiddleware==10.2.0 +keystoneauth1==5.6.0 +keystonemiddleware==10.6.0 -oslo.cache==3.3.1 -oslo.concurrency==5.1.1 -oslo.config==9.1.1 -oslo.context==5.1.1 -oslo.db==12.3.1 -oslo.i18n==6.0.0 -oslo.log==5.2.0 -oslo.messaging==14.2.4 -oslo.metrics==0.6.0 -oslo.middleware==5.1.1 -oslo.policy==4.1.1 -oslo.serialization==5.1.1 -oslo.service==3.1.1 -oslo.upgradecheck==2.1.1 -oslo.utils==6.1.0 -oslo.versionedobjects==3.1.0 \ No newline at end of file +oslo.cache==3.7.0 +oslo.concurrency==6.0.0 +oslo.config==9.4.0 +oslo.context==5.5.0 +oslo.db==15.0.0 +oslo.i18n==6.3.0 +oslo.log==5.5.1 +oslo.messaging==14.7.2 +oslo.metrics==0.8.0 +oslo.middleware==6.1.0 +oslo.policy==4.3.0 +oslo.serialization==5.4.1 +oslo.service==3.4.1 +oslo.upgradecheck==2.3.0 +oslo.utils==7.1.0 +oslo.versionedobjects==3.3.0 \ No newline at end of file diff --git a/requirements-frozen.txt b/requirements-frozen.txt index da87b596..39a93807 100644 --- a/requirements-frozen.txt +++ b/requirements-frozen.txt @@ -1,69 +1,69 @@ -alembic==1.13.2 -amqp==5.2.0 +alembic==1.14.0 +amqp==5.3.1 attrs==24.2.0 autopage==0.5.2 -barbican==16.0.0 -bcrypt==4.2.0 +barbican==18.0.0 +bcrypt==4.2.1 Beaker==1.13.0 cachetools==5.5.0 -castellan==5.1.1 +castellan==5.2.0 certifi==2024.8.30 cffi==1.17.1 -charset-normalizer==3.3.2 -cliff==4.7.0 -cmd2==2.4.3 +charset-normalizer==3.4.0 +cliff==4.8.0 +cmd2==2.5.8 cryptography==42.0.8 debtcollector==3.0.0 decorator==5.1.1 -deepdiff==8.0.1 -dnspython==2.6.1 +deepdiff==8.1.1 +dnspython==2.7.0 dogpile.cache==1.3.3 -eventlet==0.37.0 -falcon==3.1.3 +eventlet==0.38.1 +falcon==4.0.2 fasteners==0.19 fixtures==4.1.0 futurist==3.0.0 -greenlet==3.1.0 +greenlet==3.1.1 html5lib==0.9999999 httpexceptor==1.4.0 idna==3.10 iso8601==2.1.0 Jinja2==3.1.4 -jsonpath-ng==1.6.1 -jsonpickle==3.3.0 +jsonpath-ng==1.7.0 +jsonpickle==3.4.2 jsonschema==4.23.0 jsonschema-specifications==2023.12.1 -keystoneauth1==5.1.2 -keystonemiddleware==10.2.0 -kombu==5.4.1 +keystoneauth1==5.6.0 +keystonemiddleware==10.6.0 +kombu==5.4.2 ldap3==2.9.1 logutils==0.3.5 -Mako==1.3.5 -MarkupSafe==2.1.5 +Mako==1.3.8 +MarkupSafe==3.0.2 microversion-parse==2.0.0 msgpack==1.1.0 netaddr==1.3.0 netifaces==0.11.0 -networkx==3.3 -orderly-set==5.2.2 +networkx==3.4.2 +orderly-set==5.2.3 os-service-types==1.7.0 -oslo.cache==3.3.1 -oslo.concurrency==5.1.1 -oslo.config==9.1.1 -oslo.context==5.1.1 -oslo.db==12.3.1 -oslo.i18n==6.0.0 -oslo.log==5.2.0 -oslo.messaging==14.2.4 -oslo.metrics==0.6.0 -oslo.middleware==5.1.1 -oslo.policy==4.1.1 -oslo.serialization==5.1.1 -oslo.service==3.1.1 -oslo.upgradecheck==2.1.1 -oslo.utils==6.1.0 -oslo.versionedobjects==3.1.0 -packaging==24.1 +oslo.cache==3.7.0 +oslo.concurrency==6.0.0 +oslo.config==9.4.0 +oslo.context==5.5.0 +oslo.db==15.0.0 +oslo.i18n==6.3.0 +oslo.log==5.5.1 +oslo.messaging==14.7.2 +oslo.metrics==0.8.0 +oslo.middleware==6.1.0 +oslo.policy==4.3.0 +oslo.serialization==5.4.1 +oslo.service==3.4.1 +oslo.upgradecheck==2.3.0 +oslo.utils==7.1.0 +oslo.versionedobjects==3.3.0 +packaging==24.2 Paste==3.10.1 PasteDeploy==3.1.0 PasteScript==3.6.0 @@ -71,19 +71,20 @@ pbr==6.1.0 pecan==1.5.1 pip==24.1 ply==3.11 -prettytable==3.11.0 -prometheus_client==0.20.0 -psycopg2-binary==2.9.9 +prettytable==3.12.0 +prometheus_client==0.21.1 +psycopg2-binary==2.9.10 pyasn1==0.6.1 -pycadf==3.1.1 +pycadf==4.0.0 pycparser==2.22 +PyJWT==2.10.1 pylibyaml==0.1.0 -pyOpenSSL==24.2.1 -pyparsing==3.1.4 +pyOpenSSL==24.3.0 +pyparsing==3.2.0 pyperclip==1.9.0 -python-barbicanclient==5.5.0 +python-barbicanclient==5.7.0 python-dateutil==2.9.0.post0 -python-keystoneclient==5.1.0 +python-keystoneclient==5.4.0 python-memcached==1.62 python-mimeparse==2.0.0 pytz==2024.2 @@ -94,29 +95,27 @@ requests==2.32.3 resolver==0.2.1 rfc3986==2.0.0 Routes==2.5.1 -rpds-py==0.20.0 +rpds-py==0.22.3 selector==0.10.1 setuptools==70.1.0 simplejson==3.19.3 -six==1.16.0 +six==1.17.0 SQLAlchemy==1.4.54 -sqlalchemy-migrate==0.13.0 -sqlparse==0.5.1 statsd==4.0.1 -stevedore==5.3.0 -Tempita==0.5.2 +stevedore==5.4.0 testresources==2.0.1 testscenarios==0.5.0 testtools==2.7.2 tiddlyweb==2.4.3 typing_extensions==4.12.2 -urllib3==2.2.2 -uWSGI==2.0.27 +tzdata==2024.2 +urllib3==2.2.3 +uWSGI==2.0.28 vine==5.1.0 wcwidth==0.2.13 -WebOb==1.8.8 +WebOb==1.8.9 Werkzeug==2.2.3 wheel==0.43.0 -wrapt==1.16.0 +wrapt==1.17.0 xattr==0.10.1 -yappi==1.6.0 +yappi==1.6.10 diff --git a/tools/whitespace-linter.sh b/tools/whitespace-linter.sh index bfeeb560..a2ced47f 100755 --- a/tools/whitespace-linter.sh +++ b/tools/whitespace-linter.sh @@ -15,6 +15,7 @@ RES=$(find . \ -not -name "*.html" \ -not -name "favicon_32.png" \ -not -name "*.pyc" \ + -not -path "*/cover/*" \ -type f -exec egrep -l " +$" {} \;) if [[ -n $RES ]]; then diff --git a/tox.ini b/tox.ini index 94c7f757..07661a18 100644 --- a/tox.ini +++ b/tox.ini @@ -34,7 +34,7 @@ passenv = deps = -r{toxinidir}/requirements-frozen.txt -r{toxinidir}/test-requirements.txt - -c https://raw.githubusercontent.com/apache/airflow/constraints-2.10.2/constraints-3.10.txt + -c https://raw.githubusercontent.com/apache/airflow/constraints-2.10.4/constraints-3.10.txt commands = find . -type f -name "*.pyc" -delete rm -Rf .testrepository/times.dbm @@ -54,7 +54,7 @@ allowlist_externals= sh deps= -r{toxinidir}/requirements-direct.txt - -c https://raw.githubusercontent.com/apache/airflow/constraints-2.10.2/constraints-3.10.txt + -c https://raw.githubusercontent.com/apache/airflow/constraints-2.10.4/constraints-3.10.txt commands= rm -f requirements-frozen.txt sh -c "pip freeze --all | grep -vE 'deckhand|pyinotify|pkg-resources==0.0.0' > requirements-frozen.txt"