From 026b41a920fd0c82173d99f2d52d53cd34e419ae Mon Sep 17 00:00:00 2001 From: Mehdi Abaakouk Date: Wed, 22 Mar 2017 17:09:51 +0100 Subject: [PATCH] tempest: rework gabbi setup The current approach is a bit hacky and create one tempest scenario per gabbi HTTP call. The side effect is that tests continue to run even the previous one have fail. This changes the approach by running gabbi to have one scenario per yaml file from tempest point of view. This will make easier to debug the scenario in case of failure. Change-Id: I594288322d9ac5d3d128d601cba1d2291a632e20 --- .../tests/integration/hooks/post_test_hook.sh | 43 ++---- .../tempest/scenario/test_autoscaling.py | 116 --------------- .../scenario/test_telemetry_integration.py | 134 ++++++++++++++++++ 3 files changed, 146 insertions(+), 147 deletions(-) delete mode 100644 ceilometer/tests/tempest/scenario/test_autoscaling.py create mode 100644 ceilometer/tests/tempest/scenario/test_telemetry_integration.py diff --git a/ceilometer/tests/integration/hooks/post_test_hook.sh b/ceilometer/tests/integration/hooks/post_test_hook.sh index 051b4d8c2c..1e35fd4f67 100755 --- a/ceilometer/tests/integration/hooks/post_test_hook.sh +++ b/ceilometer/tests/integration/hooks/post_test_hook.sh @@ -77,17 +77,13 @@ function generate_reports_and_maybe_exit() { } -# If we're running in the gate find our keystone endpoint to give to -# gabbi tests and do a chown. Otherwise the existing environment -# should provide URL and TOKEN. -if [ -d $BASE/new/devstack ]; then - export CEILOMETER_DIR="$BASE/new/ceilometer" - STACK_USER=stack - sudo chown -R $STACK_USER:stack $CEILOMETER_DIR - source $BASE/new/devstack/openrc admin admin - # Go to the ceilometer dir - cd $CEILOMETER_DIR -fi +export CEILOMETER_DIR="$BASE/new/ceilometer" +STACK_USER=stack +sudo chown -R $STACK_USER:stack $CEILOMETER_DIR +# NOTE(sileht): on swift job permissions are wrong, I don't known why +sudo chown -R tempest:stack $BASE/new/tempest +sudo chown -R tempest:stack $BASE/data/tempest +source $BASE/new/devstack/openrc admin admin openstack catalog list export AODH_SERVICE_URL=$(openstack catalog show alarming -c endpoints -f value | awk '/public/{print $2}') @@ -99,26 +95,11 @@ export GLANCE_IMAGE_NAME=$(openstack image list | awk '/ cirros.* /{print $4; ex export ADMIN_TOKEN=$(openstack token issue -c id -f value) export OS_AUTH_TYPE=password -# Run tests with gabbi -echo "Running telemetry integration test suite" -set +e -sudo -E -H -u ${STACK_USER:-${USER}} tox -eintegration +# Run tests with tempest +cd $BASE/new/tempest +sudo -H -u tempest OS_TEST_TIMEOUT=$TEMPEST_OS_TEST_TIMEOUT tox -eall-plugin -- ceilometer.tests.tempest.scenario.test_telemetry_integration --concurrency=$TEMPEST_CONCURRENCY EXIT_CODE=$? - -if [ -d $BASE/new/devstack ]; then - export_subunit_data "integration" - generate_reports_and_maybe_exit $EXIT_CODE - - # NOTE(sileht): on swift job permissions are wrong, I don't known why - sudo chown -R tempest:stack $BASE/new/tempest - sudo chown -R tempest:stack $BASE/data/tempest - - # Run tests with tempest - cd $BASE/new/tempest - sudo -H -u tempest OS_TEST_TIMEOUT=$TEMPEST_OS_TEST_TIMEOUT tox -eall-plugin -- ceilometer.tests.tempest.scenario.test_autoscaling --concurrency=$TEMPEST_CONCURRENCY - EXIT_CODE=$? - export_subunit_data "all-plugin" - generate_reports_and_maybe_exit $EXIT_CODE -fi +export_subunit_data "all-plugin" +generate_reports_and_maybe_exit $EXIT_CODE exit $EXIT_CODE diff --git a/ceilometer/tests/tempest/scenario/test_autoscaling.py b/ceilometer/tests/tempest/scenario/test_autoscaling.py deleted file mode 100644 index 4932ed31ea..0000000000 --- a/ceilometer/tests/tempest/scenario/test_autoscaling.py +++ /dev/null @@ -1,116 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import os -import unittest - -from gabbi import driver -from tempest import config - -from ceilometer.tests.tempest.scenario import manager - - -class TestAutoscalingGabbi(manager.ScenarioTest): - credentials = ['admin', 'primary'] - - @classmethod - def skip_checks(cls): - super(TestAutoscalingGabbi, cls).skip_checks() - for name in ["aodh_plugin", "gnocchi", "nova", "heat", "panko", - "ceilometer", "glance"]: - cls._check_service(name) - - @classmethod - def _check_service(cls, name): - if not getattr(config.CONF.service_available, name, False): - raise cls.skipException("%s support is required" % - name.capitalize()) - - @classmethod - def resource_setup(cls): - super(TestAutoscalingGabbi, cls).resource_setup() - test_dir = os.path.join(os.path.dirname(__file__), '..', '..', - 'integration', 'gabbi', 'gabbits-live') - cls.tests = driver.build_tests( - test_dir, unittest.TestLoader(), - host='localhost', port='13245', - test_loader_name='tempest.scenario.telemetry-autoscaling.test') - - auth = cls.os_admin.auth_provider.get_auth() - os.environ["ADMIN_TOKEN"] = auth[0] - os.environ["AODH_SERVICE_URL"] = cls._get_endpoint_for( - auth, "alarming_plugin") - os.environ["GNOCCHI_SERVICE_URL"] = cls._get_endpoint_for( - auth, "metric") - os.environ["PANKO_SERVICE_URL"] = cls._get_endpoint_for( - auth, "event") - os.environ["HEAT_SERVICE_URL"] = cls._get_endpoint_for( - auth, "orchestration") - os.environ["NOVA_SERVICE_URL"] = cls._get_endpoint_for(auth, "compute") - os.environ["GLANCE_SERVICE_URL"] = cls._get_endpoint_for(auth, "image") - - @staticmethod - def clear_credentials(): - # FIXME(sileht): We don't want the token to be invalided, but - # for some obcurs reason, clear_credentials is called before/during run - # So, make the one used by tearDropClass a dump, and call it manually - # in run() - pass - - def run(self, result=None): - self.setUp() - os.environ["GLANCE_IMAGE_NAME"] = self.glance_image_create() - try: - self.tests.run(result) - finally: - super(TestAutoscalingGabbi, self).clear_credentials() - self.tearDown() - - @staticmethod - def _get_endpoint_for(auth, service): - opt_section = getattr(config.CONF, service) - endpoint_type = opt_section.endpoint_type - is_keystone_v3 = 'catalog' in auth[1] - - if is_keystone_v3: - if endpoint_type.endswith("URL"): - endpoint_type = endpoint_type[:-3] - catalog = auth[1]['catalog'] - endpoints = [e['endpoints'] for e in catalog - if e['type'] == opt_section.catalog_type] - if not endpoints: - raise Exception("%s endpoint not found" % - opt_section.catalog_type) - endpoints = [e['url'] for e in endpoints[0] - if e['interface'] == endpoint_type] - if not endpoints: - raise Exception("%s interface not found for endpoint %s" % - (endpoint_type, - opt_section.catalog_type)) - return endpoints[0] - - else: - if not endpoint_type.endswith("URL"): - endpoint_type += "URL" - catalog = auth[1]['serviceCatalog'] - endpoints = [e for e in catalog - if e['type'] == opt_section.catalog_type] - if not endpoints: - raise Exception("%s endpoint not found" % - opt_section.catalog_type) - return endpoints[0]['endpoints'][0][endpoint_type] - - @staticmethod - def test_fake(): - # NOTE(sileht): A fake test is needed to have the class loaded - # by the test runner - pass diff --git a/ceilometer/tests/tempest/scenario/test_telemetry_integration.py b/ceilometer/tests/tempest/scenario/test_telemetry_integration.py new file mode 100644 index 0000000000..afede751d4 --- /dev/null +++ b/ceilometer/tests/tempest/scenario/test_telemetry_integration.py @@ -0,0 +1,134 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import unittest + +from gabbi import runner +from gabbi import suitemaker +from gabbi import utils +from tempest import config + +from ceilometer.tests.tempest.scenario import manager + +TEST_DIR = os.path.join(os.path.dirname(__file__), '..', '..', + 'integration', 'gabbi', 'gabbits-live') + + +class TestTelemetryIntegration(manager.ScenarioTest): + credentials = ['admin', 'primary'] + + @classmethod + def skip_checks(cls): + super(TestTelemetryIntegration, cls).skip_checks() + for name in ["aodh_plugin", "gnocchi", "nova", "heat", "panko", + "ceilometer", "glance"]: + cls._check_service(name) + + @classmethod + def _check_service(cls, name): + if not getattr(config.CONF.service_available, name, False): + raise cls.skipException("%s support is required" % + name.capitalize()) + + @staticmethod + def _get_endpoint(auth, service): + opt_section = getattr(config.CONF, service) + endpoint_type = opt_section.endpoint_type + is_keystone_v3 = 'catalog' in auth[1] + + if is_keystone_v3: + if endpoint_type.endswith("URL"): + endpoint_type = endpoint_type[:-3] + catalog = auth[1]['catalog'] + endpoints = [e['endpoints'] for e in catalog + if e['type'] == opt_section.catalog_type] + if not endpoints: + raise Exception("%s endpoint not found" % + opt_section.catalog_type) + endpoints = [e['url'] for e in endpoints[0] + if e['interface'] == endpoint_type] + if not endpoints: + raise Exception("%s interface not found for endpoint %s" % + (endpoint_type, + opt_section.catalog_type)) + return endpoints[0] + + else: + if not endpoint_type.endswith("URL"): + endpoint_type += "URL" + catalog = auth[1]['serviceCatalog'] + endpoints = [e for e in catalog + if e['type'] == opt_section.catalog_type] + if not endpoints: + raise Exception("%s endpoint not found" % + opt_section.catalog_type) + return endpoints[0]['endpoints'][0][endpoint_type] + + def _do_test(self, filename): + auth = self.os_admin.auth_provider.get_auth() + + os.environ.update({ + "ADMIN_TOKEN": auth[0], + "AODH_SERVICE_URL": self._get_endpoint(auth, "alarming_plugin"), + "GNOCCHI_SERVICE_URL": self._get_endpoint(auth, "metric"), + "PANKO_SERVICE_URL": self._get_endpoint(auth, "event"), + "HEAT_SERVICE_URL": self._get_endpoint(auth, "orchestration"), + "NOVA_SERVICE_URL": self._get_endpoint(auth, "compute"), + "GLANCE_SERVICE_URL": self._get_endpoint(auth, "image"), + "GLANCE_IMAGE_NAME": self.glance_image_create(), + }) + + with file(os.path.join(TEST_DIR, filename)) as f: + test_suite = suitemaker.test_suite_from_dict( + loader=unittest.defaultTestLoader, + test_base_name="gabbi", + suite_dict=utils.load_yaml(f), + test_directory=TEST_DIR, + host=None, port=None, + fixture_module=None, + intercept=None, + handlers=runner.initialize_handlers([]), + test_loader_name="tempest") + + # NOTE(sileht): We hide stdout/stderr and reraise the failure + # manually, tempest will print it itself. + with open(os.devnull, 'w') as stream: + result = unittest.TextTestRunner( + stream=stream, verbosity=0, failfast=True, + ).run(test_suite) + + if not result.wasSuccessful(): + failures = (result.errors + result.failures + + result.unexpectedSuccesses) + if failures: + test, bt = failures[0] + name = test.test_data.get('name', test.id()) + msg = 'From test "%s" :\n%s' % (name, bt) + self.fail(msg) + + self.assertTrue(result.wasSuccessful()) + + +def test_maker(name, filename): + def test(self): + self._do_test(filename) + test.__name__ = name + return test + +# Create one scenario per yaml file +for filename in os.listdir(TEST_DIR): + if not filename.endswith('.yaml'): + continue + name = "test_%s" % filename[:-5].lower().replace("-", "_") + setattr(TestTelemetryIntegration, name, + test_maker(name, filename))