diff --git a/stacklight_tests/elasticsearch_kibana/api.py b/stacklight_tests/elasticsearch_kibana/api.py index e4ba9fb..91044fb 100644 --- a/stacklight_tests/elasticsearch_kibana/api.py +++ b/stacklight_tests/elasticsearch_kibana/api.py @@ -82,15 +82,74 @@ class ElasticsearchPluginApi(base_test.PluginApi): return self.helpers.check_plugin_cannot_be_uninstalled( self.settings.name, self.settings.version) - def get_current_indices(self, index_type): - indices = self.es.indices.get_aliases().keys() - return filter(lambda x: index_type in x, sorted(indices))[-2:] + def query_elasticsearch(self, index_type, time_range="now-1h", + query_filter="*", size=100): + all_indices = self.es.indices.get_aliases().keys() + indices = filter(lambda x: index_type in x, sorted(all_indices)) + return self.es.search(index=indices, body={ + "query": {"filtered": { + "query": {"bool": {"should": {"query_string": { + "query": query_filter}}}}, + "filter": {"bool": {"must": {"range": { + "Timestamp": {"from": time_range}}}}}}}, + "size": size}) - def query_nova_logs(self, indices): - query = {"query": {"filtered": { - "query": {"bool": {"should": [{"query_string": { - "query": "programname:nova*"}}]}}, - "filter": {"bool": {"must": [{"range": {"Timestamp": { - "from": "now-1h"}}}]}}}}, "size": 100} - output = self.es.search(index=indices, body=query) - return output + def make_instance_actions(self): + net_name = self.fuel_web.get_cluster_predefined_networks_name( + self.helpers.cluster_id)['private_net'] + os_conn = self.helpers.os_conn + flavors = os_conn.nova.flavors.list(sort_key="memory_mb") + logger.info("Launch an instance") + instance = os_conn.create_server_for_migration( + label=net_name, flavor=flavors[0]) + logger.info("Update the instance") + os_conn.nova.servers.update(instance, name="test-server") + self.helpers.wait_for_resource_status( + os_conn.nova.servers, instance, "ACTIVE") + image = self.helpers.os_conn._get_cirros_image() + logger.info("Rebuild the instance") + os_conn.nova.servers.rebuild( + instance, image, name="rebuilded_instance") + self.helpers.wait_for_resource_status( + os_conn.nova.servers, instance, "ACTIVE") + logger.info("Resize the instance") + os_conn.nova.servers.resize(instance, flavors[1]) + self.helpers.wait_for_resource_status( + os_conn.nova.servers, instance, "VERIFY_RESIZE") + logger.info("Confirm the resize") + os_conn.nova.servers.confirm_resize(instance) + self.helpers.wait_for_resource_status( + os_conn.nova.servers, instance, "ACTIVE") + logger.info("Resize the instance") + os_conn.nova.servers.resize(instance, flavors[2]) + self.helpers.wait_for_resource_status( + os_conn.nova.servers, instance, "VERIFY_RESIZE") + logger.info("Revert the resize") + os_conn.nova.servers.revert_resize(instance) + self.helpers.wait_for_resource_status( + os_conn.nova.servers, instance, "ACTIVE") + logger.info("Stop the instance") + os_conn.nova.servers.stop(instance) + self.helpers.wait_for_resource_status( + os_conn.nova.servers, instance, "SHUTOFF") + logger.info("Start the instance") + os_conn.nova.servers.start(instance) + self.helpers.wait_for_resource_status( + os_conn.nova.servers, instance, "ACTIVE") + logger.info("Suspend the instance") + os_conn.nova.servers.suspend(instance) + self.helpers.wait_for_resource_status( + os_conn.nova.servers, instance, "SUSPENDED") + logger.info("Resume the instance") + os_conn.nova.servers.resume(instance) + self.helpers.wait_for_resource_status( + os_conn.nova.servers, instance, "ACTIVE") + logger.info("Create an instance snapshot") + snapshot = os_conn.nova.servers.create_image(instance, "test-image") + self.helpers.wait_for_resource_status( + os_conn.nova.images, snapshot, "ACTIVE") + logger.info("Delete the instance") + os_conn.nova.servers.delete(instance) + logger.info("Check that the instance was deleted") + os_conn.verify_srv_deleted(instance) + return instance.id diff --git a/stacklight_tests/helpers/helpers.py b/stacklight_tests/helpers/helpers.py index 802a5a6..5c905ef 100644 --- a/stacklight_tests/helpers/helpers.py +++ b/stacklight_tests/helpers/helpers.py @@ -18,6 +18,7 @@ import time import urllib2 from devops.helpers import helpers +from fuelweb_test.helpers import os_actions from fuelweb_test import logger from proboscis import asserts @@ -32,6 +33,10 @@ class NotFound(Exception): pass +class TimeoutException(Exception): + pass + + def get_plugin_name(filename): """Extract the plugin name from the package filename. @@ -78,6 +83,7 @@ class PluginHelper(object): self.fuel_web = self.env.fuel_web self._cluster_id = None self.nailgun_client = self.fuel_web.client + self._os_conn = None @property def cluster_id(self): @@ -92,6 +98,13 @@ class PluginHelper(object): def cluster_id(self, value): self._cluster_id = value + @property + def os_conn(self): + if self._os_conn is None: + self._os_conn = os_actions.OpenStackActions( + self.fuel_web.get_public_vip(self.cluster_id)) + return self._os_conn + def prepare_plugin(self, plugin_path): """Upload and install plugin by path.""" self.env.admin_actions.upload_plugin(plugin=plugin_path) @@ -522,3 +535,27 @@ class PluginHelper(object): with self.fuel_web.get_ssh_for_nailgun_node(compute) as remote: for service in compute_services: remote_ops.manage_initctl_service(remote, service) + + @staticmethod + def check_notifications(got_list, expected_list): + for event_type in expected_list: + asserts.assert_true( + event_type in got_list, "{} event type not found in {}".format( + event_type, got_list)) + + @staticmethod + def wait_for_resource_status(resource_client, resource, expected_status, + timeout=180, interval=30): + start = time.time() + finish = start + timeout + while start < finish: + curr_state = resource_client.get(resource).status + if curr_state == expected_status: + return + else: + logger.debug( + "Instance is not in {} status".format(expected_status)) + time.sleep(interval) + start = time.time() + raise TimeoutException("Timed out waiting to become {}".format( + expected_status)) diff --git a/stacklight_tests/toolchain/api.py b/stacklight_tests/toolchain/api.py index 140b89e..99ed7cc 100644 --- a/stacklight_tests/toolchain/api.py +++ b/stacklight_tests/toolchain/api.py @@ -157,12 +157,10 @@ class ToolchainApi(object): ) def check_nova_logs(self): - indices = self.ELASTICSEARCH_KIBANA.get_current_indices('log') - logger.info("Found indexes {}".format(indices)) - output = self.ELASTICSEARCH_KIBANA.query_nova_logs(indices) - msg = "Indexes {} don't contain Nova logs" - asserts.assert_not_equal(output['hits']['total'], 0, msg.format( - indices)) + output = self.ELASTICSEARCH_KIBANA.query_elasticsearch( + index_type="log", query_filter="programname:nova*") + asserts.assert_not_equal(output['hits']['total'], 0, + "Indexes don't contain Nova logs") controllers = self.fuel_web.get_nailgun_cluster_nodes_by_roles( self.helpers.cluster_id, ["controller"]) computes = self.fuel_web.get_nailgun_cluster_nodes_by_roles( @@ -172,3 +170,45 @@ class ToolchainApi(object): actual_hostnames = set([hit['_source']['Hostname'] for hit in output['hits']['hits']]) asserts.assert_equal(expected_hostnames, actual_hostnames) + + def check_nova_notifications(self): + nova_event_types = [ + "compute.instance.create.start", "compute.instance.create.end", + "compute.instance.delete.start", "compute.instance.delete.end", + "compute.instance.rebuild.start", "compute.instance.rebuild.end", + "compute.instance.rebuild.scheduled", + "compute.instance.resize.prep.start", + "compute.instance.resize.prep.end", + "compute.instance.resize.confirm.start", + "compute.instance.resize.confirm.end", + "compute.instance.resize.revert.start", + "compute.instance.resize.revert.end", + "compute.instance.exists", "compute.instance.update", + "compute.instance.shutdown.start", "compute.instance.shutdown.end", + "compute.instance.power_off.start", + "compute.instance.power_off.end", + "compute.instance.power_on.start", "compute.instance.power_on.end", + "compute.instance.snapshot.start", "compute.instance.snapshot.end", + "compute.instance.resize.start", "compute.instance.resize.end", + "compute.instance.finish_resize.start", + "compute.instance.finish_resize.end", + "compute.instance.suspend.start", "compute.instance.suspend.end", + "scheduler.select_destinations.start", + "scheduler.select_destinations.end"] + instance_event_types = nova_event_types[:-2] + instance_id = self.ELASTICSEARCH_KIBANA.make_instance_actions() + output_for_instance_id = self.ELASTICSEARCH_KIBANA.query_elasticsearch( + index_type="notification", + query_filter='instance_id="{}"'.format(instance_id), size=500) + instance_id_notifications = list(set( + [hit["_source"]["event_type"] + for hit in output_for_instance_id["hits"]["hits"]])) + self.helpers.check_notifications(instance_id_notifications, + instance_event_types) + output_for_logger = self.ELASTICSEARCH_KIBANA.query_elasticsearch( + index_type="notification", query_filter="Logger:nova", size=500) + logger_notifications = list(set( + [hit["_source"]["event_type"] + for hit in output_for_logger["hits"]["hits"]])) + self.helpers.check_notifications(logger_notifications, + nova_event_types) diff --git a/stacklight_tests/toolchain/test_functional.py b/stacklight_tests/toolchain/test_functional.py index 5b78fdc..0fed9b6 100644 --- a/stacklight_tests/toolchain/test_functional.py +++ b/stacklight_tests/toolchain/test_functional.py @@ -104,3 +104,25 @@ class TestFunctionalToolchain(api.ToolchainApi): self.check_plugins_online() self.check_nova_logs() + + @test(depends_on_groups=["deploy_ha_toolchain"], + groups=["check_nova_notifications_toolchain", "toolchain", + "functional"]) + @log_snapshot_after_test + def check_nova_notifications_toolchain(self): + """Check that Nova notifications are present in Elasticsearch + + Scenario: + 1. Revert snapshot with 9 deployed nodes in HA configuration + 2. Launch, update, rebuild, resize, power-off, power-on, snapshot, + suspend, shutdown, and delete an instance + 3. Check that Nova notifications are present in current + Elasticsearch index + + Duration 25m + """ + self.env.revert_snapshot("deploy_ha_toolchain") + + self.check_plugins_online() + + self.check_nova_notifications()