Transform instance.create.error notification

Along with the transformation changes, I modified notify_about_instance_action
method so it can handle errors now. (It wraps the exception object into an
ExceptionPayload.)

Implements: bp versioned-notification-transformation-ocata

Change-Id: I171990290c8de529c9521a5ee556f97b18b741b0

doc/notification_samples/instance-create-error.json

@@ -0,0 +1,59 @@
+{
+    "event_type":"instance.create.error",
+    "payload":{
+        "nova_object.data":{
+            "architecture":"x86_64",
+            "availability_zone":null,
+            "created_at":"2012-10-29T13:42:11Z",
+            "deleted_at":null,
+            "display_name":"some-server",
+            "fault": {
+                "nova_object.data": {
+                    "exception": "FlavorDiskTooSmall",
+                    "exception_message": "The created instance's disk would be too small.",
+                    "function_name": "_build_resources",
+                    "module_name": "nova.tests.functional.notification_sample_tests.test_instance"
+                },
+                "nova_object.name": "ExceptionPayload",
+                "nova_object.namespace": "nova",
+                "nova_object.version": "1.0"
+            },
+            "host":"compute",
+            "host_name":"some-server",
+            "ip_addresses": [],
+            "kernel_id":"",
+            "launched_at":null,
+            "image_uuid": "155d900f-4e14-4e4c-a73d-069cbf4541e6",
+            "metadata":{},
+            "node":"fake-mini",
+            "os_type":null,
+            "progress":0,
+            "ramdisk_id":"",
+            "reservation_id":"r-npxv0e40",
+            "state":"building",
+            "task_state":null,
+            "power_state":"pending",
+            "tenant_id":"6f70656e737461636b20342065766572",
+            "terminated_at":null,
+            "flavor": {
+                "nova_object.name": "FlavorPayload",
+                "nova_object.data": {
+                    "flavorid": "a22d5517-147c-4147-a0d1-e698df5cd4e3",
+                    "root_gb": 1,
+                    "vcpus": 1,
+                    "ephemeral_gb": 0,
+                    "memory_mb": 512
+                },
+                "nova_object.version": "1.0",
+                "nova_object.namespace": "nova"
+            },
+            "user_id":"fake",
+            "uuid":"178b0921-8f85-4257-88b6-2e743b5a975c"
+        },
+        "nova_object.name":"InstanceActionPayload",
+        "nova_object.namespace":"nova",
+        "nova_object.version":"1.0"
+    },
+    "priority":"ERROR",
+    "publisher_id":"nova-compute:compute"
+}

nova/compute/manager.py

@@ -1930,10 +1930,18 @@ class ComputeManager(manager.Manager):
             with excutils.save_and_reraise_exception():
                 self._notify_about_instance_usage(context, instance,
                     'create.error', fault=e)
+                compute_utils.notify_about_instance_action(
+                    context, instance, self.host,
+                    action=fields.NotificationAction.CREATE,
+                    phase=fields.NotificationPhase.ERROR, exception=e)
         except exception.ComputeResourcesUnavailable as e:
             LOG.debug(e.format_message(), instance=instance)
             self._notify_about_instance_usage(context, instance,
                     'create.error', fault=e)
+            compute_utils.notify_about_instance_action(
+                    context, instance, self.host,
+                    action=fields.NotificationAction.CREATE,
+                    phase=fields.NotificationPhase.ERROR, exception=e)
             raise exception.RescheduledException(
                     instance_uuid=instance.uuid, reason=e.format_message())
         except exception.BuildAbortException as e:
@@ -1941,12 +1949,20 @@ class ComputeManager(manager.Manager):
                 LOG.debug(e.format_message(), instance=instance)
                 self._notify_about_instance_usage(context, instance,
                     'create.error', fault=e)
+                compute_utils.notify_about_instance_action(
+                    context, instance, self.host,
+                    action=fields.NotificationAction.CREATE,
+                    phase=fields.NotificationPhase.ERROR, exception=e)
         except (exception.FixedIpLimitExceeded,
                 exception.NoMoreNetworks, exception.NoMoreFixedIps) as e:
             LOG.warning(_LW('No more network or fixed IP to be allocated'),
                         instance=instance)
             self._notify_about_instance_usage(context, instance,
                     'create.error', fault=e)
+            compute_utils.notify_about_instance_action(
+                    context, instance, self.host,
+                    action=fields.NotificationAction.CREATE,
+                    phase=fields.NotificationPhase.ERROR, exception=e)
             msg = _('Failed to allocate the network(s) with error %s, '
                     'not rescheduling.') % e.format_message()
             raise exception.BuildAbortException(instance_uuid=instance.uuid,
@@ -1959,6 +1975,10 @@ class ComputeManager(manager.Manager):
                           instance=instance)
             self._notify_about_instance_usage(context, instance,
                     'create.error', fault=e)
+            compute_utils.notify_about_instance_action(
+                    context, instance, self.host,
+                    action=fields.NotificationAction.CREATE,
+                    phase=fields.NotificationPhase.ERROR, exception=e)
             msg = _('Failed to allocate the network(s), not rescheduling.')
             raise exception.BuildAbortException(instance_uuid=instance.uuid,
                     reason=msg)
@@ -1971,11 +1991,19 @@ class ComputeManager(manager.Manager):
                 exception.SignatureVerificationError) as e:
             self._notify_about_instance_usage(context, instance,
                     'create.error', fault=e)
+            compute_utils.notify_about_instance_action(
+                    context, instance, self.host,
+                    action=fields.NotificationAction.CREATE,
+                    phase=fields.NotificationPhase.ERROR, exception=e)
             raise exception.BuildAbortException(instance_uuid=instance.uuid,
                     reason=e.format_message())
         except Exception as e:
             self._notify_about_instance_usage(context, instance,
                     'create.error', fault=e)
+            compute_utils.notify_about_instance_action(
+                    context, instance, self.host,
+                    action=fields.NotificationAction.CREATE,
+                    phase=fields.NotificationPhase.ERROR, exception=e)
             raise exception.RescheduledException(
                     instance_uuid=instance.uuid, reason=six.text_type(e))
 
@@ -2007,6 +2035,10 @@ class ComputeManager(manager.Manager):
             with excutils.save_and_reraise_exception():
                 self._notify_about_instance_usage(context, instance,
                     'create.error', fault=e)
+                compute_utils.notify_about_instance_action(
+                    context, instance, self.host,
+                    action=fields.NotificationAction.CREATE,
+                    phase=fields.NotificationPhase.ERROR, exception=e)
 
         self._update_scheduler_instance_info(context, instance)
         self._notify_about_instance_usage(context, instance, 'create.end',

nova/compute/utils.py

@@ -339,28 +339,40 @@ def _get_instance_ips(instance):
     return ips
 
 
+def _get_fault_and_priority_from_exc(exception):
+    fault = None
+    priority = fields.NotificationPriority.INFO
+
+    if exception:
+        priority = fields.NotificationPriority.ERROR
+        fault = notification_exception.ExceptionPayload.from_exception(
+            exception)
+
+    return fault, priority
+
+
 def notify_about_instance_action(context, instance, host, action, phase=None,
-                                 binary='nova-compute'):
+                                 binary='nova-compute', exception=None):
     """Send versioned notification about the action made on the instance
     :param instance: the instance which the action performed on
     :param host: the host emitting the notification
     :param action: the name of the action
     :param phase: the phase of the action
     :param binary: the binary emitting the notification
+    :param exception: the thrown exception (used in error notifications)
     """
     ips = _get_instance_ips(instance)
 
     flavor = instance_notification.FlavorPayload(instance=instance)
-    # TODO(gibi): handle fault during the transformation of the first error
+    fault, priority = _get_fault_and_priority_from_exc(exception)
-    # notifications
     payload = instance_notification.InstanceActionPayload(
             instance=instance,
-            fault=None,
+            fault=fault,
             ip_addresses=ips,
             flavor=flavor)
     notification = instance_notification.InstanceActionNotification(
             context=context,
-            priority=fields.NotificationPriority.INFO,
+            priority=priority,
             publisher=notification_base.NotificationPublisher(
                     context=context, host=host, binary=binary),
             event_type=notification_base.EventType(
@@ -389,14 +401,7 @@ def notify_about_volume_swap(context, instance, host, action, phase,
 
     flavor = instance_notification.FlavorPayload(instance=instance)
 
-    if exception:
+    fault, priority = _get_fault_and_priority_from_exc(exception)
-        priority = fields.NotificationPriority.ERROR
-        fault = notification_exception.ExceptionPayload.from_exception(
-            exception)
-    else:
-        priority = fields.NotificationPriority.INFO
-        fault = None
-
     payload = instance_notification.InstanceActionVolumeSwapPayload(
         instance=instance,
         fault=fault,

nova/tests/functional/notification_sample_tests/test_instance.py

@@ -9,9 +9,11 @@
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 #    License for the specific language governing permissions and limitations
 #    under the License.
+import mock
 import time
 
 from nova import context
+from nova import exception
 from nova.tests import fixtures
 from nova.tests.functional.notification_sample_tests \
     import notification_sample_base
@@ -100,6 +102,32 @@ class TestInstanceNotificationSample(
                     'uuid': server['id']},
                 actual=fake_notifier.VERSIONED_NOTIFICATIONS[idx])
 
+    @mock.patch('nova.compute.manager.ComputeManager._build_resources')
+    def test_create_server_error(self, mock_build):
+        def _build_resources(*args, **kwargs):
+            raise exception.FlavorDiskTooSmall()
+
+        mock_build.side_effect = _build_resources
+
+        server = self._boot_a_server(
+            expected_status='ERROR',
+            extra_params={'networks': [{'port': self.neutron.port_1['id']}]})
+
+        self.assertEqual(2, len(fake_notifier.VERSIONED_NOTIFICATIONS))
+
+        self._verify_notification(
+            'instance-create-start',
+            replacements={
+                'reservation_id': server['reservation_id'],
+                'uuid': server['id']},
+            actual=fake_notifier.VERSIONED_NOTIFICATIONS[0])
+        self._verify_notification(
+            'instance-create-error',
+            replacements={
+                'reservation_id': server['reservation_id'],
+                'uuid': server['id']},
+            actual=fake_notifier.VERSIONED_NOTIFICATIONS[1])
+
     def _verify_instance_update_steps(self, steps, notifications,
                                       initial=None):
         replacements = {}

nova/tests/unit/compute/test_compute_mgr.py

@@ -4045,10 +4045,12 @@ class ComputeManagerBuildInstanceTestCase(test.NoDBTestCase):
                 mock.patch.object(self.compute,
                     '_shutdown_instance'),
                 mock.patch.object(self.compute,
-                    '_validate_instance_group_policy')
+                    '_validate_instance_group_policy'),
+                mock.patch('nova.compute.utils.notify_about_instance_action')
         ) as (spawn, save,
                 _build_networks_for_instance, _notify_about_instance_usage,
-                _shutdown_instance, _validate_instance_group_policy):
+                _shutdown_instance, _validate_instance_group_policy,
+                mock_notify):
 
             self.assertRaises(exception.BuildAbortException,
                     self.compute._build_and_run_instance, self.context,
@@ -4069,6 +4071,12 @@ class ComputeManagerBuildInstanceTestCase(test.NoDBTestCase):
                 mock.call(self.context, self.instance, 'create.error',
                     fault=exc)])
 
+            mock_notify.assert_has_calls([
+                mock.call(self.context, self.instance, 'fake-mini',
+                          action='create', phase='start'),
+                mock.call(self.context, self.instance, 'fake-mini',
+                          action='create', phase='error', exception=exc)])
+
             save.assert_has_calls([
                 mock.call(),
                 mock.call(),