From 3cef7edbc2dd72a25e46ca1962731ee3153f1fd7 Mon Sep 17 00:00:00 2001 From: Matt Riedemann Date: Fri, 30 Aug 2019 12:11:59 -0400 Subject: [PATCH] Handle VirtDriverNotReady in _cleanup_running_deleted_instances Change I6e086e0a2bffe12fd78373b3bc6215221dabc77e made the _cleanup_running_deleted_instances task run immediately on start of the nova-compute service. For some compute drivers, like ironic, the "hypervisor" might not yet be ready to list instances which results in a traceback exception in the logs. This simply handles the VirtDriverNotReady error and returns to avoid the traceback. Change-Id: I6220a59988502a9839645efa101d555c19d4b650 Closes-Bug: #1842081 --- nova/compute/manager.py | 12 +++++++++++- nova/tests/unit/compute/test_compute_mgr.py | 13 +++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 668a4a124a29..2a2b7a336e06 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -8389,7 +8389,17 @@ class ComputeManager(manager.Manager): # NOTE(sirp): admin contexts don't ordinarily return deleted records with utils.temporary_mutation(context, read_deleted="yes"): - for instance in self._running_deleted_instances(context): + + try: + instances = self._running_deleted_instances(context) + except exception.VirtDriverNotReady: + # Since this task runs immediately on startup, if the + # hypervisor is not yet ready handle it gracefully. + LOG.debug('Unable to check for running deleted instances ' + 'at this time since the hypervisor is not ready.') + return + + for instance in instances: if action == "log": LOG.warning("Detected instance with name label " "'%s' which is marked as " diff --git a/nova/tests/unit/compute/test_compute_mgr.py b/nova/tests/unit/compute/test_compute_mgr.py index e9eae67d7c7f..9a729b726c08 100644 --- a/nova/tests/unit/compute/test_compute_mgr.py +++ b/nova/tests/unit/compute/test_compute_mgr.py @@ -1925,6 +1925,19 @@ class ComputeManagerUnitTestCase(test.NoDBTestCase, power_state.NOSTATE, use_slave=True) + def test_cleanup_running_deleted_instances_virt_driver_not_ready(self): + """Tests the scenario that the driver raises VirtDriverNotReady + when listing instances so the task returns early. + """ + self.flags(running_deleted_instance_action='shutdown') + with mock.patch.object(self.compute, '_running_deleted_instances', + side_effect=exception.VirtDriverNotReady) as ls: + # Mock the virt driver to make sure nothing calls it. + with mock.patch.object(self.compute, 'driver', + new_callable=mock.NonCallableMock): + self.compute._cleanup_running_deleted_instances(self.context) + ls.assert_called_once_with(test.MatchType(context.RequestContext)) + @mock.patch.object(virt_driver.ComputeDriver, 'delete_instance_files') @mock.patch.object(objects.InstanceList, 'get_by_filters') def test_run_pending_deletes(self, mock_get, mock_delete):