openstack/nova
Code Issues Proposed changes

Add logging to find test cases leaking libvirt threads

Browse Source 
We see functional test failures due to leaked libvirt event handling
thread weaking up after its original test finished and importing
libvirt. If it happens when the libvirt package import is poisoned then
the currently executing test will fail. This patch logs the name of the
test case that leaked the libvirt event handling thread.

We will revert his before RC1.

Change-Id: I3146e9afb411056d004fc118ccfa31126a3c6b15
Related-Bug: #1946339
This commit is contained in:
Sylvain Bauza 2023-02-07 16:30:59 +01:00 committed by Balazs Gibizer
parent 7ea9aac71c
commit 1778a9c589
2 changed files with 44 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
nova/tests/fixtures/nova.py vendored
View File

@ -1822,6 +1822,24 @@ class ImportModulePoisonFixture(fixtures.Fixture):
def find_spec(self, fullname, path, target=None):
if fullname in self.modules:
current = eventlet.getcurrent()
# NOTE(gibi) not all eventlet spawn is under our control, so
# there can be senders without test_case_id set, find the first
# ancestor that was spawned from nova.utils.spawn[_n] and
# therefore has the id set.
while (
current is not None and
not getattr(current, 'test_case_id', None)
):
current = current.parent
if current is not None:
self.test.tc_id = current.test_case_id
LOG.warning(
"!!!---!!! TestCase ID %s hit the import poison while "
"importing %s. If you see this in a failed functional "
"test then please let #openstack-nova on IRC know "
"about it. !!!---!!!", current.test_case_id, fullname)
self.test.fail_message = (
f"This test imports the '{fullname}' module, which it "
f'should not in the test environment. Please add '
@ -1832,6 +1850,7 @@ class ImportModulePoisonFixture(fixtures.Fixture):
def __init__(self, module_names):
self.module_names = module_names
self.fail_message = ''
self.tc_id = None
if isinstance(module_names, str):
self.module_names = {module_names}
self.meta_path_finder = self.ForbiddenModules(self, self.module_names)
@ -1849,6 +1868,13 @@ class ImportModulePoisonFixture(fixtures.Fixture):
# there (which is also what self.assert* and self.fail() do underneath)
# will not work to cause a failure in the test.
if self.fail_message:
if self.tc_id is not None:
LOG.warning(
"!!!---!!! TestCase ID %s hit the import poison. If you "
"see this in a failed functional test then please let "
"#openstack-nova on IRC know about it. !!!---!!!",
self.tc_id
)
raise ImportError(self.fail_message)

18
nova/virt/libvirt/driver.py
View File

@ -10061,6 +10061,24 @@ class LibvirtDriver(driver.ComputeDriver):
:param instance: instance object that is in migration
"""
current = eventlet.getcurrent()
# NOTE(gibi) not all eventlet spawn is under our control, so
# there can be senders without test_case_id set, find the first
# ancestor that was spawned from nova.utils.spawn[_n] and
# therefore has the id set.
while (
current is not None and
not getattr(current, 'test_case_id', None)
):
current = current.parent
if current is not None:
LOG.warning(
"!!!---!!! live_migration_abort thread was spawned by "
"TestCase ID: %s. If you see this in a failed functional test "
"then please let #openstack-nova on IRC know about it. "
"!!!---!!!", current.test_case_id
)
guest = self._host.get_guest(instance)
dom = guest._domain