From fd656f394375231de9beb54087e2ecb910060045 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Ribaud?= Date: Tue, 18 Feb 2025 21:51:09 +0100 Subject: [PATCH] Update driver to map the targeted address for SR-IOV PCI devices This patch checks the revision of QEMU and libvirt to ensure support for VFIO SR-IOV device migration. It also updates the _live_migration_operation() function, particularly the get_updated_guest_xml() function, to map source PCI addresses to destination addresses in the destination XML file, using the data provided by the LiveMigrateData object. The target goal of these series of patch is to enable VFIO devices migration with kernel variant drivers. Partially-Implements: blueprint migrate-vfio-devices-using-kernel-variant-drivers Change-Id: I62ec475988eab8de948498f50d8d4c0d47321102 --- nova/conf/pci.py | 67 +- .../libvirt/test_pci_sriov_servers.py | 1037 ++++++++++++++++- .../tests/unit/virt/libvirt/test_migration.py | 195 ++++ nova/virt/libvirt/driver.py | 30 + nova/virt/libvirt/migration.py | 77 +- ...rnel-variant-drivers-d4180849f973012e.yaml | 8 + 6 files changed, 1407 insertions(+), 7 deletions(-) create mode 100644 releasenotes/notes/migrate-vfio-devices-using-kernel-variant-drivers-d4180849f973012e.yaml diff --git a/nova/conf/pci.py b/nova/conf/pci.py index 9a6dc2e07e16..ed9e7d4eb000 100644 --- a/nova/conf/pci.py +++ b/nova/conf/pci.py @@ -36,7 +36,24 @@ to use move operations, for each ``nova-compute`` service. Possible Values: -* A dictionary of JSON values which describe the aliases. For example:: +* A JSON dictionary which describe a PCI device. It should take + the following format:: + + alias = { + "name": "", + ["product_id": ""], + ["vendor_id": ""], + "device_type": "", + ["numa_policy": ""], + ["resource_class": ""], + ["traits": ""] + ["live_migratable": ""], + } + + Where ``[`` indicates zero or one occurrences, ``{`` indicates zero or + multiple occurrences, and ``|`` mutually exclusive options. + + For example:: alias = { "name": "QuickAssist", @@ -46,8 +63,17 @@ Possible Values: "numa_policy": "required" } - This defines an alias for the Intel QuickAssist card. (multi valued). Valid - key values are : + This defines an alias for the Intel QuickAssist card. (multi valued). + + Another example:: + + alias = { + "name": "A16_16A", + "device_type": "type-VF", + resource_class: "CUSTOM_A16_16A", + } + + Valid key values are : ``name`` Name of the PCI alias. @@ -97,6 +123,22 @@ Possible Values: scheduling the request. This field can only be used only if ``[filter_scheduler]pci_in_placement`` is enabled. + ``live_migratable`` + Specify if live-migratable devices are desired. + May have boolean-like string values case-insensitive values: + "yes" or "no". + + - ``live_migratable='yes'`` means that the user wants a device(s) + allowing live migration to a similar device(s) on another host. + + - ``live_migratable='no'`` This explicitly indicates that the user + requires a non-live migratable device, making migration impossible. + + - If not specified, the default is ``live_migratable=None``, meaning that + either a live migratable or non-live migratable device will be picked + automatically. However, in such cases, migration will **not** be + possible. + * Supports multiple aliases by repeating the option (not by specifying a list value):: @@ -112,7 +154,8 @@ Possible Values: "product_id": "0444", "vendor_id": "8086", "device_type": "type-PCI", - "numa_policy": "required" + "numa_policy": "required", + "live_migratable": "yes", } """), cfg.MultiStrOpt('device_spec', @@ -165,7 +208,9 @@ Possible values: Supported ```` values are : - ``physical_network`` + - ``trusted`` + - ``remote_managed`` - a VF is managed remotely by an off-path networking backend. May have boolean-like string values case-insensitive values: "true" or "false". By default, "false" is assumed for all devices. @@ -174,6 +219,7 @@ Possible values: VPD capability with a card serial number (either on a VF itself on its corresponding PF), otherwise they will be ignored and not available for allocation. + - ``managed`` - Specify if the PCI device is managed by libvirt. May have boolean-like string values case-insensitive values: "yes" or "no". By default, "yes" is assumed for all devices. @@ -189,6 +235,18 @@ Possible values: Warning: Incorrect configuration of this parameter may result in compute node crashes. + + - ``live_migratable`` - Specify if the PCI device is live_migratable by + libvirt. + May have boolean-like string values case-insensitive values: + "yes" or "no". By default, "no" is assumed for all devices. + + - ``live_migratable='yes'`` means that the device can be live migrated. + Of course, this requires hardware support, as well as proper system + and hypervisor configuration. + + - ``live_migratable='no'`` means that the device cannot be live migrated. + - ``resource_class`` - optional Placement resource class name to be used to track the matching PCI devices in Placement when [pci]report_in_placement is True. @@ -202,6 +260,7 @@ Possible values: device's ``vendor_id`` and ``product_id`` in the form of ``CUSTOM_PCI_{vendor_id}_{product_id}``. The ``resource_class`` can be requested from a ``[pci]alias`` + - ``traits`` - optional comma separated list of Placement trait names to report on the resource provider that will represent the matching PCI device. Each trait can be a standard trait from ``os-traits`` lib or can diff --git a/nova/tests/functional/libvirt/test_pci_sriov_servers.py b/nova/tests/functional/libvirt/test_pci_sriov_servers.py index 458cd7b2bb14..450db85d3684 100644 --- a/nova/tests/functional/libvirt/test_pci_sriov_servers.py +++ b/nova/tests/functional/libvirt/test_pci_sriov_servers.py @@ -27,6 +27,7 @@ from oslo_log import log as logging from oslo_serialization import jsonutils from oslo_utils.fixture import uuidsentinel as uuids from oslo_utils import units +from oslo_utils import versionutils import nova from nova.compute import pci_placement_translator @@ -35,11 +36,13 @@ from nova import exception from nova.network import constants from nova import objects from nova.objects import fields +from nova.objects import instance from nova.pci.utils import parse_address from nova.tests import fixtures as nova_fixtures from nova.tests.fixtures import libvirt as fakelibvirt from nova.tests.functional.api import client from nova.tests.functional.libvirt import base +from nova.virt.libvirt import driver CONF = cfg.CONF LOG = logging.getLogger(__name__) @@ -340,6 +343,7 @@ class _PCIServersWithMigrationTestBase(_PCIServersTestBase): dom.complete_job() +@ddt.ddt class SRIOVServersTest(_PCIServersWithMigrationTestBase): # TODO(stephenfin): We're using this because we want to be able to force @@ -426,6 +430,8 @@ class SRIOVServersTest(_PCIServersWithMigrationTestBase): pci_info, expected_managed, device_spec=None, + libvirt_version=None, + qemu_version=None, ): """Runs a create server test with a specified PCI setup and checks Guest.create call. @@ -443,6 +449,8 @@ class SRIOVServersTest(_PCIServersWithMigrationTestBase): ) as mock_create: compute = self.start_compute( pci_info=pci_info, + libvirt_version=libvirt_version, + qemu_version=qemu_version, ) self.host = self.computes[compute].driver._host @@ -538,6 +546,36 @@ class SRIOVServersTest(_PCIServersWithMigrationTestBase): pci_info, expected_managed="yes", device_spec=device_spec ) + def test_create_server_with_VF_and_managed_set_to_yes_fails_version(self): + device_spec = [ + { + "vendor_id": fakelibvirt.PCI_VEND_ID, + "product_id": fakelibvirt.PF_PROD_ID, + "physical_network": "physnet4", + }, + { + "vendor_id": fakelibvirt.PCI_VEND_ID, + "product_id": fakelibvirt.VF_PROD_ID, + "physical_network": "physnet4", + "live_migratable": "yes", + }, + ] + pci_info = fakelibvirt.HostPCIDevicesInfo(num_pfs=1, num_vfs=1) + + exc = self.assertRaises( + exception.InvalidConfiguration, + self._run_create_server_test, + pci_info, + expected_managed="yes", + device_spec=device_spec, + ) + + self.assertIn( + "PCI device spec is configured for " + "live_migratable but it's not supported by libvirt.", + str(exc), + ) + def test_create_server_with_PF(self): """Create a server with an SR-IOV PF-type PCI device.""" @@ -680,11 +718,12 @@ class SRIOVServersTest(_PCIServersWithMigrationTestBase): self.assertEqual(500, ex.response.status_code) self.assertIn('NoValidHost', str(ex)) - def test_live_migrate_server_with_VF(self): + def test_live_migrate_server_with_VF_legacy(self): """Live migrate an instance with a PCI VF. This should fail because it's not possible to live migrate an instance - with a PCI passthrough device, even if it's a SR-IOV VF. + with a PCI passthrough device, even if it's a SR-IOV VF. Until we have + the correct version of qemu and libvirt. """ # start two compute services @@ -711,6 +750,1000 @@ class SRIOVServersTest(_PCIServersWithMigrationTestBase): self.assertEqual(500, ex.response.status_code) self.assertIn('NoValidHost', str(ex)) + def test_live_migrate_VF_success(self): + """Live migrate an instance with a PCI VF. + This should now work with the correct version of libvirt and qemu + """ + + PCI_DEVICE_SPEC = [jsonutils.dumps(x) for x in ( + { + 'vendor_id': fakelibvirt.PCI_VEND_ID, + 'product_id': fakelibvirt.VF_PROD_ID, + "live_migratable": "yes", + }, + )] + + PCI_ALIAS = { + "alias": [ + { + "vendor_id": fakelibvirt.PCI_VEND_ID, + "product_id": fakelibvirt.VF_PROD_ID, + "name": self.VFS_ALIAS_NAME, + "device_type": fields.PciDeviceType.SRIOV_VF, + "live_migratable": "yes", + }, + ], + "qty": [1], + } + + server = self._create_lm_server(PCI_DEVICE_SPEC, PCI_ALIAS) + src_xml = self._get_xml(self.comp0, server) + self.assertPCIDeviceCounts(self.comp0, total=1, free=0) + self.assertPCIDeviceCounts(self.comp1, total=1, free=1) + + self._live_migrate(server, "completed") + dst_xml = self._get_xml(self.comp1, server) + self.assertPCIDeviceCounts(self.comp0, total=1, free=1) + self.assertPCIDeviceCounts(self.comp1, total=1, free=0) + self._assertCompareHostdevs(src_xml, dst_xml) + + def test_live_migrate_VF_fails_lm_requested_no_lm_dev(self): + """Live migrate an instance with a non migratable PCI VF. + We should fail to create the instance because we request a + live migratable PCI device and there is only a non live migratable one. + """ + + PCI_DEVICE_SPEC = [jsonutils.dumps(x) for x in ( + { + 'vendor_id': fakelibvirt.PCI_VEND_ID, + 'product_id': fakelibvirt.VF_PROD_ID, + "live_migratable": "no", + }, + )] + + PCI_ALIAS = { + "alias": [ + { + "vendor_id": fakelibvirt.PCI_VEND_ID, + "product_id": fakelibvirt.VF_PROD_ID, + "name": self.VFS_ALIAS_NAME, + "device_type": fields.PciDeviceType.SRIOV_VF, + "live_migratable": "yes", + }, + ], + "qty": [1], + } + + # The AssertionError means the server failed to be created + # it fails on the assertion in _wait_for_state_change + self.assertRaises( + AssertionError, + self._create_lm_server, + PCI_DEVICE_SPEC, + PCI_ALIAS, + ) + + self.assertPCIDeviceCounts(self.comp0, total=1, free=1) + + def test_live_migrate_VF_fails_non_lm_requested(self): + """Live migrate an instance with a non migratable PCI VF. + We should manage to create the instance but fail to live migrate it. + """ + + PCI_DEVICE_SPEC = [jsonutils.dumps(x) for x in ( + { + 'vendor_id': fakelibvirt.PCI_VEND_ID, + 'product_id': fakelibvirt.VF_PROD_ID, + "live_migratable": "no", + }, + )] + + PCI_ALIAS = { + "alias": [ + { + "vendor_id": fakelibvirt.PCI_VEND_ID, + "product_id": fakelibvirt.VF_PROD_ID, + "name": self.VFS_ALIAS_NAME, + "device_type": fields.PciDeviceType.SRIOV_VF, + "live_migratable": "no", + }, + ], + "qty": [1], + } + + server = self._create_lm_server(PCI_DEVICE_SPEC, PCI_ALIAS) + self.assertPCIDeviceCounts(self.comp0, total=1, free=0) + self.assertPCIDeviceCounts(self.comp1, total=1, free=1) + + # The OpenStackApiException means the server failed to be migrated + exc = self.assertRaises( + client.OpenStackApiException, + self._live_migrate, + server, + "completed", + ) + self.assertEqual(500, exc.response.status_code) + self.assertIn('NoValidHost', str(exc)) + + self.assertPCIDeviceCounts(self.comp0, total=1, free=0) + self.assertPCIDeviceCounts(self.comp1, total=1, free=1) + self._wait_for_state_change(server, 'ACTIVE') + + def test_live_migrate_VF_fails_non_lm_reqeusted_only_lm_dev(self): + """Live migrate an instance with a live migratable PCI VF. + We requested a non live migratable PCI device and there is only a live + migratable one. Instance creation should fail. + """ + + PCI_DEVICE_SPEC = [jsonutils.dumps(x) for x in ( + { + 'vendor_id': fakelibvirt.PCI_VEND_ID, + 'product_id': fakelibvirt.VF_PROD_ID, + "live_migratable": "yes", + }, + )] + + PCI_ALIAS = { + "alias": [ + { + "vendor_id": fakelibvirt.PCI_VEND_ID, + "product_id": fakelibvirt.VF_PROD_ID, + "name": self.VFS_ALIAS_NAME, + "device_type": fields.PciDeviceType.SRIOV_VF, + "live_migratable": "no", + }, + ], + "qty": [1], + } + + # The AssertionError means the server failed to be created + # it fails on the assertion in _wait_for_state_change + self.assertRaises( + AssertionError, + self._create_lm_server, + PCI_DEVICE_SPEC, + PCI_ALIAS, + ) + + self.assertPCIDeviceCounts(self.comp0, total=1, free=1) + + def test_live_migrate_VF_fails_lm_requested_dev_unspecified(self): + """Live migrate an instance with a live migratable PCI VF. + We requested a live migratable PCI device and there is only a + device with live migratable not specified. Instance creation should + fail. + """ + + PCI_DEVICE_SPEC = [jsonutils.dumps(x) for x in ( + { + 'vendor_id': fakelibvirt.PCI_VEND_ID, + 'product_id': fakelibvirt.VF_PROD_ID, + }, + )] + + PCI_ALIAS = { + "alias": [ + { + "vendor_id": fakelibvirt.PCI_VEND_ID, + "product_id": fakelibvirt.VF_PROD_ID, + "name": self.VFS_ALIAS_NAME, + "device_type": fields.PciDeviceType.SRIOV_VF, + "live_migratable": "yes", + }, + ], + "qty": [1], + } + + # The AssertionError means the server failed to be created + # it fails on the assertion in _wait_for_state_change + self.assertRaises( + AssertionError, + self._create_lm_server, + PCI_DEVICE_SPEC, + PCI_ALIAS, + ) + self.assertPCIDeviceCounts(self.comp0, total=1, free=1) + + def test_live_migrate_VF_fails_non_lm_requested_dev_unspecified(self): + """Live migrate an instance with a live migratable PCI VF. + We requested a non live migratable PCI device and there is only a + device with live migratable not specified. Instance creation should + fail. + """ + + PCI_DEVICE_SPEC = [jsonutils.dumps(x) for x in ( + { + 'vendor_id': fakelibvirt.PCI_VEND_ID, + 'product_id': fakelibvirt.VF_PROD_ID, + }, + )] + + PCI_ALIAS = { + "alias": [ + { + "vendor_id": fakelibvirt.PCI_VEND_ID, + "product_id": fakelibvirt.VF_PROD_ID, + "name": self.VFS_ALIAS_NAME, + "device_type": fields.PciDeviceType.SRIOV_VF, + "live_migratable": "no", + }, + ], + "qty": [1], + } + + # The AssertionError means the server failed to be created + # it fails on the assertion in _wait_for_state_change + self.assertRaises( + AssertionError, + self._create_lm_server, + PCI_DEVICE_SPEC, + PCI_ALIAS, + ) + self.assertPCIDeviceCounts(self.comp0, total=1, free=1) + + def test_live_migrate_VF_fails_lm_requested_unspecified_lm_dev(self): + """Live migrate an instance with a live migratable PCI VF. + We have not specify any kind of live migratable PCI device in the + request and we have a migratable device, we should not migrate as + we could get non migratable device on the target host. + """ + + PCI_DEVICE_SPEC = [jsonutils.dumps(x) for x in ( + { + 'vendor_id': fakelibvirt.PCI_VEND_ID, + 'product_id': fakelibvirt.VF_PROD_ID, + "live_migratable": "yes", + }, + )] + + PCI_ALIAS = { + "alias": [ + { + "vendor_id": fakelibvirt.PCI_VEND_ID, + "product_id": fakelibvirt.VF_PROD_ID, + "name": self.VFS_ALIAS_NAME, + "device_type": fields.PciDeviceType.SRIOV_VF, + }, + ], + "qty": [1], + } + + server = self._create_lm_server(PCI_DEVICE_SPEC, PCI_ALIAS) + self.assertPCIDeviceCounts(self.comp0, total=1, free=0) + self.assertPCIDeviceCounts(self.comp1, total=1, free=1) + + # The OpenStackApiException means the server failed to be migrated + exc = self.assertRaises( + client.OpenStackApiException, + self._live_migrate, + server, + "completed", + ) + self.assertEqual(500, exc.response.status_code) + self.assertIn('NoValidHost', str(exc)) + self.assertPCIDeviceCounts(self.comp0, total=1, free=0) + self.assertPCIDeviceCounts(self.comp1, total=1, free=1) + self._wait_for_state_change(server, 'ACTIVE') + + def test_live_migrate_VF_fails_lm_requested_unspecified_no_lm_dev(self): + """Live migrate an instance with a live migratable PCI VF. + We have not specify any kind of live migratable PCI device and + we have a non migratable device. + """ + + PCI_DEVICE_SPEC = [jsonutils.dumps(x) for x in ( + { + 'vendor_id': fakelibvirt.PCI_VEND_ID, + 'product_id': fakelibvirt.VF_PROD_ID, + "live_migratable": "no", + }, + )] + + PCI_ALIAS = { + "alias": [ + { + "vendor_id": fakelibvirt.PCI_VEND_ID, + "product_id": fakelibvirt.VF_PROD_ID, + "name": self.VFS_ALIAS_NAME, + "device_type": fields.PciDeviceType.SRIOV_VF, + }, + ], + "qty": [1], + } + + server = self._create_lm_server(PCI_DEVICE_SPEC, PCI_ALIAS) + self.assertPCIDeviceCounts(self.comp0, total=1, free=0) + self.assertPCIDeviceCounts(self.comp1, total=1, free=1) + + # The OpenStackApiException means the server failed to be migrated + exc = self.assertRaises( + client.OpenStackApiException, + self._live_migrate, + server, + "completed", + ) + self.assertEqual(500, exc.response.status_code) + self.assertIn('NoValidHost', str(exc)) + self.assertPCIDeviceCounts(self.comp0, total=1, free=0) + self.assertPCIDeviceCounts(self.comp1, total=1, free=1) + self._wait_for_state_change(server, 'ACTIVE') + + def test_live_migrate_VF_fails_lm_requested_unspecif_unspecif_lm_dev(self): + """Live migrate an instance with a live migratable PCI VF. + We have not specify any kind of live migratable PCI device and + we have a non migratable device. + """ + + PCI_DEVICE_SPEC = [jsonutils.dumps(x) for x in ( + { + 'vendor_id': fakelibvirt.PCI_VEND_ID, + 'product_id': fakelibvirt.VF_PROD_ID, + }, + )] + + PCI_ALIAS = { + "alias": [ + { + "vendor_id": fakelibvirt.PCI_VEND_ID, + "product_id": fakelibvirt.VF_PROD_ID, + "name": self.VFS_ALIAS_NAME, + "device_type": fields.PciDeviceType.SRIOV_VF, + }, + ], + "qty": [1], + } + + server = self._create_lm_server(PCI_DEVICE_SPEC, PCI_ALIAS) + self.assertPCIDeviceCounts(self.comp0, total=1, free=0) + self.assertPCIDeviceCounts(self.comp1, total=1, free=1) + + # The OpenStackApiException means the server failed to be migrated + exc = self.assertRaises( + client.OpenStackApiException, + self._live_migrate, + server, + "completed", + ) + self.assertEqual(500, exc.response.status_code) + self.assertIn('NoValidHost', str(exc)) + self.assertPCIDeviceCounts(self.comp0, total=1, free=0) + self.assertPCIDeviceCounts(self.comp1, total=1, free=1) + self._wait_for_state_change(server, 'ACTIVE') + + def test_live_migrate_VF_success_3_VF(self): + """Live migrate an instance with 3 x PCI VF. + """ + + PCI_DEVICE_SPEC = [jsonutils.dumps(x) for x in ( + { + "vendor_id": fakelibvirt.PCI_VEND_ID, + "product_id": fakelibvirt.VF_PROD_ID, + "live_migratable": "yes", + "address": { + "domain": "00", + "bus": "8[1-2]", + "slot": "00", + "function": "[1-3]", + }, + }, + )] + + PCI_ALIAS = { + "alias": [ + { + "vendor_id": fakelibvirt.PCI_VEND_ID, + "product_id": fakelibvirt.VF_PROD_ID, + "name": self.VFS_ALIAS_NAME, + "device_type": fields.PciDeviceType.SRIOV_VF, + "live_migratable": "yes", + }, + ], + "qty": [3], + } + + server = self._create_lm_server(PCI_DEVICE_SPEC, PCI_ALIAS, num_vfs=3) + src_xml = self._get_xml(self.comp0, server) + self.assertPCIDeviceCounts(self.comp0, total=3, free=0) + self.assertPCIDeviceCounts(self.comp1, total=3, free=3) + + self._live_migrate(server, "completed") + dst_xml = self._get_xml(self.comp1, server) + self.assertPCIDeviceCounts(self.comp0, total=3, free=3) + self.assertPCIDeviceCounts(self.comp1, total=3, free=0) + self._assertCompareHostdevs(src_xml, dst_xml) + + def test_live_migrate_VF_fails_dest_no_lm_dev(self): + """Live migrate an instance with 3 x PCI VF. + Source live_migratable, dest non live_migratable. + """ + + PCI_DEVICE_SPEC = [jsonutils.dumps(x) for x in ( + { + "vendor_id": fakelibvirt.PCI_VEND_ID, + "product_id": fakelibvirt.VF_PROD_ID, + "live_migratable": "yes", + "address": { + "domain": "00", + "bus": "81", + "slot": "00", + "function": "[1-3]", + }, + }, + { + "vendor_id": fakelibvirt.PCI_VEND_ID, + "product_id": fakelibvirt.VF_PROD_ID, + "live_migratable": "no", + "address": { + "domain": "00", + "bus": "82", + "slot": "00", + "function": "[1-3]", + }, + }, + )] + + PCI_ALIAS = { + "alias": [ + { + "vendor_id": fakelibvirt.PCI_VEND_ID, + "product_id": fakelibvirt.VF_PROD_ID, + "name": self.VFS_ALIAS_NAME, + "device_type": fields.PciDeviceType.SRIOV_VF, + "live_migratable": "yes", + }, + ], + "qty": [3], + } + + server = self._create_lm_server(PCI_DEVICE_SPEC, PCI_ALIAS, num_vfs=3) + self.assertPCIDeviceCounts(self.comp0, total=3, free=0) + self.assertPCIDeviceCounts(self.comp1, total=3, free=3) + + # The OpenStackApiException means the server failed to be migrated + exc = self.assertRaises( + client.OpenStackApiException, + self._live_migrate, + server, + "completed", + ) + self.assertEqual(500, exc.response.status_code) + self.assertIn('NoValidHost', str(exc)) + self.assertPCIDeviceCounts(self.comp0, total=3, free=0) + self.assertPCIDeviceCounts(self.comp1, total=3, free=3) + self._wait_for_state_change(server, 'ACTIVE') + + def test_live_migrate_VF_fails_dest_unspecified_lm_dev(self): + """Live migrate an instance with 3 x PCI VF. + Source live_migratable, dest non live_migratable. + """ + + PCI_DEVICE_SPEC = [jsonutils.dumps(x) for x in ( + { + "vendor_id": fakelibvirt.PCI_VEND_ID, + "product_id": fakelibvirt.VF_PROD_ID, + "live_migratable": "yes", + "address": { + "domain": "00", + "bus": "81", + "slot": "00", + "function": "[1-3]", + }, + }, + { + "vendor_id": fakelibvirt.PCI_VEND_ID, + "product_id": fakelibvirt.VF_PROD_ID, + "address": { + "domain": "00", + "bus": "82", + "slot": "00", + "function": "[1-3]", + }, + }, + )] + + PCI_ALIAS = { + "alias": [ + { + "vendor_id": fakelibvirt.PCI_VEND_ID, + "product_id": fakelibvirt.VF_PROD_ID, + "name": self.VFS_ALIAS_NAME, + "device_type": fields.PciDeviceType.SRIOV_VF, + "live_migratable": "yes", + }, + ], + "qty": [3], + } + + server = self._create_lm_server(PCI_DEVICE_SPEC, PCI_ALIAS, num_vfs=3) + self.assertPCIDeviceCounts(self.comp0, total=3, free=0) + self.assertPCIDeviceCounts(self.comp1, total=3, free=3) + + # The OpenStackApiException means the server failed to be migrated + exc = self.assertRaises( + client.OpenStackApiException, + self._live_migrate, + server, + "completed", + ) + self.assertEqual(500, exc.response.status_code) + self.assertIn('NoValidHost', str(exc)) + self.assertPCIDeviceCounts(self.comp0, total=3, free=0) + self.assertPCIDeviceCounts(self.comp1, total=3, free=3) + self._wait_for_state_change(server, 'ACTIVE') + + def test_live_migrate_VF_fails_alias_mismatch_dev_prod_id(self): + """Live migrate an instance with 3 x PCI VF. + Source live_migratable, dest non live_migratable. + Incorrect alias + """ + + PCI_DEVICE_SPEC = [jsonutils.dumps(x) for x in ( + { + "vendor_id": fakelibvirt.PCI_VEND_ID, + "product_id": fakelibvirt.VF_PROD_ID, + "live_migratable": "yes", + "address": { + "domain": "00", + "bus": "81", + "slot": "00", + "function": "[1-3]", + }, + }, + { + "vendor_id": fakelibvirt.PCI_VEND_ID, + "product_id": fakelibvirt.VF_PROD_ID, + "address": { + "domain": "00", + "bus": "82", + "slot": "00", + "function": "[1-3]", + }, + }, + )] + + PCI_ALIAS = { + "alias": [ + { + "vendor_id": fakelibvirt.PCI_VEND_ID, + "product_id": '6666', + "name": self.VFS_ALIAS_NAME, + "device_type": fields.PciDeviceType.SRIOV_VF, + "live_migratable": "yes", + }, + ], + "qty": [3], + } + + # The AssertionError means the server failed to be created + # it fails on the assertion in _wait_for_state_change + self.assertRaises( + AssertionError, + self._create_lm_server, + PCI_DEVICE_SPEC, + PCI_ALIAS, + num_vfs=3 + ) + self.assertPCIDeviceCounts(self.comp0, total=3, free=3) + + def test_live_migrate_VF_success_2_aliases(self): + PCI_DEVICE_SPEC = [jsonutils.dumps(x) for x in ( + { + "vendor_id": fakelibvirt.PCI_VEND_ID, + "product_id": fakelibvirt.VF_PROD_ID, + "live_migratable": "yes", + "address": { + "domain": "00", + "bus": "8[1-2]", + "slot": "00", + "function": "[1-7]", + }, + }, + { + "vendor_id": fakelibvirt.PCI_VEND_ID, + "product_id": "6666", + "live_migratable": "yes", + "address": { + "domain": "00", + "bus": "8[1-2]", + "slot": "01", + "function": "[1-7]", + }, + }, + )] + + PCI_ALIAS = { + "alias": [ + { + "vendor_id": fakelibvirt.PCI_VEND_ID, + "product_id": fakelibvirt.VF_PROD_ID, + "name": self.VFS_ALIAS_NAME, + "device_type": fields.PciDeviceType.SRIOV_VF, + "live_migratable": "yes", + }, + { + "vendor_id": fakelibvirt.PCI_VEND_ID, + "product_id": '6666', + "name": "vfs2", + "device_type": fields.PciDeviceType.SRIOV_VF, + "live_migratable": "yes", + }, + ], + "qty": [1, 2], + } + + server = self._create_lm_server( + PCI_DEVICE_SPEC, PCI_ALIAS, num_pfs=1, num_vfs=4, + product_ids=[fakelibvirt.VF_PROD_ID, "6666"], + ) + src_xml = self._get_xml(self.comp0, server) + self.assertPCIDeviceCounts(self.comp0, total=8, free=5) + self.assertPCIDeviceCounts(self.comp1, total=8, free=8) + + self._live_migrate(server, "completed") + dst_xml = self._get_xml(self.comp1, server) + self.assertPCIDeviceCounts(self.comp0, total=8, free=8) + self.assertPCIDeviceCounts(self.comp1, total=8, free=5) + self._assertCompareHostdevs(src_xml, dst_xml) + + def test_live_migrate_VF_success_with_pci_in_placement(self): + PCI_DEVICE_SPEC = [jsonutils.dumps(x) for x in ( + { + "vendor_id": fakelibvirt.PCI_VEND_ID, + "product_id": fakelibvirt.VF_PROD_ID, + "live_migratable": "yes", + "address": { + "domain": "00", + "bus": "8[1-2]", + "slot": "00", + "function": "1", + }, + "resource_class": "CUSTOM_A16_16A", + }, + )] + + PCI_ALIAS = { + "alias": [ + { + "resource_class": "CUSTOM_A16_16A", + "name": self.VFS_ALIAS_NAME, + "device_type": fields.PciDeviceType.SRIOV_VF, + "live_migratable": "yes", + }, + ], + "qty": [1], + } + + self.flags(group="pci", report_in_placement=True) + self.flags(group='filter_scheduler', pci_in_placement=True) + + server = self._create_lm_server( + PCI_DEVICE_SPEC, PCI_ALIAS, num_pfs=1, num_vfs=1, + ) + self.assert_placement_pci_view( + self.comp0, + inventories={"0000:81:00.0": {'CUSTOM_A16_16A': 1}}, + traits={"0000:81:00.0": []}, + usages={"0000:81:00.0": {'CUSTOM_A16_16A': 1}}, + allocations={server['id']: { + "0000:81:00.0": {'CUSTOM_A16_16A': 1}}}, + ) + self.assert_placement_pci_view( + self.comp1, + inventories={"0000:82:00.0": {'CUSTOM_A16_16A': 1}}, + traits={"0000:82:00.0": []}, + usages={"0000:82:00.0": {'CUSTOM_A16_16A': 0}}, + ) + src_xml = self._get_xml(self.comp0, server) + self.assertPCIDeviceCounts(self.comp0, total=1, free=0) + self.assertPCIDeviceCounts(self.comp1, total=1, free=1) + + self._live_migrate(server, "completed") + self.assert_placement_pci_view( + self.comp0, + inventories={"0000:81:00.0": {'CUSTOM_A16_16A': 1}}, + traits={"0000:81:00.0": []}, + usages={"0000:81:00.0": {'CUSTOM_A16_16A': 0}}, + ) + self.assert_placement_pci_view( + self.comp1, + inventories={"0000:82:00.0": {'CUSTOM_A16_16A': 1}}, + traits={"0000:82:00.0": []}, + usages={"0000:82:00.0": {'CUSTOM_A16_16A': 1}}, + allocations={server['id']: { + "0000:82:00.0": {'CUSTOM_A16_16A': 1}}}, + ) + dst_xml = self._get_xml(self.comp1, server) + self.assertPCIDeviceCounts(self.comp0, total=1, free=1) + self.assertPCIDeviceCounts(self.comp1, total=1, free=0) + self._assertCompareHostdevs(src_xml, dst_xml) + + def test_live_migrate_VF_success_with_pip_3_dev_2_requested(self): + PCI_DEVICE_SPEC = [jsonutils.dumps(x) for x in ( + { + "vendor_id": fakelibvirt.PCI_VEND_ID, + "product_id": fakelibvirt.VF_PROD_ID, + "live_migratable": "yes", + "address": { + "domain": "00", + "bus": "8[1-2]", + "slot": "00", + "function": "[1-3]", + }, + "resource_class": "CUSTOM_A16_16A", + }, + )] + + PCI_ALIAS = { + "alias": [ + { + "resource_class": "CUSTOM_A16_16A", + "name": self.VFS_ALIAS_NAME, + "device_type": fields.PciDeviceType.SRIOV_VF, + "live_migratable": "yes", + }, + ], + "qty": [2], + } + + self.flags(group="pci", report_in_placement=True) + self.flags(group='filter_scheduler', pci_in_placement=True) + + server = self._create_lm_server( + PCI_DEVICE_SPEC, PCI_ALIAS, num_pfs=1, num_vfs=3, + ) + self.assert_placement_pci_view( + self.comp0, + inventories={"0000:81:00.0": {'CUSTOM_A16_16A': 3}}, + traits={"0000:81:00.0": []}, + usages={"0000:81:00.0": {'CUSTOM_A16_16A': 2}}, + allocations={server['id']: { + "0000:81:00.0": {'CUSTOM_A16_16A': 2}}}, + ) + self.assert_placement_pci_view( + self.comp1, + inventories={"0000:82:00.0": {'CUSTOM_A16_16A': 3}}, + traits={"0000:82:00.0": []}, + usages={"0000:82:00.0": {'CUSTOM_A16_16A': 0}}, + ) + src_xml = self._get_xml(self.comp0, server) + self.assertPCIDeviceCounts(self.comp0, total=3, free=1) + self.assertPCIDeviceCounts(self.comp1, total=3, free=3) + + self._live_migrate(server, "completed") + self.assert_placement_pci_view( + self.comp0, + inventories={"0000:81:00.0": {'CUSTOM_A16_16A': 3}}, + traits={"0000:81:00.0": []}, + usages={"0000:81:00.0": {'CUSTOM_A16_16A': 0}}, + ) + self.assert_placement_pci_view( + self.comp1, + inventories={"0000:82:00.0": {'CUSTOM_A16_16A': 3}}, + traits={"0000:82:00.0": []}, + usages={"0000:82:00.0": {'CUSTOM_A16_16A': 2}}, + allocations={server['id']: { + "0000:82:00.0": {'CUSTOM_A16_16A': 2}}}, + ) + dst_xml = self._get_xml(self.comp1, server) + self.assertPCIDeviceCounts(self.comp0, total=3, free=3) + self.assertPCIDeviceCounts(self.comp1, total=3, free=1) + self._assertCompareHostdevs(src_xml, dst_xml) + + def test_live_migrate_VF_success_with_pip_2_aliases(self): + PCI_DEVICE_SPEC = [jsonutils.dumps(x) for x in ( + { + "vendor_id": fakelibvirt.PCI_VEND_ID, + "product_id": fakelibvirt.VF_PROD_ID, + "live_migratable": "yes", + "address": { + "domain": "00", + "bus": "8[1-2]", + "slot": "00", + "function": "[1-3]", + }, + "resource_class": "CUSTOM_A16_16A", + }, + { + "vendor_id": fakelibvirt.PCI_VEND_ID, + "product_id": "6666", + "live_migratable": "yes", + "address": { + "domain": "00", + "bus": "8[1-2]", + "slot": "01", + "function": "[1-3]", + }, + "resource_class": "CUSTOM_A16_8A", + }, + )] + + PCI_ALIAS = { + "alias": [ + { + "resource_class": "CUSTOM_A16_16A", + "name": self.VFS_ALIAS_NAME, + "device_type": fields.PciDeviceType.SRIOV_VF, + "live_migratable": "yes", + }, + { + "resource_class": "CUSTOM_A16_8A", + "name": "vfs2", + "device_type": fields.PciDeviceType.SRIOV_VF, + "live_migratable": "yes", + }, + ], + "qty": [2, 2], + } + + self.flags(group="pci", report_in_placement=True) + self.flags(group='filter_scheduler', pci_in_placement=True) + + server = self._create_lm_server( + PCI_DEVICE_SPEC, PCI_ALIAS, num_pfs=1, num_vfs=3, + product_ids=[fakelibvirt.VF_PROD_ID, "6666"], + ) + self.assert_placement_pci_view( + self.comp0, + inventories={ + "0000:81:00.0": {"CUSTOM_A16_16A": 3}, + "0000:81:01.0": {"CUSTOM_A16_8A": 3}, + }, + traits={"0000:81:00.0": [], "0000:81:01.0": []}, + usages={ + "0000:81:00.0": {"CUSTOM_A16_16A": 2}, + "0000:81:01.0": {"CUSTOM_A16_8A": 2}, + }, + allocations={ + server["id"]: { + "0000:81:00.0": {"CUSTOM_A16_16A": 2}, + "0000:81:01.0": {"CUSTOM_A16_8A": 2}, + } + }, + ) + self.assert_placement_pci_view( + self.comp1, + inventories={ + "0000:82:00.0": {"CUSTOM_A16_16A": 3}, + "0000:82:01.0": {"CUSTOM_A16_8A": 3}, + }, + traits={"0000:82:00.0": [], "0000:82:01.0": []}, + usages={ + "0000:82:00.0": {"CUSTOM_A16_16A": 0}, + "0000:82:01.0": {"CUSTOM_A16_8A": 0}, + }, + ) + src_xml = self._get_xml(self.comp0, server) + self.assertPCIDeviceCounts(self.comp0, total=6, free=2) + self.assertPCIDeviceCounts(self.comp1, total=6, free=6) + + self._live_migrate(server, "completed") + self.assert_placement_pci_view( + self.comp0, + inventories={ + "0000:81:00.0": {"CUSTOM_A16_16A": 3}, + "0000:81:01.0": {"CUSTOM_A16_8A": 3}, + }, + traits={"0000:81:00.0": [], "0000:81:01.0": []}, + usages={ + "0000:81:00.0": {"CUSTOM_A16_16A": 0}, + "0000:81:01.0": {"CUSTOM_A16_8A": 0}, + }, + ) + self.assert_placement_pci_view( + self.comp1, + inventories={ + "0000:82:00.0": {"CUSTOM_A16_16A": 3}, + "0000:82:01.0": {"CUSTOM_A16_8A": 3}, + }, + traits={"0000:82:00.0": [], "0000:82:01.0": []}, + usages={ + "0000:82:00.0": {"CUSTOM_A16_16A": 2}, + "0000:82:01.0": {"CUSTOM_A16_8A": 2}, + }, + allocations={ + server["id"]: { + "0000:82:00.0": {"CUSTOM_A16_16A": 2}, + "0000:82:01.0": {"CUSTOM_A16_8A": 2}, + } + }, + ) + dst_xml = self._get_xml(self.comp1, server) + self.assertPCIDeviceCounts(self.comp0, total=6, free=6) + self.assertPCIDeviceCounts(self.comp1, total=6, free=2) + self._assertCompareHostdevs(src_xml, dst_xml) + + def _create_lm_server( + self, device_spec, alias, num_pfs=1, num_vfs=1, product_ids=["1515"] + ): + + alias_def = [ + jsonutils.dumps(x) + for x in alias["alias"] + ] + + self.flags( + device_spec=device_spec, + alias=alias_def, + group='pci' + ) + + flavor_req = ",".join([ + f"{alias['alias'][index]['name']}:{alias['qty'][index]}" + for index in range(0, len(alias["alias"])) + ]) + + self.comp0 = self.start_compute( + hostname="test_compute0", + libvirt_version=versionutils.convert_version_to_int( + driver.MIN_VFIO_PCI_VARIANT_LIBVIRT_VERSION + ), + qemu_version=versionutils.convert_version_to_int( + driver.MIN_VFIO_PCI_VARIANT_QEMU_VERSION + ), + pci_info=fakelibvirt.HostPCIDevicesInfo( + num_pfs=num_pfs, num_vfs=num_vfs, product_ids=product_ids + ), + ) + + # Create a server here to ensure it goes to first compute. + extra_spec = { + "pci_passthrough:alias": f"{flavor_req}" + } + flavor_id = self._create_flavor(extra_spec=extra_spec) + server = self._create_server(flavor_id=flavor_id, networks='none') + + self.comp1 = self.start_compute( + hostname="test_compute1", + libvirt_version=versionutils.convert_version_to_int( + driver.MIN_VFIO_PCI_VARIANT_LIBVIRT_VERSION + ), + qemu_version=versionutils.convert_version_to_int( + driver.MIN_VFIO_PCI_VARIANT_QEMU_VERSION + ), + pci_info=fakelibvirt.HostPCIDevicesInfo( + num_pfs=num_pfs, num_vfs=num_vfs, + product_ids=product_ids, bus=0x82 + ), + ) + + return server + + def _get_hostdev_addresses(self, xml): + addresses = [] + tree = etree.fromstring(xml) + devices = tree.find('./devices') + hostdevs = devices.findall('./hostdev') + + for hostdev in hostdevs: + address = hostdev.find("./source/address") + addresses.append({ + 'domain': address.get('domain'), + 'bus': address.get('bus'), + 'slot': address.get('slot'), + 'function': address.get('function') + }) + + return addresses + + def _get_xml(self, compute, server): + host = self.computes[compute].driver._host + guest = host.get_guest( + instance.Instance.get_by_uuid(self.ctxt, server["id"]) + ) + xml = guest.get_xml_desc() + return xml + + def _assertCompareHostdevs(self, xml_src, xml_dst): + src_addresses = self._get_hostdev_addresses(xml_src) + dst_addresses = self._get_hostdev_addresses(xml_dst) + + self.assertEqual(len(src_addresses), len(dst_addresses)) + + for src_addr in src_addresses: + # Switch bus to destination one. + src_addr["bus"] = "0x82" + self.assertIn(src_addr, dst_addresses) + def _test_move_operation_with_neutron(self, move_operation, expect_fail=False): # The purpose here is to force an observable PCI slot update when diff --git a/nova/tests/unit/virt/libvirt/test_migration.py b/nova/tests/unit/virt/libvirt/test_migration.py index 943b22ba2c6c..38680f10cf99 100644 --- a/nova/tests/unit/virt/libvirt/test_migration.py +++ b/nova/tests/unit/virt/libvirt/test_migration.py @@ -35,6 +35,14 @@ from nova.virt.libvirt import host from nova.virt.libvirt import migration +def _normalize(xml_str): + return etree.tostring( + etree.fromstring(xml_str), + pretty_print=True, + encoding="unicode", + ).strip() + + class UtilityMigrationTestCase(test.NoDBTestCase): def test_graphics_listen_addrs(self): @@ -278,6 +286,193 @@ class UtilityMigrationTestCase(test.NoDBTestCase): self.assertRaises(exception.NovaException, migration._update_mdev_xml, doc, data.target_mdevs) + def test_update_pci_dev_xml(self): + + xml_pattern = """ + + + + +
+ + +
+ + +""" + expected_xml_pattern = """ + + + + +
+ + +
+ + +""" + data = objects.LibvirtLiveMigrateData( + pci_dev_map_src_dst={"0000:25:00.4": "0000:26:01.5"}) + doc = etree.fromstring(xml_pattern) + res = migration._update_pci_dev_xml(doc, data.pci_dev_map_src_dst) + self.assertEqual( + _normalize(expected_xml_pattern), + etree.tostring(res, encoding="unicode", pretty_print=True).strip(), + ) + + def test_update_pci_dev_xml_with_2_hostdevs(self): + + xml_pattern = """ + + + + +
+ + +
+ + + + +
+ + +
+ + +""" + expected_xml_pattern = """ + + + + +
+ + +
+ + + + +
+ + +
+ + +""" + data = objects.LibvirtLiveMigrateData( + pci_dev_map_src_dst={ + "0000:25:00.4": "0000:26:01.5", + "0000:25:01.4": "0000:26:01.4", + } + ) + doc = etree.fromstring(xml_pattern) + res = migration._update_pci_dev_xml(doc, data.pci_dev_map_src_dst) + self.assertEqual( + _normalize(expected_xml_pattern), + etree.tostring(res, encoding="unicode", pretty_print=True).strip(), + ) + + def test_update_pci_dev_xml_with_2_hostdevs_second_one_not_in_map(self): + + xml_pattern = """ + + + + +
+ + +
+ + + + +
+ + +
+ + +""" + expected_xml_pattern = """ + + + + +
+ + +
+ + + + +
+ + +
+ + +""" + data = objects.LibvirtLiveMigrateData( + pci_dev_map_src_dst={ + "0000:25:00.4": "0000:26:01.5", + } + ) + doc = etree.fromstring(xml_pattern) + res = migration._update_pci_dev_xml(doc, data.pci_dev_map_src_dst) + self.assertEqual( + _normalize(expected_xml_pattern), + etree.tostring(res, encoding="unicode", pretty_print=True).strip(), + ) + + def test_update_pci_dev_xml_fails_not_found_src_address(self): + xml_pattern = """ + + + + +
+ + +
+ + +""" + data = objects.LibvirtLiveMigrateData( + pci_dev_map_src_dst={"0000:25:00.5": "0000:26:01.5"}) + doc = etree.fromstring(xml_pattern) + exc = self.assertRaises( + exception.NovaException, + migration._update_pci_dev_xml, + doc, + data.pci_dev_map_src_dst, + ) + + norm = _normalize(xml_pattern) + + self.assertIn( + 'Unable to find the hostdev ' + f'to replace for this source PCI address: 0000:25:00.5 ' + f'in the xml: {norm}', + str(exc), + ) + def test_update_cpu_shared_set_xml(self): doc = etree.fromstring(""" diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py index cfb22b5cbd1f..f820fa91ec28 100644 --- a/nova/virt/libvirt/driver.py +++ b/nova/virt/libvirt/driver.py @@ -97,6 +97,7 @@ from nova.objects import diagnostics as diagnostics_obj from nova.objects import fields from nova.objects import migrate_data as migrate_data_obj from nova.pci import utils as pci_utils +from nova.pci import whitelist import nova.privsep.libvirt import nova.privsep.path import nova.privsep.utils @@ -266,6 +267,10 @@ MIN_LIBVIRT_STATELESS_FIRMWARE = (8, 6, 0) MIN_IGB_LIBVIRT_VERSION = (9, 3, 0) MIN_IGB_QEMU_VERSION = (8, 0, 0) +# Minimum versions supporting vfio-pci variant driver. +MIN_VFIO_PCI_VARIANT_LIBVIRT_VERSION = (10, 0, 0) +MIN_VFIO_PCI_VARIANT_QEMU_VERSION = (8, 2, 2) + REGISTER_IMAGE_PROPERTY_DEFAULTS = [ 'hw_machine_type', 'hw_cdrom_bus', @@ -902,10 +907,35 @@ class LibvirtDriver(driver.ComputeDriver): self._check_multipath() + # Even if we already checked the whitelist at startup, this driver + # needs to check specific hypervisor versions + self._check_pci_whitelist() + # Set REGISTER_IMAGE_PROPERTY_DEFAULTS in the instance system_metadata # to default values for properties that have not already been set. self._register_all_undefined_instance_details() + def _check_pci_whitelist(self): + + need_specific_version = False + + if CONF.pci.device_spec: + pci_whitelist = whitelist.Whitelist(CONF.pci.device_spec) + for spec in pci_whitelist.specs: + if spec.tags.get("live_migratable"): + need_specific_version = True + + if need_specific_version and not self._host.has_min_version( + lv_ver=MIN_VFIO_PCI_VARIANT_LIBVIRT_VERSION, + hv_ver=MIN_VFIO_PCI_VARIANT_QEMU_VERSION, + hv_type=host.HV_DRIVER_QEMU, + ): + msg = _( + "PCI device spec is configured for " + "live_migratable but it's not supported by libvirt." + ) + raise exception.InvalidConfiguration(msg) + def _update_host_specific_capabilities(self) -> None: """Update driver capabilities based on capabilities of the host.""" # TODO(stephenfin): We should also be reporting e.g. SEV functionality diff --git a/nova/virt/libvirt/migration.py b/nova/virt/libvirt/migration.py index ac632a7e2e50..53d148b5557a 100644 --- a/nova/virt/libvirt/migration.py +++ b/nova/virt/libvirt/migration.py @@ -16,7 +16,6 @@ """Utility methods to manage guests migration """ - from collections import deque from lxml import etree @@ -88,6 +87,11 @@ def get_updated_guest_xml(instance, guest, migrate_data, get_volume_config, xml_doc = _update_numa_xml(xml_doc, migrate_data) if 'target_mdevs' in migrate_data: xml_doc = _update_mdev_xml(xml_doc, migrate_data.target_mdevs) + if "pci_dev_map_src_dst" in migrate_data: + xml_doc = _update_pci_dev_xml( + xml_doc, migrate_data.pci_dev_map_src_dst + ) + if new_resources: xml_doc = _update_device_resources_xml(xml_doc, new_resources) return etree.tostring(xml_doc, encoding='unicode') @@ -149,6 +153,77 @@ def _update_mdev_xml(xml_doc, target_mdevs): return xml_doc +def _update_pci_dev_xml(xml_doc, pci_dev_map_src_dst): + hostdevs = xml_doc.findall('./devices/hostdev') + + for src_addr, dst_addr in pci_dev_map_src_dst.items(): + src_fields = _get_pci_address_fields_with_prefix(src_addr) + dst_fields = _get_pci_address_fields_with_prefix(dst_addr) + + if not _update_hostdev_address(hostdevs, src_fields, dst_fields): + _raise_hostdev_not_found_exception(xml_doc, src_addr) + + LOG.debug( + '_update_pci_xml output xml=%s', + etree.tostring(xml_doc, encoding='unicode', pretty_print=True) + ) + return xml_doc + + +def _get_pci_address_fields_with_prefix(addr): + (domain, bus, slot, func) = nova.pci.utils.get_pci_address_fields(addr) + return (f"0x{domain}", f"0x{bus}", f"0x{slot}", f"0x{func}") + + +def _update_hostdev_address(hostdevs, src_fields, dst_fields): + src_domain, src_bus, src_slot, src_function = src_fields + dst_domain, dst_bus, dst_slot, dst_function = dst_fields + + for hostdev in hostdevs: + if hostdev.get('type') != 'pci': + continue + + address_tag = hostdev.find('./source/address') + if address_tag is None: + continue + + if _address_matches( + address_tag, src_domain, src_bus, src_slot, src_function + ): + _set_address_fields( + address_tag, dst_domain, dst_bus, dst_slot, dst_function + ) + return True + + return False + + +def _address_matches(address_tag, domain, bus, slot, function): + return ( + address_tag.get('domain') == domain and + address_tag.get('bus') == bus and + address_tag.get('slot') == slot and + address_tag.get('function') == function + ) + + +def _set_address_fields(address_tag, domain, bus, slot, function): + address_tag.set('domain', domain) + address_tag.set('bus', bus) + address_tag.set('slot', slot) + address_tag.set('function', function) + + +def _raise_hostdev_not_found_exception(xml_doc, src_addr): + xml = etree.tostring( + xml_doc, encoding="unicode", pretty_print=True + ).strip() + raise exception.NovaException( + 'Unable to find the hostdev to replace for this source PCI ' + f'address: {src_addr} in the xml: {xml}' + ) + + def _update_cpu_shared_set_xml(xml_doc, migrate_data): LOG.debug('_update_cpu_shared_set_xml input xml=%s', etree.tostring(xml_doc, encoding='unicode', pretty_print=True)) diff --git a/releasenotes/notes/migrate-vfio-devices-using-kernel-variant-drivers-d4180849f973012e.yaml b/releasenotes/notes/migrate-vfio-devices-using-kernel-variant-drivers-d4180849f973012e.yaml new file mode 100644 index 000000000000..52d4ef75010b --- /dev/null +++ b/releasenotes/notes/migrate-vfio-devices-using-kernel-variant-drivers-d4180849f973012e.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + This release adds support for migrating SR-IOV devices + using the new kernel VFIO SR-IOV variant driver interface. + See the `OpenStack configuration documentation`__ for more details. + + .. __: https://docs.openstack.org/nova/latest/configuration/config.html#pci