From a234bbf80c848fb23f613383e94b68bf336231a8 Mon Sep 17 00:00:00 2001 From: Marcin Juszkiewicz Date: Thu, 15 Feb 2018 17:50:38 +0100 Subject: [PATCH] Allow to configure amount of PCIe ports On x86-64/q35 and aarch64/virt instances libvirt adds as many pcie-root-port entries (aka virtual pcie slots) as it needs and adds one free. If we want to hotplug network interfaces or storage devices then we quickly run out of available pcie slots. This patch allows to configure amount of PCIe slots in instance. Method was discussed with upstream libvirt developers. To have requested amount of pcie-root-port entries we have to create whole PCIe structure starting with pcie-root/0 and then add as many pcie-root-port/0 entries as we want slots. Too low value may get bumped by libvirt to same as amount of inserted cards. Systems not using new option will work same way as they did. Implements: bp configure-amount-of-pcie-ports Change-Id: Ic3c8761bcde3e842d1b8e1feff1d158630de59ae --- nova/conf/libvirt.py | 19 +++++ nova/tests/unit/virt/libvirt/test_driver.py | 84 +++++++++++++++++++ nova/virt/libvirt/config.py | 18 ++++ nova/virt/libvirt/driver.py | 27 ++++++ ...amount-of-pcie-ports-486bfa44e9fbdd84.yaml | 9 ++ 5 files changed, 157 insertions(+) create mode 100644 releasenotes/notes/configure-amount-of-pcie-ports-486bfa44e9fbdd84.yaml diff --git a/nova/conf/libvirt.py b/nova/conf/libvirt.py index e8aee6b3842a..0da5f9456692 100644 --- a/nova/conf/libvirt.py +++ b/nova/conf/libvirt.py @@ -636,6 +636,25 @@ Possible values: The supported events list can be found in https://libvirt.org/html/libvirt-libvirt-domain.html , which you may need to search key words ``VIR_PERF_PARAM_*`` +"""), + cfg.IntOpt('num_pcie_ports', + default=0, + min=0, + max=28, + help= """ +The number of PCIe ports an instance will get. + +Libvirt allows a custom number of PCIe ports (pcie-root-port controllers) a +target instance will get. Some will be used by default, rest will be available +for hotplug use. + +By default we have just 1-2 free ports which limits hotplug. + +More info: https://github.com/qemu/qemu/blob/master/docs/pcie.txt + +Due to QEMU limitations for aarch64/virt maximum value is set to '28'. + +Default value '0' moves calculating amount of ports to libvirt. """), ] diff --git a/nova/tests/unit/virt/libvirt/test_driver.py b/nova/tests/unit/virt/libvirt/test_driver.py index eead31416a54..dd8c61a28d8d 100644 --- a/nova/tests/unit/virt/libvirt/test_driver.py +++ b/nova/tests/unit/virt/libvirt/test_driver.py @@ -2137,6 +2137,75 @@ class LibvirtConnTestCase(test.NoDBTestCase, self.assertEqual(33550336, cfg.metadata[0].flavor.swap) + def test_get_guest_config_q35(self): + self.flags(virt_type="kvm", + group='libvirt') + + TEST_AMOUNT_OF_PCIE_SLOTS = 8 + CONF.set_override("num_pcie_ports", TEST_AMOUNT_OF_PCIE_SLOTS, + group='libvirt') + + drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True) + instance_ref = objects.Instance(**self.test_instance) + image_meta = objects.ImageMeta.from_dict({ + "disk_format": "raw", + "properties": {"hw_machine_type": + "pc-q35-test"}}) + + disk_info = blockinfo.get_disk_info(CONF.libvirt.virt_type, + instance_ref, + image_meta) + + cfg = drvr._get_guest_config(instance_ref, + _fake_network_info(self, 1), + image_meta, disk_info) + + num_ports = 0 + for device in cfg.devices: + try: + if (device.root_name == 'controller' and + device.model == 'pcie-root-port'): + num_ports += 1 + except AttributeError: + pass + + self.assertEqual(TEST_AMOUNT_OF_PCIE_SLOTS, num_ports) + + def test_get_guest_config_pcie_i440fx(self): + self.flags(virt_type="kvm", + group='libvirt') + + TEST_AMOUNT_OF_PCIE_SLOTS = 8 + CONF.set_override("num_pcie_ports", TEST_AMOUNT_OF_PCIE_SLOTS, + group='libvirt') + + drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True) + instance_ref = objects.Instance(**self.test_instance) + image_meta = objects.ImageMeta.from_dict({ + "disk_format": "raw", + "properties": {"hw_machine_type": + "pc-i440fx-test"}}) + + disk_info = blockinfo.get_disk_info(CONF.libvirt.virt_type, + instance_ref, + image_meta) + + cfg = drvr._get_guest_config(instance_ref, + _fake_network_info(self, 1), + image_meta, disk_info) + + num_ports = 0 + for device in cfg.devices: + try: + if (device.root_name == 'controller' and + device.model == 'pcie-root-port'): + num_ports += 1 + except AttributeError: + pass + + # i440fx is not pcie machine so there should be no pcie ports + self.assertEqual(0, num_ports) + def test_get_guest_config_missing_ownership_info(self): drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True) @@ -5647,6 +5716,10 @@ class LibvirtConnTestCase(test.NoDBTestCase, caps.host.cpu = cpu return caps + TEST_AMOUNT_OF_PCIE_SLOTS = 8 + CONF.set_override("num_pcie_ports", TEST_AMOUNT_OF_PCIE_SLOTS, + group='libvirt') + self.flags(virt_type="kvm", group="libvirt") @@ -5669,6 +5742,17 @@ class LibvirtConnTestCase(test.NoDBTestCase, libvirt_driver.DEFAULT_UEFI_LOADER_PATH['aarch64']) self.assertEqual(cfg.os_mach_type, "virt") + num_ports = 0 + for device in cfg.devices: + try: + if (device.root_name == 'controller' and + device.model == 'pcie-root-port'): + num_ports += 1 + except AttributeError: + pass + + self.assertEqual(TEST_AMOUNT_OF_PCIE_SLOTS, num_ports) + @mock.patch.object(libvirt_driver.LibvirtDriver, "_get_guest_storage_config") @mock.patch.object(libvirt_driver.LibvirtDriver, "_has_numa_support") diff --git a/nova/virt/libvirt/config.py b/nova/virt/libvirt/config.py index 0c0e6ee0f953..d193553a5e0c 100644 --- a/nova/virt/libvirt/config.py +++ b/nova/virt/libvirt/config.py @@ -1645,6 +1645,24 @@ class LibvirtConfigGuestUSBHostController(LibvirtConfigGuestController): self.type = 'usb' +class LibvirtConfigGuestPCIeRootController(LibvirtConfigGuestController): + + def __init__(self, **kwargs): + super(LibvirtConfigGuestPCIeRootController, self).\ + __init__(**kwargs) + self.type = 'pci' + self.model = 'pcie-root' + + +class LibvirtConfigGuestPCIeRootPortController(LibvirtConfigGuestController): + + def __init__(self, **kwargs): + super(LibvirtConfigGuestPCIeRootPortController, self).\ + __init__(**kwargs) + self.type = 'pci' + self.model = 'pcie-root-port' + + class LibvirtConfigGuestHostdev(LibvirtConfigGuestDevice): def __init__(self, **kwargs): super(LibvirtConfigGuestHostdev, self).\ diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py index cef539d09234..360739bbe9cd 100644 --- a/nova/virt/libvirt/driver.py +++ b/nova/virt/libvirt/driver.py @@ -5028,6 +5028,23 @@ class LibvirtDriver(driver.ComputeDriver): cpu_config.features.add(xf) return cpu_config + def _guest_add_pcie_root_ports(self, guest): + """Add PCI Express root ports. + + PCI Express machine can have as many PCIe devices as it has + pcie-root-port controllers (slots in virtual motherboard). + + If we want to have more PCIe slots for hotplug then we need to create + whole PCIe structure (libvirt limitation). + """ + + pcieroot = vconfig.LibvirtConfigGuestPCIeRootController() + guest.add_device(pcieroot) + + for x in range(0, CONF.libvirt.num_pcie_ports): + pcierootport = vconfig.LibvirtConfigGuestPCIeRootPortController() + guest.add_device(pcierootport) + def _guest_add_usb_host_keyboard(self, guest): """Add USB Host controller and keyboard for graphical console use. @@ -5158,6 +5175,16 @@ class LibvirtDriver(driver.ComputeDriver): if virt_type in ('qemu', 'kvm'): self._set_qemu_guest_agent(guest, flavor, instance, image_meta) + # Add PCIe root port controllers for PCI Express machines + # but only if their amount is configured + if (CONF.libvirt.num_pcie_ports and + ((caps.host.cpu.arch == fields.Architecture.AARCH64 and + guest.os_mach_type.startswith('virt')) or + (caps.host.cpu.arch == fields.Architecture.X86_64 and + guest.os_mach_type is not None and + 'q35' in guest.os_mach_type))): + self._guest_add_pcie_root_ports(guest) + self._guest_add_pci_devices(guest, instance) self._guest_add_watchdog_action(guest, flavor, image_meta) diff --git a/releasenotes/notes/configure-amount-of-pcie-ports-486bfa44e9fbdd84.yaml b/releasenotes/notes/configure-amount-of-pcie-ports-486bfa44e9fbdd84.yaml new file mode 100644 index 000000000000..d8a0e1ad037f --- /dev/null +++ b/releasenotes/notes/configure-amount-of-pcie-ports-486bfa44e9fbdd84.yaml @@ -0,0 +1,9 @@ +--- +features: + - | + The amount of PCI Express ports (slots in virtual motherboard) can now be + configured using ``num_pcie_ports`` option in ``libvirt`` section of + ``nova.conf`` file. This affects x86-64 with ``hw_machine_type`` set to + 'pc-q35' value and AArch64 instances of 'virt' ``hw_machine_type`` (which + is default for that architecture). Due to QEMU's memory map limits on + aarch64/virt maximum value is limited to 28.