diff --git a/doc/source/admin/file-backed-memory.rst b/doc/source/admin/file-backed-memory.rst index 4e77d81aa3b2..99cff4fcc3dc 100644 --- a/doc/source/admin/file-backed-memory.rst +++ b/doc/source/admin/file-backed-memory.rst @@ -21,6 +21,10 @@ file-backed memory to placement in place of the total system memory capacity. This allows the node to run more instances than would normally fit within system memory. +When available in libivrt and qemu, instance memory will be discarded by qemu +at shutdown by calling madvise(MADV_REMOVE), to avoid flushing any dirty memory +to the backing store on exit. + To enable file-backed memory, follow the steps below: #. `Configure the backing store`_ @@ -39,9 +43,11 @@ Prerequisites and Limitations Libvirt File-backed memory requires libvirt version 4.0.0 or newer + Discard capability requires libvirt ersion 4.4.0 or newer Qemu File-backed memory requires qemu version 2.6.0 or newer + Discard capability requires qemu version 2.10.0 or newer Memory overcommit File-backed memory is not compatible with memory overcommit. diff --git a/nova/objects/migrate_data.py b/nova/objects/migrate_data.py index 36bb93b00977..699c993cbd36 100644 --- a/nova/objects/migrate_data.py +++ b/nova/objects/migrate_data.py @@ -134,7 +134,8 @@ class LibvirtLiveMigrateData(LiveMigrateData): # Version 1.5: Added src_supports_native_luks # Version 1.6: Added wait_for_vif_plugged # Version 1.7: Added dst_wants_file_backed_memory - VERSION = '1.7' + # Version 1.8: Added file_backed_memory_discard + VERSION = '1.8' fields = { 'filename': fields.StringField(), @@ -155,12 +156,18 @@ class LibvirtLiveMigrateData(LiveMigrateData): 'supported_perf_events': fields.ListOfStringsField(), 'src_supports_native_luks': fields.BooleanField(), 'dst_wants_file_backed_memory': fields.BooleanField(), + # file_backed_memory_discard is ignored unless + # dst_wants_file_backed_memory is set + 'file_backed_memory_discard': fields.BooleanField(), } def obj_make_compatible(self, primitive, target_version): super(LibvirtLiveMigrateData, self).obj_make_compatible( primitive, target_version) target_version = versionutils.convert_version_to_tuple(target_version) + if target_version < (1, 8): + if 'file_backed_memory_discard' in primitive: + del primitive['file_backed_memory_discard'] if target_version < (1, 7): if 'dst_wants_file_backed_memory' in primitive: del primitive['dst_wants_file_backed_memory'] diff --git a/nova/tests/unit/objects/test_migrate_data.py b/nova/tests/unit/objects/test_migrate_data.py index d27f26264dfb..43f3618b1acd 100644 --- a/nova/tests/unit/objects/test_migrate_data.py +++ b/nova/tests/unit/objects/test_migrate_data.py @@ -220,7 +220,9 @@ class _TestLibvirtLiveMigrateData(object): instance_relative_path='foo/bar', graphics_listen_addrs={'vnc': '127.0.0.1'}, serial_listen_addr='127.0.0.1', - bdms=[test_bdmi]) + bdms=[test_bdmi], + dst_wants_file_backed_memory=True, + file_backed_memory_discard=True) obj2 = migrate_data.LibvirtLiveMigrateData() obj2.from_legacy_dict(obj.to_legacy_dict(pre_migration_result=True)) self.assertEqual(obj.to_legacy_dict(), @@ -256,6 +258,8 @@ class _TestLibvirtLiveMigrateData(object): self.assertNotIn('src_supports_native_luks', primitive) primitive = data(obj.obj_to_primitive(target_version='1.6')) self.assertNotIn('dst_wants_file_backed_memory', primitive) + primitive = data(obj.obj_to_primitive(target_version='1.7')) + self.assertNotIn('file_backed_memory_discard', primitive) def test_bdm_obj_make_compatible(self): obj = migrate_data.LibvirtLiveMigrateBDMInfo( diff --git a/nova/tests/unit/objects/test_objects.py b/nova/tests/unit/objects/test_objects.py index fae662f4f515..457446908bb2 100644 --- a/nova/tests/unit/objects/test_objects.py +++ b/nova/tests/unit/objects/test_objects.py @@ -1114,7 +1114,7 @@ object_data = { 'InstancePCIRequest': '1.2-6344dd8bd1bf873e7325c07afe47f774', 'InstancePCIRequests': '1.1-65e38083177726d806684cb1cc0136d2', 'LibvirtLiveMigrateBDMInfo': '1.1-5f4a68873560b6f834b74e7861d71aaf', - 'LibvirtLiveMigrateData': '1.7-746b3163f022f8811da62afa035ecf66', + 'LibvirtLiveMigrateData': '1.8-7806711ede43f4d0afd246d045b7d726', 'KeyPair': '1.4-1244e8d1b103cc69d038ed78ab3a8cc6', 'KeyPairList': '1.3-94aad3ac5c938eef4b5e83da0212f506', 'MemoryDiagnostics': '1.0-2c995ae0f2223bb0f8e523c5cc0b83da', diff --git a/nova/tests/unit/virt/libvirt/test_config.py b/nova/tests/unit/virt/libvirt/test_config.py index 2c231662188c..b68af754aabe 100644 --- a/nova/tests/unit/virt/libvirt/test_config.py +++ b/nova/tests/unit/virt/libvirt/test_config.py @@ -3410,12 +3410,14 @@ class LibvirtConfigGuestMemoryBackingTest(LibvirtConfigBaseTest): obj.sharedaccess = True obj.allocateimmediate = True obj.filesource = True + obj.discard = True xml = obj.to_xml() self.assertXmlEqual(xml, """ + """) diff --git a/nova/tests/unit/virt/libvirt/test_driver.py b/nova/tests/unit/virt/libvirt/test_driver.py index e2769b199a33..8f1678ead8ad 100644 --- a/nova/tests/unit/virt/libvirt/test_driver.py +++ b/nova/tests/unit/virt/libvirt/test_driver.py @@ -2446,6 +2446,54 @@ class LibvirtConnTestCase(test.NoDBTestCase, self.assertTrue(result.filesource) self.assertTrue(result.allocateimmediate) + @mock.patch.object(fakelibvirt.Connection, 'getVersion') + @mock.patch.object(fakelibvirt.Connection, 'getLibVersion') + def test_get_guest_memory_backing_config_file_backed_discard(self, + mock_lib_version, mock_version): + self.flags(file_backed_memory=1024, group='libvirt') + + mock_lib_version.return_value = versionutils.convert_version_to_int( + libvirt_driver.MIN_LIBVIRT_FILE_BACKED_DISCARD_VERSION) + mock_version.return_value = versionutils.convert_version_to_int( + libvirt_driver.MIN_QEMU_FILE_BACKED_DISCARD_VERSION) + + result = self._test_get_guest_memory_backing_config( + None, None, None + ) + self.assertTrue(result.discard) + + @mock.patch.object(fakelibvirt.Connection, 'getVersion') + @mock.patch.object(fakelibvirt.Connection, 'getLibVersion') + def test_get_guest_memory_backing_config_file_backed_discard_libvirt(self, + mock_lib_version, mock_version): + self.flags(file_backed_memory=1024, group='libvirt') + + mock_lib_version.return_value = versionutils.convert_version_to_int( + libvirt_driver.MIN_LIBVIRT_FILE_BACKED_DISCARD_VERSION) - 1 + mock_version.return_value = versionutils.convert_version_to_int( + libvirt_driver.MIN_QEMU_FILE_BACKED_DISCARD_VERSION) + + result = self._test_get_guest_memory_backing_config( + None, None, None + ) + self.assertFalse(result.discard) + + @mock.patch.object(fakelibvirt.Connection, 'getVersion') + @mock.patch.object(fakelibvirt.Connection, 'getLibVersion') + def test_get_guest_memory_backing_config_file_backed_discard_qemu(self, + mock_lib_version, mock_version): + self.flags(file_backed_memory=1024, group='libvirt') + + mock_lib_version.return_value = versionutils.convert_version_to_int( + libvirt_driver.MIN_LIBVIRT_FILE_BACKED_DISCARD_VERSION) + mock_version.return_value = versionutils.convert_version_to_int( + libvirt_driver.MIN_QEMU_FILE_BACKED_DISCARD_VERSION) - 1 + + result = self._test_get_guest_memory_backing_config( + None, None, None + ) + self.assertFalse(result.discard) + def test_get_guest_memory_backing_config_file_backed_hugepages(self): self.flags(file_backed_memory=1024, group="libvirt") host_topology = objects.NUMATopology( @@ -8561,7 +8609,8 @@ class LibvirtConnTestCase(test.NoDBTestCase, "disk_over_commit": False, "block_migration": True, "is_volume_backed": False, - "dst_wants_file_backed_memory": False}, + "dst_wants_file_backed_memory": False, + "file_backed_memory_discard": False}, matchers.DictMatches(return_value.to_legacy_dict())) @mock.patch.object(objects.Service, 'get_by_compute_host') @@ -8595,7 +8644,8 @@ class LibvirtConnTestCase(test.NoDBTestCase, "disk_over_commit": True, "block_migration": True, "is_volume_backed": False, - "dst_wants_file_backed_memory": False}, + "dst_wants_file_backed_memory": False, + "file_backed_memory_discard": False}, matchers.DictMatches(return_value.to_legacy_dict())) @mock.patch.object(objects.Service, 'get_by_compute_host') @@ -8626,7 +8676,8 @@ class LibvirtConnTestCase(test.NoDBTestCase, "disk_over_commit": False, "disk_available_mb": 409600, "is_volume_backed": False, - "dst_wants_file_backed_memory": False}, + "dst_wants_file_backed_memory": False, + "file_backed_memory_discard": False}, matchers.DictMatches(return_value.to_legacy_dict())) @mock.patch.object(objects.Service, 'get_by_compute_host') @@ -8699,7 +8750,8 @@ class LibvirtConnTestCase(test.NoDBTestCase, "disk_over_commit": False, "disk_available_mb": 1024, "is_volume_backed": False, - "dst_wants_file_backed_memory": False} + "dst_wants_file_backed_memory": False, + "file_backed_memory_discard": False} self.assertEqual(expected_result, result.to_legacy_dict()) @mock.patch.object(objects.Service, 'get_by_compute_host') @@ -8734,7 +8786,8 @@ class LibvirtConnTestCase(test.NoDBTestCase, "disk_over_commit": False, "disk_available_mb": 1024, "is_volume_backed": False, - "dst_wants_file_backed_memory": False}, + "dst_wants_file_backed_memory": False, + "file_backed_memory_discard": False}, matchers.DictMatches(return_value.to_legacy_dict())) @mock.patch.object(objects.Service, 'get_by_compute_host') @@ -8770,6 +8823,132 @@ class LibvirtConnTestCase(test.NoDBTestCase, self.assertTrue(return_value.dst_wants_file_backed_memory) + @mock.patch.object(fakelibvirt.Connection, 'getVersion') + @mock.patch.object(fakelibvirt.Connection, 'getLibVersion') + @mock.patch.object(objects.Service, 'get_by_compute_host') + @mock.patch.object(libvirt_driver.LibvirtDriver, + '_create_shared_storage_test_file') + @mock.patch.object(fakelibvirt.Connection, 'compareCPU') + def test_check_can_live_migrate_dest_file_backed_discard( + self, mock_cpu, mock_test_file, mock_svc, mock_lib_version, + mock_version): + + self.flags(file_backed_memory=1024, group='libvirt') + + mock_lib_version.return_value = versionutils.convert_version_to_int( + libvirt_driver.MIN_LIBVIRT_FILE_BACKED_DISCARD_VERSION) + mock_version.return_value = versionutils.convert_version_to_int( + libvirt_driver.MIN_QEMU_FILE_BACKED_DISCARD_VERSION) + + instance_ref = objects.Instance(**self.test_instance) + instance_ref.vcpu_model = test_vcpu_model.fake_vcpumodel + + drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), False) + compute_info = {'disk_available_least': 400, + 'cpu_info': 'asdf', + } + + filename = "file" + + svc = objects.Service() + svc.version = 32 + mock_svc.return_value = svc + + # _check_cpu_match + mock_cpu.return_value = 1 + # mounted_on_same_shared_storage + mock_test_file.return_value = filename + # No need for the src_compute_info + return_value = drvr.check_can_live_migrate_destination(self.context, + instance_ref, None, compute_info, False) + + self.assertTrue(return_value.dst_wants_file_backed_memory) + self.assertTrue(return_value.file_backed_memory_discard) + + @mock.patch.object(fakelibvirt.Connection, 'getVersion') + @mock.patch.object(fakelibvirt.Connection, 'getLibVersion') + @mock.patch.object(objects.Service, 'get_by_compute_host') + @mock.patch.object(libvirt_driver.LibvirtDriver, + '_create_shared_storage_test_file') + @mock.patch.object(fakelibvirt.Connection, 'compareCPU') + def test_check_can_live_migrate_dest_file_backed_discard_bad_libvirt( + self, mock_cpu, mock_test_file, mock_svc, mock_lib_version, + mock_version): + + self.flags(file_backed_memory=1024, group='libvirt') + + mock_lib_version.return_value = versionutils.convert_version_to_int( + libvirt_driver.MIN_LIBVIRT_FILE_BACKED_DISCARD_VERSION) - 1 + mock_version.return_value = versionutils.convert_version_to_int( + libvirt_driver.MIN_QEMU_FILE_BACKED_DISCARD_VERSION) + + instance_ref = objects.Instance(**self.test_instance) + instance_ref.vcpu_model = test_vcpu_model.fake_vcpumodel + + drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), False) + compute_info = {'disk_available_least': 400, + 'cpu_info': 'asdf', + } + + filename = "file" + + svc = objects.Service() + svc.version = 32 + mock_svc.return_value = svc + + # _check_cpu_match + mock_cpu.return_value = 1 + # mounted_on_same_shared_storage + mock_test_file.return_value = filename + # No need for the src_compute_info + return_value = drvr.check_can_live_migrate_destination(self.context, + instance_ref, None, compute_info, False) + + self.assertTrue(return_value.dst_wants_file_backed_memory) + self.assertFalse(return_value.file_backed_memory_discard) + + @mock.patch.object(fakelibvirt.Connection, 'getVersion') + @mock.patch.object(fakelibvirt.Connection, 'getLibVersion') + @mock.patch.object(objects.Service, 'get_by_compute_host') + @mock.patch.object(libvirt_driver.LibvirtDriver, + '_create_shared_storage_test_file') + @mock.patch.object(fakelibvirt.Connection, 'compareCPU') + def test_check_can_live_migrate_dest_file_backed_discard_bad_qemu( + self, mock_cpu, mock_test_file, mock_svc, mock_lib_version, + mock_version): + + self.flags(file_backed_memory=1024, group='libvirt') + + mock_lib_version.return_value = versionutils.convert_version_to_int( + libvirt_driver.MIN_LIBVIRT_FILE_BACKED_DISCARD_VERSION) + mock_version.return_value = versionutils.convert_version_to_int( + libvirt_driver.MIN_QEMU_FILE_BACKED_DISCARD_VERSION) - 1 + + instance_ref = objects.Instance(**self.test_instance) + instance_ref.vcpu_model = test_vcpu_model.fake_vcpumodel + + drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), False) + compute_info = {'disk_available_least': 400, + 'cpu_info': 'asdf', + } + + filename = "file" + + svc = objects.Service() + svc.version = 32 + mock_svc.return_value = svc + + # _check_cpu_match + mock_cpu.return_value = 1 + # mounted_on_same_shared_storage + mock_test_file.return_value = filename + # No need for the src_compute_info + return_value = drvr.check_can_live_migrate_destination(self.context, + instance_ref, None, compute_info, False) + + self.assertTrue(return_value.dst_wants_file_backed_memory) + self.assertFalse(return_value.file_backed_memory_discard) + @mock.patch.object(objects.Service, 'get_by_compute_host') @mock.patch.object(fakelibvirt.Connection, 'compareCPU') def test_check_can_live_migrate_dest_incompatible_cpu_raises( diff --git a/nova/tests/unit/virt/libvirt/test_migration.py b/nova/tests/unit/virt/libvirt/test_migration.py index fbf2b66965cd..02c515613c4b 100644 --- a/nova/tests/unit/virt/libvirt/test_migration.py +++ b/nova/tests/unit/virt/libvirt/test_migration.py @@ -595,6 +595,80 @@ class UtilityMigrationTestCase(test.NoDBTestCase): """)) + def test_update_memory_backing_discard_add(self): + data = objects.LibvirtLiveMigrateData( + dst_wants_file_backed_memory=True, file_backed_memory_discard=True) + + xml = """ + + + + + +""" + doc = etree.fromstring(xml) + res = etree.tostring(migration._update_memory_backing_xml(doc, data), + encoding='unicode') + + self.assertThat(res, matchers.XMLMatches(""" + + + + + + +""")) + + def test_update_memory_backing_discard_remove(self): + data = objects.LibvirtLiveMigrateData( + dst_wants_file_backed_memory=True, + file_backed_memory_discard=False) + + xml = """ + + + + + + +""" + doc = etree.fromstring(xml) + res = etree.tostring(migration._update_memory_backing_xml(doc, data), + encoding='unicode') + + self.assertThat(res, matchers.XMLMatches(""" + + + + + +""")) + + def test_update_memory_backing_discard_keep(self): + data = objects.LibvirtLiveMigrateData( + dst_wants_file_backed_memory=True, file_backed_memory_discard=True) + + xml = """ + + + + + + +""" + doc = etree.fromstring(xml) + res = etree.tostring(migration._update_memory_backing_xml(doc, data), + encoding='unicode') + + self.assertThat(res, matchers.XMLMatches(""" + + + + + + +""")) + class MigrationMonitorTestCase(test.NoDBTestCase): def setUp(self): diff --git a/nova/virt/libvirt/config.py b/nova/virt/libvirt/config.py index f1df5ec4e035..a9c2cf5b4e2f 100644 --- a/nova/virt/libvirt/config.py +++ b/nova/virt/libvirt/config.py @@ -2054,6 +2054,7 @@ class LibvirtConfigGuestMemoryBacking(LibvirtConfigObject): self.filesource = False self.sharedaccess = False self.allocateimmediate = False + self.discard = False def format_dom(self): root = super(LibvirtConfigGuestMemoryBacking, self).format_dom() @@ -2073,6 +2074,8 @@ class LibvirtConfigGuestMemoryBacking(LibvirtConfigObject): root.append(etree.Element("access", mode="shared")) if self.allocateimmediate: root.append(etree.Element("allocation", mode="immediate")) + if self.discard: + root.append(etree.Element("discard")) return root diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py index 001f2ddaa656..fdeea9455185 100644 --- a/nova/virt/libvirt/driver.py +++ b/nova/virt/libvirt/driver.py @@ -288,6 +288,8 @@ MIN_QEMU_LUKS_VERSION = (2, 6, 0) MIN_LIBVIRT_FILE_BACKED_VERSION = (4, 0, 0) MIN_QEMU_FILE_BACKED_VERSION = (2, 6, 0) +MIN_LIBVIRT_FILE_BACKED_DISCARD_VERSION = (4, 4, 0) +MIN_QEMU_FILE_BACKED_DISCARD_VERSION = (2, 10, 0) VGPU_RESOURCE_SEMAPHORE = "vgpu_resources" @@ -4736,6 +4738,10 @@ class LibvirtDriver(driver.ComputeDriver): membacking.filesource = True membacking.sharedaccess = True membacking.allocateimmediate = True + if self._host.has_min_version( + MIN_LIBVIRT_FILE_BACKED_DISCARD_VERSION, + MIN_QEMU_FILE_BACKED_DISCARD_VERSION): + membacking.discard = True return membacking @@ -6601,6 +6607,9 @@ class LibvirtDriver(driver.ComputeDriver): data.disk_available_mb = disk_available_mb data.dst_wants_file_backed_memory = \ CONF.libvirt.file_backed_memory > 0 + data.file_backed_memory_discard = (CONF.libvirt.file_backed_memory and + self._host.has_min_version(MIN_LIBVIRT_FILE_BACKED_DISCARD_VERSION, + MIN_QEMU_FILE_BACKED_DISCARD_VERSION)) return data diff --git a/nova/virt/libvirt/migration.py b/nova/virt/libvirt/migration.py index 4284ba31f32c..a18f2ec820f9 100644 --- a/nova/virt/libvirt/migration.py +++ b/nova/virt/libvirt/migration.py @@ -236,12 +236,16 @@ def _update_memory_backing_xml(xml_doc, migrate_data): """ old_xml_has_memory_backing = True file_backed = False + discard = False memory_backing = xml_doc.findall('./memoryBacking') if 'dst_wants_file_backed_memory' in migrate_data: file_backed = migrate_data.dst_wants_file_backed_memory + if 'file_backed_memory_discard' in migrate_data: + discard = migrate_data.file_backed_memory_discard + if not memory_backing: # Create memoryBacking element memory_backing = etree.Element("memoryBacking") @@ -249,7 +253,7 @@ def _update_memory_backing_xml(xml_doc, migrate_data): else: memory_backing = memory_backing[0] # Remove existing file-backed memory tags, if they exist. - for name in ("access", "source", "allocation"): + for name in ("access", "source", "allocation", "discard"): tag = memory_backing.findall(name) if tag: memory_backing.remove(tag[0]) @@ -263,6 +267,9 @@ def _update_memory_backing_xml(xml_doc, migrate_data): memory_backing.append(etree.Element("access", mode="shared")) memory_backing.append(etree.Element("allocation", mode="immediate")) + if discard: + memory_backing.append(etree.Element("discard")) + if not old_xml_has_memory_backing: xml_doc.append(memory_backing)