Add compute_node ratio online data migration script

This patch adds an online data migration script to process the
ratio with 0.0 or None value.

If it's an existing record with 0.0 values, we'd want to do what
the compute does, which is use the configure ``xxx_allocation_ratio``
config if it's not None, and fallback to using the
``initial_xxx_allocation_ratio`` otherwise.

Change-Id: I3a6d4d3012b3fffe94f15a724dd78707966bb522
blueprint: initial-allocation-ratios

nova/cmd/manage.py

@@ -59,6 +59,7 @@ from nova.i18n import _
 from nova import objects
 from nova.objects import block_device as block_device_obj
 from nova.objects import build_request as build_request_obj
+from nova.objects import compute_node as compute_node_obj
 from nova.objects import host_mapping as host_mapping_obj
 from nova.objects import instance as instance_obj
 from nova.objects import instance_mapping as instance_mapping_obj
@@ -418,6 +419,8 @@ class DbCommands(object):
         consumer_obj.create_incomplete_consumers,
         # Added in Rocky
         instance_mapping_obj.populate_queued_for_delete,
+        # Added in Stein
+        compute_node_obj.migrate_empty_ratio,
     )
 
     def __init__(self):

nova/objects/compute_node.py

@@ -16,6 +16,8 @@
 from oslo_serialization import jsonutils
 from oslo_utils import uuidutils
 from oslo_utils import versionutils
+from sqlalchemy import or_
+from sqlalchemy.sql import null
 
 import nova.conf
 from nova.db import api as db
@@ -469,3 +471,39 @@ class ComputeNodeList(base.ObjectListBase, base.NovaObject):
         db_computes = cls._db_compute_node_get_by_hv_type(context, hv_type)
         return base.obj_make_list(context, cls(context), objects.ComputeNode,
                                   db_computes)
+
+
+def _get_node_empty_ratio(context, max_count):
+    """Query the DB for non-deleted compute_nodes with 0.0/None alloc ratios
+
+    Results are limited by ``max_count``.
+    """
+    return context.session.query(models.ComputeNode).filter(or_(
+        models.ComputeNode.ram_allocation_ratio == '0.0',
+        models.ComputeNode.cpu_allocation_ratio == '0.0',
+        models.ComputeNode.disk_allocation_ratio == '0.0',
+        models.ComputeNode.ram_allocation_ratio == null(),
+        models.ComputeNode.cpu_allocation_ratio == null(),
+        models.ComputeNode.disk_allocation_ratio == null()
+    )).filter(models.ComputeNode.deleted == 0).limit(max_count).all()
+
+
+@sa_api.pick_context_manager_writer
+def migrate_empty_ratio(context, max_count):
+    cns = _get_node_empty_ratio(context, max_count)
+
+    # NOTE(yikun): If it's an existing record with 0.0 or None values,
+    # we need to migrate this record using 'xxx_allocation_ratio' config
+    # if it's set, and fallback to use the 'initial_xxx_allocation_ratio'
+    # otherwise.
+    for cn in cns:
+        for t in ['cpu', 'disk', 'ram']:
+            current_ratio = getattr(cn, '%s_allocation_ratio' % t)
+            if current_ratio in (0.0, None):
+                r = getattr(CONF, "%s_allocation_ratio" % t)
+                init_x_ratio = getattr(CONF, "initial_%s_allocation_ratio" % t)
+                conf_alloc_ratio = r if r else init_x_ratio
+                setattr(cn, '%s_allocation_ratio' % t, conf_alloc_ratio)
+        context.session.add(cn)
+    found = done = len(cns)
+    return found, done

nova/tests/functional/db/test_compute_node.py

@@ -15,6 +15,7 @@ import nova.conf
 from nova import context
 from nova.db import api as db
 from nova import objects
+from nova.objects import compute_node
 from nova.objects import fields as obj_fields
 from nova import test
 
@@ -188,3 +189,71 @@ class ComputeNodeTestCase(test.TestCase):
             # the ram ratio is refreshed to CONF.initial_xxx_allocation_ratio
             self.assertEqual(CONF.initial_ram_allocation_ratio,
                              cn['ram_allocation_ratio'])
+
+    def test_migrate_empty_ratio(self):
+        # we have 5 records to process, the last of which is deleted
+        for i in range(5):
+            cn = fake_compute_obj.obj_clone()
+            cn._context = self.context
+            cn.host += '-alt-%s' % i
+            cn.create()
+            db.compute_node_update(self.context, cn.id,
+                                   {'cpu_allocation_ratio': 0.0})
+            if i == 4:
+                cn.destroy()
+
+        # first only process 2
+        res = compute_node.migrate_empty_ratio(self.context, 2)
+        self.assertEqual(res, (2, 2))
+
+        # then process others - there should only be 2 found since one
+        # of the remaining compute nodes is deleted and gets filtered out
+        res = compute_node.migrate_empty_ratio(self.context, 999)
+        self.assertEqual(res, (2, 2))
+
+    def test_migrate_none_or_zero_ratio_with_none_ratio_conf(self):
+        cn1 = fake_compute_obj.obj_clone()
+        cn1._context = self.context
+        cn1.create()
+
+        db.compute_node_update(self.context, cn1.id,
+                               {'cpu_allocation_ratio': 0.0,
+                                'disk_allocation_ratio': 0.0,
+                                'ram_allocation_ratio': 0.0})
+
+        self.flags(initial_cpu_allocation_ratio=32.0)
+        self.flags(initial_ram_allocation_ratio=8.0)
+        self.flags(initial_disk_allocation_ratio=2.0)
+
+        res = compute_node.migrate_empty_ratio(self.context, 1)
+        self.assertEqual(res, (1, 1))
+
+        # the ratio is refreshed to CONF.initial_xxx_allocation_ratio
+        # beacause CONF.xxx_allocation_ratio is None
+        cns = db.compute_node_get_all(self.context)
+        # the ratio is refreshed to CONF.xxx_allocation_ratio
+        for cn in cns:
+            for x in ['cpu', 'disk', 'ram']:
+                conf_key = 'initial_%s_allocation_ratio' % x
+                key = '%s_allocation_ratio' % x
+                self.assertEqual(getattr(CONF, conf_key), cn[key])
+
+    def test_migrate_none_or_zero_ratio_with_not_empty_ratio(self):
+        cn1 = fake_compute_obj.obj_clone()
+        cn1._context = self.context
+        cn1.create()
+
+        db.compute_node_update(self.context, cn1.id,
+                               {'cpu_allocation_ratio': 32.0,
+                                'ram_allocation_ratio': 4.0,
+                                'disk_allocation_ratio': 3.0})
+
+        res = compute_node.migrate_empty_ratio(self.context, 1)
+        # the non-empty ratio will not be refreshed
+        self.assertEqual(res, (0, 0))
+
+        cns = db.compute_node_get_all(self.context)
+        for cn in cns:
+            self.assertEqual(32.0, cn['cpu_allocation_ratio'])
+            self.assertEqual(4.0, cn['ram_allocation_ratio'])
+            self.assertEqual(3.0, cn['disk_allocation_ratio'])

releasenotes/notes/add_initial_allocation_ratio-2d2666d62426a4bf.yaml

@@ -17,6 +17,11 @@ upgrade:
     initially creating the ``computes_nodes`` table record for a given
     nova-compute service.
 
+    Existing ``compute_nodes`` table records with ``0.0`` or ``None`` values
+    for ``cpu_allocation_ratio``, ``ram_allocation_ratio`` or
+    ``disk_allocation_ratio`` will be migrated online when accessed or when
+    the ``nova-manage db online_data_migrations`` command is run.
+
     For more details, refer to the `spec`__.
 
     .. __: https://specs.openstack.org/openstack/nova-specs/specs/stein/approved/initial-allocation-ratios.html