SolidFire QoS scaled by volume size

This commit implements scaled QoS for the SolidFire driver. The
following QoS and Extra specs are now recognized: scaledIOPS, scaleMin,
scaleMax, scaleBurst. ScaledIOPS presence is a flag which causes the
other 3 to scale their respective QoS starting with the original
minIOPS, maxIOPS and burstIOPS by the size of the volume.

Moved the SolidFire tests to a separate directory to accomodate extra
files for data driven testing.

Change-Id: I83cd0a1b2f978351596c20ab2bfa6dc44776ed6d
Implements: blueprint solidfire-scaled-qos

cinder/tests/unit/volume/drivers/solidfire/scaled_iops_invalid_data.json

@@ -0,0 +1,22 @@
+{
+    "test_max_greater_than_burst": [
+        {
+            "burstIOPS": 2,
+            "maxIOPS": 3,
+            "minIOPS": "100",
+            "scaleMin": "2",
+            "scaledIOPS": "True",
+            "size": 2
+        }
+    ],
+    "test_min_greater_than_max_burst": [
+        {
+            "burstIOPS": 2,
+            "maxIOPS": 2,
+            "minIOPS": "100",
+            "scaleMin": "3",
+            "scaledIOPS": "True",
+            "size": 2
+        }
+    ]
+}

cinder/tests/unit/volume/drivers/solidfire/scaled_iops_test_data.json

@@ -0,0 +1,134 @@
+{
+    "test_capping_the_maximum_of_minIOPS": [
+        {
+            "burstIOPS": "200000",
+            "maxIOPS": "200000",
+            "minIOPS": "14950",
+            "scaleMin": "100",
+            "scaledIOPS": "True",
+            "size": 2
+        },
+        {
+            "burstIOPS": 200000,
+            "maxIOPS": 200000,
+            "minIOPS": 15000
+        }
+    ],
+    "test_capping_the_maximums": [
+        {
+            "burstIOPS": "190000",
+            "maxIOPS": "190000",
+            "minIOPS": "100",
+            "scaleBurst": "10003",
+            "scaleMax": "10002",
+            "scaleMin": "2",
+            "scaledIOPS": "True",
+            "size": 2
+        },
+        {
+            "burstIOPS": 200000,
+            "maxIOPS": 200000,
+            "minIOPS": 102
+        }
+    ],
+    "test_capping_the_minimum": [
+        {
+            "burstIOPS": "300",
+            "maxIOPS": "200",
+            "minIOPS": "50",
+            "scaleBurst": "2",
+            "scaleMax": "2",
+            "scaleMin": "2",
+            "scaledIOPS": "True",
+            "size": 2
+        },
+        {
+            "burstIOPS": 302,
+            "maxIOPS": 202,
+            "minIOPS": 100
+        }
+    ],
+    "test_regular_QoS": [
+        {
+            "burstIOPS": "200",
+            "maxIOPS": "200",
+            "minIOPS": "100",
+            "size": 1
+        },
+        {
+            "burstIOPS": 200,
+            "maxIOPS": 200,
+            "minIOPS": 100
+        }
+    ],
+    "test_scaled_QoS_with_size_1": [
+        {
+            "burstIOPS": "300",
+            "maxIOPS": "200",
+            "minIOPS": "100",
+            "scaleBurst": "2",
+            "scaleMax": "2",
+            "scaleMin": "2",
+            "scaledIOPS": "True",
+            "size": 1
+        },
+        {
+            "burstIOPS": 300,
+            "maxIOPS": 200,
+            "minIOPS": 100
+        }
+    ],
+    "test_scaled_QoS_with_size_2": [
+        {
+            "burstIOPS": "300",
+            "maxIOPS": "200",
+            "minIOPS": "100",
+            "scaleBurst": "2",
+            "scaleMax": "2",
+            "scaleMin": "2",
+            "scaledIOPS": "True",
+            "size": 2
+        },
+        {
+            "burstIOPS": 302,
+            "maxIOPS": 202,
+            "minIOPS": 102
+        }
+    ],
+    "test_scoped_regular_QoS": [
+        {
+            "qos:burstIOPS": "200",
+            "qos:maxIOPS": "200",
+            "qos:minIOPS": "100",
+            "size": 1
+        },
+        {
+            "burstIOPS": 200,
+            "maxIOPS": 200,
+            "minIOPS": 100
+        }
+    ],
+    "test_when_no_valid_QoS_values_present": [
+        {
+            "key": "value",
+            "size": 2
+        },
+        {}
+    ],
+    "test_without_presence_of_the_scaled_flag": [
+        {
+            "burstIOPS": "300",
+            "maxIOPS": "200",
+            "minIOPS": "100",
+            "scaleBurst": "2",
+            "scaleMax": "2",
+            "scaleMin": "2",
+            "size": 2
+        },
+        {
+            "burstIOPS": 300,
+            "maxIOPS": 200,
+            "minIOPS": 100
+        }
+    ]
+}

cinder/tests/unit/volume/drivers/solidfire/test_solidfire.py

@@ -16,6 +16,7 @@
 
 import datetime
 
+import ddt
 import mock
 from oslo_utils import timeutils
 from oslo_utils import units
@@ -31,7 +32,9 @@ from cinder.volume import qos_specs
 from cinder.volume import volume_types
 
 
+@ddt.ddt
 class SolidFireVolumeTestCase(test.TestCase):
+
     def setUp(self):
         self.ctxt = context.get_admin_context()
         self.configuration = conf.Configuration(None)
@@ -653,6 +656,7 @@ class SolidFireVolumeTestCase(test.TestCase):
                           testvol, 2)
 
     def test_set_by_qos_spec_with_scoping(self):
+        size = 1
         sfv = solidfire.SolidFireDriver(configuration=self.configuration)
         qos_ref = qos_specs.create(self.ctxt,
                                    'qos-specs-1', {'qos:minIOPS': '1000',
@@ -665,10 +669,11 @@ class SolidFireVolumeTestCase(test.TestCase):
         qos_specs.associate_qos_with_type(self.ctxt,
                                           qos_ref['id'],
                                           type_ref['id'])
-        qos = sfv._set_qos_by_volume_type(self.ctxt, type_ref['id'])
+        qos = sfv._set_qos_by_volume_type(self.ctxt, type_ref['id'], size)
         self.assertEqual(self.expected_qos_results, qos)
 
     def test_set_by_qos_spec(self):
+        size = 1
         sfv = solidfire.SolidFireDriver(configuration=self.configuration)
         qos_ref = qos_specs.create(self.ctxt,
                                    'qos-specs-1', {'minIOPS': '1000',
@@ -681,19 +686,29 @@ class SolidFireVolumeTestCase(test.TestCase):
         qos_specs.associate_qos_with_type(self.ctxt,
                                           qos_ref['id'],
                                           type_ref['id'])
-        qos = sfv._set_qos_by_volume_type(self.ctxt, type_ref['id'])
+        qos = sfv._set_qos_by_volume_type(self.ctxt, type_ref['id'], size)
         self.assertEqual(self.expected_qos_results, qos)
 
-    def test_set_by_qos_by_type_only(self):
+    @ddt.file_data("scaled_iops_test_data.json")
+    @ddt.unpack
+    def test_scaled_qos_spec_by_type(self, argument):
         sfv = solidfire.SolidFireDriver(configuration=self.configuration)
-        type_ref = volume_types.create(self.ctxt,
+        size = argument[0].pop('size')
-                                       "type1", {"qos:minIOPS": "100",
+        type_ref = volume_types.create(self.ctxt, "type1", argument[0])
-                                                 "qos:burstIOPS": "300",
+        qos = sfv._set_qos_by_volume_type(self.ctxt, type_ref['id'], size)
-                                                 "qos:maxIOPS": "200"})
+        self.assertEqual(argument[1], qos)
-        qos = sfv._set_qos_by_volume_type(self.ctxt, type_ref['id'])
+
-        self.assertEqual({'minIOPS': 100,
+    @ddt.file_data("scaled_iops_invalid_data.json")
-                          'maxIOPS': 200,
+    @ddt.unpack
-                          'burstIOPS': 300}, qos)
+    def test_set_scaled_qos_by_type_invalid(self, inputs):
+        sfv = solidfire.SolidFireDriver(configuration=self.configuration)
+        size = inputs[0].pop('size')
+        type_ref = volume_types.create(self.ctxt, "type1", inputs[0])
+        self.assertRaises(exception.InvalidQoSSpecs,
+                          sfv._set_qos_by_volume_type,
+                          self.ctxt,
+                          type_ref['id'],
+                          size)
 
     def test_accept_transfer(self):
         sfv = solidfire.SolidFireDriver(configuration=self.configuration)
@@ -1328,7 +1343,7 @@ class SolidFireVolumeTestCase(test.TestCase):
                                return_value=vags), \
             mock.patch.object(sfv,
                               '_add_initiator_to_vag',
-                              return_value = vag_id) as add_init:
+                              return_value=vag_id) as add_init:
             res_vag_id = sfv._safe_create_vag(iqn, None)
             self.assertEqual(res_vag_id, vag_id)
             add_init.assert_called_with(iqn, vag_id)

cinder/volume/drivers/solidfire.py

@@ -154,9 +154,10 @@ class SolidFireDriver(san.SanISCSIDriver):
         2.0.5 - Try and deal with the stupid retry/clear issues from objects
                 and tflow
         2.0.6 - Add a lock decorator around the clone_image method
+        2.0.7 - Add scaled IOPS
     """
 
-    VERSION = '2.0.6'
+    VERSION = '2.0.7'
 
     # ThirdPartySystems wiki page
     CI_WIKI_NAME = "SolidFire_CI"
@@ -178,6 +179,11 @@ class SolidFireDriver(san.SanISCSIDriver):
                    'off': None}
 
     sf_qos_keys = ['minIOPS', 'maxIOPS', 'burstIOPS']
+    sf_scale_qos_keys = ['scaledIOPS', 'scaleMin', 'scaleMax', 'scaleBurst']
+    sf_iops_lim_min = {'minIOPS': 100, 'maxIOPS': 100, 'burstIOPS': 100}
+    sf_iops_lim_max = {'minIOPS': 15000,
+                       'maxIOPS': 200000,
+                       'burstIOPS': 200000}
     cluster_stats = {}
     retry_exc_tuple = (exception.SolidFireRetryableException,
                        requests.exceptions.ConnectionError)
@@ -686,8 +692,9 @@ class SolidFireDriver(san.SanISCSIDriver):
                     qos[i.key] = int(i.value)
         return qos
 
-    def _set_qos_by_volume_type(self, ctxt, type_id):
+    def _set_qos_by_volume_type(self, ctxt, type_id, vol_size):
         qos = {}
+        scale_qos = {}
         volume_type = volume_types.get_volume_type(ctxt, type_id)
         qos_specs_id = volume_type.get('qos_specs_id')
         specs = volume_type.get('extra_specs')
@@ -706,6 +713,43 @@ class SolidFireDriver(san.SanISCSIDriver):
                 key = fields[1]
             if key in self.sf_qos_keys:
                 qos[key] = int(value)
+            if key in self.sf_scale_qos_keys:
+                scale_qos[key] = value
+
+        # look for the 'scaledIOPS' key and scale QoS if set
+        if 'scaledIOPS' in scale_qos:
+            scale_qos.pop('scaledIOPS')
+            for key, value in scale_qos.items():
+                if key == 'scaleMin':
+                    qos['minIOPS'] = (qos['minIOPS'] +
+                                      (int(value) * (vol_size - 1)))
+                elif key == 'scaleMax':
+                    qos['maxIOPS'] = (qos['maxIOPS'] +
+                                      (int(value) * (vol_size - 1)))
+                elif key == 'scaleBurst':
+                    qos['burstIOPS'] = (qos['burstIOPS'] +
+                                        (int(value) * (vol_size - 1)))
+        # Cap the IOPS values at their limits
+        capped = False
+        for key, value in qos.items():
+            if value > self.sf_iops_lim_max[key]:
+                qos[key] = self.sf_iops_lim_max[key]
+                capped = True
+            if value < self.sf_iops_lim_min[key]:
+                qos[key] = self.sf_iops_lim_min[key]
+                capped = True
+        if capped:
+            LOG.debug("A SolidFire QoS value was capped at the defined limits")
+        # Check that minIOPS <= maxIOPS <= burstIOPS
+        if (qos.get('minIOPS', 0) > qos.get('maxIOPS', 0) or
+                qos.get('maxIOPS', 0) > qos.get('burstIOPS', 0)):
+            msg = (_("Scaled QoS error. Must be minIOPS <= maxIOPS <= "
+                     "burstIOPS. Currently: Min: %(min)s, Max: "
+                     "%(max)s, Burst: %(burst)s.") %
+                   {"min": qos['minIOPS'],
+                    "max": qos['maxIOPS'],
+                    "burst": qos['burstIOPS']})
+            raise exception.InvalidQoSSpecs(reason=msg)
         return qos
 
     def _get_sf_volume(self, uuid, params=None):
@@ -1156,7 +1200,8 @@ class SolidFireDriver(san.SanISCSIDriver):
         ctxt = context.get_admin_context()
         type_id = volume.get('volume_type_id', None)
         if type_id is not None:
-            qos = self._set_qos_by_volume_type(ctxt, type_id)
+            qos = self._set_qos_by_volume_type(ctxt, type_id,
+                                               volume.get('size'))
         return qos
 
     def create_volume(self, volume):
@@ -1781,7 +1826,8 @@ class SolidFireDriver(san.SanISCSIDriver):
         attributes = sf_vol['attributes']
         attributes['retyped_at'] = timeutils.utcnow().isoformat()
         params = {'volumeID': sf_vol['volumeID']}
-        qos = self._set_qos_by_volume_type(ctxt, new_type['id'])
+        qos = self._set_qos_by_volume_type(ctxt, new_type['id'],
+                                           volume.get('size'))
 
         if qos:
             params['qos'] = qos

releasenotes/notes/solidfire-scaled-qos-9b8632453909e2db.yaml

@@ -0,0 +1,20 @@
+---
+features:
+  - The SolidFire driver will recognize 4 new QoS spec keys
+    to allow an administrator to specify QoS settings which
+    are scaled by the size of the volume. 'ScaledIOPS' is a
+    flag which will tell the driver to look for 'scaleMin',
+    'scaleMax' and 'scaleBurst' which provide the scaling
+    factor from the minimum values specified by the previous
+    QoS keys ('minIOPS', 'maxIOPS', 'burstIOPS'). The
+    administrator must take care to assure that no matter what
+    the final calculated QoS values follow minIOPS <= maxIOPS
+    <= burstIOPS. A exception will be thrown if not. The QoS
+    settings are also checked against the cluster min and max
+    allowed and truncated at the min or max if they exceed.
+fixes:
+  - For SolidFire, QoS specs are now checked to make sure
+    they fall within the min and max constraints. If not
+    the QoS specs are capped at the min or max (i.e. if
+    spec says 50 and minimum supported is 100, the driver
+    will set it to 100).