Support Images API v2.11

Glance has changed the format of the cinder URIs in image locations
so that they can look like
   cinder://glance-store-name/volume_id
in addition to the legacy format
   cinder://volume_id

Change the cinder code so that it can handle both formats for
reading.  (We only need to write the legacy format.)

Change-Id: I8c176bf4c875061591bb6c94654a2cef643a4dcb
Closes-bug: #1898075

cinder/tests/unit/volume/flows/test_create_volume_flow.py

@@ -1306,12 +1306,68 @@ class CreateVolumeFlowManagerTestCase(test.TestCase):
             exception=err)
 
 
+@ddt.ddt(testNameFormat=ddt.TestNameFormat.INDEX_ONLY)
 class CreateVolumeFlowManagerGlanceCinderBackendCase(test.TestCase):
 
     def setUp(self):
         super(CreateVolumeFlowManagerGlanceCinderBackendCase, self).setUp()
         self.ctxt = context.get_admin_context()
 
+    # data for test__extract_cinder_ids
+    #   legacy glance cinder URI:    cinder://<volume-id>
+    #   new-style glance cinder URI: cinder://<glance-store>/<volume_id>
+    LEGACY_VOL2 = 'cinder://%s' % fakes.VOLUME2_ID
+    NEW_VOL3 = 'cinder://glance-store-name/%s' % fakes.VOLUME3_ID
+    # these *may* be illegal names in glance, but check anyway
+    NEW_VOL4 = 'cinder://glance/store/name/%s' % fakes.VOLUME4_ID
+    NEW_VOL5 = 'cinder://glance:store:name/%s' % fakes.VOLUME5_ID
+    NEW_VOL6 = 'cinder://glance:store,name/%s' % fakes.VOLUME6_ID
+    NOT_CINDER1 = 'rbd://%s' % fakes.UUID1
+    NOT_CINDER2 = 'http://%s' % fakes.UUID2
+    NOGOOD3 = 'cinder://glance:store,name/%s/garbage' % fakes.UUID3
+    NOGOOD4 = 'cinder://glance:store,name/%s-garbage' % fakes.UUID4
+    NOGOOD5 = fakes.UUID5
+    NOGOOD6 = 'cinder://store-name/12345678'
+    NOGOOD7 = 'cinder://'
+    NOGOOD8 = 'some-random-crap'
+    NOGOOD9 = None
+
+    TEST_CASE_DATA = (
+        # the format of these is: (input, expected output)
+        ([LEGACY_VOL2], [fakes.VOLUME2_ID]),
+        ([NEW_VOL3], [fakes.VOLUME3_ID]),
+        ([NEW_VOL4], [fakes.VOLUME4_ID]),
+        ([NEW_VOL5], [fakes.VOLUME5_ID]),
+        ([NEW_VOL6], [fakes.VOLUME6_ID]),
+        ([], []),
+        ([''], []),
+        ([NOT_CINDER1], []),
+        ([NOT_CINDER2], []),
+        ([NOGOOD3], []),
+        ([NOGOOD4], []),
+        ([NOGOOD5], []),
+        ([NOGOOD6], []),
+        ([NOGOOD7], []),
+        ([NOGOOD8], []),
+        ([NOGOOD9], []),
+        ([NOT_CINDER1, NOGOOD4], []),
+        # mix of URIs should only get the cinder IDs
+        ([LEGACY_VOL2, NOT_CINDER1, NEW_VOL3, NOT_CINDER2],
+         [fakes.VOLUME2_ID, fakes.VOLUME3_ID]),
+        # a bad cinder URI early in the list shouldn't prevent us from
+        # processing a good one later in the list
+        ([NOGOOD6, NEW_VOL3, NOGOOD7, LEGACY_VOL2],
+         [fakes.VOLUME3_ID, fakes.VOLUME2_ID]),
+    )
+
+    @ddt.data(*TEST_CASE_DATA)
+    @ddt.unpack
+    def test__extract_cinder_ids(self, url_list, id_list):
+        """Test utility function that gets IDs from Glance location URIs"""
+        klass = create_volume_manager.CreateVolumeFromSpecTask
+        actual = klass._extract_cinder_ids(url_list)
+        self.assertEqual(id_list, actual)
+
     @mock.patch('cinder.volume.flows.manager.create_volume.'
                 'CreateVolumeFromSpecTask.'
                 '_cleanup_cg_in_volume')
@@ -1373,6 +1429,10 @@ class CreateVolumeFlowManagerGlanceCinderBackendCase(test.TestCase):
             self.assertFalse(fake_driver.create_cloned_volume.called)
         mock_cleanup_cg.assert_called_once_with(volume)
 
+    LEGACY_URI = 'cinder://%s' % fakes.VOLUME_ID
+    MULTISTORE_URI = 'cinder://fake-store/%s' % fakes.VOLUME_ID
+
+    @ddt.data(LEGACY_URI, MULTISTORE_URI)
     @mock.patch('cinder.volume.flows.manager.create_volume.'
                 'CreateVolumeFromSpecTask.'
                 '_cleanup_cg_in_volume')
@@ -1381,7 +1441,8 @@ class CreateVolumeFlowManagerGlanceCinderBackendCase(test.TestCase):
                 'CreateVolumeFromSpecTask.'
                 '_handle_bootable_volume_glance_meta')
     @mock.patch('cinder.image.image_utils.qemu_img_info')
-    def test_create_from_image_volume_ignore_size(self, mock_qemu_info,
+    def test_create_from_image_volume_ignore_size(self, location_uri,
+                                                  mock_qemu_info,
                                                   handle_bootable,
                                                   mock_fetch_img,
                                                   mock_cleanup_cg,
@@ -1408,7 +1469,7 @@ class CreateVolumeFlowManagerGlanceCinderBackendCase(test.TestCase):
         # will fail because of free space being too low.
         image_info.virtual_size = '1073741824000000000000'
         mock_qemu_info.return_value = image_info
-        url = 'cinder://%s' % image_volume['id']
+        url = location_uri
         image_location = None
         if location:
             image_location = (url, [{'url': url, 'metadata': {}}])

cinder/volume/flows/manager/create_volume.py

@@ -13,7 +13,7 @@
 import binascii
 import traceback
 import typing
-from typing import Any, Dict, Optional, Tuple  # noqa: H301
+from typing import Any, Dict, List, Optional, Tuple  # noqa: H301
 
 from castellan import key_manager
 import os_brick.initiator.connectors
@@ -22,7 +22,9 @@ from oslo_config import cfg
 from oslo_log import log as logging
 from oslo_utils import excutils
 from oslo_utils import fileutils
+from oslo_utils import netutils
 from oslo_utils import timeutils
+from oslo_utils import uuidutils
 import taskflow.engines
 from taskflow.patterns import linear_flow
 from taskflow.types import failure as ft
@@ -667,6 +669,34 @@ class CreateVolumeFromSpecTask(flow_utils.CinderTask):
         self.db.volume_glance_metadata_bulk_create(context, volume_id,
                                                    volume_metadata)
 
+    @staticmethod
+    def _extract_cinder_ids(urls: List[str]) -> List[str]:
+        """Process a list of location URIs from glance
+
+        :param urls: list of glance location URIs
+        :return: list of IDs extracted from the 'cinder://' URIs
+
+        """
+        ids = []
+        for url in urls:
+            # The url can also be None and a TypeError is raised
+            # TypeError: a bytes-like object is required, not 'str'
+            if not url:
+                continue
+            parts = netutils.urlsplit(url)
+            if parts.scheme == 'cinder':
+                if parts.path:
+                    vol_id = parts.path.split('/')[-1]
+                else:
+                    vol_id = parts.netloc
+                if uuidutils.is_uuid_like(vol_id):
+                    ids.append(vol_id)
+                else:
+                    LOG.debug("Ignoring malformed image location uri "
+                              "'%(url)s'", {'url': url})
+
+        return ids
+
     def _clone_image_volume(self,
                             context: cinder_context.RequestContext,
                             volume: objects.Volume,
@@ -690,9 +720,9 @@ class CreateVolumeFromSpecTask(flow_utils.CinderTask):
 
         image_volume = None
         direct_url, locations = image_location
-        urls = set([direct_url] + [loc.get('url') for loc in locations or []])
+        urls = list(set([direct_url]
-        image_volume_ids = [url[9:] for url in urls
+                        + [loc.get('url') for loc in locations or []]))
-                            if url and url.startswith('cinder://')]
+        image_volume_ids = self._extract_cinder_ids(urls)
         image_volumes = self.db.volume_get_all_by_host(
             context, volume['host'], filters={'id': image_volume_ids})
 

releasenotes/notes/support-images-api-2.11-3699b20670db1843.yaml

@@ -0,0 +1,12 @@
+---
+fixes:
+  - |
+    `Bug #1898075
+    <https://bugs.launchpad.net/cinder/+bug/1898075>`_: When Glance added
+    support for multiple cinder stores, Images API version 2.11 modified
+    the format of the image location URI, which Cinder reads in order
+    to try to use an optimized data path when creating a volume from an
+    image.  Unfortunately, Cinder did not understand the new format and
+    when Glance multiple cinder stores were used, Cinder could not use
+    the optimized data path, and instead downloaded image data from
+    the Image service.  Cinder now supports Images API version 2.11.