Support transfer pagination

This patch adds support for transfer pagination.

Co-Authored-By: Yikun Jiang <yikunkero@gmail.com>

Closes-bug: #1814195

Change-Id: Idb04f783b2287b2b45d626131648b0005a232fbe

api-ref/source/v3/parameters.yaml

@@ -271,6 +271,17 @@ limit_group_snapshot:
   required: false
   type: integer
   min_version: 3.29
+limit_transfer:
+  description: |
+    Requests a page size of items. Returns a number
+    of items up to a limit value. Use the ``limit`` parameter to make
+    an initial limited request and use the ID of the last-seen item
+    from the response as the ``marker`` parameter value in a
+    subsequent limited request.
+  in: query
+  required: false
+  type: integer
+  min_version: 3.59
 marker:
   description: |
     The ID of the last-seen item. Use the ``limit``
@@ -290,6 +301,16 @@ marker_group_snapshot:
   required: false
   type: string
   min_version: 3.29
+marker_transfer:
+  description: |
+    The ID of the last-seen item. Use the ``limit``
+    parameter to make an initial limited request and use the ID of the
+    last-seen item from the response as the ``marker`` parameter value
+    in a subsequent limited request.
+  in: query
+  required: false
+  type: string
+  min_version: 3.59
 metadata_query:
   description: |
     Filters results by a metadata key and value pair.
@@ -324,6 +345,14 @@ offset_group_snapshot:
   required: false
   type: integer
   min_version: 3.29
+offset_transfer:
+  description: |
+    Used in conjunction with ``limit`` to return a slice of items. ``offset``
+    is where to start in the list.
+  in: query
+  required: false
+  type: integer
+  min_version: 3.59
 resource:
   description: |
     Filter for resource name.
@@ -355,6 +384,15 @@ sort_dir_group_snapshot:
   required: false
   type: string
   min_version: 3.29
+sort_dir_transfer:
+  description: |
+    Sorts by one or more sets of attribute and sort
+    direction combinations. If you omit the sort direction in a set,
+    default is ``desc``.
+  in: query
+  required: false
+  type: string
+  min_version: 3.59
 sort_key:
   description: |
     Sorts by an attribute. A valid value is ``name``,
@@ -376,6 +414,15 @@ sort_key_group_snapshot:
   required: false
   type: string
   min_version: 3.29
+sort_key_transfer:
+  description: |
+    Sorts by an attribute. Default is
+    ``created_at``. The API uses the natural sorting direction of the
+    ``sort_key`` attribute value.
+  in: query
+  required: false
+  type: string
+  min_version: 3.59
 status_query:
   description: |
     Filters results by a status. Default=None.

api-ref/source/v3/samples/versions/version-show-response.json

@@ -22,7 +22,7 @@
             "min_version": "3.0",
             "status": "CURRENT",
             "updated": "2018-07-17T00:00:00Z",
-            "version": "3.58"
+            "version": "3.59"
         }
     ]
 }

api-ref/source/v3/samples/versions/versions-response.json

@@ -46,7 +46,7 @@
             "min_version": "3.0",
             "status": "CURRENT",
             "updated": "2018-07-17T00:00:00Z",
-            "version": "3.58"
+            "version": "3.59"
         }
     ]
 }

api-ref/source/v3/vol-transfer-v3.inc

@@ -147,6 +147,11 @@ Request
 
    - project_id: project_id_path
    - all_tenants: all-tenants
+   - limit: limit_transfer
+   - offset: offset_transfer
+   - marker: marker_transfer
+   - sort_key: sort_key_transfer
+   - sort_dir: sort_dir_transfer
 
 
 Response Parameters

cinder/api/contrib/volume_transfer.py

@@ -61,7 +61,9 @@ class VolumeTransferController(wsgi.Controller):
         context = req.environ['cinder.context']
         filters = req.params.copy()
         LOG.debug('Listing volume transfers')
-        transfers = self.transfer_api.get_all(context, filters=filters)
+        transfers = self.transfer_api.get_all(context, filters=filters,
+                                              sort_keys=['created_at', 'id'],
+                                              sort_dirs=['asc', 'asc'])
         transfer_count = len(transfers)
         limited_list = common.limited(transfers, req)
 

cinder/api/microversions.py

@@ -157,6 +157,8 @@ TRANSFER_WITH_HISTORY = '3.57'
 
 GROUP_PROJECT_ID = '3.58'
 
+SUPPORT_TRANSFER_PAGINATION = '3.59'
+
 
 def get_mv_header(version):
     """Gets a formatted HTTP microversion header.

cinder/api/openstack/api_version_request.py

@@ -133,6 +133,7 @@ REST_API_VERSION_HISTORY = """
              transfer.
     * 3.58 - Add ``project_id`` attribute to response body of list groups with
              detail and show group detail APIs.
+    * 3.59 - Support volume transfer pagination.
 """
 
 # The minimum and maximum versions of the API supported
@@ -140,7 +141,7 @@ REST_API_VERSION_HISTORY = """
 # minimum version of the API supported.
 # Explicitly using /v2 endpoints will still work
 _MIN_API_VERSION = "3.0"
-_MAX_API_VERSION = "3.58"
+_MAX_API_VERSION = "3.59"
 _LEGACY_API_VERSION2 = "2.0"
 UPDATED = "2018-07-17T00:00:00Z"
 

cinder/api/openstack/rest_api_version_history.rst

@@ -457,3 +457,8 @@ related api (create/show/list detail transfer APIs) responses.
 ----
 Add ``project_id`` attribute to response body of list groups with detail and show
 group detail APIs.
+
+3.59
+----
+Support volume transfer pagination.
+

cinder/api/v3/volume_transfer.py

@@ -17,6 +17,7 @@ from oslo_utils import strutils
 from six.moves import http_client
 from webob import exc
 
+from cinder.api import common
 from cinder.api.contrib import volume_transfer as volume_transfer_v2
 from cinder.api import microversions as mv
 from cinder.api.openstack import wsgi
@@ -30,6 +31,49 @@ LOG = logging.getLogger(__name__)
 class VolumeTransferController(volume_transfer_v2.VolumeTransferController):
     """The transfer API controller for the OpenStack API V3."""
 
+    def _get_transfers(self, req, is_detail):
+        """Returns a list of transfers, transformed through view builder."""
+        context = req.environ['cinder.context']
+        req_version = req.api_version_request
+        params = req.params.copy()
+        marker = limit = offset = None
+        if req_version.matches(mv.SUPPORT_TRANSFER_PAGINATION):
+            marker, limit, offset = common.get_pagination_params(params)
+            sort_keys, sort_dirs = common.get_sort_params(params)
+        else:
+            # NOTE(yikun): After microversion SUPPORT_TRANSFER_PAGINATION,
+            # transfers list api use the ['created_at'], ['asc']
+            # as default order, but we should keep the compatible in here.
+            sort_keys, sort_dirs = ['created_at', 'id'], ['asc', 'asc']
+        filters = params
+        LOG.debug('Listing volume transfers')
+
+        transfers = self.transfer_api.get_all(context, marker=marker,
+                                              limit=limit,
+                                              sort_keys=sort_keys,
+                                              sort_dirs=sort_dirs,
+                                              filters=filters,
+                                              offset=offset)
+        transfer_count = len(transfers)
+        limited_list = common.limited(transfers, req)
+
+        if is_detail:
+            transfers = self._view_builder.detail_list(req, limited_list,
+                                                       transfer_count)
+        else:
+            transfers = self._view_builder.summary_list(req, limited_list,
+                                                        transfer_count)
+
+        return transfers
+
+    def index(self, req):
+        """Returns a summary list of transfers."""
+        return self._get_transfers(req, is_detail=False)
+
+    def detail(self, req):
+        """Returns a detailed list of transfers."""
+        return self._get_transfers(req, is_detail=True)
+
     @wsgi.response(http_client.ACCEPTED)
     @validation.schema(volume_transfer.create, mv.BASE_VERSION,
                        mv.get_prior_version(mv.TRANSFER_WITH_SNAPSHOTS))

cinder/db/api.py

@@ -1269,14 +1269,22 @@ def transfer_get(context, transfer_id):
     return IMPL.transfer_get(context, transfer_id)
 
 
-def transfer_get_all(context):
+def transfer_get_all(context, marker=None, limit=None, sort_keys=None,
+                     sort_dirs=None, filters=None, offset=None):
     """Get all volume transfer records."""
-    return IMPL.transfer_get_all(context)
+    return IMPL.transfer_get_all(context, marker=marker, limit=limit,
+                                 sort_keys=sort_keys, sort_dirs=sort_dirs,
+                                 filters=filters, offset=offset)
 
 
-def transfer_get_all_by_project(context, project_id):
+def transfer_get_all_by_project(context, project_id, marker=None,
+                                limit=None, sort_keys=None,
+                                sort_dirs=None, filters=None, offset=None):
     """Get all volume transfer records for specified project."""
-    return IMPL.transfer_get_all_by_project(context, project_id)
+    return IMPL.transfer_get_all_by_project(context, project_id, marker=marker,
+                                            limit=limit, sort_keys=sort_keys,
+                                            sort_dirs=sort_dirs,
+                                            filters=filters, offset=offset)
 
 
 def transfer_create(context, values):

cinder/db/sqlalchemy/api.py

@@ -5470,6 +5470,22 @@ def transfer_get(context, transfer_id):
     return _transfer_get(context, transfer_id)
 
 
+def _process_transfer_filters(query, filters):
+    if filters:
+        project_id = filters.pop('project_id', None)
+        # Ensure that filters' keys exist on the model
+        if not is_valid_model_filters(models.Transfer, filters):
+            return
+        if project_id:
+            volume = models.Volume
+            query = query.filter(volume.id ==
+                                 models.Transfer.volume_id,
+                                 volume.project_id == project_id)
+
+        query = query.filter_by(**filters)
+    return query
+
+
 def _translate_transfers(transfers):
     fields = ('id', 'volume_id', 'display_name', 'created_at', 'deleted',
               'no_snapshots', 'source_project_id', 'destination_project_id',
@@ -5477,21 +5493,42 @@ def _translate_transfers(transfers):
     return [{k: transfer[k] for k in fields} for transfer in transfers]
 
 
+def _transfer_get_all(context, marker=None, limit=None, sort_keys=None,
+                      sort_dirs=None, filters=None, offset=None):
+    session = get_session()
+    with session.begin():
+        # Generate the query
+        query = _generate_paginate_query(context, session, marker, limit,
+                                         sort_keys, sort_dirs, filters, offset,
+                                         models.Transfer)
+        if query is None:
+            return []
+        return _translate_transfers(query.all())
+
+
 @require_admin_context
-def transfer_get_all(context):
+def transfer_get_all(context, marker=None, limit=None, sort_keys=None,
-    results = model_query(context, models.Transfer).all()
+                     sort_dirs=None, filters=None, offset=None):
-    return _translate_transfers(results)
+    return _transfer_get_all(context, marker=marker, limit=limit,
+                             sort_keys=sort_keys, sort_dirs=sort_dirs,
+                             filters=filters, offset=offset)
+
+
+def _transfer_get_query(context, session=None, project_only=False):
+    return model_query(context, models.Transfer, session=session,
+                       project_only=project_only)
 
 
 @require_context
-def transfer_get_all_by_project(context, project_id):
+def transfer_get_all_by_project(context, project_id, marker=None,
+                                limit=None, sort_keys=None,
+                                sort_dirs=None, filters=None, offset=None):
     authorize_project_context(context, project_id)
-
+    filters = filters.copy() if filters else {}
-    query = (model_query(context, models.Transfer)
+    filters['project_id'] = project_id
-             .filter(models.Volume.id == models.Transfer.volume_id,
+    return _transfer_get_all(context, marker=marker, limit=limit,
-                     models.Volume.project_id == project_id))
+                             sort_keys=sort_keys, sort_dirs=sort_dirs,
-    results = query.all()
+                             filters=filters, offset=offset)
-    return _translate_transfers(results)
 
 
 @require_context
@@ -6836,6 +6873,8 @@ PAGINATION_HELPERS = {
     models.VolumeAttachment: (_attachment_get_query,
                               _process_attachment_filters,
                               _attachment_get),
+    models.Transfer: (_transfer_get_query, _process_transfer_filters,
+                      _transfer_get),
 }
 
 

cinder/tests/unit/api/v3/test_volume_transfer.py

@@ -16,6 +16,7 @@
 """
 Tests for volume transfer code.
 """
+import ddt
 
 from oslo_serialization import jsonutils
 from six.moves import http_client
@@ -23,6 +24,7 @@ import webob
 
 from cinder.api.contrib import volume_transfer
 from cinder.api import microversions as mv
+from cinder.api.v3 import volume_transfer as volume_transfer_v3
 from cinder import context
 from cinder import db
 from cinder.objects import fields
@@ -32,6 +34,7 @@ from cinder.tests.unit import fake_constants as fake
 import cinder.transfer
 
 
+@ddt.ddt
 class VolumeTransferAPITestCase(test.TestCase):
     """Test Case for transfers V3 API."""
 
@@ -44,6 +47,7 @@ class VolumeTransferAPITestCase(test.TestCase):
         super(VolumeTransferAPITestCase, self).setUp()
         self.volume_transfer_api = cinder.transfer.API()
         self.controller = volume_transfer.VolumeTransferController()
+        self.v3_controller = volume_transfer_v3.VolumeTransferController()
         self.user_ctxt = context.RequestContext(
             fake.USER_ID, fake.PROJECT_ID, auth_token=True, is_admin=True)
 
@@ -128,6 +132,55 @@ class VolumeTransferAPITestCase(test.TestCase):
         self.assertEqual(transfer2['id'], res_dict['transfers'][1]['id'])
         self.assertEqual('test_transfer', res_dict['transfers'][1]['name'])
 
+    def test_list_transfers_with_limit(self):
+        volume_id_1 = self._create_volume(size=5)
+        volume_id_2 = self._create_volume(size=5)
+        self._create_transfer(volume_id_1)
+        self._create_transfer(volume_id_2)
+        url = '/v3/%s/volume-transfers?limit=1' % fake.PROJECT_ID
+        req = fakes.HTTPRequest.blank(url,
+                                      version=mv.SUPPORT_TRANSFER_PAGINATION,
+                                      use_admin_context=True)
+        res_dict = self.v3_controller.index(req)
+
+        self.assertEqual(1, len(res_dict['transfers']))
+
+    def test_list_transfers_with_marker(self):
+        volume_id_1 = self._create_volume(size=5)
+        volume_id_2 = self._create_volume(size=5)
+        transfer1 = self._create_transfer(volume_id_1)
+        transfer2 = self._create_transfer(volume_id_2)
+        url = '/v3/%s/volume-transfers?marker=%s' % (fake.PROJECT_ID,
+                                                     transfer2['id'])
+        req = fakes.HTTPRequest.blank(url,
+                                      version=mv.SUPPORT_TRANSFER_PAGINATION,
+                                      use_admin_context=True)
+        res_dict = self.v3_controller.index(req)
+
+        self.assertEqual(1, len(res_dict['transfers']))
+        self.assertEqual(transfer1['id'],
+                         res_dict['transfers'][0]['id'])
+
+    @ddt.data("desc", "asc")
+    def test_list_transfers_with_sort(self, sort_dir):
+        volume_id_1 = self._create_volume(size=5)
+        volume_id_2 = self._create_volume(size=5)
+        transfer1 = self._create_transfer(volume_id_1)
+        transfer2 = self._create_transfer(volume_id_2)
+        url = '/v3/%s/volume-transfers?sort_key=id&sort_dir=%s' % (
+            fake.PROJECT_ID, sort_dir)
+        req = fakes.HTTPRequest.blank(url,
+                                      version=mv.SUPPORT_TRANSFER_PAGINATION,
+                                      use_admin_context=True)
+        res_dict = self.v3_controller.index(req)
+
+        self.assertEqual(2, len(res_dict['transfers']))
+        order_ids = sorted([transfer1['id'],
+                            transfer2['id']])
+        expect_result = order_ids[1] if sort_dir == "desc" else order_ids[0]
+        self.assertEqual(expect_result,
+                         res_dict['transfers'][0]['id'])
+
     def test_list_transfers_detail(self):
         volume_id_1 = self._create_volume(size=5)
         volume_id_2 = self._create_volume(size=5)

cinder/transfer/api.py

@@ -79,14 +79,24 @@ class API(base.Base):
         volume_utils.notify_about_volume_usage(context, volume_ref,
                                                "transfer.delete.end")
 
-    def get_all(self, context, filters=None):
+    def get_all(self, context, marker=None,
+                limit=None, sort_keys=None,
+                sort_dirs=None, filters=None, offset=None):
         filters = filters or {}
         context.authorize(policy.GET_ALL_POLICY)
         if context.is_admin and 'all_tenants' in filters:
-            transfers = self.db.transfer_get_all(context)
+            del filters['all_tenants']
+            transfers = self.db.transfer_get_all(context, marker=marker,
+                                                 limit=limit,
+                                                 sort_keys=sort_keys,
+                                                 sort_dirs=sort_dirs,
+                                                 filters=filters,
+                                                 offset=offset)
         else:
-            transfers = self.db.transfer_get_all_by_project(context,
+            transfers = self.db.transfer_get_all_by_project(
-                                                            context.project_id)
+                context, context.project_id, marker=marker,
+                limit=limit, sort_keys=sort_keys, sort_dirs=sort_dirs,
+                filters=filters, offset=offset)
         return transfers
 
     def _get_random_string(self, length):

releasenotes/notes/add-transfer-pagination-support-7y33u7y68de3cb16.yaml

@@ -0,0 +1,3 @@
+---
+features:
+  -  Added transfer pagination support since microversion 3.59.