From a5dc7f1a11b14dcf9602de5dcd761b7ccce81711 Mon Sep 17 00:00:00 2001 From: Sylvain Bauza Date: Wed, 2 Nov 2016 12:28:15 +0100 Subject: [PATCH] Expose a REST API for a specific list of RPs Now that we merged the object method for getting the list of ResourceProviders based on a specific amount request, we need to expose that method into a REST API call so that the scheduler client could be calling it. Co-Authored-By: Jay Pipes Change-Id: Ia8b534d20c064eb3a767f95ca22814925acfaa77 Implements: blueprint resource-providers-get-by-request --- .../placement/handlers/resource_provider.py | 136 +++++++++++++++--- nova/api/openstack/placement/microversion.py | 1 + .../placement/rest_api_version_history.rst | 29 ++++ nova/objects/resource_provider.py | 4 - .../api/openstack/placement/fixtures.py | 30 +++- .../placement/gabbits/microversion.yaml | 4 +- .../gabbits/resource-provider-aggregates.yaml | 8 +- .../resource-provider-resources-query.yaml | 136 ++++++++++++++++++ .../placement/gabbits/resource-provider.yaml | 4 +- .../placement/gabbits/with-allocations.yaml | 3 +- ...oviders-by-resources-0ab51c9766fe654f.yaml | 19 +++ 11 files changed, 344 insertions(+), 30 deletions(-) create mode 100644 nova/tests/functional/api/openstack/placement/gabbits/resource-provider-resources-query.yaml create mode 100644 releasenotes/notes/placement-rest-api-filter-providers-by-resources-0ab51c9766fe654f.yaml diff --git a/nova/api/openstack/placement/handlers/resource_provider.py b/nova/api/openstack/placement/handlers/resource_provider.py index 703169dd89a6..2148cb498d54 100644 --- a/nova/api/openstack/placement/handlers/resource_provider.py +++ b/nova/api/openstack/placement/handlers/resource_provider.py @@ -12,6 +12,7 @@ """Placement API handlers for resource providers.""" import copy +import jsonschema from oslo_db import exception as db_exc from oslo_serialization import jsonutils @@ -46,6 +47,102 @@ POST_RESOURCE_PROVIDER_SCHEMA = { PUT_RESOURCE_PROVIDER_SCHEMA = copy.deepcopy(POST_RESOURCE_PROVIDER_SCHEMA) PUT_RESOURCE_PROVIDER_SCHEMA['properties'].pop('uuid') +# Represents the allowed query string parameters to the GET /resource_providers +# API call +GET_RPS_SCHEMA_1_0 = { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "uuid": { + "type": "string", + "format": "uuid" + } + }, + "additionalProperties": False, +} + +# Placement API microversion 1.3 adds support for a member_of attribute +GET_RPS_SCHEMA_1_3 = copy.deepcopy(GET_RPS_SCHEMA_1_0) +GET_RPS_SCHEMA_1_3['properties']['member_of'] = { + # TODO(mriedem): At some point we need to do jsonschema and/or uuid + # validation of the value(s) here. + "type": "string" +} + +# Placement API microversion 1.4 adds support for requesting resource providers +# having some set of capacity for some resources. The query string is a +# comma-delimited set of "$RESOURCE_CLASS_NAME:$AMOUNT" strings. The validation +# of the string is left up to the helper code in the +# _normalize_resources_qs_param() function below. +GET_RPS_SCHEMA_1_4 = copy.deepcopy(GET_RPS_SCHEMA_1_3) +GET_RPS_SCHEMA_1_4['properties']['resources'] = { + "type": "string" +} + + +def _normalize_resources_qs_param(qs): + """Given a query string parameter for resources, validate it meets the + expected format and return a dict of amounts, keyed by resource class name. + + The expected format of the resources parameter looks like so: + + $RESOURCE_CLASS_NAME:$AMOUNT,$RESOURCE_CLASS_NAME:$AMOUNT + + So, if the user was looking for resource providers that had room for an + instance that will consume 2 vCPUs, 1024 MB of RAM and 50GB of disk space, + they would use the following query string: + + ?resources=VCPU:2,MEMORY_MB:1024:DISK_GB:50 + + The returned value would be: + + { + "VCPU": 2, + "MEMORY_MB": 1024, + "DISK_GB": 50, + } + + :param qs: The value of the 'resources' query string parameter + :raises `webob.exc.HTTPBadRequest` if the parameter's value isn't in the + expected format. + """ + result = {} + resource_tuples = qs.split(',') + for rt in resource_tuples: + try: + rc_name, amount = rt.split(':') + except ValueError: + msg = _('Badly formed resources parameter. Expected resources ' + 'query string parameter in form: ' + '?resources=VCPU:2,MEMORY_MB:1024. Got: %s.') + msg = msg % rt + raise webob.exc.HTTPBadRequest(msg, + json_formatter=util.json_error_formatter) + try: + amount = int(amount) + except ValueError: + msg = _('Requested resource %(resource_name)s expected positive ' + 'integer amount. Got: %(amount)s.') + msg = msg % { + 'resource_name': rc_name, + 'amount': amount, + } + raise webob.exc.HTTPBadRequest(msg, + json_formatter=util.json_error_formatter) + if amount < 1: + msg = _('Requested resource %(resource_name)s requires ' + 'amount >= 1. Got: %(amount)d.') + msg = msg % { + 'resource_name': rc_name, + 'amount': amount, + } + raise webob.exc.HTTPBadRequest(msg, + json_formatter=util.json_error_formatter) + result[rc_name] = amount + return result + def _serialize_links(environ, resource_provider): url = util.resource_provider_url(environ, resource_provider) @@ -165,24 +262,22 @@ def list_resource_providers(req): context = req.environ['placement.context'] want_version = req.environ[microversion.MICROVERSION_ENVIRON] - allowed_filters = set(objects.ResourceProviderList.allowed_filters) - if not want_version.matches((1, 3)): - allowed_filters.remove('member_of') - passed_filters = set(req.GET.keys()) - invalid_filters = passed_filters - allowed_filters - if invalid_filters: + schema = GET_RPS_SCHEMA_1_0 + if want_version == (1, 3): + schema = GET_RPS_SCHEMA_1_3 + if want_version >= (1, 4): + schema = GET_RPS_SCHEMA_1_4 + try: + jsonschema.validate(dict(req.GET), schema, + format_checker=jsonschema.FormatChecker()) + except jsonschema.ValidationError as exc: raise webob.exc.HTTPBadRequest( - _('Invalid filters: %(filters)s') % - {'filters': ', '.join(invalid_filters)}, - json_formatter=util.json_error_formatter) - - if 'uuid' in req.GET and not uuidutils.is_uuid_like(req.GET['uuid']): - raise webob.exc.HTTPBadRequest( - _('Invalid uuid value: %(uuid)s') % {'uuid': req.GET['uuid']}, + _('Invalid query string parameters: %(exc)s') % + {'exc': exc}, json_formatter=util.json_error_formatter) filters = {} - for attr in objects.ResourceProviderList.allowed_filters: + for attr in ['uuid', 'name', 'member_of']: if attr in req.GET: value = req.GET[attr] # special case member_of to always make its value a @@ -196,8 +291,17 @@ def list_resource_providers(req): else: value = [value] filters[attr] = value - resource_providers = objects.ResourceProviderList.get_all_by_filters( - context, filters) + if 'resources' in req.GET: + resources = _normalize_resources_qs_param(req.GET['resources']) + filters['resources'] = resources + try: + resource_providers = objects.ResourceProviderList.get_all_by_filters( + context, filters) + except exception.ResourceClassNotFound as exc: + raise webob.exc.HTTPBadRequest( + _('Invalid resource class in resources parameter: %(error)s') % + {'error': exc}, + json_formatter=util.json_error_formatter) response = req.response response.body = jsonutils.dumps(_serialize_providers( diff --git a/nova/api/openstack/placement/microversion.py b/nova/api/openstack/placement/microversion.py index d293dcf1ff2e..45a3f1193ed1 100644 --- a/nova/api/openstack/placement/microversion.py +++ b/nova/api/openstack/placement/microversion.py @@ -39,6 +39,7 @@ VERSIONS = [ '1.2', # Adds /resource_classes resource endpoint '1.3', # Adds 'member_of' query parameter to get resource providers # that are members of any of the listed aggregates + '1.4', # Adds resources query string parameter in GET /resource_providers ] diff --git a/nova/api/openstack/placement/rest_api_version_history.rst b/nova/api/openstack/placement/rest_api_version_history.rst index 2810af5d68d8..26310da1a57d 100644 --- a/nova/api/openstack/placement/rest_api_version_history.rst +++ b/nova/api/openstack/placement/rest_api_version_history.rst @@ -52,3 +52,32 @@ Version 1.3 adds support for listing resource providers that are members of any of the list of aggregates provided using a ``member_of`` query parameter: * /resource_providers?member_of=in:{agg1_uuid},{agg2_uuid},{agg3_uuid} + +1.4 -- Filter resource providers having requested resource capacity +------------------------------------------------------------------- + +The 1.4 version adds support for querying resource providers that have the +ability to serve a requested set of resources. A new "resources" query string +parameter is now accepted to the `GET /resource_providers` API call. This +parameter indicates the requested amounts of various resources that a provider +must have the capacity to serve. The "resources" query string parameter takes +the form: + +``?resources=$RESOURCE_CLASS_NAME:$AMOUNT,$RESOURCE_CLASS_NAME:$AMOUNT`` + +For instance, if the user wishes to see resource providers that can service a +request for 2 vCPUs, 1024 MB of RAM and 50 GB of disk space, the user can issue +a request to: + +`GET /resource_providers?resources=VCPU:2,MEMORY_MB:1024,DISK_GB:50` + +If the resource class does not exist, then it will return a HTTP 400. + +.. note:: The resources filtering is also based on the `min_unit`, `max_unit` + and `step_size` of the inventory record. For example, if the `max_unit` is + 512 for the DISK_GB inventory for a particular resource provider and a + GET request is made for `DISK_GB:1024`, that resource provider will not be + returned. The `min_unit` is the minimum amount of resource that can be + requested for a given inventory and resource provider. The `step_size` is + the increment of resource that can be requested for a given resource on a + given provider. diff --git a/nova/objects/resource_provider.py b/nova/objects/resource_provider.py index e5b2b802252a..6a716c995755 100644 --- a/nova/objects/resource_provider.py +++ b/nova/objects/resource_provider.py @@ -539,10 +539,6 @@ class ResourceProviderList(base.ObjectListBase, base.NovaObject): 'objects': fields.ListOfObjectsField('ResourceProvider'), } - allowed_filters = ( - 'name', 'uuid', 'member_of' - ) - @staticmethod @db_api.api_context_manager.reader def _get_all_by_filters_from_db(context, filters): diff --git a/nova/tests/functional/api/openstack/placement/fixtures.py b/nova/tests/functional/api/openstack/placement/fixtures.py index a55be910469b..8a3d6106bd53 100644 --- a/nova/tests/functional/api/openstack/placement/fixtures.py +++ b/nova/tests/functional/api/openstack/placement/fixtures.py @@ -91,9 +91,12 @@ class AllocationFixture(APIFixture): rp = objects.ResourceProvider( self.context, name=rp_name, uuid=rp_uuid) rp.create() + + # Create some DISK_GB inventory and allocations. inventory = objects.Inventory( self.context, resource_provider=rp, - resource_class='DISK_GB', total=2048) + resource_class='DISK_GB', total=2048, + step_size=10, min_unit=10, max_unit=600) inventory.obj_set_defaults() rp.add_inventory(inventory) allocation = objects.Allocation( @@ -108,3 +111,28 @@ class AllocationFixture(APIFixture): consumer_id=uuidutils.generate_uuid(), used=512) allocation.create() + + # Create some VCPU inventory and allocations. + inventory = objects.Inventory( + self.context, resource_provider=rp, + resource_class='VCPU', total=8, + max_unit=4) + inventory.obj_set_defaults() + rp.add_inventory(inventory) + allocation = objects.Allocation( + self.context, resource_provider=rp, + resource_class='VCPU', + consumer_id=uuidutils.generate_uuid(), + used=2) + allocation.create() + allocation = objects.Allocation( + self.context, resource_provider=rp, + resource_class='VCPU', + consumer_id=uuidutils.generate_uuid(), + used=4) + allocation.create() + + # The ALT_RP_XXX variables are for a resource provider that has + # not been created in the Allocation fixture + os.environ['ALT_RP_UUID'] = uuidutils.generate_uuid() + os.environ['ALT_RP_NAME'] = uuidutils.generate_uuid() diff --git a/nova/tests/functional/api/openstack/placement/gabbits/microversion.yaml b/nova/tests/functional/api/openstack/placement/gabbits/microversion.yaml index 56a6729e16cb..d6002dd1fceb 100644 --- a/nova/tests/functional/api/openstack/placement/gabbits/microversion.yaml +++ b/nova/tests/functional/api/openstack/placement/gabbits/microversion.yaml @@ -37,13 +37,13 @@ tests: response_strings: - "Unacceptable version header: 0.5" -- name: latest microversion is 1.3 +- name: latest microversion is 1.4 GET: / request_headers: openstack-api-version: placement latest response_headers: vary: /OpenStack-API-Version/ - openstack-api-version: placement 1.3 + openstack-api-version: placement 1.4 - name: other accept header bad version GET: / diff --git a/nova/tests/functional/api/openstack/placement/gabbits/resource-provider-aggregates.yaml b/nova/tests/functional/api/openstack/placement/gabbits/resource-provider-aggregates.yaml index 044806803295..ef6c37ccb279 100644 --- a/nova/tests/functional/api/openstack/placement/gabbits/resource-provider-aggregates.yaml +++ b/nova/tests/functional/api/openstack/placement/gabbits/resource-provider-aggregates.yaml @@ -90,11 +90,11 @@ tests: request_headers: openstack-api-version: placement 1.1 status: 400 - response_json_paths: - $.errors[0].detail: '/Invalid filters: member_of/' + response_strings: + - 'Invalid query string parameters' - name: error on bogus query parameter GET: '/resource_providers?assoc_with_aggregate=in:83a3d69d-8920-48e2-8914-cadfd8fa2f91,99652f11-9f77-46b9-80b7-4b1989be9f8c' status: 400 - response_json_paths: - $.errors[0].detail: '/Invalid filters: assoc_with_aggregate/' + response_strings: + - 'Invalid query string parameters' diff --git a/nova/tests/functional/api/openstack/placement/gabbits/resource-provider-resources-query.yaml b/nova/tests/functional/api/openstack/placement/gabbits/resource-provider-resources-query.yaml new file mode 100644 index 000000000000..3dcdc499d25f --- /dev/null +++ b/nova/tests/functional/api/openstack/placement/gabbits/resource-provider-resources-query.yaml @@ -0,0 +1,136 @@ + +fixtures: + - AllocationFixture + +defaults: + request_headers: + x-auth-token: admin + content-type: application/json + OpenStack-API-Version: placement latest + +tests: + +- name: what is at resource providers + GET: /resource_providers + response_json_paths: + $.resource_providers.`len`: 1 + $.resource_providers[0].uuid: $ENVIRON['RP_UUID'] + $.resource_providers[0].name: $ENVIRON['RP_NAME'] + $.resource_providers[0].links[?rel = "self"].href: /resource_providers/$ENVIRON['RP_UUID'] + $.resource_providers[0].links[?rel = "inventories"].href: /resource_providers/$ENVIRON['RP_UUID']/inventories + $.resource_providers[0].links[?rel = "aggregates"].href: /resource_providers/$ENVIRON['RP_UUID']/aggregates + $.resource_providers[0].links[?rel = "usages"].href: /resource_providers/$ENVIRON['RP_UUID']/usages + +- name: post new resource provider + POST: /resource_providers + data: + name: $ENVIRON['ALT_RP_NAME'] + uuid: $ENVIRON['ALT_RP_UUID'] + status: 201 + response_headers: + location: //resource_providers/[a-f0-9-]+/ + response_forbidden_headers: + - content-type + +- name: now 2 providers listed + GET: /resource_providers + response_json_paths: + $.resource_providers.`len`: 2 + +- name: list resource providers providing resources filter before API 1.4 + GET: /resource_providers?resources=VCPU:1 + request_headers: + OpenStack-API-Version: placement 1.3 + status: 400 + response_strings: + - 'Invalid query string parameters' + +- name: list resource providers providing a badly-formatted resources filter + GET: /resource_providers?resources=VCPU + status: 400 + response_strings: + - 'Badly formed resources parameter. Expected resources query string parameter in form:' + - 'Got: VCPU.' + +- name: list resource providers providing a resources filter with non-integer amount + GET: /resource_providers?resources=VCPU:fred + status: 400 + response_strings: + - 'Requested resource VCPU expected positive integer amount.' + - 'Got: fred.' + +- name: list resource providers providing a resources filter with negative amount + GET: /resource_providers?resources=VCPU:-2 + status: 400 + response_strings: + - 'Requested resource VCPU requires amount >= 1.' + - 'Got: -2.' + +- name: list resource providers providing a resource class not existing + GET: /resource_providers?resources=MYMISSINGCLASS:1 + status: 400 + response_strings: + - 'Invalid resource class in resources parameter' + +- name: list resource providers providing a bad trailing comma + GET: /resource_providers?resources=DISK_GB:500, + status: 400 + response_strings: + - 'Badly formed resources parameter. Expected resources query string parameter in form:' + # NOTE(mriedem): The value is empty because splitting on the trailing + # comma results in an empty string. + - 'Got: .' + +- name: list resource providers providing disk resources + GET: /resource_providers?resources=DISK_GB:500 + response_json_paths: + $.resource_providers.`len`: 1 + $.resource_providers[0].uuid: $ENVIRON['RP_UUID'] + +- name: list resource providers providing disk and vcpu resources + GET: /resource_providers?resources=DISK_GB:500,VCPU:2 + response_json_paths: + $.resource_providers.`len`: 1 + $.resource_providers[0].uuid: $ENVIRON['RP_UUID'] + +- name: list resource providers providing resources (no match - less than min_unit) + GET: /resource_providers?resources=DISK_GB:1 + response_json_paths: + $.resource_providers.`len`: 0 + +- name: list resource providers providing resources (no match - more than max_unit) + GET: /resource_providers?resources=DISK_GB:610 + response_json_paths: + $.resource_providers.`len`: 0 + +- name: list resource providers providing resources (no match - not enough inventory) + GET: /resource_providers?resources=DISK_GB:102400 + response_json_paths: + $.resource_providers.`len`: 0 + +- name: list resource providers providing resources (no match - bad step size) + GET: /resource_providers?resources=DISK_GB:11 + response_json_paths: + $.resource_providers.`len`: 0 + +- name: list resource providers providing resources (no match - no inventory of resource) + GET: /resource_providers?resources=MEMORY_MB:10240 + response_json_paths: + $.resource_providers.`len`: 0 + +- name: list resource providers providing resources (no match - not enough VCPU) + GET: /resource_providers?resources=DISK_GB:500,VCPU:4 + response_json_paths: + $.resource_providers.`len`: 0 + +- name: associate an aggregate with rp1 + PUT: /resource_providers/$ENVIRON['RP_UUID']/aggregates + data: + - 83a3d69d-8920-48e2-8914-cadfd8fa2f91 + status: 200 + +- name: get by aggregates with resources + GET: '/resource_providers?member_of=in:83a3d69d-8920-48e2-8914-cadfd8fa2f91&resources=VCPU:2' + response_json_paths: + $.resource_providers.`len`: 1 + $.resource_providers[0].uuid: $ENVIRON['RP_UUID'] diff --git a/nova/tests/functional/api/openstack/placement/gabbits/resource-provider.yaml b/nova/tests/functional/api/openstack/placement/gabbits/resource-provider.yaml index d564a9905f82..6ce3ebb99e09 100644 --- a/nova/tests/functional/api/openstack/placement/gabbits/resource-provider.yaml +++ b/nova/tests/functional/api/openstack/placement/gabbits/resource-provider.yaml @@ -129,13 +129,13 @@ tests: GET: /resource_providers?uuid=spameggs status: 400 response_strings: - - 'Invalid uuid value: spameggs' + - 'Invalid query string parameters' - name: list resource providers providing an invalid filter GET: /resource_providers?spam=eggs status: 400 response_strings: - - 'Invalid filters: spam' + - 'Invalid query string parameters' - name: list one resource provider filtering by uuid GET: /resource_providers?uuid=$ENVIRON['RP_UUID'] diff --git a/nova/tests/functional/api/openstack/placement/gabbits/with-allocations.yaml b/nova/tests/functional/api/openstack/placement/gabbits/with-allocations.yaml index 85d72bc63d19..592eac33a0f4 100644 --- a/nova/tests/functional/api/openstack/placement/gabbits/with-allocations.yaml +++ b/nova/tests/functional/api/openstack/placement/gabbits/with-allocations.yaml @@ -21,8 +21,9 @@ tests: # required but superfluous, is present content-type: /application/json/ response_json_paths: - $.resource_provider_generation: 1 + $.resource_provider_generation: 2 $.usages.DISK_GB: 1024 + $.usages.VCPU: 6 - name: fail to delete resource provider DELETE: /resource_providers/$ENVIRON['RP_UUID'] diff --git a/releasenotes/notes/placement-rest-api-filter-providers-by-resources-0ab51c9766fe654f.yaml b/releasenotes/notes/placement-rest-api-filter-providers-by-resources-0ab51c9766fe654f.yaml new file mode 100644 index 000000000000..86633310ea56 --- /dev/null +++ b/releasenotes/notes/placement-rest-api-filter-providers-by-resources-0ab51c9766fe654f.yaml @@ -0,0 +1,19 @@ +--- +features: + - | + A new Placement API microversion 1.4 is added. Users may now query the + Placement REST API for resource providers that have the ability to meet a + set of requested resource amounts. The `GET /resource_providers` API call + can have a "resources" query string parameter supplied that indicates the + requested amounts of various resources that a provider must have the + capacity to serve. The "resources" query string parameter takes the form: + + ``?resources=$RESOURCE_CLASS_NAME:$AMOUNT,$RESOURCE_CLASS_NAME:$AMOUNT`` + + For instance, if the user wishes to see resource providers that can service + a request for 2 vCPUs, 1024 MB of RAM and 50 GB of disk space, the user can + issue a request of:: + + ``GET /resource_providers?resources=VCPU:2,MEMORY_MB:1024,DISK_GB:50`` + + The placement API is only available to admin users.