openstack/nova
Code Issues Proposed changes
nova/nova/tests/functional/test_servers_resource_request.py
Balazs Gibizer e357ad3c23 Reject server create with extended resource req 
To prepare for the unlikely event that Neutron merges and an operator
enables the port-resource-request-groups neutron API extension before
nova adds support for it, this patch rejects server creation if such
extension is enabled in Neutron. Enabling that extension has zero
benefits without nova support hence the harsh but simple rejection.

A subsequent patch will reject server lifecycle operations in a more
sophisticated way and as soon as we support some operations, like
boot, the deployer might rightfully choose to enable the Neutron
extension.

Change-Id: I2c55d9da13a570efbc1c862116cea31aaa6aa02e
blueprint: qos-minimum-guaranteed-packet-rate
2021-08-21 09:49:54 +02:00

2649 lines
113 KiB
Python
Raw Blame History

# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import copy
import logging
from keystoneauth1 import adapter
import mock
from neutronclient.common import exceptions as neutron_exception
import os_resource_classes as orc
from oslo_config import cfg
from oslo_serialization import jsonutils
from oslo_utils.fixture import uuidsentinel as uuids
from nova.compute import instance_actions
from nova.compute import manager as compute_manager
from nova import context
from nova import exception
from nova.network import constants
from nova.network import neutron as neutronapi
from nova import objects
from nova.policies import base as base_policies
from nova.policies import servers as servers_policies
from nova.scheduler import utils
from nova import test
from nova.tests import fixtures as nova_fixtures
from nova.tests.fixtures import NeutronFixture
from nova.tests.functional.api import client
from nova.tests.functional import integrated_helpers
from nova.tests.unit import fake_requests
from nova.virt import fake
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
class ResourceRequestNeutronFixture(NeutronFixture):
port_with_sriov_resource_request = {
'id': '7059503b-a648-40fd-a561-5ca769304bee',
'name': '',
'description': '',
'network_id': NeutronFixture.network_2['id'],
'admin_state_up': True,
'status': 'ACTIVE',
'mac_address': '52:54:00:1e:59:c5',
# Do neutron really adds fixed_ips to an direct vnic_type port?
'fixed_ips': [
{
'ip_address': '192.168.13.3',
'subnet_id': NeutronFixture.subnet_2['id']
}
],
'tenant_id': NeutronFixture.tenant_id,
'project_id': NeutronFixture.tenant_id,
'device_id': '',
'resource_request': {
"resources": {
orc.NET_BW_IGR_KILOBIT_PER_SEC: 10000,
orc.NET_BW_EGR_KILOBIT_PER_SEC: 10000},
"required": ["CUSTOM_PHYSNET2", "CUSTOM_VNIC_TYPE_DIRECT"]
},
'binding:profile': {},
'binding:vif_details': {},
'binding:vif_type': 'hw_veb',
'binding:vnic_type': 'direct',
'port_security_enabled': False,
}
port_macvtap_with_resource_request = {
'id': 'cbb9707f-3559-4675-a973-4ea89c747f02',
'name': '',
'description': '',
'network_id': NeutronFixture.network_2['id'],
'admin_state_up': True,
'status': 'ACTIVE',
'mac_address': '52:54:00:1e:59:c6',
# Do neutron really adds fixed_ips to an direct vnic_type port?
'fixed_ips': [
{
'ip_address': '192.168.13.4',
'subnet_id': NeutronFixture.subnet_2['id']
}
],
'tenant_id': NeutronFixture.tenant_id,
'project_id': NeutronFixture.tenant_id,
'device_id': '',
'resource_request': {
"resources": {
orc.NET_BW_IGR_KILOBIT_PER_SEC: 10000,
orc.NET_BW_EGR_KILOBIT_PER_SEC: 10000},
"required": ["CUSTOM_PHYSNET2", "CUSTOM_VNIC_TYPE_MACVTAP"]
},
'binding:profile': {},
'binding:vif_details': {},
'binding:vif_type': 'hw_veb',
'binding:vnic_type': 'macvtap',
'port_security_enabled': False,
}
def __init__(self, test):
super().__init__(test)
# add extra ports and the related network to the neutron fixture
# specifically for resource_request tests. It cannot be added globally
# in the base fixture init as it adds a second network that makes auto
# allocation based test to fail due to ambiguous networks.
self._ports[
self.port_with_sriov_resource_request['id']] = \
copy.deepcopy(self.port_with_sriov_resource_request)
self._ports[self.sriov_port['id']] = \
copy.deepcopy(self.sriov_port)
self._networks[
self.network_2['id']] = self.network_2
self._subnets[
self.subnet_2['id']] = self.subnet_2
macvtap = self.port_macvtap_with_resource_request
self._ports[macvtap['id']] = copy.deepcopy(macvtap)
class ExtendedResourceRequestNeutronFixture(ResourceRequestNeutronFixture):
def list_extensions(self, *args, **kwargs):
extensions = super().list_extensions(*args, **kwargs)
extensions['extensions'].append(
# As defined in neutron_lib/api/definitions/
# port_resource_request_groups.py
{
"updated": "2021-08-02T10:00:00-00:00",
"name": constants.RESOURCE_REQUEST_GROUPS_EXTENSION,
"links": [],
"alias": "port-resource-request-groups",
"description":
"Support requesting multiple groups of resources and "
"traits from the same RP subtree in resource_request"
}
)
return extensions
def _translate_port_to_new_resource_request(self, port):
"""Translates the old resource request definition to the new format in
place.
"""
# NOTE(gibi): Neutron sends the new format if
# port-resource-request-groups API extension is enabled.
# TODO(gibi): make this the default definition format after nova
# not need to support the old format any more which will happen after
# Neutron does not support the old format any more.
# old format:
#
# 'resource_request': {
# "resources": {
# orc.NET_BW_IGR_KILOBIT_PER_SEC: 1000,
# orc.NET_BW_EGR_KILOBIT_PER_SEC: 1000},
# "required": ["CUSTOM_PHYSNET2", "CUSTOM_VNIC_TYPE_NORMAL"]
# },
#
# new format:
#
# 'resource_request': {
# "request_groups":
# [
# {
# "id": "group1",
# "required": [<CUSTOM_VNIC_TYPE traits>],
# "resources":
# {
# NET_KILOPACKET_PER_SEC:
# <amount requested via the QoS policy>
# }
# },
# {
# "id": "group2",
# "required": [<CUSTOM_PHYSNET_ traits>,
# <CUSTOM_VNIC_TYPE traits>],
# "resources":
# {
# <NET_BW_[E|I]GR_KILOBIT_PER_SEC resource class name>:
# <requested bandwidth amount from the QoS policy>
# }
# },
# ],
# "same_subtree": ["group1", "group2"]
# }
groups = []
same_subtree = []
# NOTE(gibi): in case of the old format Neutron sends None in the
# resource_request if the port has no QoS policy implicating
# resource request.
old_rr = port.get('resource_request')
# NOTE(gibi): In the new format Neutron also sends None if the port
# has no QoS policy implicating resource request
new_rr = None
if old_rr:
# use the port id as group id as we know that in the old format
# we can have only one group per port
old_rr['id'] = port['id']
# nest the old request as one of the groups in the new format
groups.append(old_rr)
# Neutron might generate an empty list if only one group is
# requested, but it is equally correct to list that single group
# as well. We do the later as that allows some testing already with
# a single group
same_subtree = [old_rr['id']]
new_rr = {
"request_groups": groups,
"same_subtree": same_subtree
}
port['resource_request'] = new_rr
def show_port(self, port_id, **_params):
port_dict = super().show_port(port_id, **_params)
# this is an in place transformation but it is OK as the base class
# returns a deep copy of the port
self._translate_port_to_new_resource_request(port_dict['port'])
return port_dict
def list_ports(self, is_admin, retrieve_all=True, **_params):
ports_dict = super().list_ports(is_admin, retrieve_all=True, **_params)
for port in ports_dict['ports']:
# this is an in place transformation but it is OK as the base class
# returns a deep copy of the port
self._translate_port_to_new_resource_request(port)
return ports_dict
class PortResourceRequestBasedSchedulingTestBase(
integrated_helpers.ProviderUsageBaseTestCase):
compute_driver = 'fake.FakeDriverWithPciResources'
CUSTOM_VNIC_TYPE_NORMAL = 'CUSTOM_VNIC_TYPE_NORMAL'
CUSTOM_VNIC_TYPE_DIRECT = 'CUSTOM_VNIC_TYPE_DIRECT'
CUSTOM_VNIC_TYPE_MACVTAP = 'CUSTOM_VNIC_TYPE_MACVTAP'
CUSTOM_PHYSNET1 = 'CUSTOM_PHYSNET1'
CUSTOM_PHYSNET2 = 'CUSTOM_PHYSNET2'
CUSTOM_PHYSNET3 = 'CUSTOM_PHYSNET3'
PF1 = 'pf1'
PF2 = 'pf2'
PF3 = 'pf3'
def setUp(self):
# enable PciPassthroughFilter to support SRIOV before the base class
# starts the scheduler
if 'PciPassthroughFilter' not in CONF.filter_scheduler.enabled_filters:
self.flags(
enabled_filters=CONF.filter_scheduler.enabled_filters +
['PciPassthroughFilter'],
group='filter_scheduler')
self.useFixture(
fake.FakeDriverWithPciResources.
FakeDriverWithPciResourcesConfigFixture())
super(PortResourceRequestBasedSchedulingTestBase, self).setUp()
# override the default neutron fixture by mocking over it
self.neutron = self.useFixture(
ResourceRequestNeutronFixture(self))
# Make ComputeManager._allocate_network_async synchronous to detect
# errors in tests that involve rescheduling.
self.useFixture(nova_fixtures.SpawnIsSynchronousFixture())
self.compute1 = self._start_compute('host1')
self.compute1_rp_uuid = self._get_provider_uuid_by_host('host1')
self.compute1_service_id = self.admin_api.get_services(
host='host1', binary='nova-compute')[0]['id']
self.ovs_agent_rp_per_host = {}
self.ovs_bridge_rp_per_host = {}
self.sriov_dev_rp_per_host = {}
self.flavor = self.api.get_flavors()[0]
self.flavor_with_group_policy = self.api.get_flavors()[1]
# Setting group policy for placement. This is mandatory when more than
# one request group is included in the allocation candidate request and
# we have tests with two ports both having resource request modelled as
# two separate request groups.
self.admin_api.post_extra_spec(
self.flavor_with_group_policy['id'],
{'extra_specs': {'group_policy': 'isolate'}})
self._create_networking_rp_tree('host1', self.compute1_rp_uuid)
def assertComputeAllocationMatchesFlavor(
self, allocations, compute_rp_uuid, flavor):
compute_allocations = allocations[compute_rp_uuid]['resources']
self.assertEqual(
self._resources_from_flavor(flavor),
compute_allocations)
def _create_server(self, flavor, networks, host=None):
server_req = self._build_server(
image_uuid='76fa36fc-c930-4bf3-8c8a-ea2a2420deb6',
flavor_id=flavor['id'],
networks=networks,
host=host)
return self.api.post_server({'server': server_req})
def _set_provider_inventories(self, rp_uuid, inventories):
rp = self.placement.get(
'/resource_providers/%s' % rp_uuid).body
inventories['resource_provider_generation'] = rp['generation']
return self._update_inventory(rp_uuid, inventories)
def _create_ovs_networking_rp_tree(self, compute_rp_uuid):
# we need uuid sentinel for the test to make pep8 happy but we need a
# unique one per compute so here is some ugliness
ovs_agent_rp_uuid = getattr(uuids, compute_rp_uuid + 'ovs agent')
agent_rp_req = {
"name": ovs_agent_rp_uuid,
"uuid": ovs_agent_rp_uuid,
"parent_provider_uuid": compute_rp_uuid
}
self.placement.post(
'/resource_providers', body=agent_rp_req, version='1.20')
self.ovs_agent_rp_per_host[compute_rp_uuid] = ovs_agent_rp_uuid
ovs_bridge_rp_uuid = getattr(uuids, ovs_agent_rp_uuid + 'ovs br')
ovs_bridge_req = {
"name": ovs_bridge_rp_uuid,
"uuid": ovs_bridge_rp_uuid,
"parent_provider_uuid": ovs_agent_rp_uuid
}
self.placement.post(
'/resource_providers', body=ovs_bridge_req, version='1.20')
self.ovs_bridge_rp_per_host[compute_rp_uuid] = ovs_bridge_rp_uuid
self._set_provider_inventories(
ovs_agent_rp_uuid,
{"inventories": {
orc.NET_PACKET_RATE_KILOPACKET_PER_SEC: {"total": 10000},
}})
self._set_provider_inventories(
ovs_bridge_rp_uuid,
{"inventories": {
orc.NET_BW_IGR_KILOBIT_PER_SEC: {"total": 10000},
orc.NET_BW_EGR_KILOBIT_PER_SEC: {"total": 10000},
}})
self._create_trait(self.CUSTOM_VNIC_TYPE_NORMAL)
self._create_trait(self.CUSTOM_PHYSNET2)
self._set_provider_traits(
ovs_agent_rp_uuid, [self.CUSTOM_VNIC_TYPE_NORMAL])
self._set_provider_traits(
ovs_bridge_rp_uuid,
[self.CUSTOM_VNIC_TYPE_NORMAL, self.CUSTOM_PHYSNET2])
def _create_pf_device_rp(
self, device_rp_uuid, parent_rp_uuid, inventories, traits,
device_rp_name=None):
"""Create a RP in placement for a physical function network device with
traits and inventories.
"""
if not device_rp_name:
device_rp_name = device_rp_uuid
sriov_pf_req = {
"name": device_rp_name,
"uuid": device_rp_uuid,
"parent_provider_uuid": parent_rp_uuid
}
self.placement.post('/resource_providers',
body=sriov_pf_req,
version='1.20')
self._set_provider_inventories(
device_rp_uuid,
{"inventories": inventories})
for trait in traits:
self._create_trait(trait)
self._set_provider_traits(
device_rp_uuid,
traits)
def _create_sriov_networking_rp_tree(self, hostname, compute_rp_uuid):
# Create a matching RP tree in placement for the PCI devices added to
# the passthrough_whitelist config during setUp() and PCI devices
# present in the FakeDriverWithPciResources virt driver.
#
# * PF1 represents the PCI device 0000:01:00, it will be mapped to
# physnet1 and it will have bandwidth inventory.
# * PF2 represents the PCI device 0000:02:00, it will be mapped to
# physnet2 it will have bandwidth inventory.
# * PF3 represents the PCI device 0000:03:00 and, it will be mapped to
# physnet2 but it will not have bandwidth inventory.
self.sriov_dev_rp_per_host[compute_rp_uuid] = {}
sriov_agent_rp_uuid = getattr(uuids, compute_rp_uuid + 'sriov agent')
agent_rp_req = {
"name": "%s:NIC Switch agent" % hostname,
"uuid": sriov_agent_rp_uuid,
"parent_provider_uuid": compute_rp_uuid
}
self.placement.post('/resource_providers',
body=agent_rp_req,
version='1.20')
dev_rp_name_prefix = ("%s:NIC Switch agent:" % hostname)
sriov_pf1_rp_uuid = getattr(uuids, sriov_agent_rp_uuid + 'PF1')
self.sriov_dev_rp_per_host[
compute_rp_uuid][self.PF1] = sriov_pf1_rp_uuid
inventories = {
orc.NET_BW_IGR_KILOBIT_PER_SEC: {"total": 100000},
orc.NET_BW_EGR_KILOBIT_PER_SEC: {"total": 100000},
}
traits = [self.CUSTOM_VNIC_TYPE_DIRECT, self.CUSTOM_PHYSNET1]
self._create_pf_device_rp(
sriov_pf1_rp_uuid, sriov_agent_rp_uuid, inventories, traits,
device_rp_name=dev_rp_name_prefix + "%s-ens1" % hostname)
sriov_pf2_rp_uuid = getattr(uuids, sriov_agent_rp_uuid + 'PF2')
self.sriov_dev_rp_per_host[
compute_rp_uuid][self.PF2] = sriov_pf2_rp_uuid
inventories = {
orc.NET_BW_IGR_KILOBIT_PER_SEC: {"total": 100000},
orc.NET_BW_EGR_KILOBIT_PER_SEC: {"total": 100000},
}
traits = [self.CUSTOM_VNIC_TYPE_DIRECT, self.CUSTOM_VNIC_TYPE_MACVTAP,
self.CUSTOM_PHYSNET2]
self._create_pf_device_rp(
sriov_pf2_rp_uuid, sriov_agent_rp_uuid, inventories, traits,
device_rp_name=dev_rp_name_prefix + "%s-ens2" % hostname)
sriov_pf3_rp_uuid = getattr(uuids, sriov_agent_rp_uuid + 'PF3')
self.sriov_dev_rp_per_host[
compute_rp_uuid][self.PF3] = sriov_pf3_rp_uuid
inventories = {}
traits = [self.CUSTOM_VNIC_TYPE_DIRECT, self.CUSTOM_PHYSNET2]
self._create_pf_device_rp(
sriov_pf3_rp_uuid, sriov_agent_rp_uuid, inventories, traits,
device_rp_name=dev_rp_name_prefix + "%s-ens3" % hostname)
def _create_networking_rp_tree(self, hostname, compute_rp_uuid):
# let's simulate what the neutron would do
self._create_ovs_networking_rp_tree(compute_rp_uuid)
self._create_sriov_networking_rp_tree(hostname, compute_rp_uuid)
def assertPortMatchesAllocation(self, port, allocations):
port_request = port[constants.RESOURCE_REQUEST]['resources']
for rc, amount in allocations.items():
self.assertEqual(port_request[rc], amount,
'port %s requested %d %s '
'resources but got allocation %d' %
(port['id'], port_request[rc], rc,
amount))
def _create_server_with_ports(self, *ports):
server = self._create_server(
flavor=self.flavor_with_group_policy,
networks=[{'port': port['id']} for port in ports],
host='host1')
return self._wait_for_state_change(server, 'ACTIVE')
def _check_allocation(
self, server, compute_rp_uuid, non_qos_port, qos_port,
qos_sriov_port, flavor, migration_uuid=None,
source_compute_rp_uuid=None, new_flavor=None):
updated_non_qos_port = self.neutron.show_port(
non_qos_port['id'])['port']
updated_qos_port = self.neutron.show_port(qos_port['id'])['port']
updated_qos_sriov_port = self.neutron.show_port(
qos_sriov_port['id'])['port']
allocations = self.placement.get(
'/allocations/%s' % server['id']).body['allocations']
# if there is new_flavor then we either have an in progress resize or
# a confirmed resize. In both cases the instance allocation should be
# according to the new_flavor
current_flavor = (new_flavor if new_flavor else flavor)
# We expect one set of allocations for the compute resources on the
# compute rp and two sets for the networking resources one on the ovs
# bridge rp due to the qos_port resource request and one one the
# sriov pf2 due to qos_sriov_port resource request
self.assertEqual(3, len(allocations))
self.assertComputeAllocationMatchesFlavor(
allocations, compute_rp_uuid, current_flavor)
ovs_allocations = allocations[
self.ovs_bridge_rp_per_host[compute_rp_uuid]]['resources']
self.assertPortMatchesAllocation(qos_port, ovs_allocations)
sriov_allocations = allocations[
self.sriov_dev_rp_per_host[compute_rp_uuid][self.PF2]]['resources']
self.assertPortMatchesAllocation(qos_sriov_port, sriov_allocations)
# We expect that only the RP uuid of the networking RP having the port
# allocation is sent in the port binding for the port having resource
# request
qos_binding_profile = updated_qos_port['binding:profile']
self.assertEqual(self.ovs_bridge_rp_per_host[compute_rp_uuid],
qos_binding_profile['allocation'])
qos_sriov_binding_profile = updated_qos_sriov_port['binding:profile']
self.assertEqual(self.sriov_dev_rp_per_host[compute_rp_uuid][self.PF2],
qos_sriov_binding_profile['allocation'])
# And we expect not to have any allocation set in the port binding for
# the port that doesn't have resource request
self.assertEqual({}, updated_non_qos_port['binding:profile'])
if migration_uuid:
migration_allocations = self.placement.get(
'/allocations/%s' % migration_uuid).body['allocations']
# We expect one set of allocations for the compute resources on the
# compute rp and two sets for the networking resources one on the
# ovs bridge rp due to the qos_port resource request and one one
# the sriov pf2 due to qos_sriov_port resource request
self.assertEqual(3, len(migration_allocations))
self.assertComputeAllocationMatchesFlavor(
migration_allocations, source_compute_rp_uuid, flavor)
ovs_allocations = migration_allocations[
self.ovs_bridge_rp_per_host[
source_compute_rp_uuid]]['resources']
self.assertPortMatchesAllocation(qos_port, ovs_allocations)
sriov_allocations = migration_allocations[
self.sriov_dev_rp_per_host[
source_compute_rp_uuid][self.PF2]]['resources']
self.assertPortMatchesAllocation(qos_sriov_port, sriov_allocations)
def _delete_server_and_check_allocations(
self, server, qos_port, qos_sriov_port):
self._delete_and_check_allocations(server)
# assert that unbind removes the allocation from the binding of the
# ports that got allocation during the bind
updated_qos_port = self.neutron.show_port(qos_port['id'])['port']
binding_profile = updated_qos_port['binding:profile']
self.assertNotIn('allocation', binding_profile)
updated_qos_sriov_port = self.neutron.show_port(
qos_sriov_port['id'])['port']
binding_profile = updated_qos_sriov_port['binding:profile']
self.assertNotIn('allocation', binding_profile)
def _create_server_with_ports_and_check_allocation(
self, non_qos_normal_port, qos_normal_port, qos_sriov_port):
server = self._create_server_with_ports(
non_qos_normal_port, qos_normal_port, qos_sriov_port)
# check that the server allocates from the current host properly
self._check_allocation(
server, self.compute1_rp_uuid, non_qos_normal_port,
qos_normal_port, qos_sriov_port, self.flavor_with_group_policy)
return server
def _assert_pci_request_pf_device_name(self, server, device_name):
ctxt = context.get_admin_context()
pci_requests = objects.InstancePCIRequests.get_by_instance_uuid(
ctxt, server['id'])
self.assertEqual(1, len(pci_requests.requests))
self.assertEqual(1, len(pci_requests.requests[0].spec))
self.assertEqual(
device_name,
pci_requests.requests[0].spec[0]['parent_ifname'])
class UnsupportedPortResourceRequestBasedSchedulingTest(
PortResourceRequestBasedSchedulingTestBase):
"""Tests for handling servers with ports having resource requests """
def _add_resource_request_to_a_bound_port(self, port_id):
# NOTE(gibi): self.neutron._ports contains a copy of each neutron port
# defined on class level in the fixture. So modifying what is in the
# _ports list is safe as it is re-created for each Neutron fixture
# instance therefore for each individual test using that fixture.
bound_port = self.neutron._ports[port_id]
bound_port[constants.RESOURCE_REQUEST] = (
self.neutron.port_with_resource_request[
constants.RESOURCE_REQUEST])
def test_interface_attach_with_resource_request_old_compute(self):
# create a server
server = self._create_server(
flavor=self.flavor,
networks=[{'port': self.neutron.port_1['id']}])
self._wait_for_state_change(server, 'ACTIVE')
# simulate that the compute the instance is running on is older than
# when support is added for attach, older than service version 55
orig_get_service = objects.Service.get_by_host_and_binary
def fake_get_service(context, host, binary):
service = orig_get_service(context, host, binary)
service.version = 54
return service
with mock.patch(
'nova.objects.Service.get_by_host_and_binary',
side_effect=fake_get_service
):
# try to add a port with resource request
post = {
'interfaceAttachment': {
'port_id': self.neutron.port_with_resource_request['id']
}}
ex = self.assertRaises(
client.OpenStackApiException, self.api.attach_interface,
server['id'], post)
self.assertEqual(400, ex.response.status_code)
self.assertIn('Attaching interfaces with QoS policy is '
'not supported for instance',
str(ex))
@mock.patch('nova.tests.fixtures.NeutronFixture.create_port')
def test_interface_attach_with_network_create_port_has_resource_request(
self, mock_neutron_create_port):
# create a server
server = self._create_server(
flavor=self.flavor,
networks=[{'port': self.neutron.port_1['id']}])
self._wait_for_state_change(server, 'ACTIVE')
# the interfaceAttach operation below will result in a new port being
# created in the network that is attached. Make sure that neutron
# returns a port that has resource request.
mock_neutron_create_port.return_value = (
{'port': copy.deepcopy(self.neutron.port_with_resource_request)})
# try to attach a network
post = {
'interfaceAttachment': {
'net_id': self.neutron.network_1['id']
}}
ex = self.assertRaises(client.OpenStackApiException,
self.api.attach_interface,
server['id'], post)
self.assertEqual(400, ex.response.status_code)
self.assertIn('Using networks with QoS policy is not supported for '
'instance',
str(ex))
@mock.patch('nova.tests.fixtures.NeutronFixture.create_port')
def test_create_server_with_network_create_port_has_resource_request(
self, mock_neutron_create_port):
# the server create operation below will result in a new port being
# created in the network. Make sure that neutron returns a port that
# has resource request.
mock_neutron_create_port.return_value = (
{'port': copy.deepcopy(self.neutron.port_with_resource_request)})
server = self._create_server(
flavor=self.flavor,
networks=[{'uuid': self.neutron.network_1['id']}])
server = self._wait_for_state_change(server, 'ERROR')
self.assertEqual(500, server['fault']['code'])
self.assertIn('Failed to allocate the network',
server['fault']['message'])
def test_create_server_with_port_resource_request_old_microversion(self):
# NOTE(gibi): 2.71 is the last microversion where nova does not support
# this kind of create server
self.api.microversion = '2.71'
ex = self.assertRaises(
client.OpenStackApiException, self._create_server,
flavor=self.flavor,
networks=[{'port': self.neutron.port_with_resource_request['id']}])
self.assertEqual(400, ex.response.status_code)
self.assertIn(
"Creating servers with ports having resource requests, like a "
"port with a QoS minimum bandwidth policy, is not supported "
"until microversion 2.72.",
str(ex))
def test_unshelve_not_offloaded_server_with_port_resource_request(
self):
"""If the server is not offloaded then unshelving does not cause a new
resource allocation therefore having port resource request is
irrelevant. This test asserts that such unshelve request is not
rejected.
"""
server = self._create_server(
flavor=self.flavor,
networks=[{'port': self.neutron.port_1['id']}])
self._wait_for_state_change(server, 'ACTIVE')
# avoid automatic shelve offloading
self.flags(shelved_offload_time=-1)
req = {
'shelve': {}
}
self.api.post_server_action(server['id'], req)
self._wait_for_server_parameter(server, {'status': 'SHELVED'})
# We need to simulate that the above server has a port that has
# resource request; we cannot boot with such a port but legacy servers
# can exist with such a port.
self._add_resource_request_to_a_bound_port(self.neutron.port_1['id'])
self.api.post_server_action(server['id'], {'unshelve': None})
self._wait_for_state_change(server, 'ACTIVE')
class NonAdminUnsupportedPortResourceRequestBasedSchedulingTest(
UnsupportedPortResourceRequestBasedSchedulingTest):
def setUp(self):
super(
NonAdminUnsupportedPortResourceRequestBasedSchedulingTest,
self).setUp()
# switch to non admin api
self.api = self.api_fixture.api
self.api.microversion = self.microversion
# allow non-admin to call the operations
self.policy.set_rules({
'os_compute_api:servers:create': '@',
'os_compute_api:servers:create:attach_network': '@',
'os_compute_api:servers:show': '@',
'os_compute_api:os-attach-interfaces': '@',
'os_compute_api:os-attach-interfaces:create': '@',
'os_compute_api:os-shelve:shelve': '@',
'os_compute_api:os-shelve:unshelve': '@',
'os_compute_api:os-migrate-server:migrate_live': '@',
'os_compute_api:os-evacuate': '@',
})
class PortResourceRequestBasedSchedulingTest(
PortResourceRequestBasedSchedulingTestBase):
"""Tests creating a server with a pre-existing port that has a resource
request for a QoS minimum bandwidth policy.
"""
def test_boot_server_with_two_ports_one_having_resource_request(self):
non_qos_port = self.neutron.port_1
qos_port = self.neutron.port_with_resource_request
server = self._create_server(
flavor=self.flavor,
networks=[{'port': non_qos_port['id']},
{'port': qos_port['id']}])
server = self._wait_for_state_change(server, 'ACTIVE')
updated_non_qos_port = self.neutron.show_port(
non_qos_port['id'])['port']
updated_qos_port = self.neutron.show_port(qos_port['id'])['port']
allocations = self.placement.get(
'/allocations/%s' % server['id']).body['allocations']
# We expect one set of allocations for the compute resources on the
# compute rp and one set for the networking resources on the ovs bridge
# rp due to the qos_port resource request
self.assertEqual(2, len(allocations))
self.assertComputeAllocationMatchesFlavor(
allocations, self.compute1_rp_uuid, self.flavor)
network_allocations = allocations[
self.ovs_bridge_rp_per_host[self.compute1_rp_uuid]]['resources']
self.assertPortMatchesAllocation(qos_port, network_allocations)
# We expect that only the RP uuid of the networking RP having the port
# allocation is sent in the port binding for the port having resource
# request
qos_binding_profile = updated_qos_port['binding:profile']
self.assertEqual(self.ovs_bridge_rp_per_host[self.compute1_rp_uuid],
qos_binding_profile['allocation'])
# And we expect not to have any allocation set in the port binding for
# the port that doesn't have resource request
self.assertEqual({}, updated_non_qos_port['binding:profile'])
self._delete_and_check_allocations(server)
# assert that unbind removes the allocation from the binding of the
# port that got allocation during the bind
updated_qos_port = self.neutron.show_port(qos_port['id'])['port']
binding_profile = updated_qos_port['binding:profile']
self.assertNotIn('allocation', binding_profile)
def test_one_ovs_one_sriov_port(self):
ovs_port = self.neutron.port_with_resource_request
sriov_port = self.neutron.port_with_sriov_resource_request
server = self._create_server(flavor=self.flavor_with_group_policy,
networks=[{'port': ovs_port['id']},
{'port': sriov_port['id']}])
server = self._wait_for_state_change(server, 'ACTIVE')
ovs_port = self.neutron.show_port(ovs_port['id'])['port']
sriov_port = self.neutron.show_port(sriov_port['id'])['port']
allocations = self.placement.get(
'/allocations/%s' % server['id']).body['allocations']
# We expect one set of allocations for the compute resources on the
# compute rp and one set for the networking resources on the ovs bridge
# rp and on the sriov PF rp.
self.assertEqual(3, len(allocations))
self.assertComputeAllocationMatchesFlavor(
allocations, self.compute1_rp_uuid, self.flavor_with_group_policy)
ovs_allocations = allocations[
self.ovs_bridge_rp_per_host[self.compute1_rp_uuid]]['resources']
sriov_allocations = allocations[
self.sriov_dev_rp_per_host[
self.compute1_rp_uuid][self.PF2]]['resources']
self.assertPortMatchesAllocation(ovs_port, ovs_allocations)
self.assertPortMatchesAllocation(sriov_port, sriov_allocations)
# We expect that only the RP uuid of the networking RP having the port
# allocation is sent in the port binding for the port having resource
# request
ovs_binding = ovs_port['binding:profile']
self.assertEqual(self.ovs_bridge_rp_per_host[self.compute1_rp_uuid],
ovs_binding['allocation'])
sriov_binding = sriov_port['binding:profile']
self.assertEqual(
self.sriov_dev_rp_per_host[self.compute1_rp_uuid][self.PF2],
sriov_binding['allocation'])
def test_interface_attach_with_resource_request(self):
server = self._create_server(
flavor=self.flavor,
networks=[{'port': self.neutron.port_1['id']}])
self._wait_for_state_change(server, 'ACTIVE')
# start a second compute to show that resources are only allocated from
# the compute the instance currently runs on
self.compute2 = self._start_compute('host2')
self.compute2_rp_uuid = self._get_provider_uuid_by_host('host2')
self._create_networking_rp_tree('host2', self.compute2_rp_uuid)
self.compute2_service_id = self.admin_api.get_services(
host='host2', binary='nova-compute')[0]['id']
# attach an OVS port with resource request
ovs_port = self.neutron.port_with_resource_request
post = {
'interfaceAttachment': {
'port_id': ovs_port['id']
}}
self.api.attach_interface(server['id'], post)
ovs_port = self.neutron.show_port(ovs_port['id'])['port']
allocations = self.placement.get(
'/allocations/%s' % server['id']).body['allocations']
# We expect one set of allocations for the compute resources on the
# compute RP and one set for the networking resources on the OVS
# bridge RP.
self.assertEqual(2, len(allocations))
self.assertComputeAllocationMatchesFlavor(
allocations, self.compute1_rp_uuid, self.flavor)
ovs_allocations = allocations[
self.ovs_bridge_rp_per_host[self.compute1_rp_uuid]]['resources']
self.assertPortMatchesAllocation(ovs_port, ovs_allocations)
# We expect that only the RP uuid of the networking RP having the port
# allocation is sent in the port binding for the port having resource
# request
ovs_binding = ovs_port['binding:profile']
self.assertEqual(self.ovs_bridge_rp_per_host[self.compute1_rp_uuid],
ovs_binding['allocation'])
# now attach an SRIOV port
sriov_port = self.neutron.port_with_sriov_resource_request
post = {
'interfaceAttachment': {
'port_id': sriov_port['id']
}}
self.api.attach_interface(server['id'], post)
ovs_port = self.neutron.show_port(ovs_port['id'])['port']
sriov_port = self.neutron.show_port(sriov_port['id'])['port']
allocations = self.placement.get(
'/allocations/%s' % server['id']).body['allocations']
# We expect one set of allocations for the compute resources on the
# compute RP and one set each for the networking resources on the OVS
# bridge RP and on the SRIOV PF RP.
self.assertEqual(3, len(allocations))
self.assertComputeAllocationMatchesFlavor(
allocations, self.compute1_rp_uuid, self.flavor)
ovs_allocations = allocations[
self.ovs_bridge_rp_per_host[self.compute1_rp_uuid]]['resources']
sriov_allocations = allocations[
self.sriov_dev_rp_per_host[
self.compute1_rp_uuid][self.PF2]]['resources']
self.assertPortMatchesAllocation(ovs_port, ovs_allocations)
self.assertPortMatchesAllocation(sriov_port, sriov_allocations)
# We expect that only the RP uuid of the networking RP having the port
# allocation is sent in the port binding for the port having resource
# request
ovs_binding = ovs_port['binding:profile']
self.assertEqual(self.ovs_bridge_rp_per_host[self.compute1_rp_uuid],
ovs_binding['allocation'])
sriov_binding = sriov_port['binding:profile']
self.assertEqual(
self.sriov_dev_rp_per_host[self.compute1_rp_uuid][self.PF2],
sriov_binding['allocation'])
def test_interface_attach_with_resource_request_no_candidates(self):
server = self._create_server(
flavor=self.flavor,
networks=[{'port': self.neutron.port_1['id']}])
self._wait_for_state_change(server, 'ACTIVE')
# attach an OVS port with too big resource request
ovs_port = self.neutron.port_with_resource_request
resources = self.neutron._ports[
ovs_port['id']]['resource_request']['resources']
resources['NET_BW_IGR_KILOBIT_PER_SEC'] = 1000000
post = {
'interfaceAttachment': {
'port_id': ovs_port['id']
}}
ex = self.assertRaises(
client.OpenStackApiException, self.api.attach_interface,
server['id'], post)
self.assertEqual(400, ex.response.status_code)
self.assertIn('Failed to allocate additional resources', str(ex))
def test_interface_attach_with_resource_request_pci_claim_fails(self):
# boot a server with a single SRIOV port that has no resource request
sriov_port = self.neutron.sriov_port
server = self._create_server(
flavor=self.flavor,
networks=[{'port': sriov_port['id']}])
self._wait_for_state_change(server, 'ACTIVE')
sriov_port = self.neutron.show_port(sriov_port['id'])['port']
sriov_binding = sriov_port['binding:profile']
# We expect that this consume the last available VF from the PF2
self.assertEqual(
fake.FakeDriverWithPciResources.PCI_ADDR_PF2_VF1,
sriov_binding['pci_slot'])
# Now attach a second port to this server that has resource request
# At this point PF2 has available bandwidth but no available VF
# and PF3 has available VF but no available bandwidth so we expect
# the attach to fail.
sriov_port_with_res_req = self.neutron.port_with_sriov_resource_request
post = {
'interfaceAttachment': {
'port_id': sriov_port_with_res_req['id']
}}
ex = self.assertRaises(
client.OpenStackApiException, self.api.attach_interface,
server['id'], post)
self.assertEqual(400, ex.response.status_code)
self.assertIn('Failed to claim PCI device', str(ex))
sriov_port_with_res_req = self.neutron.show_port(
sriov_port_with_res_req['id'])['port']
allocations = self.placement.get(
'/allocations/%s' % server['id']).body['allocations']
# We expect only one allocations that is on the compute RP as the
# allocation made towards the PF2 RP has been rolled back when the PCI
# claim failed
self.assertEqual([self.compute1_rp_uuid], list(allocations))
self.assertComputeAllocationMatchesFlavor(
allocations, self.compute1_rp_uuid, self.flavor)
# We expect that the port binding is not updated with any RP uuid as
# the attach failed.
sriov_binding = sriov_port_with_res_req['binding:profile']
self.assertNotIn('allocation', sriov_binding)
def test_interface_attach_sriov_with_qos_pci_update_fails(self):
# Update the name of the network device RP of PF2 on host2 to something
# unexpected. This will cause
# update_pci_request_spec_with_allocated_interface_name() to raise
# when the sriov interface is attached.
rsp = self.placement.put(
'/resource_providers/%s'
% self.sriov_dev_rp_per_host[self.compute1_rp_uuid][self.PF2],
{"name": "invalid-device-rp-name"})
self.assertEqual(200, rsp.status)
server = self._create_server(
flavor=self.flavor,
networks=[{'port': self.neutron.port_1['id']}])
self._wait_for_state_change(server, 'ACTIVE')
sriov_port = self.neutron.port_with_sriov_resource_request
post = {
'interfaceAttachment': {
'port_id': sriov_port['id']
}}
ex = self.assertRaises(
client.OpenStackApiException, self.api.attach_interface,
server['id'], post)
self.assertEqual(500, ex.response.status_code)
self.assertIn('UnexpectedResourceProviderNameForPCIRequest', str(ex))
sriov_port = self.neutron.show_port(sriov_port['id'])['port']
allocations = self.placement.get(
'/allocations/%s' % server['id']).body['allocations']
# We expect only one allocations that is on the compute RP as the
# allocation made towards the PF2 RP has been rolled back when the PCI
# update failed
self.assertEqual([self.compute1_rp_uuid], list(allocations))
self.assertComputeAllocationMatchesFlavor(
allocations, self.compute1_rp_uuid, self.flavor)
# We expect that the port binding is not updated with any RP uuid as
# the attach failed.
sriov_binding = sriov_port['binding:profile']
self.assertNotIn('allocation', sriov_binding)
def test_interface_attach_sriov_with_qos_pci_update_fails_cleanup_fails(
self
):
# Update the name of the network device RP of PF2 on host2 to something
# unexpected. This will cause
# update_pci_request_spec_with_allocated_interface_name() to raise
# when the sriov interface is attached.
rsp = self.placement.put(
'/resource_providers/%s'
% self.sriov_dev_rp_per_host[self.compute1_rp_uuid][self.PF2],
{"name": "invalid-device-rp-name"})
self.assertEqual(200, rsp.status)
server = self._create_server(
flavor=self.flavor,
networks=[{'port': self.neutron.port_1['id']}])
self._wait_for_state_change(server, 'ACTIVE')
sriov_port = self.neutron.port_with_sriov_resource_request
post = {
'interfaceAttachment': {
'port_id': sriov_port['id']
}}
orig_put = adapter.Adapter.put
conflict_rsp = fake_requests.FakeResponse(
409,
jsonutils.dumps(
{'errors': [
{'code': 'placement.concurrent_update',
'detail': 'consumer generation conflict'}]}))
self.adapter_put_call_count = 0
def fake_put(_self, url, **kwargs):
self.adapter_put_call_count += 1
if self.adapter_put_call_count == 1:
# allocation update to add the port resource request
return orig_put(_self, url, **kwargs)
else:
# cleanup calls to remove the port resource allocation
return conflict_rsp
# this mock makes sure that the placement cleanup will fail with
# conflict
with mock.patch('keystoneauth1.adapter.Adapter.put', new=fake_put):
ex = self.assertRaises(
client.OpenStackApiException, self.api.attach_interface,
server['id'], post)
self.assertEqual(500, ex.response.status_code)
self.assertIn('AllocationUpdateFailed', str(ex))
# we have a proper log about the leak
PF_rp_uuid = self.sriov_dev_rp_per_host[
self.compute1_rp_uuid][self.PF2]
self.assertIn(
"nova.exception.AllocationUpdateFailed: Failed to update "
"allocations for consumer %s. Error: Cannot remove "
"resources {'%s': "
"{'resources': {'NET_BW_EGR_KILOBIT_PER_SEC': 10000, "
"'NET_BW_IGR_KILOBIT_PER_SEC': 10000}}} from the allocation "
"due to multiple successive generation conflicts in "
"placement." % (server['id'], PF_rp_uuid),
self.stdlog.logger.output)
# assert that we retried the cleanup multiple times
self.assertEqual(5, self.adapter_put_call_count)
sriov_port = self.neutron.show_port(sriov_port['id'])['port']
allocations = self.placement.get(
'/allocations/%s' % server['id']).body['allocations']
# As the cleanup failed we leaked allocation in placement
self.assertEqual(2, len(allocations))
self.assertComputeAllocationMatchesFlavor(
allocations, self.compute1_rp_uuid, self.flavor)
sriov_dev_rp = self.sriov_dev_rp_per_host[
self.compute1_rp_uuid][self.PF2]
sriov_allocations = allocations[sriov_dev_rp]['resources']
# this is the leaked allocation in placement
self.assertPortMatchesAllocation(sriov_port, sriov_allocations)
allocations[sriov_dev_rp].pop('generation')
leaked_allocation = {sriov_dev_rp: allocations[sriov_dev_rp]}
self.assertIn(
f'Failed to update allocations for consumer {server["id"]}. '
f'Error: Cannot remove resources {leaked_allocation} from the '
f'allocation due to multiple successive generation conflicts in '
f'placement. To clean up the leaked resource allocation you can '
f'use nova-manage placement audit.',
self.stdlog.logger.output)
# We expect that the port binding is not updated with any RP uuid as
# the attach failed.
sriov_binding = sriov_port['binding:profile']
self.assertNotIn('allocation', sriov_binding)
def test_interface_detach_with_port_with_bandwidth_request(self):
port = self.neutron.port_with_resource_request
# create a server
server = self._create_server(
flavor=self.flavor,
networks=[{'port': port['id']}])
self._wait_for_state_change(server, 'ACTIVE')
allocations = self.placement.get(
'/allocations/%s' % server['id']).body['allocations']
# We expect one set of allocations for the compute resources on the
# compute rp and one set for the networking resources on the ovs bridge
# rp due to the port resource request
self.assertEqual(2, len(allocations))
self.assertComputeAllocationMatchesFlavor(
allocations, self.compute1_rp_uuid, self.flavor)
network_allocations = allocations[
self.ovs_bridge_rp_per_host[self.compute1_rp_uuid]]['resources']
self.assertPortMatchesAllocation(port, network_allocations)
# We expect that only the RP uuid of the networking RP having the port
# allocation is sent in the port binding for the port having resource
# request
updated_port = self.neutron.show_port(port['id'])['port']
binding_profile = updated_port['binding:profile']
self.assertEqual(self.ovs_bridge_rp_per_host[self.compute1_rp_uuid],
binding_profile['allocation'])
self.api.detach_interface(
server['id'], self.neutron.port_with_resource_request['id'])
self.notifier.wait_for_versioned_notifications(
'instance.interface_detach.end')
updated_port = self.neutron.show_port(
self.neutron.port_with_resource_request['id'])['port']
allocations = self.placement.get(
'/allocations/%s' % server['id']).body['allocations']
# We expect that the port related resource allocations are removed
self.assertEqual(1, len(allocations))
self.assertComputeAllocationMatchesFlavor(
allocations, self.compute1_rp_uuid, self.flavor)
# We expect that the allocation is removed from the port too
binding_profile = updated_port['binding:profile']
self.assertNotIn('allocation', binding_profile)
def test_delete_bound_port_in_neutron_with_resource_request(self):
"""Neutron sends a network-vif-deleted os-server-external-events
notification to nova when a bound port is deleted. Nova detaches the
vif from the server. If the port had a resource allocation then that
allocation is leaked. This test makes sure that 1) an ERROR is logged
when the leak happens. 2) the leaked resource is reclaimed when the
server is deleted.
"""
port = self.neutron.port_with_resource_request
# create a server
server = self._create_server(
flavor=self.flavor,
networks=[{'port': port['id']}])
server = self._wait_for_state_change(server, 'ACTIVE')
allocations = self.placement.get(
'/allocations/%s' % server['id']).body['allocations']
# We expect one set of allocations for the compute resources on the
# compute rp and one set for the networking resources on the ovs bridge
# rp due to the port resource request
self.assertEqual(2, len(allocations))
compute_allocations = allocations[self.compute1_rp_uuid]['resources']
network_allocations = allocations[
self.ovs_bridge_rp_per_host[self.compute1_rp_uuid]]['resources']
self.assertEqual(self._resources_from_flavor(self.flavor),
compute_allocations)
self.assertPortMatchesAllocation(port, network_allocations)
# We expect that only the RP uuid of the networking RP having the port
# allocation is sent in the port binding for the port having resource
# request
updated_port = self.neutron.show_port(port['id'])['port']
binding_profile = updated_port['binding:profile']
self.assertEqual(self.ovs_bridge_rp_per_host[self.compute1_rp_uuid],
binding_profile['allocation'])
# neutron is faked in the functional test so this test just sends in
# a os-server-external-events notification to trigger the
# detach + ERROR log.
events = {
"events": [
{
"name": "network-vif-deleted",
"server_uuid": server['id'],
"tag": port['id'],
}
]
}
response = self.api.api_post('/os-server-external-events', events).body
self.assertEqual(200, response['events'][0]['code'])
port_rp_uuid = self.ovs_bridge_rp_per_host[self.compute1_rp_uuid]
# 1) Nova logs an ERROR about the leak
self._wait_for_log(
'ERROR [nova.compute.manager] The bound port %(port_id)s is '
'deleted in Neutron but the resource allocation on the resource '
'provider %(rp_uuid)s is leaked until the server %(server_uuid)s '
'is deleted.'
% {'port_id': port['id'],
'rp_uuid': port_rp_uuid,
'server_uuid': server['id']})
allocations = self.placement.get(
'/allocations/%s' % server['id']).body['allocations']
# Nova leaks the port allocation so the server still has the same
# allocation before the port delete.
self.assertEqual(2, len(allocations))
compute_allocations = allocations[self.compute1_rp_uuid]['resources']
network_allocations = allocations[port_rp_uuid]['resources']
self.assertEqual(self._resources_from_flavor(self.flavor),
compute_allocations)
self.assertPortMatchesAllocation(port, network_allocations)
# 2) Also nova will reclaim the leaked resource during the server
# delete
self._delete_and_check_allocations(server)
def test_two_sriov_ports_one_with_request_two_available_pfs(self):
"""Verify that the port's bandwidth allocated from the same PF as
the allocated VF.
One compute host:
* PF1 (0000:01:00) is configured for physnet1
* PF2 (0000:02:00) is configured for physnet2, with 1 VF and bandwidth
inventory
* PF3 (0000:03:00) is configured for physnet2, with 1 VF but without
bandwidth inventory
One instance will be booted with two neutron ports, both ports
requested to be connected to physnet2. One port has resource request
the other does not have resource request. The port having the resource
request cannot be allocated to PF3 and PF1 while the other port that
does not have resource request can be allocated to PF2 or PF3.
For the detailed compute host config see the FakeDriverWithPciResources
class. For the necessary passthrough_whitelist config see the setUp of
the PortResourceRequestBasedSchedulingTestBase class.
"""
sriov_port = self.neutron.sriov_port
sriov_port_with_res_req = self.neutron.port_with_sriov_resource_request
server = self._create_server(
flavor=self.flavor_with_group_policy,
networks=[
{'port': sriov_port_with_res_req['id']},
{'port': sriov_port['id']}])
server = self._wait_for_state_change(server, 'ACTIVE')
sriov_port = self.neutron.show_port(sriov_port['id'])['port']
sriov_port_with_res_req = self.neutron.show_port(
sriov_port_with_res_req['id'])['port']
allocations = self.placement.get(
'/allocations/%s' % server['id']).body['allocations']
# We expect one set of allocations for the compute resources on the
# compute rp and one set for the networking resources on the sriov PF2
# rp.
self.assertEqual(2, len(allocations))
self.assertComputeAllocationMatchesFlavor(
allocations, self.compute1_rp_uuid, self.flavor_with_group_policy)
sriov_allocations = allocations[
self.sriov_dev_rp_per_host[
self.compute1_rp_uuid][self.PF2]]['resources']
self.assertPortMatchesAllocation(
sriov_port_with_res_req, sriov_allocations)
# We expect that only the RP uuid of the networking RP having the port
# allocation is sent in the port binding for the port having resource
# request
sriov_with_req_binding = sriov_port_with_res_req['binding:profile']
self.assertEqual(
self.sriov_dev_rp_per_host[self.compute1_rp_uuid][self.PF2],
sriov_with_req_binding['allocation'])
# and the port without resource request does not have allocation
sriov_binding = sriov_port['binding:profile']
self.assertNotIn('allocation', sriov_binding)
# We expect that the selected PCI device matches with the RP from
# where the bandwidth is allocated from. The bandwidth is allocated
# from 0000:02:00 (PF2) so the PCI device should be a VF of that PF
self.assertEqual(
fake.FakeDriverWithPciResources.PCI_ADDR_PF2_VF1,
sriov_with_req_binding['pci_slot'])
# But also the port that has no resource request still gets a pci slot
# allocated. The 0000:02:00 has no more VF available but 0000:03:00 has
# one VF available and that PF is also on physnet2
self.assertEqual(
fake.FakeDriverWithPciResources.PCI_ADDR_PF3_VF1,
sriov_binding['pci_slot'])
def test_one_sriov_port_no_vf_and_bandwidth_available_on_the_same_pf(self):
"""Verify that if there is no PF that both provides bandwidth and VFs
then the boot will fail.
"""
# boot a server with a single sriov port that has no resource request
sriov_port = self.neutron.sriov_port
server = self._create_server(
flavor=self.flavor_with_group_policy,
networks=[{'port': sriov_port['id']}])
self._wait_for_state_change(server, 'ACTIVE')
sriov_port = self.neutron.show_port(sriov_port['id'])['port']
sriov_binding = sriov_port['binding:profile']
# We expect that this consume the last available VF from the PF2
self.assertEqual(
fake.FakeDriverWithPciResources.PCI_ADDR_PF2_VF1,
sriov_binding['pci_slot'])
# Now boot a second server with a port that has resource request
# At this point PF2 has available bandwidth but no available VF
# and PF3 has available VF but no available bandwidth so we expect
# the boot to fail.
sriov_port_with_res_req = self.neutron.port_with_sriov_resource_request
server = self._create_server(
flavor=self.flavor_with_group_policy,
networks=[{'port': sriov_port_with_res_req['id']}])
# NOTE(gibi): It should be NoValidHost in an ideal world but that would
# require the scheduler to detect the situation instead of the pci
# claim. However that is pretty hard as the scheduler does not know
# anything about allocation candidates (e.g. that the only candidate
# for the port in this case is PF2) it see the whole host as a
# candidate and in our host there is available VF for the request even
# if that is on the wrong PF.
server = self._wait_for_state_change(server, 'ERROR')
self.assertIn(
'Exceeded maximum number of retries. Exhausted all hosts '
'available for retrying build failures for instance',
server['fault']['message'])
def test_sriov_macvtap_port_with_resource_request(self):
"""Verify that vnic type macvtap is also supported"""
port = self.neutron.port_macvtap_with_resource_request
server = self._create_server(
flavor=self.flavor,
networks=[{'port': port['id']}])
server = self._wait_for_state_change(server, 'ACTIVE')
port = self.neutron.show_port(port['id'])['port']
allocations = self.placement.get(
'/allocations/%s' % server['id']).body['allocations']
# We expect one set of allocations for the compute resources on the
# compute rp and one set for the networking resources on the sriov PF2
# rp.
self.assertEqual(2, len(allocations))
self.assertComputeAllocationMatchesFlavor(
allocations, self.compute1_rp_uuid, self.flavor)
sriov_allocations = allocations[self.sriov_dev_rp_per_host[
self.compute1_rp_uuid][self.PF2]]['resources']
self.assertPortMatchesAllocation(
port, sriov_allocations)
# We expect that only the RP uuid of the networking RP having the port
# allocation is sent in the port binding for the port having resource
# request
port_binding = port['binding:profile']
self.assertEqual(
self.sriov_dev_rp_per_host[self.compute1_rp_uuid][self.PF2],
port_binding['allocation'])
# We expect that the selected PCI device matches with the RP from
# where the bandwidth is allocated from. The bandwidth is allocated
# from 0000:02:00 (PF2) so the PCI device should be a VF of that PF
self.assertEqual(
fake.FakeDriverWithPciResources.PCI_ADDR_PF2_VF1,
port_binding['pci_slot'])
class ServerMoveWithPortResourceRequestTest(
PortResourceRequestBasedSchedulingTestBase):
def setUp(self):
# Use our custom weigher defined above to make sure that we have
# a predictable host order in the alternate list returned by the
# scheduler for migration.
self.useFixture(nova_fixtures.HostNameWeigherFixture())
super(ServerMoveWithPortResourceRequestTest, self).setUp()
self.compute2 = self._start_compute('host2')
self.compute2_rp_uuid = self._get_provider_uuid_by_host('host2')
self._create_networking_rp_tree('host2', self.compute2_rp_uuid)
self.compute2_service_id = self.admin_api.get_services(
host='host2', binary='nova-compute')[0]['id']
# create a bigger flavor to use in resize test
self.flavor_with_group_policy_bigger = self.admin_api.post_flavor(
{'flavor': {
'ram': self.flavor_with_group_policy['ram'],
'vcpus': self.flavor_with_group_policy['vcpus'],
'name': self.flavor_with_group_policy['name'] + '+',
'disk': self.flavor_with_group_policy['disk'] + 1,
}})
self.admin_api.post_extra_spec(
self.flavor_with_group_policy_bigger['id'],
{'extra_specs': {'group_policy': 'isolate'}})
def _test_resize_or_migrate_server_with_qos_ports(self, new_flavor=None):
non_qos_normal_port = self.neutron.port_1
qos_normal_port = self.neutron.port_with_resource_request
qos_sriov_port = self.neutron.port_with_sriov_resource_request
server = self._create_server_with_ports_and_check_allocation(
non_qos_normal_port, qos_normal_port, qos_sriov_port)
if new_flavor:
self.api_fixture.api.post_server_action(
server['id'], {'resize': {"flavorRef": new_flavor['id']}})
else:
self.api.post_server_action(server['id'], {'migrate': None})
self._wait_for_state_change(server, 'VERIFY_RESIZE')
migration_uuid = self.get_migration_uuid_for_instance(server['id'])
# check that server allocates from the new host properly
self._check_allocation(
server, self.compute2_rp_uuid, non_qos_normal_port,
qos_normal_port, qos_sriov_port, self.flavor_with_group_policy,
migration_uuid, source_compute_rp_uuid=self.compute1_rp_uuid,
new_flavor=new_flavor)
self._assert_pci_request_pf_device_name(server, 'host2-ens2')
self._confirm_resize(server)
# check that allocation is still OK
self._check_allocation(
server, self.compute2_rp_uuid, non_qos_normal_port,
qos_normal_port, qos_sriov_port, self.flavor_with_group_policy,
new_flavor=new_flavor)
migration_allocations = self.placement.get(
'/allocations/%s' % migration_uuid).body['allocations']
self.assertEqual({}, migration_allocations)
self._delete_server_and_check_allocations(
server, qos_normal_port, qos_sriov_port)
def test_migrate_server_with_qos_ports(self):
self._test_resize_or_migrate_server_with_qos_ports()
def test_resize_server_with_qos_ports(self):
self._test_resize_or_migrate_server_with_qos_ports(
new_flavor=self.flavor_with_group_policy_bigger)
def _test_resize_or_migrate_revert_with_qos_ports(self, new_flavor=None):
non_qos_port = self.neutron.port_1
qos_port = self.neutron.port_with_resource_request
qos_sriov_port = self.neutron.port_with_sriov_resource_request
server = self._create_server_with_ports_and_check_allocation(
non_qos_port, qos_port, qos_sriov_port)
if new_flavor:
self.api_fixture.api.post_server_action(
server['id'], {'resize': {"flavorRef": new_flavor['id']}})
else:
self.api.post_server_action(server['id'], {'migrate': None})
self._wait_for_state_change(server, 'VERIFY_RESIZE')
migration_uuid = self.get_migration_uuid_for_instance(server['id'])
# check that server allocates from the new host properly
self._check_allocation(
server, self.compute2_rp_uuid, non_qos_port, qos_port,
qos_sriov_port, self.flavor_with_group_policy, migration_uuid,
source_compute_rp_uuid=self.compute1_rp_uuid,
new_flavor=new_flavor)
self.api.post_server_action(server['id'], {'revertResize': None})
self._wait_for_state_change(server, 'ACTIVE')
# check that allocation is moved back to the source host
self._check_allocation(
server, self.compute1_rp_uuid, non_qos_port, qos_port,
qos_sriov_port, self.flavor_with_group_policy)
# check that the target host allocation is cleaned up.
self.assertRequestMatchesUsage(
{'VCPU': 0, 'MEMORY_MB': 0, 'DISK_GB': 0,
'NET_BW_IGR_KILOBIT_PER_SEC': 0, 'NET_BW_EGR_KILOBIT_PER_SEC': 0},
self.compute2_rp_uuid)
migration_allocations = self.placement.get(
'/allocations/%s' % migration_uuid).body['allocations']
self.assertEqual({}, migration_allocations)
self._delete_server_and_check_allocations(
server, qos_port, qos_sriov_port)
def test_migrate_revert_with_qos_ports(self):
self._test_resize_or_migrate_revert_with_qos_ports()
def test_resize_revert_with_qos_ports(self):
self._test_resize_or_migrate_revert_with_qos_ports(
new_flavor=self.flavor_with_group_policy_bigger)
def _test_resize_or_migrate_server_with_qos_port_reschedule_success(
self, new_flavor=None):
self._start_compute('host3')
compute3_rp_uuid = self._get_provider_uuid_by_host('host3')
self._create_networking_rp_tree('host3', compute3_rp_uuid)
non_qos_port = self.neutron.port_1
qos_port = self.neutron.port_with_resource_request
qos_sriov_port = self.neutron.port_with_sriov_resource_request
server = self._create_server_with_ports_and_check_allocation(
non_qos_port, qos_port, qos_sriov_port)
# Yes this isn't great in a functional test, but it's simple.
original_prep_resize = compute_manager.ComputeManager._prep_resize
prep_resize_calls = []
def fake_prep_resize(_self, *args, **kwargs):
# Make the first prep_resize fail and the rest passing through
# the original _prep_resize call
if not prep_resize_calls:
prep_resize_calls.append(_self.host)
raise test.TestingException('Simulated prep_resize failure.')
prep_resize_calls.append(_self.host)
original_prep_resize(_self, *args, **kwargs)
# The patched compute manager will raise from _prep_resize on the
# first host of the migration. Then the migration
# is reschedule on the other host where it will succeed
with mock.patch.object(
compute_manager.ComputeManager, '_prep_resize',
new=fake_prep_resize):
if new_flavor:
self.api_fixture.api.post_server_action(
server['id'], {'resize': {"flavorRef": new_flavor['id']}})
else:
self.api.post_server_action(server['id'], {'migrate': None})
self._wait_for_state_change(server, 'VERIFY_RESIZE')
# ensure that resize is tried on two hosts, so we had a re-schedule
self.assertEqual(['host2', 'host3'], prep_resize_calls)
migration_uuid = self.get_migration_uuid_for_instance(server['id'])
# check that server allocates from the final host properly while
# the migration holds the allocation on the source host
self._check_allocation(
server, compute3_rp_uuid, non_qos_port, qos_port, qos_sriov_port,
self.flavor_with_group_policy, migration_uuid,
source_compute_rp_uuid=self.compute1_rp_uuid,
new_flavor=new_flavor)
self._assert_pci_request_pf_device_name(server, 'host3-ens2')
self._confirm_resize(server)
# check that allocation is still OK
self._check_allocation(
server, compute3_rp_uuid, non_qos_port, qos_port, qos_sriov_port,
self.flavor_with_group_policy, new_flavor=new_flavor)
migration_allocations = self.placement.get(
'/allocations/%s' % migration_uuid).body['allocations']
self.assertEqual({}, migration_allocations)
self._delete_server_and_check_allocations(
server, qos_port, qos_sriov_port)
def test_migrate_server_with_qos_port_reschedule_success(self):
self._test_resize_or_migrate_server_with_qos_port_reschedule_success()
def test_resize_server_with_qos_port_reschedule_success(self):
self._test_resize_or_migrate_server_with_qos_port_reschedule_success(
new_flavor=self.flavor_with_group_policy_bigger)
def _test_resize_or_migrate_server_with_qos_port_reschedule_failure(
self, new_flavor=None):
non_qos_port = self.neutron.port_1
qos_port = self.neutron.port_with_resource_request
qos_sriov_port = self.neutron.port_with_sriov_resource_request
server = self._create_server_with_ports_and_check_allocation(
non_qos_port, qos_port, qos_sriov_port)
# The patched compute manager on host2 will raise from _prep_resize.
# Then the migration is reschedule but there is no other host to
# choose from.
with mock.patch.object(
compute_manager.ComputeManager, '_prep_resize',
side_effect=test.TestingException(
'Simulated prep_resize failure.')):
if new_flavor:
self.api_fixture.api.post_server_action(
server['id'], {'resize': {"flavorRef": new_flavor['id']}})
else:
self.api.post_server_action(server['id'], {'migrate': None})
self._wait_for_server_parameter(server,
{'OS-EXT-SRV-ATTR:host': 'host1',
'status': 'ERROR'})
self._wait_for_migration_status(server, ['error'])
migration_uuid = self.get_migration_uuid_for_instance(server['id'])
# as the migration is failed we expect that the migration allocation
# is deleted
migration_allocations = self.placement.get(
'/allocations/%s' % migration_uuid).body['allocations']
self.assertEqual({}, migration_allocations)
# and the instance allocates from the source host
self._check_allocation(
server, self.compute1_rp_uuid, non_qos_port, qos_port,
qos_sriov_port, self.flavor_with_group_policy)
def test_migrate_server_with_qos_port_reschedule_failure(self):
self._test_resize_or_migrate_server_with_qos_port_reschedule_failure()
def test_resize_server_with_qos_port_reschedule_failure(self):
self._test_resize_or_migrate_server_with_qos_port_reschedule_failure(
new_flavor=self.flavor_with_group_policy_bigger)
def test_migrate_server_with_qos_port_pci_update_fail_not_reschedule(self):
# Update the name of the network device RP of PF2 on host2 to something
# unexpected. This will cause
# update_pci_request_spec_with_allocated_interface_name() to raise
# when the instance is migrated to the host2.
rsp = self.placement.put(
'/resource_providers/%s'
% self.sriov_dev_rp_per_host[self.compute2_rp_uuid][self.PF2],
{"name": "invalid-device-rp-name"})
self.assertEqual(200, rsp.status)
self._start_compute('host3')
compute3_rp_uuid = self._get_provider_uuid_by_host('host3')
self._create_networking_rp_tree('host3', compute3_rp_uuid)
non_qos_port = self.neutron.port_1
qos_port = self.neutron.port_with_resource_request
qos_sriov_port = self.neutron.port_with_sriov_resource_request
server = self._create_server_with_ports_and_check_allocation(
non_qos_port, qos_port, qos_sriov_port)
# The compute manager on host2 will raise from
# update_pci_request_spec_with_allocated_interface_name which will
# intentionally not trigger a re-schedule even if there is host3 as an
# alternate.
self.api.post_server_action(server['id'], {'migrate': None})
server = self._wait_for_server_parameter(server,
{'OS-EXT-SRV-ATTR:host': 'host1',
# Note that we have to wait for the task_state to be reverted
# to None since that happens after the fault is recorded.
'OS-EXT-STS:task_state': None,
'status': 'ERROR'})
self._wait_for_migration_status(server, ['error'])
self.assertIn(
'Build of instance %s aborted' % server['id'],
server['fault']['message'])
self._wait_for_action_fail_completion(
server, instance_actions.MIGRATE, 'compute_prep_resize')
self.notifier.wait_for_versioned_notifications(
'instance.resize_prep.end')
self.notifier.wait_for_versioned_notifications(
'compute.exception')
migration_uuid = self.get_migration_uuid_for_instance(server['id'])
# as the migration is failed we expect that the migration allocation
# is deleted
migration_allocations = self.placement.get(
'/allocations/%s' % migration_uuid).body['allocations']
self.assertEqual({}, migration_allocations)
# and the instance allocates from the source host
self._check_allocation(
server, self.compute1_rp_uuid, non_qos_port, qos_port,
qos_sriov_port, self.flavor_with_group_policy)
def _check_allocation_during_evacuate(
self, server, flavor, source_compute_rp_uuid, dest_compute_rp_uuid,
non_qos_port, qos_port, qos_sriov_port):
# evacuate is the only case when the same consumer has allocation from
# two different RP trees so we need special checks
updated_non_qos_port = self.neutron.show_port(
non_qos_port['id'])['port']
updated_qos_port = self.neutron.show_port(qos_port['id'])['port']
updated_qos_sriov_port = self.neutron.show_port(
qos_sriov_port['id'])['port']
allocations = self.placement.get(
'/allocations/%s' % server['id']).body['allocations']
# We expect two sets of allocations. One set for the source compute
# and one set for the dest compute. Each set we expect 3 allocations
# one for the compute RP according to the flavor, one for the ovs port
# and one for the SRIOV port.
self.assertEqual(6, len(allocations), allocations)
# 1. source compute allocation
compute_allocations = allocations[source_compute_rp_uuid]['resources']
self.assertEqual(
self._resources_from_flavor(flavor),
compute_allocations)
# 2. source ovs allocation
ovs_allocations = allocations[
self.ovs_bridge_rp_per_host[source_compute_rp_uuid]]['resources']
self.assertPortMatchesAllocation(qos_port, ovs_allocations)
# 3. source sriov allocation
sriov_allocations = allocations[
self.sriov_dev_rp_per_host[
source_compute_rp_uuid][self.PF2]]['resources']
self.assertPortMatchesAllocation(qos_sriov_port, sriov_allocations)
# 4. dest compute allocation
compute_allocations = allocations[dest_compute_rp_uuid]['resources']
self.assertEqual(
self._resources_from_flavor(flavor),
compute_allocations)
# 5. dest ovs allocation
ovs_allocations = allocations[
self.ovs_bridge_rp_per_host[dest_compute_rp_uuid]]['resources']
self.assertPortMatchesAllocation(qos_port, ovs_allocations)
# 6. dest SRIOV allocation
sriov_allocations = allocations[
self.sriov_dev_rp_per_host[
dest_compute_rp_uuid][self.PF2]]['resources']
self.assertPortMatchesAllocation(qos_sriov_port, sriov_allocations)
# the qos ports should have their binding pointing to the RPs in the
# dest compute RP tree
qos_binding_profile = updated_qos_port['binding:profile']
self.assertEqual(
self.ovs_bridge_rp_per_host[dest_compute_rp_uuid],
qos_binding_profile['allocation'])
qos_sriov_binding_profile = updated_qos_sriov_port['binding:profile']
self.assertEqual(
self.sriov_dev_rp_per_host[dest_compute_rp_uuid][self.PF2],
qos_sriov_binding_profile['allocation'])
# And we expect not to have any allocation set in the port binding for
# the port that doesn't have resource request
self.assertEqual({}, updated_non_qos_port['binding:profile'])
def _check_allocation_after_evacuation_source_recovered(
self, server, flavor, dest_compute_rp_uuid, non_qos_port,
qos_port, qos_sriov_port):
# check that source allocation is cleaned up and the dest allocation
# and the port bindings are not touched.
updated_non_qos_port = self.neutron.show_port(
non_qos_port['id'])['port']
updated_qos_port = self.neutron.show_port(qos_port['id'])['port']
updated_qos_sriov_port = self.neutron.show_port(
qos_sriov_port['id'])['port']
allocations = self.placement.get(
'/allocations/%s' % server['id']).body['allocations']
self.assertEqual(3, len(allocations), allocations)
# 1. dest compute allocation
compute_allocations = allocations[dest_compute_rp_uuid]['resources']
self.assertEqual(
self._resources_from_flavor(flavor),
compute_allocations)
# 2. dest ovs allocation
ovs_allocations = allocations[
self.ovs_bridge_rp_per_host[dest_compute_rp_uuid]]['resources']
self.assertPortMatchesAllocation(qos_port, ovs_allocations)
# 3. dest SRIOV allocation
sriov_allocations = allocations[
self.sriov_dev_rp_per_host[
dest_compute_rp_uuid][self.PF2]]['resources']
self.assertPortMatchesAllocation(qos_sriov_port, sriov_allocations)
# the qos ports should have their binding pointing to the RPs in the
# dest compute RP tree
qos_binding_profile = updated_qos_port['binding:profile']
self.assertEqual(
self.ovs_bridge_rp_per_host[dest_compute_rp_uuid],
qos_binding_profile['allocation'])
qos_sriov_binding_profile = updated_qos_sriov_port['binding:profile']
self.assertEqual(
self.sriov_dev_rp_per_host[dest_compute_rp_uuid][self.PF2],
qos_sriov_binding_profile['allocation'])
# And we expect not to have any allocation set in the port binding for
# the port that doesn't have resource request
self.assertEqual({}, updated_non_qos_port['binding:profile'])
def test_evacuate_with_qos_port(self, host=None):
non_qos_normal_port = self.neutron.port_1
qos_normal_port = self.neutron.port_with_resource_request
qos_sriov_port = self.neutron.port_with_sriov_resource_request
server = self._create_server_with_ports_and_check_allocation(
non_qos_normal_port, qos_normal_port, qos_sriov_port)
# force source compute down
self.compute1.stop()
self.admin_api.put_service(
self.compute1_service_id, {'forced_down': 'true'})
self._evacuate_server(
server, {'host': host} if host else {}, expected_host='host2')
self._check_allocation_during_evacuate(
server, self.flavor_with_group_policy, self.compute1_rp_uuid,
self.compute2_rp_uuid, non_qos_normal_port, qos_normal_port,
qos_sriov_port)
self._assert_pci_request_pf_device_name(server, 'host2-ens2')
# recover source compute
self.compute1 = self.restart_compute_service(self.compute1)
self.admin_api.put_service(
self.compute1_service_id, {'forced_down': 'false'})
# check that source allocation is cleaned up and the dest allocation
# and the port bindings are not touched.
self._check_allocation_after_evacuation_source_recovered(
server, self.flavor_with_group_policy, self.compute2_rp_uuid,
non_qos_normal_port, qos_normal_port, qos_sriov_port)
self._delete_server_and_check_allocations(
server, qos_normal_port, qos_sriov_port)
def test_evacuate_with_target_host_with_qos_port(self):
self.test_evacuate_with_qos_port(host='host2')
def test_evacuate_with_qos_port_fails_recover_source_compute(self):
non_qos_normal_port = self.neutron.port_1
qos_normal_port = self.neutron.port_with_resource_request
qos_sriov_port = self.neutron.port_with_sriov_resource_request
server = self._create_server_with_ports_and_check_allocation(
non_qos_normal_port, qos_normal_port, qos_sriov_port)
# force source compute down
self.compute1.stop()
self.admin_api.put_service(
self.compute1_service_id, {'forced_down': 'true'})
with mock.patch(
'nova.compute.resource_tracker.ResourceTracker.rebuild_claim',
side_effect=exception.ComputeResourcesUnavailable(
reason='test evacuate failure')):
# Evacuate does not have reschedule loop so evacuate expected to
# simply fail and the server remains on the source host
server = self._evacuate_server(
server, expected_host='host1', expected_task_state=None,
expected_migration_status='failed')
# As evacuation failed the resource allocation should be untouched
self._check_allocation(
server, self.compute1_rp_uuid, non_qos_normal_port,
qos_normal_port, qos_sriov_port, self.flavor_with_group_policy)
# recover source compute
self.compute1 = self.restart_compute_service(self.compute1)
self.admin_api.put_service(
self.compute1_service_id, {'forced_down': 'false'})
# check again that even after source host recovery the source
# allocation is intact
self._check_allocation(
server, self.compute1_rp_uuid, non_qos_normal_port,
qos_normal_port, qos_sriov_port, self.flavor_with_group_policy)
self._delete_server_and_check_allocations(
server, qos_normal_port, qos_sriov_port)
def test_evacuate_with_qos_port_pci_update_fail(self):
# Update the name of the network device RP of PF2 on host2 to something
# unexpected. This will cause
# update_pci_request_spec_with_allocated_interface_name() to raise
# when the instance is evacuated to the host2.
rsp = self.placement.put(
'/resource_providers/%s'
% self.sriov_dev_rp_per_host[self.compute2_rp_uuid][self.PF2],
{"name": "invalid-device-rp-name"})
self.assertEqual(200, rsp.status)
non_qos_port = self.neutron.port_1
qos_port = self.neutron.port_with_resource_request
qos_sriov_port = self.neutron.port_with_sriov_resource_request
server = self._create_server_with_ports_and_check_allocation(
non_qos_port, qos_port, qos_sriov_port)
# force source compute down
self.compute1.stop()
self.admin_api.put_service(
self.compute1_service_id, {'forced_down': 'true'})
# The compute manager on host2 will raise from
# update_pci_request_spec_with_allocated_interface_name
server = self._evacuate_server(
server, expected_host='host1', expected_state='ERROR',
expected_task_state=None, expected_migration_status='failed')
self.assertIn(
'does not have a properly formatted name',
server['fault']['message'])
self._wait_for_action_fail_completion(
server, instance_actions.EVACUATE, 'compute_rebuild_instance')
self.notifier.wait_for_versioned_notifications(
'instance.rebuild.error')
self.notifier.wait_for_versioned_notifications(
'compute.exception')
# and the instance allocates from the source host
self._check_allocation(
server, self.compute1_rp_uuid, non_qos_port, qos_port,
qos_sriov_port, self.flavor_with_group_policy)
def test_live_migrate_with_qos_port(self, host=None):
non_qos_normal_port = self.neutron.port_1
qos_normal_port = self.neutron.port_with_resource_request
qos_sriov_port = self.neutron.port_with_sriov_resource_request
server = self._create_server_with_ports_and_check_allocation(
non_qos_normal_port, qos_normal_port, qos_sriov_port)
self.api.post_server_action(
server['id'],
{
'os-migrateLive': {
'host': host,
'block_migration': 'auto'
}
}
)
self._wait_for_server_parameter(
server,
{'OS-EXT-SRV-ATTR:host': 'host2',
'status': 'ACTIVE'})
self._check_allocation(
server, self.compute2_rp_uuid, non_qos_normal_port,
qos_normal_port, qos_sriov_port, self.flavor_with_group_policy)
self._assert_pci_request_pf_device_name(server, 'host2-ens2')
self._delete_server_and_check_allocations(
server, qos_normal_port, qos_sriov_port)
def test_live_migrate_with_qos_port_with_target_host(self):
self.test_live_migrate_with_qos_port(host='host2')
def test_live_migrate_with_qos_port_reschedule_success(self):
self._start_compute('host3')
compute3_rp_uuid = self._get_provider_uuid_by_host('host3')
self._create_networking_rp_tree('host3', compute3_rp_uuid)
non_qos_normal_port = self.neutron.port_1
qos_normal_port = self.neutron.port_with_resource_request
qos_sriov_port = self.neutron.port_with_sriov_resource_request
server = self._create_server_with_ports_and_check_allocation(
non_qos_normal_port, qos_normal_port, qos_sriov_port)
orig_check = fake.FakeDriver.check_can_live_migrate_destination
def fake_check_can_live_migrate_destination(
context, instance, src_compute_info, dst_compute_info,
block_migration=False, disk_over_commit=False):
if dst_compute_info['host'] == 'host2':
raise exception.MigrationPreCheckError(
reason='test_live_migrate_pre_check_fails')
else:
return orig_check(
context, instance, src_compute_info, dst_compute_info,
block_migration, disk_over_commit)
with mock.patch('nova.virt.fake.FakeDriver.'
'check_can_live_migrate_destination',
side_effect=fake_check_can_live_migrate_destination):
self.api.post_server_action(
server['id'],
{
'os-migrateLive': {
'host': None,
'block_migration': 'auto'
}
}
)
# The first migration attempt was to host2. So we expect that the
# instance lands on host3.
self._wait_for_server_parameter(
server,
{'OS-EXT-SRV-ATTR:host': 'host3',
'status': 'ACTIVE'})
self._check_allocation(
server, compute3_rp_uuid, non_qos_normal_port,
qos_normal_port, qos_sriov_port, self.flavor_with_group_policy)
self._assert_pci_request_pf_device_name(server, 'host3-ens2')
self._delete_server_and_check_allocations(
server, qos_normal_port, qos_sriov_port)
def test_live_migrate_with_qos_port_reschedule_fails(self):
non_qos_normal_port = self.neutron.port_1
qos_normal_port = self.neutron.port_with_resource_request
qos_sriov_port = self.neutron.port_with_sriov_resource_request
server = self._create_server_with_ports_and_check_allocation(
non_qos_normal_port, qos_normal_port, qos_sriov_port)
with mock.patch(
'nova.virt.fake.FakeDriver.check_can_live_migrate_destination',
side_effect=exception.MigrationPreCheckError(
reason='test_live_migrate_pre_check_fails')):
self.api.post_server_action(
server['id'],
{
'os-migrateLive': {
'host': None,
'block_migration': 'auto'
}
}
)
# The every migration target host will fail the pre check so
# the conductor will run out of target host and the migration will
# fail
self._wait_for_migration_status(server, ['error'])
# the server will remain on host1
self._wait_for_server_parameter(
server,
{'OS-EXT-SRV-ATTR:host': 'host1',
'status': 'ACTIVE'})
self._check_allocation(
server, self.compute1_rp_uuid, non_qos_normal_port,
qos_normal_port, qos_sriov_port, self.flavor_with_group_policy)
# Assert that the InstancePCIRequests also rolled back to point to
# host1
self._assert_pci_request_pf_device_name(server, 'host1-ens2')
self._delete_server_and_check_allocations(
server, qos_normal_port, qos_sriov_port)
def test_live_migrate_with_qos_port_pci_update_fails(self):
# Update the name of the network device RP of PF2 on host2 to something
# unexpected. This will cause
# update_pci_request_spec_with_allocated_interface_name() to raise
# when the instance is live migrated to the host2.
rsp = self.placement.put(
'/resource_providers/%s'
% self.sriov_dev_rp_per_host[self.compute2_rp_uuid][self.PF2],
{"name": "invalid-device-rp-name"})
self.assertEqual(200, rsp.status)
non_qos_normal_port = self.neutron.port_1
qos_normal_port = self.neutron.port_with_resource_request
qos_sriov_port = self.neutron.port_with_sriov_resource_request
server = self._create_server_with_ports_and_check_allocation(
non_qos_normal_port, qos_normal_port, qos_sriov_port)
self.api.post_server_action(
server['id'],
{
'os-migrateLive': {
'host': None,
'block_migration': 'auto'
}
}
)
# pci update will fail after scheduling to host2
self._wait_for_migration_status(server, ['error'])
server = self._wait_for_server_parameter(
server,
{'OS-EXT-SRV-ATTR:host': 'host1',
'status': 'ERROR'})
self.assertIn(
'does not have a properly formatted name',
server['fault']['message'])
self._check_allocation(
server, self.compute1_rp_uuid, non_qos_normal_port,
qos_normal_port, qos_sriov_port, self.flavor_with_group_policy)
# Assert that the InstancePCIRequests still point to host1
self._assert_pci_request_pf_device_name(server, 'host1-ens2')
self._delete_server_and_check_allocations(
server, qos_normal_port, qos_sriov_port)
def test_unshelve_offloaded_server_with_qos_port(self):
non_qos_normal_port = self.neutron.port_1
qos_normal_port = self.neutron.port_with_resource_request
qos_sriov_port = self.neutron.port_with_sriov_resource_request
server = self._create_server_with_ports_and_check_allocation(
non_qos_normal_port, qos_normal_port, qos_sriov_port)
# with default config shelve means immediate offload as well
req = {
'shelve': {}
}
self.api.post_server_action(server['id'], req)
self._wait_for_server_parameter(
server, {'status': 'SHELVED_OFFLOADED'})
allocations = self.placement.get(
'/allocations/%s' % server['id']).body['allocations']
self.assertEqual(0, len(allocations))
self.api.post_server_action(server['id'], {'unshelve': None})
self._wait_for_server_parameter(
server,
{'OS-EXT-SRV-ATTR:host': 'host1',
'status': 'ACTIVE'})
self._check_allocation(
server, self.compute1_rp_uuid, non_qos_normal_port,
qos_normal_port, qos_sriov_port, self.flavor_with_group_policy)
self._assert_pci_request_pf_device_name(server, 'host1-ens2')
# shelve offload again and then make host1 unusable so the subsequent
# unshelve needs to select host2
req = {
'shelve': {}
}
self.api.post_server_action(server['id'], req)
self._wait_for_server_parameter(
server, {'status': 'SHELVED_OFFLOADED'})
allocations = self.placement.get(
'/allocations/%s' % server['id']).body['allocations']
self.assertEqual(0, len(allocations))
self.admin_api.put_service(
self.compute1_service_id, {"status": "disabled"})
self.api.post_server_action(server['id'], {'unshelve': None})
self._wait_for_server_parameter(
server,
{'OS-EXT-SRV-ATTR:host': 'host2',
'status': 'ACTIVE'})
self._check_allocation(
server, self.compute2_rp_uuid, non_qos_normal_port,
qos_normal_port, qos_sriov_port, self.flavor_with_group_policy)
self._assert_pci_request_pf_device_name(server, 'host2-ens2')
self._delete_server_and_check_allocations(
server, qos_normal_port, qos_sriov_port)
def test_unshelve_offloaded_server_with_qos_port_pci_update_fails(self):
# Update the name of the network device RP of PF2 on host2 to something
# unexpected. This will cause
# update_pci_request_spec_with_allocated_interface_name() to raise
# when the instance is unshelved to the host2.
rsp = self.placement.put(
'/resource_providers/%s'
% self.sriov_dev_rp_per_host[self.compute2_rp_uuid][self.PF2],
{"name": "invalid-device-rp-name"})
self.assertEqual(200, rsp.status)
non_qos_normal_port = self.neutron.port_1
qos_normal_port = self.neutron.port_with_resource_request
qos_sriov_port = self.neutron.port_with_sriov_resource_request
server = self._create_server_with_ports_and_check_allocation(
non_qos_normal_port, qos_normal_port, qos_sriov_port)
# with default config shelve means immediate offload as well
req = {
'shelve': {}
}
self.api.post_server_action(server['id'], req)
self._wait_for_server_parameter(
server, {'status': 'SHELVED_OFFLOADED'})
allocations = self.placement.get(
'/allocations/%s' % server['id']).body['allocations']
self.assertEqual(0, len(allocations))
# make host1 unusable so the subsequent unshelve needs to select host2
self.admin_api.put_service(
self.compute1_service_id, {"status": "disabled"})
self.api.post_server_action(server['id'], {'unshelve': None})
# Unshelve fails on host2 due to
# update_pci_request_spec_with_allocated_interface_name fails so the
# instance goes back to shelve offloaded state
self.notifier.wait_for_versioned_notifications(
'instance.unshelve.start')
error_notification = self.notifier.wait_for_versioned_notifications(
'compute.exception')[0]
self.assertEqual(
'UnexpectedResourceProviderNameForPCIRequest',
error_notification['payload']['nova_object.data']['exception'])
server = self._wait_for_server_parameter(
server,
{'OS-EXT-STS:task_state': None,
'status': 'SHELVED_OFFLOADED'})
allocations = self.placement.get(
'/allocations/%s' % server['id']).body['allocations']
self.assertEqual(0, len(allocations))
self._delete_server_and_check_allocations(
server, qos_normal_port, qos_sriov_port)
def test_unshelve_offloaded_server_with_qos_port_fails_due_to_neutron(
self):
non_qos_normal_port = self.neutron.port_1
qos_normal_port = self.neutron.port_with_resource_request
qos_sriov_port = self.neutron.port_with_sriov_resource_request
server = self._create_server_with_ports_and_check_allocation(
non_qos_normal_port, qos_normal_port, qos_sriov_port)
# with default config shelve means immediate offload as well
req = {
'shelve': {}
}
self.api.post_server_action(server['id'], req)
self._wait_for_server_parameter(
server, {'status': 'SHELVED_OFFLOADED'})
allocations = self.placement.get(
'/allocations/%s' % server['id']).body['allocations']
self.assertEqual(0, len(allocations))
# Simulate that port update fails during unshelve due to neutron is
# unavailable
with mock.patch(
'nova.tests.fixtures.NeutronFixture.'
'update_port') as mock_update_port:
mock_update_port.side_effect = neutron_exception.ConnectionFailed(
reason='test')
req = {'unshelve': None}
self.api.post_server_action(server['id'], req)
self.notifier.wait_for_versioned_notifications(
'instance.unshelve.start')
self._wait_for_server_parameter(
server,
{'status': 'SHELVED_OFFLOADED',
'OS-EXT-STS:task_state': None})
# As the instance went back to offloaded state we expect no allocation
allocations = self.placement.get(
'/allocations/%s' % server['id']).body['allocations']
self.assertEqual(0, len(allocations))
self._delete_server_and_check_allocations(
server, qos_normal_port, qos_sriov_port)
class LiveMigrateAbortWithPortResourceRequestTest(
PortResourceRequestBasedSchedulingTestBase):
compute_driver = "fake.FakeLiveMigrateDriverWithPciResources"
def setUp(self):
# Use a custom weigher to make sure that we have a predictable host
# order in the alternate list returned by the scheduler for migration.
self.useFixture(nova_fixtures.HostNameWeigherFixture())
super(LiveMigrateAbortWithPortResourceRequestTest, self).setUp()
self.compute2 = self._start_compute('host2')
self.compute2_rp_uuid = self._get_provider_uuid_by_host('host2')
self._create_networking_rp_tree('host2', self.compute2_rp_uuid)
self.compute2_service_id = self.admin_api.get_services(
host='host2', binary='nova-compute')[0]['id']
def test_live_migrate_with_qos_port_abort_migration(self):
non_qos_normal_port = self.neutron.port_1
qos_normal_port = self.neutron.port_with_resource_request
qos_sriov_port = self.neutron.port_with_sriov_resource_request
server = self._create_server_with_ports_and_check_allocation(
non_qos_normal_port, qos_normal_port, qos_sriov_port)
# The special virt driver will keep the live migration running until it
# is aborted.
self.api.post_server_action(
server['id'],
{
'os-migrateLive': {
'host': None,
'block_migration': 'auto'
}
}
)
# wait for the migration to start
migration = self._wait_for_migration_status(server, ['running'])
# delete the migration to abort it
self.api.delete_migration(server['id'], migration['id'])
self._wait_for_migration_status(server, ['cancelled'])
self._wait_for_server_parameter(
server,
{'OS-EXT-SRV-ATTR:host': 'host1',
'status': 'ACTIVE'})
self._check_allocation(
server, self.compute1_rp_uuid, non_qos_normal_port,
qos_normal_port, qos_sriov_port, self.flavor_with_group_policy)
# Assert that the InstancePCIRequests rolled back to point to host1
self._assert_pci_request_pf_device_name(server, 'host1-ens2')
self._delete_server_and_check_allocations(
server, qos_normal_port, qos_sriov_port)
class PortResourceRequestReSchedulingTest(
PortResourceRequestBasedSchedulingTestBase):
"""Similar to PortResourceRequestBasedSchedulingTest
except this test uses FakeRescheduleDriver which will test reschedules
during server create work as expected, i.e. that the resource request
allocations are moved from the initially selected compute to the
alternative compute.
"""
compute_driver = 'fake.FakeRescheduleDriver'
def setUp(self):
super(PortResourceRequestReSchedulingTest, self).setUp()
self.compute2 = self._start_compute('host2')
self.compute2_rp_uuid = self._get_provider_uuid_by_host('host2')
self._create_networking_rp_tree('host2', self.compute2_rp_uuid)
def _create_networking_rp_tree(self, hostname, compute_rp_uuid):
# let's simulate what the neutron would do
self._create_ovs_networking_rp_tree(compute_rp_uuid)
def test_boot_reschedule_success(self):
port = self.neutron.port_with_resource_request
server = self._create_server(
flavor=self.flavor,
networks=[{'port': port['id']}])
server = self._wait_for_state_change(server, 'ACTIVE')
updated_port = self.neutron.show_port(port['id'])['port']
dest_hostname = server['OS-EXT-SRV-ATTR:host']
dest_compute_rp_uuid = self._get_provider_uuid_by_host(dest_hostname)
failed_compute_rp = (self.compute1_rp_uuid
if dest_compute_rp_uuid == self.compute2_rp_uuid
else self.compute2_rp_uuid)
allocations = self.placement.get(
'/allocations/%s' % server['id']).body['allocations']
# We expect one set of allocations for the compute resources on the
# compute rp and one set for the networking resources on the ovs bridge
# rp
self.assertEqual(2, len(allocations))
self.assertComputeAllocationMatchesFlavor(
allocations, dest_compute_rp_uuid, self.flavor)
network_allocations = allocations[
self.ovs_bridge_rp_per_host[dest_compute_rp_uuid]]['resources']
self.assertPortMatchesAllocation(port, network_allocations)
# assert that the allocations against the host where the spawn
# failed are cleaned up properly
self.assertEqual(
{'VCPU': 0, 'MEMORY_MB': 0, 'DISK_GB': 0},
self._get_provider_usages(failed_compute_rp))
self.assertEqual(
{'NET_BW_EGR_KILOBIT_PER_SEC': 0, 'NET_BW_IGR_KILOBIT_PER_SEC': 0},
self._get_provider_usages(
self.ovs_bridge_rp_per_host[failed_compute_rp]))
# We expect that only the RP uuid of the networking RP having the port
# allocation is sent in the port binding
binding_profile = updated_port['binding:profile']
self.assertEqual(self.ovs_bridge_rp_per_host[dest_compute_rp_uuid],
binding_profile['allocation'])
self._delete_and_check_allocations(server)
# assert that unbind removes the allocation from the binding
updated_port = self.neutron.show_port(port['id'])['port']
binding_profile = updated_port['binding:profile']
self.assertNotIn('allocation', binding_profile)
def test_boot_reschedule_fill_provider_mapping_raises(self):
"""Verify that if the _fill_provider_mapping raises during re-schedule
then the instance is properly put into ERROR state.
"""
port = self.neutron.port_with_resource_request
# First call is during boot, we want that to succeed normally. Then the
# fake virt driver triggers a re-schedule. During that re-schedule the
# fill is called again, and we simulate that call raises.
original_fill = utils.fill_provider_mapping
def stub_fill_provider_mapping(*args, **kwargs):
if not mock_fill.called:
return original_fill(*args, **kwargs)
raise exception.ResourceProviderTraitRetrievalFailed(
uuid=uuids.rp1)
with mock.patch(
'nova.scheduler.utils.fill_provider_mapping',
side_effect=stub_fill_provider_mapping) as mock_fill:
server = self._create_server(
flavor=self.flavor,
networks=[{'port': port['id']}])
server = self._wait_for_state_change(server, 'ERROR')
self.assertIn(
'Failed to get traits for resource provider',
server['fault']['message'])
self._delete_and_check_allocations(server)
# assert that unbind removes the allocation from the binding
updated_port = self.neutron.show_port(port['id'])['port']
binding_profile = neutronapi.get_binding_profile(updated_port)
self.assertNotIn('allocation', binding_profile)
class CrossCellResizeWithQoSPort(PortResourceRequestBasedSchedulingTestBase):
NUMBER_OF_CELLS = 2
def setUp(self):
# Use our custom weigher defined above to make sure that we have
# a predictable host order in the alternate list returned by the
# scheduler for migration.
self.useFixture(nova_fixtures.HostNameWeigherFixture())
super(CrossCellResizeWithQoSPort, self).setUp()
# start compute2 in cell2, compute1 is started in cell1 by default
self.compute2 = self._start_compute('host2', cell_name='cell2')
self.compute2_rp_uuid = self._get_provider_uuid_by_host('host2')
self._create_networking_rp_tree('host2', self.compute2_rp_uuid)
self.compute2_service_id = self.admin_api.get_services(
host='host2', binary='nova-compute')[0]['id']
# Enable cross-cell resize policy since it defaults to not allow
# anyone to perform that type of operation. For these tests we'll
# just allow admins to perform cross-cell resize.
self.policy.set_rules({
servers_policies.CROSS_CELL_RESIZE:
base_policies.RULE_ADMIN_API},
overwrite=False)
def test_cross_cell_migrate_server_with_qos_ports(self):
"""Test that cross cell migration is not supported with qos ports and
nova therefore falls back to do a same cell migration instead.
To test this properly we first make sure that there is no valid host
in the same cell but there is valid host in another cell and observe
that the migration fails with NoValidHost. Then we start a new compute
in the same cell the instance is in and retry the migration that is now
expected to pass.
"""
non_qos_normal_port = self.neutron.port_1
qos_normal_port = self.neutron.port_with_resource_request
qos_sriov_port = self.neutron.port_with_sriov_resource_request
server = self._create_server_with_ports_and_check_allocation(
non_qos_normal_port, qos_normal_port, qos_sriov_port)
orig_create_binding = self.neutron.create_port_binding
hosts = {
'host1': self.compute1_rp_uuid, 'host2': self.compute2_rp_uuid,
}
# Add an extra check to our neutron fixture. This check makes sure that
# the RP sent in the binding corresponds to host of the binding. In a
# real deployment this is checked by the Neutron server. As bug
# 1907522 showed we fail this check for cross cell migration with qos
# ports in a real deployment. So to reproduce that bug we need to have
# the same check in our test env too.
def spy_on_create_binding(port_id, data):
host_rp_uuid = hosts[data['binding']['host']]
device_rp_uuid = data['binding']['profile'].get('allocation')
if port_id == qos_normal_port['id']:
if device_rp_uuid != self.ovs_bridge_rp_per_host[host_rp_uuid]:
raise exception.PortBindingFailed(port_id=port_id)
elif port_id == qos_sriov_port['id']:
if (
device_rp_uuid not in
self.sriov_dev_rp_per_host[host_rp_uuid].values()
):
raise exception.PortBindingFailed(port_id=port_id)
return orig_create_binding(port_id, data)
with mock.patch(
'nova.tests.fixtures.NeutronFixture.create_port_binding',
side_effect=spy_on_create_binding, autospec=True
):
# We expect the migration to fail as the only available target
# host is in a different cell and while cross cell migration is
# enabled it is not supported for neutron ports with resource
# request.
self.api.post_server_action(server['id'], {'migrate': None})
self._wait_for_migration_status(server, ['error'])
self._wait_for_server_parameter(
server,
{'status': 'ACTIVE', 'OS-EXT-SRV-ATTR:host': 'host1'})
event = self._wait_for_action_fail_completion(
server, 'migrate', 'conductor_migrate_server')
self.assertIn(
'exception.NoValidHost', event['traceback'])
self.assertIn(
'Request is allowed by policy to perform cross-cell resize '
'but the instance has ports with resource request and '
'cross-cell resize is not supported with such ports.',
self.stdlog.logger.output)
self.assertNotIn(
'nova.exception.PortBindingFailed: Binding failed for port',
self.stdlog.logger.output)
# Now start a new compute in the same cell as the instance and retry
# the migration.
self._start_compute('host3', cell_name='cell1')
self.compute3_rp_uuid = self._get_provider_uuid_by_host('host3')
self._create_networking_rp_tree('host3', self.compute3_rp_uuid)
with mock.patch(
'nova.tests.fixtures.NeutronFixture.create_port_binding',
side_effect=spy_on_create_binding, autospec=True
):
server = self._migrate_server(server)
self.assertEqual('host3', server['OS-EXT-SRV-ATTR:host'])
self._delete_server_and_check_allocations(
server, qos_normal_port, qos_sriov_port)
class ExtendedResourceRequestTempNegativeTest(
PortResourceRequestBasedSchedulingTestBase):
"""A set of temporary tests to show that nova currently rejects requests
that uses the extended-resource-request Neutron API extension. These test
are expected to be removed when support for the extension is implemented
in nova.
"""
def setUp(self):
super().setUp()
self.neutron = self.useFixture(
ExtendedResourceRequestNeutronFixture(self))
def test_boot(self):
"""The neutron fixture used in this test class enables the
extended-resource-request API extension. This results in any new
server to boot. This is harsh but without nova support for this
extension there is no way that this extension is helpful. So treat
this as a deployment configuration error.
"""
ex = self.assertRaises(
client.OpenStackApiException,
self._create_server,
flavor=self.flavor,
networks=[{'port': self.neutron.port_1['id']}],
)
self.assertEqual(400, ex.response.status_code)
self.assertIn(
'The port-resource-request-groups neutron API extension is not '
'yet supported by Nova. Please turn off this extension in '
'Neutron.',
str(ex)
)
View Git Blame Copy Permalink