# Copyright 2024 Red Hat, Inc. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import random import time import testtools import netaddr from neutron_lib import constants from neutron_tempest_plugin.common import ssh from neutron_tempest_plugin.scenario import constants as neutron_constants from oslo_log import log from pyroute2 import IPRoute from tempest.common import utils from tempest.common import waiters from tempest import config from tempest.lib.common.utils import data_utils from tempest.lib.common.utils import test_utils from tempest.lib import decorators from whitebox_neutron_tempest_plugin.common import utils as local_utils from whitebox_neutron_tempest_plugin.tests.scenario import base CONF = config.CONF WB_CONF = config.CONF.whitebox_neutron_plugin_options LOG = log.getLogger(__name__) class OvnDvrBase(base.TrafficFlowTest, base.BaseTempestTestCaseOvn): @classmethod def resource_setup(cls): super(OvnDvrBase, cls).resource_setup() cls.setup_api_microversion_fixture( compute_microversion='2.74') msg = "DVR is not enabled" if len(cls.nodes) < 2: raise cls.skipException( "The tests require environment with at least 2 nodes") for node in cls.nodes: if WB_CONF.openstack_type == 'devstack': if node['is_controller'] is not True: continue cls.check_service_setting( host=node, service='', config_files=[WB_CONF.neutron_config], param='enable_dvr') cls.check_service_setting( host=node, service='', config_files=[WB_CONF.neutron_config], param='router_distributed') cls.check_service_setting( host=node, service='', config_files=[WB_CONF.ml2_plugin_config], section='ovn', param='enable_distributed_floating_ip') # (rsafrono) checks for bgp are probably suitable for tripleo # and should be adjusted for devstack and podified cases cls.bgp_expose_tenant_networks = ( WB_CONF.bgp and cls.check_service_setting( host=node, service='ovn_bgp_agent', config_files=[WB_CONF.bgp_agent_config], param='expose_tenant_networks', skip_if_fails=False)) if WB_CONF.openstack_type == 'podified': config_files = cls.get_configs_of_service() cls.check_service_setting( {'client': cls.proxy_host_client}, config_files=config_files, section='ovn', param='enable_distributed_floating_ip', msg=msg) def _setup(self, router=None): router = self.create_router_by_client() self.router_port = self.os_admin.network_client.list_ports( device_id=router['id'], device_owner=constants.DEVICE_OWNER_ROUTER_GW)['ports'][0] self.chassis_list = self.get_router_gateway_chassis_list( self.router_port['id']) chassis_name = self.get_router_gateway_chassis_by_id( self.chassis_list[0]) LOG.debug("router chassis name = {}".format(chassis_name)) # Since we are going to spawn VMs with 'host' option which # is available only for admin user, we create security group # and keypair also as admin secgroup = self.os_admin.network_client.create_security_group( name=data_utils.rand_name('secgroup')) self.security_groups.append(secgroup['security_group']) self.os_admin.network_client.create_security_group_rule( security_group_id=secgroup['security_group']['id'], protocol=constants.PROTO_NAME_ICMP, direction=constants.INGRESS_DIRECTION) self.os_admin.network_client.create_security_group_rule( security_group_id=secgroup['security_group']['id'], protocol=constants.PROTO_NAME_TCP, direction=constants.INGRESS_DIRECTION, port_range_min=22, port_range_max=22) self.addCleanup( test_utils.call_and_ignore_notfound_exc, self.os_admin.network_client.delete_security_group, secgroup['security_group']['id']) self.keypair = self.os_admin.keypairs_client.create_keypair( name=data_utils.rand_name('keypair'))['keypair'] self.network = self.create_network() self.subnet = self.create_subnet(self.network) self.create_router_interface(router['id'], self.subnet['id']) # We create VMs on compute hosts that are not on the same host # as router gateway port, i.e. the test is capable to work even # on environments that schedule ovn routers on compute nodes self.exclude_hosts = [chassis_name] network_details = self.os_admin.network_client.show_network( self.network['id']) if network_details['network']['provider:network_type'] == 'vlan': # This helps to avoid false positives with vlan+dvr,see BZ2192633 self.ignore_outbound = True else: self.ignore_outbound = False self.server = self._create_server( exclude_hosts=self.exclude_hosts) self.compute = self.get_host_for_server(self.server['server']['id']) self.exclude_hosts.append(self.compute) self.server_ssh_client = ssh.Client( self.server['fip']['floating_ip_address'], CONF.validation.image_ssh_user, pkey=self.keypair['private_key']) self.fip_port_mac = self.get_fip_port_details( self.server['fip'])['mac_address'] LOG.debug("FIP port MAC: {}".format(self.fip_port_mac)) class OvnDvrTest(OvnDvrBase): @decorators.idempotent_id('1561819a-b19f-45a7-8131-d001b2d7c945') def test_validate_floatingip_compute_egress(self): """Check that VM with a floating ip goes out through compute node. The aim of the test is to verify egress DVR functionality. Currently only OVN DVR environments are supported. Topology: Any topology with separate controller and compute nodes is valid for running the test. Recommended topology: 3 controller nodes with networking services and 2 compute nodes with configured access to external network. Scenario: 1. Create network, subnet, pingable and loginable security groups, keypair, run a vm instance (server) and create a fip for it. 2. Check on which compute node the server runs. 3. Capture traffic on all nodes and ping an IP address in the external network from the server (see details in _check_north_south_flow()) 4. Search for the server fip mac address in all capture files. 5. Verify that we found the server fip mac address in the capture file on the compute node where the server runs and not found in captures on other nodes. """ self._setup() self.check_north_south_icmp_flow( dst_ip=self.gateway_external_ip, expected_routing_nodes=[self.compute], expected_mac=self.fip_port_mac, ssh_client=self.server_ssh_client) @decorators.idempotent_id('682167ba-6250-4f3c-8fdf-1c768825cb8c') @testtools.skipIf(WB_CONF.openstack_type == 'podified', 'Not yet adapted for podified environment') def test_validate_floatingip_compute_ingress_delete_fip_restart_instance( self): """Check that traffic to a VM with a floating ip enters through compute node. The aim of the test is to verify ingress DVR functionality. Currently only OVN DVR environments are supported. The test also removes the first floating ip from the server and adds a new one, verifying that it does not affect routing to ingress traffic. Finally, it verifies that an instance reboot does not affect the connectivity or/and the routing. Topology: Any topology with separate controller and compute nodes is valid for running the test. Recommended topology: 3 controller nodes with networking services and 2 compute nodes with configured access to external network. Scenario: 1. Create network, subnet, pingable and loginable security groups, keypair, run a vm instance (server) and create a fip for it. 2. Check on which compute node the server runs. 3. Capture traffic on all nodes and ping the server from an IP address in the external network (see details in _check_north_south_flow()) 4. Search for the server fip mac address in all capture files. 5. Verify that we found the server fip mac address in the capture file on the compute node where the server runs and not found in captures on other nodes. 6. Remove the FIP from the test server and add a new one to it; verify that traffic again is passing through compute node where the server was spawned 7. Restart the server and verify that routing has not changed """ self._setup() self.check_north_south_icmp_flow( dst_ip=self.server['fip']['floating_ip_address'], expected_routing_nodes=[self.compute], expected_mac=self.fip_port_mac, ssh_client=self.proxy_host_client) # Delete fip LOG.debug('Deleting floating ip') self.os_admin.network_client.delete_floatingip( self.server['fip']['id']) # Add a new fip to the test server and make sure that routing is # via compute again. LOG.debug('Adding new floating ip to vm') fip = self.os_admin.network_client.create_floatingip( port_id=self.server['port']['id'], floating_network_id=CONF.network.public_network_id)['floatingip'] fip_port_mac = self.get_fip_port_details(fip)['mac_address'] self.check_north_south_icmp_flow( dst_ip=fip['floating_ip_address'], expected_routing_nodes=[self.compute], expected_mac=fip_port_mac, ssh_client=self.proxy_host_client) # Reboot the server and make sure that routing is still via compute. LOG.debug('Rebooting vm') self.os_primary.servers_client.reboot_server( self.server['server']['id'], type='SOFT') waiters.wait_for_server_status(self.os_primary.servers_client, self.server['server']['id'], neutron_constants.SERVER_STATUS_ACTIVE) self.check_north_south_icmp_flow( dst_ip=fip['floating_ip_address'], expected_routing_nodes=[self.compute], expected_mac=fip_port_mac, ssh_client=self.proxy_host_client) @decorators.idempotent_id('0fcf9f97-6368-4c5d-a5f5-ff8a7643e3b6') def test_validate_fip2fip_compute(self): """Check that traffic between VMs with a floating ip running on different compute nodes passes only through the compute nodes. The aim of the test is to verify fip to fip DVR functionality. Currently only OVN DVR environments are supported. Topology: Any topology with separate controller and compute nodes is valid for running the test. Recommended topology: 3 controller nodes with networking services and 2 compute nodes with configured access to external network. Scenario: 1. Create network, subnet, pingable and loginable security groups, keypair, run 2 vm instances (servers) and create a fip for each. 2. Check on which compute node the server runs. 3. Capture traffic on all nodes and ping one server from the other in the external network (see details in _check_north_south_flow()) 4. Search for the server fip mac address in all capture files. 5. Verify that we found the server fip mac address in the capture file on compute nodes where both server run and not found in captures on other nodes. """ self._setup() server2 = self._create_server(exclude_hosts=self.exclude_hosts) compute2 = self.get_host_for_server( server2['server']['id']).split('.')[0] LOG.debug("compute = {}, compute2 = {}".format(self.compute, compute2)) if self.compute == compute2: self.skipTest( "Servers are running on same compute - the test can provide " "wrong results, skipping") # with FIP to FIP it is expected to see packets on both computes self.check_north_south_icmp_flow( dst_ip=server2['fip']['floating_ip_address'], expected_routing_nodes=[self.compute, compute2], expected_mac=self.fip_port_mac, ssh_client=self.server_ssh_client) @decorators.idempotent_id('f8fd0fbd-4ad3-4b0b-b805-6c59228bc5d8') @testtools.skipUnless(CONF.compute_feature_enabled.live_migration, 'Live migration is not available.') @testtools.skipUnless(CONF.compute.min_compute_nodes > 1, 'Less than 2 compute nodes, skipping multinode ' 'tests.') @testtools.skipIf(WB_CONF.openstack_type == 'podified', 'Not yet adapted for podified environment') @decorators.attr(type='slow') @utils.services('compute', 'network') def test_validate_dvr_connectivity_live_migration(self): """Check that after VM migration to a new compute node, traffic is correctly routed through that new node. The aim of the test is to verify egress DVR functionality after a VM live migration. Currently only OVN DVR environments are supported. Topology: Two compute nodes are required for this test. Recommended topology: 3 controller nodes with networking services and 2 compute nodes with configured access to external network. Scenario: 1. Create network, subnet, pingable and loginable security groups, keypair, run two vm instances (self.server and server2) and create a fip for each of them. 2. Check on which compute node each server runs. 3. North-South Traffic verification: a. Capture traffic on all nodes and ping an IP address in the external network from self.server (see details in _check_north_south_flow()). b. Search for the server fip mac address in all capture files. c. Verify that we found the server fip mac address in the capture file on the compute node where the server runs and not found in captures on other nodes. b. Repeat steps a to c using server2. 4. East-West Traffic verification: a. Capture traffic on all nodes and ping server2 internal IP address from self.server. b. Search for both self.server and server2 internal mac addresses in all capture files. c. Verify that we found the server both mac addresses in the capture file on both compute nodes where the servers run and not found in captures on other nodes. 5. Apply live migration on both self.server and server2. 6. Verify that each server is migrated to a different compute node. 7. Repeat steps 3 and 4. """ router = self.create_router_by_client() self._setup(router=router) server2 = self._create_server( scheduler_hints={'different_host': self.server['server']['id']}) server2_ip = server2['port']['fixed_ips'][0]['ip_address'] server2_mac = server2['port']['mac_address'] server2_fip_ip = server2['fip']['floating_ip_address'] server2_fip_mac = self.get_fip_port_details( server2['fip'])['mac_address'] server2_ssh_client = ssh.Client(server2_fip_ip, CONF.validation.image_ssh_user, pkey=self.keypair['private_key'], proxy_client=self.server_ssh_client) server2_host_full = self.get_host_for_server( server2['server']['server']['id']) server2_host = server2_host_full.split('.')[0] servers_host_suffix = '.'.join(server2_host_full.split('.')[1:]) # verify N/S connection with self.server self.check_north_south_icmp_flow( dst_ip=self.gateway_external_ip, expected_routing_nodes=[self.compute], expected_mac=self.fip_port_mac, ssh_client=self.server_ssh_client) # verify N/S connection with server2 self.check_north_south_icmp_flow( dst_ip=self.gateway_external_ip, expected_routing_nodes=[server2_host], expected_mac=server2_fip_mac, ssh_client=server2_ssh_client) # verify E/W connection between self.server and server2 # remove duplicates expected_routing_nodes = list(set([self.compute, server2_host])) if len(expected_routing_nodes) == 1: self.skipTest( "Servers are running on same compute - Please check if " "DifferentHostFilter is configured within the " "NovaSchedulerDefaultFilters list") self.check_east_west_icmp_flow( dst_ip=server2_ip, expected_routing_nodes=expected_routing_nodes, expected_macs=(self.port['mac_address'], server2_mac), ssh_client=self.server_ssh_client) block_migration = (CONF.compute_feature_enabled. block_migration_for_live_migration) # migrate self.server self.os_admin.servers_client.live_migrate_server( self.server['server']['id'], host=None, block_migration=block_migration, disk_over_commit=False) self.wait_for_server_active(self.server['server']) new_host = self.get_host_for_server( self.server['server']['id']).split('.')[0] self.assertNotEqual(self.compute, new_host, 'Server1 did not migrate') # migrate server2 compute_names = [ node['name'] for node in self.nodes if node['type'] == 'compute' and node['name'] not in ( new_host, server2_host)] host = '.'.join((random.choice(compute_names), servers_host_suffix)) self.os_admin.servers_client.live_migrate_server( server2['server']['server']['id'], host=host, block_migration=block_migration, disk_over_commit=False) self.wait_for_server_active(server2['server']['server']) new_server2_host = self.get_host_for_server( server2['server']['server']['id']).split('.')[0] self.assertNotEqual(server2_host, new_server2_host, 'Server2 did not migrate') # verify N/S connection with self.server self.check_north_south_icmp_flow( dst_ip=self.gateway_external_ip, expected_routing_nodes=[new_host], expected_mac=self.fip_port_mac, ssh_client=self.server_ssh_client) # verify N/S connection with server2 self.check_north_south_icmp_flow( dst_ip=self.gateway_external_ip, expected_routing_nodes=[new_server2_host], expected_mac=server2_fip_mac, ssh_client=server2_ssh_client) # verify E/W connection between self.server and server2 # remove duplicates expected_routing_nodes = list(set([new_host, new_server2_host])) if len(expected_routing_nodes) == 1: self.skipTest( "Servers are running on same compute - Please check if " "DifferentHostFilter is configured within the " "NovaSchedulerDefaultFilters list") self.check_east_west_icmp_flow( dst_ip=server2_ip, expected_routing_nodes=expected_routing_nodes, expected_macs=(self.port['mac_address'], server2_mac), ssh_client=self.server_ssh_client) @decorators.idempotent_id('609997ab-bffc-40e5-a858-635099df4db9') @testtools.skipUnless(CONF.compute_feature_enabled.live_migration, 'Live migration is not available.') @testtools.skipUnless(CONF.compute.min_compute_nodes > 1, 'Less than 2 compute nodes, skipping multinode ' 'tests.') @testtools.skipIf(WB_CONF.openstack_type == 'podified', 'Not yet adapted for podified environment') @decorators.attr(type='slow') @utils.services('compute', 'network') def test_validate_dvr_connectivity_live_migration_different_networks(self): """This test is like test_validate_dvr_connectivity_live_migration but VM instances are created in different tenant networks Plase see previous test description for more details """ self.create_security_group( name=data_utils.rand_name('secgroup')) self.create_loginable_secgroup_rule( secgroup_id=self.security_groups[-1]['id']) self.create_pingable_secgroup_rule( secgroup_id=self.security_groups[-1]['id']) self.keypair = self.create_keypair() router = self.create_router_by_client() networks = [] subnets = [] servers = [] servers_host = [] servers_fip_mac = [] servers_ssh_client = [] router_port_subnet_macs = [] for i in range(2): networks.append(self.create_network()) subnets.append(self.create_subnet(network=networks[i])) self.create_router_interface(router['id'], subnets[i]['id']) scheduler_hints = ( {'different_host': servers[0]['server']['server']['id']} if i > 0 else {}) servers.append(self._create_server( network=networks[i], scheduler_hints=scheduler_hints)) servers_host.append(self.get_host_for_server( servers[i]['server']['server']['id']).split('.')[0]) servers_fip_mac.append(self.get_fip_port_details( servers[i]['fip'])['mac_address']) servers_ssh_client.append(ssh.Client( servers[i]['fip']['floating_ip_address'], CONF.validation.image_ssh_user, pkey=self.keypair['private_key'])) # verify N/S connection with servers self.check_north_south_icmp_flow( dst_ip=self.gateway_external_ip, expected_routing_nodes=[servers_host[i]], expected_mac=servers_fip_mac[i], ssh_client=servers_ssh_client[i]) # obtain router mac for each network router_port_subnet_macs.append( self.os_admin.network_client.list_ports( device_id=router['id'], device_owner=constants.DEVICE_OWNER_ROUTER_INTF, network_id=networks[i]['id'])['ports'][0]['mac_address']) # verify E/W connection between servers ew_expected_macs = [ (servers[0]['port']['mac_address'], router_port_subnet_macs[0]), (router_port_subnet_macs[1], servers[1]['port']['mac_address'])] # remove duplicates expected_routing_nodes = list(set([host for host in servers_host])) if len(expected_routing_nodes) == 1: self.skipTest( "Servers are running on same compute - Please check if " "DifferentHostFilter is configured within the " "NovaSchedulerDefaultFilters list") self.check_east_west_icmp_flow( dst_ip=servers[1]['port']['fixed_ips'][0]['ip_address'], expected_routing_nodes=expected_routing_nodes, expected_macs=ew_expected_macs, ssh_client=servers_ssh_client[0]) block_migration = (CONF.compute_feature_enabled. block_migration_for_live_migration) # migrate self.server new_servers_host = [] servers_host_suffix = '.'.join( self.get_host_for_server( servers[0]['server']['server']['id']).split('.')[1:]) for i, server in enumerate(servers): if i < 1: host = None else: compute_names = [ node['name'] for node in self.nodes if node['type'] == 'compute' and node['name'] not in ( new_servers_host[0], servers_host[1])] host = '.'.join((random.choice(compute_names), servers_host_suffix)) self.os_admin.servers_client.live_migrate_server( server['server']['server']['id'], host=host, block_migration=block_migration, disk_over_commit=False) self.wait_for_server_active(server['server']['server']) new_servers_host.append(self.get_host_for_server( server['server']['server']['id']).split('.')[0]) self.assertNotEqual(servers_host[i], new_servers_host[i], 'Server%d did not migrate' % i) # verify N/S connection with servers self.check_north_south_icmp_flow( dst_ip=self.gateway_external_ip, expected_routing_nodes=[new_servers_host[i]], expected_mac=servers_fip_mac[i], ssh_client=servers_ssh_client[i]) # verify E/W connection between servers # remove duplicates expected_routing_nodes = list(set([host for host in new_servers_host])) if len(expected_routing_nodes) == 1: self.skipTest( "Servers are running on same compute - Please check if " "DifferentHostFilter is configured within the " "NovaSchedulerDefaultFilters list") self.check_east_west_icmp_flow( dst_ip=servers[1]['port']['fixed_ips'][0]['ip_address'], expected_routing_nodes=expected_routing_nodes, expected_macs=ew_expected_macs, ssh_client=servers_ssh_client[0]) @decorators.idempotent_id('0423e5b5-ac6a-4d4a-ad98-b0465e3ad71d') @testtools.skipIf(WB_CONF.openstack_type == 'podified', 'Not yet adapted for podified environment') def test_dvr_create_delete_fip_restart_instance(self): """Check that traffic from a VM with a FIP passes through compute node where VM is running and traffic from VM without a FIP passes through controller node where router is scheduled. The aim of the test is to verify that routing is distributed only when VM has a FIP and centralized when VM does not have a FIP. Currently only OVN DVR environments are supported. The test also verifies that an instance reboot does not affect the connectivity or/and the routing. Topology: Any topology with separate controller and compute nodes is valid for running the test. Recommended topology: 3 controller nodes with networking services and 2 compute nodes with configured access to external network. Scenario: 1. Create network, subnet, router, pingable and loginable security groups, keypair, run 2 VM instances (servers). One of the VMs is a test server without a FIP and second one is a 'proxy' (with a FIP) in order to have ability to access the test server even in case it does not have a FIP. 2. Check on which controller node the router was scheduled. 3. Capture traffic on all nodes on the interface connected to the external network and ping an external address from the test server. 4. Search for the icmp packets with router gateway port mac address in all capture files. 5. Verify that icmp traffic with the router gateway port mac address was found on controller node where the router was scheduled. 6. Create a FIP for the test server, ping external address and this time verify that traffic is passing through compute node where the test server is running. This time we are looking for the test server FIP port mac. 7. Delete the FIP from the test server and verify that traffic again is passing through controller node where router is scheduled. 8. Add a new FIP to the test server and verify that traffic again is passing through compute node where router is scheduled. 9. Restart the server and verify that routing has not changed """ router = self.create_router_by_client() self._setup(router=router) # self.server will be used as a proxy server for accessing test_server. test_server = self._create_server(create_floating_ip=False) test_server_ip = test_server['port']['fixed_ips'][0]['ip_address'] test_server_client = ssh.Client(test_server_ip, CONF.validation.image_ssh_user, pkey=self.keypair['private_key'], proxy_client=self.server_ssh_client) router_port = self.os_admin.network_client.list_ports( device_id=router['id'], device_owner=constants.DEVICE_OWNER_ROUTER_GW)['ports'][0] router_gateway_chassis = self.get_router_gateway_chassis( router_port['id']) self.check_north_south_icmp_flow( dst_ip=self.gateway_external_ip, expected_routing_nodes=[router_gateway_chassis], expected_mac=router_port['mac_address'], ssh_client=test_server_client, ignore_outbound=self.ignore_outbound) # Now add a fip to the test server and make sure that routing now # via compute. LOG.debug('Adding floating ip to source vm') fip = self.create_floatingip(port=test_server['port']) fip_port_mac = self.get_fip_port_details(fip)['mac_address'] test_server_compute = self.get_host_for_server( test_server['server']['server']['id']).split('.')[0] self.check_north_south_icmp_flow( dst_ip=self.gateway_external_ip, expected_routing_nodes=[test_server_compute], expected_mac=fip_port_mac, ssh_client=test_server_client) # Delete fip and make sure that traffic goes via router chassis. LOG.debug('Deleting floating ip from source vm') self.delete_floatingip(fip) self.check_north_south_icmp_flow( dst_ip=self.gateway_external_ip, expected_routing_nodes=[router_gateway_chassis], expected_mac=router_port['mac_address'], ssh_client=test_server_client, ignore_outbound=self.ignore_outbound) # Add a new fip to the test server and make sure that routing is # via compute again. LOG.debug('Adding new floating ip to source vm') fip = self.create_floatingip(port=test_server['port']) fip_port_mac = self.get_fip_port_details(fip)['mac_address'] self.check_north_south_icmp_flow( dst_ip=self.gateway_external_ip, expected_routing_nodes=[test_server_compute], expected_mac=fip_port_mac, ssh_client=test_server_client) # Reboot the server and make sure that routing is still via compute. LOG.debug('Rebooting vm') self.os_primary.servers_client.reboot_server( test_server['server']['server']['id'], type='SOFT') waiters.wait_for_server_status(self.os_primary.servers_client, test_server['server']['server']['id'], neutron_constants.SERVER_STATUS_ACTIVE) self.check_north_south_icmp_flow( dst_ip=self.gateway_external_ip, expected_routing_nodes=[test_server_compute], expected_mac=fip_port_mac, ssh_client=test_server_client) @testtools.skipIf(WB_CONF.openstack_type == 'podified', 'Not yet adapted for podified environment') class OvnDvrAdvancedTest(base.BaseTempestTestCaseAdvanced, OvnDvrBase): @classmethod def create_loginable_secgroup_rule(cls, secgroup_id=None, client=None): super(OvnDvrAdvancedTest, cls).create_loginable_secgroup_rule( secgroup_id=secgroup_id, client=client) # the parent method only creates an ssh rule for IPv4 traffic and IPv6 # is needed too cls.create_security_group_rule( security_group_id=secgroup_id, client=client, protocol='tcp', direction='ingress', ip_version=6, port_range_min=22, port_range_max=22) @classmethod def resource_setup(cls): super(OvnDvrAdvancedTest, cls).resource_setup() cls.keypair = cls.create_keypair() cls.secgroup = cls.os_primary.network_client.create_security_group( name=data_utils.rand_name('secgroup')) cls.security_groups.append(cls.secgroup['security_group']) cls.create_loginable_secgroup_rule( secgroup_id=cls.secgroup['security_group']['id']) cls.router = cls.create_router_by_client() @staticmethod def _configure_ip_address_in_vm(ssh_client, nic, ip_address): ssh_client.execute_script( 'ip addr add %s dev %s' % (ip_address, nic), become_root=True) if netaddr.valid_ipv4(ip_address): # We need this delay in order to simulate behavior on a busy # environment, i.e. cause a race condition between garp and tcp # traffic, see rhbz#2035079. # Without the delay the race condition does not occur and the test # passes successfully even with broken core OVN. time.sleep(5) ssh_client.execute_script( 'arping -c 1 -A -I %s %s' % (nic, ip_address), become_root=True) @staticmethod def _remove_ip_address_from_vm(ssh_client, nic, ip_address): ssh_client.execute_script( 'ip addr del {ip} dev {nic}; ip addr'.format( ip=ip_address, nic=nic), become_root=True) @staticmethod def _start_tcp_connection(ssh_client): ssh_process = ssh_client.open_session() ip_address = WB_CONF.global_ip_address cmd = ('ping {}'.format(ip_address)) ssh_process.exec_command(cmd) def _failover_vip( self, master_vm, new_master_vm, vip_ssh, nic, ip_address): self._start_tcp_connection(vip_ssh) self._remove_ip_address_from_vm( master_vm['ssh_client'], nic, ip_address) self._configure_ip_address_in_vm( new_master_vm['ssh_client'], nic, ip_address) @staticmethod def _get_src_ip_from_route(dst): # if there is no local IP from the external subnet's CIDR (this happens # with BGP setups), the src IP from the route to the vip(fip) is used # instead ipr = IPRoute() routes = ipr.get_routes(dst=dst) ipr.close() for route in routes: for attr in route.get('attrs', []): if attr[0] == 'RTA_PREFSRC': return attr[1] def _get_filters(self, local_ip, vip_mac, vip_ip): if self.external_network['provider:network_type'] == 'vlan': filters = 'vlan {} and '.format( self.external_network['provider:segmentation_id']) else: filters = '' if not WB_CONF.bgp and vip_mac is not None: filters += 'ether host {} and dst host {}'.format( vip_mac, local_ip) else: filters += 'src host {} and dst host {}'.format( vip_ip, local_ip) filters += ' and tcp src port 22' return filters def _capture_and_test_failover_vip( self, filters, vm1, vm2, nic, vip_ip, vip_ssh_client): self._start_captures(WB_CONF.node_ext_interface, filters) self._failover_vip(vm1, vm2, vip_ssh_client, nic, vip_ip) # BZ#2035079 reproduced reliably when 2 failovers happen self._failover_vip(vm2, vm1, vip_ssh_client, nic, vip_ip) self._stop_captures() LOG.debug('Expected routing nodes: {}'.format( ','.join(self.expected_routing_nodes))) actual_routing_nodes = [node['name'] for node in self.nodes if not node['capture'].is_empty()] LOG.debug('Actual routing nodes: {}'.format( ','.join(actual_routing_nodes))) self.assertCountEqual( self.expected_routing_nodes, actual_routing_nodes) @decorators.idempotent_id('509d1432-3879-40d4-9378-e6a0d972a292') def test_dvr_vip_failover(self): """Test DVR during VIP failover using a tenant network and FIPs The test checks that during VIP failover on DVR environment traffic remains distributed. Scenario: 1. Create private network, router, connect the network to the router and the router to the public network. 2. Spawn 2 VMs connected to the private network. 3. Create a port on the private network that will work as a virtual ip address (VIP) for the VMs. Make sure that VM ports have this ip address configured in the allowed address pairs list. 4. Create FIPs on the public network for VMs and the VIP. 5. Configure VIP address on one of the VMs, initiate a TCP connection to the VIP FIP. 6. Start traffic captures on all overcloud nodes filtering VIP FIP MAC address. 7. Failover the VIP from one VM to another with a delay before sending GARP in order to simulate behaviour on a busy system. See relevant customer issue for details, BZ#2035079. 8. Check traffic captures on all nodes to see where VIP FIP traffic really passed. Expected behavior is to see traffic on compute nodes rather than any networker node. 9. In case BGP is configured with expose_tenant_networks, the test proceeds to remove the FIPs, disable snat from the router and check connectivity by repeating steps 6, 7 and 8 using tenant IPs instead of FIPs. The only expected routing node is the controller hosting the router gateway port. """ vm1, vm2 = self._create_vms_by_topology() vm1_port = self.client.list_ports(device_id=vm1['id'])['ports'][0] vip_port = self.client.create_port( network_id=vm1_port['network_id'])['port'] self.ports.append(vip_port) vip_ip = vip_port['fixed_ips'][0]['ip_address'] self.client.update_port( vm1_port['id'], allowed_address_pairs=[{"ip_address": vip_ip}]) vm2_port = self.client.list_ports(device_id=vm2['id'])['ports'][0] self.client.update_port( vm2_port['id'], allowed_address_pairs=[{"ip_address": vip_ip}]) vip_fip = self.create_floatingip(port=vip_port) self.expected_routing_nodes = [] for vm in [vm1, vm2]: nic = local_utils.get_default_interface(vm['ssh_client']) self.expected_routing_nodes.append( self.get_host_for_server(vm['id']).split('.')[0]) vip_ssh_client = ssh.Client( vip_fip['floating_ip_address'], self.username, pkey=self.keypair['private_key']) vip_fip_mac = self.get_fip_port_details(vip_fip)['mac_address'] # Let's set vip first on vm1 and then will do the vip failover to vm2 self._configure_ip_address_in_vm(vm1['ssh_client'], nic, vip_ip) self.external_network = self.os_admin.network_client.show_network( CONF.network.public_network_id)['network'] local_ip = self.get_local_ssh_client(self.external_network).host if local_ip is None: local_ip = self._get_src_ip_from_route( vip_fip['floating_ip_address']) filters = self._get_filters( local_ip, vip_fip_mac, vip_fip['floating_ip_address']) self._capture_and_test_failover_vip( filters, vm1, vm2, nic, vip_ip, vip_ssh_client) # Test connectivity to tenant VIP if BGP is configured with # expose_tenant_networks if self.bgp_expose_tenant_networks: # remove FIPs vm1_fip = self.client.list_floatingips( port_id=vm1_port['id'])['floatingips'][0] vm2_fip = self.client.list_floatingips( port_id=vm2_port['id'])['floatingips'][0] self.delete_floatingip(vm1_fip) self.delete_floatingip(vm2_fip) self.delete_floatingip(vip_fip) # update ssh clients with tenant IPs vm1_ip = vm1_port['fixed_ips'][0]['ip_address'] vm2_ip = vm2_port['fixed_ips'][0]['ip_address'] vm1['ssh_client'].host = vm1_ip vm2['ssh_client'].host = vm2_ip vip_ssh_client.host = vip_ip # disable snat from the router self.os_admin.network_client.update_router_with_snat_gw_info( self.router['id'], external_gateway_info={ 'network_id': CONF.network.public_network_id, 'enable_snat': False}) # generate updated tcpdump filters local_ip = self._get_src_ip_from_route(vip_ip) filters = self._get_filters( local_ip, vip_port['mac_address'], vip_ip) # calculate new expected_routing_nodes = chassis hosting the router # gateway is the only expected match (traffic from that # chassis/controller to the computes hosting the two VMs goes # through a geneve tunnel) router_port = self.os_admin.network_client.list_ports( device_id=self.router['id'], device_owner=constants.DEVICE_OWNER_ROUTER_GW)['ports'][0] router_gateway_chassis = self.get_router_gateway_chassis( router_port['id']) self.expected_routing_nodes = [router_gateway_chassis] self._capture_and_test_failover_vip( filters, vm1, vm2, nic, vip_ip, vip_ssh_client) @decorators.idempotent_id('5d812adb-ba7c-4ce3-a589-4cc3426d1578') def test_dvr_vip_failover_external_network(self): """Test DVR during VIP failover using an external network with IPv4 and IPv6 subnets Repeat test test_dvr_vip_failover using VMs connected directly to the external network. The test checks that during VIP failover on DVR environment traffic remains distributed. Scenario: 1. Spawn 2 VMs connected to the external network. 3. Create a port on the external network that will work as a virtual ip address (VIP) for the VMs. Make sure that VM ports have this ip address configured in the allowed address pairs list. 4. Configure VIP address on one of the VMs, initiate a TCP connection to the VIP FIP. 5. Start traffic captures on all overcloud nodes filtering VIP MAC address. 6. Failover the VIP from one VM to another with a delay before sending GARP in order to simulate behaviour on a busy system. See relevant customer issue for details, BZ#2035079. 7. Check traffic captures on all nodes to see where VIP traffic really passed. Expected behavior is to see traffic on compute nodes rather than any networker node. 8. If the external network has an IPv6 subnet too, repeat steps 2 to 7 with an IPv6 VIP port. """ vm1, vm2 = self._create_vms_by_topology(topology='external') vm1_port = self.client.list_ports(device_id=vm1['id'])['ports'][0] vm2_port = self.client.list_ports(device_id=vm2['id'])['ports'][0] vip_port = self.client.create_port( network_id=vm1_port['network_id'])['port'] self.ports.append(vip_port) vip_ip = vip_port['fixed_ips'][0]['ip_address'] aap = [{"ip_address": vip_ip}] vip_ssh_client = ssh.Client( vip_ip, self.username, pkey=self.keypair['private_key']) self.expected_routing_nodes = [] for vm in [vm1, vm2]: nic = local_utils.get_default_interface(vm['ssh_client']) self.expected_routing_nodes.append( self.get_host_for_server(vm['id']).split('.')[0]) # checking whether an external IPv6 subnet exists or not ip_versions = [ subnet['ip_version'] for subnet in self.os_admin.network_client.list_subnets( network_id=CONF.network.public_network_id)['subnets']] if len(ip_versions) > 1 and ip_versions[1] == 6: skip_ipv6 = False vip_portv6 = self.client.create_port( network_id=vm1_port['network_id'])['port'] self.ports.append(vip_portv6) vip_ipv6 = vip_portv6['fixed_ips'][1]['ip_address'] vip_ssh_clientv6 = ssh.Client( vip_ipv6, self.username, pkey=self.keypair['private_key']) # adding the vip_ipv6 address to the allowed-address-pairs list aap.append({"ip_address": vip_ipv6}) else: skip_ipv6 = True LOG.info('The test cannot be executed with an IPv6 address') self.os_admin.network_client.update_port( vm1_port['id'], allowed_address_pairs=aap) self.os_admin.network_client.update_port( vm2_port['id'], allowed_address_pairs=aap) # Let's set vip first on vm1 and then will do the vip failover to vm2 self._configure_ip_address_in_vm(vm1['ssh_client'], nic, vip_ip) self.external_network = self.os_admin.network_client.show_network( CONF.network.public_network_id)['network'] local_ip = self.get_local_ssh_client(self.external_network).host if local_ip is None: local_ip = self._get_src_ip_from_route(vip_ip) # vip_mac is set to None because the filter should be based on the # vip_ip instead in this case, where no FIPs are used filters = self._get_filters(local_ip, None, vip_ip) self._capture_and_test_failover_vip( filters, vm1, vm2, nic, vip_ip, vip_ssh_client) # removing the IPv4 VIP address self._remove_ip_address_from_vm(vm1['ssh_client'], nic, vip_ip) # now, with the external ipv6 address # Let's set vip first on vm2 and then will do the vip failover to vm1 if skip_ipv6 is False: self._configure_ip_address_in_vm(vm2['ssh_client'], nic, vip_ipv6) local_ip = self._get_src_ip_from_route(vip_ipv6) # vip_mac is set to None because the filter should be based on the # vip_ip instead in this case, where no FIPs are used filters = self._get_filters(local_ip, None, vip_ipv6) self._capture_and_test_failover_vip( filters, vm2, vm1, nic, vip_ipv6, vip_ssh_clientv6)