Fix identity url version detection

Before we could diferentiate the api version for identity because
the use of /v2 or /v3. Even though v2 is now deprecated, we need
to continue support previous versions of openstack. In this case
We cannot relay on the /v2 or /v3 in the url. The best user case
is to do a request in the base url (without any version) and
check if we have v3 version stable. If so, we assume identity
version is v3 otherwise, is v2.

Change-Id: Ia41d21ebad9329ae9fa506868957a72e6f9a5ca5

config_tempest/credentials.py

@@ -13,6 +13,9 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+import re
+import requests
+from six.moves import urllib
 from tempest.lib import auth
 
 
@@ -62,15 +65,36 @@ class Credentials(object):
             # tool keeps them in identity section for further usage
             return self._conf.get_defaulted('identity', key)
 
+    def _get_base_url(self, endpoint):
+        """Return the base url.
+
+        :param key: endpoint
+        :type key: string
+        :returns: base_url
+        :rtype: string
+        """
+        url = urllib.parse.urlsplit(endpoint)
+        new_path = re.split(r'(^|/)+v\d+(\.\d+)?', url.path)[0]
+        url = list(url)
+        url[2] = new_path + '/'
+        return urllib.parse.urlunsplit(url)
+
+    def _list_versions(self, base_url):
+        resp = requests.get(base_url)
+        data = resp.json()
+        return data["versions"]["values"]
+
     def _get_identity_version(self):
         """Looks for identity version in TempestConf object.
 
         :returns: identity version
         :rtype: string
         """
-        if "v3" in self._conf.get("identity", "uri"):
+        base_url = self._get_base_url(self._conf.get("identity", "uri"))
+        versions = self._list_versions(base_url)
+        for version in versions:
+            if version["status"] == "stable" and "v3" in version["id"]:
                 return "v3"
-        else:
         return "v2"
 
     def _get_creds_kwargs(self):
@@ -111,6 +135,11 @@ class Credentials(object):
         :rtype: auth.KeystoneV2AuthProvider/auth.KeystoneV3AuthProvider
         """
         if isinstance(self.tempest_creds, auth.KeystoneV3Credentials):
+            # We set uri and uri_v3 to /v3 here because if the endpoint on the
+            # rc file don't set the /v3 it will fail with a error 404
+            uri = self._conf.get_defaulted('identity', 'uri_v3')
+            uri = self._get_base_url(uri) + 'v3'
+            self._conf.set('identity', 'uri_v3', uri)
             return auth.KeystoneV3AuthProvider(
                 self.tempest_creds,
                 self._conf.get_defaulted('identity', 'uri_v3'),

config_tempest/tests/base.py

@@ -29,6 +29,25 @@ class BaseConfigTempestTest(base.BaseTestCase):
 
     """Test case base class for all config_tempest unit tests"""
 
+    FAKE_V3_VERSIONS = (
+        [{
+            'status': 'stable',
+            'id': 'v3.8',
+        }, {
+            'status': 'deprecated',
+            'id': 'v2.0',
+        }]
+    )
+    FAKE_V2_VERSIONS = (
+        [{
+            'status': 'deprecated',
+            'id': 'v3.8',
+        }, {
+            'status': 'stable',
+            'id': 'v2.0',
+        }]
+    )
+
     def _get_conf(self, V2, V3):
         """Creates fake conf for testing purposes"""
         conf = tempest_conf.TempestConf()
@@ -63,14 +82,21 @@ class BaseConfigTempestTest(base.BaseTestCase):
         conf.set("auth", "use_dynamic_credentials", "True")
         return conf
 
-    def _get_creds(self, conf, admin=False):
+    def _get_creds(self, conf, admin=False, v2=False):
+        # We return creds configured to v2 or v3
+        func2mock = 'config_tempest.credentials.Credentials._list_versions'
+        return_value = self.FAKE_V3_VERSIONS
+        if v2:
+            return_value = self.FAKE_V2_VERSIONS
+        mock_function = mock.Mock(return_value=return_value)
+        self.useFixture(MonkeyPatch(func2mock, mock_function))
         return Credentials(conf, admin)
 
     @mock.patch('os_client_config.cloud_config.CloudConfig')
     def _get_clients(self, conf, mock_args, creds=None):
         """Returns ClientManager instance"""
         if creds is None:
-            creds = self._get_creds(conf)
+            creds = self._get_creds(conf, v2=True)
         mock_function = mock.Mock(return_value=False)
         func2mock = 'os_client_config.cloud_config.CloudConfig.config.get'
         self.useFixture(MonkeyPatch(func2mock, mock_function))

config_tempest/tests/services/test_services.py

@@ -37,15 +37,15 @@ class TestServices(BaseConfigTempestTest):
         super(TestServices, self).setUp()
 
     @mock.patch('config_tempest.services.services.Services.discover')
-    def _create_services_instance(self, mock_discover):
+    def _create_services_instance(self, mock_discover, v2=False):
         conf = self._get_conf('v2', 'v3')
-        creds = self._get_creds(conf)
+        creds = self._get_creds(conf, v2=v2)
         clients = mock.Mock()
         services = Services(clients, conf, creds)
         return services
 
     def test_get_endpoints_api_2(self):
-        services = self._create_services_instance()
+        services = self._create_services_instance(v2=True)
         services._region = 'my_region'
         resp = services.get_endpoints(self.FAKE_ENTRY)
         self.assertEqual(resp, self.FAKE_ENTRY['endpoints'][0])
@@ -72,13 +72,13 @@ class TestServices(BaseConfigTempestTest):
         self.assertEqual(resp, [])
 
     def test_set_catalog_and_url(self):
-        services = self._create_services_instance()
         # api version = 2
+        services = self._create_services_instance(v2=True)
         services.set_catalog_and_url()
         self.assertEqual(services.service_catalog, 'serviceCatalog')
         self.assertEqual(services.public_url, 'publicURL')
         # api version = 3
-        services._creds.api_version = 3
+        services = self._create_services_instance()
         services.set_catalog_and_url()
         self.assertEqual(services.service_catalog, 'catalog')
         self.assertEqual(services.public_url, 'url')
@@ -101,10 +101,10 @@ class TestServices(BaseConfigTempestTest):
         url = services.parse_endpoints(ep, "ServiceName")
         self.assertEqual('http://10.0.0.107:8386/v1.1/96409a589d', url)
         ep['url'] = 'https://10.0.0.101/identity'
-        auth_url = 'https://10.0.0.101:13000/v2.0'
+        auth_url = 'https://10.0.0.101:13000/v3.0'
         services._clients.auth_provider.auth_url = auth_url
         url = services.parse_endpoints(ep, 'ServiceName')
-        self.assertEqual('https://10.0.0.101:13000/identity/v2', url)
+        self.assertEqual('https://10.0.0.101:13000/identity/v3', url)
 
     def test_parse_endpoints_not_ip_hostname(self):
         services = self._create_services_instance()
@@ -119,8 +119,24 @@ class TestServices(BaseConfigTempestTest):
         url_resp = services.parse_endpoints(ep, "ServiceName")
         self.assertEqual(url, url_resp)
 
-    def test_edit_identity_url(self):
+    def test_edit_identity_url_v3(self):
         services = self._create_services_instance()
+        url_port = 'https://10.0.0.101:13000/v3.0'
+        identity_url = 'https://10.0.0.101/identity'
+        url_no_port = 'https://10.0.0.101/v333'
+        services._clients.auth_provider.auth_url = url_port
+        url = services.edit_identity_url(url_port)
+        self.assertEqual(url_port, url)
+        url = services.edit_identity_url(identity_url)
+        self.assertEqual("https://10.0.0.101:13000/identity/v3", url)
+        url = services.edit_identity_url(url_no_port)
+        self.assertEqual(url_no_port, url)
+        services._clients.auth_provider.auth_url = url_no_port
+        url = services.edit_identity_url(identity_url)
+        self.assertEqual(identity_url + "/v3", url)
+
+    def test_edit_identity_url_v2(self):
+        services = self._create_services_instance(v2=True)
         url_port = 'https://10.0.0.101:13000/v2.0'
         identity_url = 'https://10.0.0.101/identity'
         url_no_port = 'https://10.0.0.101/v2.0'

config_tempest/tests/test_credentials.py

@@ -27,6 +27,7 @@ class TestCredentials(BaseConfigTempestTest):
         super(TestCredentials, self).setUp()
         self.conf = self._get_conf("v2.0", "v3")
         self.creds = self._get_creds(self.conf)
+        self.creds_v2 = self._get_creds(self.conf, v2=True)
 
     def test_get_credential(self):
         # set conf containing the newer values (admin creds in auth section)
@@ -39,7 +40,7 @@ class TestCredentials(BaseConfigTempestTest):
         self.assertEqual(resp, "admin")
 
     def test_get_identity_version_v2(self):
-        resp = self.creds._get_identity_version()
+        resp = self.creds_v2._get_identity_version()
         self.assertEqual(resp, 'v2')
 
     def test_get_identity_version_v3(self):
@@ -54,8 +55,8 @@ class TestCredentials(BaseConfigTempestTest):
             'password': 'secret',
             'project_name': 'demo'
         }
-        self.assertEqual(self.creds._get_creds_kwargs(), expected_resp)
+        self.assertEqual(self.creds_v2._get_creds_kwargs(), expected_resp)
-        self.creds.identity_version = 'v3'
+        self.creds_v2.identity_version = 'v3'
         expected_resp = {
             'username': 'demo',
             'password': 'secret',
@@ -69,10 +70,10 @@ class TestCredentials(BaseConfigTempestTest):
         mock_function = mock.Mock()
         function2mock = 'config_tempest.credentials.auth.get_credentials'
         self.useFixture(MonkeyPatch(function2mock, mock_function))
-        self.creds.username = "name"
+        self.creds_v2.username = "name"
-        self.creds.password = "pass"
+        self.creds_v2.password = "pass"
-        self.creds.project_name = "Tname"
+        self.creds_v2.project_name = "Tname"
-        self.creds.set_credentials()
+        self.creds_v2.set_credentials()
         mock_function.assert_called_with(
             auth_url=None, fill_in=False, identity_version='v2',
             disable_ssl_certificate_validation='true',
@@ -103,11 +104,11 @@ class TestCredentials(BaseConfigTempestTest):
         # mock V2Provider, if other provider is called, it fails
         func2mock = 'config_tempest.credentials.auth.KeystoneV2AuthProvider'
         self.useFixture(MonkeyPatch(func2mock, mock_function))
-        resp = self.creds.get_auth_provider()
+        resp = self.creds_v2.get_auth_provider()
         self.assertEqual(resp, mock_function())
         # check parameters of returned function
-        self.creds.get_auth_provider()
+        self.creds_v2.get_auth_provider()
-        mock_function.assert_called_with(self.creds.tempest_creds,
+        mock_function.assert_called_with(self.creds_v2.tempest_creds,
                                          'http://172.16.52.151:5000/v2.0',
                                          'true', None)
 

config_tempest/tests/test_users.py

@@ -23,6 +23,9 @@ from tempest.lib import exceptions
 class TestUsers(BaseConfigTempestTest):
 
     def setUp(self):
+        # TODO(arxcruz): All these tests are running on identity v2 only, we
+        # need to create tests for v3 too!
+        # Story 2003388
         super(TestUsers, self).setUp()
         self.conf = self._get_conf("v2.0", "v3")
         projects_client = self._get_clients(self.conf).projects