Add cpid failover

If the client can't get the identity service ID from the cloud,
the client will use a hash of the endpoint's hostname as the CPID.

Change-Id: I0d61ccb05caf40131a2590bd89b6fce7c62cedd2
Implements-Spec: https://review.openstack.org/#/c/255607/

refstack_client/refstack_client.py

@@ -27,6 +27,7 @@ Tempest configuration file.
 import argparse
 import binascii
 import ConfigParser
+import hashlib
 import itertools
 import json
 import logging
@@ -42,6 +43,7 @@ from keystoneclient.v3 import client as ksclient3
 import requests
 import requests.exceptions
 import six.moves
+from six.moves.urllib import parse
 from subunit_processor import SubunitProcessor
 from list_parser import TestListParser
 import yaml
@@ -225,20 +227,32 @@ class RefstackClient:
             # If a Key or Index Error was raised, one of the expected keys or
             # indices for retrieving the identity service ID was not found.
             except (KeyError, IndexError) as e:
-                raise RuntimeError('Unable to retrieve CPID from Keystone %s '
+                self.logger.warning('Unable to retrieve CPID from Keystone %s '
-                                   'catalog. The catalog or the identity '
+                                    'catalog. The catalog or the identity '
-                                   'service endpoint was not '
+                                    'service endpoint was not '
-                                   'found.' % auth_version)
+                                    'found.' % auth_version)
             except Exception as e:
-                self.logger.error('Problems retreiving CPID from Keystone '
+                self.logger.warning('Problems retrieving CPID from Keystone '
-                                  'using %s endpoint' % auth_version)
+                                    'using %s endpoint (%s)' % (auth_version,
-                raise
+                                    args['auth_url']))
 
+            return self._generate_cpid_from_endpoint(args['auth_url'])
         except ConfigParser.Error as e:
             # Most likely a missing section or option in the config file.
             self.logger.error("Invalid Config File: %s" % e)
             exit(1)
 
+    def _generate_cpid_from_endpoint(self, endpoint):
+        '''This method will md5 hash the hostname of a Keystone endpoint to
+           generate a CPID.'''
+        self.logger.info('Creating hash from endpoint to use as CPID.')
+        url_parts = parse.urlparse(endpoint)
+        if url_parts.scheme not in ('http', 'https'):
+            raise ValueError('Invalid Keystone endpoint format. Make sure '
+                             'the endpoint (%s) includes the URL scheme '
+                             '(i.e. http/https).' % endpoint)
+        return hashlib.md5(url_parts.hostname).hexdigest()
+
     def _form_result_content(self, cpid, duration, results):
         '''This method will create the content for the request. The spec at
            github.com/openstack/refstack/blob/master/specs/approved/api-v1.md.

refstack_client/tests/unit/refstack-client.test.conf

@@ -1,10 +1,10 @@
 [identity]
 auth_version = v2
-uri = 0.0.0.0:35357/v2.0
+uri = http://0.0.0.0:35357/v2.0
-uri_v3 = 0.0.0.0:35357/v3
+uri_v3 = http://0.0.0.0:35357/v3
 username = admin
 password = test
 tenant_id = admin_tenant_id
 
 [identity-feature-enabled]
-api_v2 = true
+api_v2 = true

refstack_client/tests/unit/test_client.py

@@ -14,6 +14,7 @@
 #    under the License.
 #
 
+import hashlib
 import json
 import logging
 import os
@@ -174,7 +175,8 @@ class TestRefstackClient(unittest.TestCase):
         cpid = client._get_cpid_from_keystone(client.conf)
         self.ks2_client_builder.assert_called_with(
             username='admin', tenant_id='admin_tenant_id',
-            password='test', auth_url='0.0.0.0:35357/v2.0', insecure=False
+            password='test', auth_url='http://0.0.0.0:35357/v2.0',
+            insecure=False
         )
         self.assertEqual('test-id', cpid)
 
@@ -192,7 +194,8 @@ class TestRefstackClient(unittest.TestCase):
         cpid = client._get_cpid_from_keystone(client.conf)
         self.ks2_client_builder.assert_called_with(
             username='admin', tenant_name='tenant_name',
-            password='test', auth_url='0.0.0.0:35357/v2.0', insecure=False
+            password='test', auth_url='http://0.0.0.0:35357/v2.0',
+            insecure=False
         )
         self.assertEqual('test-id', cpid)
 
@@ -214,7 +217,8 @@ class TestRefstackClient(unittest.TestCase):
         cpid = client._get_cpid_from_keystone(client.conf)
         self.ks2_client_builder.assert_called_with(
             username='admin', tenant_name='tenant_name',
-            password='test', auth_url='0.0.0.0:35357/v2.0', insecure=False
+            password='test', auth_url='http://0.0.0.0:35357/v2.0',
+            insecure=False
         )
         self.assertEqual('test-id', cpid)
 
@@ -247,7 +251,8 @@ class TestRefstackClient(unittest.TestCase):
         cpid = client._get_cpid_from_keystone(client.conf)
         self.ks2_client_builder.assert_called_with(
             username='admin', tenant_id='tenant_id',
-            password='test', auth_url='0.0.0.0:35357/v2.0', insecure=False
+            password='test', auth_url='http://0.0.0.0:35357/v2.0',
+            insecure=False
         )
         self.assertEqual('test-id', cpid)
 
@@ -307,7 +312,8 @@ class TestRefstackClient(unittest.TestCase):
         client._get_cpid_from_keystone(client.conf)
         self.ks2_client_builder.assert_called_with(
             username='admin', tenant_id='admin_tenant_id',
-            password='test', auth_url='0.0.0.0:35357/v2.0', insecure=True
+            password='test', auth_url='http://0.0.0.0:35357/v2.0',
+            insecure=True
         )
 
     def test_get_cpid_from_keystone_v3(self):
@@ -325,7 +331,8 @@ class TestRefstackClient(unittest.TestCase):
         cpid = client._get_cpid_from_keystone(client.conf)
         self.ks3_client_builder.assert_called_with(
             username='admin', tenant_name='tenant_name',
-            password='test', auth_url='0.0.0.0:35357/v3', insecure=False
+            password='test', auth_url='http://0.0.0.0:35357/v3',
+            insecure=False
         )
         self.assertEqual('test-id', cpid)
 
@@ -339,6 +346,8 @@ class TestRefstackClient(unittest.TestCase):
         client.tempest_dir = self.test_path
         client._prep_test()
 
+        client._generate_cpid_from_endpoint = MagicMock()
+
         # Test when the identity endpoints is empty
         self.mock_ks2_client = MagicMock(
             name='ks_client',
@@ -349,8 +358,11 @@ class TestRefstackClient(unittest.TestCase):
             'refstack_client.refstack_client.ksclient2.Client',
             return_value=self.mock_ks2_client
         )
-        with self.assertRaises(RuntimeError):
+        client._get_cpid_from_keystone(client.conf)
-            client._get_cpid_from_keystone(client.conf)
+        # Failover CPID should be generated.
+        client._generate_cpid_from_endpoint.assert_called_with(
+            'http://0.0.0.0:35357/v2.0'
+        )
 
         # Test when the catalog is empty
         self.mock_ks2_client = MagicMock(
@@ -361,8 +373,11 @@ class TestRefstackClient(unittest.TestCase):
             'refstack_client.refstack_client.ksclient2.Client',
             return_value=self.mock_ks2_client
         )
-        with self.assertRaises(RuntimeError):
+        client._get_cpid_from_keystone(client.conf)
-            client._get_cpid_from_keystone(client.conf)
+        # Failover CPID should be generated.
+        client._generate_cpid_from_endpoint.assert_called_with(
+            'http://0.0.0.0:35357/v2.0'
+        )
 
         # Test when there is no service catalog
         self.mock_ks2_client = MagicMock(name='ks_client', **{'auth_ref': {}})
@@ -370,8 +385,11 @@ class TestRefstackClient(unittest.TestCase):
             'refstack_client.refstack_client.ksclient2.Client',
             return_value=self.mock_ks2_client
         )
-        with self.assertRaises(RuntimeError):
+        client._get_cpid_from_keystone(client.conf)
-            client._get_cpid_from_keystone(client.conf)
+        # Failover CPID should be generated.
+        client._generate_cpid_from_endpoint.assert_called_with(
+            'http://0.0.0.0:35357/v2.0'
+        )
 
         # Test when catalog has other non-identity services.
         self.mock_ks2_client = MagicMock(
@@ -402,6 +420,8 @@ class TestRefstackClient(unittest.TestCase):
         client.conf.set('identity', 'tenant_name', 'tenant_name')
         client.conf.set('identity-feature-enabled', 'api_v3', 'true')
 
+        client._generate_cpid_from_endpoint = MagicMock()
+
         # Test when the identity ID is None.
         self.mock_ks3_client = MagicMock(
             name='ks_client',
@@ -411,8 +431,11 @@ class TestRefstackClient(unittest.TestCase):
             'refstack_client.refstack_client.ksclient3.Client',
             return_value=self.mock_ks3_client
         )
-        with self.assertRaises(RuntimeError):
+        client._get_cpid_from_keystone(client.conf)
-            client._get_cpid_from_keystone(client.conf)
+        # Failover CPID should be generated.
+        client._generate_cpid_from_endpoint.assert_called_with(
+            'http://0.0.0.0:35357/v3'
+        )
 
         # Test when the catalog is empty.
         self.mock_ks3_client = MagicMock(
@@ -423,8 +446,11 @@ class TestRefstackClient(unittest.TestCase):
             'refstack_client.refstack_client.ksclient3.Client',
             return_value=self.mock_ks3_client
         )
-        with self.assertRaises(RuntimeError):
+        client._get_cpid_from_keystone(client.conf)
-            client._get_cpid_from_keystone(client.conf)
+        # Failover CPID should be generated.
+        client._generate_cpid_from_endpoint.assert_called_with(
+            'http://0.0.0.0:35357/v3'
+        )
 
         # Test when there is no service catalog.
         self.mock_ks3_client = MagicMock(name='ks_client', **{'auth_ref': {}})
@@ -432,8 +458,11 @@ class TestRefstackClient(unittest.TestCase):
             'refstack_client.refstack_client.ksclient3.Client',
             return_value=self.mock_ks3_client
         )
-        with self.assertRaises(RuntimeError):
+        client._get_cpid_from_keystone(client.conf)
-            client._get_cpid_from_keystone(client.conf)
+        # Failover CPID should be generated.
+        client._generate_cpid_from_endpoint.assert_called_with(
+            'http://0.0.0.0:35357/v3'
+        )
 
         #Test when catalog has other non-identity services.
         self.mock_ks3_client = MagicMock(
@@ -450,6 +479,19 @@ class TestRefstackClient(unittest.TestCase):
         cpid = client._get_cpid_from_keystone(client.conf)
         self.assertEqual('test-id2', cpid)
 
+    def test_generate_cpid_from_endpoint(self):
+        """
+        Test that an endpoint's hostname is properly hashed.
+        """
+        args = rc.parse_cli_args(self.mock_argv())
+        client = rc.RefstackClient(args)
+        cpid = client._generate_cpid_from_endpoint('http://some.url:5000/v2')
+        expected = hashlib.md5('some.url').hexdigest()
+        self.assertEqual(expected, cpid)
+
+        with self.assertRaises(ValueError):
+            client._generate_cpid_from_endpoint('some.url:5000/v2')
+
     def test_form_result_content(self):
         """
         Test that the request content is formed into the expected format.