openinfra/refstack-client
Code Issues Proposed changes

Add ability to upload test results

Browse Source 
Split refstack-client into two subcommands: upload and test.
The 'test' command will run Tempest like before, while 'upload'
takes in a result file argument and will upload it to the Refstack
API url. Testing with refstack-client now saves a JSON result file
containing the content of what will or would be posted to the
Refstack API.

Change-Id: I28bcb75b51b77872f39e144ae8ffa7e64b26b233
This commit is contained in:
Paul Van Eck 2014-10-14 16:44:12 -07:00
parent 6e6ced6436
commit 6c3f3a1a02
5 changed files with 194 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
README.rst
View File

@ -5,13 +5,13 @@ refstack-client is a command line utility that allows you to execute Tempest
test runs based on configurations you specify. When finished running Tempest test runs based on configurations you specify. When finished running Tempest
it sends the passed test data back to the Refstack API server. it sends the passed test data back to the Refstack API server.
**Usage** **Setup**
We've created an "easy button" for Ubuntu, Centos, RHEL and openSuSe. We've created an "easy button" for Ubuntu, Centos, RHEL and openSuSe.
$ ./setup_env.sh $ ./setup_env
**Start testing** **Usage**
1. Prepare a tempest configuration file that is customized to your cloud 1. Prepare a tempest configuration file that is customized to your cloud
environment. environment.
@ -21,10 +21,9 @@ $ ./setup_env.sh
source .venv/bin/activate source .venv/bin/activate
4. Execute test by typing: 4. Test your cloud by typing:
./refstack-client -c "Path of the tempest configuration file\ ./refstack-client test -c <Path of the tempest configuration file to use>
to use"
**Note:** **Note:**
@ -33,5 +32,21 @@ $ ./setup_env.sh
c. Adding -t option will only test a particular test case or a test group. c. Adding -t option will only test a particular test case or a test group.
This option can be used for quick verification of the target test cases This option can be used for quick verification of the target test cases
(i.e. -t "tempest.api.identity.admin.test_roles"). (i.e. -t "tempest.api.identity.admin.test_roles").
d. Adding --url option will upload the test results to a Refstack API server d. Adding --url option will upload the test results to the specified
instead of the default Refstack API server. Refstack API server instead of the default Refstack API server.
server instead of the default Refstack API server.
e. Adding --offline option will have your test results not be uploaded.
**Upload:**
If you previously ran a test with refstack-client using the --offline
option, you can upload your results to a Refstack API server by using the
following command:
./refstack-client upload <Path of results file>
The results file is a JSON file generated by refstack-client when a test has
completed. This is saved in .venv/src/tempest/.testrepository. When you use
the 'upload' command, you can also override the Refstack API server
uploaded to with the --url option.

2
refstack-client
View File

@ -28,4 +28,4 @@ from refstack_client import refstack_client
if __name__ == '__main__': if __name__ == '__main__':
args = refstack_client.parse_cli_args() args = refstack_client.parse_cli_args()
test = refstack_client.RefstackClient(args) test = refstack_client.RefstackClient(args)
test.run() getattr(test, args.func)()

141
refstack_client/refstack_client.py
View File

@ -48,16 +48,17 @@ class RefstackClient:
logging.Formatter(self.log_format)) logging.Formatter(self.log_format))
self.logger.addHandler(self.console_log_handle) self.logger.addHandler(self.console_log_handle)
if args.verbose > 1: self.args = args
if self.args.verbose > 1:
self.logger.setLevel(logging.DEBUG) self.logger.setLevel(logging.DEBUG)
elif args.verbose == 1: elif self.args.verbose == 1:
self.logger.setLevel(logging.INFO) self.logger.setLevel(logging.INFO)
else: else:
self.logger.setLevel(logging.ERROR) self.logger.setLevel(logging.ERROR)
self.args = args def _prep_test(self):
self.tempest_script = os.path.join(self.args.tempest_dir, '''Prepare a tempest test against a cloud.'''
'run_tempest.sh')
# Check that the config file exists. # Check that the config file exists.
if not os.path.isfile(self.args.conf_file): if not os.path.isfile(self.args.conf_file):
@ -76,6 +77,16 @@ class RefstackClient:
self.conf_file = self.args.conf_file self.conf_file = self.args.conf_file
self.conf = ConfigParser.SafeConfigParser() self.conf = ConfigParser.SafeConfigParser()
self.conf.read(self.args.conf_file) self.conf.read(self.args.conf_file)
self.tempest_script = os.path.join(self.args.tempest_dir,
'run_tempest.sh')
def _prep_upload(self):
'''Prepare an upload to the Refstack_api'''
if not os.path.isfile(self.args.file):
self.logger.error("File not valid: %s" % self.args.file)
exit(1)
self.logger.setLevel(logging.DEBUG)
self.upload_file = self.args.file
def _get_next_stream_subunit_output_file(self, tempest_dir): def _get_next_stream_subunit_output_file(self, tempest_dir):
'''This method reads from the next-stream file in the .testrepository '''This method reads from the next-stream file in the .testrepository
@ -131,6 +142,12 @@ class RefstackClient:
content['results'] = results content['results'] = results
return content return content
def _save_json_results(self, results, path):
'''Save the output results from the Tempest run as a JSON file'''
file = open(path, "w+")
file.write(json.dumps(results, indent=4, separators=(',', ': ')))
file.close()
def get_passed_tests(self, result_file): def get_passed_tests(self, result_file):
'''Get a list of tests IDs that passed Tempest from a subunit file.''' '''Get a list of tests IDs that passed Tempest from a subunit file.'''
subunit_processor = SubunitProcessor(result_file) subunit_processor = SubunitProcessor(result_file)
@ -145,8 +162,9 @@ class RefstackClient:
json_content = json.dumps(content) json_content = json.dumps(content)
self.logger.debug('API request content: %s ' % json_content) self.logger.debug('API request content: %s ' % json_content)
def run(self): def test(self):
'''Execute tempest test against the cloud.''' '''Execute Tempest test against the cloud.'''
self._prep_test()
results_file = self._get_next_stream_subunit_output_file( results_file = self._get_next_stream_subunit_output_file(
self.args.tempest_dir) self.args.tempest_dir)
cpid = self._get_cpid_from_keystone(self.conf) cpid = self._get_cpid_from_keystone(self.conf)
@ -189,6 +207,11 @@ class RefstackClient:
results = self.get_passed_tests(results_file) results = self.get_passed_tests(results_file)
self.logger.info("Number of passed tests: %d" % len(results)) self.logger.info("Number of passed tests: %d" % len(results))
content = self._form_result_content(cpid, duration, results)
json_path = results_file + ".json"
self._save_json_results(content, json_path)
self.logger.info('JSON results saved in: %s' % json_path)
# If the user did not specify the offline argument, then upload # If the user did not specify the offline argument, then upload
# the results. # the results.
if not self.args.offline: if not self.args.offline:
@ -198,50 +221,82 @@ class RefstackClient:
self.logger.error("Problem executing Tempest script. Exit code %d", self.logger.error("Problem executing Tempest script. Exit code %d",
process.returncode) process.returncode)
def upload(self):
'''Perform upload to Refstack URL.'''
self._prep_upload()
json_file = open(self.upload_file)
json_data = json.load(json_file)
json_file.close()
self.post_results(self.args.url, json_data)
def parse_cli_args(args=None): def parse_cli_args(args=None):
parser = argparse.ArgumentParser(description='Starts a tempest test', usage_string = ('refstack-client [-h] {upload,test} ...\n\n'
'To see help on specific argument, do:\n'
'refstack-client <ARG> -h')
parser = argparse.ArgumentParser(description='Refstack-client arguments',
formatter_class=argparse. formatter_class=argparse.
ArgumentDefaultsHelpFormatter) ArgumentDefaultsHelpFormatter,
usage=usage_string)
parser.add_argument('-v', '--verbose', subparsers = parser.add_subparsers(help='Available subcommands.')
action='count',
help='Show verbose output. Note that -vv will show '
'Tempest test result output.')
parser.add_argument('--offline', # Arguments that go with all subcommands.
action='store_true', shared_args = argparse.ArgumentParser(add_help=False)
help='Do not upload test results after running ' shared_args.add_argument('-v', '--verbose',
'Tempest.') action='count',
help='Show verbose output.')
parser.add_argument('--url', shared_args.add_argument('--url',
action='store', action='store',
required=False, required=False,
default='https://api.refstack.org', default='https://api.refstack.org',
type=str, type=str,
help='Refstack API URL to post results to (e.g. --url ' help='Refstack API URL to upload results to '
'https://127.0.0.1:8000).') '(--url https://127.0.0.1:8000).')
parser.add_argument('--tempest-dir', # Upload command
action='store', parser_upload = subparsers.add_parser('upload', parents=[shared_args],
required=False, help='Upload an existing result '
dest='tempest_dir', 'file. ')
default='.venv/src/tempest', parser_upload.add_argument('file',
help='Path of the tempest project directory.') type=str,
help='Path of JSON results file.')
parser_upload.set_defaults(func="upload")
parser.add_argument('-c', '--conf-file', # Test command
action='store', parser_test = subparsers.add_parser('test', parents=[shared_args],
required=True, help='Run Tempest against a cloud.')
dest='conf_file',
type=str, parser_test.add_argument('--tempest-dir',
help='Path of the tempest configuration file to use.') action='store',
required=False,
dest='tempest_dir',
default='.venv/src/tempest',
help='Path of the Tempest project directory.')
parser_test.add_argument('-c', '--conf-file',
action='store',
required=True,
dest='conf_file',
type=str,
help='Path of the Tempest configuration file to '
'use.')
parser_test.add_argument('-t', '--test-cases',
action='store',
required=False,
dest='test_cases',
type=str,
help='Specify a subset of test cases to run '
'(e.g. --test-cases tempest.api.compute).')
parser_test.add_argument('--offline',
action='store_true',
help='Do not upload test results after running '
'Tempest.')
parser_test.set_defaults(func="test")
parser.add_argument('-t', '--test-cases',
action='store',
required=False,
dest='test_cases',
type=str,
help='Specify a subset of test cases to run '
'(e.g. --test-cases tempest.api.compute).')
return parser.parse_args(args=args) return parser.parse_args(args=args)

8
refstack_client/tests/unit/.testrepository/0.json Normal file
View File

@ -0,0 +1,8 @@
{
"duration_seconds": 0,
"cpid": "test-id",
"results": [
"tempest.passed.test"
]
}

70
refstack_client/tests/unit/test_client.py
View File

@ -15,7 +15,9 @@
# #
import logging import logging
import json
import os import os
import tempfile
import mock import mock
from mock import MagicMock from mock import MagicMock
@ -51,7 +53,8 @@ class TestRefstackClient(unittest.TestCase):
conf_file_name = self.conf_file_name conf_file_name = self.conf_file_name
if tempest_dir is None: if tempest_dir is None:
tempest_dir = self.test_path tempest_dir = self.test_path
argv = ['-c', conf_file_name, argv = ['test',
'-c', conf_file_name,
'--tempest-dir', tempest_dir, '--tempest-dir', tempest_dir,
'--test-cases', 'tempest.api.compute', '--test-cases', 'tempest.api.compute',
'--url', '0.0.0.0'] '--url', '0.0.0.0']
@ -86,14 +89,17 @@ class TestRefstackClient(unittest.TestCase):
""" """
args = rc.parse_cli_args(self.mock_argv()) args = rc.parse_cli_args(self.mock_argv())
client = rc.RefstackClient(args) client = rc.RefstackClient(args)
client._prep_test()
self.assertEqual(client.logger.level, logging.ERROR) self.assertEqual(client.logger.level, logging.ERROR)
args = rc.parse_cli_args(self.mock_argv(verbose='-v')) args = rc.parse_cli_args(self.mock_argv(verbose='-v'))
client = rc.RefstackClient(args) client = rc.RefstackClient(args)
client._prep_test()
self.assertEqual(client.logger.level, logging.INFO) self.assertEqual(client.logger.level, logging.INFO)
args = rc.parse_cli_args(self.mock_argv(verbose='-vv')) args = rc.parse_cli_args(self.mock_argv(verbose='-vv'))
client = rc.RefstackClient(args) client = rc.RefstackClient(args)
client._prep_test()
self.assertEqual(client.logger.level, logging.DEBUG) self.assertEqual(client.logger.level, logging.DEBUG)
def test_get_next_stream_subunit_output_file(self): def test_get_next_stream_subunit_output_file(self):
@ -128,6 +134,7 @@ class TestRefstackClient(unittest.TestCase):
""" """
args = rc.parse_cli_args(self.mock_argv()) args = rc.parse_cli_args(self.mock_argv())
client = rc.RefstackClient(args) client = rc.RefstackClient(args)
client._prep_test()
self.mock_keystone() self.mock_keystone()
cpid = client._get_cpid_from_keystone(client.conf) cpid = client._get_cpid_from_keystone(client.conf)
self.ks_client_builder.assert_called_with( self.ks_client_builder.assert_called_with(
@ -142,6 +149,7 @@ class TestRefstackClient(unittest.TestCase):
""" """
args = rc.parse_cli_args(self.mock_argv()) args = rc.parse_cli_args(self.mock_argv())
client = rc.RefstackClient(args) client = rc.RefstackClient(args)
client._prep_test()
client.conf.remove_option('identity', 'admin_tenant_id') client.conf.remove_option('identity', 'admin_tenant_id')
client.conf.set('identity', 'admin_tenant_name', 'admin_tenant_name') client.conf.set('identity', 'admin_tenant_name', 'admin_tenant_name')
self.mock_keystone() self.mock_keystone()
@ -159,6 +167,7 @@ class TestRefstackClient(unittest.TestCase):
""" """
args = rc.parse_cli_args(self.mock_argv(verbose='-vv')) args = rc.parse_cli_args(self.mock_argv(verbose='-vv'))
client = rc.RefstackClient(args) client = rc.RefstackClient(args)
client._prep_test()
client.conf.remove_option('identity', 'admin_tenant_id') client.conf.remove_option('identity', 'admin_tenant_id')
self.assertRaises(SystemExit, client._get_cpid_from_keystone, self.assertRaises(SystemExit, client._get_cpid_from_keystone,
client.conf) client.conf)
@ -175,6 +184,25 @@ class TestRefstackClient(unittest.TestCase):
'results': ['tempest.sample.test']} 'results': ['tempest.sample.test']}
self.assertEqual(expected, content) self.assertEqual(expected, content)
def test_save_json_result(self):
"""
Test that the results are properly written to a JSON file.
"""
args = rc.parse_cli_args(self.mock_argv())
client = rc.RefstackClient(args)
results = {'cpid': 1,
'duration_seconds': 1,
'results': ['tempest.sample.test']}
temp_file = tempfile.NamedTemporaryFile()
client._save_json_results(results, temp_file.name)
# Get the JSON that was written to the file and make sure it
# matches the expected value.
json_file = open(temp_file.name)
json_data = json.load(json_file)
json_file.close()
self.assertEqual(results, json_data)
def test_get_passed_tests(self): def test_get_passed_tests(self):
""" """
Test that only passing tests are retrieved from a subunit file. Test that only passing tests are retrieved from a subunit file.
@ -201,7 +229,7 @@ class TestRefstackClient(unittest.TestCase):
client.get_passed_tests = MagicMock(return_value=['test']) client.get_passed_tests = MagicMock(return_value=['test'])
client.post_results = MagicMock() client.post_results = MagicMock()
client._save_json_results = MagicMock() client._save_json_results = MagicMock()
client.run() client.test()
mock_popen.assert_called_with( mock_popen.assert_called_with(
('%s/run_tempest.sh' % self.test_path, '-C', self.conf_file_name, ('%s/run_tempest.sh' % self.test_path, '-C', self.conf_file_name,
'-N', '-t', '--', 'tempest.api.compute'), '-N', '-t', '--', 'tempest.api.compute'),
@ -229,7 +257,7 @@ class TestRefstackClient(unittest.TestCase):
client.get_passed_tests = MagicMock(return_value=['test']) client.get_passed_tests = MagicMock(return_value=['test'])
client.post_results = MagicMock() client.post_results = MagicMock()
client._save_json_results = MagicMock() client._save_json_results = MagicMock()
client.run() client.test()
mock_popen.assert_called_with( mock_popen.assert_called_with(
('%s/run_tempest.sh' % self.test_path, '-C', self.conf_file_name, ('%s/run_tempest.sh' % self.test_path, '-C', self.conf_file_name,
'-N', '-t', '--', 'tempest.api.compute'), '-N', '-t', '--', 'tempest.api.compute'),
@ -245,14 +273,16 @@ class TestRefstackClient(unittest.TestCase):
Test when a nonexistent configuration file is passed in. Test when a nonexistent configuration file is passed in.
""" """
args = rc.parse_cli_args(self.mock_argv(conf_file_name='ptn-khl')) args = rc.parse_cli_args(self.mock_argv(conf_file_name='ptn-khl'))
self.assertRaises(SystemExit, rc.RefstackClient, args) client = rc.RefstackClient(args)
self.assertRaises(SystemExit, client.test)
def test_run_tempest_nonexisting_directory(self): def test_run_tempest_nonexisting_directory(self):
""" """
Test when a nonexistent Tempest directory is passed in. Test when a nonexistent Tempest directory is passed in.
""" """
args = rc.parse_cli_args(self.mock_argv(tempest_dir='/does/not/exist')) args = rc.parse_cli_args(self.mock_argv(tempest_dir='/does/not/exist'))
self.assertRaises(SystemExit, rc.RefstackClient, args) client = rc.RefstackClient(args)
self.assertRaises(SystemExit, client.test)
def test_failed_run(self): def test_failed_run(self):
""" """
@ -264,5 +294,33 @@ class TestRefstackClient(unittest.TestCase):
args = rc.parse_cli_args(self.mock_argv(verbose='-vv')) args = rc.parse_cli_args(self.mock_argv(verbose='-vv'))
client = rc.RefstackClient(args) client = rc.RefstackClient(args)
client.logger.error = MagicMock() client.logger.error = MagicMock()
client.run() client.test()
self.assertTrue(client.logger.error.called) self.assertTrue(client.logger.error.called)
def test_upload(self):
"""
Test that the upload command runs as expected.
"""
upload_file_path = self.test_path + "/.testrepository/0.json"
args = rc.parse_cli_args(['upload', upload_file_path,
'--url', 'http://api.test.org'])
client = rc.RefstackClient(args)
client.post_results = MagicMock()
client.upload()
expected_json = {'duration_seconds': 0,
'cpid': 'test-id',
'results': ['tempest.passed.test']}
client.post_results.assert_called_with('http://api.test.org',
expected_json)
def test_upload_nonexisting_file(self):
"""
Test that the upload file does not exist
"""
upload_file_path = self.test_path + "/.testrepository/foo.json"
args = rc.parse_cli_args(['upload', upload_file_path,
'--url', 'http://api.test.org'])
client = rc.RefstackClient(args)
self.assertRaises(SystemExit, client.upload)