From 3e005e2606504d18eb135d36fa0035761c806c89 Mon Sep 17 00:00:00 2001 From: Ilya Shakhat Date: Thu, 16 Jan 2014 18:34:41 +0400 Subject: [PATCH] Module groups based on governance program list Process official programs list and add module groups based on it. Module groups are added for every program (like OpenStack Networking), every type of project (like integrated) and all official projects. * Added config parameter program-list-uri * Removed manual group configurations in default_data that duplicate official list * Fixed issue with spaces in module group name in UI Change-Id: I9503d46f009a388aafc27d5ed31e1e8cab40ea2c --- dashboard/vault.py | 4 +-- etc/default_data.json | 58 -------------------------------- etc/stackalytics.conf | 3 ++ etc/test_default_data.json | 2 +- etc/test_programs.yaml | 9 +++++ requirements.txt | 1 + stackalytics/processor/config.py | 4 +++ stackalytics/processor/main.py | 53 +++++++++++++++++++++++++++++ 8 files changed, 73 insertions(+), 61 deletions(-) create mode 100644 etc/test_programs.yaml diff --git a/dashboard/vault.py b/dashboard/vault.py index 0858609a2..9ee7db035 100644 --- a/dashboard/vault.py +++ b/dashboard/vault.py @@ -94,7 +94,7 @@ def init_module_groups(vault): for module_group in module_groups: module_group_name = module_group['module_group_name'] - module_group_id = utils.safe_encode(module_group_name.lower()) + module_group_id = module_group_name.lower() module_id_index[module_group_id] = { 'group': True, @@ -112,7 +112,7 @@ def init_module_groups(vault): for module in memory_storage_inst.get_modules(): module_id_index[module] = { - 'id': utils.safe_encode(module.lower()), + 'id': module.lower(), 'text': module, 'modules': [module.lower()], } diff --git a/etc/default_data.json b/etc/default_data.json index ddff3b448..01dda4ec1 100644 --- a/etc/default_data.json +++ b/etc/default_data.json @@ -6053,72 +6053,14 @@ } ], "module_groups": [ - { - "module_group_name": "openstack-core", - "modules": [ - "nova", "python-novaclient", - "neutron", "python-neutronclient", - "keystone", "python-keystoneclient", - "cinder", "python-cinderclient", - "glance", "python-glanceclient", - "swift", "python-swiftclient", - "horizon" - ] - }, - { - "module_group_name": "openstack-docs", - "modules": ["openstack-manuals", "api-site", "operations-guide", "openstack-doc-tools", - "compute-api", "netconn-api", "identity-api", "volume-api", "image-api"] - }, - { - "module_group_name": "nova-group", - "modules": ["nova", "python-novaclient"] - }, - { - "module_group_name": "neutron-group", - "modules": ["neutron", "python-neutronclient"] - }, - { - "module_group_name": "keystone-group", - "modules": ["keystone", "python-keystoneclient"] - }, - { - "module_group_name": "cinder-group", - "modules": ["cinder", "python-cinderclient"] - }, - { - "module_group_name": "glance-group", - "modules": ["glance", "python-glanceclient"] - }, - { - "module_group_name": "swift-group", - "modules": ["swift", "python-swiftclient"] - }, { "module_group_name": "libra-group", "modules": ["libra", "python-libraclient"] }, - { - "module_group_name": "ironic-group", - "modules": ["ironic", "python-ironicclient", "pyghmi"] - }, - { - "module_group_name": "tripleo-group", - "modules": ["tripleo-incubator", "tripleo-image-elements", "tripleo-heat-templates", "diskimage-builder", - "os-apply-config", "os-collect-config", "os-refresh-config", "tuskar", "tuskar-ui", "python-tuskarclient"] - }, - { - "module_group_name": "oslo-group", - "modules": ["oslo-incubator", "oslo.config", "oslo.messaging", "oslo.sphinx", "hacking", "pbr"] - }, { "module_group_name": "fuel-group", "modules": ["fuel-main", "fuel-web", "fuel-astute", "fuel-ostf", "fuel-library", "fuel-docs", "fuel-devops"] }, - { - "module_group_name": "savanna-group", - "modules": ["savanna", "python-savannaclient", "savanna-dashboard", "savanna-extra", "savanna-image-elements", "puppet-savanna"] - }, { "module_group_name": "murano-group", "modules": ["murano-agent", "python-muranoclient", "murano-api", "murano-common", "murano-conductor", "murano-dashboard", "murano-deployment", "murano-docs", "murano-tests", "murano-metadataclient", "murano-repository"] diff --git a/etc/stackalytics.conf b/etc/stackalytics.conf index b539b1502..73a628c3f 100644 --- a/etc/stackalytics.conf +++ b/etc/stackalytics.conf @@ -31,3 +31,6 @@ # Forcibly read default data and update records # force_update = False + +# The address of file with list of programs +# program_list_uri = https://raw.github.com/openstack/governance/master/reference/programs.yaml diff --git a/etc/test_default_data.json b/etc/test_default_data.json index 84709a218..7d7095dfc 100644 --- a/etc/test_default_data.json +++ b/etc/test_default_data.json @@ -110,7 +110,7 @@ "module_groups": [ { - "module_group_name": "glance-group", + "module_group_name": "manual-glance-group", "modules": ["glance", "python-glanceclient"] } ], diff --git a/etc/test_programs.yaml b/etc/test_programs.yaml new file mode 100644 index 000000000..9e6c828a3 --- /dev/null +++ b/etc/test_programs.yaml @@ -0,0 +1,9 @@ +Image Service: + codename: Glance + ptl: Mark Washenberger (markwash) + url: https://wiki.openstack.org/wiki/Glance + projects: + integrated: + - openstack/glance + other: + - openstack/python-glanceclient diff --git a/requirements.txt b/requirements.txt index 0901f7233..47378a6bf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,5 +8,6 @@ paramiko>=1.8.0 psutil<=1.0.1 PyGithub python-memcached>=1.48 +PyYAML>=3.1.0 sh six>=1.4.1 diff --git a/stackalytics/processor/config.py b/stackalytics/processor/config.py index e8dcd296b..6a8273a0d 100644 --- a/stackalytics/processor/config.py +++ b/stackalytics/processor/config.py @@ -40,4 +40,8 @@ OPTS = [ help='SSH username for gerrit review system access'), cfg.BoolOpt('force-update', default=False, help='Forcibly read default data and update records'), + cfg.StrOpt('program-list-uri', + default=('https://raw.github.com/openstack/governance/' + 'master/reference/programs.yaml'), + help='The address of file with list of programs'), ] diff --git a/stackalytics/processor/main.py b/stackalytics/processor/main.py index 3e256fb9a..7101cd0d5 100644 --- a/stackalytics/processor/main.py +++ b/stackalytics/processor/main.py @@ -13,8 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +import collections + from oslo.config import cfg import psutil +import six +import yaml from stackalytics.openstack.common import log as logging from stackalytics.openstack.common.py3kcompat import urlutils @@ -168,6 +172,53 @@ def apply_corrections(uri, runtime_storage_inst): runtime_storage_inst.apply_corrections(valid_corrections) +def _make_module_group(name, modules): + module_group = {'module_group_name': name, 'modules': modules} + LOG.debug('New module group: %s', module_group) + return module_group + + +def _read_module_groups(program_list_uri): + LOG.debug('Process list of programs from uri: %s', program_list_uri) + content = yaml.safe_load(utils.read_uri(program_list_uri)) + module_groups = [] + modules_by_types = collections.defaultdict(list) + for name, info in six.iteritems(content): + if 'codename' in info: + name += ' (%s)' % info['codename'] + + all_modules = [] + for project_type, project_list in six.iteritems(info['projects']): + module_list = [s.split('/')[1] for s in project_list] + modules_by_types[project_type] += module_list + all_modules += module_list + + module_groups.append(_make_module_group(name, all_modules)) + + all_modules = [] + for project_type, modules_list in six.iteritems(modules_by_types): + all_modules += modules_list + module_groups.append( + _make_module_group('OpenStack ' + project_type.capitalize(), + modules_list)) + module_groups.append(_make_module_group('OpenStack All Official', + all_modules)) + return module_groups + + +def process_program_list(runtime_storage_inst, program_list_uri): + stored_module_groups = runtime_storage_inst.get_by_key('module_groups') + mg_dict = dict([(mg['module_group_name'], mg['modules']) + for mg in stored_module_groups]) + for mg in _read_module_groups(program_list_uri): + mg_dict[mg['module_group_name']] = mg['modules'] + + stored_module_groups = [{'module_group_name': name, 'modules': modules} + for name, modules in six.iteritems(mg_dict)] + + runtime_storage_inst.set_by_key('module_groups', stored_module_groups) + + def main(): # init conf and logging conf = cfg.CONF @@ -190,6 +241,8 @@ def main(): cfg.CONF.sources_root, cfg.CONF.force_update) + process_program_list(runtime_storage_inst, cfg.CONF.program_list_uri) + update_pids(runtime_storage_inst) update_records(runtime_storage_inst)