x/stackalytics
Code Issues Proposed changes

Support of a new format of projects.yaml

Browse Source 
Recently the program definition file was changed in governance repo.
The new format is project oriented, incubation status is removed and
integration status is specified as tag.

Closes bug 1417385

Change-Id: I06d8c27d7afac2660f09175e17173b8e089485e1
This commit is contained in:
Ilya Shakhat 2015-02-06 16:04:36 +03:00
parent 1fca64ab72
commit 570887cd57
10 changed files with 53 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
etc/default_data.json
View File

@ -11694,7 +11694,7 @@
{
"id": "openstack",
"title": "OpenStack",
"modules": ["official-integrated", "official-incubated", "official-other", "governance", "python-openstackclient", "api-wg"]
"modules": ["official-integrated", "official-other", "governance", "python-openstackclient", "api-wg"]
},
{
"id": "integrated",
@ -11702,12 +11702,6 @@
"child": true,
"modules": ["official-integrated"]
},
{
"id": "incubated",
"title": "incubated",
"child": true,
"modules": ["official-incubated"]
},
{
"id": "other",
"title": "other",

4
etc/stackalytics.conf
View File

@ -38,8 +38,8 @@
# Forcibly read default data and update records
# force_update = False
# The address of file with list of programs
# program_list_uri = https://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml
# The address of file with the official projects list
# project_list_uri = https://git.openstack.org/cgit/openstack/governance/plain/reference/projects.yaml
# The address of DriverLog data
# driverlog_data_uri = https://git.openstack.org/cgit/stackforge/driverlog/plain/etc/default_data.json

8
etc/test_default_data.json
View File

@ -183,7 +183,7 @@
{
"id": "openstack",
"title": "OpenStack",
"modules": ["openstack", "openstack-infra", "openstack-dev"]
"modules": ["official-integrated", "official-other", "governance", "python-openstackclient", "api-wg"]
},
{
"id": "integrated",
@ -191,12 +191,6 @@
"child": true,
"modules": ["official-integrated"]
},
{
"id": "incubated",
"title": "incubated",
"child": true,
"modules": ["official-incubated"]
},
{
"id": "other",
"title": "other",

9
etc/test_programs.yaml
View File

@ -1,9 +0,0 @@
Image Service:
codename: Glance
ptl: Mark Washenberger (markwash)
url: https://wiki.openstack.org/wiki/Glance
projects:
- repo: openstack/glance
integrated-since: havana
incubated-since: grizzly
- repo: openstack/python-glanceclient

16
etc/test_projects.yaml Normal file
View File

@ -0,0 +1,16 @@
Sahara:
ptl: Sergey Lukjanov (SergeyLukjanov)
mission: >
To provide a scalable data processing stack and associated management
interfaces.
url: https://wiki.openstack.org/wiki/Sahara
projects:
- repo: openstack/sahara
tags:
- name: integrated-release
since: juno
- repo: openstack/python-saharaclient
- repo: openstack/sahara-dashboard
- repo: openstack/sahara-extra
- repo: openstack/sahara-image-elements
- repo: openstack/sahara-specs

4
stackalytics/dashboard/templates/_macros/module_details.html
View File

@ -28,8 +28,8 @@
<h2>${name}</h2>
<div style="font-style: italic; margin-bottom: 0.5em;">
{%if tag == 'program' %}
The official OpenStack program as defined in
<a href="https://git.openstack.org/cgit/openstack/governance/tree/reference/programs.yaml" target="_blank">programs.yaml</a>
The official OpenStack project as defined in
<a href="https://git.openstack.org/cgit/openstack/governance/tree/reference/projects.yaml" target="_blank">projects.yaml</a>
{%/if%}
{%if tag == 'group' %}
Custom module group as defined in

4
stackalytics/dashboard/templates/layout.html
View File

@ -54,12 +54,12 @@ Stackalytics {% if page_title %}| {{ page_title }} {% endif %}
</div>
<div class="drop">
<label for="project_type_selector" title="Project type groups modules of same kind: official (integrated, incubated, other) or belonging to same organization (stackforge, infra)">Project Type</label>
<label for="project_type_selector" title="Project type groups modules of same kind: official (integrated-release, other) or belonging to same organization (openstack, stackforge)">Project Type</label>
<input type="hidden" id="project_type_selector" style="width: 140px" data-placeholder="Select project type"/>
</div>
<div class="drop">
<label for="module_selector" title="Module represents a repo (black), official program (violet) or pre-configured group of modules (cyan)">Module</label>
<label for="module_selector" title="Module represents a repo (black), official project (violet) or pre-configured group of modules (cyan)">Module</label>
<input type="hidden" id="module_selector" style="width: 140px" data-placeholder="Any module"/>
</div>

2
stackalytics/dashboard/templates/widget.html
View File

@ -121,7 +121,7 @@
</div>
<div class="drop" style="margin-right: 15px;">
<label for="project_type_selector" title="Project type groups modules of same kind: official (integrated, incubated, other) or belonging to same organization (stackforge, infra)">Project Type</label>
<label for="project_type_selector" title="Project type groups modules of same kind: official (integrated-release, other) or belonging to same organization (openstack, stackforge)">Project Type</label>
<input type="hidden" id="project_type_selector" style="width: 95px" data-placeholder="Select project type"/>
</div>

6
stackalytics/processor/config.py
View File

@ -45,10 +45,10 @@ OPTS = [
help='SSH username for gerrit review system access'),
cfg.BoolOpt('force-update', default=False,
help='Forcibly read default data and update records'),
cfg.StrOpt('program-list-uri',
cfg.StrOpt('project-list-uri',
default=('https://git.openstack.org/cgit/'
'openstack/governance/plain/reference/programs.yaml'),
help='The address of file with list of programs'),
'openstack/governance/plain/reference/projects.yaml'),
help='The address of file with the official projects list'),
cfg.StrOpt('driverlog-data-uri',
default='https://git.openstack.org/cgit/'
'stackforge/driverlog/plain/etc/default_data.json',

41
stackalytics/processor/main.py
View File

@ -240,30 +240,24 @@ def apply_corrections(uri, runtime_storage_inst):
runtime_storage_inst.apply_corrections(valid_corrections)
def _read_official_programs_yaml(program_list_uri, release_names):
LOG.debug('Process list of programs from uri: %s', program_list_uri)
content = yaml.safe_load(utils.read_uri(program_list_uri))
def _read_official_projects_yaml(project_list_uri, release_names):
LOG.debug('Process list of projects from uri: %s', project_list_uri)
content = yaml.safe_load(utils.read_uri(project_list_uri))
module_groups = collections.defaultdict(
lambda: {'modules': [], 'releases': collections.defaultdict(list)})
official_integrated = module_groups['official-integrated']
official_integrated['tag'] = 'project_type'
official_integrated['module_group_name'] = 'official-integrated'
official_incubated = module_groups['official-incubated']
official_incubated['tag'] = 'project_type'
official_incubated['module_group_name'] = 'official-incubated'
official_other = module_groups['official-other']
official_other['tag'] = 'project_type'
official_other['module_group_name'] = 'official-other'
for name, info in six.iteritems(content):
# for one program
group_id = name.lower()
if 'codename' in info:
name = '%s (%s)' % (info['codename'], name)
group_id = '%s-group' % info['codename'].lower()
# take one official project
module_groups[group_id]['module_group_name'] = name
group_id = '%s-group' % name.lower()
module_groups[group_id]['module_group_name'] = '%s Official' % name
module_groups[group_id]['tag'] = 'program'
for module in info['projects']:
@ -271,17 +265,22 @@ def _read_official_programs_yaml(program_list_uri, release_names):
module_groups[group_id]['modules'].append(module_name)
if ('integrated-since' in module) or ('incubated-since' in module):
type_matched = False
if 'tags' in module:
for tag in module.get('tags'):
tag_name = tag.get('name')
if tag_name == 'integrated-release':
type_matched = True # project type is matched here
project_type = 'official-other'
for release_name in release_names:
if release_name == module.get('incubated-since'):
project_type = 'official-incubated'
elif release_name == module.get('integrated-since'):
if release_name == tag.get('since'):
project_type = 'official-integrated'
module_groups[project_type]['releases'][
release_name].append(module_name)
else:
if not type_matched:
module_groups['official-other']['modules'].append(module_name)
# set ids for module groups
@ -291,13 +290,13 @@ def _read_official_programs_yaml(program_list_uri, release_names):
return module_groups
def process_program_list(runtime_storage_inst, program_list_uri):
def process_project_list(runtime_storage_inst, project_list_uri):
module_groups = runtime_storage_inst.get_by_key('module_groups') or {}
release_names = [r['release_name'].lower()
for r in runtime_storage_inst.get_by_key('releases')[1:]]
official_module_groups = _read_official_programs_yaml(
program_list_uri, release_names)
official_module_groups = _read_official_projects_yaml(
project_list_uri, release_names)
LOG.debug('Update module groups with official: %s', official_module_groups)
module_groups.update(official_module_groups)
@ -335,7 +334,7 @@ def main():
default_data,
cfg.CONF.driverlog_data_uri)
process_program_list(runtime_storage_inst, cfg.CONF.program_list_uri)
process_project_list(runtime_storage_inst, cfg.CONF.project_list_uri)
update_pids(runtime_storage_inst)