diff --git a/dashboard/memory_storage.py b/dashboard/memory_storage.py index 769e54818..6da2ef56e 100644 --- a/dashboard/memory_storage.py +++ b/dashboard/memory_storage.py @@ -20,8 +20,9 @@ class CachedMemoryStorage(MemoryStorage): self.release_index = {} self.dates = [] for record in records: - self.records[record['record_id']] = record - self.index(record) + if record['company_name'] != '*robots': # ignore robots + self.records[record['record_id']] = record + self.index(record) self.dates = sorted(self.date_index) self.company_name_mapping = dict((c.lower(), c) for c in self.company_index.keys()) diff --git a/dashboard/templates/base.html b/dashboard/templates/base.html index 6d8d55b4e..f5940f1dc 100644 --- a/dashboard/templates/base.html +++ b/dashboard/templates/base.html @@ -133,7 +133,7 @@
-

Stackalytics | {{ self.title() }}

+

Stackalytics | {{ self.title() }}

Community heartbeat

diff --git a/dashboard/templates/engineer_details.html b/dashboard/templates/engineer_details.html index 811047abc..9578a701d 100644 --- a/dashboard/templates/engineer_details.html +++ b/dashboard/templates/engineer_details.html @@ -26,7 +26,7 @@

Commits history

{% if not commits %} -
There are no commits for selected period or project type.
+
There are no commits for selected release or project type.
{% endif %} {% for rec in commits %} diff --git a/dashboard/templates/layout.html b/dashboard/templates/layout.html index 314976293..fb970a991 100644 --- a/dashboard/templates/layout.html +++ b/dashboard/templates/layout.html @@ -117,7 +117,11 @@ } else { index++; } - var link = make_link(link_prefix, data[i].id, data[i].name); + if (data[i].id) { + var link = make_link(link_prefix, data[i].id, data[i].name); + } else { + var link = data[i].name + } tableData.push({"index": index_label, "link": link, "metric": data[i].metric}); } @@ -197,12 +201,12 @@ {# if (getRelease() != 'havana') {#} options['release'] = getRelease(); {# }#} - if (getMetric() != 'loc') { +{# if (getMetric() != 'loc') {#} options['metric'] = getMetric(); - } - if (getProjectType() != 'incubation') { +{# }#} +{# if (getProjectType() != 'incubation') {#} options['project_type'] = getProjectType(); - } +{# }#} return options; } @@ -249,9 +253,8 @@
diff --git a/dashboard/templates/module_details.html b/dashboard/templates/module_details.html index 271820eb7..f9dc4222b 100644 --- a/dashboard/templates/module_details.html +++ b/dashboard/templates/module_details.html @@ -6,8 +6,8 @@ {% block scripts %} {% endblock %} diff --git a/dashboard/web.py b/dashboard/web.py index 79ca50946..607bb2901 100644 --- a/dashboard/web.py +++ b/dashboard/web.py @@ -1,3 +1,18 @@ +# Copyright (c) 2013 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import cgi import datetime import functools @@ -41,6 +56,9 @@ def get_vault(): releases = vault['persistent_storage'].get_releases() vault['releases'] = dict((r['release_name'].lower(), r) for r in releases) + modules = vault['persistent_storage'].get_repos() + vault['modules'] = dict((r['module'].lower(), + r['project_type'].lower()) for r in modules) app.stackalytics_vault = vault return vault @@ -49,7 +67,32 @@ def get_memory_storage(): return get_vault()['memory_storage'] -def record_filter(parameter_getter=lambda x: flask.request.args.get(x)): +def get_default(param_name): + if param_name in DEFAULTS: + return DEFAULTS[param_name] + else: + return None + + +def get_parameter(kwargs, singular_name, plural_name, use_default=True): + if singular_name in kwargs: + p = kwargs[singular_name] + else: + p = (flask.request.args.get(singular_name) or + flask.request.args.get(plural_name)) + if p: + return p.split(',') + elif use_default: + default = get_default(singular_name) + return [default] if default else None + else: + return [] + + +def record_filter(ignore=None, use_default=True): + if not ignore: + ignore = [] + def decorator(f): @functools.wraps(f) def decorated_function(*args, **kwargs): @@ -58,34 +101,42 @@ def record_filter(parameter_getter=lambda x: flask.request.args.get(x)): memory_storage = vault['memory_storage'] record_ids = memory_storage.get_record_ids() - param = parameter_getter('modules') - if param: - record_ids &= memory_storage.get_record_ids_by_modules( - param.split(',')) + if 'module' not in ignore: + param = get_parameter(kwargs, 'module', 'modules', use_default) + if param: + record_ids &= ( + memory_storage.get_record_ids_by_modules(param)) - if 'launchpad_id' in kwargs: - param = kwargs['launchpad_id'] - else: - param = (parameter_getter('launchpad_id') or - parameter_getter('launchpad_ids')) - if param: - record_ids &= memory_storage.get_record_ids_by_launchpad_ids( - param.split(',')) + if 'project_type' not in ignore: + param = get_parameter(kwargs, 'project_type', 'project_types', + use_default) + if param: + modules = [module for module, project_type + in vault['modules'].iteritems() + if project_type in param] + record_ids &= ( + memory_storage.get_record_ids_by_modules(modules)) - if 'company' in kwargs: - param = kwargs['company'] - else: - param = (parameter_getter('company') or - parameter_getter('companies')) - if param: - record_ids &= memory_storage.get_record_ids_by_companies( - param.split(',')) + if 'launchpad_id' not in ignore: + param = get_parameter(kwargs, 'launchpad_id', 'launchpad_ids') + if param: + record_ids &= ( + memory_storage.get_record_ids_by_launchpad_ids(param)) - param = parameter_getter('release') or parameter_getter('releases') - if param: - if param != 'all': - record_ids &= memory_storage.get_record_ids_by_releases( - c.lower() for c in param.split(',')) + if 'company' not in ignore: + param = get_parameter(kwargs, 'company', 'companies') + if param: + record_ids &= ( + memory_storage.get_record_ids_by_companies(param)) + + if 'release' not in ignore: + param = get_parameter(kwargs, 'release', 'releases', + use_default) + if param: + if 'all' not in param: + record_ids &= ( + memory_storage.get_record_ids_by_releases( + c.lower() for c in param)) kwargs['records'] = memory_storage.get_records(record_ids) return f(*args, **kwargs) @@ -100,14 +151,15 @@ def aggregate_filter(): @functools.wraps(f) def decorated_function(*args, **kwargs): - metric_filter = lambda r: r['loc'] - metric_param = flask.request.args.get('metric') - if metric_param: - metric = metric_param.lower() - if metric == 'commits': - metric_filter = lambda r: 1 - elif metric != 'loc': - raise Exception('Invalid metric %s' % metric) + metric_param = (flask.request.args.get('metric') or + DEFAULTS['metric']) + metric = metric_param.lower() + if metric == 'commits': + metric_filter = lambda r: 1 + elif metric == 'loc': + metric_filter = lambda r: r['loc'] + else: + raise Exception('Invalid metric %s' % metric) kwargs['metric_filter'] = metric_filter return f(*args, **kwargs) @@ -132,9 +184,11 @@ def exception_handler(): return decorator -DEFAULT_METRIC = 'loc' -DEFAULT_RELEASE = 'havana' -DEFAULT_PROJECT_TYPE = 'incubation' +DEFAULTS = { + 'metric': 'commits', + 'release': 'havana', + 'project_type': 'openstack', +} INDEPENDENT = '*independent' @@ -144,9 +198,8 @@ METRIC_LABELS = { } PROJECT_TYPES = { - 'core': ['core'], - 'incubation': ['core', 'incubation'], - 'all': ['core', 'incubation', 'dev'], + 'openstack': 'OpenStack', + 'stackforge': 'StackForge', } ISSUE_TYPES = ['bug', 'blueprint'] @@ -172,9 +225,15 @@ def templated(template=None): metric = flask.request.args.get('metric') if metric not in METRIC_LABELS: metric = None - ctx['metric'] = metric or DEFAULT_METRIC + ctx['metric'] = metric or DEFAULTS['metric'] ctx['metric_label'] = METRIC_LABELS[ctx['metric']] + project_type = flask.request.args.get('project_type') + if project_type not in PROJECT_TYPES: + project_type = None + ctx['project_type'] = project_type or DEFAULTS['project_type'] + ctx['project_type_label'] = PROJECT_TYPES[ctx['project_type']] + release = flask.request.args.get('release') releases = vault['releases'] if release: @@ -183,7 +242,7 @@ def templated(template=None): release = None else: release = releases[release]['release_name'] - ctx['release'] = (release or DEFAULT_RELEASE).lower() + ctx['release'] = (release or DEFAULTS['release']).lower() return flask.render_template(template_name, **ctx) @@ -329,22 +388,16 @@ def get_engineers(records, metric_filter): @app.route('/data/timeline') @exception_handler() -@record_filter(parameter_getter=lambda x: flask.request.args.get(x) - if (x != "release") and (x != "releases") else None) -def timeline(records): - +@record_filter(ignore='release') +def timeline(records, **kwargs): # find start and end dates - release_name = flask.request.args.get('release') - - if not release_name: - release_name = DEFAULT_RELEASE - else: - release_name = release_name.lower() - + release_names = get_parameter(kwargs, 'release', 'releases') releases = get_vault()['releases'] - if release_name not in releases: + if not release_names: flask.abort(404) - release = releases[release_name] + if not (set(release_names) & set(releases.keys())): + flask.abort(404) + release = releases[release_names[0]] start_date = release_start_date = user_utils.timestamp_to_week( user_utils.date_to_timestamp(release['start_date'])) @@ -365,6 +418,7 @@ def timeline(records): weeks = range(start_date, end_date) week_stat_loc = dict((c, 0) for c in weeks) week_stat_commits = dict((c, 0) for c in weeks) + week_stat_commits_hl = dict((c, 0) for c in weeks) # fill stats with the data for record in records: @@ -372,6 +426,8 @@ def timeline(records): if week in weeks: week_stat_loc[week] += record['loc'] week_stat_commits[week] += 1 + if 'all' in release_names or record['release'] in release_names: + week_stat_commits_hl[week] += 1 # form arrays in format acceptable to timeline plugin array_loc = [] @@ -381,9 +437,8 @@ def timeline(records): for week in weeks: week_str = user_utils.week_to_date(week) array_loc.append([week_str, week_stat_loc[week]]) - if release_start_date <= week <= release_end_date: - array_commits_hl.append([week_str, week_stat_commits[week]]) array_commits.append([week_str, week_stat_commits[week]]) + array_commits_hl.append([week_str, week_stat_commits_hl[week]]) return json.dumps([array_commits, array_commits_hl, array_loc]) @@ -408,37 +463,35 @@ def safe_encode(s): @app.template_filter('link') def make_link(title, uri=None): + param_names = ('release', 'metric', 'project_type') + param_values = {} + for param_name in param_names: + v = get_parameter({}, param_name, param_name) + if v: + param_values[param_name] = ','.join(v) + if param_values: + uri += '?' + '&'.join(['%s=%s' % (n, v) + for n, v in param_values.iteritems()]) return '%(title)s' % {'uri': uri, 'title': title} -def clear_text(s): - return cgi.escape(re.sub(r'\n{2,}', '\n', s, flags=re.MULTILINE)) - - -def link_blueprint(s, module): - return re.sub(r'(blueprint\s+)([\w-]+)', - r'\1\2', - s, flags=re.IGNORECASE) - - -def link_bug(s): - return re.sub(r'(bug\s+)#?([\d]{5,7})', - r'\1\2', - s, flags=re.IGNORECASE) - - -def link_change_id(s): - return re.sub(r'\s+(I[0-9a-f]{40})', - r' \1', - s) - - @app.template_filter('commit_message') def make_commit_message(record): + s = record['message'] + module = record['module'] - return link_change_id(link_bug(link_blueprint(clear_text( - record['message']), record['module']))) + # clear text + s = cgi.escape(re.sub(re.compile('\n{2,}', flags=re.MULTILINE), '\n', s)) + + # insert links + s = re.sub(re.compile('(blueprint\s+)([\w-]+)', flags=re.IGNORECASE), + r'\1\2', s) + s = re.sub(re.compile('(bug\s+)#?([\d]{5,7})', flags=re.IGNORECASE), + r'\1\2', s) + s = re.sub(r'\s+(I[0-9a-f]{40})', + r' \1', s) + return s gravatar = gravatar_ext.Gravatar(app, diff --git a/etc/default_data.json b/etc/default_data.json index 41061f73f..4bb6f83b5 100644 --- a/etc/default_data.json +++ b/etc/default_data.json @@ -4,6 +4,19 @@ }, "users": [ + { + "launchpad_id": "openstack", + "companies": [ + { + "company_name": "*robots", + "end_date": null + } + ], + "user_name": "OpenStack Robot", + "emails": [ + "review@openstack.org", "jenkins@review.openstack.org", "jenkins@openstack.org", "hudson@openstack.org" + ] + }, { "launchpad_id": "akamyshnikova", "companies": [ @@ -12661,6 +12674,10 @@ "companies": [ { "company_name": "Rackspace", + "end_date": "2012-Nov-01" + }, + { + "company_name": "SwiftStack", "end_date": null } ], @@ -13429,6 +13446,10 @@ "domains": ["vexxhost.com"], "company_name": "VexxHost" }, + { + "domains": ["vbridges.com"], + "company_name": "VirtualBridges" + }, { "domains": ["virtualtech.jp"], "company_name": "Virtualtech" @@ -13454,8 +13475,8 @@ "repos": [ { "branches": ["master"], - "name": "Neutron Client", - "type": "core", + "module": "python-neutronclient", + "project_type": "openstack", "uri": "git://github.com/openstack/python-neutronclient.git", "releases": [ { @@ -13477,8 +13498,8 @@ }, { "branches": ["master"], - "name": "Keystone", - "type": "core", + "module": "keystone", + "project_type": "openstack", "uri": "git://github.com/openstack/keystone.git", "releases": [ { @@ -13505,8 +13526,64 @@ }, { "branches": ["master"], - "name": "Murano API", - "type": "stackforge", + "module": "nova", + "project_type": "openstack", + "uri": "git://github.com/openstack/nova.git", + "releases": [ + { + "release_name": "Essex", + "tag_from": "2011.3", + "tag_to": "2012.1" + }, + { + "release_name": "Folsom", + "tag_from": "2012.1", + "tag_to": "2012.2" + }, + { + "release_name": "Grizzly", + "tag_from": "2012.2", + "tag_to": "2013.1" + }, + { + "release_name": "Havana", + "tag_from": "2013.1", + "tag_to": "HEAD" + } + ] + }, + { + "branches": ["master"], + "module": "neutron", + "project_type": "openstack", + "uri": "git://github.com/openstack/neutron.git", + "releases": [ + { + "release_name": "Essex", + "tag_from": "2011.3", + "tag_to": "2012.1" + }, + { + "release_name": "Folsom", + "tag_from": "2012.1", + "tag_to": "2012.2" + }, + { + "release_name": "Grizzly", + "tag_from": "2012.2", + "tag_to": "2013.1" + }, + { + "release_name": "Havana", + "tag_from": "2013.1", + "tag_to": "HEAD" + } + ] + }, + { + "branches": ["master"], + "module": "murano-api", + "project_type": "stackforge", "uri": "git://github.com/stackforge/murano-api.git", "releases": [ { diff --git a/etc/test_default_data.json b/etc/test_default_data.json index 9f719d3c3..cdbe66b22 100644 --- a/etc/test_default_data.json +++ b/etc/test_default_data.json @@ -39,7 +39,7 @@ "repos": [ { "branches": ["master"], - "name": "Quantum Client", + "module": "python-quantumclient", "type": "core", "uri": "git://github.com/openstack/python-quantumclient.git", "releases": [ @@ -62,7 +62,7 @@ }, { "branches": ["master"], - "name": "Keystone", + "module": "keystone", "type": "core", "uri": "git://github.com/openstack/keystone.git", "releases": [ diff --git a/tests/unit/test_web_utils.py b/tests/unit/test_web_utils.py new file mode 100644 index 000000000..bda37ba9c --- /dev/null +++ b/tests/unit/test_web_utils.py @@ -0,0 +1,66 @@ +import testtools + +from dashboard import web + + +class TestWebUtils(testtools.TestCase): + def setUp(self): + super(TestWebUtils, self).setUp() + + def test_make_commit_message(self): + message = ''' +During finish_migration the manager calls initialize_connection but doesn't +update the block_device_mapping with the potentially new connection_info +returned. + + +Fixes bug 1076801 +Change-Id: Ie49ccd2138905e178843b375a9b16c3fe572d1db''' + + module = 'test' + + record = { + 'message': message, + 'module': module, + } + + expected = ''' +During finish_migration the manager calls initialize_connection but doesn't +update the block_device_mapping with the potentially new connection_info +returned. +Fixes bug 1076801 +''' + ('Change-Id: ' + 'Ie49ccd2138905e178843b375a9b16c3fe572d1db') + + observed = web.make_commit_message(record) + + self.assertEqual(expected, observed, + 'Commit message should be processed correctly') + + def test_make_commit_message_blueprint_link(self): + message = ''' +Implemented new driver for Cinder <: +Implements Blueprint super-driver +Change-Id: Ie49ccd2138905e178843b375a9b16c3fe572d1db''' + + module = 'cinder' + + record = { + 'message': message, + 'module': module, + } + + expected = ''' +Implemented new driver for Cinder <: +Implements Blueprint ''' + ( + 'super-driver' + '\n' + + 'Change-Id: ' + 'Ie49ccd2138905e178843b375a9b16c3fe572d1db') + + observed = web.make_commit_message(record) + + self.assertEqual(expected, observed, + 'Commit message should be processed correctly')