# Copyright (c) 2013 Mirantis Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import collections import functools import json import flask import six from werkzeug import exceptions from dashboard import helpers from dashboard import parameters from dashboard import vault from stackalytics.openstack.common import log as logging from stackalytics.processor import utils from stackalytics import version as stackalytics_version LOG = logging.getLogger(__name__) def _get_time_filter(kwargs): start_date = parameters.get_single_parameter(kwargs, 'start_date') if start_date: start_date = utils.date_to_timestamp_ext(start_date) else: start_date = 0 end_date = parameters.get_single_parameter(kwargs, 'end_date') if end_date: end_date = utils.date_to_timestamp_ext(end_date) else: end_date = utils.date_to_timestamp_ext('now') def time_filter(records): for record in records: if start_date <= record['date'] <= end_date: yield record return time_filter def record_filter(ignore=None, use_default=True): if not ignore: ignore = [] def decorator(f): @functools.wraps(f) def record_filter_decorated_function(*args, **kwargs): memory_storage_inst = vault.get_memory_storage() record_ids = set(memory_storage_inst.get_record_ids()) # a copy if 'module' not in ignore: param = parameters.get_parameter(kwargs, 'module', 'modules', use_default) if param: record_ids &= ( memory_storage_inst.get_record_ids_by_modules( vault.resolve_modules(param))) if 'project_type' not in ignore: param = parameters.get_parameter(kwargs, 'project_type', 'project_types', use_default) if param: record_ids &= ( memory_storage_inst.get_record_ids_by_modules( vault.resolve_project_types(param))) if 'user_id' not in ignore: param = parameters.get_parameter(kwargs, 'user_id', 'user_ids') param = [u for u in param if vault.get_user_from_runtime_storage(u)] if param: record_ids &= ( memory_storage_inst.get_record_ids_by_user_ids(param)) if 'company' not in ignore: param = parameters.get_parameter(kwargs, 'company', 'companies') if param: record_ids &= ( memory_storage_inst.get_record_ids_by_companies(param)) if 'release' not in ignore: param = parameters.get_parameter(kwargs, 'release', 'releases', use_default) if param: if 'all' not in param: record_ids &= ( memory_storage_inst.get_record_ids_by_releases( c.lower() for c in param)) if 'metric' not in ignore: metrics = parameters.get_parameter(kwargs, 'metric') if 'all' not in metrics: for metric in metrics: if metric in parameters.METRIC_TO_RECORD_TYPE: record_ids &= ( memory_storage_inst.get_record_ids_by_type( parameters.METRIC_TO_RECORD_TYPE[metric])) if 'tm_marks' in metrics: filtered_ids = [] review_nth = int(parameters.get_parameter( kwargs, 'review_nth')[0]) for record in memory_storage_inst.get_records(record_ids): parent = memory_storage_inst.get_record_by_primary_key( record['review_id']) if (parent and ('review_number' in parent) and (parent['review_number'] <= review_nth)): filtered_ids.append(record['record_id']) record_ids = filtered_ids if 'blueprint_id' not in ignore: param = parameters.get_parameter(kwargs, 'blueprint_id') if param: record_ids &= ( memory_storage_inst.get_record_ids_by_blueprint_ids( param)) time_filter = _get_time_filter(kwargs) kwargs['records'] = time_filter( memory_storage_inst.get_records(record_ids)) return f(*args, **kwargs) return record_filter_decorated_function return decorator def incremental_filter(result, record, param_id): result[record[param_id]]['metric'] += 1 def loc_filter(result, record, param_id): result[record[param_id]]['metric'] += record['loc'] def mark_filter(result, record, param_id): result_by_param = result[record[param_id]] if record['type'] == 'APRV': value = 'A' else: value = record['value'] result_by_param['metric'] += 1 result_by_param[value] = result_by_param.get(value, 0) + 1 if record.get('x'): result_by_param['disagreements'] = ( result_by_param.get('disagreements', 0) + 1) def mark_finalize(record): new_record = record.copy() positive = 0 numeric = 0 mark_distribution = [] for key in [-2, -1, 1, 2, 'A']: if key in record: if key in [1, 2]: positive += record[key] if key in [-2, -1, 1, 2]: numeric += record[key] mark_distribution.append(str(record[key])) else: mark_distribution.append('0') new_record[key] = 0 new_record['disagreements'] = record.get('disagreements', 0) if numeric: positive_ratio = '%.1f%%' % ( (positive * 100.0) / numeric) new_record['disagreement_ratio'] = '%.1f%%' % ( (record.get('disagreements', 0) * 100.0) / numeric) else: positive_ratio = helpers.INFINITY_HTML new_record['disagreement_ratio'] = helpers.INFINITY_HTML new_record['mark_ratio'] = ( '|'.join(mark_distribution) + ' (' + positive_ratio + ')') new_record['positive_ratio'] = positive_ratio return new_record def man_days_filter(result, record, param_id): if record['record_type'] == 'commit': # commit is attributed with the date of the merge which is not an # effort of the author (author's effort is represented in patches) return day = record['date'] // (24 * 3600) result_by_param = result[record[param_id]] if 'days' not in result_by_param: result_by_param['days'] = collections.defaultdict(set) user = vault.get_user_from_runtime_storage(record['user_id']) result_by_param['days'][day] |= set([user['seq']]) result_by_param['metric'] = 1 def man_days_finalize(result_item): metric = 0 for day_set in six.itervalues(result_item['days']): metric += len(day_set) del result_item['days'] result_item['metric'] = metric return result_item def aggregate_filter(): def decorator(f): @functools.wraps(f) def aggregate_filter_decorated_function(*args, **kwargs): metric_param = (flask.request.args.get('metric') or parameters.get_default('metric')) metric = metric_param.lower() metric_to_filters_map = { 'commits': (incremental_filter, None), 'loc': (loc_filter, None), 'marks': (mark_filter, mark_finalize), 'tm_marks': (mark_filter, mark_finalize), 'emails': (incremental_filter, None), 'bpd': (incremental_filter, None), 'bpc': (incremental_filter, None), 'members': (incremental_filter, None), 'man-days': (man_days_filter, man_days_finalize), } if metric not in metric_to_filters_map: metric = parameters.get_default('metric') kwargs['metric_filter'] = metric_to_filters_map[metric][0] kwargs['finalize_handler'] = metric_to_filters_map[metric][1] return f(*args, **kwargs) return aggregate_filter_decorated_function return decorator def exception_handler(): def decorator(f): @functools.wraps(f) def exception_handler_decorated_function(*args, **kwargs): try: return f(*args, **kwargs) except Exception as e: if isinstance(e, exceptions.HTTPException): raise # ignore Flask exceptions LOG.exception(e) flask.abort(404) return exception_handler_decorated_function return decorator def templated(template=None, return_code=200): def decorator(f): @functools.wraps(f) def templated_decorated_function(*args, **kwargs): vault_inst = vault.get_vault() template_name = template if template_name is None: template_name = (flask.request.endpoint.replace('.', '/') + '.html') ctx = f(*args, **kwargs) if ctx is None: ctx = {} # put parameters into template metric = flask.request.args.get('metric') if metric not in parameters.METRIC_LABELS: metric = None ctx['metric'] = metric or parameters.get_default('metric') ctx['metric_label'] = parameters.METRIC_LABELS[ctx['metric']] project_type = flask.request.args.get('project_type') if not vault.is_project_type_valid(project_type): project_type = parameters.get_default('project_type') ctx['project_type'] = project_type release = flask.request.args.get('release') releases = vault_inst['releases'] if release: release = release.lower() if release != 'all': if release not in releases: release = None else: release = releases[release]['release_name'] ctx['release'] = (release or parameters.get_default('release')).lower() ctx['review_nth'] = (flask.request.args.get('review_nth') or parameters.get_default('review_nth')) ctx['project_type_options'] = vault.get_project_types() ctx['release_options'] = vault.get_release_options() ctx['metric_options'] = sorted(parameters.METRIC_LABELS.items(), key=lambda x: x[0]) ctx['company'] = parameters.get_single_parameter(kwargs, 'company') ctx['company_original'] = ( vault.get_memory_storage().get_original_company_name( ctx['company'])) module = parameters.get_single_parameter(kwargs, 'module') ctx['module'] = module if module and module in vault_inst['module_id_index']: ctx['module_inst'] = vault_inst['module_id_index'][module] ctx['user_id'] = parameters.get_single_parameter(kwargs, 'user_id') ctx['page_title'] = helpers.make_page_title( ctx['company'], ctx['user_id'], ctx['module'], ctx['release']) ctx['stackalytics_version'] = ( stackalytics_version.version_info.version_string()) ctx['stackalytics_release'] = ( stackalytics_version.version_info.release_string()) return flask.render_template(template_name, **ctx), return_code return templated_decorated_function return decorator def jsonify(root='data'): def decorator(func): @functools.wraps(func) def jsonify_decorated_function(*args, **kwargs): callback = flask.app.request.args.get('callback', False) data = json.dumps({root: func(*args, **kwargs)}) if callback: data = str(callback) + '(' + data + ')' mimetype = 'application/javascript' else: mimetype = 'application/json' return flask.current_app.response_class(data, mimetype=mimetype) return jsonify_decorated_function return decorator