418a4a8140
The metric counts number of days a unique user made some action. Implements bp contribution-effort Change-Id: I00fbb2ab2c0c9617ef2e4d949ac06e7a64facdac
362 lines
13 KiB
Python
362 lines
13 KiB
Python
# Copyright (c) 2013 Mirantis Inc.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
# implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import collections
|
|
import functools
|
|
import json
|
|
|
|
import flask
|
|
import six
|
|
from werkzeug import exceptions
|
|
|
|
from dashboard import helpers
|
|
from dashboard import parameters
|
|
from dashboard import vault
|
|
from stackalytics.openstack.common import log as logging
|
|
from stackalytics.processor import utils
|
|
from stackalytics import version as stackalytics_version
|
|
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
|
|
def _get_time_filter(kwargs):
|
|
start_date = parameters.get_single_parameter(kwargs, 'start_date')
|
|
if start_date:
|
|
start_date = utils.date_to_timestamp_ext(start_date)
|
|
else:
|
|
start_date = 0
|
|
end_date = parameters.get_single_parameter(kwargs, 'end_date')
|
|
if end_date:
|
|
end_date = utils.date_to_timestamp_ext(end_date)
|
|
else:
|
|
end_date = utils.date_to_timestamp_ext('now')
|
|
|
|
def time_filter(records):
|
|
for record in records:
|
|
if start_date <= record['date'] <= end_date:
|
|
yield record
|
|
|
|
return time_filter
|
|
|
|
|
|
def record_filter(ignore=None, use_default=True):
|
|
if not ignore:
|
|
ignore = []
|
|
|
|
def decorator(f):
|
|
@functools.wraps(f)
|
|
def record_filter_decorated_function(*args, **kwargs):
|
|
|
|
memory_storage_inst = vault.get_memory_storage()
|
|
record_ids = set(memory_storage_inst.get_record_ids()) # a copy
|
|
|
|
if 'module' not in ignore:
|
|
param = parameters.get_parameter(kwargs, 'module', 'modules',
|
|
use_default)
|
|
if param:
|
|
record_ids &= (
|
|
memory_storage_inst.get_record_ids_by_modules(
|
|
vault.resolve_modules(param)))
|
|
|
|
if 'project_type' not in ignore:
|
|
param = parameters.get_parameter(kwargs, 'project_type',
|
|
'project_types', use_default)
|
|
if param:
|
|
record_ids &= (
|
|
memory_storage_inst.get_record_ids_by_modules(
|
|
vault.resolve_project_types(param)))
|
|
|
|
if 'user_id' not in ignore:
|
|
param = parameters.get_parameter(kwargs, 'user_id', 'user_ids')
|
|
param = [u for u in param
|
|
if vault.get_user_from_runtime_storage(u)]
|
|
if param:
|
|
record_ids &= (
|
|
memory_storage_inst.get_record_ids_by_user_ids(param))
|
|
|
|
if 'company' not in ignore:
|
|
param = parameters.get_parameter(kwargs, 'company',
|
|
'companies')
|
|
if param:
|
|
record_ids &= (
|
|
memory_storage_inst.get_record_ids_by_companies(param))
|
|
|
|
if 'release' not in ignore:
|
|
param = parameters.get_parameter(kwargs, 'release', 'releases',
|
|
use_default)
|
|
if param:
|
|
if 'all' not in param:
|
|
record_ids &= (
|
|
memory_storage_inst.get_record_ids_by_releases(
|
|
c.lower() for c in param))
|
|
|
|
if 'metric' not in ignore:
|
|
metrics = parameters.get_parameter(kwargs, 'metric')
|
|
if 'all' not in metrics:
|
|
for metric in metrics:
|
|
if metric in parameters.METRIC_TO_RECORD_TYPE:
|
|
record_ids &= (
|
|
memory_storage_inst.get_record_ids_by_type(
|
|
parameters.METRIC_TO_RECORD_TYPE[metric]))
|
|
|
|
if 'tm_marks' in metrics:
|
|
filtered_ids = []
|
|
review_nth = int(parameters.get_parameter(
|
|
kwargs, 'review_nth')[0])
|
|
for record in memory_storage_inst.get_records(record_ids):
|
|
parent = memory_storage_inst.get_record_by_primary_key(
|
|
record['review_id'])
|
|
if (parent and ('review_number' in parent) and
|
|
(parent['review_number'] <= review_nth)):
|
|
filtered_ids.append(record['record_id'])
|
|
record_ids = filtered_ids
|
|
|
|
if 'blueprint_id' not in ignore:
|
|
param = parameters.get_parameter(kwargs, 'blueprint_id')
|
|
if param:
|
|
record_ids &= (
|
|
memory_storage_inst.get_record_ids_by_blueprint_ids(
|
|
param))
|
|
|
|
time_filter = _get_time_filter(kwargs)
|
|
|
|
kwargs['records'] = time_filter(
|
|
memory_storage_inst.get_records(record_ids))
|
|
return f(*args, **kwargs)
|
|
|
|
return record_filter_decorated_function
|
|
|
|
return decorator
|
|
|
|
|
|
def incremental_filter(result, record, param_id):
|
|
result[record[param_id]]['metric'] += 1
|
|
|
|
|
|
def loc_filter(result, record, param_id):
|
|
result[record[param_id]]['metric'] += record['loc']
|
|
|
|
|
|
def mark_filter(result, record, param_id):
|
|
result_by_param = result[record[param_id]]
|
|
if record['type'] == 'APRV':
|
|
value = 'A'
|
|
else:
|
|
value = record['value']
|
|
result_by_param['metric'] += 1
|
|
result_by_param[value] = result_by_param.get(value, 0) + 1
|
|
|
|
if record.get('x'):
|
|
result_by_param['disagreements'] = (
|
|
result_by_param.get('disagreements', 0) + 1)
|
|
|
|
|
|
def mark_finalize(record):
|
|
new_record = record.copy()
|
|
|
|
positive = 0
|
|
numeric = 0
|
|
mark_distribution = []
|
|
for key in [-2, -1, 1, 2, 'A']:
|
|
if key in record:
|
|
if key in [1, 2]:
|
|
positive += record[key]
|
|
if key in [-2, -1, 1, 2]:
|
|
numeric += record[key]
|
|
mark_distribution.append(str(record[key]))
|
|
else:
|
|
mark_distribution.append('0')
|
|
new_record[key] = 0
|
|
|
|
new_record['disagreements'] = record.get('disagreements', 0)
|
|
if numeric:
|
|
positive_ratio = '%.1f%%' % (
|
|
(positive * 100.0) / numeric)
|
|
new_record['disagreement_ratio'] = '%.1f%%' % (
|
|
(record.get('disagreements', 0) * 100.0) / numeric)
|
|
else:
|
|
positive_ratio = helpers.INFINITY_HTML
|
|
new_record['disagreement_ratio'] = helpers.INFINITY_HTML
|
|
new_record['mark_ratio'] = (
|
|
'|'.join(mark_distribution) + ' (' + positive_ratio + ')')
|
|
new_record['positive_ratio'] = positive_ratio
|
|
|
|
return new_record
|
|
|
|
|
|
def man_days_filter(result, record, param_id):
|
|
if record['record_type'] == 'commit':
|
|
# commit is attributed with the date of the merge which is not an
|
|
# effort of the author (author's effort is represented in patches)
|
|
return
|
|
|
|
day = record['date'] // (24 * 3600)
|
|
|
|
result_by_param = result[record[param_id]]
|
|
if 'days' not in result_by_param:
|
|
result_by_param['days'] = collections.defaultdict(set)
|
|
user = vault.get_user_from_runtime_storage(record['user_id'])
|
|
result_by_param['days'][day] |= set([user['seq']])
|
|
result_by_param['metric'] = 1
|
|
|
|
|
|
def man_days_finalize(result_item):
|
|
metric = 0
|
|
for day_set in six.itervalues(result_item['days']):
|
|
metric += len(day_set)
|
|
del result_item['days']
|
|
result_item['metric'] = metric
|
|
return result_item
|
|
|
|
|
|
def aggregate_filter():
|
|
def decorator(f):
|
|
@functools.wraps(f)
|
|
def aggregate_filter_decorated_function(*args, **kwargs):
|
|
|
|
metric_param = (flask.request.args.get('metric') or
|
|
parameters.get_default('metric'))
|
|
metric = metric_param.lower()
|
|
|
|
metric_to_filters_map = {
|
|
'commits': (incremental_filter, None),
|
|
'loc': (loc_filter, None),
|
|
'marks': (mark_filter, mark_finalize),
|
|
'tm_marks': (mark_filter, mark_finalize),
|
|
'emails': (incremental_filter, None),
|
|
'bpd': (incremental_filter, None),
|
|
'bpc': (incremental_filter, None),
|
|
'members': (incremental_filter, None),
|
|
'man-days': (man_days_filter, man_days_finalize),
|
|
}
|
|
if metric not in metric_to_filters_map:
|
|
metric = parameters.get_default('metric')
|
|
|
|
kwargs['metric_filter'] = metric_to_filters_map[metric][0]
|
|
kwargs['finalize_handler'] = metric_to_filters_map[metric][1]
|
|
return f(*args, **kwargs)
|
|
|
|
return aggregate_filter_decorated_function
|
|
|
|
return decorator
|
|
|
|
|
|
def exception_handler():
|
|
def decorator(f):
|
|
@functools.wraps(f)
|
|
def exception_handler_decorated_function(*args, **kwargs):
|
|
try:
|
|
return f(*args, **kwargs)
|
|
except Exception as e:
|
|
if isinstance(e, exceptions.HTTPException):
|
|
raise # ignore Flask exceptions
|
|
LOG.exception(e)
|
|
flask.abort(404)
|
|
|
|
return exception_handler_decorated_function
|
|
|
|
return decorator
|
|
|
|
|
|
def templated(template=None, return_code=200):
|
|
def decorator(f):
|
|
@functools.wraps(f)
|
|
def templated_decorated_function(*args, **kwargs):
|
|
|
|
vault_inst = vault.get_vault()
|
|
template_name = template
|
|
if template_name is None:
|
|
template_name = (flask.request.endpoint.replace('.', '/') +
|
|
'.html')
|
|
ctx = f(*args, **kwargs)
|
|
if ctx is None:
|
|
ctx = {}
|
|
|
|
# put parameters into template
|
|
metric = flask.request.args.get('metric')
|
|
if metric not in parameters.METRIC_LABELS:
|
|
metric = None
|
|
ctx['metric'] = metric or parameters.get_default('metric')
|
|
ctx['metric_label'] = parameters.METRIC_LABELS[ctx['metric']]
|
|
|
|
project_type = flask.request.args.get('project_type')
|
|
if not vault.is_project_type_valid(project_type):
|
|
project_type = parameters.get_default('project_type')
|
|
ctx['project_type'] = project_type
|
|
|
|
release = flask.request.args.get('release')
|
|
releases = vault_inst['releases']
|
|
if release:
|
|
release = release.lower()
|
|
if release != 'all':
|
|
if release not in releases:
|
|
release = None
|
|
else:
|
|
release = releases[release]['release_name']
|
|
ctx['release'] = (release or
|
|
parameters.get_default('release')).lower()
|
|
ctx['review_nth'] = (flask.request.args.get('review_nth') or
|
|
parameters.get_default('review_nth'))
|
|
|
|
ctx['project_type_options'] = vault.get_project_types()
|
|
ctx['release_options'] = vault.get_release_options()
|
|
ctx['metric_options'] = sorted(parameters.METRIC_LABELS.items(),
|
|
key=lambda x: x[0])
|
|
|
|
ctx['company'] = parameters.get_single_parameter(kwargs, 'company')
|
|
ctx['company_original'] = (
|
|
vault.get_memory_storage().get_original_company_name(
|
|
ctx['company']))
|
|
|
|
module = parameters.get_single_parameter(kwargs, 'module')
|
|
ctx['module'] = module
|
|
if module and module in vault_inst['module_id_index']:
|
|
ctx['module_inst'] = vault_inst['module_id_index'][module]
|
|
|
|
ctx['user_id'] = parameters.get_single_parameter(kwargs, 'user_id')
|
|
ctx['page_title'] = helpers.make_page_title(
|
|
ctx['company'], ctx['user_id'], ctx['module'], ctx['release'])
|
|
ctx['stackalytics_version'] = (
|
|
stackalytics_version.version_info.version_string())
|
|
ctx['stackalytics_release'] = (
|
|
stackalytics_version.version_info.release_string())
|
|
|
|
return flask.render_template(template_name, **ctx), return_code
|
|
|
|
return templated_decorated_function
|
|
|
|
return decorator
|
|
|
|
|
|
def jsonify(root='data'):
|
|
def decorator(func):
|
|
@functools.wraps(func)
|
|
def jsonify_decorated_function(*args, **kwargs):
|
|
callback = flask.app.request.args.get('callback', False)
|
|
data = json.dumps({root: func(*args, **kwargs)})
|
|
|
|
if callback:
|
|
data = str(callback) + '(' + data + ')'
|
|
mimetype = 'application/javascript'
|
|
else:
|
|
mimetype = 'application/json'
|
|
|
|
return flask.current_app.response_class(data, mimetype=mimetype)
|
|
|
|
return jsonify_decorated_function
|
|
|
|
return decorator
|