Added new metric counting man-days effort

The metric counts number of days a unique user made some action.

Implements bp contribution-effort

Change-Id: I00fbb2ab2c0c9617ef2e4d949ac06e7a64facdac

dashboard/decorators.py

@@ -13,10 +13,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import collections
 import functools
 import json
 
 import flask
+import six
 from werkzeug import exceptions
 
 from dashboard import helpers
@@ -105,6 +107,7 @@ def record_filter(ignore=None, use_default=True):
                 metrics = parameters.get_parameter(kwargs, 'metric')
                 if 'all' not in metrics:
                     for metric in metrics:
+                        if metric in parameters.METRIC_TO_RECORD_TYPE:
                             record_ids &= (
                                 memory_storage_inst.get_record_ids_by_type(
                                     parameters.METRIC_TO_RECORD_TYPE[metric]))
@@ -194,6 +197,31 @@ def mark_finalize(record):
     return new_record
 
 
+def man_days_filter(result, record, param_id):
+    if record['record_type'] == 'commit':
+        # commit is attributed with the date of the merge which is not an
+        # effort of the author (author's effort is represented in patches)
+        return
+
+    day = record['date'] // (24 * 3600)
+
+    result_by_param = result[record[param_id]]
+    if 'days' not in result_by_param:
+        result_by_param['days'] = collections.defaultdict(set)
+    user = vault.get_user_from_runtime_storage(record['user_id'])
+    result_by_param['days'][day] |= set([user['seq']])
+    result_by_param['metric'] = 1
+
+
+def man_days_finalize(result_item):
+    metric = 0
+    for day_set in six.itervalues(result_item['days']):
+        metric += len(day_set)
+    del result_item['days']
+    result_item['metric'] = metric
+    return result_item
+
+
 def aggregate_filter():
     def decorator(f):
         @functools.wraps(f)
@@ -212,6 +240,7 @@ def aggregate_filter():
                 'bpd': (incremental_filter, None),
                 'bpc': (incremental_filter, None),
                 'members': (incremental_filter, None),
+                'man-days': (man_days_filter, man_days_finalize),
             }
             if metric not in metric_to_filters_map:
                 metric = parameters.get_default('metric')

dashboard/parameters.py

@@ -35,6 +35,7 @@ METRIC_LABELS = {
     'emails': 'Emails',
     'bpd': 'Drafted Blueprints',
     'bpc': 'Completed Blueprints',
+    'man-days': "Man-days effort"
 }
 
 METRIC_TO_RECORD_TYPE = {

dashboard/web.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import collections
 import operator
 import os
 import re
@@ -88,8 +89,8 @@ def _get_aggregated_stats(records, metric_filter, keys, param_id,
         result[record[param_id]]['name'] = record[param_title]
 
     response = [r for r in result.values() if r['metric']]
-    response.sort(key=lambda x: x['metric'], reverse=True)
     response = [item for item in map(finalize_handler, response) if item]
+    response.sort(key=lambda x: x['metric'], reverse=True)
     utils.add_index(response, item_filter=lambda x: x['id'] != '*independent')
     return response
 
@@ -102,7 +103,8 @@ def _get_aggregated_stats(records, metric_filter, keys, param_id,
 def get_companies(records, metric_filter, finalize_handler):
     return _get_aggregated_stats(records, metric_filter,
                                  vault.get_memory_storage().get_companies(),
-                                 'company_name')
+                                 'company_name',
+                                 finalize_handler=finalize_handler)
 
 
 @app.route('/api/1.0/stats/modules')
@@ -113,7 +115,7 @@ def get_companies(records, metric_filter, finalize_handler):
 def get_modules(records, metric_filter, finalize_handler):
     return _get_aggregated_stats(records, metric_filter,
                                  vault.get_memory_storage().get_modules(),
-                                 'module')
+                                 'module', finalize_handler=finalize_handler)
 
 
 def get_core_engineer_branch(user, modules):
@@ -492,21 +494,45 @@ def timeline(records, **kwargs):
     week_stat_commits = dict((c, 0) for c in weeks)
     week_stat_commits_hl = dict((c, 0) for c in weeks)
 
-    param = parameters.get_parameter(kwargs, 'metric')
+    metric = parameters.get_parameter(kwargs, 'metric')
-    if ('commits' in param) or ('loc' in param):
+    if ('commits' in metric) or ('loc' in metric):
         handler = lambda record: record['loc']
     else:
         handler = lambda record: 0
 
     # fill stats with the data
+    if 'man-days' in metric:
+        # special case for man-day effort metric
+        release_stat = collections.defaultdict(set)
+        all_stat = collections.defaultdict(set)
+        for record in records:
+            if ((record['record_type'] == 'commit') or
+                    (record['week'] not in weeks)):
+                continue
+
+            day = record['date'] // (24 * 3600)
+            user = vault.get_user_from_runtime_storage(record['user_id'])
+            if record['release'] == release_name:
+                release_stat[day] |= set([user['seq']])
+            all_stat[day] |= set([user['seq']])
+        for day, users in six.iteritems(release_stat):
+            week = utils.timestamp_to_week(day * 24 * 3600)
+            week_stat_commits_hl[week] += len(users)
+        for day, users in six.iteritems(all_stat):
+            week = utils.timestamp_to_week(day * 24 * 3600)
+            week_stat_commits[week] += len(users)
+    else:
         for record in records:
             week = record['week']
             if week in weeks:
                 week_stat_loc[week] += handler(record)
                 week_stat_commits[week] += 1
-            if 'all' == release_name or record['release'] == release_name:
+                if record['release'] == release_name:
                     week_stat_commits_hl[week] += 1
 
+    if 'all' == release_name:
+        week_stat_commits_hl = week_stat_commits
+
     # form arrays in format acceptable to timeline plugin
     array_loc = []
     array_commits = []