Import old who-approves.py script

This was last used roughly a decade ago (when we were still relying
on Gerrit's internal gitweb service, before we switched to cgit) to
create approximate lists of core reviewers for official OpenStack
deliverables. It does not work, but is being copied verbatim from
system-config as a first step, in preparation to revitalize and
generalize it.

Change-Id: Icbc3c635cb5312e44dc8377614e805d5ea8629cc

engagement/who-approves.py

@@ -0,0 +1,157 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2015 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Description: When run using OpenStack's Gerrit server, this builds
+# JSON and YAML representations of repos with information on the
+# official owning project team if any, deliverable tags, and groups
+# with approve rights listing the members of each along with their
+# Gerrit preferred E-mail addresses and usernames when available.
+
+# Rationale: It was done as a demonstration to a representative of a
+# foundation member company who requested a list of the "core
+# reviewers" for official projects, optionally broken down by
+# integrated vs. other. I'm attempting to show that this data is
+# already publicly available and can be extracted/analyzed by anyone
+# without needing to request it.
+
+# Use: This needs your Gerrit username passed as the command-line
+# parameter, found at https://review.opendev.org/#/settings/ when
+# authenticated in the WebUI. It also prompts for an HTTP password
+# which https://review.opendev.org/#/settings/http-password will
+# allow you to generate. The results end up in files named
+# approvers.json and approvers.yaml. At the time of writing, it
+# takes approximately 6.5 minutes to run on a well-connected machine
+# with 70-80ms round-trip latency to review.opendev.org.
+
+# Example:
+#
+#     $ virtualenv approvers
+#     [...]
+#     $ ./approvers/bin/pip install pyyaml requests
+#     [...]
+#     $ ./approvers/bin/python tools/who-approves.py fungi
+#     Password:
+#     [wait for completion]
+#     $ ./approvers/bin/python
+#     >>> import yaml
+#     >>>
+#     >>> def get_approvers(repos):
+#     ...     approvers = set()
+#     ...     for repo in repos:
+#     ...         for group in repos[repo]['approvers']:
+#     ...             for approver in repos[repo]['approvers'][group]:
+#     ...                 approvers.add(approver)
+#     ...     return(approvers)
+#     ...
+#     >>> p = yaml.safe_load(open('approvers.yaml'))
+#     >>> print('Total repos: %s' % len(p))
+#     Total repos: 751
+#     >>> print('Total approvers: %s' % len(get_approvers(p)))
+#     Total approvers: 849
+#     >>>
+#     >>> o = {k: v for k, v in p.iteritems() if 'team' in v}
+#     >>> print('Repos for official teams: %s' % len(o))
+#     Repos for official teams: 380
+#     >>> print('OpenStack repo approvers: %s' % len(get_approvers(o)))
+#     OpenStack repo approvers: 456
+#     >>>
+#     >>> i = {k: v for k, v in p.iteritems() if 'tags' in v
+#     ...      and 'release:managed' in v['tags']}
+#     >>> print('Repos under release management: %s' % len(i))
+#     Repos under release management: 77
+#     >>> print('Managed release repo approvers: %s' % len(get_approvers(i)))
+#     Managed release repo approvers: 245
+
+import getpass
+import json
+import re
+import sys
+
+import requests
+import yaml
+
+gerrit_url = 'https://review.opendev.org/'
+try:
+    gerrit_auth = requests.auth.HTTPDigestAuth(sys.argv[1], getpass.getpass())
+except IndexError:
+    sys.stderr.write("Usage: %s USERNAME\n" % sys.argv[0])
+    sys.exit(1)
+acl_path = 'gitweb?p=%s.git;a=blob_plain;f=project.config;hb=refs/meta/config'
+group_path = 'a/groups/%s/members/?recursive&pp=0'
+projects_file = ('gitweb?p=openstack/governance.git;a=blob_plain;'
+                 'f=reference/projects.yaml;hb=%s')
+ref_name = 'refs/heads/master'
+aprv_pattern = 'label-Workflow = .*\.\.\+1 group (.*)'
+projects = requests.get(gerrit_url + projects_file % ref_name)
+projects.encoding = 'utf-8'  # Workaround for Gitweb encoding
+projects = yaml.safe_load(projects.text)
+repos_dump = json.loads(requests.get(
+    gerrit_url + 'projects/?pp=0').text[4:])
+all_groups = json.loads(requests.get(gerrit_url + 'a/groups/',
+                                     auth=gerrit_auth).text[4:])
+repos = {}
+aprv_groups = {}
+for repo in repos_dump:
+    repos[repo.encode('utf-8')] = {'approvers': {}}
+    acl_ini = requests.get(gerrit_url + acl_path % repo).text
+    for aprv_group in [str(x) for x in re.findall(aprv_pattern, acl_ini)]:
+        if aprv_group not in repos[repo]['approvers']:
+            repos[repo]['approvers'][aprv_group] = []
+        if aprv_group not in aprv_groups:
+            aprv_groups[aprv_group] = []
+for team in projects:
+    if 'deliverables' in projects[team]:
+        for deli in projects[team]['deliverables']:
+            if 'repos' in projects[team]['deliverables'][deli]:
+                drepos = projects[team]['deliverables'][deli]['repos']
+                for repo in drepos:
+                    if repo in repos:
+                        repos[repo]['team'] = team
+                        if 'tags' in projects[team]['deliverables'][deli]:
+                            repos[repo]['tags'] = \
+                                projects[team]['deliverables'][deli]['tags']
+for aprv_group in aprv_groups.keys():
+    # It's possible for built-in metagroups in recent Gerrit releases to
+    # appear in ACLs but not in the groups list
+    if aprv_group in all_groups:
+        aprv_groups[aprv_group] = json.loads(requests.get(
+            gerrit_url + group_path % all_groups[aprv_group]['id'],
+            auth=gerrit_auth).text[4:])
+    else:
+        sys.stderr.write('Ignoring nonexistent "%s" group.\n' % aprv_group)
+for repo in repos:
+    for aprv_group in repos[repo]['approvers'].keys():
+        for approver in aprv_groups[aprv_group]:
+            if 'name' in approver:
+                approver_details = '"%s"' % approver['name']
+            else:
+                approver_details = ''
+            if 'email' in approver:
+                if approver_details:
+                    approver_details += ' '
+                approver_details += '<%s>' % approver['email']
+            if 'username' in approver:
+                if approver_details:
+                    approver_details += ' '
+                approver_details += '(%s)' % approver['username']
+            repos[repo]['approvers'][aprv_group].append(
+                approver_details.encode('utf-8'))
+approvers_yaml = open('approvers.yaml', 'w')
+yaml.dump(repos, approvers_yaml, allow_unicode=True, encoding='utf-8',
+          default_flow_style=False)
+approvers_json = open('approvers.json', 'w')
+json.dump(repos, approvers_json, indent=2)