Broke launchpad user sync into its own class.

Additionally, the file installation of the scripts wasn't working,
so the sync script wasn't actually getting installed. This moves
the underlying scripts to be installed by the gerrit module,
because it owns /usr/local/gerrit/scripts, and then manages the
gerrit hooks which call those scripts in the openstack_project
class, since that's where the config choice to enable those
functions really should live.

Change-Id: I54fb9edd9fb0c634d8d9de4e57f9ddad6af63a99
This commit is contained in:
Monty Taylor 2012-07-28 11:27:47 -05:00
parent 777524bba0
commit 6c9634c502
4 changed files with 537 additions and 0 deletions

89
notify_doc_impact.py Executable file
View File

@ -0,0 +1,89 @@
#!/usr/bin/env python
# Copyright (c) 2012 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is designed to be called by a gerrit hook. It searched new
# patchsets for strings like "bug FOO" and updates corresponding Launchpad
# bugs status.
import argparse
import re
import subprocess
import smtplib
from email.mime.text import MIMEText
BASE_DIR = '/home/gerrit2/review_site'
EMAIL_TEMPLATE = """
Hi, I'd like you to take a look at this patch for potential
documentation impact.
%s
Log:
%s
"""
DEST_ADDRESS = 'openstack-docs@lists.openstack.org'
def process_impact(git_log, args):
"""Notify doc team of doc impact"""
email_content = EMAIL_TEMPLATE % (args.change_url, git_log)
msg = MIMEText(email_content)
msg['Subject'] = '[%s] DocImpact review request' % args.project
msg['From'] = 'gerrit2@review.openstack.org'
msg['To'] = DEST_ADDRESS
s = smtplib.SMTP('localhost')
s.sendmail('gerrit2@review.openstack.org', DEST_ADDRESS, msg.as_string())
s.quit()
def docs_impacted(git_log):
"""Determine if a changes log indicates there is a doc impact"""
impact_regexp = r'DocImpact'
return re.search(impact_regexp, git_log, re.IGNORECASE)
def extract_git_log(args):
"""Extract git log of all merged commits"""
cmd = ['git',
'--git-dir=' + BASE_DIR + '/git/' + args.project + '.git',
'log', '--no-merges', args.commit + '^1..' + args.commit]
return subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]
def main():
parser = argparse.ArgumentParser()
parser.add_argument('hook')
#common
parser.add_argument('--change', default=None)
parser.add_argument('--change-url', default=None)
parser.add_argument('--project', default=None)
parser.add_argument('--branch', default=None)
parser.add_argument('--commit', default=None)
#change-merged
parser.add_argument('--submitter', default=None)
#patchset-created
parser.add_argument('--uploader', default=None)
parser.add_argument('--patchset', default=None)
args = parser.parse_args()
# Get git log
git_log = extract_git_log(args)
# Process doc_impacts found in git log
if docs_impacted(git_log):
process_impact(git_log, args)
if __name__ == '__main__':
main()

139
update_blueprint.py Executable file
View File

@ -0,0 +1,139 @@
#!/usr/bin/env python
# Copyright (c) 2011 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is designed to be called by a gerrit hook. It searched new
# patchsets for strings like "blueprint FOO" or "bp FOO" and updates
# corresponding Launchpad blueprints with links back to the change.
from launchpadlib.launchpad import Launchpad
from launchpadlib.uris import LPNET_SERVICE_ROOT
import os
import argparse
import re
import subprocess
import StringIO
import ConfigParser
import MySQLdb
BASE_DIR = '/home/gerrit2/review_site'
GERRIT_CACHE_DIR = os.path.expanduser(os.environ.get('GERRIT_CACHE_DIR',
'~/.launchpadlib/cache'))
GERRIT_CREDENTIALS = os.path.expanduser(os.environ.get('GERRIT_CREDENTIALS',
'~/.launchpadlib/creds'))
GERRIT_CONFIG = os.environ.get('GERRIT_CONFIG',
'/home/gerrit2/review_site/etc/gerrit.config')
GERRIT_SECURE_CONFIG = os.environ.get('GERRIT_SECURE_CONFIG',
'/home/gerrit2/review_site/etc/secure.config')
SPEC_RE = re.compile(r'(blueprint|bp)\s*[#:]?\s*(\S+)', re.I)
BODY_RE = re.compile(r'^\s+.*$')
def get_broken_config(filename):
""" gerrit config ini files are broken and have leading tabs """
text = ""
with open(filename,"r") as conf:
for line in conf.readlines():
text = "%s%s" % (text, line.lstrip())
fp = StringIO.StringIO(text)
c=ConfigParser.ConfigParser()
c.readfp(fp)
return c
GERRIT_CONFIG = get_broken_config(GERRIT_CONFIG)
SECURE_CONFIG = get_broken_config(GERRIT_SECURE_CONFIG)
DB_USER = GERRIT_CONFIG.get("database", "username")
DB_PASS = SECURE_CONFIG.get("database","password")
DB_DB = GERRIT_CONFIG.get("database","database")
def update_spec(launchpad, project, name, subject, link, topic=None):
# For testing, if a project doesn't match openstack/foo, use
# the openstack-ci project instead.
group, project = project.split('/')
if group != 'openstack':
project = 'openstack-ci'
spec = launchpad.projects[project].getSpecification(name=name)
if not spec: return
if spec.whiteboard:
wb = spec.whiteboard.strip()
else:
wb = ''
changed = False
if topic:
topiclink = '%s/#q,topic:%s,n,z' % (link[:link.find('/',8)],
topic)
if topiclink not in wb:
wb += "\n\n\nGerrit topic: %(link)s" % dict(link=topiclink)
changed = True
if link not in wb:
wb += "\n\n\nAddressed by: %(link)s\n %(subject)s\n" % dict(subject=subject,
link=link)
changed = True
if changed:
spec.whiteboard = wb
spec.lp_save()
def find_specs(launchpad, dbconn, args):
git_log = subprocess.Popen(['git',
'--git-dir=' + BASE_DIR + '/git/' + args.project + '.git',
'log', '--no-merges',
args.commit + '^1..' + args.commit],
stdout=subprocess.PIPE).communicate()[0]
cur = dbconn.cursor()
cur.execute("select subject, topic from changes where change_key=%s", args.change)
subject, topic = cur.fetchone()
specs = set([m.group(2) for m in SPEC_RE.finditer(git_log)])
if topic:
topicspec = topic.split('/')[-1]
specs |= set([topicspec])
for spec in specs:
update_spec(launchpad, args.project, spec, subject,
args.change_url, topic)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('hook')
#common
parser.add_argument('--change', default=None)
parser.add_argument('--change-url', default=None)
parser.add_argument('--project', default=None)
parser.add_argument('--branch', default=None)
parser.add_argument('--commit', default=None)
#change-merged
parser.add_argument('--submitter', default=None)
# patchset-created
parser.add_argument('--uploader', default=None)
parser.add_argument('--patchset', default=None)
args = parser.parse_args()
launchpad = Launchpad.login_with('Gerrit User Sync', LPNET_SERVICE_ROOT,
GERRIT_CACHE_DIR,
credentials_file = GERRIT_CREDENTIALS,
version='devel')
conn = MySQLdb.connect(user = DB_USER, passwd = DB_PASS, db = DB_DB)
find_specs(launchpad, conn, args)
if __name__ == '__main__':
main()

238
update_bug.py Executable file
View File

@ -0,0 +1,238 @@
#!/usr/bin/env python
# Copyright (c) 2011 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is designed to be called by a gerrit hook. It searched new
# patchsets for strings like "bug FOO" and updates corresponding Launchpad
# bugs status.
from launchpadlib.launchpad import Launchpad
from launchpadlib.uris import LPNET_SERVICE_ROOT
import os
import argparse
import re
import subprocess
BASE_DIR = '/home/gerrit2/review_site'
GERRIT_CACHE_DIR = os.path.expanduser(os.environ.get('GERRIT_CACHE_DIR',
'~/.launchpadlib/cache'))
GERRIT_CREDENTIALS = os.path.expanduser(os.environ.get('GERRIT_CREDENTIALS',
'~/.launchpadlib/creds'))
def add_change_proposed_message(bugtask, change_url, project, branch):
subject = 'Fix proposed to %s (%s)' % (short_project(project), branch)
body = 'Fix proposed to branch: %s\nReview: %s' % (branch, change_url)
bugtask.bug.newMessage(subject=subject, content=body)
def add_change_merged_message(bugtask, change_url, project, commit,
submitter, branch, git_log):
subject = 'Fix merged to %s (%s)' % (short_project(project), branch)
git_url = 'http://github.com/%s/commit/%s' % (project, commit)
body = '''Reviewed: %s
Committed: %s
Submitter: %s
Branch: %s\n''' % (change_url, git_url, submitter, branch)
body = body + '\n' + git_log
bugtask.bug.newMessage(subject=subject, content=body)
def set_in_progress(bugtask, launchpad, uploader, change_url):
"""Set bug In progress with assignee being the uploader"""
# Retrieve uploader from Launchpad. Use email as search key if
# provided, and only set if there is a clear match.
try:
searchkey = uploader[uploader.rindex("(") + 1:-1]
except ValueError:
searchkey = uploader
persons = launchpad.people.findPerson(text=searchkey)
if len(persons) == 1:
bugtask.assignee = persons[0]
bugtask.status = "In Progress"
bugtask.lp_save()
def set_fix_committed(bugtask):
"""Set bug fix committed"""
bugtask.status = "Fix Committed"
bugtask.lp_save()
def set_fix_released(bugtask):
"""Set bug fix released"""
bugtask.status = "Fix Released"
bugtask.lp_save()
def release_fixcommitted(bugtask):
"""Set bug FixReleased if it was FixCommitted"""
if bugtask.status == u'Fix Committed':
set_fix_released(bugtask)
def tag_in_branchname(bugtask, branch):
"""Tag bug with in-branch-name tag (if name is appropriate)"""
lp_bug = bugtask.bug
branch_name = branch.replace('/', '-')
if branch_name.replace('-', '').isalnum():
lp_bug.tags = lp_bug.tags + ["in-%s" % branch_name]
lp_bug.tags.append("in-%s" % branch_name)
lp_bug.lp_save()
def short_project(full_project_name):
"""Return the project part of the git repository name"""
return full_project_name.split('/')[-1]
def git2lp(full_project_name):
"""Convert Git repo name to Launchpad project"""
project_map = {
'openstack/openstack-ci-puppet': 'openstack-ci',
'openstack-ci/devstack-gate': 'openstack-ci',
'openstack-ci/gerrit': 'openstack-ci',
'openstack-ci/lodgeit': 'openstack-ci',
'openstack-ci/meetbot': 'openstack-ci',
}
return project_map.get(full_project_name, short_project(full_project_name))
def is_direct_release(full_project_name):
"""Test against a list of projects who directly release changes."""
return full_project_name in [
'openstack-ci/devstack-gate',
'openstack-ci/lodgeit',
'openstack-ci/meetbot',
'openstack-dev/devstack',
'openstack/openstack-ci',
'openstack/openstack-ci-puppet',
'openstack/openstack-manuals',
]
def process_bugtask(launchpad, bugtask, git_log, args):
"""Apply changes to bugtask, based on hook / branch..."""
if args.hook == "change-merged":
if args.branch == 'master':
if is_direct_release(args.project):
set_fix_released(bugtask)
else:
set_fix_committed(bugtask)
elif args.branch == 'milestone-proposed':
release_fixcommitted(bugtask)
elif args.branch.startswith('stable/'):
series = args.branch[7:]
# Look for a related task matching the series
for reltask in bugtask.related_tasks:
if reltask.bug_target_name.endswith("/" + series):
# Use fixcommitted if there is any
set_fix_committed(reltask)
break
else:
# Use tagging if there isn't any
tag_in_branchname(bugtask, args.branch)
add_change_merged_message(bugtask, args.change_url, args.project,
args.commit, args.submitter, args.branch,
git_log)
if args.hook == "patchset-created":
if args.branch == 'master':
set_in_progress(bugtask, launchpad, args.uploader, args.change_url)
elif args.branch.startswith('stable/'):
series = args.branch[7:]
for reltask in bugtask.related_tasks:
if reltask.bug_target_name.endswith("/" + series):
set_in_progress(reltask, launchpad,
args.uploader, args.change_url)
break
if args.patchset == '1':
add_change_proposed_message(bugtask, args.change_url,
args.project, args.branch)
def find_bugs(launchpad, git_log, args):
"""Find bugs referenced in the git log and return related bugtasks"""
bug_regexp = r'([Bb]ug|[Ll][Pp])[\s#:]*(\d+)'
tokens = re.split(bug_regexp, git_log)
# Extract unique bug tasks
bugtasks = {}
for token in tokens:
if re.match('^\d+$', token) and (token not in bugtasks):
try:
lp_bug = launchpad.bugs[token]
for lp_task in lp_bug.bug_tasks:
if lp_task.bug_target_name == git2lp(args.project):
bugtasks[token] = lp_task
break
except KeyError:
# Unknown bug
pass
return bugtasks.values()
def extract_git_log(args):
"""Extract git log of all merged commits"""
cmd = ['git',
'--git-dir=' + BASE_DIR + '/git/' + args.project + '.git',
'log', '--no-merges', args.commit + '^1..' + args.commit]
return subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]
def main():
parser = argparse.ArgumentParser()
parser.add_argument('hook')
#common
parser.add_argument('--change', default=None)
parser.add_argument('--change-url', default=None)
parser.add_argument('--project', default=None)
parser.add_argument('--branch', default=None)
parser.add_argument('--commit', default=None)
#change-merged
parser.add_argument('--submitter', default=None)
#patchset-created
parser.add_argument('--uploader', default=None)
parser.add_argument('--patchset', default=None)
args = parser.parse_args()
# Connect to Launchpad
launchpad = Launchpad.login_with('Gerrit User Sync', LPNET_SERVICE_ROOT,
GERRIT_CACHE_DIR,
credentials_file=GERRIT_CREDENTIALS,
version='devel')
# Get git log
git_log = extract_git_log(args)
# Process bugtasks found in git log
for bugtask in find_bugs(launchpad, git_log, args):
process_bugtask(launchpad, bugtask, git_log, args)
if __name__ == '__main__':
main()

71
update_cla_group.py Executable file
View File

@ -0,0 +1,71 @@
#! /usr/bin/env python
# Copyright (C) 2011 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# Add launchpad ids listed in the wiki CLA page to the CLA group in LP.
import os
import urllib
import re
from launchpadlib.launchpad import Launchpad
from launchpadlib.uris import LPNET_SERVICE_ROOT
DEBUG = False
LP_CACHE_DIR = '~/.launchpadlib/cache'
LP_CREDENTIALS = '~/.launchpadlib/creds'
CONTRIBUTOR_RE = re.compile(r'.*?\|\|\s*(?P<name>.*?)\s*\|\|\s*(?P<login>.*?)\s*\|\|\s*(?P<trans>.*?)\s*\|\|.*?')
LINK_RE = re.compile(r'\[\[.*\|\s*(?P<name>.*)\s*\]\]')
for check_path in (os.path.dirname(LP_CACHE_DIR),
os.path.dirname(LP_CREDENTIALS)):
if not os.path.exists(check_path):
os.makedirs(check_path)
wiki_members = []
for line in urllib.urlopen('http://wiki.openstack.org/Contributors?action=raw'):
m = CONTRIBUTOR_RE.match(line)
if m and m.group('login') and m.group('trans'):
login = m.group('login')
if login=="<#c0c0c0>'''Launchpad ID'''": continue
l = LINK_RE.match(login)
if l:
login = l.group('name')
wiki_members.append(login)
launchpad = Launchpad.login_with('CLA Team Sync', LPNET_SERVICE_ROOT,
LP_CACHE_DIR,
credentials_file = LP_CREDENTIALS)
lp_members = []
team = launchpad.people['openstack-cla']
for detail in team.members_details:
user = None
# detail.self_link ==
# 'https://api.launchpad.net/1.0/~team/+member/${username}'
login = detail.self_link.split('/')[-1]
status = detail.status
lp_members.append(login)
for wm in wiki_members:
if wm not in lp_members:
print "Need to add %s to LP" % (wm)
try:
person = launchpad.people[wm]
except:
print 'Unable to find %s on LP'%wm
continue
status = team.addMember(person=person, status="Approved")