Merge "Retire bandersnatch mirroring"
This commit is contained in:
Zuul
Gerrit Code Review
commit
0626de0b59
@ -1,57 +0,0 @@
|
|||||||
:title: Bandersnatch
|
|
||||||
|
|
||||||
.. _bandersnatch:
|
|
||||||
|
|
||||||
Bandersnatch
|
|
||||||
############
|
|
||||||
|
|
||||||
A pypi mirror tool
|
|
||||||
|
|
||||||
At a Glance
|
|
||||||
===========
|
|
||||||
|
|
||||||
:Hosts:
|
|
||||||
* http://mirror.bhs1.ovh.openstack.org/pypi
|
|
||||||
* http://mirror.ca-ymq-1.vexxhost.openstack.org/pypi
|
|
||||||
* http://mirror.dfw.rax.openstack.org/pypi
|
|
||||||
* http://mirror.gra1.ovh.openstack.org/pypi
|
|
||||||
* http://mirror.iad.rax.openstack.org/pypi
|
|
||||||
* http://mirror.ord.rax.openstack.org/pypi
|
|
||||||
:Puppet:
|
|
||||||
* :cgit_file:`modules/openstack_project/manifests/static.pp`
|
|
||||||
:Projects:
|
|
||||||
* https://pypi.python.org/pypi/bandersnatch
|
|
||||||
* https://git.openstack.org/cgit/openstack-infra/puppet-bandersnatch
|
|
||||||
:Documentation:
|
|
||||||
* https://pypi.python.org/pypi/bandersnatch#configuration
|
|
||||||
:Bugs:
|
|
||||||
* https://storyboard.openstack.org/#!/project/748
|
|
||||||
* https://bitbucket.org/pypa/bandersnatch/issues?status=new&status=open
|
|
||||||
|
|
||||||
Overview
|
|
||||||
========
|
|
||||||
|
|
||||||
Bandersnatch is a tool we run on the static.openstack.org host to
|
|
||||||
build a complete mirror of pypi.python.org. Cron execs bandersnatch
|
|
||||||
on an interval with logs going to ``/var/log/bandersnatch``.
|
|
||||||
|
|
||||||
Stale Packages
|
|
||||||
==============
|
|
||||||
|
|
||||||
There is an issue with pypi.python.org syncing to its CDN occasionally
|
|
||||||
resulting in stale package artifacts. You will notice this in the
|
|
||||||
bandersnatch logs as::
|
|
||||||
|
|
||||||
2014-07-11 01:30:04,592 INFO: Syncing package: python-novaclient (serial 1154164)
|
|
||||||
2014-07-11 01:30:04,592 DEBUG: Getting /pypi/python-novaclient/json (serial 1154164)
|
|
||||||
2014-07-11 01:30:04,599 DEBUG: Expected PyPI serial 1154164 for request https://pypi.python.org/pypi/python-novaclient/json but got 1154163
|
|
||||||
2014-07-11 01:30:04,599 ERROR: Stale serial for package python-novaclient
|
|
||||||
2014-07-11 01:30:04,599 ERROR: Stale serial for python-novaclient (1154164) not updating. Giving up.
|
|
||||||
|
|
||||||
The fix for this is to issue a PURGE against the url specified above::
|
|
||||||
|
|
||||||
curl -X PURGE https://pypi.python.org/pypi/python-novaclient/json
|
|
||||||
|
|
||||||
The next run of bandersnatch will sync the package. Note this PURGE
|
|
||||||
step should be performed automatically by our bandersnatch wrapper
|
|
||||||
script, but can be performed by hand safely if necessary.
|
|
||||||
@ -26,7 +26,6 @@ Major Systems
|
|||||||
planet
|
planet
|
||||||
puppet
|
puppet
|
||||||
static
|
static
|
||||||
bandersnatch
|
|
||||||
reprepro
|
reprepro
|
||||||
lists
|
lists
|
||||||
wiki
|
wiki
|
||||||
|
|||||||
@ -508,7 +508,6 @@ node /^mirror-update\d*\.openstack\.org$/ {
|
|||||||
$group = "afsadmin"
|
$group = "afsadmin"
|
||||||
|
|
||||||
class { 'openstack_project::mirror_update':
|
class { 'openstack_project::mirror_update':
|
||||||
bandersnatch_keytab => hiera('bandersnatch_keytab'),
|
|
||||||
admin_keytab => hiera('afsadmin_keytab'),
|
admin_keytab => hiera('afsadmin_keytab'),
|
||||||
fedora_keytab => hiera('fedora_keytab'),
|
fedora_keytab => hiera('fedora_keytab'),
|
||||||
opensuse_keytab => hiera('opensuse_keytab'),
|
opensuse_keytab => hiera('opensuse_keytab'),
|
||||||
|
|||||||
@ -83,7 +83,6 @@ INTEGRATION_MODULES["$OPENSTACK_GIT_ROOT/openstack-infra/puppet-accessbot"]="ori
|
|||||||
INTEGRATION_MODULES["$OPENSTACK_GIT_ROOT/openstack-infra/puppet-ansible"]="origin/master"
|
INTEGRATION_MODULES["$OPENSTACK_GIT_ROOT/openstack-infra/puppet-ansible"]="origin/master"
|
||||||
INTEGRATION_MODULES["$OPENSTACK_GIT_ROOT/openstack-infra/puppet-askbot"]="origin/master"
|
INTEGRATION_MODULES["$OPENSTACK_GIT_ROOT/openstack-infra/puppet-askbot"]="origin/master"
|
||||||
INTEGRATION_MODULES["$OPENSTACK_GIT_ROOT/openstack-infra/puppet-asterisk"]="origin/master"
|
INTEGRATION_MODULES["$OPENSTACK_GIT_ROOT/openstack-infra/puppet-asterisk"]="origin/master"
|
||||||
INTEGRATION_MODULES["$OPENSTACK_GIT_ROOT/openstack-infra/puppet-bandersnatch"]="origin/master"
|
|
||||||
INTEGRATION_MODULES["$OPENSTACK_GIT_ROOT/openstack-infra/puppet-bugdaystats"]="origin/master"
|
INTEGRATION_MODULES["$OPENSTACK_GIT_ROOT/openstack-infra/puppet-bugdaystats"]="origin/master"
|
||||||
INTEGRATION_MODULES["$OPENSTACK_GIT_ROOT/openstack-infra/puppet-bup"]="origin/master"
|
INTEGRATION_MODULES["$OPENSTACK_GIT_ROOT/openstack-infra/puppet-bup"]="origin/master"
|
||||||
INTEGRATION_MODULES["$OPENSTACK_GIT_ROOT/openstack-infra/puppet-cgit"]="origin/master"
|
INTEGRATION_MODULES["$OPENSTACK_GIT_ROOT/openstack-infra/puppet-cgit"]="origin/master"
|
||||||
|
|||||||
@ -1,119 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
|
|
||||||
# Copyright 2016 IBM Corp.
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
||||||
# not use this file except in compliance with the License. You may obtain
|
|
||||||
# a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
||||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
||||||
# License for the specific language governing permissions and limitations
|
|
||||||
# under the License.
|
|
||||||
|
|
||||||
set -e
|
|
||||||
|
|
||||||
# Set up logging, see:
|
|
||||||
# http://www.tldp.org/LDP/abs/html/x17974.html
|
|
||||||
LOG_FILE=$1
|
|
||||||
# Open STDOUT as $LOG_FILE file for write appending.
|
|
||||||
exec 1>>$LOG_FILE
|
|
||||||
# Redirect STDERR to STDOUT
|
|
||||||
exec 2>&1
|
|
||||||
|
|
||||||
START_TIME=$(date --iso-8601=ns)
|
|
||||||
echo $START_TIME
|
|
||||||
echo "Obtaining bandersnatch tokens and running bandersnatch."
|
|
||||||
# Note that the set -e is important above as it will cause us
|
|
||||||
# to not do a vos release if bandersnatch fails. Below we check
|
|
||||||
# additional conditions on whether or not to do a vos release.
|
|
||||||
k5start -t -f /etc/bandersnatch.keytab service/bandersnatch -- timeout -k 2m 4h run-bandersnatch
|
|
||||||
|
|
||||||
# Make sure logs made it to disk
|
|
||||||
sync
|
|
||||||
|
|
||||||
# This is what it looks like when bandersnatch logs it.
|
|
||||||
# 2017-06-09 19:40:02,545 INFO: Syncing package: shodan (serial 2939083)
|
|
||||||
# Need to get package name (shodan) and compare it to upper-constraints in
|
|
||||||
# openstack/requirements (all branches)
|
|
||||||
|
|
||||||
# We get the list of packages out of our own log. There is a lot happening
|
|
||||||
# with sed below so lets talk about it.
|
|
||||||
# First we don't print every line we process (-n) we only print those lines
|
|
||||||
# that match using the trailing /p
|
|
||||||
# Next we only match beginning at our start time to the end of the file
|
|
||||||
# (/from_pattern/,to_pattern)
|
|
||||||
# Then we oonly look for lines that say Syncing package as these actually
|
|
||||||
# give us the package name. We extract the package name from here and print
|
|
||||||
# it.
|
|
||||||
sed -n -e "/$START_TIME/,\$s/.*Syncing\spackage:\s\(.*\)\s(serial\s[0-9]\+)/\1/p" $LOG_FILE | sort -u > /tmp/bandersnatch_updated_packages
|
|
||||||
|
|
||||||
LAST_VOS_RELEASE=$(vos examine mirror.pypi.readonly -format | grep 'updateDate' | head -1 | sed -e 's/updateDate\s\([0-9]\+\)\s.*/\1/')
|
|
||||||
NOW=$(date +%s)
|
|
||||||
DELTA=$((NOW - LAST_VOS_RELEASE))
|
|
||||||
|
|
||||||
NEED_RELEASE="no"
|
|
||||||
if [[ "$DELTA" -gt "14400" ]] ; then
|
|
||||||
NEED_RELEASE="yes"
|
|
||||||
elif [[ $(wc -l /tmp/bandersnatch_updated_packages | cut -d' ' -f 1) -gt "512" ]] ; then
|
|
||||||
# If there are a lot of packages updated just go ahead and sync.
|
|
||||||
NEED_RELEASE="yes"
|
|
||||||
else
|
|
||||||
date --iso-8601=ns
|
|
||||||
echo "Checking package updates against requirements"
|
|
||||||
|
|
||||||
REPO_PATH=/opt/pypi_mirror_update/requirements
|
|
||||||
if ! [ -d $REPO_PATH ] ; then
|
|
||||||
mkdir -p $REPO_PATH
|
|
||||||
fi
|
|
||||||
|
|
||||||
export GIT_DIR="$REPO_PATH/.git"
|
|
||||||
if ! [ -d $GIT_DIR ] ; then
|
|
||||||
git clone https://git.openstack.org/openstack/requirements $REPO_PATH
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Ensure repo contents are up to date
|
|
||||||
git remote update
|
|
||||||
git prune
|
|
||||||
|
|
||||||
PACKAGES=$(cat /tmp/bandersnatch_updated_packages)
|
|
||||||
for BRANCH in `git branch -a | grep 'remotes/origin' | grep -v 'HEAD'` ; do
|
|
||||||
# Make best effort at listing packages but don't fail if we can't
|
|
||||||
# list then on a specific branch for some reason.
|
|
||||||
set +e
|
|
||||||
git show "$BRANCH:upper-constraints.txt" > /tmp/pypi-update-constraints
|
|
||||||
git show "$BRANCH:blacklist.txt" > /tmp/pypi-update-blacklist
|
|
||||||
set -e
|
|
||||||
for PACKAGE in $PACKAGES ; do
|
|
||||||
# Note this regex is as specific as possible to avoid matching
|
|
||||||
# substrings inadverdently.
|
|
||||||
if grep -q -i "^$PACKAGE\(=\+.*\)\?$" /tmp/pypi-update-constraints /tmp/pypi-update-blacklist ; then
|
|
||||||
NEED_RELEASE="yes"
|
|
||||||
break
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
if [[ "$NEED_RELEASE" == "yes" ]] ; then
|
|
||||||
break
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
unset GIT_DIR
|
|
||||||
fi
|
|
||||||
|
|
||||||
date --iso-8601=ns
|
|
||||||
if [[ "$NEED_RELEASE" == "yes" ]] ; then
|
|
||||||
echo "Bandersnatch completed successfully, running vos release."
|
|
||||||
k5start -t -f /etc/afsadmin.keytab service/afsadmin -- vos release -v mirror.pypi
|
|
||||||
else
|
|
||||||
# We minimize vos releases as a release causes the remote AFS caches to
|
|
||||||
# update file metadata on reads. This significantly tanks the performance
|
|
||||||
# of remote caches on the other side of the world. Note this appears to
|
|
||||||
# happen even if vos release doesn't update any data.
|
|
||||||
echo "Bandersnatch completed successfully, not updating as no constrained package was updated."
|
|
||||||
fi
|
|
||||||
|
|
||||||
date --iso-8601=ns
|
|
||||||
echo "Done."
|
|
||||||
@ -1,7 +1,6 @@
|
|||||||
# == Class: openstack_project::mirror_update
|
# == Class: openstack_project::mirror_update
|
||||||
#
|
#
|
||||||
class openstack_project::mirror_update (
|
class openstack_project::mirror_update (
|
||||||
$bandersnatch_keytab = '',
|
|
||||||
$reprepro_keytab = '',
|
$reprepro_keytab = '',
|
||||||
$admin_keytab = '',
|
$admin_keytab = '',
|
||||||
$gem_keytab = '',
|
$gem_keytab = '',
|
||||||
@ -20,45 +19,6 @@ class openstack_project::mirror_update (
|
|||||||
|
|
||||||
class { 'openstack_project::gem_mirror': }
|
class { 'openstack_project::gem_mirror': }
|
||||||
|
|
||||||
class { 'bandersnatch':
|
|
||||||
bandersnatch_source => 'pip3',
|
|
||||||
}
|
|
||||||
|
|
||||||
class { 'bandersnatch::mirror':
|
|
||||||
mirror_root => '/afs/.openstack.org/mirror/pypi',
|
|
||||||
static_root => '/afs/.openstack.org/mirror',
|
|
||||||
hash_index => true,
|
|
||||||
package_blacklist => [
|
|
||||||
# These packages are quite large and release often. Ignore them.
|
|
||||||
tensorflow,
|
|
||||||
tf-nightly,
|
|
||||||
tf-nightly-gpu,
|
|
||||||
tfp-nightly,
|
|
||||||
tfp-nightly-gpu,
|
|
||||||
tensorboard,
|
|
||||||
tb-nightly,
|
|
||||||
mxnet,
|
|
||||||
mxnet-mkl,
|
|
||||||
mxnet-cu75,
|
|
||||||
mxnet-cu75mkl,
|
|
||||||
mxnet-cu80,
|
|
||||||
mxnet-cu80mkl,
|
|
||||||
mxnet-cu80-win,
|
|
||||||
mxnet-cu90,
|
|
||||||
mxnet-cu90mkl,
|
|
||||||
mxnet-cu91,
|
|
||||||
mxnet-cu91mkl,
|
|
||||||
],
|
|
||||||
require => Class['bandersnatch'],
|
|
||||||
}
|
|
||||||
|
|
||||||
file { '/etc/bandersnatch.keytab':
|
|
||||||
owner => 'root',
|
|
||||||
group => 'root',
|
|
||||||
mode => '0400',
|
|
||||||
content => $bandersnatch_keytab,
|
|
||||||
}
|
|
||||||
|
|
||||||
file { '/etc/gem.keytab':
|
file { '/etc/gem.keytab':
|
||||||
owner => 'rubygems',
|
owner => 'rubygems',
|
||||||
group => 'root',
|
group => 'root',
|
||||||
@ -74,14 +34,6 @@ class openstack_project::mirror_update (
|
|||||||
content => $admin_keytab,
|
content => $admin_keytab,
|
||||||
}
|
}
|
||||||
|
|
||||||
file { '/usr/local/bin/bandersnatch-mirror-update':
|
|
||||||
ensure => present,
|
|
||||||
owner => 'root',
|
|
||||||
group => 'root',
|
|
||||||
mode => '0755',
|
|
||||||
source => 'puppet:///modules/openstack_project/bandersnatch-mirror-update.sh',
|
|
||||||
}
|
|
||||||
|
|
||||||
file { '/usr/local/bin/gem-mirror-update':
|
file { '/usr/local/bin/gem-mirror-update':
|
||||||
ensure => present,
|
ensure => present,
|
||||||
owner => 'root',
|
owner => 'root',
|
||||||
@ -90,22 +42,6 @@ class openstack_project::mirror_update (
|
|||||||
source => 'puppet:///modules/openstack_project/gem-mirror-update.sh',
|
source => 'puppet:///modules/openstack_project/gem-mirror-update.sh',
|
||||||
}
|
}
|
||||||
|
|
||||||
cron { 'bandersnatch':
|
|
||||||
# Disabled until we sort out how to mirror without unbound growth.
|
|
||||||
# We may just switch to caching proxy long term.
|
|
||||||
ensure => absent,
|
|
||||||
user => $user,
|
|
||||||
minute => '*/5',
|
|
||||||
command => 'flock -n /var/run/bandersnatch/mirror.lock bandersnatch-mirror-update /var/log/bandersnatch/mirror.log',
|
|
||||||
environment => 'PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin',
|
|
||||||
require => [
|
|
||||||
File['/usr/local/bin/bandersnatch-mirror-update'],
|
|
||||||
File['/etc/afsadmin.keytab'],
|
|
||||||
File['/etc/bandersnatch.keytab'],
|
|
||||||
Class['bandersnatch::mirror']
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
||||||
file { '/etc/reprepro.keytab':
|
file { '/etc/reprepro.keytab':
|
||||||
owner => 'root',
|
owner => 'root',
|
||||||
group => 'root',
|
group => 'root',
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user