Start building a bandersnatch mirror

Bandersnatch is more efficient than pypi-mirror, so we can just
run it in cron on the mirror host. Overlay the two mirrors in
the same dir so that we can transition. Landing this patch will
involve running a manual copy command on the host.

Change-Id: Iad1f3d24016cb8e262d69102ba8847e9dfaa0ca0

modules/openstack_project/files/bandersnatch.conf

@@ -0,0 +1,39 @@
+[mirror]
+; The directory where the mirror data will be stored.
+directory = /srv/static/mirror
+
+; The PyPI server which will be mirrored.
+; master = https://testpypi.python.org
+; scheme for PyPI server MUST be https
+master = https://pypi.python.org
+
+; The network socket timeout to use for all connections. This is set to a
+; somewhat aggressively low value: rather fail quickly temporarily and re-run
+; the client soon instead of having a process hang infinitely and have TCP not
+; catching up for ages.
+timeout = 10
+
+; Number of worker threads to use for parallel downloads.
+; Recommendations for worker thread setting:
+; - leave the default of 3 to avoid overloading the pypi master
+; - official servers located in data centers could run 20 workers
+; - anything beyond 50 is probably unreasonable and avoided by bandersnatch
+workers = 3
+
+; Whether to stop a sync quickly after an error is found or whether to continue
+; syncing but not marking the sync as successful. Value should be "true" or
+; "false".
+stop-on-error = false
+
+; Whether or not files that have been deleted on the master should be deleted
+; on the mirror, too.
+; IMPORTANT: if you are running an official mirror than you *need* to leave
+; this on.
+delete-packages = true
+
+[statistics]
+; A glob pattern matching all access log files that should be processed to
+; generate daily access statistics that will be aggregated on the master PyPI.
+access-log-pattern = /var/log/apache2/pypi.openstack.org_access.*
+
+; vim: set ft=cfg:

modules/openstack_project/files/jenkins_job_builder/config/macros.yaml

@@ -292,7 +292,7 @@
           #!/bin/bash -xe
           run-mirror -c /home/jenkins/pypimirror/etc/pypi-mirror.yaml
           find /home/jenkins/pypimirror/mirror/ \( -name index.html -or -name full.html \) -delete
-          rsync -a --ignore-existing --itemize-changes /home/jenkins/pypimirror/mirror/ jenkins@static.openstack.org:/srv/static/pypi/
+          rsync -a --ignore-existing --itemize-changes /home/jenkins/pypimirror/mirror/openstack/ jenkins@static.openstack.org:/srv/static/mirror/web/openstack/
 
 - builder:
     name: install-buck

modules/openstack_project/manifests/static.pp

@@ -174,26 +174,56 @@ class openstack_project::static (
   ###########################################################
   # Pypi Mirror
 
-  apache::vhost { 'pypi.openstack.org':
+  file { '/srv/static/mirror':
-    port     => 80,
+    ensure  => directory,
-    priority => '50',
+    owner   => 'root',
-    docroot  => '/srv/static/pypi',
+    group   => 'root',
-    require  => File['/srv/static/pypi'],
   }
 
-  file { '/srv/static/pypi':
+  file { '/srv/static/mirror/web':
+    ensure  => directory,
+    owner   => 'root',
+    group   => 'root',
+    require => File['/srv/static/mirror'],
+  }
+
+  file { '/srv/static/mirror/web/openstack':
     ensure  => directory,
     owner   => 'jenkins',
     group   => 'jenkins',
-    require => User['jenkins'],
+    require => [User['jenkins'], File['/srv/static/mirror']],
   }
 
-  file { '/srv/static/pypi/robots.txt':
+  apache::vhost { 'pypi.openstack.org':
+    port     => 80,
+    priority => '50',
+    docroot  => '/srv/static/mirror/web',
+    require  => File['/srv/static/mirror/web'],
+  }
+
+  file { '/srv/static/mirror/web/robots.txt':
     ensure  => present,
     owner   => 'root',
     group   => 'root',
     mode    => '0444',
     source  => 'puppet:///modules/openstack_project/disallow_robots.txt',
-    require => File['/srv/static/pypi'],
+    require => File['/srv/static/mirror/web'],
   }
+
+  package { 'bandersnatch':
+    ensure   => 'present',
+    provider => 'pip',
+  }
+
+  file { '/etc/bandersnatch.conf':
+    ensure  => present,
+    source  => 'puppet:///modules/openstack_project/bandersnatch.conf',
+  }
+
+  cron { 'bandersnatch':
+    minute      => '*/5',
+    command     => 'bandersnatch mirror',
+    environment => 'PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin',
+  }
+
 }