diff --git a/modules/openstack_project/files/bandersnatch.conf b/modules/openstack_project/files/bandersnatch.conf new file mode 100644 index 0000000000..bbef5ac1d1 --- /dev/null +++ b/modules/openstack_project/files/bandersnatch.conf @@ -0,0 +1,39 @@ +[mirror] +; The directory where the mirror data will be stored. +directory = /srv/static/mirror + +; The PyPI server which will be mirrored. +; master = https://testpypi.python.org +; scheme for PyPI server MUST be https +master = https://pypi.python.org + +; The network socket timeout to use for all connections. This is set to a +; somewhat aggressively low value: rather fail quickly temporarily and re-run +; the client soon instead of having a process hang infinitely and have TCP not +; catching up for ages. +timeout = 10 + +; Number of worker threads to use for parallel downloads. +; Recommendations for worker thread setting: +; - leave the default of 3 to avoid overloading the pypi master +; - official servers located in data centers could run 20 workers +; - anything beyond 50 is probably unreasonable and avoided by bandersnatch +workers = 3 + +; Whether to stop a sync quickly after an error is found or whether to continue +; syncing but not marking the sync as successful. Value should be "true" or +; "false". +stop-on-error = false + +; Whether or not files that have been deleted on the master should be deleted +; on the mirror, too. +; IMPORTANT: if you are running an official mirror than you *need* to leave +; this on. +delete-packages = true + +[statistics] +; A glob pattern matching all access log files that should be processed to +; generate daily access statistics that will be aggregated on the master PyPI. +access-log-pattern = /var/log/apache2/pypi.openstack.org_access.* + +; vim: set ft=cfg: diff --git a/modules/openstack_project/files/jenkins_job_builder/config/macros.yaml b/modules/openstack_project/files/jenkins_job_builder/config/macros.yaml index ca54f1cbcb..c94e302c67 100644 --- a/modules/openstack_project/files/jenkins_job_builder/config/macros.yaml +++ b/modules/openstack_project/files/jenkins_job_builder/config/macros.yaml @@ -292,7 +292,7 @@ #!/bin/bash -xe run-mirror -c /home/jenkins/pypimirror/etc/pypi-mirror.yaml find /home/jenkins/pypimirror/mirror/ \( -name index.html -or -name full.html \) -delete - rsync -a --ignore-existing --itemize-changes /home/jenkins/pypimirror/mirror/ jenkins@static.openstack.org:/srv/static/pypi/ + rsync -a --ignore-existing --itemize-changes /home/jenkins/pypimirror/mirror/openstack/ jenkins@static.openstack.org:/srv/static/mirror/web/openstack/ - builder: name: install-buck diff --git a/modules/openstack_project/manifests/static.pp b/modules/openstack_project/manifests/static.pp index c523693903..20f4b195d7 100644 --- a/modules/openstack_project/manifests/static.pp +++ b/modules/openstack_project/manifests/static.pp @@ -174,26 +174,56 @@ class openstack_project::static ( ########################################################### # Pypi Mirror - apache::vhost { 'pypi.openstack.org': - port => 80, - priority => '50', - docroot => '/srv/static/pypi', - require => File['/srv/static/pypi'], + file { '/srv/static/mirror': + ensure => directory, + owner => 'root', + group => 'root', } - file { '/srv/static/pypi': + file { '/srv/static/mirror/web': + ensure => directory, + owner => 'root', + group => 'root', + require => File['/srv/static/mirror'], + } + + file { '/srv/static/mirror/web/openstack': ensure => directory, owner => 'jenkins', group => 'jenkins', - require => User['jenkins'], + require => [User['jenkins'], File['/srv/static/mirror']], } - file { '/srv/static/pypi/robots.txt': + apache::vhost { 'pypi.openstack.org': + port => 80, + priority => '50', + docroot => '/srv/static/mirror/web', + require => File['/srv/static/mirror/web'], + } + + file { '/srv/static/mirror/web/robots.txt': ensure => present, owner => 'root', group => 'root', mode => '0444', source => 'puppet:///modules/openstack_project/disallow_robots.txt', - require => File['/srv/static/pypi'], + require => File['/srv/static/mirror/web'], } + + package { 'bandersnatch': + ensure => 'present', + provider => 'pip', + } + + file { '/etc/bandersnatch.conf': + ensure => present, + source => 'puppet:///modules/openstack_project/bandersnatch.conf', + } + + cron { 'bandersnatch': + minute => '*/5', + command => 'bandersnatch mirror', + environment => 'PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin', + } + }