Add new pip provider to use pypi mirrors

The upstream pip provider hard-codes pypi.python.org as the index to
search when determining the latest available release of a package. This
means puppet won't use our mirrors on the second run of a pip package
resource using ensure => latest. This patch adds a new provider that
inherits from the upstream one and overrides the latest() method to
just use `pip list --outdated` to see what the latest version is. This could
potentially be proposed upstream in the future depending on what
versions of pip support 'list --outdated' and what systems require
support.

To use this, package resources currently using the 'pip' provider will
have to switch to the 'openstack_pip' provider.

Change-Id: I6c24c8f99fb3f879a30d21f38d1ad883f96f7937

Gemfile

@@ -27,4 +27,8 @@ group :development, :test do
 
 end
 
+group :system_tests do
+  gem 'beaker-rspec', :require => false
+end
+
 # vim:ft=ruby

lib/puppet/provider/package/openstack_pip.rb

@@ -0,0 +1,29 @@
+require 'puppet/provider/package'
+require 'net/http'
+require 'xmlrpc/client'
+require 'puppet/util/http_proxy'
+
+Puppet::Type.type(:package).provide(:openstack_pip, :parent => :pip) do
+
+  desc "Python packages via `pip` with mirrors."
+
+  commands :pip => 'pip'
+
+  def self.outdated
+    @outdated ||= pip(['list', '--outdated'])
+  end
+
+  def latest
+    outdated = self.class.outdated
+    if outdated =~ /#{@resource[:name]}/
+      latest = outdated.split('-')[1].match('Latest: (.*) ')[1]
+    else
+      package_info = lazy_pip(['show', @resource[:name]])
+      current = package_info.split("\n").select { |line|
+        line =~ /^Version/
+      }.first.split(': ')[1]
+      latest = current
+    end
+    return latest
+  end
+end

spec/acceptance/fixtures/openstack_pip.pp

@@ -0,0 +1,35 @@
+include pip
+
+$packages = [
+  'Babel',
+  'bandersnatch',
+  'elasticsearch-curator',
+  'gear',
+  'git-review',
+  'irclog2html',
+  'gerritbot',
+  'keyring',
+  'ndg-httpsclient',
+  'pyasn1',
+  'PyGithub',
+  'pyOpenSSL',
+  'python-jenkins',
+  'python-subunit',
+  'python-swiftclient',
+  'PyYAML',
+  'requests',
+  'requestsexceptions',
+  'shade',
+  'statsd',
+  'SQLAlchemy',
+  'subunit2sql',
+  'testtools',
+  'tox',
+  'virtualenv',
+  'yappi',
+]
+package { $packages:
+  ensure   => latest,
+  provider => openstack_pip,
+  require  => Class['pip'],
+}

spec/acceptance/openstack_pip_provider_spec.rb

@@ -0,0 +1,48 @@
+require 'spec_helper_acceptance'
+
+describe 'custom pip provider' do
+
+  pp = File.read(File.join(File.dirname(__FILE__), 'fixtures/openstack_pip.pp'))
+
+  context 'using mirrors' do
+
+    before :all do
+      # Set up pip.conf for testers playing at home
+      pip_conf =<<EOF
+[global]
+timeout = 60
+index-url = http://mirror.dfw.rax.openstack.org/pypi/simple
+trusted-host = mirror.dfw.rax.openstack.org
+extra-index-url = http://mirror.dfw.rax.openstack.org/wheel/ubuntu-14.04-x86_64
+EOF
+      shell("if [ ! -f /etc/pip.conf ] ; then echo '#{pip_conf}' > /etc/pip.conf ; fi")
+      # Block pypi.python.org so we know the mirror is working
+      shell("iptables -A OUTPUT -d pypi.python.org -j DROP")
+    end
+
+
+    it 'should work with no errors' do
+      apply_manifest(pp, catch_failures: true)
+    end
+
+    # This is where latest will be checked
+    it 'should be idempotent' do
+      apply_manifest(pp, catch_changes: true)
+    end
+
+  end
+
+  context 'without mirrors' do
+
+    before :all do
+      shell("iptables -D OUTPUT -d pypi.python.org -j DROP")
+      shell("rm /etc/pip.conf")
+    end
+
+    it 'should be idempotent' do
+      apply_manifest(pp, catch_changes: true)
+    end
+
+  end
+
+end