Get py34 subunit.run test discovery to work

Currently the tox py34 target uses testtools.run to run
a subset of our test harness. We need to be able to use
pretty_tox.sh just like py27 as we make progress with py34
support.

The first step is to make sure we can discover all the
tests using:
python -m subunit.run discover -t . ./nova/tests/ --list

So, we need to fix a bunch of things for the discovery
to work including updating to a new version of websockify.

In the xen code, we should keep the original import and
add except for py34 as xen uses an older python that does
not work with six.moves.

Depends-On: Ib4ef2e79b28b7180e564b3d6dc2188456c66c08a
Change-Id: I88b6746da6136a7386a173f6cacd42f0b470deee

nova/api/openstack/api_version_request.py

@@ -104,6 +104,10 @@ class APIVersionRequest(object):
         return cmp((self.ver_major, self.ver_minor),
                    (other.ver_major, other.ver_minor))
 
+    def __lt__(self, other):
+        return ((self.ver_major, self.ver_minor) <
+                (other.ver_major, other.ver_minor))
+
     def matches(self, min_version, max_version):
         """Returns whether the version object represents a version
         greater than or equal to the minimum version and less than

nova/console/websocketproxy.py

@@ -18,10 +18,10 @@ Websocket proxy that is compatible with OpenStack Nova.
 Leverages websockify.py by Joel Martin
 '''
 
-import Cookie
+from six.moves import http_cookies as Cookie
+import six.moves.urllib.parse as urlparse
 import socket
 import sys
-import urlparse
 
 from oslo_log import log as logging
 import websockify

nova/tests/unit/api/ec2/test_api.py

@@ -18,7 +18,7 @@
 
 import random
 import re
-import StringIO
+from six.moves import StringIO
 
 import boto
 import boto.connection
@@ -28,7 +28,7 @@ from boto import exception as boto_exc
 if hasattr(boto.connection, 'HTTPResponse'):
     httplib = boto.connection
 else:
-    import httplib
+    from six.moves import http_client as httplib
 import fixtures
 from oslo_utils import versionutils
 import webob
@@ -47,7 +47,7 @@ class FakeHttplibSocket(object):
     """a fake socket implementation for httplib.HTTPResponse, trivial."""
     def __init__(self, response_string):
         self.response_string = response_string
-        self._buffer = StringIO.StringIO(response_string)
+        self._buffer = StringIO(response_string)
 
     def makefile(self, _mode, _other):
         """Returns the socket's internal buffer."""

nova/tests/unit/api/openstack/compute/contrib/test_extended_ips.py

@@ -84,7 +84,7 @@ for cache in NW_CACHE:
             ALL_IPS.append(sanitized)
             for floating in fixed['floating_ips']:
                 ALL_IPS.append(floating)
-ALL_IPS.sort()
+ALL_IPS.sort(key=lambda x: str(x))
 
 
 def fake_compute_get(*args, **kwargs):

nova/tests/unit/api/openstack/compute/contrib/test_extended_ips_mac.py

@@ -90,7 +90,7 @@ for cache in NW_CACHE:
                 sanitized['mac_address'] = cache['address']
                 sanitized.pop('type')
                 ALL_IPS.append(sanitized)
-ALL_IPS.sort()
+ALL_IPS.sort(key=lambda x: '%s-%s' % (x['address'], x['mac_address']))
 
 
 def fake_compute_get(*args, **kwargs):

nova/tests/unit/api/openstack/compute/contrib/test_hypervisors.py

@@ -479,32 +479,34 @@ class HypervisorsTestV2(HypervisorsTestV21):
                           self.TEST_HYPERS_OBJ[0].id)
 
 
+_CELL_PATH = 'cell1'
+
+
 class CellHypervisorsTestV21(HypervisorsTestV21):
-    cell_path = 'cell1'
+    TEST_HYPERS_OBJ = [cells_utils.ComputeNodeProxy(obj, _CELL_PATH)
-    TEST_HYPERS_OBJ = [cells_utils.ComputeNodeProxy(obj, cell_path)
                        for obj in TEST_HYPERS_OBJ]
-    TEST_SERVICES = [cells_utils.ServiceProxy(obj, cell_path)
+    TEST_SERVICES = [cells_utils.ServiceProxy(obj, _CELL_PATH)
                      for obj in TEST_SERVICES]
 
     TEST_SERVERS = [dict(server,
-                         host=cells_utils.cell_with_item(cell_path,
+                         host=cells_utils.cell_with_item(_CELL_PATH,
                                                          server['host']))
                     for server in TEST_SERVERS]
 
     DETAIL_HYPERS_DICTS = copy.deepcopy(HypervisorsTestV21.DETAIL_HYPERS_DICTS)
-    DETAIL_HYPERS_DICTS = [dict(hyp, id=cells_utils.cell_with_item(cell_path,
+    DETAIL_HYPERS_DICTS = [dict(hyp, id=cells_utils.cell_with_item(_CELL_PATH,
                                                                    hyp['id']),
                                 service=dict(hyp['service'],
                                              id=cells_utils.cell_with_item(
-                                                 cell_path,
+                                                 _CELL_PATH,
                                                  hyp['service']['id']),
                                              host=cells_utils.cell_with_item(
-                                                 cell_path,
+                                                 _CELL_PATH,
                                                  hyp['service']['host'])))
                            for hyp in DETAIL_HYPERS_DICTS]
 
     INDEX_HYPER_DICTS = copy.deepcopy(HypervisorsTestV21.INDEX_HYPER_DICTS)
-    INDEX_HYPER_DICTS = [dict(hyp, id=cells_utils.cell_with_item(cell_path,
+    INDEX_HYPER_DICTS = [dict(hyp, id=cells_utils.cell_with_item(_CELL_PATH,
                                                                  hyp['id']))
                          for hyp in INDEX_HYPER_DICTS]
 
@@ -559,21 +561,20 @@ class CellHypervisorsTestV21(HypervisorsTestV21):
 
 
 class CellHypervisorsTestV2(HypervisorsTestV2, CellHypervisorsTestV21):
-    cell_path = 'cell1'
     DETAIL_HYPERS_DICTS = copy.deepcopy(HypervisorsTestV2.DETAIL_HYPERS_DICTS)
-    DETAIL_HYPERS_DICTS = [dict(hyp, id=cells_utils.cell_with_item(cell_path,
+    DETAIL_HYPERS_DICTS = [dict(hyp, id=cells_utils.cell_with_item(_CELL_PATH,
                                                                    hyp['id']),
                                 service=dict(hyp['service'],
                                              id=cells_utils.cell_with_item(
-                                                 cell_path,
+                                                 _CELL_PATH,
                                                  hyp['service']['id']),
                                              host=cells_utils.cell_with_item(
-                                                 cell_path,
+                                                 _CELL_PATH,
                                                  hyp['service']['host'])))
                            for hyp in DETAIL_HYPERS_DICTS]
 
     INDEX_HYPER_DICTS = copy.deepcopy(HypervisorsTestV2.INDEX_HYPER_DICTS)
-    INDEX_HYPER_DICTS = [dict(hyp, id=cells_utils.cell_with_item(cell_path,
+    INDEX_HYPER_DICTS = [dict(hyp, id=cells_utils.cell_with_item(_CELL_PATH,
                                                                  hyp['id']))
                          for hyp in INDEX_HYPER_DICTS]
 

nova/tests/unit/api/openstack/compute/test_limits.py

@@ -17,8 +17,8 @@
 Tests dealing with HTTP rate-limiting.
 """
 
-import httplib
+from six.moves import http_client as httplib
-import StringIO
+from six.moves import StringIO
 
 import mock
 from oslo_serialization import jsonutils
@@ -726,7 +726,7 @@ class FakeHttplibSocket(object):
 
     def __init__(self, response_string):
         """Initialize new `FakeHttplibSocket`."""
-        self._buffer = StringIO.StringIO(response_string)
+        self._buffer = StringIO(response_string)
 
     def makefile(self, _mode, _other):
         """Returns the socket's internal buffer."""

nova/tests/unit/cmd/test_baseproxy.py

@@ -47,9 +47,9 @@ class BaseProxyTestCase(test.NoDBTestCase):
     @mock.patch('os.path.exists', return_value=True)
     @mock.patch.object(logging, 'setup')
     @mock.patch.object(gmr.TextGuruMeditation, 'setup_autorun')
-    @mock.patch.object(websocketproxy.NovaWebSocketProxy, '__init__',
+    @mock.patch('nova.console.websocketproxy.NovaWebSocketProxy.__init__',
                        return_value=None)
-    @mock.patch.object(websocketproxy.NovaWebSocketProxy, 'start_server')
+    @mock.patch('nova.console.websocketproxy.NovaWebSocketProxy.start_server')
     def test_proxy(self, mock_start, mock_init, mock_gmr, mock_log,
                    mock_exists):
         baseproxy.proxy('0.0.0.0', '6080')

nova/tests/unit/image/test_fake.py

@@ -13,7 +13,7 @@
 #    under the License.
 
 import datetime
-import StringIO
+from six.moves import StringIO
 
 from nova import context
 from nova import exception
@@ -108,10 +108,10 @@ class FakeImageServiceTestCase(test.NoDBTestCase):
 
     def test_create_then_get(self):
         blob = 'some data'
-        s1 = StringIO.StringIO(blob)
+        s1 = StringIO(blob)
         self.image_service.create(self.context,
                                   {'id': '32', 'foo': 'bar'},
                                   data=s1)
-        s2 = StringIO.StringIO()
+        s2 = StringIO()
         self.image_service.download(self.context, '32', data=s2)
         self.assertEqual(s2.getvalue(), blob, 'Did not get blob back intact')

nova/tests/unit/image/test_glance.py

@@ -15,7 +15,7 @@
 
 
 import datetime
-import StringIO
+from six.moves import StringIO
 
 import glanceclient.exc
 import mock
@@ -543,7 +543,7 @@ class TestDownloadNoDirectUri(test.NoDBTestCase):
         class FakeDiskException(Exception):
             pass
 
-        class Exceptionator(StringIO.StringIO):
+        class Exceptionator(StringIO):
             def write(self, _):
                 raise FakeDiskException('Disk full!')
 

nova/tests/unit/image/test_transfer_modules.py

@@ -12,7 +12,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import urlparse
+import six.moves.urllib.parse as urlparse
 
 import mock
 

nova/tests/unit/test_metadata.py

@@ -54,7 +54,7 @@ from nova.virt import netutils
 
 CONF = cfg.CONF
 
-USER_DATA_STRING = ("This is an encoded string")
+USER_DATA_STRING = (b"This is an encoded string")
 ENCODE_USER_DATA_STRING = base64.b64encode(USER_DATA_STRING)
 
 

nova/tests/unit/test_nova_manage.py

@@ -13,7 +13,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import StringIO
+from six.moves import StringIO
 import sys
 
 import fixtures
@@ -59,7 +59,7 @@ class FixedIpCommandsTestCase(test.TestCase):
 
     def test_list(self):
         self.useFixture(fixtures.MonkeyPatch('sys.stdout',
-                                             StringIO.StringIO()))
+                                             StringIO()))
         self.commands.list()
         self.assertNotEqual(1, sys.stdout.getvalue().find('192.168.0.100'))
 
@@ -71,7 +71,7 @@ class FixedIpCommandsTestCase(test.TestCase):
             'nova.db.fixed_ip_get_by_host',
             fake_fixed_ip_get_by_host))
         self.useFixture(fixtures.MonkeyPatch('sys.stdout',
-                                             StringIO.StringIO()))
+                                             StringIO()))
         self.commands.list('banana')
         self.assertNotEqual(1, sys.stdout.getvalue().find('192.168.0.100'))
 
@@ -212,7 +212,7 @@ class NetworkCommandsTestCase(test.TestCase):
         def fake_network_get_all(context):
             return [db_fakes.FakeModel(self.net)]
         self.stubs.Set(db, 'network_get_all', fake_network_get_all)
-        output = StringIO.StringIO()
+        output = StringIO()
         sys.stdout = output
         self.commands.list()
         sys.stdout = sys.__stdout__
@@ -316,7 +316,7 @@ class ProjectCommandsTestCase(test.TestCase):
         self.commands = manage.ProjectCommands()
 
     def test_quota(self):
-        output = StringIO.StringIO()
+        output = StringIO()
         sys.stdout = output
         self.commands.quota(project_id='admin',
                             key='instances',
@@ -339,7 +339,7 @@ class VmCommandsTestCase(test.TestCase):
         self.fake_flavor = objects.Flavor(**test_flavors.DEFAULT_FLAVORS[0])
 
     def test_list_without_host(self):
-        output = StringIO.StringIO()
+        output = StringIO()
         sys.stdout = output
         with mock.patch.object(objects.InstanceList, 'get_by_filters') as get:
             get.return_value = objects.InstanceList(
@@ -357,7 +357,7 @@ class VmCommandsTestCase(test.TestCase):
         self.assertIn('foo-host', result)
 
     def test_list_with_host(self):
-        output = StringIO.StringIO()
+        output = StringIO()
         sys.stdout = output
         with mock.patch.object(objects.InstanceList, 'get_by_host') as get:
             get.return_value = objects.InstanceList(
@@ -387,7 +387,7 @@ class DBCommandsTestCase(test.TestCase):
                        return_value={'foo': 0})
     def test_null_instance_uuid_scan_no_records_found(self, mock_scan):
         self.useFixture(fixtures.MonkeyPatch('sys.stdout',
-                                             StringIO.StringIO()))
+                                             StringIO()))
         self.commands.null_instance_uuid_scan()
         self.assertIn("There were no records found", sys.stdout.getvalue())
 
@@ -395,7 +395,7 @@ class DBCommandsTestCase(test.TestCase):
                        return_value={'foo': 1, 'bar': 0})
     def _test_null_instance_uuid_scan(self, mock_scan, delete):
         self.useFixture(fixtures.MonkeyPatch('sys.stdout',
-                                             StringIO.StringIO()))
+                                             StringIO()))
         self.commands.null_instance_uuid_scan(delete)
         output = sys.stdout.getvalue()
 

nova/tests/unit/test_policy.py

@@ -16,7 +16,7 @@
 """Test of Policy Engine For Nova."""
 
 import os.path
-import StringIO
+from six.moves import StringIO
 
 import mock
 import six.moves.urllib.request as urlrequest
@@ -100,17 +100,17 @@ class PolicyTestCase(test.NoDBTestCase):
         result = policy.enforce(self.context, action, self.target)
         self.assertEqual(result, True)
 
-    @mock.patch.object(urlrequest, 'urlopen',
+    @mock.patch.object(urlrequest, 'urlopen')
-                       return_value=StringIO.StringIO("True"))
     def test_enforce_http_true(self, mock_urlrequest):
+        mock_urlrequest.return_value = StringIO("True")
         action = "example:get_http"
         target = {}
         result = policy.enforce(self.context, action, target)
         self.assertEqual(result, True)
 
-    @mock.patch.object(urlrequest, 'urlopen',
+    @mock.patch.object(urlrequest, 'urlopen')
-                       return_value=StringIO.StringIO("False"))
     def test_enforce_http_false(self, mock_urlrequest):
+        mock_urlrequest.return_value = StringIO("False")
         action = "example:get_http"
         target = {}
         self.assertRaises(exception.PolicyNotAuthorized, policy.enforce,

nova/tests/unit/virt/libvirt/fake_libvirt_utils.py

@@ -13,7 +13,7 @@
 #    under the License.
 
 import os
-import StringIO
+from six.moves import StringIO
 
 from nova.virt.libvirt import utils as libvirt_utils
 
@@ -137,9 +137,9 @@ def extract_snapshot(disk_path, source_fmt, out_path, dest_fmt):
 class File(object):
     def __init__(self, path, mode=None):
         if path in files:
-            self.fp = StringIO.StringIO(files[path])
+            self.fp = StringIO(files[path])
         else:
-            self.fp = StringIO.StringIO(files[os.path.split(path)[-1]])
+            self.fp = StringIO(files[os.path.split(path)[-1]])
 
     def __enter__(self):
         return self.fp

nova/tests/unit/virt/libvirt/test_driver.py

@@ -13,7 +13,6 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import __builtin__
 import contextlib
 import copy
 import datetime
@@ -45,6 +44,7 @@ from oslo_utils import timeutils
 from oslo_utils import units
 from oslo_utils import uuidutils
 import six
+from six.moves import builtins
 from six.moves import range
 
 from nova.api.metadata import base as instance_metadata
@@ -3511,9 +3511,9 @@ class LibvirtConnTestCase(test.NoDBTestCase):
     def test_get_guest_config_sysinfo_serial_os(self):
         self.flags(sysinfo_serial="os", group="libvirt")
 
-        real_open = __builtin__.open
+        real_open = builtins.open
         with contextlib.nested(
-                mock.patch.object(__builtin__, "open"),
+                mock.patch.object(builtins, "open"),
         ) as (mock_open, ):
             theuuid = "56b40135-a973-4eb3-87bb-a2382a3e6dbc"
 
@@ -3554,10 +3554,10 @@ class LibvirtConnTestCase(test.NoDBTestCase):
         self.flags(sysinfo_serial="auto", group="libvirt")
 
         real_exists = os.path.exists
-        real_open = __builtin__.open
+        real_open = builtins.open
         with contextlib.nested(
                 mock.patch.object(os.path, "exists"),
-                mock.patch.object(__builtin__, "open"),
+                mock.patch.object(builtins, "open"),
         ) as (mock_exists, mock_open):
             def fake_exists(filename):
                 if filename == "/etc/machine-id":

nova/tests/unit/virt/libvirt/test_guest.py

@@ -14,6 +14,8 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import sys
+
 import mock
 from oslo_config import cfg
 from oslo_utils import encodeutils
@@ -33,6 +35,9 @@ libvirt_guest.libvirt = fakelibvirt
 
 CONF = cfg.CONF
 
+if sys.version_info > (3,):
+    long = int
+
 
 class GuestTestCase(test.NoDBTestCase):
 
@@ -130,13 +135,13 @@ class GuestTestCase(test.NoDBTestCase):
         self.domain.resume.assert_called_once_with()
 
     def test_get_vcpus_info(self):
-        self.domain.vcpus.return_value = ([(0, 1, 10290000000L, 2)],
+        self.domain.vcpus.return_value = ([(0, 1, long(10290000000), 2)],
                                      [(True, True)])
         vcpus = list(self.guest.get_vcpus_info())
         self.assertEqual(0, vcpus[0].id)
         self.assertEqual(2, vcpus[0].cpu)
         self.assertEqual(1, vcpus[0].state)
-        self.assertEqual(10290000000L, vcpus[0].time)
+        self.assertEqual(long(10290000000), vcpus[0].time)
 
     def test_delete_configuration(self):
         self.guest.delete_configuration()

nova/tests/unit/virt/libvirt/test_imagecache.py

@@ -15,7 +15,6 @@
 
 
 import contextlib
-import cStringIO
 import hashlib
 import os
 import time
@@ -27,6 +26,7 @@ from oslo_log import formatters
 from oslo_log import log as logging
 from oslo_serialization import jsonutils
 from oslo_utils import importutils
+from six.moves import cStringIO
 
 from nova import conductor
 from nova import context
@@ -47,7 +47,7 @@ CONF.import_opt('host', 'nova.netconf')
 def intercept_log_messages():
     try:
         mylog = logging.getLogger('nova')
-        stream = cStringIO.StringIO()
+        stream = cStringIO()
         handler = logging.logging.StreamHandler(stream)
         handler.setFormatter(formatters.ContextFormatter())
         mylog.logger.addHandler(handler)

nova/virt/xenapi/client/session.py

@@ -13,11 +13,15 @@
 #    under the License.
 
 import contextlib
-import cPickle as pickle
+
+try:
+    import cPickle as pickle
+except ImportError:
+    import pickle
+
 import errno
 import socket
 import time
-import xmlrpclib
 
 from eventlet import queue
 from eventlet import timeout
@@ -26,6 +30,11 @@ from oslo_log import log as logging
 from oslo_utils import versionutils
 from six.moves import range
 
+try:
+    import xmlrpclib
+except ImportError:
+    import six.moves.xmlrpc_client as xmlrpclib
+
 from nova import context
 from nova import exception
 from nova.i18n import _, _LE, _LW

plugins/xenserver/xenapi/etc/xapi.d/plugins/glance

@@ -23,7 +23,11 @@
 
 """Handle the uploading and downloading of images via Glance."""
 
-import httplib
+try:
+    import httplib
+except ImportError:
+    from six.moves import http_client as httplib
+
 import md5
 import socket
 import urllib2

plugins/xenserver/xenapi/etc/xapi.d/plugins/utils.py

@@ -17,7 +17,11 @@
 
 """Various utilities used by XenServer plugins."""
 
-import cPickle as pickle
+try:
+    import cPickle as pickle
+except ImportError:
+    import pickle
+
 import errno
 import logging
 import os

plugins/xenserver/xenapi/etc/xapi.d/plugins/xenhost

@@ -32,7 +32,6 @@ import logging
 import re
 import sys
 import time
-import xmlrpclib
 
 import utils
 
@@ -40,6 +39,11 @@ import pluginlib_nova as pluginlib
 import XenAPI
 import XenAPIPlugin
 
+try:
+    import xmlrpclib
+except ImportError:
+    import six.moves.xmlrpc_client as xmlrpclib
+
 
 pluginlib.configure_logging("xenhost")
 _ = pluginlib._

tox.ini

@@ -35,6 +35,7 @@ deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
 commands =
   find . -type f -name "*.pyc" -delete
+  python -m subunit.run discover -t . ./nova/tests/ --list
   python -m testtools.run \
     nova.tests.unit.compute.test_keypairs \
     nova.tests.unit.db.test_db_api \