openstack/nova
Code Issues Proposed changes

Rewrite image code to use python-glanceclient

Browse Source 
This patch rewrites the Glance-specific code within Nova to use
the new python-glanceclient library. The old client is deprecated
in Folsom and should not be used in conjunction with
python-glanceclient.

This removes the dependency on installing 'glance' through pip.

Implements bp integrate-python-glanceclient
Fixes bug 880964

Change-Id: Ifb9e75593abd36a2d1c08773d02f192e5e5627fc
This commit is contained in:
Brian Waldon 2012-07-30 13:12:18 -07:00
parent 3fc3dbe6ab
commit 95015ad42a
10 changed files with 163 additions and 200 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

176
nova/image/glance.py
View File

@ -26,8 +26,8 @@ import sys
import time
import urlparse
import glance.client
from glance.common import exception as glance_exception
import glanceclient
import glanceclient.exc
from nova import exception
from nova import flags
@ -56,16 +56,12 @@ def _parse_image_ref(image_href):
def _create_glance_client(context, host, port):
"""Instantiate a new glanceclient.Client object"""
params = {}
if FLAGS.auth_strategy == 'keystone':
params['creds'] = {
'strategy': 'keystone',
'username': context.user_id,
'tenant': context.project_id,
}
params['auth_tok'] = context.auth_token
return glance.client.Client(host, port, **params)
params['token'] = context.auth_token
endpoint = 'http://%s:%s' % (host, port)
return glanceclient.Client('1', endpoint, **params)
def get_api_servers():
@ -110,15 +106,15 @@ class GlanceClientWrapper(object):
Call a glance client method. If we get a connection error,
retry the request according to FLAGS.glance_num_retries.
"""
retry_excs = (glance_exception.ClientConnectionError,
glance_exception.ServiceUnavailable)
retry_excs = (glanceclient.exc.ServiceUnavailable,
glanceclient.exc.InvalidEndpoint,
glanceclient.exc.CommunicationError)
num_attempts = 1 + FLAGS.glance_num_retries
for attempt in xrange(1, num_attempts + 1):
client = self.client or self._create_onetime_client(context)
try:
return getattr(client, method)(*args, **kwargs)
return getattr(client.images, method)(*args, **kwargs)
except retry_excs as e:
host = self.host
port = self.port
@ -143,14 +139,17 @@ class GlanceImageService(object):
def detail(self, context, **kwargs):
"""Calls out to Glance for a list of detailed image information."""
params = self._extract_query_params(kwargs)
image_metas = self._get_images(context, **params)
try:
images = self._client.call(context, 'list', **params)
except Exception:
_reraise_translated_exception()
images = []
for image_meta in image_metas:
if self._is_image_available(context, image_meta):
base_image_meta = self._translate_from_glance(image_meta)
images.append(base_image_meta)
return images
_images = []
for image in images:
if self._is_image_available(context, image):
_images.append(self._translate_from_glance(image))
return _images
def _extract_query_params(self, params):
_params = {}
@ -159,68 +158,31 @@ class GlanceImageService(object):
for param in accepted_params:
if param in params:
_params[param] = params.get(param)
return _params
def _get_images(self, context, **kwargs):
"""Get image entitites from images service"""
# ensure filters is a dict
kwargs['filters'] = kwargs.get('filters') or {}
params.setdefault('filters', {})
# NOTE(vish): don't filter out private images
kwargs['filters'].setdefault('is_public', 'none')
params['filters'].setdefault('is_public', 'none')
return self._fetch_images(context, 'get_images_detailed', **kwargs)
def _fetch_images(self, context, fetch_method, **kwargs):
"""Paginate through results from glance server"""
try:
images = self._client.call(context, fetch_method, **kwargs)
except Exception:
_reraise_translated_exception()
if not images:
# break out of recursive loop to end pagination
return
for image in images:
yield image
try:
# attempt to advance the marker in order to fetch next page
kwargs['marker'] = images[-1]['id']
except KeyError:
raise exception.ImagePaginationFailed()
try:
kwargs['limit'] = kwargs['limit'] - len(images)
# break if we have reached a provided limit
if kwargs['limit'] <= 0:
return
except KeyError:
# ignore missing limit, just proceed without it
pass
for image in self._fetch_images(context, fetch_method, **kwargs):
yield image
return _params
def show(self, context, image_id):
"""Returns a dict with image data for the given opaque image id."""
try:
image_meta = self._client.call(context, 'get_image_meta',
image_id)
image = self._client.call(context, 'get', image_id)
except Exception:
_reraise_translated_image_exception(image_id)
if not self._is_image_available(context, image_meta):
if not self._is_image_available(context, image):
raise exception.ImageNotFound(image_id=image_id)
base_image_meta = self._translate_from_glance(image_meta)
base_image_meta = self._translate_from_glance(image)
return base_image_meta
def download(self, context, image_id, data):
"""Calls out to Glance for metadata and data and writes data."""
try:
image_meta, image_chunks = self._client.call(context,
'get_image', image_id)
image_chunks = self._client.call(context, 'data', image_id)
except Exception:
_reraise_translated_image_exception(image_id)
@ -228,34 +190,34 @@ class GlanceImageService(object):
data.write(chunk)
def create(self, context, image_meta, data=None):
"""Store the image data and return the new image id.
:raises: AlreadyExists if the image already exist.
"""
"""Store the image data and return the new image object."""
sent_service_image_meta = self._translate_to_glance(image_meta)
recv_service_image_meta = self._client.call(context,
'add_image', sent_service_image_meta, data)
base_image_meta = self._translate_from_glance(recv_service_image_meta)
return base_image_meta
def update(self, context, image_id, image_meta, data=None, features=None):
"""Replace the contents of the given image with the new data.
if data:
sent_service_image_meta['data'] = data
:raises: ImageNotFound if the image does not exist.
recv_service_image_meta = self._client.call(context, 'create',
**sent_service_image_meta)
"""
# NOTE(vish): show is to check if image is available
self.show(context, image_id)
return self._translate_from_glance(recv_service_image_meta)
def update(self, context, image_id, image_meta, data=None,
purge_props=True):
"""Modify the given image with the new data."""
image_meta = self._translate_to_glance(image_meta)
image_meta['purge_props'] = purge_props
#NOTE(bcwaldon): id is not an editable field, but it is likely to be
# passed in by calling code. Let's be nice and ignore it.
image_meta.pop('id', None)
if data:
image_meta['data'] = data
try:
image_meta = self._client.call(context, 'update_image',
image_id, image_meta, data, features)
image_meta = self._client.call(context, 'update',
image_id, **image_meta)
except Exception:
_reraise_translated_image_exception(image_id)
base_image_meta = self._translate_from_glance(image_meta)
return base_image_meta
else:
return self._translate_from_glance(image_meta)
def delete(self, context, image_id):
"""Delete the given image.
@ -264,13 +226,11 @@ class GlanceImageService(object):
:raises: NotAuthorized if the user is not an owner.
"""
# NOTE(vish): show is to check if image is available
self.show(context, image_id)
try:
result = self._client.call(context, 'delete_image', image_id)
except glance_exception.NotFound:
self._client.call(context, 'delete', image_id)
except glanceclient.exc.NotFound:
raise exception.ImageNotFound(image_id=image_id)
return result
return True
@staticmethod
def _translate_to_glance(image_meta):
@ -279,14 +239,14 @@ class GlanceImageService(object):
return image_meta
@staticmethod
def _translate_from_glance(image_meta):
image_meta = _limit_attributes(image_meta)
def _translate_from_glance(image):
image_meta = _extract_attributes(image)
image_meta = _convert_timestamps_to_datetimes(image_meta)
image_meta = _convert_from_string(image_meta)
return image_meta
@staticmethod
def _is_image_available(context, image_meta):
def _is_image_available(context, image):
"""Check image availability.
This check is needed in case Nova and Glance are deployed
@ -297,10 +257,10 @@ class GlanceImageService(object):
if hasattr(context, 'auth_token') and context.auth_token:
return True
if image_meta['is_public'] or context.is_admin:
if image.is_public or context.is_admin:
return True
properties = image_meta['properties']
properties = image.properties
if context.project_id and ('owner_id' in properties):
return str(properties['owner_id']) == str(context.project_id)
@ -359,7 +319,7 @@ def _convert_to_string(metadata):
return _convert(_json_dumps, metadata)
def _limit_attributes(image_meta):
def _extract_attributes(image):
IMAGE_ATTRIBUTES = ['size', 'disk_format', 'owner',
'container_format', 'checksum', 'id',
'name', 'created_at', 'updated_at',
@ -367,15 +327,15 @@ def _limit_attributes(image_meta):
'min_disk', 'min_ram', 'is_public']
output = {}
for attr in IMAGE_ATTRIBUTES:
output[attr] = image_meta.get(attr)
output[attr] = getattr(image, attr, None)
output['properties'] = image_meta.get('properties', {})
output['properties'] = getattr(image, 'properties', {})
return output
def _remove_read_only(image_meta):
IMAGE_ATTRIBUTES = ['updated_at', 'created_at', 'deleted_at']
IMAGE_ATTRIBUTES = ['status', 'updated_at', 'created_at', 'deleted_at']
output = copy.deepcopy(image_meta)
for attr in IMAGE_ATTRIBUTES:
if attr in output:
@ -398,25 +358,23 @@ def _reraise_translated_exception():
def _translate_image_exception(image_id, exc_type, exc_value):
if exc_type in (glance_exception.Forbidden,
glance_exception.NotAuthenticated,
glance_exception.MissingCredentialError):
if exc_type in (glanceclient.exc.Forbidden,
glanceclient.exc.Unauthorized):
return exception.ImageNotAuthorized(image_id=image_id)
if exc_type is glance_exception.NotFound:
if exc_type is glanceclient.exc.NotFound:
return exception.ImageNotFound(image_id=image_id)
if exc_type is glance_exception.Invalid:
if exc_type is glanceclient.exc.BadRequest:
return exception.Invalid(exc_value)
return exc_value
def _translate_plain_exception(exc_type, exc_value):
if exc_type in (glance_exception.Forbidden,
glance_exception.NotAuthenticated,
glance_exception.MissingCredentialError):
if exc_type in (glanceclient.exc.Forbidden,
glanceclient.exc.Unauthorized):
return exception.NotAuthorized(exc_value)
if exc_type is glance_exception.NotFound:
if exc_type is glanceclient.exc.NotFound:
return exception.NotFound(exc_value)
if exc_type is glance_exception.Invalid:
if exc_type is glanceclient.exc.BadRequest:
return exception.Invalid(exc_value)
return exc_value

12
nova/image/s3.py
View File

@ -281,15 +281,13 @@ class S3ImageService(object):
def _update_image_state(context, image_uuid, image_state):
metadata = {'properties': {'image_state': image_state}}
headers = {'x-glance-registry-purge-props': False}
self.service.update(context, image_uuid, metadata, None,
headers)
self.service.update(context, image_uuid, metadata,
purge_props=False)
def _update_image_data(context, image_uuid, image_data):
metadata = {}
headers = {'x-glance-registry-purge-props': False}
self.service.update(context, image_uuid, metadata, image_data,
headers)
purge_props=False)
_update_image_state(context, image_uuid, 'downloading')
@ -354,8 +352,8 @@ class S3ImageService(object):
metadata = {'status': 'active',
'properties': {'image_state': 'available'}}
headers = {'x-glance-registry-purge-props': False}
self.service.update(context, image_uuid, metadata, None, headers)
self.service.update(context, image_uuid, metadata,
purge_props=False)
shutil.rmtree(image_path)

2
nova/tests/api/openstack/compute/test_server_actions.py
View File

@ -74,7 +74,7 @@ class ServerActionsControllerTest(test.TestCase):
service_class = 'nova.image.glance.GlanceImageService'
self.service = importutils.import_object(service_class)
self.sent_to_glance = {}
fakes.stub_out_glance_add_image(self.stubs, self.sent_to_glance)
fakes.stub_out_glanceclient_create(self.stubs, self.sent_to_glance)
self.flags(allow_instance_snapshots=True,
enable_instance_password=True)
self.uuid = FAKE_UUID

10
nova/tests/api/openstack/fakes.py
View File

@ -17,7 +17,7 @@
import datetime
from glance import client as glance_client
import glanceclient.v1.images
import routes
import webob
import webob.dec
@ -245,19 +245,19 @@ def _make_image_fixtures():
return fixtures
def stub_out_glance_add_image(stubs, sent_to_glance):
def stub_out_glanceclient_create(stubs, sent_to_glance):
"""
We return the metadata sent to glance by modifying the sent_to_glance dict
in place.
"""
orig_add_image = glance_client.Client.add_image
orig_add_image = glanceclient.v1.images.ImageManager.create
def fake_add_image(context, metadata, data=None):
def fake_create(context, metadata, data=None):
sent_to_glance['metadata'] = metadata
sent_to_glance['data'] = data
return orig_add_image(metadata, data)
stubs.Set(glance_client.Client, 'add_image', fake_add_image)
stubs.Set(glanceclient.v1.images.ImageManager, 'create', fake_create)
def stub_out_glance(stubs):

98
nova/tests/glance/stubs.py
View File

@ -14,7 +14,7 @@
# License for the specific language governing permissions and limitations
# under the License.
from glance.common import exception as glance_exception
import glanceclient.exc
NOW_GLANCE_FORMAT = "2010-10-11T10:30:22"
@ -23,64 +23,90 @@ NOW_GLANCE_FORMAT = "2010-10-11T10:30:22"
class StubGlanceClient(object):
def __init__(self, images=None):
self.images = []
self._images = []
_images = images or []
map(lambda image: self.add_image(image, None), _images)
map(lambda image: self.create(**image), _images)
def set_auth_token(self, auth_tok):
pass
def get_image_meta(self, image_id):
for image in self.images:
if image['id'] == str(image_id):
return image
raise glance_exception.NotFound()
#NOTE(bcwaldon): HACK to get client.images.* to work
self.images = lambda: None
for fn in ('list', 'get', 'data', 'create', 'update', 'delete'):
setattr(self.images, fn, getattr(self, fn))
#TODO(bcwaldon): implement filters
def get_images_detailed(self, filters=None, marker=None, limit=3):
def list(self, filters=None, marker=None, limit=30):
if marker is None:
index = 0
else:
for index, image in enumerate(self.images):
if image['id'] == str(marker):
for index, image in enumerate(self._images):
if image.id == str(marker):
index += 1
break
else:
raise glance_exception.Invalid()
raise glanceclient.exc.BadRequest('Marker not found')
return self.images[index:index + limit]
return self._images[index:index + limit]
def get_image(self, image_id):
return self.get_image_meta(image_id), []
def get(self, image_id):
for image in self._images:
if image.id == str(image_id):
return image
raise glanceclient.exc.NotFound(image_id)
def add_image(self, metadata, data):
def data(self, image_id):
self.get(image_id)
return []
def create(self, **metadata):
metadata['created_at'] = NOW_GLANCE_FORMAT
metadata['updated_at'] = NOW_GLANCE_FORMAT
self.images.append(metadata)
self._images.append(FakeImage(metadata))
try:
image_id = str(metadata['id'])
except KeyError:
# auto-generate an id if one wasn't provided
image_id = str(len(self.images))
image_id = str(len(self._images))
self.images[-1]['id'] = image_id
self._images[-1].id = image_id
return self.images[-1]
return self._images[-1]
def update_image(self, image_id, metadata, data, features):
for i, image in enumerate(self.images):
if image['id'] == str(image_id):
if 'id' in metadata:
metadata['id'] = str(metadata['id'])
self.images[i].update(metadata)
return self.images[i]
raise glance_exception.NotFound()
def update(self, image_id, **metadata):
for i, image in enumerate(self._images):
if image.id == str(image_id):
for k, v in metadata.items():
setattr(self._images[i], k, v)
return self._images[i]
raise glanceclient.exc.NotFound(image_id)
def delete_image(self, image_id):
for i, image in enumerate(self.images):
if image['id'] == image_id:
del self.images[i]
def delete(self, image_id):
for i, image in enumerate(self._images):
if image.id == image_id:
del self._images[i]
return
raise glance_exception.NotFound()
raise glanceclient.exc.NotFound(image_id)
class FakeImage(object):
def __init__(self, metadata):
IMAGE_ATTRIBUTES = ['size', 'disk_format', 'owner',
'container_format', 'checksum', 'id',
'name', 'created_at', 'updated_at',
'deleted', 'status',
'min_disk', 'min_ram', 'is_public']
raw = dict.fromkeys(IMAGE_ATTRIBUTES)
raw.update(metadata)
self.__dict__['raw'] = raw
def __getattr__(self, key):
try:
return self.__dict__['raw'][key]
except KeyError:
raise AttributeError(key)
def __setattr__(self, key, value):
try:
self.__dict__['raw'][key] = value
except KeyError:
raise AttributeError(key)

8
nova/tests/image/fake.py
View File

@ -188,7 +188,7 @@ class _FakeImageService(object):
return self.images[image_id]
def update(self, context, image_id, metadata, data=None,
headers=None):
purge_props=False):
"""Replace the contents of the given image with the new data.
:raises: ImageNotFound if the image does not exist.
@ -196,11 +196,7 @@ class _FakeImageService(object):
"""
if not self.images.get(image_id):
raise exception.ImageNotFound(image_id=image_id)
try:
purge = headers['x-glance-registry-purge-props']
except Exception:
purge = True
if purge:
if purge_props:
self.images[image_id] = copy.deepcopy(metadata)
else:
image = self.images[image_id]

45
nova/tests/image/test_glance.py
View File

@ -20,7 +20,7 @@ import datetime
import random
import time
import glance.common.exception as glance_exception
import glanceclient.exc
from nova import context
from nova import exception
@ -331,7 +331,9 @@ class TestGlanceImageService(test.TestCase):
def test_update(self):
fixture = self._make_fixture(name='test image')
image_id = self.service.create(self.context, fixture)['id']
image = self.service.create(self.context, fixture)
print image
image_id = image['id']
fixture['name'] = 'new image name'
self.service.update(self.context, image_id, fixture)
@ -395,17 +397,6 @@ class TestGlanceImageService(test.TestCase):
self.context,
image_id)
def test_show_raises_on_missing_credential(self):
def raise_missing_credentials(*args, **kwargs):
raise glance_exception.MissingCredentialError()
self.stubs.Set(glance_stubs.StubGlanceClient, 'get_image_meta',
raise_missing_credentials)
self.assertRaises(exception.ImageNotAuthorized,
self.service.show,
self.context,
'test-image-id')
def test_detail_passes_through_to_client(self):
fixture = self._make_fixture(name='image10', is_public=True)
image_id = self.service.create(self.context, fixture)['id']
@ -451,12 +442,12 @@ class TestGlanceImageService(test.TestCase):
class MyGlanceStubClient(glance_stubs.StubGlanceClient):
"""A client that fails the first time, then succeeds."""
def get_image(self, image_id):
def get(self, image_id):
if tries[0] == 0:
tries[0] = 1
raise glance_exception.ClientConnectionError()
raise glanceclient.exc.ServiceUnavailable('')
else:
return {}, []
return {}
client = MyGlanceStubClient()
service = self._create_image_service(client)
@ -476,8 +467,8 @@ class TestGlanceImageService(test.TestCase):
def test_client_raises_forbidden(self):
class MyGlanceStubClient(glance_stubs.StubGlanceClient):
"""A client that fails the first time, then succeeds."""
def get_image(self, image_id):
raise glance_exception.Forbidden()
def get(self, image_id):
raise glanceclient.exc.Forbidden(image_id)
client = MyGlanceStubClient()
service = self._create_image_service(client)
@ -507,11 +498,11 @@ class TestGlanceImageService(test.TestCase):
def _create_failing_glance_client(info):
class MyGlanceStubClient(glance_stubs.StubGlanceClient):
"""A client that fails the first time, then succeeds."""
def get_image(self, image_id):
def get(self, image_id):
info['num_calls'] += 1
if info['num_calls'] == 1:
raise glance_exception.ClientConnectionError()
return {}, []
raise glanceclient.exc.ServiceUnavailable('')
return {}
return MyGlanceStubClient()
@ -548,7 +539,7 @@ class TestGlanceClientWrapper(test.TestCase):
client = glance.GlanceClientWrapper(context=ctxt,
host=fake_host, port=fake_port)
self.assertRaises(exception.GlanceConnectionFailed,
client.call, ctxt, 'get_image', 'meow')
client.call, ctxt, 'get', 'meow')
self.assertEqual(info['num_calls'], 1)
def test_default_client_without_retries(self):
@ -576,7 +567,7 @@ class TestGlanceClientWrapper(test.TestCase):
client = glance.GlanceClientWrapper()
client2 = glance.GlanceClientWrapper()
self.assertRaises(exception.GlanceConnectionFailed,
client.call, ctxt, 'get_image', 'meow')
client.call, ctxt, 'get', 'meow')
self.assertEqual(info['num_calls'], 1)
info = {'num_calls': 0,
@ -590,7 +581,7 @@ class TestGlanceClientWrapper(test.TestCase):
self.stubs.Set(random, 'shuffle', _fake_shuffle2)
self.assertRaises(exception.GlanceConnectionFailed,
client2.call, ctxt, 'get_image', 'meow')
client2.call, ctxt, 'get', 'meow')
self.assertEqual(info['num_calls'], 1)
def test_static_client_with_retries(self):
@ -612,7 +603,7 @@ class TestGlanceClientWrapper(test.TestCase):
client = glance.GlanceClientWrapper(context=ctxt,
host=fake_host, port=fake_port)
client.call(ctxt, 'get_image', 'meow')
client.call(ctxt, 'get', 'meow')
self.assertEqual(info['num_calls'], 2)
def test_default_client_with_retries(self):
@ -642,7 +633,7 @@ class TestGlanceClientWrapper(test.TestCase):
client = glance.GlanceClientWrapper()
client2 = glance.GlanceClientWrapper()
client.call(ctxt, 'get_image', 'meow')
client.call(ctxt, 'get', 'meow')
self.assertEqual(info['num_calls'], 2)
def _fake_shuffle2(servers):
@ -657,5 +648,5 @@ class TestGlanceClientWrapper(test.TestCase):
'host1': 'host3',
'port1': 9294}
client2.call(ctxt, 'get_image', 'meow')
client2.call(ctxt, 'get', 'meow')
self.assertEqual(info['num_calls'], 2)

3
nova/tests/image/test_s3.py
View File

@ -191,8 +191,7 @@ class TestS3ImageService(test.TestCase):
uuid = translated['id']
image_service = fake.FakeImageService()
updated_image = image_service.update(self.context, uuid,
{'is_public': True}, None,
{'x-glance-registry-purge-props': False})
{'is_public': True}, purge_props=False)
self.assertTrue(updated_image['is_public'])
self.assertEqual(updated_image['status'], 'active')
self.assertEqual(updated_image['properties']['image_state'],

7
nova/virt/vmwareapi/read_write_util.py
View File

@ -27,8 +27,6 @@ import urllib
import urllib2
import urlparse
from glance import client
from nova import flags
from nova.openstack.common import log as logging
@ -38,10 +36,7 @@ FLAGS = flags.FLAGS
USER_AGENT = "OpenStack-ESX-Adapter"
try:
READ_CHUNKSIZE = client.BaseClient.CHUNKSIZE
except AttributeError:
READ_CHUNKSIZE = 65536
READ_CHUNKSIZE = 65536
class GlanceFileRead(object):

2
tools/pip-requires
View File

@ -14,7 +14,6 @@ PasteDeploy==1.5.0
paste
sqlalchemy-migrate>=0.7.2
netaddr
glance>=2011.3.1
suds==0.4
paramiko
feedparser
@ -23,3 +22,4 @@ iso8601>=0.1.4
httplib2
setuptools_git>=0.4
python-quantumclient>=0.1,<0.2
python-glanceclient<2