From f8933f4abda4ecfc07ee41f84fd5fd8f6667e95a Mon Sep 17 00:00:00 2001 From: liusheng Date: Mon, 1 Aug 2016 15:35:14 +0800 Subject: [PATCH] Switch to use Glance v2 in image pollsters Since the Glance v1 APIs won't be maintained any more, it is better to switch use Glance v2 for in image pollsters. Change-Id: Ib2df3bb4fdd12649bddffd624714707e1369f6af --- ceilometer/image/discovery.py | 43 +++ ceilometer/image/glance.py | 105 ++----- ceilometer/nova_client.py | 2 +- ceilometer/opts.py | 5 +- ceilometer/tests/unit/image/test_glance.py | 261 ++++++------------ ...2-in-image-pollsters-137a315577d5dc4c.yaml | 8 + setup.cfg | 1 + 7 files changed, 158 insertions(+), 267 deletions(-) create mode 100644 ceilometer/image/discovery.py create mode 100644 releasenotes/notes/use-glance-v2-in-image-pollsters-137a315577d5dc4c.yaml diff --git a/ceilometer/image/discovery.py b/ceilometer/image/discovery.py new file mode 100644 index 0000000000..68c1ed41ad --- /dev/null +++ b/ceilometer/image/discovery.py @@ -0,0 +1,43 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import glanceclient +from oslo_config import cfg + +from ceilometer.agent import plugin_base +from ceilometer import keystone_client + +SERVICE_OPTS = [ + cfg.StrOpt('glance', + default='image', + help='Glance service type.'), +] + +cfg.CONF.register_opts(SERVICE_OPTS, group='service_types') +cfg.CONF.import_group('service_credentials', 'ceilometer.keystone_client') + + +class ImagesDiscovery(plugin_base.DiscoveryBase): + def __init__(self): + super(ImagesDiscovery, self).__init__() + conf = cfg.CONF.service_credentials + self.glance_client = glanceclient.Client( + version='2', + session=keystone_client.get_session(), + region_name=conf.region_name, + interface=conf.interface, + service_type=cfg.CONF.service_types.glance) + + def discover(self, manager, param=None): + """Discover resources to monitor.""" + return self.glance_client.images.list() diff --git a/ceilometer/image/glance.py b/ceilometer/image/glance.py index e25aaedd54..46518cf6a6 100644 --- a/ceilometer/image/glance.py +++ b/ceilometer/image/glance.py @@ -17,63 +17,14 @@ from __future__ import absolute_import -import glanceclient -from oslo_config import cfg - from ceilometer.agent import plugin_base -from ceilometer import keystone_client from ceilometer import sample -OPTS = [ - cfg.IntOpt('glance_page_size', - default=0, - help="Number of items to request in " - "each paginated Glance API request " - "(parameter used by glanceclient). " - "If this is less than or equal to 0, " - "page size is not specified " - "(default value in glanceclient is used)."), -] - -SERVICE_OPTS = [ - cfg.StrOpt('glance', - default='image', - help='Glance service type.'), -] - -cfg.CONF.register_opts(OPTS) -cfg.CONF.register_opts(SERVICE_OPTS, group='service_types') - - class _Base(plugin_base.PollsterBase): - @property def default_discovery(self): - return 'endpoint:%s' % cfg.CONF.service_types.glance - - @staticmethod - def get_glance_client(ksclient, endpoint): - # hard-code v1 glance API version selection while v2 API matures - return glanceclient.Client('1', - session=keystone_client.get_session(), - endpoint=endpoint, - auth=ksclient.session.auth) - - def _get_images(self, ksclient, endpoint): - client = self.get_glance_client(ksclient, endpoint) - page_size = cfg.CONF.glance_page_size - kwargs = {} - if page_size > 0: - kwargs['page_size'] = page_size - return client.images.list(filters={"is_public": None}, **kwargs) - - def _iter_images(self, ksclient, cache, endpoint): - """Iterate over all images.""" - key = '%s-images' % endpoint - if key not in cache: - cache[key] = list(self._get_images(ksclient, endpoint)) - return iter(cache[key]) + return 'images' @staticmethod def extract_image_metadata(image): @@ -81,49 +32,45 @@ class _Base(plugin_base.PollsterBase): for k in [ "status", - "is_public", + "visibility", "name", - "deleted", "container_format", "created_at", "disk_format", "updated_at", - "properties", "min_disk", "protected", "checksum", - "deleted_at", "min_ram", - "size", ]) + "tags", + "virtual_size"]) class ImagePollster(_Base): def get_samples(self, manager, cache, resources): - for endpoint in resources: - for image in self._iter_images(manager.keystone, cache, endpoint): - yield sample.Sample( - name='image', - type=sample.TYPE_GAUGE, - unit='image', - volume=1, - user_id=None, - project_id=image.owner, - resource_id=image.id, - resource_metadata=self.extract_image_metadata(image), - ) + for image in resources: + yield sample.Sample( + name='image', + type=sample.TYPE_GAUGE, + unit='image', + volume=1, + user_id=None, + project_id=image.owner, + resource_id=image.id, + resource_metadata=self.extract_image_metadata(image), + ) class ImageSizePollster(_Base): def get_samples(self, manager, cache, resources): - for endpoint in resources: - for image in self._iter_images(manager.keystone, cache, endpoint): - yield sample.Sample( - name='image.size', - type=sample.TYPE_GAUGE, - unit='B', - volume=image.size, - user_id=None, - project_id=image.owner, - resource_id=image.id, - resource_metadata=self.extract_image_metadata(image), - ) + for image in resources: + yield sample.Sample( + name='image.size', + type=sample.TYPE_GAUGE, + unit='B', + volume=image.size, + user_id=None, + project_id=image.owner, + resource_id=image.id, + resource_metadata=self.extract_image_metadata(image), + ) diff --git a/ceilometer/nova_client.py b/ceilometer/nova_client.py index eaf03d2861..55d5cde6d0 100644 --- a/ceilometer/nova_client.py +++ b/ceilometer/nova_client.py @@ -40,7 +40,7 @@ SERVICE_OPTS = [ cfg.CONF.register_opts(OPTS) cfg.CONF.register_opts(SERVICE_OPTS, group='service_types') cfg.CONF.import_opt('http_timeout', 'ceilometer.service') -cfg.CONF.import_opt('glance', 'ceilometer.image.glance', 'service_types') +cfg.CONF.import_opt('glance', 'ceilometer.image.discovery', 'service_types') cfg.CONF.import_group('service_credentials', 'ceilometer.keystone_client') LOG = log.getLogger(__name__) diff --git a/ceilometer/opts.py b/ceilometer/opts.py index 8f9f93a1b6..3c82bf1461 100644 --- a/ceilometer/opts.py +++ b/ceilometer/opts.py @@ -34,7 +34,7 @@ import ceilometer.dispatcher.gnocchi import ceilometer.energy.kwapi import ceilometer.event.converter import ceilometer.hardware.discovery -import ceilometer.image.glance +import ceilometer.image.discovery import ceilometer.ipmi.notifications.ironic import ceilometer.ipmi.platform.intel_node_manager import ceilometer.ipmi.pollsters @@ -67,7 +67,6 @@ def list_opts(): ceilometer.compute.virt.inspector.OPTS, ceilometer.compute.virt.libvirt.inspector.OPTS, ceilometer.dispatcher.OPTS, - ceilometer.image.glance.OPTS, ceilometer.ipmi.notifications.ironic.OPTS, ceilometer.middleware.OPTS, ceilometer.network.notifications.OPTS, @@ -113,7 +112,7 @@ def list_opts(): loading.get_auth_plugin_conf_options('password'))), ('service_types', itertools.chain(ceilometer.energy.kwapi.SERVICE_OPTS, - ceilometer.image.glance.SERVICE_OPTS, + ceilometer.image.discovery.SERVICE_OPTS, ceilometer.neutron_client.SERVICE_OPTS, ceilometer.nova_client.SERVICE_OPTS, ceilometer.objectstore.rgw.SERVICE_OPTS, diff --git a/ceilometer/tests/unit/image/test_glance.py b/ceilometer/tests/unit/image/test_glance.py index c9a16cb923..b37fb9e8cd 100644 --- a/ceilometer/tests/unit/image/test_glance.py +++ b/ceilometer/tests/unit/image/test_glance.py @@ -14,214 +14,107 @@ # under the License. import mock -from oslo_config import fixture as fixture_config -from oslotest import base -from oslotest import mockpatch from ceilometer.agent import manager from ceilometer.image import glance +import ceilometer.tests.base as base IMAGE_LIST = [ type('Image', (object,), - {u'status': u'queued', - u'name': "some name", - u'deleted': False, - u'container_format': None, - u'created_at': u'2012-09-18T16:29:46', - u'disk_format': None, - u'updated_at': u'2012-09-18T16:29:46', - u'properties': {}, - u'min_disk': 0, - u'protected': False, - u'id': u'1d21a8d0-25f4-4e0a-b4ec-85f40237676b', - u'location': None, - u'checksum': None, - u'owner': u'4c8364fc20184ed7971b76602aa96184', - u'is_public': True, - u'deleted_at': None, + {u'status': u'active', + u'tags': [], + u'kernel_id': u'fd24d91a-dfd5-4a3c-b990-d4563eb27396', + u'container_format': u'ami', u'min_ram': 0, - u'size': 2048}), + u'ramdisk_id': u'd629522b-ebaa-4c92-9514-9e31fe760d18', + u'updated_at': u'2016-06-20T13: 34: 41Z', + u'visibility': u'public', + u'owner': u'6824974c08974d4db864bbaa6bc08303', + u'file': u'/v2/images/fda54a44-3f96-40bf-ab07-0a4ce9e1761d/file', + u'min_disk': 0, + u'virtual_size': None, + u'id': u'fda54a44-3f96-40bf-ab07-0a4ce9e1761d', + u'size': 25165824, + u'name': u'cirros-0.3.4-x86_64-uec', + u'checksum': u'eb9139e4942121f22bbc2afc0400b2a4', + u'created_at': u'2016-06-20T13: 34: 40Z', + u'disk_format': u'ami', + u'protected': False, + u'schema': u'/v2/schemas/image'}), type('Image', (object,), {u'status': u'active', - u'name': "hello world", - u'deleted': False, - u'container_format': None, - u'created_at': u'2012-09-18T16:27:41', - u'disk_format': None, - u'updated_at': u'2012-09-18T16:27:41', - u'properties': {}, - u'min_disk': 0, - u'protected': False, - u'id': u'22be9f90-864d-494c-aa74-8035fd535989', - u'location': None, - u'checksum': None, - u'owner': u'9e4f98287a0246daa42eaf4025db99d4', - u'is_public': True, - u'deleted_at': None, + u'tags': [], + u'container_format': u'ari', u'min_ram': 0, - u'size': 0}), + u'updated_at': u'2016-06-20T13: 34: 38Z', + u'visibility': u'public', + u'owner': u'6824974c08974d4db864bbaa6bc08303', + u'file': u'/v2/images/d629522b-ebaa-4c92-9514-9e31fe760d18/file', + u'min_disk': 0, + u'virtual_size': None, + u'id': u'd629522b-ebaa-4c92-9514-9e31fe760d18', + u'size': 3740163, + u'name': u'cirros-0.3.4-x86_64-uec-ramdisk', + u'checksum': u'be575a2b939972276ef675752936977f', + u'created_at': u'2016-06-20T13: 34: 37Z', + u'disk_format': u'ari', + u'protected': False, + u'schema': u'/v2/schemas/image'}), type('Image', (object,), - {u'status': u'queued', - u'name': None, - u'deleted': False, - u'container_format': None, - u'created_at': u'2012-09-18T16:23:27', - u'disk_format': "raw", - u'updated_at': u'2012-09-18T16:23:27', - u'properties': {}, - u'min_disk': 0, - u'protected': False, - u'id': u'8d133f6c-38a8-403c-b02c-7071b69b432d', - u'location': None, - u'checksum': None, - u'owner': u'5f8806a76aa34ee8b8fc8397bd154319', - u'is_public': True, - u'deleted_at': None, + {u'status': u'active', + u'tags': [], + u'container_format': u'aki', u'min_ram': 0, - u'size': 1024}), - type('Image', (object,), - {u'status': u'queued', - u'name': "some name", - u'deleted': False, - u'container_format': None, - u'created_at': u'2012-09-18T16:29:46', - u'disk_format': None, - u'updated_at': u'2012-09-18T16:29:46', - u'properties': {}, + u'updated_at': u'2016-06-20T13: 34: 35Z', + u'visibility': u'public', + u'owner': u'6824974c08974d4db864bbaa6bc08303', + u'file': u'/v2/images/fd24d91a-dfd5-4a3c-b990-d4563eb27396/file', u'min_disk': 0, + u'virtual_size': None, + u'id': u'fd24d91a-dfd5-4a3c-b990-d4563eb27396', + u'size': 4979632, + u'name': u'cirros-0.3.4-x86_64-uec-kernel', + u'checksum': u'8a40c862b5735975d82605c1dd395796', + u'created_at': u'2016-06-20T13: 34: 35Z', + u'disk_format': u'aki', u'protected': False, - u'id': u'e753b196-49b4-48e8-8ca5-09ebd9805f40', - u'location': None, - u'checksum': None, - u'owner': u'4c8364fc20184ed7971b76602aa96184', - u'is_public': True, - u'deleted_at': None, - u'min_ram': 0, - u'size': 2048}), + u'schema': u'/v2/schemas/image'}), ] -ENDPOINT = 'end://point' - - -class _BaseObject(object): - pass - - -class FakeGlanceClient(object): - class images(object): - pass - - -class TestManager(manager.AgentManager): - - def __init__(self): - super(TestManager, self).__init__() - self._keystone = mock.Mock() - access = self._keystone.session.auth.get_access.return_value - access.service_catalog.get_endpoints = mock.Mock( - return_value={'image': mock.ANY}) - class TestImagePollsterPageSize(base.BaseTestCase): - - @staticmethod - def fake_get_glance_client(ksclient, endpoint): - glanceclient = FakeGlanceClient() - glanceclient.images.list = mock.MagicMock(return_value=IMAGE_LIST) - return glanceclient - @mock.patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock()) def setUp(self): super(TestImagePollsterPageSize, self).setUp() - self.manager = TestManager() - self.useFixture(mockpatch.PatchObject( - glance._Base, 'get_glance_client', - side_effect=self.fake_get_glance_client)) - self.CONF = self.useFixture(fixture_config.Config()).conf + self.manager = manager.AgentManager() + self.pollster = glance.ImageSizePollster() - def _do_test_iter_images(self, page_size=0, length=0): - self.CONF.set_override("glance_page_size", page_size) - images = list(glance.ImagePollster(). - _iter_images(self.manager.keystone, {}, ENDPOINT)) - kwargs = {} - if page_size > 0: - kwargs['page_size'] = page_size - FakeGlanceClient.images.list.assert_called_with( - filters={'is_public': None}, **kwargs) - self.assertEqual(length, len(images)) - - def test_page_size(self): - self._do_test_iter_images(100, 4) - - def test_page_size_default(self): - self._do_test_iter_images(length=4) - - def test_page_size_negative_number(self): - self._do_test_iter_images(-1, 4) + def test_image_pollster(self): + image_samples = list( + self.pollster.get_samples(self.manager, {}, resources=IMAGE_LIST)) + self.assertEqual(3, len(image_samples)) + self.assertEqual('image.size', image_samples[0].name) + self.assertEqual(25165824, image_samples[0].volume) + self.assertEqual('6824974c08974d4db864bbaa6bc08303', + image_samples[0].project_id) + self.assertEqual('fda54a44-3f96-40bf-ab07-0a4ce9e1761d', + image_samples[0].resource_id) -class TestImagePollster(base.BaseTestCase): - - @staticmethod - def fake_get_glance_client(ksclient, endpoint): - glanceclient = _BaseObject() - setattr(glanceclient, "images", _BaseObject()) - setattr(glanceclient.images, - "list", lambda *args, **kwargs: iter(IMAGE_LIST)) - return glanceclient - +class TestImagePageSize(base.BaseTestCase): @mock.patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock()) def setUp(self): - super(TestImagePollster, self).setUp() - self.manager = TestManager() - self.useFixture(mockpatch.PatchObject( - glance._Base, 'get_glance_client', - side_effect=self.fake_get_glance_client)) + super(TestImagePageSize, self).setUp() + self.manager = manager.AgentManager() + self.pollster = glance.ImagePollster() - def test_default_discovery(self): - pollster = glance.ImagePollster() - self.assertEqual('endpoint:image', pollster.default_discovery) - - def test_iter_images(self): - # Tests whether the iter_images method returns a unique image - # list when there is nothing in the cache - images = list(glance.ImagePollster(). - _iter_images(self.manager.keystone, {}, ENDPOINT)) - self.assertEqual(len(set(image.id for image in images)), len(images)) - - def test_iter_images_cached(self): - # Tests whether the iter_images method returns the values from - # the cache - cache = {'%s-images' % ENDPOINT: []} - images = list(glance.ImagePollster(). - _iter_images(self.manager.keystone, cache, - ENDPOINT)) - self.assertEqual([], images) - - def test_image(self): - samples = list(glance.ImagePollster().get_samples(self.manager, {}, - [ENDPOINT])) - self.assertEqual(4, len(samples)) - for sample in samples: - self.assertEqual(1, sample.volume) - - def test_image_size(self): - samples = list(glance.ImageSizePollster().get_samples(self.manager, - {}, - [ENDPOINT])) - self.assertEqual(4, len(samples)) - for image in IMAGE_LIST: - self.assertTrue( - any(map(lambda sample: sample.volume == image.size, - samples))) - - def test_image_get_sample_names(self): - samples = list(glance.ImagePollster().get_samples(self.manager, {}, - [ENDPOINT])) - self.assertEqual(set(['image']), set([s.name for s in samples])) - - def test_image_size_get_sample_names(self): - samples = list(glance.ImageSizePollster().get_samples(self.manager, - {}, - [ENDPOINT])) - self.assertEqual(set(['image.size']), set([s.name for s in samples])) + def test_image_pollster(self): + image_samples = list( + self.pollster.get_samples(self.manager, {}, resources=IMAGE_LIST)) + self.assertEqual(3, len(image_samples)) + self.assertEqual('image', image_samples[0].name) + self.assertEqual(1, image_samples[0].volume) + self.assertEqual('6824974c08974d4db864bbaa6bc08303', + image_samples[0].project_id) + self.assertEqual('fda54a44-3f96-40bf-ab07-0a4ce9e1761d', + image_samples[0].resource_id) diff --git a/releasenotes/notes/use-glance-v2-in-image-pollsters-137a315577d5dc4c.yaml b/releasenotes/notes/use-glance-v2-in-image-pollsters-137a315577d5dc4c.yaml new file mode 100644 index 0000000000..6f7e6e8faa --- /dev/null +++ b/releasenotes/notes/use-glance-v2-in-image-pollsters-137a315577d5dc4c.yaml @@ -0,0 +1,8 @@ +--- +features: + - Since the Glance v1 APIs won't be maintained any more, this change + add the support of glance v2 in images pollsters. + +upgrade: + - > + The option `glance_page_size' has been removed because it's not actually needed. diff --git a/setup.cfg b/setup.cfg index c409bf32d3..30c84a48bc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -75,6 +75,7 @@ ceilometer.discover.central = fw_policy = ceilometer.network.services.discovery:FirewallPolicyDiscovery tripleo_overcloud_nodes = ceilometer.hardware.discovery:NodesDiscoveryTripleO fip_services = ceilometer.network.services.discovery:FloatingIPDiscovery + images = ceilometer.image.discovery:ImagesDiscovery ceilometer.discover.ipmi = local_node = ceilometer.agent.discovery.localnode:LocalNodeDiscovery