Merge "Create dynamic pollster feature"

.gitignore

@@ -23,3 +23,9 @@ releasenotes/build
 
 #IntelJ Idea
 .idea/
+
+#venv
+venv/
+
+#Pyenv files
+.python-version

ceilometer/cmd/polling.py

@@ -69,7 +69,7 @@ CLI_OPTS = [
                     default=['compute', 'central'],
                     dest='polling_namespaces',
                     help='Polling namespace(s) to be used while '
-                         'resource polling'),
+                         'resource polling')
 ]
 
 

ceilometer/declarative.py

@@ -42,6 +42,10 @@ class ResourceDefinitionException(DefinitionException):
     pass
 
 
+class DynamicPollsterDefinitionException(DefinitionException):
+    pass
+
+
 class Definition(object):
     JSONPATH_RW_PARSER = parser.ExtentedJsonPathParser()
     GETTERS_CACHE = {}

ceilometer/opts.py

@@ -73,6 +73,7 @@ def list_opts():
                          ceilometer.compute.virt.libvirt.utils.OPTS,
                          ceilometer.objectstore.swift.OPTS,
                          ceilometer.pipeline.base.OPTS,
+                         ceilometer.polling.manager.POLLING_OPTS,
                          ceilometer.sample.OPTS,
                          ceilometer.utils.OPTS,
                          OPTS)),

ceilometer/polling/dynamic_pollster.py

@@ -0,0 +1,231 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""Dynamic pollster component
+    This component enables operators to create new pollsters on the fly
+    via configuration. The configuration files are read from
+    '/etc/ceilometer/pollsters.d/'. The pollster are defined in YAML files
+    similar to the idea used for handling notifications.
+"""
+
+from oslo_log import log
+from oslo_utils import timeutils
+from requests import RequestException
+
+from ceilometer import declarative
+from ceilometer.polling import plugin_base
+from ceilometer import sample
+
+
+import requests
+from six.moves.urllib import parse as url_parse
+
+LOG = log.getLogger(__name__)
+
+
+class DynamicPollster(plugin_base.PollsterBase):
+
+    OPTIONAL_POLLSTER_FIELDS = ['metadata_fields', 'skip_sample_values',
+                                'value_mapping', 'default_value',
+                                'metadata_mapping',
+                                'preserve_mapped_metadata']
+
+    REQUIRED_POLLSTER_FIELDS = ['name', 'sample_type', 'unit',
+                                'value_attribute', 'endpoint_type',
+                                'url_path']
+
+    ALL_POLLSTER_FIELDS = OPTIONAL_POLLSTER_FIELDS + REQUIRED_POLLSTER_FIELDS
+
+    name = ""
+
+    def __init__(self, pollster_definitions, conf=None):
+        super(DynamicPollster, self).__init__(conf)
+        LOG.debug("Dynamic pollster created with [%s]",
+                  pollster_definitions)
+
+        self.pollster_definitions = pollster_definitions
+        self.validate_pollster_definition()
+
+        if 'metadata_fields' in self.pollster_definitions:
+            LOG.debug("Metadata fields configured to [%s].",
+                      self.pollster_definitions['metadata_fields'])
+
+        self.name = self.pollster_definitions['name']
+        self.obj = self
+
+        if 'skip_sample_values' not in self.pollster_definitions:
+            self.pollster_definitions['skip_sample_values'] = []
+
+        if 'value_mapping' not in self.pollster_definitions:
+            self.pollster_definitions['value_mapping'] = {}
+
+        if 'default_value' not in self.pollster_definitions:
+            self.pollster_definitions['default_value'] = -1
+
+        if 'preserve_mapped_metadata' not in self.pollster_definitions:
+            self.pollster_definitions['preserve_mapped_metadata'] = True
+
+        if 'metadata_mapping' not in self.pollster_definitions:
+            self.pollster_definitions['metadata_mapping'] = {}
+
+    def validate_pollster_definition(self):
+        missing_required_fields = \
+            [field for field in self.REQUIRED_POLLSTER_FIELDS
+             if field not in self.pollster_definitions]
+
+        if missing_required_fields:
+            raise declarative.DynamicPollsterDefinitionException(
+                "Required fields %s not specified."
+                % missing_required_fields, self.pollster_definitions)
+
+        sample_type = self.pollster_definitions['sample_type']
+        if sample_type not in sample.TYPES:
+            raise declarative.DynamicPollsterDefinitionException(
+                "Invalid sample type [%s]. Valid ones are [%s]."
+                % (sample_type, sample.TYPES), self.pollster_definitions)
+
+        for definition_key in self.pollster_definitions:
+            if definition_key not in self.ALL_POLLSTER_FIELDS:
+                LOG.warning(
+                    "Field [%s] defined in [%s] is unknown "
+                    "and will be ignored. Valid fields are [%s].",
+                    definition_key, self.pollster_definitions,
+                    self.ALL_POLLSTER_FIELDS)
+
+    def get_samples(self, manager, cache, resources):
+        if not resources:
+            LOG.debug("No resources received for processing.")
+            yield None
+
+        for endpoint in resources:
+            LOG.debug("Executing get sample on URL [%s].", endpoint)
+
+            samples = list([])
+            try:
+                samples = self.execute_request_get_samples(
+                    keystone_client=manager._keystone, endpoint=endpoint)
+            except RequestException as e:
+                LOG.warning("Error [%s] while loading samples for [%s] "
+                            "for dynamic pollster [%s].",
+                            e, endpoint, self.name)
+
+            for pollster_sample in samples:
+                response_value_attribute_name = self.pollster_definitions[
+                    'value_attribute']
+                value = pollster_sample[response_value_attribute_name]
+
+                skip_sample_values = \
+                    self.pollster_definitions['skip_sample_values']
+                if skip_sample_values and value in skip_sample_values:
+                    LOG.debug("Skipping sample [%s] because value [%s] "
+                              "is configured to be skipped in skip list [%s].",
+                              pollster_sample, value, skip_sample_values)
+                    continue
+
+                value = self.execute_value_mapping(value)
+
+                user_id = None
+                if 'user_id' in pollster_sample:
+                    user_id = pollster_sample["user_id"]
+
+                project_id = None
+                if 'project_id' in pollster_sample:
+                    project_id = pollster_sample["project_id"]
+
+                metadata = []
+                if 'metadata_fields' in self.pollster_definitions:
+                    metadata = dict((k, pollster_sample.get(k))
+                                    for k in self.pollster_definitions[
+                                        'metadata_fields'])
+                self.generate_new_metadata_fields(metadata=metadata)
+                yield sample.Sample(
+                    timestamp=timeutils.isotime(),
+
+                    name=self.pollster_definitions['name'],
+                    type=self.pollster_definitions['sample_type'],
+                    unit=self.pollster_definitions['unit'],
+                    volume=value,
+
+                    user_id=user_id,
+                    project_id=project_id,
+                    resource_id=pollster_sample["id"],
+
+                    resource_metadata=metadata
+                )
+
+    def execute_value_mapping(self, value):
+        value_mapping = self.pollster_definitions['value_mapping']
+        if value_mapping:
+            if value in value_mapping:
+                old_value = value
+                value = value_mapping[value]
+                LOG.debug("Value mapped from [%s] to [%s]",
+                          old_value, value)
+            else:
+                default_value = \
+                    self.pollster_definitions['default_value']
+                LOG.warning(
+                    "Value [%s] was not found in value_mapping [%s]; "
+                    "therefore, we will use the default [%s].",
+                    value, value_mapping, default_value)
+                value = default_value
+        return value
+
+    def generate_new_metadata_fields(self, metadata=None):
+        metadata_mapping = self.pollster_definitions['metadata_mapping']
+        if not metadata_mapping or not metadata:
+            return
+
+        metadata_keys = list(metadata.keys())
+        for k in metadata_keys:
+            if k not in metadata_mapping:
+                continue
+
+            new_key = metadata_mapping[k]
+            metadata[new_key] = metadata[k]
+            LOG.debug("Generating new key [%s] with content [%s] of key [%s]",
+                      new_key, metadata[k], k)
+            if self.pollster_definitions['preserve_mapped_metadata']:
+                continue
+
+            k_value = metadata.pop(k)
+            LOG.debug("Removed key [%s] with value [%s] from "
+                      "metadata set that is sent to Gnocchi.", k, k_value)
+
+    @property
+    def default_discovery(self):
+        return 'endpoint:' + self.pollster_definitions['endpoint_type']
+
+    def execute_request_get_samples(self, keystone_client, endpoint):
+        url = url_parse.urljoin(
+            endpoint, self.pollster_definitions['url_path'])
+        resp = keystone_client.session.get(url, authenticated=True)
+        if resp.status_code != requests.codes.ok:
+            resp.raise_for_status()
+
+        response_json = resp.json()
+
+        entry_size = len(response_json)
+        LOG.debug("Entries [%s] in the JSON for request [%s] "
+                  "for dynamic pollster [%s].",
+                  response_json, url, self.name)
+
+        if entry_size > 0:
+            first_entry_name = None
+            try:
+                first_entry_name = next(iter(response_json))
+            except RuntimeError as e:
+                LOG.debug("Generator threw a StopIteration "
+                          "and we need to catch it [%s].",  e)
+            return response_json[first_entry_name]
+        return []

ceilometer/polling/manager.py

@@ -15,8 +15,10 @@
 # under the License.
 
 import collections
+import glob
 import itertools
 import logging
+import os
 import random
 import uuid
 
@@ -34,8 +36,10 @@ from stevedore import extension
 from tooz import coordination
 
 from ceilometer import agent
+from ceilometer import declarative
 from ceilometer import keystone_client
 from ceilometer import messaging
+from ceilometer.polling import dynamic_pollster
 from ceilometer.polling import plugin_base
 from ceilometer.publisher import utils as publisher_utils
 from ceilometer import utils
@@ -58,6 +62,10 @@ POLLING_OPTS = [
                default=50,
                help='Batch size of samples to send to notification agent, '
                     'Set to 0 to disable'),
+    cfg.MultiStrOpt('pollsters_definitions_dirs',
+                    default=["/etc/ceilometer/pollsters.d"],
+                    help="List of directories with YAML files used "
+                         "to created pollsters.")
 ]
 
 
@@ -93,7 +101,7 @@ class Resources(object):
                     not self.agent_manager.partition_coordinator or
                     self.agent_manager.hashrings[
                         static_resources_group].belongs_to_self(
-                            six.text_type(v))] + source_discovery
+                        six.text_type(v))] + source_discovery
 
         return source_discovery
 
@@ -245,8 +253,12 @@ class AgentManager(cotyledon.Service):
         extensions_fb = (self._extensions_from_builder('poll', namespace)
                          for namespace in namespaces)
 
+        # Create dynamic pollsters
+        extensions_dynamic_pollsters = self.create_dynamic_pollsters()
+
         self.extensions = list(itertools.chain(*list(extensions))) + list(
-            itertools.chain(*list(extensions_fb)))
+            itertools.chain(*list(extensions_fb))) + list(
+            extensions_dynamic_pollsters)
 
         if not self.extensions:
             LOG.warning('No valid pollsters can be loaded from %s '
@@ -280,6 +292,70 @@ class AgentManager(cotyledon.Service):
         self._keystone = None
         self._keystone_last_exception = None
 
+    def create_dynamic_pollsters(self):
+        """Creates dynamic pollsters
+
+        This method Creates dynamic pollsters based on configurations placed on
+        'pollsters_definitions_dirs'
+
+        :return: a list with the dynamic pollsters defined by the operator.
+        """
+
+        pollsters_definitions_dirs = self.conf.pollsters_definitions_dirs
+        if not pollsters_definitions_dirs:
+            LOG.info("Variable 'pollsters_definitions_dirs' not defined.")
+            return []
+
+        LOG.info("Looking for dynamic pollsters configurations at [%s].",
+                 pollsters_definitions_dirs)
+        pollsters_definitions_files = []
+        for directory in pollsters_definitions_dirs:
+            files = glob.glob(os.path.join(directory, "*.yaml"))
+            if not files:
+                LOG.info("No dynamic pollsters found in folder [%s].",
+                         directory)
+                continue
+            for filepath in sorted(files):
+                if filepath is not None:
+                    pollsters_definitions_files.append(filepath)
+
+        if not pollsters_definitions_files:
+            LOG.info("No dynamic pollsters file found in dirs [%s].",
+                     pollsters_definitions_dirs)
+            return []
+
+        pollsters_definitions = {}
+        for pollsters_definitions_file in pollsters_definitions_files:
+            pollsters_cfg = declarative.load_definitions(
+                self.conf, {}, pollsters_definitions_file)
+
+            LOG.info("File [%s] has [%s] dynamic pollster configurations.",
+                     pollsters_definitions_file, len(pollsters_cfg))
+
+            for pollster_cfg in pollsters_cfg:
+                pollster_name = pollster_cfg['name']
+                if pollster_name not in pollsters_definitions:
+                    LOG.info("Loading dynamic pollster [%s] from file [%s].",
+                             pollster_name, pollsters_definitions_file)
+                    try:
+                        dynamic_pollster_object = dynamic_pollster.\
+                            DynamicPollster(pollster_cfg, self.conf)
+                        pollsters_definitions[pollster_name] = \
+                            dynamic_pollster_object
+                    except Exception as e:
+                        LOG.error(
+                            "Error [%s] while loading dynamic pollster [%s].",
+                            e, pollster_name)
+
+                else:
+                    LOG.info(
+                        "Dynamic pollster [%s] is already defined."
+                        "Therefore, we are skipping it.", pollster_name)
+
+        LOG.debug("Total of dynamic pollsters [%s] loaded.",
+                  len(pollsters_definitions))
+        return pollsters_definitions.values()
+
     @staticmethod
     def _get_ext_mgr(namespace, *args, **kwargs):
         def _catch_extension_load_error(mgr, ep, exc):
@@ -371,7 +447,6 @@ class AgentManager(cotyledon.Service):
             futures.ThreadPoolExecutor(max_workers=len(data)))
 
         for interval, polling_task in data.items():
-
             @periodics.periodic(spacing=interval, run_immediately=True)
             def task(running_task):
                 self.interval_task(running_task)
@@ -461,9 +536,9 @@ class AgentManager(cotyledon.Service):
                         service_type = getattr(
                             self.conf.service_types,
                             discoverer.KEYSTONE_REQUIRED_FOR_SERVICE)
-                        if not keystone_client.get_service_catalog(
-                                self.keystone).get_endpoints(
-                                    service_type=service_type):
+                        if not keystone_client.\
+                                get_service_catalog(self.keystone).\
+                                get_endpoints(service_type=service_type):
                             LOG.warning(
                                 'Skipping %(name)s, %(service_type)s service '
                                 'is not registered in keystone',

ceilometer/tests/unit/polling/test_dynamic_pollster.py

@@ -0,0 +1,377 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""Tests for ceilometer/central/manager.py
+"""
+
+
+from ceilometer.declarative import DynamicPollsterDefinitionException
+from ceilometer.polling import dynamic_pollster
+from ceilometer import sample
+
+import copy
+import logging
+import mock
+
+from oslotest import base
+
+import requests
+
+LOG = logging.getLogger(__name__)
+
+
+class TestDynamicPollster(base.BaseTestCase):
+    class FakeResponse(object):
+        status_code = None
+        json_object = None
+
+        def json(self):
+            return self.json_object
+
+        def raise_for_status(self):
+            raise requests.HTTPError("Mock HTTP error.", response=self)
+
+    class FakeManager(object):
+        _keystone = None
+
+    def setUp(self):
+        super(TestDynamicPollster, self).setUp()
+        self.pollster_definition_only_required_fields = {
+            'name': "test-pollster", 'sample_type': "gauge", 'unit': "test",
+            'value_attribute': "volume", 'endpoint_type': "test",
+            'url_path': "v1/test/endpoint/fake"}
+
+        self.pollster_definition_all_fields = {
+            'metadata_fields': "metadata-field-name",
+            'skip_sample_values': ["I-do-not-want-entries-with-this-value"],
+            'value_mapping': {
+                'value-to-map': 'new-value', 'value-to-map-to-numeric': 12
+            },
+            'default_value_mapping': 0,
+            'metadata_mapping': {
+                'old-metadata-name': "new-metadata-name"
+            },
+            'preserve_mapped_metadata': False}
+        self.pollster_definition_all_fields.update(
+            self.pollster_definition_only_required_fields)
+
+    def execute_basic_asserts(self, pollster, pollster_definition):
+        self.assertEqual(pollster, pollster.obj)
+        self.assertEqual(pollster_definition['name'], pollster.name)
+
+        for key in pollster.REQUIRED_POLLSTER_FIELDS:
+            self.assertEqual(pollster_definition[key],
+                             pollster.pollster_definitions[key])
+
+        self.assertEqual(pollster_definition, pollster.pollster_definitions)
+
+    def test_all_required_fields_ok(self):
+        pollster = dynamic_pollster.DynamicPollster(
+            self.pollster_definition_only_required_fields)
+
+        self.execute_basic_asserts(
+            pollster, self.pollster_definition_only_required_fields)
+
+        self.assertEqual(
+            0, len(pollster.pollster_definitions['skip_sample_values']))
+        self.assertEqual(
+            0, len(pollster.pollster_definitions['value_mapping']))
+        self.assertEqual(
+            -1, pollster.pollster_definitions['default_value'])
+        self.assertEqual(
+            0, len(pollster.pollster_definitions['metadata_mapping']))
+        self.assertEqual(
+            True, pollster.pollster_definitions['preserve_mapped_metadata'])
+
+    def test_all_fields_ok(self):
+        pollster = dynamic_pollster.DynamicPollster(
+            self.pollster_definition_all_fields)
+
+        self.execute_basic_asserts(pollster,
+                                   self.pollster_definition_all_fields)
+
+        self.assertEqual(
+            1, len(pollster.pollster_definitions['skip_sample_values']))
+        self.assertEqual(
+            2, len(pollster.pollster_definitions['value_mapping']))
+        self.assertEqual(
+            0, pollster.pollster_definitions['default_value_mapping'])
+        self.assertEqual(
+            1, len(pollster.pollster_definitions['metadata_mapping']))
+        self.assertEqual(
+            False, pollster.pollster_definitions['preserve_mapped_metadata'])
+
+    def test_all_required_fields_exceptions(self):
+        for key in dynamic_pollster.\
+                DynamicPollster.REQUIRED_POLLSTER_FIELDS:
+            pollster_definition = copy.deepcopy(
+                self.pollster_definition_only_required_fields)
+            pollster_definition.pop(key)
+            exception = self.assertRaises(DynamicPollsterDefinitionException,
+                                          dynamic_pollster.DynamicPollster,
+                                          pollster_definition)
+            self.assertEqual("Required fields ['%s'] not specified."
+                             % key, exception.brief_message)
+
+    def test_invalid_sample_type(self):
+        self.pollster_definition_only_required_fields[
+            'sample_type'] = "invalid_sample_type"
+        exception = self.assertRaises(
+            DynamicPollsterDefinitionException,
+            dynamic_pollster.DynamicPollster,
+            self.pollster_definition_only_required_fields)
+        self.assertEqual("Invalid sample type [invalid_sample_type]. "
+                         "Valid ones are [('gauge', 'delta', 'cumulative')].",
+                         exception.brief_message)
+
+    def test_all_valid_sample_type(self):
+        for sample_type in sample.TYPES:
+            self.pollster_definition_only_required_fields[
+                'sample_type'] = sample_type
+            pollster = dynamic_pollster.DynamicPollster(
+                self.pollster_definition_only_required_fields)
+
+            self.execute_basic_asserts(
+                pollster, self.pollster_definition_only_required_fields)
+
+    def test_default_discovery_method(self):
+        pollster = dynamic_pollster.DynamicPollster(
+            self.pollster_definition_only_required_fields)
+
+        self.assertEqual("endpoint:test", pollster.default_discovery)
+
+    @mock.patch('keystoneclient.v2_0.client.Client')
+    def test_execute_request_get_samples_empty_response(self, client_mock):
+        pollster = dynamic_pollster.DynamicPollster(
+            self.pollster_definition_only_required_fields)
+
+        return_value = self.FakeResponse()
+        return_value.status_code = requests.codes.ok
+        return_value.json_object = {}
+
+        client_mock.session.get.return_value = return_value
+
+        samples = pollster.execute_request_get_samples(
+            client_mock, "https://endpoint.server.name/")
+
+        self.assertEqual(0, len(samples))
+
+    @mock.patch('keystoneclient.v2_0.client.Client')
+    def test_execute_request_get_samples_response_non_empty(
+            self, client_mock):
+        pollster = dynamic_pollster.DynamicPollster(
+            self.pollster_definition_only_required_fields)
+
+        return_value = self.FakeResponse()
+        return_value.status_code = requests.codes.ok
+        return_value.json_object = {"firstElement": [{}, {}, {}]}
+
+        client_mock.session.get.return_value = return_value
+
+        samples = pollster.execute_request_get_samples(
+            client_mock, "https://endpoint.server.name/")
+
+        self.assertEqual(3, len(samples))
+
+    @mock.patch('keystoneclient.v2_0.client.Client')
+    def test_execute_request_get_samples_exception_on_request(
+            self, client_mock):
+        pollster = dynamic_pollster.DynamicPollster(
+            self.pollster_definition_only_required_fields)
+
+        return_value = self.FakeResponse()
+        return_value.status_code = requests.codes.bad
+
+        client_mock.session.get.return_value = return_value
+
+        exception = self.assertRaises(requests.HTTPError,
+                                      pollster.execute_request_get_samples,
+                                      client_mock,
+                                      "https://endpoint.server.name/")
+        self.assertEqual("Mock HTTP error.", str(exception))
+
+    def test_generate_new_metadata_fields_no_metadata_mapping(self):
+        metadata = {'name': 'someName',
+                    'value': 1}
+
+        metadata_before_call = copy.deepcopy(metadata)
+
+        self.pollster_definition_only_required_fields['metadata_mapping'] = {}
+        pollster = dynamic_pollster.DynamicPollster(
+            self.pollster_definition_only_required_fields)
+        pollster.generate_new_metadata_fields(metadata)
+
+        self.assertEqual(metadata_before_call, metadata)
+
+    def test_generate_new_metadata_fields_preserve_old_key(self):
+        metadata = {'name': 'someName', 'value': 2}
+
+        expected_metadata = copy.deepcopy(metadata)
+        expected_metadata['balance'] = metadata['value']
+
+        self.pollster_definition_only_required_fields[
+            'metadata_mapping'] = {'value': 'balance'}
+        self.pollster_definition_only_required_fields[
+            'preserve_mapped_metadata'] = True
+        pollster = dynamic_pollster.DynamicPollster(
+            self.pollster_definition_only_required_fields)
+        pollster.generate_new_metadata_fields(metadata)
+
+        self.assertEqual(expected_metadata, metadata)
+
+    def test_generate_new_metadata_fields_preserve_old_key_equals_false(self):
+        metadata = {'name': 'someName', 'value': 1}
+
+        expected_clean_metadata = copy.deepcopy(metadata)
+        expected_clean_metadata['balance'] = metadata['value']
+        expected_clean_metadata.pop('value')
+
+        self.pollster_definition_only_required_fields[
+            'metadata_mapping'] = {'value': 'balance'}
+        self.pollster_definition_only_required_fields[
+            'preserve_mapped_metadata'] = False
+        pollster = dynamic_pollster.DynamicPollster(
+            self.pollster_definition_only_required_fields)
+        pollster.generate_new_metadata_fields(metadata)
+
+        self.assertEqual(expected_clean_metadata, metadata)
+
+    def test_execute_value_mapping_no_value_mapping(self):
+        self.pollster_definition_only_required_fields['value_mapping'] = {}
+        pollster = dynamic_pollster.DynamicPollster(
+            self.pollster_definition_only_required_fields)
+
+        value_to_be_mapped = "test"
+        expected_value = value_to_be_mapped
+        value = pollster.execute_value_mapping(value_to_be_mapped)
+
+        self.assertEqual(expected_value, value)
+
+    def test_execute_value_mapping_no_value_mapping_found_with_default(self):
+        self.pollster_definition_only_required_fields[
+            'value_mapping'] = {'some-possible-value': 15}
+        pollster = dynamic_pollster.DynamicPollster(
+            self.pollster_definition_only_required_fields)
+
+        value_to_be_mapped = "test"
+        expected_value = -1
+        value = pollster.execute_value_mapping(value_to_be_mapped)
+
+        self.assertEqual(expected_value, value)
+
+    def test_execute_value_mapping_no_value_mapping_found_with_custom_default(
+            self):
+        self.pollster_definition_only_required_fields[
+            'value_mapping'] = {'some-possible-value': 5}
+        self.pollster_definition_only_required_fields[
+            'default_value'] = 0
+        pollster = dynamic_pollster.DynamicPollster(
+            self.pollster_definition_only_required_fields)
+
+        value_to_be_mapped = "test"
+        expected_value = 0
+        value = pollster.execute_value_mapping(value_to_be_mapped)
+
+        self.assertEqual(expected_value, value)
+
+    def test_execute_value_mapping(self):
+        self.pollster_definition_only_required_fields[
+            'value_mapping'] = {'test': 'new-value'}
+        pollster = dynamic_pollster.DynamicPollster(
+            self.pollster_definition_only_required_fields)
+
+        value_to_be_mapped = "test"
+        expected_value = 'new-value'
+        value = pollster.execute_value_mapping(value_to_be_mapped)
+
+        self.assertEqual(expected_value, value)
+
+    def test_get_samples_no_resources(self):
+        pollster = dynamic_pollster.DynamicPollster(
+            self.pollster_definition_only_required_fields)
+        samples = pollster.get_samples(None, None, None)
+
+        self.assertEqual(None, next(samples))
+
+    @mock.patch('ceilometer.polling.dynamic_pollster.'
+                'DynamicPollster.execute_request_get_samples')
+    def test_get_samples_empty_samples(self, execute_request_get_samples_mock):
+        execute_request_get_samples_mock.side_effect = []
+
+        pollster = dynamic_pollster.DynamicPollster(
+            self.pollster_definition_only_required_fields)
+
+        fake_manager = self.FakeManager()
+        samples = pollster.get_samples(
+            fake_manager, None, ["https://endpoint.server.name.com/"])
+
+        samples_list = list()
+        try:
+            for s in samples:
+                samples_list.append(s)
+        except RuntimeError as e:
+            LOG.debug("Generator threw a StopIteration "
+                      "and we need to catch it [%s]." % e)
+
+        self.assertEqual(0, len(samples_list))
+
+    def fake_sample_list(self, keystone_client=None, endpoint=None):
+        samples_list = list()
+        samples_list.append(
+            {'name': "sample5", 'volume': 5, 'description': "desc-sample-5",
+             'user_id': "924d1f77-5d75-4b96-a755-1774d6be17af",
+             'project_id': "6c7a0e87-7f2e-45d3-89ca-5a2dbba71a0e",
+             'id': "e335c317-dfdd-4f22-809a-625bd9a5992d"
+             }
+        )
+        samples_list.append(
+            {'name': "sample1", 'volume': 2, 'description': "desc-sample-2",
+             'user_id': "20b5a704-b481-4603-a99e-2636c144b876",
+             'project_id': "6c7a0e87-7f2e-45d3-89ca-5a2dbba71a0e",
+             'id': "2e350554-6c05-4fda-8109-e47b595a714c"
+             }
+        )
+        return samples_list
+
+    @mock.patch.object(
+        dynamic_pollster.DynamicPollster,
+        'execute_request_get_samples',
+        fake_sample_list)
+    def test_get_samples(self):
+        pollster = dynamic_pollster.DynamicPollster(
+            self.pollster_definition_only_required_fields)
+
+        fake_manager = self.FakeManager()
+        samples = pollster.get_samples(
+            fake_manager, None, ["https://endpoint.server.name.com/"])
+
+        samples_list = list(samples)
+        self.assertEqual(2, len(samples_list))
+
+        first_element = [
+            s for s in samples_list
+            if s.resource_id == "e335c317-dfdd-4f22-809a-625bd9a5992d"][0]
+        self.assertEqual(5, first_element.volume)
+        self.assertEqual(
+            "6c7a0e87-7f2e-45d3-89ca-5a2dbba71a0e", first_element.project_id)
+        self.assertEqual(
+            "924d1f77-5d75-4b96-a755-1774d6be17af", first_element.user_id)
+
+        second_element = [
+            s for s in samples_list
+            if s.resource_id == "2e350554-6c05-4fda-8109-e47b595a714c"][0]
+        self.assertEqual(2, second_element.volume)
+        self.assertEqual(
+            "6c7a0e87-7f2e-45d3-89ca-5a2dbba71a0e", second_element.project_id)
+        self.assertEqual(
+            "20b5a704-b481-4603-a99e-2636c144b876", second_element.user_id)

doc/source/admin/index.rst

@@ -20,6 +20,7 @@ Configuration
    telemetry-data-collection
    telemetry-data-pipelines
    telemetry-best-practices
+   telemetry-dynamic-pollster
 
 Data Types
 ==========

doc/source/admin/telemetry-data-collection.rst

@@ -294,6 +294,11 @@ Some of the services polled with this agent are:
 To install and configure this service use the :ref:`install_rdo`
 section in the Installation Tutorials and Guides.
 
+Although Ceilometer has a set of default polling agents, operators can
+add new pollsters dynamically via the dynamic pollsters subsystem
+:ref:`telemetry_dynamic_pollster`.
+
+
 .. _telemetry-ipmi-agent:
 
 IPMI agent

doc/source/admin/telemetry-dynamic-pollster.rst

@@ -0,0 +1,231 @@
+.. _telemetry_dynamic_pollster:
+
+Introduction to dynamic pollster subsystem
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The dynamic pollster feature allows system administrators to
+create/update REST API pollsters on the fly (without changing code).
+The system reads YAML configures that are found in
+``pollsters_definitions_dirs`` parameter, which has the default at
+``/etc/ceilometer/pollsters.d``. Operators can use a single file per
+dynamic pollster or multiple dynamic pollsters per file.
+
+
+Current limitations of the dynamic pollster system
+--------------------------------------------------
+Currently, the following types of APIs are not supported by the
+dynamic pollster system:
+
+*  Paging APIs: if a user configures a dynamic pollster to gather data
+   from a paging API, the pollster will use only the entries from the first
+   page.
+
+*  Tenant APIs: Tenant APIs are the ones that need to be polled in a tenant
+   fashion. This feature is "a nice" to have, but is currently not
+   implemented.
+
+*  non-OpenStack APIs such as RadosGW (currently in development)
+
+*  APIs that return a list of entries directly, without a first key for the
+   list. An example is Aodh alarm list.
+
+
+The dynamic pollsters system configuration
+------------------------------------------
+Each YAML file in the dynamic pollster feature can use the following
+attributes to define a dynamic pollster:
+
+*  ``name``: mandatory field. It specifies the name/key of the dynamic
+   pollster. For instance, a pollster for magnum can use the name
+   ``dynamic.magnum.cluster``;
+
+*  ``sample_type``: mandatory field; it defines the sample type. It must
+   be one of the values: ``gauge``, ``delta``, ``cumulative``;
+
+*  ``unit``: mandatory field; defines the unit of the metric that is
+   being collected. For magnum, for instance, one can use ``cluster`` as
+   the unit or some other meaningful String value;
+
+*  ``value_attribute``: mandatory attribute; defines the attribute in the
+   JSON response from the URL of the component being polled. In our magnum
+   example, we can use ``status`` as the value attribute;
+
+*  ``endpoint_type``: mandatory field; defines the endpoint type that is
+   used to discover the base URL of the component to be monitored; for
+   magnum, one can use ``container-infra``. Other values are accepted such
+   as ``volume`` for cinder endpoints, ``object-store`` for swift, and so
+   on;
+
+*  ``url_path``: mandatory attribute. It defines the path of the request
+   that we execute on the endpoint to gather data. For example, to gather
+   data from magnum, one can use ``v1/clusters/detail``;
+
+*  ``metadata_fields``: optional field. It is a list of all fields that
+   the response of the request executed with ``url_path`` that we want to
+   retrieve. As an example, for magnum, one can use the following values:
+
+  .. code-block:: yaml
+
+    metadata_fields:
+      - "labels"
+      - "updated_at"
+      - "keypair"
+      - "master_flavor_id"
+      - "api_address"
+      - "master_addresses"
+      - "node_count"
+      - "docker_volume_size"
+      - "master_count"
+      - "node_addresses"
+      - "status_reason"
+      - "coe_version"
+      - "cluster_template_id"
+      - "name"
+      - "stack_id"
+      - "created_at"
+      - "discovery_url"
+      - "container_version"
+
+*  ``skip_sample_values``: optional field. It defines the values that
+   might come in the ``value_attribute`` that we want to ignore. For
+   magnun, one could for instance, ignore some of the status it has for
+   clusters. Therefore, data is not gathered for clusters in the defined
+   status.
+
+  .. code-block:: yaml
+
+    skip_sample_values:
+      - "CREATE_FAILED"
+      - "DELETE_FAILED"
+
+*  ``value_mapping``: optional attribute. It defines a mapping for the
+   values that the dynamic pollster is handling. This is the actual value
+   that is sent to Gnocchi or other backends. If there is no mapping
+   specified, we will use the raw value that is obtained with the use of
+   ``value_attribute``. An example for magnum, one can use:
+
+  .. code-block:: yaml
+
+    value_mapping:
+      CREATE_IN_PROGRESS: "0"
+      CREATE_FAILED: "1"
+      CREATE_COMPLETE: "2"
+      UPDATE_IN_PROGRESS: "3"
+      UPDATE_FAILED: "4"
+      UPDATE_COMPLETE: "5"
+      DELETE_IN_PROGRESS: "6"
+      DELETE_FAILED: "7"
+      DELETE_COMPLETE: "8"
+      RESUME_COMPLETE: "9"
+      RESUME_FAILED: "10"
+      RESTORE_COMPLETE: "11"
+      ROLLBACK_IN_PROGRESS: "12"
+      ROLLBACK_FAILED: "13"
+      ROLLBACK_COMPLETE: "14"
+      SNAPSHOT_COMPLETE: "15"
+      CHECK_COMPLETE: "16"
+      ADOPT_COMPLETE: "17"
+
+*  ``default_value``: optional parameter. The default value for
+   the value mapping in case the variable value receives data that is not
+   mapped to something in the ``value_mapping`` configuration. This
+   attribute is only used when ``value_mapping`` is defined. Moreover, it
+   has a default of ``-1``.
+
+*  ``metadata_mapping``: the map used to create new metadata fields. The key
+   is a metadata name that exists in the response of the request we make,
+   and the value of this map is the new desired metadata field that will be
+   created with the content of the metadata that we are mapping.
+   The ``metadata_mapping`` can be created as follows:
+
+  .. code-block:: yaml
+
+    metadata_mapping:
+      name: "display_name"
+      some_attribute: "new_attribute_name"
+
+*  ``preserve_mapped_metadata``: indicates if we preserve the old metadata name
+   when it gets mapped to a new one. The default value is ``True``.
+
+The complete YAML configuration to gather data from Magnum (that has been used
+as an example) is the following:
+
+.. code-block:: yaml
+
+  ---
+
+  - name: "dynamic.magnum.cluster"
+    sample_type: "gauge"
+    unit: "cluster"
+    value_attribute: "status"
+    endpoint_type: "container-infra"
+    url_path: "v1/clusters/detail"
+    metadata_fields:
+      - "labels"
+      - "updated_at"
+      - "keypair"
+      - "master_flavor_id"
+      - "api_address"
+      - "master_addresses"
+      - "node_count"
+      - "docker_volume_size"
+      - "master_count"
+      - "node_addresses"
+      - "status_reason"
+      - "coe_version"
+      - "cluster_template_id"
+      - "name"
+      - "stack_id"
+      - "created_at"
+      - "discovery_url"
+      - "container_version"
+    value_mapping:
+      CREATE_IN_PROGRESS: "0"
+      CREATE_FAILED: "1"
+      CREATE_COMPLETE: "2"
+      UPDATE_IN_PROGRESS: "3"
+      UPDATE_FAILED: "4"
+      UPDATE_COMPLETE: "5"
+      DELETE_IN_PROGRESS: "6"
+      DELETE_FAILED: "7"
+      DELETE_COMPLETE: "8"
+      RESUME_COMPLETE: "9"
+      RESUME_FAILED: "10"
+      RESTORE_COMPLETE: "11"
+      ROLLBACK_IN_PROGRESS: "12"
+      ROLLBACK_FAILED: "13"
+      ROLLBACK_COMPLETE: "14"
+      SNAPSHOT_COMPLETE: "15"
+      CHECK_COMPLETE: "16"
+      ADOPT_COMPLETE: "17"
+
+We can also replicate and enhance some hardcoded pollsters.
+For instance, the pollster to gather VPN connections. Currently,
+it is always persisting `1` for all of the VPN connections it finds.
+However, the VPN connection can have multiple statuses, and we should
+normally only bill for active resources, and not resources on `ERROR`
+states. An example to gather VPN connections data is the following
+(this is just an example, and one can adapt and configure as he/she
+desires):
+
+.. code-block:: yaml
+
+  ---
+
+  - name: "dynamic.network.services.vpn.connection"
+  sample_type: "gauge"
+  unit: "ipsec_site_connection"
+  value_attribute: "status"
+  endpoint_type: "network"
+  url_path: "v2.0/vpn/ipsec-site-connections"
+  metadata_fields:
+    - "name"
+    - "vpnservice_id"
+    - "description"
+    - "status"
+    - "peer_address"
+  value_mapping:
+    ACTIVE: "1"
+  metadata_mapping:
+    name: "display_name"
+  default_value: 0

releasenotes/notes/dynamic-pollster-system-6b45c8c973201b2b.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Add dynamic pollster system. The dynamic pollster system enables operators
+    to gather new metrics on the fly (without needing to code pollsters).