From f05939d742233b44240036e1151a07bb71b4159d Mon Sep 17 00:00:00 2001 From: Hanxi Liu Date: Thu, 19 Jan 2017 15:33:32 +0800 Subject: [PATCH] Support loading multiple meter definition files Closes-Bug: #1479775 Change-Id: Iad15476cabd1f35f13322a0903f4ff0abf9a0160 --- .../{meter/data => data/meters.d}/meters.yaml | 0 ceilometer/meter/notifications.py | 70 +++++--- .../tests/unit/meter/test_notifications.py | 163 +++++++++++++++--- ...ter-definition-files-e3ce1fa73ef2e1de.yaml | 7 + 4 files changed, 196 insertions(+), 44 deletions(-) rename ceilometer/{meter/data => data/meters.d}/meters.yaml (100%) create mode 100644 releasenotes/notes/support-multiple-meter-definition-files-e3ce1fa73ef2e1de.yaml diff --git a/ceilometer/meter/data/meters.yaml b/ceilometer/data/meters.d/meters.yaml similarity index 100% rename from ceilometer/meter/data/meters.yaml rename to ceilometer/data/meters.d/meters.yaml diff --git a/ceilometer/meter/notifications.py b/ceilometer/meter/notifications.py index 8ac36e07c1..49d8bf63ef 100644 --- a/ceilometer/meter/notifications.py +++ b/ceilometer/meter/notifications.py @@ -10,8 +10,10 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. - +import glob import itertools +import os + import pkg_resources import six @@ -27,9 +29,24 @@ from ceilometer import sample as sample_util OPTS = [ cfg.StrOpt('meter_definitions_cfg_file', - default="meters.yaml", - help="Configuration file for defining meter notifications." + deprecated_for_removal=True, + help="Configuration file for defining meter " + "notifications. This option is deprecated " + "and use meter_definitions_dirs to " + "configure meter notification file. Meter " + "definitions configuration file will be sought " + "according to the parameter." ), + cfg.MultiStrOpt('meter_definitions_dirs', + default=["/etc/ceilometer/meters.d", + os.path.abspath( + os.path.join( + os.path.split( + os.path.dirname(__file__))[0], + "data", "meters.d"))], + help="List directory to find files of " + "defining meter notifications." + ), ] LOG = log.getLogger(__name__) @@ -176,26 +193,35 @@ class ProcessMeterNotifications(notification.NotificationProcessBase): def _load_definitions(self): plugin_manager = extension.ExtensionManager( namespace='ceilometer.event.trait_plugin') - meters_cfg = declarative.load_definitions( - self.manager.conf, {}, - self.manager.conf.meter.meter_definitions_cfg_file, - pkg_resources.resource_filename(__name__, "data/meters.yaml")) - definitions = {} - for meter_cfg in reversed(meters_cfg['metric']): - if meter_cfg.get('name') in definitions: - # skip duplicate meters - LOG.warning("Skipping duplicate meter definition %s" - % meter_cfg) - continue - try: - md = MeterDefinition(meter_cfg, self.manager.conf, - plugin_manager) - except declarative.DefinitionException as e: - errmsg = "Error loading meter definition: %s" - LOG.error(errmsg, six.text_type(e)) - else: - definitions[meter_cfg['name']] = md + mfs = [] + for dir in self.manager.conf.meter.meter_definitions_dirs: + for filepath in sorted(glob.glob(os.path.join(dir, "*.yaml"))): + if filepath is not None: + mfs.append(filepath) + if self.manager.conf.meter.meter_definitions_cfg_file is not None: + mfs.append( + pkg_resources.resource_filename( + self.manager.conf.meter.meter_definitions_cfg_file) + ) + for mf in mfs: + meters_cfg = declarative.load_definitions( + self.manager.conf, {}, mf) + + for meter_cfg in reversed(meters_cfg['metric']): + if meter_cfg.get('name') in definitions: + # skip duplicate meters + LOG.warning("Skipping duplicate meter definition %s" + % meter_cfg) + continue + try: + md = MeterDefinition(meter_cfg, self.manager.conf, + plugin_manager) + except declarative.DefinitionException as e: + errmsg = "Error loading meter definition: %s" + LOG.error(errmsg, six.text_type(e)) + else: + definitions[meter_cfg['name']] = md return definitions.values() def process_notification(self, notification_body): diff --git a/ceilometer/tests/unit/meter/test_notifications.py b/ceilometer/tests/unit/meter/test_notifications.py index f2767293cc..6b703b1d0b 100644 --- a/ceilometer/tests/unit/meter/test_notifications.py +++ b/ceilometer/tests/unit/meter/test_notifications.py @@ -13,15 +13,14 @@ """Tests for ceilometer.meter.notifications """ import copy +import fixtures import mock -import os import six import yaml from oslo_utils import encodeutils from oslo_utils import fileutils -import ceilometer from ceilometer import declarative from ceilometer.meter import notifications from ceilometer import service as ceilometer_service @@ -284,30 +283,24 @@ class TestMeterProcessing(test.BaseTestCase): def setUp(self): super(TestMeterProcessing, self).setUp() self.CONF = ceilometer_service.prepare_service([], []) + self.path = self.useFixture(fixtures.TempDir()).path self.handler = notifications.ProcessMeterNotifications( mock.Mock(conf=self.CONF)) - def test_fallback_meter_path(self): - self.CONF.set_override('meter_definitions_cfg_file', - '/not/existing/path', group='meter') - with mock.patch('ceilometer.declarative.open', - mock.mock_open(read_data='---\nmetric: []'), - create=True) as mock_open: - self.handler._load_definitions() + def _load_meter_def_file(self, cfgs=None): + self.CONF.set_override('meter_definitions_dirs', + [self.path], group='meter') + cfgs = cfgs or [] + if not isinstance(cfgs, list): + cfgs = [cfgs] + meter_cfg_files = list() + for cfg in cfgs: if six.PY3: - path = os.path.dirname(ceilometer.__file__) - else: - path = "ceilometer" - mock_open.assert_called_with(path + "/meter/data/meters.yaml") - - def _load_meter_def_file(self, cfg): - if six.PY3: - cfg = cfg.encode('utf-8') - meter_cfg_file = fileutils.write_to_tempfile(content=cfg, - prefix="meters", - suffix="yaml") - self.CONF.set_override('meter_definitions_cfg_file', - meter_cfg_file, group='meter') + cfg = cfg.encode('utf-8') + meter_cfg_files.append(fileutils.write_to_tempfile(content=cfg, + path=self.path, + prefix="meters", + suffix=".yaml")) self.handler.definitions = self.handler._load_definitions() @mock.patch('ceilometer.meter.notifications.LOG') @@ -768,3 +761,129 @@ class TestMeterProcessing(test.BaseTestCase): self._load_meter_def_file(cfg) c = list(self.handler.process_notification(NOTIFICATION)) self.assertEqual(1, len(c)) + + def test_multi_files_multi_meters(self): + cfg1 = yaml.dump( + {'metric': [dict(name="test1", + event_type="test.create", + type="delta", + unit="B", + volume="$.payload.volume", + resource_id="$.payload.resource_id", + project_id="$.payload.project_id")]}) + cfg2 = yaml.dump( + {'metric': [dict(name="test2", + event_type="test.create", + type="delta", + unit="B", + volume="$.payload.volume", + resource_id="$.payload.resource_id", + project_id="$.payload.project_id")]}) + self._load_meter_def_file([cfg1, cfg2]) + data = list(self.handler.process_notification(NOTIFICATION)) + self.assertEqual(2, len(data)) + expected_names = ['test1', 'test2'] + for s in data: + self.assertIn(s.as_dict()['name'], expected_names) + + def test_multi_files_duplicate_meter(self): + cfg1 = yaml.dump( + {'metric': [dict(name="test", + event_type="test.create", + type="delta", + unit="B", + volume="$.payload.volume", + resource_id="$.payload.resource_id", + project_id="$.payload.project_id")]}) + cfg2 = yaml.dump( + {'metric': [dict(name="test", + event_type="test.create", + type="delta", + unit="B", + volume="$.payload.volume", + resource_id="$.payload.resource_id", + project_id="$.payload.project_id")]}) + self._load_meter_def_file([cfg1, cfg2]) + data = list(self.handler.process_notification(NOTIFICATION)) + self.assertEqual(1, len(data)) + self.assertEqual(data[0].as_dict()['name'], 'test') + + def test_multi_files_empty_payload(self): + event = copy.deepcopy(MIDDLEWARE_EVENT) + del event['payload']['measurements'] + cfg1 = yaml.dump( + {'metric': [dict(name="$.payload.measurements.[*].metric.[*].name", + event_type="objectstore.http.request", + type="delta", + unit="$.payload.measurements.[*].metric.[*].unit", + volume="$.payload.measurements.[*].result", + resource_id="$.payload.target_id", + project_id="$.payload.initiator.project_id", + lookup="name")]}) + cfg2 = yaml.dump( + {'metric': [dict(name="$.payload.measurements.[*].metric.[*].name", + event_type="objectstore.http.request", + type="delta", + unit="$.payload.measurements.[*].metric.[*].unit", + volume="$.payload.measurements.[*].result", + resource_id="$.payload.target_id", + project_id="$.payload.initiator.project_id", + lookup="name")]}) + self._load_meter_def_file([cfg1, cfg2]) + data = list(self.handler.process_notification(event)) + self.assertEqual(0, len(data)) + + def test_multi_files_unmatched_meter(self): + cfg1 = yaml.dump( + {'metric': [dict(name="test1", + event_type="test.create", + type="delta", + unit="B", + volume="$.payload.volume", + resource_id="$.payload.resource_id", + project_id="$.payload.project_id")]}) + cfg2 = yaml.dump( + {'metric': [dict(name="test2", + event_type="test.update", + type="delta", + unit="B", + volume="$.payload.volume", + resource_id="$.payload.resource_id", + project_id="$.payload.project_id")]}) + self._load_meter_def_file([cfg1, cfg2]) + data = list(self.handler.process_notification(NOTIFICATION)) + self.assertEqual(1, len(data)) + self.assertEqual(data[0].as_dict()['name'], 'test1') + + @mock.patch('ceilometer.meter.notifications.LOG') + def test_multi_files_bad_meter(self, LOG): + cfg1 = yaml.dump( + {'metric': [dict(name="test1", + event_type="test.create", + type="delta", + unit="B", + volume="$.payload.volume", + resource_id="$.payload.resource_id", + project_id="$.payload.project_id"), + dict(name="bad_test", + type="bad_type", + event_type="bar.create", + unit="foo", volume="bar", + resource_id="bea70e51c7340cb9d555b15cbfcaec23")]}) + cfg2 = yaml.dump( + {'metric': [dict(name="test2", + event_type="test.create", + type="delta", + unit="B", + volume="$.payload.volume", + resource_id="$.payload.resource_id", + project_id="$.payload.project_id")]}) + self._load_meter_def_file([cfg1, cfg2]) + data = list(self.handler.process_notification(NOTIFICATION)) + self.assertEqual(2, len(data)) + expected_names = ['test1', 'test2'] + for s in data: + self.assertIn(s.as_dict()['name'], expected_names) + args, kwargs = LOG.error.call_args_list[0] + self.assertEqual("Error loading meter definition: %s", args[0]) + self.assertTrue(args[1].endswith("Invalid type bad_type specified")) diff --git a/releasenotes/notes/support-multiple-meter-definition-files-e3ce1fa73ef2e1de.yaml b/releasenotes/notes/support-multiple-meter-definition-files-e3ce1fa73ef2e1de.yaml new file mode 100644 index 0000000000..a706f79e78 --- /dev/null +++ b/releasenotes/notes/support-multiple-meter-definition-files-e3ce1fa73ef2e1de.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Support loading multiple meter definition files and + allow users to add their own meter definitions into + several files according to different types of metrics + under the directory of /etc/ceilometer/meters.d. \ No newline at end of file