Reduce complexity of storage engine API

In the process of implementing the first storage
backend I realized that the proposed API was
much more complicated than necessary. This
changeset streamlines the API to make it easier
to implement the plugins.

- add ceilometer.storage.get_connection() convenience function
- add get_volume_max() method
- clean up query arg handling by using EventFilter
- tighten up the DB query API by removing separate "by_user"
  and "by_project" methods

Change-Id: Id4dffcc59dbee44fd4670bbe55b5e3380ea240e5
Signed-off-by: Doug Hellmann <doug.hellmann@dreamhost.com>

ceilometer/storage/__init__.py

@@ -49,6 +49,8 @@ def get_engine(conf):
     """Load the configured engine and return an instance.
     """
     engine_name = conf.metering_storage_engine
+    LOG.debug('looking for %r driver in %r',
+              engine_name, STORAGE_ENGINE_NAMESPACE)
     for ep in pkg_resources.iter_entry_points(STORAGE_ENGINE_NAMESPACE,
                                               engine_name):
         try:
@@ -59,7 +61,40 @@ def get_engine(conf):
                       engine_name, err)
             LOG.exception(err)
             raise
-        LOG.info('Loaded %s storage engine', engine_name)
+        LOG.info('Loaded %s storage engine %r', engine_name, ep)
         return engine
     else:
         raise RuntimeError('No %r storage engine found' % engine_name)
+
+
+def get_connection(conf):
+    """Return an open connection to the database.
+    """
+    engine = get_engine(conf)
+    engine.register_opts(conf)
+    db = engine.get_connection(conf)
+    return db
+
+
+class EventFilter(object):
+    """Holds the properties for building a query to filter events.
+
+    :param user: The event owner.
+    :param project: The event owner.
+    :param start: Earliest timestamp to include.
+    :param end: Only include events with timestamp less than this.
+    :param resource: Optional filter for resource id.
+    :param meter: Optional filter for meter type using the meter name.
+    :param source: Optional source filter.
+    """
+    def __init__(self, user=None, project=None, start=None, end=None,
+                 resource=None, meter=None, source=None):
+        self.user = user
+        self.project = project
+        self.start = start
+        self.end = end
+        self.resource = resource
+        self.meter = meter
+        self.source = source
+        if not (self.user or self.project):
+            raise RuntimeError('Must provide one of "user" or "project".')

ceilometer/storage/base.py

@@ -67,56 +67,6 @@ class Connection(object):
         :param source: Optional source filter.
         """
 
-    @abc.abstractmethod
-    def get_resources_by_user(self, user=None, source=None):
-        """Return an iterable of tuples containing resource ids and
-        the most recent version of the metadata for the resource.
-
-        :param user: The event owner.
-        :param source: Optional source filter.
-        """
-
-    @abc.abstractmethod
-    def get_raw_events_by_user(self, user, start=None, end=None,
-                               resource=None, meter=None, source=None):
-        """Return an iterable of event data.
-
-        :param user: The event owner.
-        :param start: Earliest timestamp to include.
-        :param end: Only include events with timestamp less than this.
-        :param resource: Optional filter for resource id.
-        :param meter: Optional filter for meter type using the meter name.
-        :param source: Optional source filter.
-        """
-
-    @abc.abstractmethod
-    def get_volume_sum_by_user(self, user, meter, start=None, end=None,
-                               resource=None, source=None):
-        """Return the sum of the volume field for the events
-        described by the query parameters.
-
-        :param user: The event owner.
-        :param meter: Filter for meter type using the meter name.
-        :param start: Earliest timestamp to include.
-        :param end: Only include events with timestamp less than this.
-        :param resource: Optional filter for resource id.
-        :param source: Optional source filter.
-        """
-
-    @abc.abstractmethod
-    def get_duration_sum_by_user(self, user, meter, start=None, end=None,
-                                 resource=None, source=None):
-        """Return the sum of time for the events described by the
-        query parameters.
-
-        :param user: The event owner.
-        :param meter: Filter for meter type using the meter name.
-        :param start: Earliest timestamp to include.
-        :param end: Only include events with timestamp less than this.
-        :param resource: Optional filter for resource id.
-        :param source: Optional source filter.
-        """
-
     @abc.abstractmethod
     def get_projects(self, source=None):
         """Return an iterable of project id strings.
@@ -125,51 +75,55 @@ class Connection(object):
         """
 
     @abc.abstractmethod
-    def get_resources_by_project(self, project=None, source=None):
+    def get_resources(self, user=None, project=None, source=None):
-        """Return an iterable of tuples containing resource ids and
+        """Return an iterable of dictionaries containing resource information.
-        the most recent version of the metadata for the resource.
 
-        :param project: The event owner.
+        { 'resource_id': UUID of the resource,
+          'project_id': UUID of project owning the resource,
+          'user_id': UUID of user owning the resource,
+          'timestamp': UTC datetime of last update to the resource,
+          'metadata': most current metadata for the resource,
+          'meter': list of the meters reporting data for the resource,
+          }
+
+        :param user: Optional resource owner.
+        :param project: Optional resource owner.
         :param source: Optional source filter.
         """
 
     @abc.abstractmethod
-    def get_raw_events_by_project(self, project, start=None, end=None,
+    def get_raw_events(self, event_filter):
-                               resource=None, meter=None, source=None):
+        """Return an iterable of raw event data.
-        """Return an iterable of event data.
-
-        :param project: The event owner.
-        :param start: Earliest timestamp to include.
-        :param end: Only include events with timestamp less than this.
-        :param resource: Optional filter for resource id.
-        :param meter: Optional filter for meter type using the meter name.
-        :param source: Optional source filter.
         """
 
     @abc.abstractmethod
-    def get_volume_sum_by_project(self, project, meter, start=None, end=None,
+    def get_volume_sum(self, event_filter):
-                               resource=None, source=None):
         """Return the sum of the volume field for the events
         described by the query parameters.
 
-        :param project: The event owner.
+        The filter must have a meter value set.
-        :param meter: Optional filter for meter type using the meter name.
+
-        :param start: Earliest timestamp to include.
+        { 'resource_id': UUID string for the resource,
-        :param end: Only include events with timestamp less than this.
+          'value': The sum for the volume.
-        :param resource: Optional filter for resource id.
+          }
-        :param source: Optional source filter.
         """
 
     @abc.abstractmethod
-    def get_duration_sum_by_project(self, project, meter, start=None, end=None,
+    def get_volume_max(self, event_filter):
-                                 resource=None, source=None):
+        """Return the maximum of the volume field for the events
+        described by the query parameters.
+
+        The filter must have a meter value set.
+
+        { 'resource_id': UUID string for the resource,
+          'value': The max for the volume.
+          }
+        """
+
+    @abc.abstractmethod
+    def get_duration_sum(self, event_filter):
         """Return the sum of time for the events described by the
         query parameters.
 
-        :param project: The event owner.
+        The filter must have a meter value set.
-        :param meter: Optional filter for meter type using the meter name.
-        :param start: Earliest timestamp to include.
-        :param end: Only include events with timestamp less than this.
-        :param resource: Optional filter for resource id.
-        :param source: Optional source filter.
         """

ceilometer/storage/impl_log.py

@@ -62,100 +62,35 @@ class Connection(base.Connection):
         :param source: Optional source filter.
         """
 
-    def get_resources_by_user(self, user=None, source=None):
-        """Return an iterable of tuples containing resource ids and
-        the most recent version of the metadata for the resource.
-
-        :param user: The event owner.
-        :param source: Optional source filter.
-        """
-
-    def get_raw_events_by_user(self, user, start=None, end=None,
-                               resource=None, meter=None, source=None):
-        """Return an iterable of event data.
-
-        :param user: The event owner.
-        :param start: Earliest timestamp to include.
-        :param end: Only include events with timestamp less than this.
-        :param resource: Optional filter for resource id.
-        :param meter: Optional filter for meter type using the meter name.
-        :param source: Optional source filter.
-        """
-
-    def get_volume_sum_by_user(self, user, meter, start=None, end=None,
-                               resource=None, source=None):
-        """Return the sum of the volume field for the events
-        described by the query parameters.
-
-        :param user: The event owner.
-        :param meter: Filter for meter type using the meter name.
-        :param start: Earliest timestamp to include.
-        :param end: Only include events with timestamp less than this.
-        :param resource: Optional filter for resource id.
-        :param source: Optional source filter.
-        """
-
-    def get_duration_sum_by_user(self, user, meter, start=None, end=None,
-                                 resource=None, source=None):
-        """Return the sum of time for the events described by the
-        query parameters.
-
-        :param user: The event owner.
-        :param meter: Filter for meter type using the meter name.
-        :param start: Earliest timestamp to include.
-        :param end: Only include events with timestamp less than this.
-        :param resource: Optional filter for resource id.
-        :param source: Optional source filter.
-        """
-
     def get_projects(self, source=None):
         """Return an iterable of project id strings.
 
         :param source: Optional source filter.
         """
 
-    def get_resources_by_project(self, project=None, source=None):
+    def get_resources(self, user=None, project=None, source=None):
         """Return an iterable of tuples containing resource ids and
         the most recent version of the metadata for the resource.
 
-        :param project: The event owner.
+        :param user: The event owner.
         :param source: Optional source filter.
         """
 
-    def get_raw_events_by_project(self, project, start=None, end=None,
+    def get_raw_events(self, event_filter):
-                               resource=None, meter=None, source=None):
         """Return an iterable of event data.
-
-        :param project: The event owner.
-        :param start: Earliest timestamp to include.
-        :param end: Only include events with timestamp less than this.
-        :param resource: Optional filter for resource id.
-        :param meter: Optional filter for meter type using the meter name.
-        :param source: Optional source filter.
         """
 
-    def get_volume_sum_by_project(self, project, meter, start=None, end=None,
+    def get_volume_sum(self, event_filter):
-                               resource=None, source=None):
         """Return the sum of the volume field for the events
         described by the query parameters.
-
-        :param project: The event owner.
-        :param meter: Optional filter for meter type using the meter name.
-        :param start: Earliest timestamp to include.
-        :param end: Only include events with timestamp less than this.
-        :param resource: Optional filter for resource id.
-        :param source: Optional source filter.
         """
 
-    def get_duration_sum_by_project(self, project, meter, start=None, end=None,
+    def get_volume_max(self, event_filter):
-                                 resource=None, source=None):
+        """Return the maximum of the volume field for the events
+        described by the query parameters.
+        """
+
+    def get_duration_sum(self, event_filter):
         """Return the sum of time for the events described by the
         query parameters.
-
-        :param project: The event owner.
-        :param meter: Optional filter for meter type using the meter name.
-        :param start: Earliest timestamp to include.
-        :param end: Only include events with timestamp less than this.
-        :param resource: Optional filter for resource id.
-        :param source: Optional source filter.
         """

run_tests.sh

@@ -9,4 +9,5 @@ if [ ! -z "$VIRTUAL_ENV" ]
 then
 	rm -f $VIRTUAL_ENV/lib/python*/no-global-site-packages.txt
 fi
+
 nosetests "$@"

tools/pip-requires

@@ -1,4 +1,6 @@
-https://github.com/openstack/nova/zipball/master#egg=nova
+#https://github.com/openstack/nova/zipball/master#egg=nova
+# Work-around for packaging issue in nova:
+http://nova.openstack.org/tarballs/nova-2012.2~f2~20120629.14648.tar.gz
 webob
 kombu
 iso8601