openstack/ceilometer
Code Issues Proposed changes

Add Disk Meters for ceilometer

Browse Source 
The new disk meter reading correspond to libvirt domblkinfo Statistics :
capacity, allocation and physical(usage) .

Storage meters per instance
disk.capacity = Disk Capacity for all the disks of the instance
disk.usage = Disk Usage for all the disks of the instance
disk.allocation = Disk Allocation for all the disks of the instance

Storage Meters per disk per instance
disk.device.capacity = Disk Capacity for each individual disk of an instance
disk.device.usage = Disk Usage for each individual disk of an instance
disk.device.allocation =Disk Allocation for each individual disk of an instance

DocImpact

Implements: blueprint disk-info-meters
Change-Id: Ie5accb8fb75b0d3a6641d090fa273fc8fe100ab3
This commit is contained in:
Saba Ahmed 2015-02-10 14:42:02 -05:00
parent 3b8c820dd1
commit 707c286cf5
8 changed files with 317 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

176
ceilometer/compute/pollsters/disk.py
View File

@ -50,6 +50,12 @@ DiskIOPSData = collections.namedtuple('DiskIOPSData',
['iops_count', ['iops_count',
'per_disk_iops']) 'per_disk_iops'])
DiskInfoData = collections.namedtuple('DiskInfoData',
['capacity',
'allocation',
'physical',
'per_disk_info'])
@six.add_metaclass(abc.ABCMeta) @six.add_metaclass(abc.ABCMeta)
class _Base(pollsters.BaseComputePollster): class _Base(pollsters.BaseComputePollster):
@ -635,3 +641,173 @@ class PerDeviceDiskIOPSPollster(_DiskIOPSPollsterBase):
resource_id="%s-%s" % (instance.id, disk) resource_id="%s-%s" % (instance.id, disk)
)) ))
return samples return samples
@six.add_metaclass(abc.ABCMeta)
class _DiskInfoPollsterBase(pollsters.BaseComputePollster):
CACHE_KEY_DISK_INFO = 'diskinfo'
def _populate_cache(self, inspector, cache, instance):
i_cache = cache.setdefault(self.CACHE_KEY_DISK_INFO, {})
if instance.id not in i_cache:
all_capacity = 0
all_allocation = 0
all_physical = 0
per_disk_capacity = {}
per_disk_allocation = {}
per_disk_physical = {}
disk_info = inspector.inspect_disk_info(
instance)
for disk, info in disk_info:
all_capacity += info.capacity
all_allocation += info.allocation
all_physical += info.physical
per_disk_capacity[disk.device] = info.capacity
per_disk_allocation[disk.device] = info.allocation
per_disk_physical[disk.device] = info.physical
per_disk_info = {
'capacity': per_disk_capacity,
'allocation': per_disk_allocation,
'physical': per_disk_physical,
}
i_cache[instance.id] = DiskInfoData(
all_capacity,
all_allocation,
all_physical,
per_disk_info
)
return i_cache[instance.id]
@abc.abstractmethod
def _get_samples(self, instance, disk_info):
"""Return one or more Sample."""
def get_samples(self, manager, cache, resources):
for instance in resources:
try:
disk_size_info = self._populate_cache(
self.inspector,
cache,
instance,
)
for disk_info in self._get_samples(instance, disk_size_info):
yield disk_info
except virt_inspector.InstanceNotFoundException as err:
# Instance was deleted while getting samples. Ignore it.
LOG.debug(_('Exception while getting samples %s'), err)
except virt_inspector.InstanceShutOffException as e:
LOG.warn(_LW('Instance %(instance_id)s was shut off while '
'getting samples of %(pollster)s: %(exc)s'),
{'instance_id': instance.id,
'pollster': self.__class__.__name__, 'exc': e})
except ceilometer.NotImplementedError:
# Selected inspector does not implement this pollster.
LOG.debug(_('%(inspector)s does not provide data for '
' %(pollster)s'), (
{'inspector': manager.inspector.__class__.__name__,
'pollster': self.__class__.__name__}))
except Exception as err:
instance_name = util.instance_name(instance)
LOG.exception(_('Ignoring instance %(name)s '
'(%(instance_id)s) : %(error)s') % (
{'name': instance_name,
'instance_id': instance.id,
'error': err}))
class CapacityPollster(_DiskInfoPollsterBase):
def _get_samples(self, instance, disk_info):
return [util.make_sample_from_instance(
instance,
name='disk.capacity',
type=sample.TYPE_GAUGE,
unit='B',
volume=disk_info.capacity,
additional_metadata={
'device': disk_info.per_disk_info[
'capacity'].keys()},
)]
class PerDeviceCapacityPollster(_DiskInfoPollsterBase):
def _get_samples(self, instance, disk_info):
samples = []
for disk, value in six.iteritems(disk_info.per_disk_info[
'capacity']):
samples.append(util.make_sample_from_instance(
instance,
name='disk.device.capacity',
type=sample.TYPE_GAUGE,
unit='B',
volume=value,
resource_id="%s-%s" % (instance.id, disk),
))
return samples
class AllocationPollster(_DiskInfoPollsterBase):
def _get_samples(self, instance, disk_info):
return [util.make_sample_from_instance(
instance,
name='disk.allocation',
type=sample.TYPE_GAUGE,
unit='B',
volume=disk_info.allocation,
additional_metadata={
'device': disk_info.per_disk_info[
'allocation'].keys()},
)]
class PerDeviceAllocationPollster(_DiskInfoPollsterBase):
def _get_samples(self, instance, disk_info):
samples = []
for disk, value in six.iteritems(disk_info.per_disk_info[
'allocation']):
samples.append(util.make_sample_from_instance(
instance,
name='disk.device.allocation',
type=sample.TYPE_GAUGE,
unit='B',
volume=value,
resource_id="%s-%s" % (instance.id, disk),
))
return samples
class PhysicalPollster(_DiskInfoPollsterBase):
def _get_samples(self, instance, disk_info):
return [util.make_sample_from_instance(
instance,
name='disk.usage',
type=sample.TYPE_GAUGE,
unit='B',
volume=disk_info.physical,
additional_metadata={
'device': disk_info.per_disk_info[
'physical'].keys()},
)]
class PerDevicePhysicalPollster(_DiskInfoPollsterBase):
def _get_samples(self, instance, disk_info):
samples = []
for disk, value in six.iteritems(disk_info.per_disk_info[
'physical']):
samples.append(util.make_sample_from_instance(
instance,
name='disk.device.usage',
type=sample.TYPE_GAUGE,
unit='B',
volume=value,
resource_id="%s-%s" % (instance.id, disk),
))
return samples

20
ceilometer/compute/virt/inspector.py
View File

@ -143,6 +143,18 @@ DiskIOPSStats = collections.namedtuple('DiskIOPSStats',
['iops_count']) ['iops_count'])
# Named tuple representing disk Information.
#
# capacity: capacity of the disk
# allocation: allocation of the disk
# physical: usage of the disk
DiskInfo = collections.namedtuple('DiskInfo',
['capacity',
'allocation',
'physical'])
# Exception types # Exception types
# #
class InspectorException(Exception): class InspectorException(Exception):
@ -250,6 +262,14 @@ class Inspector(object):
""" """
raise ceilometer.NotImplementedError raise ceilometer.NotImplementedError
def inspect_disk_info(self, instance):
"""Inspect the disk information for an instance.
:param instance: the target instance
:return: for each disk , capacity , alloaction and usage
"""
raise ceilometer.NotImplementedError
def get_hypervisor_inspector(): def get_hypervisor_inspector():
try: try:

16
ceilometer/compute/virt/libvirt/inspector.py
View File

@ -196,3 +196,19 @@ class LibvirtInspector(virt_inspector.Inspector):
'can not get info from libvirt: %(error)s') % { 'can not get info from libvirt: %(error)s') % {
'instance_uuid': instance.id, 'error': e} 'instance_uuid': instance.id, 'error': e}
raise virt_inspector.NoDataException(msg) raise virt_inspector.NoDataException(msg)
def inspect_disk_info(self, instance):
domain = self._get_domain_not_shut_off_or_raise(instance)
tree = etree.fromstring(domain.XMLDesc(0))
for device in filter(
bool,
[target.get("dev")
for target in tree.findall('devices/disk/target')]):
disk = virt_inspector.Disk(device=device)
block_info = domain.blockInfo(device)
info = virt_inspector.DiskInfo(capacity=block_info[0],
allocation=block_info[1],
physical=block_info[2])
yield (disk, info)

4
ceilometer/tests/agent/test_manager.py
View File

@ -39,8 +39,8 @@ class TestManager(base.BaseTestCase):
def test_load_plugins_pollster_list(self): def test_load_plugins_pollster_list(self):
mgr = manager.AgentManager(pollster_list=['disk.*']) mgr = manager.AgentManager(pollster_list=['disk.*'])
# currently we do have 20 disk-related pollsters # currently we do have 26 disk-related pollsters
self.assertEqual(20, len(list(mgr.extensions))) self.assertEqual(26, len(list(mgr.extensions)))
def test_load_plugins_no_intersection(self): def test_load_plugins_no_intersection(self):
# Let's test nothing will be polled if namespace and pollsters # Let's test nothing will be polled if namespace and pollsters

55
ceilometer/tests/compute/pollsters/test_diskio.py
View File

@ -304,3 +304,58 @@ class TestDiskIOPSPollsters(TestBaseDiskIO):
self._check_per_device_samples(disk.PerDeviceDiskIOPSPollster, self._check_per_device_samples(disk.PerDeviceDiskIOPSPollster,
'disk.device.iops', 20L, 'disk2') 'disk.device.iops', 20L, 'disk2')
class TestDiskInfoPollsters(TestBaseDiskIO):
DISKS = [
(virt_inspector.Disk(device='vda1'),
virt_inspector.DiskInfo(capacity=3L, allocation=2L, physical=1L)),
(virt_inspector.Disk(device='vda2'),
virt_inspector.DiskInfo(capacity=4L, allocation=3L, physical=2L)),
]
TYPE = 'gauge'
CACHE_KEY = "CACHE_KEY_DISK_INFO"
def setUp(self):
super(TestDiskInfoPollsters, self).setUp()
self.inspector.inspect_disk_info = mock.Mock(return_value=self.DISKS)
def test_disk_capacity(self):
self._check_aggregate_samples(disk.CapacityPollster,
'disk.capacity', 7L,
expected_device=['vda1', 'vda2'])
def test_disk_allocation(self):
self._check_aggregate_samples(disk.AllocationPollster,
'disk.allocation', 5L,
expected_device=['vda1', 'vda2'])
def test_disk_physical(self):
self._check_aggregate_samples(disk.PhysicalPollster,
'disk.usage', 3L,
expected_device=['vda1', 'vda2'])
def test_per_disk_capacity(self):
self._check_per_device_samples(disk.PerDeviceCapacityPollster,
'disk.device.capacity', 3L,
'vda1')
self._check_per_device_samples(disk.PerDeviceCapacityPollster,
'disk.device.capacity', 4L,
'vda2')
def test_per_disk_allocation(self):
self._check_per_device_samples(disk.PerDeviceAllocationPollster,
'disk.device.allocation', 2L,
'vda1')
self._check_per_device_samples(disk.PerDeviceAllocationPollster,
'disk.device.allocation', 3L,
'vda2')
def test_per_disk_physical(self):
self._check_per_device_samples(disk.PerDevicePhysicalPollster,
'disk.device.usage', 1L,
'vda1')
self._check_per_device_samples(disk.PerDevicePhysicalPollster,
'disk.device.usage', 2L,
'vda2')

36
ceilometer/tests/compute/virt/libvirt/test_inspector.py
View File

@ -245,6 +245,42 @@ class TestLibvirtInspection(base.BaseTestCase):
self.instance) self.instance)
self.assertEqual(25600L / units.Ki, memory.usage) self.assertEqual(25600L / units.Ki, memory.usage)
def test_inspect_disk_info(self):
dom_xml = """
<domain type='kvm'>
<devices>
<disk type='file' device='disk'>
<driver name='qemu' type='qcow2' cache='none'/>
<source file='/path/instance-00000001/disk'/>
<target dev='vda' bus='virtio'/>
<alias name='virtio-disk0'/>
<address type='pci' domain='0x0000' bus='0x00'
slot='0x04' function='0x0'/>
</disk>
</devices>
</domain>
"""
with contextlib.nested(mock.patch.object(self.inspector.connection,
'lookupByUUIDString',
return_value=self.domain),
mock.patch.object(self.domain, 'XMLDesc',
return_value=dom_xml),
mock.patch.object(self.domain, 'blockInfo',
return_value=(1L, 2L, 3L,
-1)),
mock.patch.object(self.domain, 'info',
return_value=(0L, 0L, 0L,
2L, 999999L))):
disks = list(self.inspector.inspect_disk_info(self.instance))
self.assertEqual(1, len(disks))
disk0, info0 = disks[0]
self.assertEqual('vda', disk0.device)
self.assertEqual(1L, info0.capacity)
self.assertEqual(2L, info0.allocation)
self.assertEqual(3L, info0.physical)
def test_inspect_memory_usage_with_domain_shutoff(self): def test_inspect_memory_usage_with_domain_shutoff(self):
connection = self.inspector.connection connection = self.inspector.connection
with mock.patch.object(connection, 'lookupByUUIDString', with mock.patch.object(connection, 'lookupByUUIDString',

6
doc/source/measurements.rst
View File

@ -97,6 +97,12 @@ disk.device.latency g ms disk ID p 2
disk.device.iops g count/s disk ID p 2 Average disk iops per device disk.device.iops g count/s disk ID p 2 Average disk iops per device
disk.root.size g GB inst ID n 1, 2 Size of root disk disk.root.size g GB inst ID n 1, 2 Size of root disk
disk.ephemeral.size g GB inst ID n 1, 2 Size of ephemeral disk disk.ephemeral.size g GB inst ID n 1, 2 Size of ephemeral disk
disk.capacity g B inst ID p 1 Capacity of disk
disk.allocation g B inst ID p 1 Allocation of disk
disk.usage g B inst ID p 1 Usage of virtual disk
disk.device.capacity g B disk ID p 1 Capacity per device of disk
disk.device.allocation g B disk ID p 1 Allocation per device of disk
disk.device.usage g B disk ID p 1 Usage per device of virtual disk
network.incoming.bytes c B iface ID p 1, 2 Number of incoming bytes network.incoming.bytes c B iface ID p 1, 2 Number of incoming bytes
network.incoming.bytes.rate g B/s iface ID p 1, 2, 3, 4 Average rate of incoming bytes network.incoming.bytes.rate g B/s iface ID p 1, 2, 3, 4 Average rate of incoming bytes
network.outgoing.bytes c B iface ID p 1, 2 Number of outgoing bytes network.outgoing.bytes c B iface ID p 1, 2 Number of outgoing bytes

6
setup.cfg
View File

@ -140,6 +140,12 @@ ceilometer.poll.compute =
instance = ceilometer.compute.pollsters.instance:InstancePollster instance = ceilometer.compute.pollsters.instance:InstancePollster
instance_flavor = ceilometer.compute.pollsters.instance:InstanceFlavorPollster instance_flavor = ceilometer.compute.pollsters.instance:InstanceFlavorPollster
memory.usage = ceilometer.compute.pollsters.memory:MemoryUsagePollster memory.usage = ceilometer.compute.pollsters.memory:MemoryUsagePollster
disk.capacity = ceilometer.compute.pollsters.disk:CapacityPollster
disk.allocation = ceilometer.compute.pollsters.disk:AllocationPollster
disk.usage = ceilometer.compute.pollsters.disk:PhysicalPollster
disk.device.capacity = ceilometer.compute.pollsters.disk:PerDeviceCapacityPollster
disk.device.allocation = ceilometer.compute.pollsters.disk:PerDeviceAllocationPollster
disk.device.usage = ceilometer.compute.pollsters.disk:PerDevicePhysicalPollster
ceilometer.poll.ipmi = ceilometer.poll.ipmi =
hardware.ipmi.node.power = ceilometer.ipmi.pollsters.node:PowerPollster hardware.ipmi.node.power = ceilometer.ipmi.pollsters.node:PowerPollster