Add the beginnings of the nova-conductor service

This adds a new service called "nova-conductor" which will provide
services for nova-compute, such as completing database updates and
handling long-running tasks.

Right now, this just adds a fraction of the database-related work
so that we can proceed with incremental changes to nova-compute.

For compatibility/flexibility, both LocalAPI and (RPC)API classes
are provided. Users concerned with performance or compatibility
who wish to maintain compute-node-local database accesses can
set the [conductor] use_local=True, which will mimic the legacy
behavior.

DocImpact

Change-Id: Ib81d879f40b7daceeffad5af0631c7db68168661

bin/nova-conductor

@@ -0,0 +1,51 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+#    Copyright 2012 IBM Corp.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Starter script for Nova Conductor."""
+
+import eventlet
+eventlet.monkey_patch()
+
+import os
+import sys
+
+# If ../nova/__init__.py exists, add ../ to Python search path, so that
+# it will override what happens to be installed in /usr/(local/)lib/python...
+possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
+                                   os.pardir,
+                                   os.pardir))
+if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')):
+    sys.path.insert(0, possible_topdir)
+
+
+from nova import config
+from nova.openstack.common import log as logging
+from nova import service
+from nova import utils
+
+CONF = config.CONF
+CONF.import_opt('topic', 'nova.conductor.api', group='conductor')
+
+if __name__ == '__main__':
+    config.parse_args(sys.argv)
+    logging.setup("nova")
+    utils.monkey_patch()
+    server = service.Service.create(binary='nova-conductor',
+                                    topic=CONF.conductor.topic,
+                                    manager=CONF.conductor.manager)
+    service.serve(server)
+    service.wait()

doc/source/conf.py

@@ -154,7 +154,9 @@ man_pages = [
     ('man/nova-scheduler', 'nova-scheduler', u'Cloud controller fabric',
      [u'OpenStack'], 1),
     ('man/nova-xvpvncproxy', 'nova-xvpvncproxy', u'Cloud controller fabric',
-     [u'OpenStack'], 1)
+     [u'OpenStack'], 1),
+    ('man/nova-conductor', 'nova-conductor', u'Cloud controller fabric',
+     [u'OpenStack'], 1),
 ]
 
 # -- Options for HTML output --------------------------------------------------

doc/source/man/nova-conductor.rst

@@ -0,0 +1,45 @@
+==========
+nova-conductor
+==========
+
+--------------------------------
+Server for the Nova Conductor
+--------------------------------
+
+:Author: openstack@lists.launchpad.net
+:Date:   2012-11-16
+:Copyright: OpenStack LLC
+:Version: 2012.1
+:Manual section: 1
+:Manual group: cloud computing
+
+SYNOPSIS
+========
+
+  nova-conductor [options]
+
+DESCRIPTION
+===========
+
+nova-conductor is a server daemon that serves the Nova Conductor service, which provides coordination and database query support for Nova.
+
+OPTIONS
+=======
+
+ **General options**
+
+FILES
+========
+
+* /etc/nova/nova.conf
+
+SEE ALSO
+========
+
+* `OpenStack Nova <http://nova.openstack.org>`__
+* `OpenStack Nova <http://nova.openstack.org>`__
+
+BUGS
+====
+
+* Nova is sourced in Launchpad so you can view current bugs at `OpenStack Nova <http://nova.openstack.org>`__

nova/conductor/__init__.py

@@ -0,0 +1,25 @@
+#    Copyright 2012 IBM Corp.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from nova.conductor import api as conductor_api
+import nova.config
+import nova.openstack.common.importutils
+
+
+def API(*args, **kwargs):
+    if nova.config.CONF.conductor.use_local:
+        api = conductor_api.LocalAPI
+    else:
+        api = conductor_api.API
+    return api(*args, **kwargs)

nova/conductor/api.py

@@ -0,0 +1,61 @@
+#    Copyright 2012 IBM Corp.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Handles all requests to the conductor service"""
+
+from nova.conductor import manager
+from nova.conductor import rpcapi
+from nova import config
+from nova.openstack.common import cfg
+
+conductor_opts = [
+    cfg.BoolOpt('use_local',
+                default=False,
+                help='Perform nova-conductor operations locally'),
+    cfg.StrOpt('topic',
+               default='conductor',
+               help='the topic conductor nodes listen on'),
+    cfg.StrOpt('manager',
+               default='nova.conductor.manager.ConductorManager',
+               help='full class name for the Manager for conductor'),
+]
+conductor_group = cfg.OptGroup(name='conductor',
+                               title='Conductor Options')
+CONF = config.CONF
+CONF.register_group(conductor_group)
+CONF.register_opts(conductor_opts, conductor_group)
+
+
+class LocalAPI(object):
+    """A local version of the conductor API that does database updates
+    locally instead of via RPC"""
+
+    def __init__(self):
+        self._manager = manager.ConductorManager()
+
+    def instance_update(self, context, instance_uuid, **updates):
+        """Perform an instance update in the database"""
+        return self._manager.instance_update(context, instance_uuid, updates)
+
+
+class API(object):
+    """Conductor API that does updates via RPC to the ConductorManager"""
+
+    def __init__(self):
+        self.conductor_rpcapi = rpcapi.ConductorAPI()
+
+    def instance_update(self, context, instance_uuid, **updates):
+        """Perform an instance update in the database"""
+        return self.conductor_rpcapi.instance_update(context, instance_uuid,
+                                                     updates)

nova/conductor/manager.py

@@ -0,0 +1,51 @@
+#    Copyright 2012 IBM Corp.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Handles database requests from other nova services"""
+
+from nova import manager
+from nova import notifications
+from nova.openstack.common import jsonutils
+from nova.openstack.common import log as logging
+
+
+LOG = logging.getLogger(__name__)
+
+allowed_updates = ['task_state', 'vm_state', 'expected_task_state',
+                   'power_state', 'access_ip_v4', 'access_ip_v6',
+                   'launched_at', 'terminated_at', 'host',
+                   'memory_mb', 'vcpus', 'root_gb', 'ephemeral_gb',
+                   'instance_type_id',
+                   ]
+
+
+class ConductorManager(manager.SchedulerDependentManager):
+    """Mission: TBD"""
+
+    RPC_API_VERSION = '1.0'
+
+    def __init__(self, *args, **kwargs):
+        super(ConductorManager, self).__init__(service_name='conductor',
+                                               *args, **kwargs)
+
+    def instance_update(self, context, instance_uuid, updates):
+        for key in updates:
+            if key not in allowed_updates:
+                LOG.error(_("Instance update attempted for "
+                            "'%(key)s' on %(instance_uuid)s") % locals())
+                raise KeyError("unexpected update keyword '%s'" % key)
+        old_ref, instance_ref = self.db.instance_update_and_get_original(
+            context, instance_uuid, updates)
+        notifications.send_update(context, old_ref, instance_ref)
+        return jsonutils.to_primitive(instance_ref)

nova/conductor/rpcapi.py

@@ -0,0 +1,43 @@
+#    Copyright 2012 IBM Corp.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Client side of the conductor RPC API"""
+
+from nova import config
+import nova.openstack.common.rpc.proxy
+
+CONF = config.CONF
+
+
+class ConductorAPI(nova.openstack.common.rpc.proxy.RpcProxy):
+    """Client side of the conductor RPC API
+
+    API version history:
+
+    1.0 - Initial version.
+    """
+
+    BASE_RPC_API_VERSION = '1.0'
+
+    def __init__(self):
+        super(ConductorAPI, self).__init__(
+            topic=CONF.conductor.topic,
+            default_version=self.BASE_RPC_API_VERSION)
+
+    def instance_update(self, context, instance_uuid, updates):
+        return self.call(context,
+                         self.make_msg('instance_update',
+                                       instance_uuid=instance_uuid,
+                                       updates=updates),
+                         topic=self.topic)

nova/tests/conductor/test_conductor.py

@@ -0,0 +1,133 @@
+#    Copyright 2012 IBM Corp.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Tests for the conductor service"""
+
+from nova.compute import instance_types
+from nova.compute import vm_states
+from nova import conductor
+from nova.conductor import api as conductor_api
+from nova.conductor import manager as conductor_manager
+from nova.conductor import rpcapi as conductor_rpcapi
+from nova import context
+from nova import db
+from nova import notifications
+from nova import test
+
+
+FAKE_IMAGE_REF = 'fake-image-ref'
+
+
+class BaseTestCase(test.TestCase):
+    def setUp(self):
+        super(BaseTestCase, self).setUp()
+        self.user_id = 'fake'
+        self.project_id = 'fake'
+        self.context = context.RequestContext(self.user_id,
+                                              self.project_id)
+
+    def _create_fake_instance(self, params=None, type_name='m1.tiny'):
+        if not params:
+            params = {}
+
+        inst = {}
+        inst['vm_state'] = vm_states.ACTIVE
+        inst['image_ref'] = FAKE_IMAGE_REF
+        inst['reservation_id'] = 'r-fakeres'
+        inst['launch_time'] = '10'
+        inst['user_id'] = self.user_id
+        inst['project_id'] = self.project_id
+        inst['host'] = 'fake_host'
+        type_id = instance_types.get_instance_type_by_name(type_name)['id']
+        inst['instance_type_id'] = type_id
+        inst['ami_launch_index'] = 0
+        inst['memory_mb'] = 0
+        inst['vcpus'] = 0
+        inst['root_gb'] = 0
+        inst['ephemeral_gb'] = 0
+        inst['architecture'] = 'x86_64'
+        inst['os_type'] = 'Linux'
+        inst.update(params)
+        return db.instance_create(self.context, inst)
+
+
+class ConductorTestCase(BaseTestCase):
+    """Conductor Manager Tests"""
+    def setUp(self):
+        super(ConductorTestCase, self).setUp()
+        self.conductor = conductor_manager.ConductorManager()
+        self.db = None
+
+    def _do_update(self, instance_uuid, **updates):
+        return self.conductor.instance_update(self.context, instance_uuid,
+                                              updates)
+
+    def test_instance_update(self):
+        instance = self._create_fake_instance()
+        new_inst = self._do_update(instance['uuid'],
+                                   vm_state=vm_states.STOPPED)
+        instance = db.instance_get_by_uuid(self.context, instance['uuid'])
+        self.assertEqual(instance['vm_state'], vm_states.STOPPED)
+        self.assertEqual(new_inst['vm_state'], instance['vm_state'])
+
+    def test_instance_update_invalid_key(self):
+        # NOTE(danms): the real DB API call ignores invalid keys
+        if self.db == None:
+            self.assertRaises(KeyError,
+                              self._do_update, 'any-uuid', foobar=1)
+
+
+class ConductorRPCAPITestCase(ConductorTestCase):
+    """Conductor RPC API Tests"""
+    def setUp(self):
+        super(ConductorRPCAPITestCase, self).setUp()
+        self.conductor_service = self.start_service(
+            'conductor', manager='nova.conductor.manager.ConductorManager')
+        self.conductor = conductor_rpcapi.ConductorAPI()
+
+
+class ConductorLocalAPITestCase(ConductorTestCase):
+    """Conductor LocalAPI Tests"""
+    def setUp(self):
+        super(ConductorLocalAPITestCase, self).setUp()
+        self.conductor = conductor_api.LocalAPI()
+        self.db = db
+
+    def _do_update(self, instance_uuid, **updates):
+        # NOTE(danms): the public API takes actual keyword arguments,
+        # so override the base class here to make the call correctly
+        return self.conductor.instance_update(self.context, instance_uuid,
+                                              **updates)
+
+
+class ConductorAPITestCase(ConductorLocalAPITestCase):
+    """Conductor API Tests"""
+    def setUp(self):
+        super(ConductorAPITestCase, self).setUp()
+        self.conductor_service = self.start_service(
+            'conductor', manager='nova.conductor.manager.ConductorManager')
+        self.conductor = conductor_api.API()
+        self.db = None
+
+
+class ConductorImportTest(test.TestCase):
+    def test_import_conductor_local(self):
+        self.flags(use_local=True, group='conductor')
+        self.assertTrue(isinstance(conductor.API(),
+                                   conductor_api.LocalAPI))
+
+    def test_import_conductor_rpc(self):
+        self.flags(use_local=False, group='conductor')
+        self.assertTrue(isinstance(conductor.API(),
+                                   conductor_api.API))