Merge "Add a new scheduler for the l3 HA"

neutron/db/l3_agentschedulers_db.py

@@ -23,12 +23,14 @@ from sqlalchemy import func
 from sqlalchemy import orm
 from sqlalchemy.orm import exc
 from sqlalchemy.orm import joinedload
+from sqlalchemy import sql
 
 from neutron.common import constants
 from neutron.common import utils as n_utils
 from neutron import context as n_ctx
 from neutron.db import agents_db
 from neutron.db import agentschedulers_db
+from neutron.db import l3_attrs_db
 from neutron.db import model_base
 from neutron.extensions import l3agentscheduler
 from neutron import manager
@@ -114,7 +116,13 @@ class L3AgentSchedulerDbMixin(l3agentscheduler.L3AgentSchedulerPluginBase,
             context.session.query(RouterL3AgentBinding).
             join(agents_db.Agent).
             filter(agents_db.Agent.heartbeat_timestamp < cutoff,
-                   agents_db.Agent.admin_state_up))
+                   agents_db.Agent.admin_state_up).
+            outerjoin(l3_attrs_db.RouterExtraAttributes,
+                      l3_attrs_db.RouterExtraAttributes.router_id ==
+                      RouterL3AgentBinding.router_id).
+            filter(sa.or_(l3_attrs_db.RouterExtraAttributes.ha == sql.false(),
+                          l3_attrs_db.RouterExtraAttributes.ha == sql.null())))
+
         for binding in down_bindings:
             LOG.warn(_LW("Rescheduling router %(router)s from agent %(agent)s "
                          "because the agent did not report to the server in "

neutron/db/l3_hascheduler_db.py

@@ -0,0 +1,59 @@
+# Copyright (C) 2014 eNovance SAS <licensing@enovance.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from sqlalchemy import func
+from sqlalchemy import sql
+
+from neutron.db import agents_db
+from neutron.db import l3_agentschedulers_db as l3_sch_db
+from neutron.db import l3_attrs_db
+from neutron.db import l3_db
+
+
+class L3_HA_scheduler_db_mixin(l3_sch_db.L3AgentSchedulerDbMixin):
+
+    def get_ha_routers_l3_agents_count(self, context):
+        """Return a map between HA routers and how many agents every
+        router is scheduled to.
+        """
+
+        # Postgres requires every column in the select to be present in
+        # the group by statement when using an aggregate function.
+        # One solution is to generate a subquery and join it with the desired
+        # columns.
+        binding_model = l3_sch_db.RouterL3AgentBinding
+        sub_query = (context.session.query(
+            binding_model.router_id,
+            func.count(binding_model.router_id).label('count')).
+            join(l3_attrs_db.RouterExtraAttributes,
+                 binding_model.router_id ==
+                 l3_attrs_db.RouterExtraAttributes.router_id).
+            join(l3_db.Router).
+            filter(l3_attrs_db.RouterExtraAttributes.ha == sql.true()).
+            group_by(binding_model.router_id).subquery())
+
+        query = (context.session.query(
+                 l3_db.Router.id, l3_db.Router.tenant_id, sub_query.c.count).
+                 join(sub_query))
+        return query
+
+    def get_l3_agents_ordered_by_num_routers(self, context, agent_ids):
+        query = (context.session.query(agents_db.Agent, func.count(
+            l3_sch_db.RouterL3AgentBinding.router_id).label('count')).
+            outerjoin(l3_sch_db.RouterL3AgentBinding).
+            group_by(agents_db.Agent.id).
+            filter(agents_db.Agent.id.in_(agent_ids)).
+            order_by('count'))
+
+        return [record[0] for record in query]

neutron/scheduler/l3_agent_scheduler.py

@@ -14,24 +14,34 @@
 #    under the License.
 
 import abc
+import itertools
 import random
 
+from oslo.config import cfg
 from oslo.db import exception as db_exc
 import six
 from sqlalchemy import sql
 
 from neutron.common import constants
+from neutron.common import utils
 from neutron.db import l3_agentschedulers_db
 from neutron.db import l3_db
+from neutron.db import l3_hamode_db
+from neutron.openstack.common.gettextutils import _LE
 from neutron.openstack.common import log as logging
 
 
 LOG = logging.getLogger(__name__)
+cfg.CONF.register_opts(l3_hamode_db.L3_HA_OPTS)
 
 
 @six.add_metaclass(abc.ABCMeta)
 class L3Scheduler(object):
 
+    def __init__(self):
+        self.min_ha_agents = cfg.CONF.min_l3_agents_per_router
+        self.max_ha_agents = cfg.CONF.max_l3_agents_per_router
+
     @abc.abstractmethod
     def schedule(self, plugin, context, router_id,
                  candidates=None, hints=None):
@@ -41,6 +51,15 @@ class L3Scheduler(object):
         """
         pass
 
+    def router_has_binding(self, context, router_id, l3_agent_id):
+        router_binding_model = l3_agentschedulers_db.RouterL3AgentBinding
+
+        query = context.session.query(router_binding_model)
+        query = query.filter(router_binding_model.router_id == router_id,
+                             router_binding_model.l3_agent_id == l3_agent_id)
+
+        return query.count() > 0
+
     def filter_unscheduled_routers(self, context, plugin, routers):
         """Filter from list of routers the ones that are not scheduled."""
         unscheduled_routers = []
@@ -126,7 +145,10 @@ class L3Scheduler(object):
         unscheduled_routers = self.get_routers_to_schedule(
             context, plugin, router_ids, exclude_distributed=True)
         if not unscheduled_routers:
-            return False
+            if utils.is_extension_supported(
+                    plugin, constants.L3_HA_MODE_EXT_ALIAS):
+                return self.schedule_ha_routers_to_additional_agent(
+                    plugin, context, l3_agent)
 
         target_routers = self.get_routers_can_schedule(
             context, plugin, unscheduled_routers, l3_agent)
@@ -135,7 +157,7 @@ class L3Scheduler(object):
                        ' on host %s'), host)
             return False
 
-        self.bind_routers(context, target_routers, l3_agent)
+        self.bind_routers(context, plugin, target_routers, l3_agent)
         return True
 
     def get_candidates(self, plugin, context, sync_router):
@@ -173,8 +195,15 @@ class L3Scheduler(object):
 
             return candidates
 
-    def bind_routers(self, context, routers, l3_agent):
+    def bind_routers(self, context, plugin, routers, l3_agent):
         for router in routers:
+            if router.get('ha'):
+                if not self.router_has_binding(context, router['id'],
+                                               l3_agent.id):
+                    self.create_ha_router_binding(
+                        plugin, context, router['id'],
+                        router['tenant_id'], l3_agent)
+            else:
                 self.bind_router(context, router['id'], l3_agent)
 
     def bind_router(self, context, router_id, chosen_agent):
@@ -221,6 +250,12 @@ class L3Scheduler(object):
         if router_distributed:
             for chosen_agent in candidates:
                 self.bind_router(context, router_id, chosen_agent)
+        elif sync_router.get('ha', False):
+            chosen_agents = self.bind_ha_router(plugin, context,
+                                                router_id, candidates)
+            if not chosen_agents:
+                return
+            chosen_agent = chosen_agents[-1]
         else:
             chosen_agent = self._choose_router_agent(
                 plugin, context, candidates)
@@ -232,6 +267,82 @@ class L3Scheduler(object):
         """Choose an agent from candidates based on a specific policy."""
         pass
 
+    @abc.abstractmethod
+    def _choose_router_agents_for_ha(self, plugin, context, candidates):
+        """Choose agents from candidates based on a specific policy."""
+        pass
+
+    def get_num_of_agents_for_ha(self, candidates_count):
+        return (min(self.max_ha_agents, candidates_count) if self.max_ha_agents
+                else candidates_count)
+
+    def enough_candidates_for_ha(self, candidates):
+        if not candidates or len(candidates) < self.min_ha_agents:
+            LOG.error(_LE("Not enough candidates, a HA router needs at least "
+                          "%s agents"), self.min_ha_agents)
+            return False
+        return True
+
+    def create_ha_router_binding(self, plugin, context, router_id, tenant_id,
+                                 agent):
+        """Creates and binds a new HA port for this agent."""
+        ha_network = plugin.get_ha_network(context, tenant_id)
+        port_binding = plugin.add_ha_port(context.elevated(), router_id,
+                                          ha_network.network.id, tenant_id)
+        port_binding.l3_agent_id = agent['id']
+        self.bind_router(context, router_id, agent)
+
+    def schedule_ha_routers_to_additional_agent(self, plugin, context, agent):
+        """Bind already scheduled routers to the agent.
+
+        Retrieve the number of agents per router and check if the router has
+        to be scheduled on the given agent if max_l3_agents_per_router
+        is not yet reached.
+        """
+
+        routers_agents = plugin.get_ha_routers_l3_agents_count(context)
+
+        scheduled = False
+        admin_ctx = context.elevated()
+        for router_id, tenant_id, agents in routers_agents:
+            max_agents_not_reached = (
+                not self.max_ha_agents or agents < self.max_ha_agents)
+            if max_agents_not_reached:
+                if not self.router_has_binding(admin_ctx, router_id, agent.id):
+                    self.create_ha_router_binding(plugin, admin_ctx,
+                                                  router_id, tenant_id,
+                                                  agent)
+                    scheduled = True
+
+        return scheduled
+
+    def bind_ha_router_to_agents(self, plugin, context, router_id,
+                                 chosen_agents):
+        port_bindings = plugin.get_ha_router_port_bindings(context,
+                                                           [router_id])
+        for port_binding, agent in itertools.izip(port_bindings,
+                                                  chosen_agents):
+            port_binding.l3_agent_id = agent.id
+            self.bind_router(context, router_id, agent)
+
+            LOG.debug('HA Router %(router_id)s is scheduled to L3 agent '
+                      '%(agent_id)s)',
+                      {'router_id': router_id, 'agent_id': agent.id})
+
+    def bind_ha_router(self, plugin, context, router_id, candidates):
+        """Bind a HA router to agents based on a specific policy."""
+
+        if not self.enough_candidates_for_ha(candidates):
+            return
+
+        chosen_agents = self._choose_router_agents_for_ha(
+            plugin, context, candidates)
+
+        self.bind_ha_router_to_agents(plugin, context, router_id,
+                                      chosen_agents)
+
+        return chosen_agents
+
 
 class ChanceScheduler(L3Scheduler):
     """Randomly allocate an L3 agent for a router."""
@@ -244,6 +355,10 @@ class ChanceScheduler(L3Scheduler):
     def _choose_router_agent(self, plugin, context, candidates):
         return random.choice(candidates)
 
+    def _choose_router_agents_for_ha(self, plugin, context, candidates):
+        num_agents = self.get_num_of_agents_for_ha(len(candidates))
+        return random.sample(candidates, num_agents)
+
 
 class LeastRoutersScheduler(L3Scheduler):
     """Allocate to an L3 agent with the least number of routers bound."""
@@ -258,3 +373,9 @@ class LeastRoutersScheduler(L3Scheduler):
         chosen_agent = plugin.get_l3_agent_with_min_routers(
             context, candidate_ids)
         return chosen_agent
+
+    def _choose_router_agents_for_ha(self, plugin, context, candidates):
+        num_agents = self.get_num_of_agents_for_ha(len(candidates))
+        ordered_agents = plugin.get_l3_agents_ordered_by_num_routers(
+            context, [candidate['id'] for candidate in candidates])
+        return ordered_agents[:num_agents]

neutron/services/l3_router/l3_router_plugin.py

@@ -27,6 +27,7 @@ from neutron.db import extraroute_db
 from neutron.db import l3_dvrscheduler_db
 from neutron.db import l3_gwmode_db
 from neutron.db import l3_hamode_db
+from neutron.db import l3_hascheduler_db
 from neutron.openstack.common import importutils
 from neutron.plugins.common import constants
 
@@ -35,7 +36,8 @@ class L3RouterPlugin(common_db_mixin.CommonDbMixin,
                      extraroute_db.ExtraRoute_db_mixin,
                      l3_gwmode_db.L3_NAT_db_mixin,
                      l3_dvrscheduler_db.L3_DVRsch_db_mixin,
-                     l3_hamode_db.L3_HA_NAT_db_mixin):
+                     l3_hamode_db.L3_HA_NAT_db_mixin,
+                     l3_hascheduler_db.L3_HA_scheduler_db_mixin):
 
     """Implementation of the Neutron L3 Router Service Plugin.
 

neutron/tests/unit/test_l3_schedulers.py

@@ -17,23 +17,26 @@
 # @author: Emilien Macchi, eNovance SAS
 
 import contextlib
+import datetime
 import uuid
 
 import mock
 from oslo.config import cfg
 from sqlalchemy.orm import query
 
-from neutron.api.v2 import attributes as attr
 from neutron.common import constants
 from neutron.common import topics
 from neutron import context as q_context
 from neutron.db import agents_db
 from neutron.db import common_db_mixin
+from neutron.db import db_base_plugin_v2 as db_v2
 from neutron.db import l3_agentschedulers_db
 from neutron.db import l3_db
 from neutron.db import l3_dvrscheduler_db
-from neutron.extensions import l3 as ext_l3
+from neutron.db import l3_hamode_db
+from neutron.db import l3_hascheduler_db
 from neutron import manager
+from neutron.openstack.common import importutils
 from neutron.openstack.common import timeutils
 from neutron.scheduler import l3_agent_scheduler
 from neutron.tests import base
@@ -62,6 +65,26 @@ SECOND_L3_AGENT = {
     'start_flag': True
 }
 
+HOST_3 = 'my_l3_host_3'
+THIRD_L3_AGENT = {
+    'binary': 'neutron-l3-agent',
+    'host': HOST_3,
+    'topic': topics.L3_AGENT,
+    'configurations': {},
+    'agent_type': constants.AGENT_TYPE_L3,
+    'start_flag': True
+}
+
+HOST_4 = 'my_l3_host_4'
+FOURTH_L3_AGENT = {
+    'binary': 'neutron-l3-agent',
+    'host': HOST_4,
+    'topic': topics.L3_AGENT,
+    'configurations': {},
+    'agent_type': constants.AGENT_TYPE_L3,
+    'start_flag': True
+}
+
 HOST_DVR = 'my_l3_host_dvr'
 DVR_L3_AGENT = {
     'binary': 'neutron-l3-agent',
@@ -82,9 +105,6 @@ DVR_SNAT_L3_AGENT = {
     'start_flag': True
 }
 
-DB_PLUGIN_KLASS = ('neutron.plugins.openvswitch.ovs_neutron_plugin.'
-                   'OVSNeutronPluginV2')
-
 
 class FakeL3Scheduler(l3_agent_scheduler.L3Scheduler):
 
@@ -94,6 +114,9 @@ class FakeL3Scheduler(l3_agent_scheduler.L3Scheduler):
     def _choose_router_agent(self):
         pass
 
+    def _choose_router_agents_for_ha(self):
+        pass
+
 
 class L3SchedulerBaseTestCase(base.BaseTestCase):
 
@@ -123,9 +146,11 @@ class L3SchedulerBaseTestCase(base.BaseTestCase):
         self.assertFalse(result)
 
     def test_auto_schedule_routers_no_unscheduled_routers(self):
+        type(self.plugin).supported_extension_aliases = (
+            mock.PropertyMock(return_value=[]))
         with mock.patch.object(self.scheduler,
                                'get_routers_to_schedule') as mock_routers:
-            mock_routers.return_value = None
+            mock_routers.return_value = []
             result = self.scheduler.auto_schedule_routers(
                 self.plugin, mock.ANY, mock.ANY, mock.ANY)
         self.assertTrue(self.plugin.get_enabled_agent_on_host.called)
@@ -218,55 +243,49 @@ class L3SchedulerBaseTestCase(base.BaseTestCase):
     def test_bind_routers_centralized(self):
         routers = [{'id': 'foo_router'}]
         with mock.patch.object(self.scheduler, 'bind_router') as mock_bind:
-            self.scheduler.bind_routers(mock.ANY, routers, mock.ANY)
+            self.scheduler.bind_routers(mock.ANY, mock.ANY, routers, mock.ANY)
         mock_bind.assert_called_once_with(mock.ANY, 'foo_router', mock.ANY)
 
+    def _test_bind_routers_ha(self, has_binding):
+        routers = [{'id': 'foo_router', 'ha': True, 'tenant_id': '42'}]
+        agent = agents_db.Agent(id='foo_agent')
+        with contextlib.nested(
+            mock.patch.object(self.scheduler, 'router_has_binding',
+                              return_value=has_binding),
+            mock.patch.object(self.scheduler, 'create_ha_router_binding')) as (
+                mock_has_binding, mock_bind):
+            self.scheduler.bind_routers(mock.ANY, mock.ANY, routers, agent)
+        mock_has_binding.assert_called_once_with(mock.ANY, 'foo_router',
+                                                 'foo_agent')
+        self.assertEqual(not has_binding, mock_bind.called)
 
-class L3SchedulerTestExtensionManager(object):
+    def test_bind_routers_ha_has_binding(self):
+        self._test_bind_routers_ha(has_binding=True)
 
-    def get_resources(self):
+    def test_bind_routers_ha_no_binding(self):
-        attr.RESOURCE_ATTRIBUTE_MAP.update(ext_l3.RESOURCE_ATTRIBUTE_MAP)
+        self._test_bind_routers_ha(has_binding=False)
-        l3_res = ext_l3.L3.get_resources()
-        return l3_res
-
-    def get_actions(self):
-        return []
-
-    def get_request_extensions(self):
-        return []
 
 
-class L3SchedulerTestCase(l3_agentschedulers_db.L3AgentSchedulerDbMixin,
+class L3SchedulerBaseMixin(object):
-                          l3_db.L3_NAT_db_mixin,
-                          common_db_mixin.CommonDbMixin,
-                          test_db_plugin.NeutronDbPluginV2TestCase,
-                          test_l3_plugin.L3NatTestCaseMixin):
 
-    def setUp(self):
+    def _register_l3_agent(self, agent, plugin=None):
-        ext_mgr = L3SchedulerTestExtensionManager()
+        if not plugin:
-        super(L3SchedulerTestCase, self).setUp(plugin=DB_PLUGIN_KLASS,
+            plugin = self.plugin
-                                               ext_mgr=ext_mgr)
 
-        self.adminContext = q_context.get_admin_context()
-        self.plugin = manager.NeutronManager.get_plugin()
-        self._register_l3_agents()
-
-    def _register_l3_agents(self):
         callback = agents_db.AgentExtRpcCallback()
         callback.report_state(self.adminContext,
-                              agent_state={'agent_state': FIRST_L3_AGENT},
+                              agent_state={'agent_state': agent},
                               time=timeutils.strtime())
-        agent_db = self.plugin.get_agents_db(self.adminContext,
+        agent_db = plugin.get_agents_db(self.adminContext,
-                                             filters={'host': [HOST]})
+                                        filters={'host': [agent['host']]})
-        self.agent_id1 = agent_db[0].id
+        return agent_db[0]
-        self.agent1 = agent_db[0]
 
-        callback.report_state(self.adminContext,
+    def _register_l3_agents(self, plugin=None):
-                              agent_state={'agent_state': SECOND_L3_AGENT},
+        self.agent1 = self._register_l3_agent(FIRST_L3_AGENT, plugin)
-                              time=timeutils.strtime())
+        self.agent_id1 = self.agent1.id
-        agent_db = self.plugin.get_agents_db(self.adminContext,
+
-                                             filters={'host': [HOST]})
+        self.agent2 = self._register_l3_agent(SECOND_L3_AGENT, plugin)
-        self.agent_id2 = agent_db[0].id
+        self.agent_id2 = self.agent2.id
 
     def _register_l3_dvr_agents(self):
         callback = agents_db.AgentExtRpcCallback()
@@ -289,6 +308,13 @@ class L3SchedulerTestCase(l3_agentschedulers_db.L3AgentSchedulerDbMixin,
         update = {'agent': {'admin_state_up': state}}
         self.plugin.update_agent(context, agent_id, update)
 
+    def _set_l3_agent_dead(self, agent_id):
+        update = {
+            'agent': {
+                'heartbeat_timestamp':
+                timeutils.utcnow() - datetime.timedelta(hours=1)}}
+        self.plugin.update_agent(self.adminContext, agent_id, update)
+
     @contextlib.contextmanager
     def router_with_ext_gw(self, name='router1', admin_state_up=True,
                            fmt=None, tenant_id=str(uuid.uuid4()),
@@ -308,6 +334,9 @@ class L3SchedulerTestCase(l3_agentschedulers_db.L3AgentSchedulerDbMixin,
             router['router']['id'], subnet['subnet']['network_id'])
         self._delete('routers', router['router']['id'])
 
+
+class L3SchedulerTestBaseMixin(object):
+
     def _test_add_router_to_l3_agent(self,
                                      distributed=False,
                                      already_scheduled=False):
@@ -614,6 +643,30 @@ class L3SchedulerTestCase(l3_agentschedulers_db.L3AgentSchedulerDbMixin,
             self.assertTrue(val)
 
 
+class L3SchedulerTestCase(l3_agentschedulers_db.L3AgentSchedulerDbMixin,
+                          l3_db.L3_NAT_db_mixin,
+                          common_db_mixin.CommonDbMixin,
+                          test_db_plugin.NeutronDbPluginV2TestCase,
+                          test_l3_plugin.L3NatTestCaseMixin,
+                          L3SchedulerBaseMixin,
+                          L3SchedulerTestBaseMixin):
+
+    def setUp(self):
+        self.mock_rescheduling = False
+        ext_mgr = test_l3_plugin.L3TestExtensionManager()
+        plugin_str = ('neutron.tests.unit.test_l3_plugin.'
+                      'TestL3NatIntAgentSchedulingPlugin')
+        super(L3SchedulerTestCase, self).setUp(plugin=plugin_str,
+                                               ext_mgr=ext_mgr)
+
+        self.adminContext = q_context.get_admin_context()
+        self.plugin = manager.NeutronManager.get_plugin()
+        self.plugin.router_scheduler = importutils.import_object(
+            'neutron.scheduler.l3_agent_scheduler.ChanceScheduler'
+        )
+        self._register_l3_agents()
+
+
 class L3AgentChanceSchedulerTestCase(L3SchedulerTestCase):
 
     def test_random_scheduling(self):
@@ -644,14 +697,38 @@ class L3AgentChanceSchedulerTestCase(L3SchedulerTestCase):
 
         random_patch.stop()
 
+    def test_scheduler_auto_schedule_when_agent_added(self):
+        self._set_l3_agent_admin_state(self.adminContext,
+                                       self.agent_id1, False)
+        self._set_l3_agent_admin_state(self.adminContext,
+                                       self.agent_id2, False)
+
+        with self.subnet() as subnet:
+            self._set_net_external(subnet['subnet']['network_id'])
+            with self.router_with_ext_gw(name='r1', subnet=subnet) as r1:
+                agents = self.get_l3_agents_hosting_routers(
+                    self.adminContext, [r1['router']['id']],
+                    admin_state_up=True)
+                self.assertEqual(0, len(agents))
+
+                self._set_l3_agent_admin_state(self.adminContext,
+                                               self.agent_id1, True)
+                self.plugin.auto_schedule_routers(self.adminContext,
+                                                  FIRST_L3_AGENT['host'],
+                                                  [r1['router']['id']])
+
+                agents = self.get_l3_agents_hosting_routers(
+                    self.adminContext, [r1['router']['id']],
+                    admin_state_up=True)
+                self.assertEqual(FIRST_L3_AGENT['host'], agents[0]['host'])
+
 
 class L3AgentLeastRoutersSchedulerTestCase(L3SchedulerTestCase):
     def setUp(self):
-        cfg.CONF.set_override('router_scheduler_driver',
-                              'neutron.scheduler.l3_agent_scheduler.'
-                              'LeastRoutersScheduler')
-
         super(L3AgentLeastRoutersSchedulerTestCase, self).setUp()
+        self.plugin.router_scheduler = importutils.import_object(
+            'neutron.scheduler.l3_agent_scheduler.LeastRoutersScheduler'
+        )
 
     def test_scheduler(self):
         # disable one agent to force the scheduling to the only one.
@@ -957,3 +1034,244 @@ class L3DvrSchedulerTestCase(testlib_api.SqlTestCase,
         self.assertTrue(mock_delete.call_count)
         core_plugin.assert_called_once_with()
         l3_notifier.assert_called_once_with()
+
+
+class L3HAPlugin(db_v2.NeutronDbPluginV2,
+                 l3_hamode_db.L3_HA_NAT_db_mixin,
+                 l3_hascheduler_db.L3_HA_scheduler_db_mixin):
+    supported_extension_aliases = ["l3-ha"]
+
+
+class L3HATestCaseMixin(testlib_api.SqlTestCase,
+                        L3SchedulerBaseMixin,
+                        testlib_plugin.PluginSetupHelper):
+
+    def setUp(self):
+        super(L3HATestCaseMixin, self).setUp()
+
+        self.adminContext = q_context.get_admin_context()
+        self.plugin = L3HAPlugin()
+
+        self.setup_coreplugin('neutron.plugins.ml2.plugin.Ml2Plugin')
+        mock.patch.object(l3_hamode_db.L3_HA_NAT_db_mixin,
+                          '_notify_ha_interfaces_updated').start()
+
+        cfg.CONF.set_override('max_l3_agents_per_router', 0)
+        self.plugin.router_scheduler = importutils.import_object(
+            'neutron.scheduler.l3_agent_scheduler.ChanceScheduler'
+        )
+
+        self._register_l3_agents()
+
+    def _create_ha_router(self, ha=True, tenant_id='tenant1'):
+        router = {'name': 'router1', 'admin_state_up': True}
+        if ha is not None:
+            router['ha'] = ha
+        return self.plugin._create_router_db(self.adminContext,
+                                             router, tenant_id)
+
+
+class L3_HA_scheduler_db_mixinTestCase(L3HATestCaseMixin):
+
+    def _register_l3_agents(self, plugin=None):
+        super(L3_HA_scheduler_db_mixinTestCase,
+              self)._register_l3_agents(plugin=plugin)
+
+        self.agent3 = self._register_l3_agent(THIRD_L3_AGENT, plugin)
+        self.agent_id3 = self.agent3.id
+
+        self.agent4 = self._register_l3_agent(FOURTH_L3_AGENT, plugin)
+        self.agent_id4 = self.agent4.id
+
+    def test_get_ha_routers_l3_agents_count(self):
+        router1 = self._create_ha_router()
+        router2 = self._create_ha_router()
+        router3 = self._create_ha_router(ha=False)
+        self.plugin.schedule_router(self.adminContext, router1['id'])
+        self.plugin.schedule_router(self.adminContext, router2['id'])
+        self.plugin.schedule_router(self.adminContext, router3['id'])
+        result = self.plugin.get_ha_routers_l3_agents_count(
+            self.adminContext).all()
+
+        self.assertEqual(2, len(result))
+        self.assertIn((router1['id'], router1['tenant_id'], 4), result)
+        self.assertIn((router2['id'], router2['tenant_id'], 4), result)
+        self.assertNotIn((router3['id'], router3['tenant_id'], mock.ANY),
+                         result)
+
+    def test_get_ordered_l3_agents_by_num_routers(self):
+        router1 = self._create_ha_router()
+        router2 = self._create_ha_router()
+        router3 = self._create_ha_router(ha=False)
+        router4 = self._create_ha_router(ha=False)
+
+        # Agent 1 will host 0 routers, agent 2 will host 1, agent 3 will
+        # host 2, and agent 4 will host 3.
+        self.plugin.schedule_router(self.adminContext, router1['id'],
+                                    candidates=[self.agent2, self.agent4])
+        self.plugin.schedule_router(self.adminContext, router2['id'],
+                                    candidates=[self.agent3, self.agent4])
+        self.plugin.schedule_router(self.adminContext, router3['id'],
+                                    candidates=[self.agent3])
+        self.plugin.schedule_router(self.adminContext, router4['id'],
+                                    candidates=[self.agent4])
+
+        agent_ids = [self.agent_id1, self.agent_id2, self.agent_id3,
+                     self.agent_id4]
+        result = self.plugin.get_l3_agents_ordered_by_num_routers(
+            self.adminContext, agent_ids)
+
+        self.assertEqual(agent_ids, [record['id'] for record in result])
+
+
+class L3AgentSchedulerDbMixinTestCase(L3HATestCaseMixin):
+
+    def test_reschedule_ha_routers_from_down_agents(self):
+        router = self._create_ha_router()
+        self.plugin.schedule_router(self.adminContext, router['id'])
+        agents = self.plugin.get_l3_agents_hosting_routers(
+            self.adminContext, [router['id']],
+            admin_state_up=True)
+        self.assertEqual(2, len(agents))
+        self._set_l3_agent_dead(self.agent_id1)
+        with mock.patch.object(self.plugin, 'reschedule_router') as reschedule:
+            self.plugin.reschedule_routers_from_down_agents()
+            self.assertFalse(reschedule.called)
+
+
+class L3HAChanceSchedulerTestCase(L3HATestCaseMixin):
+
+    def test_scheduler_with_ha_enabled(self):
+        router = self._create_ha_router()
+        self.plugin.schedule_router(self.adminContext, router['id'])
+        agents = self.plugin.get_l3_agents_hosting_routers(
+            self.adminContext, [router['id']],
+            admin_state_up=True)
+        self.assertEqual(2, len(agents))
+
+        for agent in agents:
+            sync_data = self.plugin.get_ha_sync_data_for_host(
+                self.adminContext, router_ids=[router['id']],
+                host=agent.host)
+            self.assertEqual(1, len(sync_data))
+            interface = sync_data[0][constants.HA_INTERFACE_KEY]
+            self.assertIsNotNone(interface)
+
+    def test_auto_schedule(self):
+        router = self._create_ha_router()
+        self.plugin.auto_schedule_routers(
+            self.adminContext, self.agent1.host, None)
+        self.plugin.auto_schedule_routers(
+            self.adminContext, self.agent2.host, None)
+        agents = self.plugin.get_l3_agents_hosting_routers(
+            self.adminContext, [router['id']])
+        self.assertEqual(2, len(agents))
+
+    def test_auto_schedule_specific_router_when_agent_added(self):
+        self._auto_schedule_when_agent_added(True)
+
+    def test_auto_schedule_all_routers_when_agent_added(self):
+        self._auto_schedule_when_agent_added(False)
+
+    def _auto_schedule_when_agent_added(self, specific_router):
+        router = self._create_ha_router()
+        self.plugin.schedule_router(self.adminContext, router['id'])
+        agents = self.plugin.get_l3_agents_hosting_routers(
+            self.adminContext, [router['id']],
+            admin_state_up=True)
+        self.assertEqual(2, len(agents))
+        agent_ids = [agent['id'] for agent in agents]
+        self.assertIn(self.agent_id1, agent_ids)
+        self.assertIn(self.agent_id2, agent_ids)
+
+        agent = self._register_l3_agent(THIRD_L3_AGENT)
+        self.agent_id3 = agent.id
+        routers_to_auto_schedule = [router['id']] if specific_router else []
+        self.plugin.auto_schedule_routers(self.adminContext,
+                                          THIRD_L3_AGENT['host'],
+                                          routers_to_auto_schedule)
+
+        agents = self.plugin.get_l3_agents_hosting_routers(
+            self.adminContext, [router['id']],
+            admin_state_up=True)
+        self.assertEqual(3, len(agents))
+
+        # Simulate agent restart to make sure we don't try to re-bind
+        self.plugin.auto_schedule_routers(self.adminContext,
+                                          THIRD_L3_AGENT['host'],
+                                          routers_to_auto_schedule)
+
+    def test_scheduler_with_ha_enabled_not_enough_agent(self):
+        r1 = self._create_ha_router()
+        self.plugin.schedule_router(self.adminContext, r1['id'])
+        agents = self.plugin.get_l3_agents_hosting_routers(
+            self.adminContext, [r1['id']],
+            admin_state_up=True)
+        self.assertEqual(2, len(agents))
+
+        self._set_l3_agent_admin_state(self.adminContext,
+                                       self.agent_id2, False)
+
+        r2 = self._create_ha_router()
+        self.plugin.schedule_router(self.adminContext, r2['id'])
+        agents = self.plugin.get_l3_agents_hosting_routers(
+            self.adminContext, [r2['id']],
+            admin_state_up=True)
+        self.assertEqual(0, len(agents))
+
+        self._set_l3_agent_admin_state(self.adminContext,
+                                       self.agent_id2, True)
+
+
+class L3HALeastRoutersSchedulerTestCase(L3HATestCaseMixin):
+
+    def _register_l3_agents(self, plugin=None):
+        super(L3HALeastRoutersSchedulerTestCase,
+              self)._register_l3_agents(plugin=plugin)
+
+        agent = self._register_l3_agent(THIRD_L3_AGENT, plugin)
+        self.agent_id3 = agent.id
+
+        agent = self._register_l3_agent(FOURTH_L3_AGENT, plugin)
+        self.agent_id4 = agent.id
+
+    def setUp(self):
+        super(L3HALeastRoutersSchedulerTestCase, self).setUp()
+        self.plugin.router_scheduler = importutils.import_object(
+            'neutron.scheduler.l3_agent_scheduler.LeastRoutersScheduler'
+        )
+
+    def test_scheduler(self):
+        cfg.CONF.set_override('max_l3_agents_per_router', 2)
+
+        # disable the third agent to be sure that the router will
+        # be scheduled of the two firsts
+        self._set_l3_agent_admin_state(self.adminContext,
+                                       self.agent_id3, False)
+        self._set_l3_agent_admin_state(self.adminContext,
+                                       self.agent_id4, False)
+
+        r1 = self._create_ha_router()
+        self.plugin.schedule_router(self.adminContext, r1['id'])
+        agents = self.plugin.get_l3_agents_hosting_routers(
+            self.adminContext, [r1['id']],
+            admin_state_up=True)
+        self.assertEqual(2, len(agents))
+        agent_ids = [agent['id'] for agent in agents]
+        self.assertIn(self.agent_id1, agent_ids)
+        self.assertIn(self.agent_id2, agent_ids)
+
+        self._set_l3_agent_admin_state(self.adminContext,
+                                       self.agent_id3, True)
+        self._set_l3_agent_admin_state(self.adminContext,
+                                       self.agent_id4, True)
+
+        r2 = self._create_ha_router()
+        self.plugin.schedule_router(self.adminContext, r2['id'])
+        agents = self.plugin.get_l3_agents_hosting_routers(
+            self.adminContext, [r2['id']],
+            admin_state_up=True)
+        self.assertEqual(2, len(agents))
+        agent_ids = [agent['id'] for agent in agents]
+        self.assertIn(self.agent_id3, agent_ids)
+        self.assertIn(self.agent_id4, agent_ids)