CLI changes and release notes for remote virtualenvs

doc/source/configuration/kayobe.rst

@@ -115,3 +115,19 @@ configuration files may be encrypted.  Since encryption can make working with
 Kayobe difficult, it is recommended to follow `best practice
 <http://docs.ansible.com/ansible/playbooks_best_practices.html#best-practices-for-variables-and-vaults>`_,
 adding a layer of indirection and using encryption only where necessary.
+
+Remote Execution Environment
+----------------------------
+
+By default, ansible executes modules remotely using the system python
+interpreter, even if the ansible control process is executed from within a
+virtual environment (unless the ``local`` connection plugin is used).
+This is not ideal if there are python dependencies that must be installed
+without isolation from the system python packages. Ansible can be configured to
+use a virtualenv by setting the host variable ``ansible_python_interpreter``
+to a path to a python interpreter in an existing virtual environment.
+
+If kayobe detects that ``ansible_python_interpreter`` is set and references a
+virtual environment, it will create the virtual environment if it does not
+exist. Typically this variable should be set via a group variable for hosts in
+the ``seed``, ``seed-hypervisor``, and/or ``overcloud`` groups.

doc/source/configuration/kolla-ansible.rst

@@ -27,6 +27,23 @@ kolla-ansible is installed and executed.
                           the kolla-ansible virtualenv will be created.
    ====================== ================================================== ============================
 
+Remote Execution Environment
+============================
+
+By default, ansible executes modules remotely using the system python
+interpreter, even if the ansible control process is executed from within a
+virtual environment (unless the ``local`` connection plugin is used).
+This is not ideal if there are python dependencies that must be installed
+without isolation from the system python packages. Ansible can be configured to
+use a virtualenv by setting the host variable ``ansible_python_interpreter``
+to a path to a python interpreter in an existing virtual environment.
+
+If the variable ``kolla_ansible_target_venv`` is set, kolla-ansible will be
+configured to create and use a virtual environment on the remote hosts.
+This variable is by default set to ``{{ virtualenv_path }}/kolla-ansible``.
+The previous behaviour of installing python dependencies directly to the host
+can be used by setting ``kolla_ansible_target_venv`` to ``None``.
+
 Control Plane Services
 ======================
 

doc/source/release-notes.rst

@@ -31,6 +31,16 @@ Features
 * Adds commands for management of baremetal compute nodes - ``kayobe baremetal
   compute inspect``, ``kayobe baremetal compute manage``, and ``kayobe
   baremetal compute provide``.
+* Adds support for installation and use of a python virtual environment for
+  remote execution of ansible modules, providing isolation from the system's
+  python packages. This is enabled by setting a host variable,
+  ``ansible_python_interpreter``, to the path to a python interpreter in a
+  virtualenv, noting that Jinja2 templating is not supported for this variable.
+* Adds support for configuration of a python virtual environment for remote
+  execution of ansible modules in kolla-ansible, providing isolation from the
+  system's python packages. This is enabled by setting the variable
+  ``kolla_ansible_target_venv`` to a path to the virtualenv. The default for
+  this variable is ``{{ virtualenv_path }}/kolla-ansible``.
 
 Upgrade Notes
 -------------
@@ -56,6 +66,10 @@ Upgrade Notes
   images for the seed were built on the seed, and container images for the
   overcloud were built on the controllers.  The new design is intended to
   encourage a build, push, pull workflow.
+* The default behaviour is now to configure kolla-ansible to use a virtual
+  environment for remote execution of ansible modules. The previous behaviour
+  of installing python dependencies directly to the host can be used by
+  setting ``kolla_ansible_target_venv`` to ``None``
 
 Kayobe 3.0.0
 ============

kayobe/cli/commands.py

@@ -263,6 +263,7 @@ class SeedHypervisorHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin,
 
     * Allocate IP addresses for all configured networks.
     * Add the host to SSH known hosts.
+    * Optionally, create a virtualenv for remote target hosts.
     * Configure user accounts, group associations, and authorised SSH keys.
     * Configure Yum repos.
     * Configure the host's network interfaces.
@@ -274,8 +275,9 @@ class SeedHypervisorHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin,
     def take_action(self, parsed_args):
         self.app.LOG.debug("Configuring seed hypervisor host OS")
         playbooks = _build_playbook_list(
-            "ip-allocation", "ssh-known-host", "users", "yum", "dev-tools",
+            "ip-allocation", "ssh-known-host", "kayobe-target-venv", "users",
-            "network", "sysctl", "ntp", "seed-hypervisor-libvirt-host")
+            "yum", "dev-tools", "network", "sysctl", "ntp",
+            "seed-hypervisor-libvirt-host")
         self.run_kayobe_playbooks(parsed_args, playbooks,
                                   limit="seed-hypervisor")
 
@@ -319,6 +321,7 @@ class SeedHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
     * Allocate IP addresses for all configured networks.
     * Add the host to SSH known hosts.
     * Configure a user account for use by kayobe for SSH access.
+    * Optionally, create a virtualenv for remote target hosts.
     * Optionally, wipe unmounted disk partitions (--wipe-disks).
     * Configure user accounts, group associations, and authorised SSH keys.
     * Configure Yum repos.
@@ -329,6 +332,7 @@ class SeedHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
     * Disable bootstrap interface configuration.
     * Configure NTP.
     * Configure LVM volumes.
+    * Optionally, create a virtualenv for kolla-ansible.
     * Configure a user account for kolla-ansible.
     * Configure Docker engine.
     """
@@ -344,14 +348,25 @@ class SeedHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
 
     def take_action(self, parsed_args):
         self.app.LOG.debug("Configuring seed host OS")
-        ansible_user = self.run_kayobe_config_dump(
+
-            parsed_args, host="seed", var_name="kayobe_ansible_user")
+        # Query some kayobe ansible variables.
+        hostvars = self.run_kayobe_config_dump(parsed_args, hosts="seed")
+        if not hostvars:
+            self.app.LOG.error("No hosts in the seed group")
+            sys.exit(1)
+        hostvars = hostvars.values()[0]
+        ansible_user = hostvars.get("kayobe_ansible_user")
         if not ansible_user:
             self.app.LOG.error("Could not determine kayobe_ansible_user "
                                "variable for seed host")
             sys.exit(1)
+        python_interpreter = hostvars.get("ansible_python_interpreter")
+        kolla_target_venv = hostvars.get("kolla_ansible_target_venv")
+
+        # Run kayobe playbooks.
         playbooks = _build_playbook_list(
-            "ip-allocation", "ssh-known-host", "kayobe-ansible-user")
+            "ip-allocation", "ssh-known-host", "kayobe-ansible-user",
+            "kayobe-target-venv")
         if parsed_args.wipe_disks:
             playbooks += _build_playbook_list("wipe-disks")
         playbooks += _build_playbook_list(
@@ -360,8 +375,25 @@ class SeedHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
         self.run_kayobe_playbooks(parsed_args, playbooks, limit="seed")
         playbooks = _build_playbook_list("kolla-ansible")
         self.run_kayobe_playbooks(parsed_args, playbooks, tags="config")
+
+        # Run kolla-ansible bootstrap-servers.
+        # This command should be run as the kayobe ansible user because at this
+        # point the kolla user may not exist.
+        extra_vars = {"ansible_user": ansible_user}
+        if python_interpreter:
+            # Use the kayobe virtualenv, as this is the executing user.
+            extra_vars["ansible_python_interpreter"] = python_interpreter
+        elif kolla_target_venv:
+            # Override the kolla-ansible virtualenv, use the system python
+            # instead.
+            extra_vars["ansible_python_interpreter"] = "/usr/bin/python"
+        if kolla_target_venv:
+            # Specify a virtualenv in which to install python packages.
+            extra_vars["virtualenv"] = kolla_target_venv
         self.run_kolla_ansible_seed(parsed_args, "bootstrap-servers",
-                                    extra_vars={"ansible_user": ansible_user})
+                                    extra_vars=extra_vars)
+
+        # Run final kayobe playbooks.
         playbooks = _build_playbook_list("kolla-host", "docker")
         self.run_kayobe_playbooks(parsed_args, playbooks, limit="seed")
 
@@ -559,6 +591,7 @@ class OvercloudHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
     * Allocate IP addresses for all configured networks.
     * Add the host to SSH known hosts.
     * Configure a user account for use by kayobe for SSH access.
+    * Optionally, create a virtualenv for remote target hosts.
     * Optionally, wipe unmounted disk partitions (--wipe-disks).
     * Configure user accounts, group associations, and authorised SSH keys.
     * Configure Yum repos.
@@ -568,6 +601,7 @@ class OvercloudHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
     * Disable bootstrap interface configuration.
     * Configure NTP.
     * Configure LVM volumes.
+    * Optionally, create a virtualenv for kolla-ansible.
     * Configure a user account for kolla-ansible.
     * Configure Docker engine.
     """
@@ -583,15 +617,25 @@ class OvercloudHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
 
     def take_action(self, parsed_args):
         self.app.LOG.debug("Configuring overcloud host OS")
-        ansible_user = self.run_kayobe_config_dump(
+
-            parsed_args, hosts="overcloud", var_name="kayobe_ansible_user")
+        # Query some kayobe ansible variables.
+        hostvars = self.run_kayobe_config_dump(parsed_args, hosts="overcloud")
+        if not hostvars:
+            self.app.LOG.error("No hosts in the overcloud group")
+            sys.exit(1)
+        hostvars = hostvars.values()[0]
+        ansible_user = hostvars.get("kayobe_ansible_user")
         if not ansible_user:
             self.app.LOG.error("Could not determine kayobe_ansible_user "
                                "variable for overcloud hosts")
             sys.exit(1)
-        ansible_user = ansible_user.values()[0]
+        python_interpreter = hostvars.get("ansible_python_interpreter")
+        kolla_target_venv = hostvars.get("kolla_ansible_target_venv")
+
+        # Kayobe playbooks.
         playbooks = _build_playbook_list(
-            "ip-allocation", "ssh-known-host", "kayobe-ansible-user")
+            "ip-allocation", "ssh-known-host", "kayobe-ansible-user",
+            "kayobe-target-venv")
         if parsed_args.wipe_disks:
             playbooks += _build_playbook_list("wipe-disks")
         playbooks += _build_playbook_list(
@@ -600,9 +644,26 @@ class OvercloudHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
         self.run_kayobe_playbooks(parsed_args, playbooks, limit="overcloud")
         playbooks = _build_playbook_list("kolla-ansible")
         self.run_kayobe_playbooks(parsed_args, playbooks, tags="config")
+
+        # Kolla-ansible bootstrap-servers.
+        # The kolla-ansible bootstrap-servers command should be run as the
+        # kayobe ansible user because at this point the kolla user may not
+        # exist.
         extra_vars = {"ansible_user": ansible_user}
+        if python_interpreter:
+            # Use the kayobe virtualenv, as this is the executing user.
+            extra_vars["ansible_python_interpreter"] = python_interpreter
+        elif kolla_target_venv:
+            # Override the kolla-ansible virtualenv, use the system python
+            # instead.
+            extra_vars["ansible_python_interpreter"] = "/usr/bin/python"
+        if kolla_target_venv:
+            # Specify a virtualenv in which to install python packages.
+            extra_vars["virtualenv"] = kolla_target_venv
         self.run_kolla_ansible_overcloud(parsed_args, "bootstrap-servers",
                                          extra_vars=extra_vars)
+
+        # Further kayobe playbooks.
         playbooks = _build_playbook_list("kolla-host", "docker")
         self.run_kayobe_playbooks(parsed_args, playbooks, limit="overcloud")
 

kayobe/tests/unit/cli/test_commands.py

@@ -98,6 +98,7 @@ class TestCase(unittest.TestCase):
                 [
                     "ansible/ip-allocation.yml",
                     "ansible/ssh-known-host.yml",
+                    "ansible/kayobe-target-venv.yml",
                     "ansible/users.yml",
                     "ansible/yum.yml",
                     "ansible/dev-tools.yml",
@@ -121,13 +122,15 @@ class TestCase(unittest.TestCase):
         command = commands.SeedHostConfigure(TestApp(), [])
         parser = command.get_parser("test")
         parsed_args = parser.parse_args([])
-        mock_dump.return_value = "stack"
+        mock_dump.return_value = {
+            "seed": {"kayobe_ansible_user": "stack"}
+        }
 
         result = command.run(parsed_args)
         self.assertEqual(0, result)
 
         expected_calls = [
-            mock.call(mock.ANY, host="seed", var_name="kayobe_ansible_user")
+            mock.call(mock.ANY, hosts="seed")
         ]
         self.assertEqual(expected_calls, mock_dump.call_args_list)
 
@@ -138,6 +141,7 @@ class TestCase(unittest.TestCase):
                     "ansible/ip-allocation.yml",
                     "ansible/ssh-known-host.yml",
                     "ansible/kayobe-ansible-user.yml",
+                    "ansible/kayobe-target-venv.yml",
                     "ansible/users.yml",
                     "ansible/yum.yml",
                     "ansible/dev-tools.yml",
@@ -177,6 +181,108 @@ class TestCase(unittest.TestCase):
         ]
         self.assertEqual(expected_calls, mock_kolla_run.call_args_list)
 
+    @mock.patch.object(commands.KayobeAnsibleMixin,
+                       "run_kayobe_config_dump")
+    @mock.patch.object(commands.KayobeAnsibleMixin,
+                       "run_kayobe_playbooks")
+    @mock.patch.object(commands.KollaAnsibleMixin,
+                       "run_kolla_ansible_seed")
+    def test_seed_host_configure_kayobe_venv(self, mock_kolla_run, mock_run,
+                                             mock_dump):
+        command = commands.SeedHostConfigure(TestApp(), [])
+        parser = command.get_parser("test")
+        parsed_args = parser.parse_args([])
+        mock_dump.return_value = {
+            "seed": {
+                "ansible_python_interpreter": "/kayobe/venv/bin/python",
+                "kayobe_ansible_user": "stack",
+            }
+        }
+
+        result = command.run(parsed_args)
+        self.assertEqual(0, result)
+
+        expected_calls = [
+            mock.call(
+                mock.ANY,
+                "bootstrap-servers",
+                extra_vars={
+                    "ansible_python_interpreter": "/kayobe/venv/bin/python",
+                    "ansible_user": "stack",
+                },
+            ),
+        ]
+        self.assertEqual(expected_calls, mock_kolla_run.call_args_list)
+
+    @mock.patch.object(commands.KayobeAnsibleMixin,
+                       "run_kayobe_config_dump")
+    @mock.patch.object(commands.KayobeAnsibleMixin,
+                       "run_kayobe_playbooks")
+    @mock.patch.object(commands.KollaAnsibleMixin,
+                       "run_kolla_ansible_seed")
+    def test_seed_host_configure_kolla_venv(self, mock_kolla_run, mock_run,
+                                            mock_dump):
+        command = commands.SeedHostConfigure(TestApp(), [])
+        parser = command.get_parser("test")
+        parsed_args = parser.parse_args([])
+        mock_dump.return_value = {
+            "seed": {
+                "kayobe_ansible_user": "stack",
+                "kolla_ansible_target_venv": "/kolla/venv/bin/python",
+            }
+        }
+
+        result = command.run(parsed_args)
+        self.assertEqual(0, result)
+
+        expected_calls = [
+            mock.call(
+                mock.ANY,
+                "bootstrap-servers",
+                extra_vars={
+                    "ansible_python_interpreter": "/usr/bin/python",
+                    "ansible_user": "stack",
+                    "virtualenv": "/kolla/venv/bin/python",
+                },
+            ),
+        ]
+        self.assertEqual(expected_calls, mock_kolla_run.call_args_list)
+
+    @mock.patch.object(commands.KayobeAnsibleMixin,
+                       "run_kayobe_config_dump")
+    @mock.patch.object(commands.KayobeAnsibleMixin,
+                       "run_kayobe_playbooks")
+    @mock.patch.object(commands.KollaAnsibleMixin,
+                       "run_kolla_ansible_seed")
+    def test_seed_host_configure_both_venvs(self, mock_kolla_run, mock_run,
+                                            mock_dump):
+        command = commands.SeedHostConfigure(TestApp(), [])
+        parser = command.get_parser("test")
+        parsed_args = parser.parse_args([])
+        mock_dump.return_value = {
+            "seed": {
+                "ansible_python_interpreter": "/kayobe/venv/bin/python",
+                "kayobe_ansible_user": "stack",
+                "kolla_ansible_target_venv": "/kolla/venv/bin/python",
+            }
+        }
+
+        result = command.run(parsed_args)
+        self.assertEqual(0, result)
+
+        expected_calls = [
+            mock.call(
+                mock.ANY,
+                "bootstrap-servers",
+                extra_vars={
+                    "ansible_python_interpreter": "/kayobe/venv/bin/python",
+                    "ansible_user": "stack",
+                    "virtualenv": "/kolla/venv/bin/python",
+                },
+            ),
+        ]
+        self.assertEqual(expected_calls, mock_kolla_run.call_args_list)
+
     @mock.patch.object(commands.KayobeAnsibleMixin,
                        "run_kayobe_playbooks")
     def test_seed_container_image_build(self, mock_run):
@@ -238,15 +344,15 @@ class TestCase(unittest.TestCase):
         parser = command.get_parser("test")
         parsed_args = parser.parse_args([])
         mock_dump.return_value = {
-            "controller0": "stack"
+            "controller0": {"kayobe_ansible_user": "stack"}
         }
 
         result = command.run(parsed_args)
         self.assertEqual(0, result)
 
         expected_calls = [
-            mock.call(mock.ANY, hosts="overcloud",
+            mock.call(mock.ANY, hosts="overcloud")
-                      var_name="kayobe_ansible_user")]
+        ]
         self.assertEqual(expected_calls, mock_dump.call_args_list)
 
         expected_calls = [
@@ -256,6 +362,7 @@ class TestCase(unittest.TestCase):
                     "ansible/ip-allocation.yml",
                     "ansible/ssh-known-host.yml",
                     "ansible/kayobe-ansible-user.yml",
+                    "ansible/kayobe-target-venv.yml",
                     "ansible/users.yml",
                     "ansible/yum.yml",
                     "ansible/dev-tools.yml",
@@ -293,6 +400,108 @@ class TestCase(unittest.TestCase):
         ]
         self.assertEqual(expected_calls, mock_kolla_run.call_args_list)
 
+    @mock.patch.object(commands.KayobeAnsibleMixin,
+                       "run_kayobe_config_dump")
+    @mock.patch.object(commands.KayobeAnsibleMixin,
+                       "run_kayobe_playbooks")
+    @mock.patch.object(commands.KollaAnsibleMixin,
+                       "run_kolla_ansible_overcloud")
+    def test_overcloud_host_configure_kayobe_venv(self, mock_kolla_run,
+                                                  mock_run, mock_dump):
+        command = commands.OvercloudHostConfigure(TestApp(), [])
+        parser = command.get_parser("test")
+        parsed_args = parser.parse_args([])
+        mock_dump.return_value = {
+            "controller0": {
+                "ansible_python_interpreter": "/kayobe/venv/bin/python",
+                "kayobe_ansible_user": "stack",
+            }
+        }
+
+        result = command.run(parsed_args)
+        self.assertEqual(0, result)
+
+        expected_calls = [
+            mock.call(
+                mock.ANY,
+                "bootstrap-servers",
+                extra_vars={
+                    "ansible_python_interpreter": "/kayobe/venv/bin/python",
+                    "ansible_user": "stack",
+                }
+            ),
+        ]
+        self.assertEqual(expected_calls, mock_kolla_run.call_args_list)
+
+    @mock.patch.object(commands.KayobeAnsibleMixin,
+                       "run_kayobe_config_dump")
+    @mock.patch.object(commands.KayobeAnsibleMixin,
+                       "run_kayobe_playbooks")
+    @mock.patch.object(commands.KollaAnsibleMixin,
+                       "run_kolla_ansible_overcloud")
+    def test_overcloud_host_configure_kolla_venv(self, mock_kolla_run,
+                                                 mock_run, mock_dump):
+        command = commands.OvercloudHostConfigure(TestApp(), [])
+        parser = command.get_parser("test")
+        parsed_args = parser.parse_args([])
+        mock_dump.return_value = {
+            "controller0": {
+                "kayobe_ansible_user": "stack",
+                "kolla_ansible_target_venv": "/kolla/venv/bin/python",
+            }
+        }
+
+        result = command.run(parsed_args)
+        self.assertEqual(0, result)
+
+        expected_calls = [
+            mock.call(
+                mock.ANY,
+                "bootstrap-servers",
+                extra_vars={
+                    "ansible_python_interpreter": "/usr/bin/python",
+                    "ansible_user": "stack",
+                    "virtualenv": "/kolla/venv/bin/python",
+                }
+            ),
+        ]
+        self.assertEqual(expected_calls, mock_kolla_run.call_args_list)
+
+    @mock.patch.object(commands.KayobeAnsibleMixin,
+                       "run_kayobe_config_dump")
+    @mock.patch.object(commands.KayobeAnsibleMixin,
+                       "run_kayobe_playbooks")
+    @mock.patch.object(commands.KollaAnsibleMixin,
+                       "run_kolla_ansible_overcloud")
+    def test_overcloud_host_configure_both_venvs(self, mock_kolla_run,
+                                                 mock_run, mock_dump):
+        command = commands.OvercloudHostConfigure(TestApp(), [])
+        parser = command.get_parser("test")
+        parsed_args = parser.parse_args([])
+        mock_dump.return_value = {
+            "controller0": {
+                "ansible_python_interpreter": "/kayobe/venv/bin/python",
+                "kayobe_ansible_user": "stack",
+                "kolla_ansible_target_venv": "/kolla/venv/bin/python",
+            }
+        }
+
+        result = command.run(parsed_args)
+        self.assertEqual(0, result)
+
+        expected_calls = [
+            mock.call(
+                mock.ANY,
+                "bootstrap-servers",
+                extra_vars={
+                    "ansible_python_interpreter": "/kayobe/venv/bin/python",
+                    "ansible_user": "stack",
+                    "virtualenv": "/kolla/venv/bin/python",
+                }
+            ),
+        ]
+        self.assertEqual(expected_calls, mock_kolla_run.call_args_list)
+
     @mock.patch.object(commands.KayobeAnsibleMixin,
                        "run_kayobe_playbooks")
     def test_overcloud_container_image_build(self, mock_run):