From efb8b8bd27d08469c94f103b7314e9905cf6aa0e Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 24 Oct 2019 13:43:30 +0100 Subject: [PATCH] Use docker_custom_config variable In the Train cycle, Kolla Ansible added support for docker_custom_config, and writes out configuration to /etc/docker/daemon.json. This will conflict with Kayobe's configuration of that file, and changes made by kayobe will be reversed when kolla-ansible bootstrap-servers is run. This change uses the new variable to pass daemon.json configuration through to kolla ansible. Because the ordering has changed, we also need to separate out the devicemapper setup and run this prior to starting docker. Change-Id: Idc3fa9fefd8242ef9db76d4d773885e3594b453a Depends-On: https://review.opendev.org/691001 Story: 2006764 Task: 37277 --- ansible/docker-devicemapper.yml | 11 +++++ ansible/docker.yml | 1 - ansible/kolla-ansible.yml | 1 + .../docker-devicemapper/defaults/main.yml | 25 ++++++++++ .../docker-devicemapper/handlers/main.yml | 13 +++++ .../tasks/main.yml} | 48 +++++++++++-------- .../templates/docker-thinpool.profile.j2 | 0 ansible/roles/docker/defaults/main.yml | 34 ------------- ansible/roles/docker/handlers/main.yml | 6 --- ansible/roles/docker/tasks/config.yml | 22 --------- ansible/roles/docker/tasks/main.yml | 33 +++++-------- ansible/roles/kolla-ansible/defaults/main.yml | 27 +++++++++++ ansible/roles/kolla-ansible/tasks/config.yml | 2 + .../templates/daemon.json.j2 | 1 - .../kolla-ansible/templates/globals.yml.j2 | 2 + kayobe/cli/commands.py | 4 +- kayobe/tests/unit/cli/test_commands.py | 4 ++ ...docker-custom-config-5103260d5ddb7223.yaml | 5 ++ 18 files changed, 132 insertions(+), 107 deletions(-) create mode 100644 ansible/docker-devicemapper.yml create mode 100644 ansible/roles/docker-devicemapper/defaults/main.yml create mode 100644 ansible/roles/docker-devicemapper/handlers/main.yml rename ansible/roles/{docker/tasks/storage.yml => docker-devicemapper/tasks/main.yml} (54%) rename ansible/roles/{docker => docker-devicemapper}/templates/docker-thinpool.profile.j2 (100%) delete mode 100644 ansible/roles/docker/tasks/config.yml rename ansible/roles/{docker => kolla-ansible}/templates/daemon.json.j2 (93%) create mode 100644 releasenotes/notes/docker-custom-config-5103260d5ddb7223.yaml diff --git a/ansible/docker-devicemapper.yml b/ansible/docker-devicemapper.yml new file mode 100644 index 000000000..e61c97644 --- /dev/null +++ b/ansible/docker-devicemapper.yml @@ -0,0 +1,11 @@ +--- +- name: Ensure docker devicemapper storage is configured + hosts: docker + tags: + - docker + - docker-devicemapper + tasks: + - name: Ensure docker devicemapper storage is configured + include_role: + name: docker-devicemapper + when: docker_storage_driver == 'devicemapper' diff --git a/ansible/docker.yml b/ansible/docker.yml index 712fda784..c2405444a 100644 --- a/ansible/docker.yml +++ b/ansible/docker.yml @@ -7,4 +7,3 @@ - docker_upper_constraints_file: "{{ pip_upper_constraints_file }}" roles: - role: docker - docker_daemon_mtu: "{{ public_net_name | net_mtu | default }}" diff --git a/ansible/kolla-ansible.yml b/ansible/kolla-ansible.yml index b5b7bb218..50481f5a5 100644 --- a/ansible/kolla-ansible.yml +++ b/ansible/kolla-ansible.yml @@ -303,3 +303,4 @@ # While kayobe has its own support for installing an NTP daemon, the # kolla-ansible baremetal role does a one-time sync which is useful. kolla_enable_host_ntp: "{{ ntp_service_enabled }}" + docker_daemon_mtu: "{{ public_net_name | net_mtu | default }}" diff --git a/ansible/roles/docker-devicemapper/defaults/main.yml b/ansible/roles/docker-devicemapper/defaults/main.yml new file mode 100644 index 000000000..b9f813b0b --- /dev/null +++ b/ansible/roles/docker-devicemapper/defaults/main.yml @@ -0,0 +1,25 @@ +--- +# Name of the docker storage driver. +docker_storage_driver: devicemapper + +# Name of the docker storage LVM volume group. +docker_storage_volume_group: + +# Name of the docker storage data LVM volume. +docker_storage_volume_thinpool: + +# Size of the docker storage data LVM volume (see lvol module size argument). +docker_storage_volume_thinpool_size: + +# Name of the docker storage metadata LVM volume. +docker_storage_volume_thinpool_meta: + +# Size of the docker storage metadata LVM volume (see lvol module size +# argument). +docker_storage_volume_thinpool_meta_size: + +# Threshold at which to extend thin-provisioned docker storage volumes. +docker_storage_thinpool_autoextend_threshold: 80 + +# Percentage by which to extend thin-provisioned docker storage volumes. +docker_storage_thinpool_autoextend_percent: 20 diff --git a/ansible/roles/docker-devicemapper/handlers/main.yml b/ansible/roles/docker-devicemapper/handlers/main.yml new file mode 100644 index 000000000..b7bf0f830 --- /dev/null +++ b/ansible/roles/docker-devicemapper/handlers/main.yml @@ -0,0 +1,13 @@ +--- +- name: Ensure the docker storage volume is converted to a thinpool + command: > + lvconvert -y --zero n -c 512K + --thinpool {{ docker_storage_volume_group }}/{{ docker_storage_volume_thinpool }} + --poolmetadata {{ docker_storage_volume_group }}/{{ docker_storage_volume_thinpool_meta }} + become: True + +- name: Ensure the docker storage metadata profile is applied + command: > + lvchange --metadataprofile docker-thinpool + {{ docker_storage_volume_group }}/{{ docker_storage_volume_thinpool }} + become: True diff --git a/ansible/roles/docker/tasks/storage.yml b/ansible/roles/docker-devicemapper/tasks/main.yml similarity index 54% rename from ansible/roles/docker/tasks/storage.yml rename to ansible/roles/docker-devicemapper/tasks/main.yml index eed6b6a13..fbda5a607 100644 --- a/ansible/roles/docker/tasks/storage.yml +++ b/ansible/roles/docker-devicemapper/tasks/main.yml @@ -1,21 +1,33 @@ --- -- name: Ensure the docker daemon is stopped - service: - name: docker - state: stopped - become: True - notify: restart docker service +- name: Query docker daemon information + command: "docker info" + register: docker_info + changed_when: False + failed_when: False -- name: Ensure loopback storage state is absent - file: - path: "{{ item }}" - state: absent - with_items: - - "/var/lib/docker/devicemapper" - - "/var/lib/docker/images" - - "/var/lib/docker/containers" +- name: Fail when non-devicemapper containers or images exist + fail: + msg: > + Not configuring docker storage in {{ docker_storage_driver }} mode as + non-devicemapper containers or images exist. + when: + - docker_info.rc == 0 + - "'Data loop file' in docker_info.stdout or 'devicemapper' not in docker_info.stdout" + - "'Images: 0' not in docker_info.stdout or 'Containers: 0' not in docker_info.stdout" + +- name: Ensure the docker storage metadata profile exists + template: + src: docker-thinpool.profile.j2 + dest: /etc/lvm/profile/docker-thinpool.profile become: True +- name: Query LVM thinpool volume + command: "lvs {{ docker_storage_volume_group }}/{{ docker_storage_volume_thinpool }}" + register: lvs_result + changed_when: false + failed_when: false + become: true + - block: - name: Ensure the docker storage data and metadata volumes exist lvol: @@ -38,15 +50,9 @@ --poolmetadata {{ docker_storage_volume_group }}/{{ docker_storage_volume_thinpool_meta }} become: True - - name: Ensure the docker storage metadata profile exists - template: - src: docker-thinpool.profile.j2 - dest: /etc/lvm/profile/docker-thinpool.profile - become: True - - name: Ensure the docker storage metadata profile is applied command: > lvchange --metadataprofile docker-thinpool {{ docker_storage_volume_group }}/{{ docker_storage_volume_thinpool }} become: True - when: docker_storage_driver == 'devicemapper' + when: lvs_result.rc != 0 diff --git a/ansible/roles/docker/templates/docker-thinpool.profile.j2 b/ansible/roles/docker-devicemapper/templates/docker-thinpool.profile.j2 similarity index 100% rename from ansible/roles/docker/templates/docker-thinpool.profile.j2 rename to ansible/roles/docker-devicemapper/templates/docker-thinpool.profile.j2 diff --git a/ansible/roles/docker/defaults/main.yml b/ansible/roles/docker/defaults/main.yml index 59165a251..756d5cca3 100644 --- a/ansible/roles/docker/defaults/main.yml +++ b/ansible/roles/docker/defaults/main.yml @@ -1,44 +1,10 @@ --- -# Name of the docker storage driver. -docker_storage_driver: devicemapper - -# Name of the docker storage LVM volume group. -docker_storage_volume_group: - -# Name of the docker storage data LVM volume. -docker_storage_volume_thinpool: - -# Size of the docker storage data LVM volume (see lvol module size argument). -docker_storage_volume_thinpool_size: - -# Name of the docker storage metadata LVM volume. -docker_storage_volume_thinpool_meta: - -# Size of the docker storage metadata LVM volume (see lvol module size -# argument). -docker_storage_volume_thinpool_meta_size: - -# Threshold at which to extend thin-provisioned docker storage volumes. -docker_storage_thinpool_autoextend_threshold: 80 - -# Percentage by which to extend thin-provisioned docker storage volumes. -docker_storage_thinpool_autoextend_percent: 20 - # URL of docker registry docker_registry: # CA of docker registry docker_registry_ca: -# List of Docker registry mirrors. -docker_registry_mirrors: [] - -# MTU to pass through to containers not using net=host -docker_daemon_mtu: 1500 - -# Enable live-restore on docker daemon -docker_daemon_live_restore: false - # Upper constraints file which is passed to pip when installing packages # into a venv. docker_upper_constraints_file: diff --git a/ansible/roles/docker/handlers/main.yml b/ansible/roles/docker/handlers/main.yml index 8a285ad69..356c6b622 100644 --- a/ansible/roles/docker/handlers/main.yml +++ b/ansible/roles/docker/handlers/main.yml @@ -1,10 +1,4 @@ --- -- name: restart docker service - service: - name: docker - state: restarted - become: True - - name: reload docker service service: name: docker diff --git a/ansible/roles/docker/tasks/config.yml b/ansible/roles/docker/tasks/config.yml deleted file mode 100644 index cd01d350f..000000000 --- a/ansible/roles/docker/tasks/config.yml +++ /dev/null @@ -1,22 +0,0 @@ ---- -- name: Ensure the docker daemon configuration file exists - template: - src: daemon.json.j2 - dest: /etc/docker/daemon.json - become: True - notify: restart docker service - -- name: Ensure the path for CA file for private registry exists - file: - path: "/etc/docker/certs.d/{{ docker_registry }}" - state: directory - become: True - when: docker_registry is not none and docker_registry_ca is not none - -- name: Ensure the CA file for private registry exists - copy: - src: "{{ docker_registry_ca }}" - dest: "/etc/docker/certs.d/{{ docker_registry }}/ca.crt" - become: True - when: docker_registry is not none and docker_registry_ca is not none - notify: reload docker service diff --git a/ansible/roles/docker/tasks/main.yml b/ansible/roles/docker/tasks/main.yml index 9a792723e..494656e37 100644 --- a/ansible/roles/docker/tasks/main.yml +++ b/ansible/roles/docker/tasks/main.yml @@ -57,24 +57,17 @@ state: started become: True -- name: Query docker daemon information - command: "docker info" - register: docker_info - changed_when: False - until: docker_info is success - retries: 3 - delay: 5 +- name: Ensure the path for CA file for private registry exists + file: + path: "/etc/docker/certs.d/{{ docker_registry }}" + state: directory + become: True + when: docker_registry is not none and docker_registry_ca is not none -- name: Fail when loopback-mode containers or images exist - fail: - msg: > - Not configuring docker storage in {{ docker_storage_driver }} mode as - loopback-backed containers or images exist. - when: - - "'Data loop file' in docker_info.stdout or docker_storage_driver not in docker_info.stdout" - - "'Images: 0' not in docker_info.stdout or 'Containers: 0' not in docker_info.stdout" - -- include_tasks: storage.yml - when: "'Data loop file' in docker_info.stdout or docker_storage_driver not in docker_info.stdout" - -- include_tasks: config.yml +- name: Ensure the CA file for private registry exists + copy: + src: "{{ docker_registry_ca }}" + dest: "/etc/docker/certs.d/{{ docker_registry }}/ca.crt" + become: True + when: docker_registry is not none and docker_registry_ca is not none + notify: reload docker service diff --git a/ansible/roles/kolla-ansible/defaults/main.yml b/ansible/roles/kolla-ansible/defaults/main.yml index e239cf0de..d7a440a21 100644 --- a/ansible/roles/kolla-ansible/defaults/main.yml +++ b/ansible/roles/kolla-ansible/defaults/main.yml @@ -312,3 +312,30 @@ kolla_selinux_state: # Whether to enable the NTP daemon. kolla_enable_host_ntp: + +############################################################################### +# Docker configuration. + +# Name of the docker storage driver. +docker_storage_driver: devicemapper + +# Name of the docker storage LVM volume group. +docker_storage_volume_group: + +# Name of the docker storage data LVM volume. +docker_storage_volume_thinpool: + +# URL of docker registry +docker_registry: + +# CA of docker registry +docker_registry_ca: + +# List of Docker registry mirrors. +docker_registry_mirrors: [] + +# MTU to pass through to containers not using net=host +docker_daemon_mtu: 1500 + +# Enable live-restore on docker daemon +docker_daemon_live_restore: false diff --git a/ansible/roles/kolla-ansible/tasks/config.yml b/ansible/roles/kolla-ansible/tasks/config.yml index 55849026c..6519fc892 100644 --- a/ansible/roles/kolla-ansible/tasks/config.yml +++ b/ansible/roles/kolla-ansible/tasks/config.yml @@ -45,6 +45,8 @@ src: "globals.yml.j2" dest: "{{ kolla_config_path }}/globals.yml" mode: 0640 + vars: + kolla_docker_custom_config: "{{ lookup('template', 'daemon.json.j2') }}" - name: Ensure the Kolla seed inventory file exists copy: diff --git a/ansible/roles/docker/templates/daemon.json.j2 b/ansible/roles/kolla-ansible/templates/daemon.json.j2 similarity index 93% rename from ansible/roles/docker/templates/daemon.json.j2 rename to ansible/roles/kolla-ansible/templates/daemon.json.j2 index f2d63d78b..da8ede525 100644 --- a/ansible/roles/docker/templates/daemon.json.j2 +++ b/ansible/roles/kolla-ansible/templates/daemon.json.j2 @@ -6,7 +6,6 @@ {%- endfor %} ], {%- endif %} - "storage-driver": "{{ docker_storage_driver }}", {% if docker_daemon_mtu %} "mtu": {{ docker_daemon_mtu }}, {% endif %} diff --git a/ansible/roles/kolla-ansible/templates/globals.yml.j2 b/ansible/roles/kolla-ansible/templates/globals.yml.j2 index 05fc332d9..f34a66f2e 100644 --- a/ansible/roles/kolla-ansible/templates/globals.yml.j2 +++ b/ansible/roles/kolla-ansible/templates/globals.yml.j2 @@ -68,6 +68,8 @@ docker_namespace: "{{ kolla_docker_namespace }}" docker_registry_username: "{{ kolla_docker_registry_username }}" docker_registry_password: "{{ kolla_docker_registry_password }}" {% endif %} +docker_storage_driver: "{{ docker_storage_driver }}" +docker_custom_config: {{ kolla_docker_custom_config | to_nice_json | indent(2) }} ################### # Messaging options diff --git a/kayobe/cli/commands.py b/kayobe/cli/commands.py index 26a7418a5..06c7b841b 100644 --- a/kayobe/cli/commands.py +++ b/kayobe/cli/commands.py @@ -551,7 +551,7 @@ class SeedHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin, playbooks += _build_playbook_list( "users", "yum", "dev-tools", "disable-selinux", "network", "sysctl", "ip-routing", "snat", "disable-glean", "ntp", "mdadm", - "lvm") + "lvm", "docker-devicemapper") self.run_kayobe_playbooks(parsed_args, playbooks, limit="seed") self.generate_kolla_ansible_config(parsed_args, service_config=False) @@ -951,7 +951,7 @@ class OvercloudHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin, playbooks += _build_playbook_list( "users", "yum", "dev-tools", "disable-selinux", "network", "sysctl", "disable-glean", "disable-cloud-init", "ntp", "mdadm", - "lvm") + "lvm", "docker-devicemapper") self.run_kayobe_playbooks(parsed_args, playbooks, limit="overcloud") self.generate_kolla_ansible_config(parsed_args, service_config=False) diff --git a/kayobe/tests/unit/cli/test_commands.py b/kayobe/tests/unit/cli/test_commands.py index 9d38ea6a0..441edca88 100644 --- a/kayobe/tests/unit/cli/test_commands.py +++ b/kayobe/tests/unit/cli/test_commands.py @@ -513,6 +513,8 @@ class TestCase(unittest.TestCase): utils.get_data_files_path("ansible", "ntp.yml"), utils.get_data_files_path("ansible", "mdadm.yml"), utils.get_data_files_path("ansible", "lvm.yml"), + utils.get_data_files_path("ansible", + "docker-devicemapper.yml"), ], limit="seed", ), @@ -1138,6 +1140,8 @@ class TestCase(unittest.TestCase): utils.get_data_files_path("ansible", "ntp.yml"), utils.get_data_files_path("ansible", "mdadm.yml"), utils.get_data_files_path("ansible", "lvm.yml"), + utils.get_data_files_path("ansible", + "docker-devicemapper.yml"), ], limit="overcloud", ), diff --git a/releasenotes/notes/docker-custom-config-5103260d5ddb7223.yaml b/releasenotes/notes/docker-custom-config-5103260d5ddb7223.yaml new file mode 100644 index 000000000..327658603 --- /dev/null +++ b/releasenotes/notes/docker-custom-config-5103260d5ddb7223.yaml @@ -0,0 +1,5 @@ +--- +upgrade: + - | + Uses the new Kolla Ansible variable ``docker_custom_config`` to populate + Docker's ``daemon.json`` configuration file.