From c16597aa2daf76d7dd31991c5db938eee291a098 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Fri, 6 Apr 2018 19:47:32 +0100 Subject: [PATCH] Add seed VM provisioning CI job Adds the kayobe-seed-vm-centos8 CI job to configure the Zuul VM as a seed hypervisor, and use nested virt to provision a seed VM. This ensures that the seed hypervisor code paths are tested. The job uses a Cirros image for the seed VM rather than the usual CentOS cloud image. This is to reduce bandwidth required to download the image. It does mean that the resulting seed VM cannot be used as a seed, but nested virt would make this slow and unreliable anyway. Cirros does not load cdrom drivers by default, so we add the configdrive as a disk rather than a cdrom device. Depends-On: https://review.opendev.org/617161 Change-Id: I2268a1ddf9a2870c713f32a40689e1686365aabd Story: 2001655 Task: 6683 --- ansible/seed-vm-provision.yml | 3 +- dev/functions | 18 ++++-- .../kayobe-seed-vm-base/overrides.yml.j2 | 44 +++++++++++++ playbooks/kayobe-seed-vm-base/pre.yml | 63 +++++++++++++++++++ playbooks/kayobe-seed-vm-base/run.yml | 15 +++++ .../kayobe-seed-vm-base/seed-group-vars.j2 | 5 ++ requirements.yml | 2 +- zuul.d/jobs.yaml | 17 +++++ zuul.d/project.yaml | 2 + 9 files changed, 163 insertions(+), 6 deletions(-) create mode 100644 playbooks/kayobe-seed-vm-base/overrides.yml.j2 create mode 100644 playbooks/kayobe-seed-vm-base/pre.yml create mode 100644 playbooks/kayobe-seed-vm-base/run.yml create mode 100644 playbooks/kayobe-seed-vm-base/seed-group-vars.j2 diff --git a/ansible/seed-vm-provision.yml b/ansible/seed-vm-provision.yml index af6fb2e56..cc5e3839a 100644 --- a/ansible/seed-vm-provision.yml +++ b/ansible/seed-vm-provision.yml @@ -82,12 +82,13 @@ roles: - role: stackhpc.libvirt-vm + seed_vm_configdrive_device: cdrom seed_vm_configdrive_volume: name: "{{ hostvars[seed_host].seed_vm_name }}-configdrive" pool: "{{ hostvars[seed_host].seed_vm_pool }}" # Round size up to next multiple of 4096. capacity: "{{ (stat_result.stat.size + 4095) // 4096 * 4096 }}" - device: "cdrom" + device: "{{ seed_vm_configdrive_device }}" format: "raw" image: "{{ seed_vm_configdrive_path }}" libvirt_vm_image_cache_path: "{{ image_cache_path }}" diff --git a/dev/functions b/dev/functions index da3fa83a4..d8c06b6df 100644 --- a/dev/functions +++ b/dev/functions @@ -30,10 +30,16 @@ function config_defaults { # Whether to provision a VM for the seed host. export KAYOBE_SEED_VM_PROVISION=${KAYOBE_SEED_VM_PROVISION:-1} + # Whether to configure the seed host. + export KAYOBE_SEED_HOST_CONFIGURE=${KAYOBE_SEED_HOST_CONFIGURE:-1} + # Whether to build container images for the seed services. If 0, they will # be pulled. export KAYOBE_SEED_CONTAINER_IMAGE_BUILD=${KAYOBE_SEED_CONTAINER_IMAGE_BUILD:-0} + # Whether to deploy seed services. + export KAYOBE_SEED_SERVICE_DEPLOY=${KAYOBE_SEED_SERVICE_DEPLOY:-1} + # Whether to build container images for the overcloud services. If 0, they # will be pulled if $KAYOBE_OVERCLOUD_CONTAINER_IMAGE_PULL is 1. export KAYOBE_OVERCLOUD_CONTAINER_IMAGE_BUILD=${KAYOBE_OVERCLOUD_CONTAINER_IMAGE_BUILD:-0} @@ -262,8 +268,10 @@ function seed_deploy { run_kayobe seed vm provision fi - echo "Configuring the seed host" - run_kayobe seed host configure + if [[ ${KAYOBE_SEED_HOST_CONFIGURE} = 1 ]]; then + echo "Configuring the seed host" + run_kayobe seed host configure + fi # Note: This must currently be done before host configure, because host # configure runs kolla-ansible.yml, which validates the presence of the @@ -283,8 +291,10 @@ function seed_deploy { #run_kayobe seed container image pull fi - echo "Deploying containerised seed services" - run_kayobe seed service deploy + if [[ ${KAYOBE_SEED_SERVICE_DEPLOY} = 1 ]]; then + echo "Deploying containerised seed services" + run_kayobe seed service deploy + fi } function seed_upgrade { diff --git a/playbooks/kayobe-seed-vm-base/overrides.yml.j2 b/playbooks/kayobe-seed-vm-base/overrides.yml.j2 new file mode 100644 index 000000000..22061d14e --- /dev/null +++ b/playbooks/kayobe-seed-vm-base/overrides.yml.j2 @@ -0,0 +1,44 @@ +--- +# NOTE(mgoddard): Don't reboot after disabling SELinux during CI testing, as +# Ansible is run directly on the controller. +disable_selinux_do_reboot: false + +# Use the OpenStack infra's Dockerhub mirror. +docker_registry_mirrors: + - "http://{{ zuul_site_mirror_fqdn }}:8082/" + +kolla_source_url: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['opendev.org/openstack/kolla'].src_dir }}" +kolla_source_version: "{{ zuul.projects['opendev.org/openstack/kolla'].checkout }}" +kolla_ansible_source_url: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['opendev.org/openstack/kolla-ansible'].src_dir }}" +kolla_ansible_source_version: "{{ zuul.projects['opendev.org/openstack/kolla-ansible'].checkout }}" +kolla_openstack_logging_debug: True +pip_upper_constraints_file: "/tmp/upper-constraints.txt" + +# The hosts used by Zuul may or may not have Virtualization Technology (VT) +# enabled. Don't fail if it's disabled. +libvirt_host_require_vt: false + +# Use the CI infra's PyPI mirror. +pip_local_mirror: true +pip_index_url: "http://{{ zuul_site_mirror_fqdn }}/pypi/simple" +pip_trusted_hosts: + - "{{ zuul_site_mirror_fqdn }}" + +# Try with only a single VCPU, word on the street is that QEMU doesn't play +# nicely with more than one. +seed_vm_vcpus: 1 + +# Reduce the memory footprint of the seed VM. +seed_vm_memory_mb: "{{ 1 * 1024 }}" + +# Use cirros rather than CentOS for the VM. +seed_bootstrap_user: cirros +seed_vm_root_image: https://download.cirros-cloud.net/0.5.1/cirros-0.5.1-x86_64-disk.img + +# Cirros doesn't load cdom drivers by default. +seed_vm_configdrive_device: disk + +# Cirros is Debian family, but doesn't support path globs in +# /etc/network/interfaces. +configdrive_os_family: Debian +configdrive_debian_network_interfaces_supports_glob: false diff --git a/playbooks/kayobe-seed-vm-base/pre.yml b/playbooks/kayobe-seed-vm-base/pre.yml new file mode 100644 index 000000000..85635fa53 --- /dev/null +++ b/playbooks/kayobe-seed-vm-base/pre.yml @@ -0,0 +1,63 @@ +--- +- hosts: primary + environment: + KAYOBE_CONFIG_SOURCE_PATH: "{{ kayobe_config_src_dir }}" + tasks: + # NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes + # precedence over the standard config files. + - name: Ensure kayobe-config override config file exists + template: + src: overrides.yml.j2 + dest: "{{ kayobe_config_src_dir }}/etc/kayobe/zz-overrides.yml" + + - name: Ensure seed group variables exist + template: + src: seed-group-vars.j2 + dest: "{{ kayobe_config_src_dir }}/etc/kayobe/inventory/group_vars/seed/network-interfaces" + + # NOTE(mgoddard): The kayobe dev config by default expects a bridge - + # braio - to exist with an IP address of 192.168.33.4. + - name: Ensure all-in-one network bridge interface exists + command: "{{ item }}" + become: true + with_items: + - "ip l add braio type bridge" + - "ip l set braio up" + - "ip a add 192.168.33.4/24 dev braio" + # NOTE(mgoddard): CentOS 8 removes interfaces from their bridge during + # ifdown, and removes the bridge if there are no interfaces left. When + # Kayobe bounces veth links plugged into the bridge, it causes the + # bridge which has the IP we are using for SSH to be removed. Use a + # dummy interface. + - "ip l add dummy1 type dummy" + - "ip l set dummy1 up" + - "ip l set dummy1 master braio" + + # NOTE(mgoddard): Configure IP forwarding and NAT to allow communication + # from the seed VM to the outside world. + + # FIXME(mgoddard): use a libvirt network? + - name: Ensure NAT is configured + iptables: + chain: POSTROUTING + table: nat + out_interface: "{{ ansible_default_ipv4.interface }}" + jump: MASQUERADE + become: true + + # FIXME(mgoddard): use a libvirt network? + - name: Ensure IP forwarding is enabled + sysctl: + name: net.ipv4.conf.all.forwarding + value: 1 + become: true + + - name: Ensure SELinux is disabled + selinux: + state: disabled + become: True + + - name: Ensure kayobe is installed + shell: + cmd: dev/install.sh &> {{ logs_dir }}/ansible/install + chdir: "{{ kayobe_src_dir }}" diff --git a/playbooks/kayobe-seed-vm-base/run.yml b/playbooks/kayobe-seed-vm-base/run.yml new file mode 100644 index 000000000..bfda7dda6 --- /dev/null +++ b/playbooks/kayobe-seed-vm-base/run.yml @@ -0,0 +1,15 @@ +--- +- hosts: primary + environment: + KAYOBE_CONFIG_SOURCE_PATH: "{{ kayobe_config_src_dir }}" + # The CirrOS image does not support much beyond logging in. + KAYOBE_SEED_HOST_CONFIGURE: 0 + KAYOBE_SEED_SERVICE_DEPLOY: 0 + tasks: + - name: Ensure seed hypervisor is deployed + shell: + cmd: "{{ kayobe_src_dir }}/dev/seed-hypervisor-deploy.sh > {{ logs_dir }}/ansible/seed-hypervisor-deploy" + + - name: Ensure seed is deployed + shell: + cmd: "{{ kayobe_src_dir }}/dev/seed-deploy.sh &> {{ logs_dir }}/ansible/seed-deploy" diff --git a/playbooks/kayobe-seed-vm-base/seed-group-vars.j2 b/playbooks/kayobe-seed-vm-base/seed-group-vars.j2 new file mode 100644 index 000000000..109707e11 --- /dev/null +++ b/playbooks/kayobe-seed-vm-base/seed-group-vars.j2 @@ -0,0 +1,5 @@ +--- +aio_interface: eth0 + +# Route via the seed-hypervisor to the outside world. +aio_gateway: 192.168.33.4 diff --git a/requirements.yml b/requirements.yml index f43aeb8f8..4a5da2fab 100644 --- a/requirements.yml +++ b/requirements.yml @@ -3,7 +3,7 @@ version: 1.3.1 - src: jriguera.configdrive # There are no versioned releases of this role. - version: 56daf0017783bd4fa5722da4ee91b66ddd4e6b6a + version: 8438592c84585c86e62ae07e526d3da53629b377 - src: MichaelRigart.interfaces version: v1.5.0 - src: mrlesmithjr.manage-lvm diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 14db735bf..cc2903777 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -165,3 +165,20 @@ name: kayobe-seed-upgrade-centos8 parent: kayobe-seed-upgrade-base nodeset: kayobe-centos8 + +- job: + name: kayobe-seed-vm-base + parent: kayobe-base + description: | + Base job for testing seed VM provisioning. + + Configures the primary VM as a libvirt hypervisor, and provisions a seed + as a VM. + pre-run: playbooks/kayobe-seed-vm-base/pre.yml + run: playbooks/kayobe-seed-vm-base/run.yml + timeout: 5400 + +- job: + name: kayobe-seed-vm-centos8 + parent: kayobe-seed-vm-base + nodeset: kayobe-centos8 diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index 0621b1c30..80375c810 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -15,6 +15,7 @@ - kayobe-overcloud-upgrade-centos8 - kayobe-seed-centos8 - kayobe-seed-upgrade-centos8 + - kayobe-seed-vm-centos8 gate: queue: kayobe @@ -27,3 +28,4 @@ - kayobe-overcloud-upgrade-centos8 - kayobe-seed-centos8 - kayobe-seed-upgrade-centos8 + - kayobe-seed-vm-centos8