diff --git a/.zuul.d/cinder.yaml b/.zuul.d/cinder.yaml index 06768535..96730714 100644 --- a/.zuul.d/cinder.yaml +++ b/.zuul.d/cinder.yaml @@ -16,6 +16,7 @@ project: cinder required-projects: - openstack/loci + - openstack/requirements - openstack/cinder - job: diff --git a/.zuul.d/glance.yaml b/.zuul.d/glance.yaml index 2ee11ef6..159e07c0 100644 --- a/.zuul.d/glance.yaml +++ b/.zuul.d/glance.yaml @@ -16,6 +16,7 @@ project: glance required-projects: - openstack/loci + - openstack/requirements - openstack/glance - job: diff --git a/.zuul.d/heat.yaml b/.zuul.d/heat.yaml index 4c4dbacd..c05834f9 100644 --- a/.zuul.d/heat.yaml +++ b/.zuul.d/heat.yaml @@ -16,6 +16,7 @@ project: heat required-projects: - openstack/loci + - openstack/requirements - openstack/heat - job: diff --git a/.zuul.d/horizon.yaml b/.zuul.d/horizon.yaml index a142d712..9c3e706b 100644 --- a/.zuul.d/horizon.yaml +++ b/.zuul.d/horizon.yaml @@ -16,6 +16,7 @@ project: horizon required-projects: - openstack/loci + - openstack/requirements - openstack/horizon - job: diff --git a/.zuul.d/ironic.yaml b/.zuul.d/ironic.yaml index 11ca1464..14ffa504 100644 --- a/.zuul.d/ironic.yaml +++ b/.zuul.d/ironic.yaml @@ -16,6 +16,7 @@ project: ironic required-projects: - openstack/loci + - openstack/requirements - openstack/ironic - job: diff --git a/.zuul.d/keystone.yaml b/.zuul.d/keystone.yaml index da7db6f2..8f09f420 100644 --- a/.zuul.d/keystone.yaml +++ b/.zuul.d/keystone.yaml @@ -16,6 +16,7 @@ project: keystone required-projects: - openstack/loci + - openstack/requirements - openstack/keystone - job: diff --git a/.zuul.d/neutron.yaml b/.zuul.d/neutron.yaml index 3f9d0ab7..f91d47a7 100644 --- a/.zuul.d/neutron.yaml +++ b/.zuul.d/neutron.yaml @@ -16,6 +16,7 @@ project: neutron required-projects: - openstack/loci + - openstack/requirements - openstack/neutron - job: diff --git a/.zuul.d/nova.yaml b/.zuul.d/nova.yaml index 78e8ec43..e683d3a3 100644 --- a/.zuul.d/nova.yaml +++ b/.zuul.d/nova.yaml @@ -16,6 +16,7 @@ project: nova required-projects: - openstack/loci + - openstack/requirements - openstack/nova - job: diff --git a/.zuul.d/octavia.yaml b/.zuul.d/octavia.yaml index e4965a4d..d64542b9 100644 --- a/.zuul.d/octavia.yaml +++ b/.zuul.d/octavia.yaml @@ -16,6 +16,7 @@ project: octavia required-projects: - openstack/loci + - openstack/requirements - openstack/octavia - job: diff --git a/playbooks/loci-builder.yaml b/playbooks/loci-builder.yaml index ca179c4c..a57d58c7 100644 --- a/playbooks/loci-builder.yaml +++ b/playbooks/loci-builder.yaml @@ -9,7 +9,13 @@ - "pydep.txt" environment: LC_ALL: C - + # NOTE(evrardjp): While reuse_requirements is very nice and optimises + # checks and gating, there is a race condition here, because + # we are consuming prebuild wheels (see vars.yaml) by default: + # The jobs in zuul can be building a new "requirements" image, working + # And the new "nova" image would still build on previous "requirements" + # image that was last published. This could cause an issue. Instead + # in gating we should build directly what will be consumed. # NOTE(SamYaple): This process is so we can take advantage of the infra # DockerHub mirroring as configured through the Docker daemon. We do this # instead of calling fetch_wheels initially. All-in-all this saves @@ -25,6 +31,7 @@ async: 1000 poll: 0 register: pull + - async_status: jid: "{{ item.ansible_job_id }}" with_items: "{{ pull.results }}" @@ -34,11 +41,14 @@ - pull_result.finished retries: 1000 delay: 5 - when: project != 'requirements' + when: + - reuse_requirements | bool + - project != 'requirements' - name: Build base images block: - - docker_image: + - name: "Build base image for {{ item.name }} asynchronously" + docker_image: path: "{{ loci_src_dir }}/dockerfiles/{{ item.name }}" name: base tag: "{{ item.name }}" @@ -47,6 +57,7 @@ async: 1000 poll: 0 register: base + - async_status: jid: "{{ item.ansible_job_id }}" with_items: "{{ base.results }}" @@ -57,18 +68,49 @@ retries: 1000 delay: 5 - - name: Build project images + - name: Build requirements image block: - - docker_image: + - name: "Build requirements image for {{ item.name }}" + docker_image: path: "{{ loci_src_dir }}" - name: loci/{{ project }} + name: loci/requirements tag: "{{ branch }}-{{ item.name }}" + repository: 172.17.0.1:5000/loci/requirements + push: yes pull: False - buildargs: "{{ item.buildargs.project }}" + buildargs: "{{ item.buildargs.requirements }}" with_items: "{{ distros }}" async: 1000 poll: 0 register: build + + - async_status: + jid: "{{ item.ansible_job_id }}" + with_items: "{{ build.results }}" + register: build_result + until: + - build_result.finished is defined + - build_result.finished + retries: 1000 + delay: 5 + when: + - (not reuse_requirements) | bool + - project != 'requirements' + + - name: Build project images + block: + - name: "Build {{ project }} image for {{ item.name }}" + docker_image: + path: "{{ loci_src_dir }}" + name: loci/{{ project }} + tag: "{{ branch }}-{{ item.name }}" + pull: False + buildargs: "{{ item.buildargs.project }}" + with_items: "{{ distros }}" + async: 1000 + poll: 0 + register: build + - async_status: jid: "{{ item.ansible_job_id }}" with_items: "{{ build.results }}" diff --git a/playbooks/vars.yaml b/playbooks/vars.yaml index bf5dc801..735d60b2 100644 --- a/playbooks/vars.yaml +++ b/playbooks/vars.yaml @@ -7,6 +7,8 @@ docker_daemon: insecure-registries: - 172.17.0.1:5000 +reuse_requirements: False + # NOTE(SamYaple): When running in the loci repo, the project is "loci", but # when running as a post job for cinder, the project is "cinder". We statically # declare the path rather than using zuul variables so thats not an issue @@ -26,6 +28,10 @@ distros: PROJECT_REPO: http://172.17.0.1/git/openstack/{{ project }} WHEELS: 172.17.0.1:5000/loci/requirements:{{ branch }}-centos FROM: base:centos + requirements: + PROJECT: requirements + PROJECT_REPO: http://172.17.0.1/git/openstack/requirements + FROM: base:centos - name: ubuntu image: ubuntu:xenial buildargs: @@ -41,6 +47,10 @@ distros: PROJECT_REPO: http://172.17.0.1/git/openstack/{{ project }} WHEELS: 172.17.0.1:5000/loci/requirements:{{ branch }}-ubuntu FROM: base:ubuntu + requirements: + PROJECT: requirements + PROJECT_REPO: http://172.17.0.1/git/openstack/requirements + FROM: base:ubuntu # - name: debian # image: debian:stretch # buildargs: @@ -57,3 +67,7 @@ distros: # PROJECT_REPO: http://172.17.0.1/git/openstack/{{ project }} # WHEELS: 172.17.0.1:5000/loci/requirements:master-debian # FROM: base:debian +# requirements: +# PROJECT: requirements +# PROJECT_REPO: http://172.17.0.1/git/openstack/requirements +# FROM: base:debian