openstack/loci
Code Issues Proposed changes

Always build requirements image in the pipeline

Browse Source 
Without this patch, the loci builder automatically consumes a
previously published requirements image when the project is not
"requirements". This also means the first step in checks/gates
for non requirement images would be to pull a requirement.

Currently there is neither a "pipeline" nor consumption of
artifacts between jobs. Which means the "requirements" build
and the other image build have to be considered as independant.

However, by default, the "non-requirements" image built is
consuming previously published "requirements" image, but not
immediately previous built requirements image.

This is a problem, as it leaves us open to a race condition
where a new build of "requirements" and "non-requirements"
can succeed in a commit, but would cause all the next
"non-requirements" builds to fail, as it is possible the
newly published "requirements" to be incompatible with
the next "non-requirements", as they are not tested together.

This should fix the problem by ensuring that "requirements"
are always built and consumed in the "non-requirement"
building process, instead of re-using the previously built
requirements. For this, project needs to be overriden to
build the requirement image, and therefore a new profile
'requirements' was added to the buildargs of the distros.
Requirements repo also needs to be cloned in the gating
environment, and therefore was added to all the required
projects.

This will also allow new branches to be created, as there
would be no need of a previous build and publish
of requirements in the gating.

Change-Id: I093e4dfc0eef03c47d2d029011fdb5429242ae79
This commit is contained in:
Jean-Philippe Evrard 2018-10-11 11:13:51 +02:00
parent 2df96283b9
commit baf6a5b524
11 changed files with 72 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.zuul.d/cinder.yaml
View File

@ -16,6 +16,7 @@
project: cinder
required-projects:
- openstack/loci
- openstack/requirements
- openstack/cinder
- job:

1
.zuul.d/glance.yaml
View File

@ -16,6 +16,7 @@
project: glance
required-projects:
- openstack/loci
- openstack/requirements
- openstack/glance
- job:

1
.zuul.d/heat.yaml
View File

@ -16,6 +16,7 @@
project: heat
required-projects:
- openstack/loci
- openstack/requirements
- openstack/heat
- job:

1
.zuul.d/horizon.yaml
View File

@ -16,6 +16,7 @@
project: horizon
required-projects:
- openstack/loci
- openstack/requirements
- openstack/horizon
- job:

1
.zuul.d/ironic.yaml
View File

@ -16,6 +16,7 @@
project: ironic
required-projects:
- openstack/loci
- openstack/requirements
- openstack/ironic
- job:

1
.zuul.d/keystone.yaml
View File

@ -16,6 +16,7 @@
project: keystone
required-projects:
- openstack/loci
- openstack/requirements
- openstack/keystone
- job:

1
.zuul.d/neutron.yaml
View File

@ -16,6 +16,7 @@
project: neutron
required-projects:
- openstack/loci
- openstack/requirements
- openstack/neutron
- job:

1
.zuul.d/nova.yaml
View File

@ -16,6 +16,7 @@
project: nova
required-projects:
- openstack/loci
- openstack/requirements
- openstack/nova
- job:

1
.zuul.d/octavia.yaml
View File

@ -16,6 +16,7 @@
project: octavia
required-projects:
- openstack/loci
- openstack/requirements
- openstack/octavia
- job:

56
playbooks/loci-builder.yaml
View File

@ -9,7 +9,13 @@
- "pydep.txt"
environment:
LC_ALL: C
# NOTE(evrardjp): While reuse_requirements is very nice and optimises
# checks and gating, there is a race condition here, because
# we are consuming prebuild wheels (see vars.yaml) by default:
# The jobs in zuul can be building a new "requirements" image, working
# And the new "nova" image would still build on previous "requirements"
# image that was last published. This could cause an issue. Instead
# in gating we should build directly what will be consumed.
# NOTE(SamYaple): This process is so we can take advantage of the infra
# DockerHub mirroring as configured through the Docker daemon. We do this
# instead of calling fetch_wheels initially. All-in-all this saves
@ -25,6 +31,7 @@
async: 1000
poll: 0
register: pull
- async_status:
jid: "{{ item.ansible_job_id }}"
with_items: "{{ pull.results }}"
@ -34,11 +41,14 @@
- pull_result.finished
retries: 1000
delay: 5
when: project != 'requirements'
when:
- reuse_requirements | bool
- project != 'requirements'
- name: Build base images
block:
- docker_image:
- name: "Build base image for {{ item.name }} asynchronously"
docker_image:
path: "{{ loci_src_dir }}/dockerfiles/{{ item.name }}"
name: base
tag: "{{ item.name }}"
@ -47,6 +57,7 @@
async: 1000
poll: 0
register: base
- async_status:
jid: "{{ item.ansible_job_id }}"
with_items: "{{ base.results }}"
@ -57,18 +68,49 @@
retries: 1000
delay: 5
- name: Build project images
- name: Build requirements image
block:
- docker_image:
- name: "Build requirements image for {{ item.name }}"
docker_image:
path: "{{ loci_src_dir }}"
name: loci/{{ project }}
name: loci/requirements
tag: "{{ branch }}-{{ item.name }}"
repository: 172.17.0.1:5000/loci/requirements
push: yes
pull: False
buildargs: "{{ item.buildargs.project }}"
buildargs: "{{ item.buildargs.requirements }}"
with_items: "{{ distros }}"
async: 1000
poll: 0
register: build
- async_status:
jid: "{{ item.ansible_job_id }}"
with_items: "{{ build.results }}"
register: build_result
until:
- build_result.finished is defined
- build_result.finished
retries: 1000
delay: 5
when:
- (not reuse_requirements) | bool
- project != 'requirements'
- name: Build project images
block:
- name: "Build {{ project }} image for {{ item.name }}"
docker_image:
path: "{{ loci_src_dir }}"
name: loci/{{ project }}
tag: "{{ branch }}-{{ item.name }}"
pull: False
buildargs: "{{ item.buildargs.project }}"
with_items: "{{ distros }}"
async: 1000
poll: 0
register: build
- async_status:
jid: "{{ item.ansible_job_id }}"
with_items: "{{ build.results }}"

14
playbooks/vars.yaml
View File

@ -7,6 +7,8 @@ docker_daemon:
insecure-registries:
- 172.17.0.1:5000
reuse_requirements: False
# NOTE(SamYaple): When running in the loci repo, the project is "loci", but
# when running as a post job for cinder, the project is "cinder". We statically
# declare the path rather than using zuul variables so thats not an issue
@ -26,6 +28,10 @@ distros:
PROJECT_REPO: http://172.17.0.1/git/openstack/{{ project }}
WHEELS: 172.17.0.1:5000/loci/requirements:{{ branch }}-centos
FROM: base:centos
requirements:
PROJECT: requirements
PROJECT_REPO: http://172.17.0.1/git/openstack/requirements
FROM: base:centos
- name: ubuntu
image: ubuntu:xenial
buildargs:
@ -41,6 +47,10 @@ distros:
PROJECT_REPO: http://172.17.0.1/git/openstack/{{ project }}
WHEELS: 172.17.0.1:5000/loci/requirements:{{ branch }}-ubuntu
FROM: base:ubuntu
requirements:
PROJECT: requirements
PROJECT_REPO: http://172.17.0.1/git/openstack/requirements
FROM: base:ubuntu
# - name: debian
# image: debian:stretch
# buildargs:
@ -57,3 +67,7 @@ distros:
# PROJECT_REPO: http://172.17.0.1/git/openstack/{{ project }}
# WHEELS: 172.17.0.1:5000/loci/requirements:master-debian
# FROM: base:debian
# requirements:
# PROJECT: requirements
# PROJECT_REPO: http://172.17.0.1/git/openstack/requirements
# FROM: base:debian