Reapply "Bootstrap-bridge as top-level job"

This reverts commit 03816fa43363d9162749bf3cf418f788acfee7cc.

This is a partial reapplication of the previously broken change. We make
a small edit to the ansible playbook to run zuul_return in a valid
context. Specifically as a task against localhost.

We also move the infra-prod-bootstrap-bridge dependency into the PPC
because the PPC dependencies override job dpendencies.

Change-Id: Icc2e0871abfed28937eb96bc14bb2be6b0d882d8

playbooks/zuul/run-production-bootstrap-bridge.yaml

@@ -3,3 +3,11 @@
     - add-bastion-host
 
 - import_playbook: ../bootstrap-bridge.yaml
+
+- hosts: localhost
+  tasks:
+    - name: Wait for child jobs
+      zuul_return:
+        data:
+          zuul:
+            pause: true

zuul.d/infra-prod.yaml

@@ -2,14 +2,59 @@
 # in projects.yaml because it's easier to keep an overall view of
 # what's happening in there.
 
-# Make sure only one run of a system-config playbook happens at a time
+# Make sure only one run happens at a time.  The deploy pipeline
+# should keep things in order, but this is to stop perodic jobs
+# jumping in.
 - semaphore:
-    name: infra-prod-playbook
+    name: infra-prod-deployment
+    max: 1
+
+# This semaphore limits the total number of production playbook
+# jobs that can run on bridge at one time. We want things to run in
+# parallel but we have a lot of jobs (particularly in the periodic
+# pipeline) that we don't want to run all at once.
+- semaphore:
+    name: infra-prod-playbook-limit
+    # TODO(clarkb) this semaphore allows us to stage the rollout of
+    # parallel infra-prod job exceution in two steps. First we reorganize
+    # everything but roughly keep the same behaviors as before (max: 1).
+    # When we are happy with that we can bump this to 2 or higher and see
+    # things run in parallel.
     max: 1
 
+- job:
+    name: infra-prod-bootstrap-bridge
+    parent: opendev-infra-prod-setup-src
+    semaphores: infra-prod-deployment
+    description: |
+        Configure the bastion host (bridge)
+
+        This job does minimal configuration on the bastion host
+        (bridge.openstack.org) to allow it to run system-config
+        playbooks against our production hosts.  It sets up Ansible
+        and root keys on the host.  It also synchronizes the
+        system-config repo from the executor to the bastion.
+
+        Note that this is separate to infra-prod-service-bridge;
+        bridge in it's role as the bastion host actaully runs that
+        against itself; it includes things not strictly needed to make
+        the host able to deploy system-config.
+
+        This job is the parent of all deployment jobs, and will pause
+        until they finish.  This prevents conflicts between deployment
+        jobs from changes and periodic runs (which use HEAD of
+        master).
+    run: playbooks/zuul/run-production-bootstrap-bridge.yaml
+    # Do not set file matchers on this job. We must always run this job
+    # before any other infra-prod jobs to ensure system-config is up to
+    # date on bridge before we run our playbooks.
+    nodeset:
+      nodes: []
+
 - job:
     name: infra-prod-playbook
-    parent: opendev-infra-prod-base
+    parent: opendev-infra-prod-setup-keys
+    semaphores: infra-prod-playbook-limit
     description: |
       Run specified playbook against productions hosts.
 
@@ -19,7 +64,6 @@
       /home/zuul/src/opendev.org/opendev/system-config/playbooks
       on the bastion host.
     abstract: true
-    semaphores: infra-prod-playbook
     run: playbooks/zuul/run-production-playbook.yaml
     post-run: playbooks/zuul/run-production-playbook-post.yaml
     required-projects:
@@ -31,41 +75,6 @@
     nodeset:
       nodes: []
 
-- job:
-    name: infra-prod-bootstrap-bridge
-    parent: opendev-infra-prod-setup-src
-    description: |
-        Configure the bastion host (bridge)
-
-        This job does minimal configuration on the bastion host
-        (bridge.openstack.org) to allow it to run system-config
-        playbooks against our production hosts.  It sets up Ansible
-        and root keys on the host. It also synchronizes the system-config
-        repo from the executor to the bastion. This is necessary to
-        emit an up to date known_hosts file when adding new hosts to
-        the inventory.
-
-        Note that this is separate to infra-prod-service-bridge;
-        bridge in it's role as the bastion host actaully runs that
-        against itself; it includes things not strictly needed to make
-        the host able to deploy system-config.
-    # While we don't run the infra-prod-playbook in this job we do run
-    # system-config git repo updates. Until we're ready to stop running
-    # system-config updates in every job we use this semaphore to ensure
-    # exclusivity.
-    semaphores: infra-prod-playbook
-    run: playbooks/zuul/run-production-bootstrap-bridge.yaml
-    files:
-      - playbooks/bootstrap-bridge.yaml
-      - playbooks/zuul/run-production-bootstrap-bridge.yaml
-      - playbooks/zuul/run-production-bootstrap-bridge-add-rootkey.yaml
-      - playbooks/roles/install-ansible/
-      - playbooks/roles/root-keys/
-      - inventory/base/hosts.yaml
-      - inventory/service/group_vars/bastion.yaml
-    nodeset:
-      nodes: []
-
 - job:
     name: infra-prod-base
     parent: infra-prod-playbook

zuul.d/project.yaml

@@ -348,14 +348,19 @@
         - infra-prod-bootstrap-bridge
 
         # From now on, all jobs should depend on base
+        # infra-prod-bootstrap-bridge is a hard dependency of all jobs
+        # because we require the bootstrap job to have run before we
+        # start any playbook jobs, otherwise our buildset would not
+        # hold the bridge semaphore and we may not have the correct
+        # system-config state on bridge.
         - infra-prod-base: &infra-prod-base
             dependencies:
               - name: infra-prod-bootstrap-bridge
-                soft: true
 
         # Legacy puppet hosts
         - infra-prod-remote-puppet-else: &infra-prod-remote-puppet-else
             dependencies:
+              - name: infra-prod-bootstrap-bridge
               - name: infra-prod-base
                 soft: true
 
@@ -365,21 +370,25 @@
 
         - infra-prod-service-bridge: &infra-prod-service-bridge
             dependencies:
+              - name: infra-prod-bootstrap-bridge
               - name: infra-prod-base
                 soft: true
         - infra-prod-run-cloud-launcher: &infra-prod-run-cloud-launcher
             dependencies:
               # depends on the cloud config written out by
               # service-bridge
+              - name: infra-prod-bootstrap-bridge
               - name: infra-prod-service-bridge
                 soft: true
 
         - infra-prod-service-kerberos: &infra-prod-service-kerberos
             dependencies:
+              - name: infra-prod-bootstrap-bridge
               - name: infra-prod-base
                 soft: true
         - infra-prod-service-afs: &infra-prod-service-afs
             dependencies:
+              - name: infra-prod-bootstrap-bridge
               - name: infra-prod-base
                 soft: true
               # NOTE(ianw) in theory we'd want auth changes before
@@ -391,11 +400,13 @@
 
         - infra-prod-service-nameserver: &infra-prod-service-nameserver
             dependencies:
+              - name: infra-prod-bootstrap-bridge
               - name: infra-prod-base
                 soft: true
 
         - infra-prod-service-mirror-update: &infra-prod-service-mirror-update
             dependencies:
+              - name: infra-prod-bootstrap-bridge
               - name: infra-prod-base
                 soft: true
 
@@ -404,6 +415,7 @@
         #
         - infra-prod-service-gitea-lb: &infra-prod-service-gitea-lb
             dependencies:
+              - name: infra-prod-bootstrap-bridge
               - name: infra-prod-base
                 soft: true
               - name: system-config-promote-image-haproxy-statsd
@@ -411,10 +423,12 @@
 
         - infra-prod-service-zuul-db: &infra-prod-service-zuul-db
             dependencies:
+              - name: infra-prod-bootstrap-bridge
               - name: infra-prod-base
                 soft: true
         - infra-prod-service-zuul-lb: &infra-prod-service-zuul-lb
             dependencies:
+              - name: infra-prod-bootstrap-bridge
               - name: infra-prod-base
                 soft: true
               - name: system-config-promote-image-haproxy-statsd
@@ -429,6 +443,7 @@
         # role to work.
         - infra-prod-service-borg-backup: &infra-prod-service-borg-backup
             dependencies:
+              - name: infra-prod-bootstrap-bridge
               - name: infra-prod-base
                 soft: true
 
@@ -438,6 +453,7 @@
         # this job.
         - infra-prod-letsencrypt: &infra-prod-letsencrypt
             dependencies:
+              - name: infra-prod-bootstrap-bridge
               - name: infra-prod-base
                 soft: true
               - name: infra-prod-service-nameserver
@@ -446,12 +462,14 @@
         # letsencrypt depdencies.  keep in alphabetical order
         - infra-prod-service-codesearch: &infra-prod-service-codesearch
             dependencies:
+              - name: infra-prod-bootstrap-bridge
               - name: infra-prod-letsencrypt
                 soft: true
               - name: system-config-promote-image-hound
                 soft: true
         - infra-prod-service-eavesdrop: &infra-prod-service-eavesdrop
             dependencies:
+              - name: infra-prod-bootstrap-bridge
               - name: infra-prod-service-borg-backup
                 soft: true
               - name: infra-prod-letsencrypt
@@ -462,6 +480,7 @@
                 soft: true
         - infra-prod-service-etherpad: &infra-prod-service-etherpad
             dependencies:
+              - name: infra-prod-bootstrap-bridge
               - name: infra-prod-service-borg-backup
                 soft: true
               - name: infra-prod-letsencrypt
@@ -470,6 +489,7 @@
                 soft: true
         - infra-prod-service-gitea: &infra-prod-service-gitea
             dependencies:
+              - name: infra-prod-bootstrap-bridge
               - name: infra-prod-service-borg-backup
                 soft: true
               - name: infra-prod-letsencrypt
@@ -478,22 +498,27 @@
                 soft: true
         - infra-prod-service-grafana: &infra-prod-service-grafana
             dependencies:
+              - name: infra-prod-bootstrap-bridge
               - name: infra-prod-letsencrypt
                 soft: true
         - infra-prod-service-graphite: &infra-prod-service-graphite
             dependencies:
+              - name: infra-prod-bootstrap-bridge
               - name: infra-prod-letsencrypt
                 soft: true
         - infra-prod-service-keycloak: &infra-prod-service-keycloak
             dependencies:
+              - name: infra-prod-bootstrap-bridge
               - name: infra-prod-letsencrypt
                 soft: true
         - infra-prod-service-meetpad: &infra-prod-service-meetpad
             dependencies:
+              - name: infra-prod-bootstrap-bridge
               - name: infra-prod-letsencrypt
                 soft: true
         - infra-prod-service-lists3: &infra-prod-service-lists3
             dependencies:
+              - name: infra-prod-bootstrap-bridge
               - name: infra-prod-service-borg-backup
                 soft: true
               - name: infra-prod-letsencrypt
@@ -502,28 +527,34 @@
                 soft: true
         - infra-prod-service-mirror: &infra-prod-service-mirror
             dependencies:
+              - name: infra-prod-bootstrap-bridge
               - name: infra-prod-letsencrypt
                 soft: true
         - infra-prod-service-nodepool: &infra-prod-service-nodepool
             dependencies:
+              - name: infra-prod-bootstrap-bridge
               - name: infra-prod-letsencrypt
                 soft: true
         - infra-prod-service-static: &infra-prod-service-static
             dependencies:
+              - name: infra-prod-bootstrap-bridge
               - name: infra-prod-letsencrypt
                 soft: true
         - infra-prod-service-paste: &infra-prod-service-paste
             dependencies:
+              - name: infra-prod-bootstrap-bridge
               - name: infra-prod-service-borg-backup
                 soft: true
               - name: infra-prod-letsencrypt
                 soft: true
         - infra-prod-service-registry: &infra-prod-service-registry
             dependencies:
+              - name: infra-prod-bootstrap-bridge
               - name: infra-prod-letsencrypt
                 soft: true
         - infra-prod-service-refstack: &infra-prod-service-refstack
             dependencies:
+              - name: infra-prod-bootstrap-bridge
               - name: infra-prod-service-borg-backup
                 soft: true
               - name: infra-prod-letsencrypt
@@ -532,6 +563,7 @@
                 soft: true
         - infra-prod-service-review: &infra-prod-service-review
             dependencies:
+              - name: infra-prod-bootstrap-bridge
               - name: infra-prod-service-borg-backup
                 soft: true
               - name: infra-prod-letsencrypt
@@ -540,16 +572,19 @@
                 soft: true
         - infra-prod-service-tracing: &infra-prod-service-tracing
             dependencies:
+              - name: infra-prod-bootstrap-bridge
               - name: infra-prod-letsencrypt
                 soft: true
         - infra-prod-service-zookeeper: &infra-prod-service-zookeeper
             dependencies:
+              - name: infra-prod-bootstrap-bridge
               - name: infra-prod-letsencrypt
                 soft: true
               - name: system-config-promote-image-zookeeper-statsd
                 soft: true
         - infra-prod-service-zuul: &infra-prod-service-zuul
             dependencies:
+              - name: infra-prod-bootstrap-bridge
               - name: infra-prod-service-borg-backup
                 soft: true
               - name: infra-prod-letsencrypt
@@ -559,6 +594,7 @@
                 soft: true
         - infra-prod-service-zuul-preview: &infra-prod-service-zuul-preview
             dependencies:
+              - name: infra-prod-bootstrap-bridge
               - name: infra-prod-letsencrypt
                 soft: true
 
@@ -569,6 +605,7 @@
         # accessbot should run on a setup eavesdrop host
         - infra-prod-run-accessbot: &infra-prod-run-accessbot
             dependencies:
+              - name: infra-prod-bootstrap-bridge
               - name: infra-prod-base
                 soft: true
               - name: infra-prod-service-eavesdrop
@@ -580,6 +617,7 @@
         # a setup review host.  also sets up gitea
         - infra-prod-manage-projects: &infra-prod-manage-projects
             dependencies:
+              - name: infra-prod-bootstrap-bridge
               - name: infra-prod-base
                 soft: true
               - name: infra-prod-service-review