[Testing Infra] Apply here the improvements done to tripleo-validations

This patch brings out the improvements done from tripleo-validations to
validations-common regarding the molecule testing infrastructure with
the UBI8 images.

This patch integrates molecule fix after release of cryptography3.4.
See https://review.opendev.org/c/openstack/tripleo-validations/+/774747

Related-Bug: #1915101
Co-Authored-By: David Vallee Delisle <dvd@redhat.com>
Change-Id: I2e5ebebdd48e4013aa87868a22807e91feb3ffab
Signed-off-by: Gael Chamoulaud (Strider) <gchamoul@redhat.com>

ansible-test-env.rc

@@ -1,6 +1,5 @@
 export VALIDATIONS_COMMON_WORKPATH="$(dirname $(readlink -f ${BASH_SOURCE[0]}))"
 export ANSIBLE_STDOUT_CALLBACK=debug
-export ANSIBLE_CALLBACK_PLUGINS="${VALIDATIONS_COMMON_WORKPATH}/validations_common/callback_plugins"
 export ANSIBLE_LIBRARY="${VALIDATIONS_COMMON_WORKPATH}/validations_common/library"
 
 # TODO(gchamoul): uncomment once we host lookup plugins

bindep.txt

@@ -24,6 +24,10 @@ libxml2-devel     [platform:rpm]
 libxslt-devel     [platform:rpm]
 libxslt1-dev      [platform:dpkg]
 libxslt-dev       [platform:apk]
+
+# RH Mechanisms
+python-rhsm-certificates [platform: redhat]
+
 podman            [platform:rpm]
 python-devel      [platform:rpm !platform:rhel-8 !platform:centos-8]
 python3-devel     [platform:rpm !platform:rhel-7 !platform:centos-7]

molecule-requirements.txt

@@ -1,11 +1,11 @@
 # this is required for the molecule jobs
 ansible>=2.8,!=2.8.9,!=2.9.12,<2.10.0
 ansi2html
+jinja2>=2.8.0,<3
+mock
+molecule>=3.0,<3.1
 pytest
 pytest-cov
 pytest-html
 pytest-xdist
-mock
-molecule>=3.0,<3.1
 selinux  # MIT
-jinja2>=2.8.0,<3

playbooks/molecule/pre.yml

@@ -1,6 +1,10 @@
 ---
 - hosts: all
   pre_tasks:
+    - name: Set project path fact
+      set_fact:
+        validations_common_project_path: "{{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/validations-common'].src_dir }}"
+
     - name: Ensure output dirs
       file:
         path: "{{ ansible_user_dir }}/zuul-output/logs"
@@ -18,18 +22,25 @@
       pip:
         name: "bindep"
         virtualenv: "{{ ansible_user_dir }}/test-python"
+        virtualenv_command: "{{ ensure_pip_virtualenv_command }}"
         virtualenv_site_packages: true
 
     - name: Run bindep
       shell: |-
         . {{ ansible_user_dir }}/test-python/bin/activate
-        {{ ansible_user_dir }}/{{ zuul.project.src_dir }}/scripts/bindep-install
+        {{ validations_common_project_path }}/scripts/bindep-install
       become: true
       changed_when: false
 
+    - name: Ensure a recent version of pip is installed in virtualenv
+      pip:
+        name: "pip>=19.1.1"
+        virtualenv: "{{ ansible_user_dir }}/test-python"
+        virtualenv_command: "{{ ensure_pip_virtualenv_command }}"
+
     - name: Setup test-python
       pip:
-        requirements: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/molecule-requirements.txt"
+        requirements: "{{ validations_common_project_path }}/molecule-requirements.txt"
         virtualenv: "{{ ansible_user_dir }}/test-python"
         virtualenv_command: "{{ ensure_pip_virtualenv_command }}"
         virtualenv_site_packages: true
@@ -76,6 +87,18 @@
               * hard nproc 4096
             dest: /etc/security/limits.d/containers.conf
 
+        - name: Install epel-release
+          package:
+            name: epel-release
+            state: present
+
+        - name: Be sure to enable epel repo
+          become: true
+          command: >-
+            dnf config-manager --set-enabled epel
+          args:
+            warn: false
+
         - name: Copy repos directory to opt workspace
           copy:
             src: /etc/yum.repos.d

playbooks/molecule/run-local.yml

@@ -4,7 +4,8 @@
     - name: set basic zuul fact
       set_fact:
         zuul:
-          project:
+          projects:
+            "opendev.org/openstack/validations-common":
               src_dir: "{{ tripleo_src }}"
         ansible_connection: ssh
 

playbooks/molecule/run.yml

@@ -2,18 +2,38 @@
 - hosts: all
   environment:
     ANSIBLE_LOG_PATH: "{{ ansible_user_dir }}/zuul-output/logs/ansible-execution.log"
+  pre_tasks:
+
+    - name: Set project path fact
+      set_fact:
+        validations_common_project_path: "{{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/validations-common'].src_dir }}"
+
+    - name: Set roles path fact
+      set_fact:
+        validations_common_roles_paths:
+          - "{{ validations_common_project_path }}/validations_common/roles"
+          - "/usr/share/ansible/roles"
+
+    - name: Set library path fact
+      set_fact:
+        validations_common_library_paths:
+          - "{{ validations_common_project_path }}/validations_common/library"
+          - "/usr/share/ansible/library"
+
   tasks:
     - name: Run role test job
       shell: |-
         . {{ ansible_user_dir }}/test-python/bin/activate
-        . {{ ansible_user_dir }}/{{ zuul.project.src_dir }}/ansible-test-env.rc
+        . {{ validations_common_project_path }}/ansible-test-env.rc
-        pytest --color=no \
+        pytest --color=yes \
                --html={{ ansible_user_dir }}/zuul-output/logs/reports.html \
                --self-contained-html \
                --ansible-args='{{ tripleo_job_ansible_args | default("") }}' \
-               {{ ansible_user_dir }}/{{ zuul.project.src_dir }}/tests/test_molecule.py
+               {{ validations_common_project_path }}/tests/test_molecule.py
       args:
         chdir:
-          "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/validations_common/roles/{{ validations_common_role_name }}"
+          "{{ validations_common_project_path }}/validations_common/roles/{{ validations_common_role_name }}"
         executable: /bin/bash
-      changed_when: false
+      environment:
+        ANSIBLE_ROLES_PATH: "{{ validations_common_roles_paths | join(':') }}"
+        ANSIBLE_LIBRARY: "{{ validations_common_library_paths | join(':') }}"

scripts/run-local-test

@@ -15,16 +15,25 @@
 # under the License.
 
 
-## Shell Opts ----------------------------------------------------------------
+## Functions -----------------------------------------------------------------
-
+function usage {
-set -o pipefail
+    echo "Usage: ROLE_NAME=ROLE_NAME ${0##*/} or ${0##*/} ROLE_NAME"
-set -xeuo
+}
 
 ## Vars ----------------------------------------------------------------------
 
 export PROJECT_DIR="$(dirname $(readlink -f ${BASH_SOURCE[0]}))/../"
+if [ "${ROLE_NAME}x" = "x" -a "${1}x" = "x" ]; then
+    usage;
+    exit 2
+fi
 export ROLE_NAME="${ROLE_NAME:-$1}"
-export TRIPLEO_JOB_ANSIBLE_ARGS=${TRIPLEO_JOB_ANSIBLE_ARGS:-"-v"}
+export TRIPLEO_JOB_ANSIBLE_ARGS=${TRIPLEO_JOB_ANSIBLE_ARGS:-""}
+
+## Shell Opts ----------------------------------------------------------------
+
+set -o pipefail
+set -xeuo
 
 ## Main ----------------------------------------------------------------------
 
@@ -47,8 +56,11 @@ sudo touch /etc/ci/mirror_info.sh
 # Create a virtual env
 "${PYTHON_EXEC}" -m virtualenv --system-site-packages "${HOME}/test-python"
 
+# Activate a virtual env
+PS1="[\u@\h \W]\$" source "${HOME}/test-python/bin/activate"
+
 # Run bindep
-"${HOME}/test-python/bin/pip" install pip setuptools bindep --upgrade
+"${HOME}/test-python/bin/pip" install "pip>=19.1.1" setuptools bindep --upgrade
 "${PROJECT_DIR}/scripts/bindep-install"
 
 # Install local requirements
@@ -61,7 +73,6 @@ fi
                               -r "${PROJECT_DIR}/molecule-requirements.txt"
 
 # Run local test
-PS1="[\u@\h \W]\$" source "${HOME}/test-python/bin/activate"
 source "${PROJECT_DIR}/ansible-test-env.rc"
 export ANSIBLE_ROLES_PATH="${ANSIBLE_ROLES_PATH}:${HOME}/zuul-jobs/roles"
 ansible-playbook -i "${PROJECT_DIR}/tests/hosts.ini" \
@@ -71,4 +82,5 @@ ansible-playbook -i "${PROJECT_DIR}/tests/hosts.ini" \
                  -e "ansible_user=${USER}" \
                  -e "ansible_user_dir=${HOME}" \
                  "${PROJECT_DIR}/tests/prepare-test-host.yml" \
-                 "${PROJECT_DIR}/playbooks/molecule/run-local.yml"
+                 "${PROJECT_DIR}/playbooks/molecule/run-local.yml" \
+                 -v

tests/prepare-test-host.yml

@@ -44,11 +44,9 @@
         mode: "0700"
 
     - name: Create ssh key pair
-      user:
+      openssh_keypair:
-        name: "{{ ansible_user }}"
+        path: "{{ ansible_user_dir }}/.ssh/id_rsa"
-        generate_ssh_key: true
+        size: 2048
-        ssh_key_bits: 2048
-        ssh_key_file: "{{ ansible_user_dir }}/.ssh/id_rsa"
 
     - name: Slurp pub key
       slurp:

validations_common/roles/advanced_format_512e_support/molecule/default/molecule.yml

@@ -11,7 +11,7 @@ platforms:
     registry:
       url: registry.access.redhat.com
     dockerfile: Dockerfile
-    pkg_extras: python*setuptools python*-PyYAML
+    pkg_extras: python*setuptools python*-pyyaml
     volumes:
       - /etc/ci/mirror_info.sh:/etc/ci/mirror_info.sh:ro
       - /etc/pki/rpm-gpg:/etc/pki/rpm-gpg
@@ -34,7 +34,7 @@ provisioner:
   log: true
   env:
     ANSIBLE_STDOUT_CALLBACK: yaml
-    ANSIBLE_LIBRARY: "../../../../library"
+    ANSIBLE_LIBRARY: "${ANSIBLE_LIBRARY:-/usr/share/ansible/plugins/modules}"
 
 scenario:
   test_sequence:

validations_common/roles/check_cpu/molecule/default/molecule.yml

@@ -34,6 +34,7 @@ provisioner:
   log: true
   env:
     ANSIBLE_STDOUT_CALLBACK: yaml
+    ANSIBLE_LIBRARY: "${ANSIBLE_LIBRARY:-/usr/share/ansible/plugins/modules}"
 
 scenario:
   test_sequence:

validations_common/roles/check_disk_space/molecule/default/molecule.yml

@@ -34,6 +34,7 @@ provisioner:
   log: true
   env:
     ANSIBLE_STDOUT_CALLBACK: yaml
+    ANSIBLE_LIBRARY: "${ANSIBLE_LIBRARY:-/usr/share/ansible/plugins/modules}"
 
 scenario:
   test_sequence:

validations_common/roles/check_latest_packages_version/molecule/default/molecule.yml

@@ -11,7 +11,7 @@ platforms:
     registry:
       url: registry.access.redhat.com
     dockerfile: Dockerfile
-    pkg_extras: python*setuptools python*-PyYAML
+    pkg_extras: python*setuptools python*-pyyaml
     volumes:
       - /etc/ci/mirror_info.sh:/etc/ci/mirror_info.sh:ro
       - /etc/pki/rpm-gpg:/etc/pki/rpm-gpg
@@ -34,7 +34,7 @@ provisioner:
   log: true
   env:
     ANSIBLE_STDOUT_CALLBACK: yaml
-    ANSIBLE_LIBRARY: "../../../../library"
+    ANSIBLE_LIBRARY: "${ANSIBLE_LIBRARY:-/usr/share/ansible/plugins/modules}"
 
 scenario:
   test_sequence:

validations_common/roles/check_ram/molecule/default/molecule.yml

@@ -34,6 +34,7 @@ provisioner:
   log: true
   env:
     ANSIBLE_STDOUT_CALLBACK: yaml
+    ANSIBLE_LIBRARY: "${ANSIBLE_LIBRARY:-/usr/share/ansible/plugins/modules}"
 
 scenario:
   test_sequence:

validations_common/roles/check_selinux_mode/molecule/default/molecule.yml

@@ -34,7 +34,7 @@ provisioner:
   log: true
   env:
     ANSIBLE_STDOUT_CALLBACK: yaml
-    ANSIBLE_LIBRARY: "../../../../library"
+    ANSIBLE_LIBRARY: "${ANSIBLE_LIBRARY:-/usr/share/ansible/plugins/modules}"
 
 scenario:
   test_sequence:

validations_common/roles/dns/molecule/default/molecule.yml

@@ -34,6 +34,7 @@ provisioner:
   log: true
   env:
     ANSIBLE_STDOUT_CALLBACK: yaml
+    ANSIBLE_LIBRARY: "${ANSIBLE_LIBRARY:-/usr/share/ansible/plugins/modules}"
 
 scenario:
   test_sequence:

validations_common/roles/haproxy/molecule/default/molecule.yml

@@ -34,7 +34,7 @@ provisioner:
   log: true
   env:
     ANSIBLE_STDOUT_CALLBACK: yaml
-    ANSIBLE_LIBRARY: "../../../../library"
+    ANSIBLE_LIBRARY: "${ANSIBLE_LIBRARY:-/usr/share/ansible/plugins/modules}"
 
 scenario:
   test_sequence:

validations_common/roles/ntp/molecule/default/molecule.yml

@@ -34,7 +34,7 @@ provisioner:
   log: true
   env:
     ANSIBLE_STDOUT_CALLBACK: yaml
-    ANSIBLE_LIBRARY: "../../../../library"
+    ANSIBLE_LIBRARY: "${ANSIBLE_LIBRARY:-/usr/share/ansible/plugins/modules}"
 
 scenario:
   test_sequence:

validations_common/roles/service_status/molecule/default/molecule.yml

@@ -12,7 +12,7 @@ platforms:
       url: registry.access.redhat.com
     dockerfile: Dockerfile
     command: /sbin/init
-    pkg_extras: python*-setuptools python*-enum34 python*-netaddr ruby python*-PyYAML
+    pkg_extras: python*-setuptools python*-pyyaml
     volumes:
       - /etc/ci/mirror_info.sh:/etc/ci/mirror_info.sh:ro
       - /etc/pki/rpm-gpg:/etc/pki/rpm-gpg
@@ -35,7 +35,7 @@ provisioner:
   log: true
   env:
     ANSIBLE_STDOUT_CALLBACK: yaml
-    ANSIBLE_LIBRARY: "../../../../library"
+    ANSIBLE_LIBRARY: "${ANSIBLE_LIBRARY:-/usr/share/ansible/plugins/modules}"
 
 scenario:
   test_sequence:

validations_common/roles/service_status/molecule/docker/molecule.yml

@@ -11,7 +11,7 @@ platforms:
     registry:
       url: registry.access.redhat.com
     dockerfile: Dockerfile
-    pkg_extras: python*-setuptools python*-enum34 python*-netaddr ruby python*-PyYAML python*-libselinux
+    pkg_extras: python*-setuptools python*-pyyaml python*-libselinux
     command: /sbin/init
     capabilities:
       - SYS_ADMIN
@@ -36,7 +36,7 @@ provisioner:
   log: true
   env:
     ANSIBLE_STDOUT_CALLBACK: yaml
-    ANSIBLE_LIBRARY: "../../../../library"
+    ANSIBLE_LIBRARY: "${ANSIBLE_LIBRARY:-/usr/share/ansible/plugins/modules}"
 
 scenario:
   test_sequence:

validations_common/roles/service_status/molecule/podman/molecule.yml

@@ -11,7 +11,7 @@ platforms:
     registry:
       url: registry.access.redhat.com
     dockerfile: Dockerfile
-    pkg_extras: python*-setuptools python*-enum34 python*-netaddr ruby python*-PyYAML
+    pkg_extras: python*-setuptools python*-pyyaml
     volumes:
       - /etc/ci/mirror_info.sh:/etc/ci/mirror_info.sh:ro
       - /etc/pki/rpm-gpg:/etc/pki/rpm-gpg
@@ -34,7 +34,7 @@ provisioner:
   log: true
   env:
     ANSIBLE_STDOUT_CALLBACK: yaml
-    ANSIBLE_LIBRARY: "../../../../library"
+    ANSIBLE_LIBRARY: "${ANSIBLE_LIBRARY:-/usr/share/ansible/plugins/modules}"
 
 scenario:
   test_sequence:

validations_common/roles/service_status/molecule/systemd/molecule.yml

@@ -5,13 +5,18 @@ driver:
 log: true
 
 platforms:
-  - name: centos7
+  - name: ubi8
-    hostname: centos7
+    hostname: ubi8
-    image: centos:7
+    image: ubi8/ubi-init
-    pkg_extras: python-setuptools python-enum34 python-netaddr ruby epel-release PyYAML
+    registry:
+      url: registry.access.redhat.com
+    dockerfile: Dockerfile
+    pkg_extras: python*-setuptools python*-pyyaml
+    volumes:
       - /etc/ci/mirror_info.sh:/etc/ci/mirror_info.sh:ro
-    easy_install:
+      - /etc/pki/rpm-gpg:/etc/pki/rpm-gpg
-      - pip
+      - /opt/yum.repos.d:/etc/yum.repos.d:rw
+    privileged: true
     command: /sbin/init
     environment: &env
       http_proxy: "{{ lookup('env', 'http_proxy') }}"
@@ -19,23 +24,18 @@ platforms:
     ulimits: &ulimit
       - host
 
-  - name: centos8
-    hostname: centos8
-    image: centos:8
-    command: /sbin/init
-    pkg_extras: python*-setuptools python*-enum34 python*-netaddr ruby python*-PyYAML
-    volumes:
-      - /etc/ci/mirror_info.sh:/etc/ci/mirror_info.sh:ro
-    environment:
-      <<: *env
-    ulimits: *ulimit
-
 provisioner:
   name: ansible
+  inventory:
+    hosts:
+      all:
+        hosts:
+          ubi8:
+            ansible_python_interpreter: /usr/bin/python3
   log: true
   env:
     ANSIBLE_STDOUT_CALLBACK: yaml
-    ANSIBLE_LIBRARY: "../../../../library"
+    ANSIBLE_LIBRARY: "${ANSIBLE_LIBRARY:-/usr/share/ansible/plugins/modules}"
 
 scenario:
   test_sequence: