From 7a4cf0d03eac8c3f83b39ede391cd9fcc32bc3fa Mon Sep 17 00:00:00 2001 From: Thiago Brito Date: Wed, 25 May 2022 11:30:52 -0300 Subject: [PATCH] FluxCD: Add image versions from image record files This commit aims to add support to override image versions with the image record file's provided images on the FluxCD manifest files. Since the apps that goes with build-helm-charts.sh needs to have a specific rpm structure, I used [1] to test. TEST PLAN PASS Built stx-openstack app for FluxCD and verify if the keys were overwritten on the fluxcd-manifests folder REGRESSION PLAN PASS Built stx-openstack app for Armada and verify if the keys were overwritten on the manifest.yaml Logs FluxCD: https://paste.opendev.org/show/bHeUDAxZC18Zq2e04cNp/ Logs Armada: https://paste.opendev.org/show/brj2YFSoeK54aR2bfelR/ NOTE: Since [1] is just a partial migration of FluxCD, only 3 overrides are executed for FluxCD vs. a couple of dozen in Armada. [1] https://review.opendev.org/c/starlingx/openstack-armada-app/+/840432 Story: 2009138 Task: 45472 Signed-off-by: Thiago Brito Change-Id: I27e491523c87f41841951a275c7b79e1c85d3cf8 --- build-tools/build-helm-charts.sh | 109 ++++++++++++++++++++++++------- build-tools/helm_chart_modify.py | 16 ++++- 2 files changed, 97 insertions(+), 28 deletions(-) diff --git a/build-tools/build-helm-charts.sh b/build-tools/build-helm-charts.sh index 270de8c1..53b625dc 100755 --- a/build-tools/build-helm-charts.sh +++ b/build-tools/build-helm-charts.sh @@ -97,31 +97,37 @@ function is_in { return 1 } + +function get_image_record_file { + local image_record=$1 + + if [[ ${image_record} =~ ^https?://.*(.lst|.txt)$ ]]; then + wget --quiet --no-clobber ${image_record} \ + --directory-prefix ${IMAGE_RECORD_PATH} + + if [ $? -ne 0 ]; then + echo "Failed to download image record file from ${image_record}" >&2 + exit 1 + fi + elif [[ -f ${image_record} && ${image_record} =~ .lst|.txt ]]; then + cp ${image_record} ${IMAGE_RECORD_PATH} + if [ $? -ne 0 ]; then + echo "Failed to copy ${image_record} to ${IMAGE_RECORD_PATH}" >&2 + exit 1 + fi + else + echo "Cannot recognize the provided image record file:${image_record}" >&2 + exit 1 + fi +} + # Read the image versions from the passed image # record files and build them into armada manifest -function build_image_versions_to_manifest { +function build_image_versions_to_armada_manifest { local manifest_file=$1 for image_record in ${IMAGE_RECORDS[@]}; do - - if [[ ${image_record} =~ ^https?://.*(.lst|.txt)$ ]]; then - wget --quiet --no-clobber ${image_record} \ - --directory-prefix ${IMAGE_RECORD_PATH} - - if [ $? -ne 0 ]; then - echo "Failed to download image record file from ${image_record}" >&2 - exit 1 - fi - elif [[ -f ${image_record} && ${image_record} =~ .lst|.txt ]]; then - cp ${image_record} ${IMAGE_RECORD_PATH} - if [ $? -ne 0 ]; then - echo "Failed to copy ${image_record} to ${IMAGE_RECORD_PATH}" >&2 - exit 1 - fi - else - echo "Cannot recognize the provided image record file:${image_record}" >&2 - exit 1 - fi + get_image_record_file ${image_record} # An image record file contains a list of images with the following format: # ///.../: @@ -166,6 +172,61 @@ function build_image_versions_to_manifest { done } + +# Read the image versions from the passed image +# record files and build them into fluxcd manifests +function build_image_versions_to_fluxcd_manifests { + local manifest_folder=$1 + + for image_record in ${IMAGE_RECORDS[@]}; do + get_image_record_file ${image_record} + + # An image record file contains a list of images with the following format: + # ///.../: + # + # An example of the content of an image record file: + # e.g. images-centos-dev-latest.lst + # docker.io/starlingx/stx-aodh:master-centos-dev-latest + # docker.io/starlingx/stx-ceilometer:master-centos-dev-latest + # docker.io/starlingx/stx-cinder:master-centos-dev-latest + # ... + # + # An example of the usage of an image reference in manifest file: + # e.g. manifest.yaml + # images: + # tags: + # aodh_api: docker.io/starlingx/stx-aodh:master-centos-stable-latest + # aodh_db_sync: docker.io/starlingx/stx-aodh:master-centos-stable-latest + # ... + # + # To replace the images in the manifest file with the images in image record file: + # For each image reference in the image record file, + # 1. extract image name + # e.g. image_name = stx-aodh + # + # 2. search the image reference in manifest yaml via image_name + # e.g. old_image_reference = docker.io/starlingx/stx-aodh:master-centos-stable-latest + # + # 3. update the manifest file to replace the old image references with the new one + # e.g. manifest.yaml + # images: + # tags: + # aodh_api: docker.io/starlingx/stx-aodh:master-centos-dev-latest + # aodh_db_sync: docker.io/starlingx/stx-aodh:master-centos-dev-latest + # + image_record=${IMAGE_RECORD_PATH}/$(basename ${image_record}) + find ${manifest_folder} -name "*.yaml" | while read manifest_file; do + ${PYTHON2:-python2} $BUILD_HELM_CHARTS_DIR/helm_chart_modify.py ${manifest_file} ${manifest_file}.tmp ${image_record} + if [ $? -ne 0 ]; then + echo "Failed to update manifest file" >&2 + exit 1 + fi + \mv -f ${manifest_file}.tmp ${manifest_file} + done + done +} + + function build_application_tarball { if [ -n "$1" ] ; then @@ -180,7 +241,7 @@ function build_application_tarball_armada { manifest_file=$(basename ${manifest}) manifest_name=${manifest_file%.yaml} deprecated_tarball_name="helm-charts-${manifest_name}" - build_image_versions_to_manifest ${manifest} + build_image_versions_to_armada_manifest ${manifest} cp ${manifest} staging/. if [ $? -ne 0 ]; then @@ -246,10 +307,9 @@ function build_application_tarball_fluxcd { exit 1 fi - # TODO: implement build image version mapping to fluxcd manifests - # build_image_versions_to_manifests - cd staging + build_image_versions_to_fluxcd_manifests ${FLUXCD_MANIFEST_DIR} + # Add metadata file touch metadata.yaml if [ -n "${LABEL}" ]; then @@ -848,7 +908,6 @@ if [ ! -d "usr/lib/fluxcd" ] ; then else echo echo "WARNING: Merging yaml manifests is currently not supported for FluxCD applications" >&2 - echo "WARNING: Adding image versions from image record files is currently not supported for FluxCD applications" >&2 echo echo "Results:" build_application_tarball diff --git a/build-tools/helm_chart_modify.py b/build-tools/helm_chart_modify.py index 9b6b5b57..6385eeae 100755 --- a/build-tools/helm_chart_modify.py +++ b/build-tools/helm_chart_modify.py @@ -164,9 +164,19 @@ def main(argv): Loader=yaml.RoundTripLoader, preserve_quotes=True, version=(1, 1)): - document_name = (document['schema'], - document['metadata']['schema'], - document['metadata']['name']) + if 'schema' in document and 'armada' in document.get('schema'): + # Armada manifest, need to drop them all in the same file so + # storing in the OrderedDict using tuple as key to differentiate + # between entities + document_name = ( + document['schema'], + document['metadata']['schema'], + document['metadata']['name'] + ) + else: + # FluxCD manifest, plain yaml file, should be simply dumped + document_name = "" + modify_yaml(document, '', '', new_image_dict) document_out[document_name] = document