15e02fec78
When GRENADE_USE_EXTERNAL_DEVSTACK is set to True, the initial steps performed by grenade are skipped, namely: - grabbing and configuring the base and the target devstacks; - running devstacks on the base target. This change is required to allow a native Zuul v3 job to use the existing workflow to setup and run devstack. Change-Id: I232db8de05141849c81851dd29440959cb0d8533
387 lines
13 KiB
Bash
387 lines
13 KiB
Bash
#!/bin/bash
|
|
#
|
|
# Copyright 2015 Hewlett-Packard Development Company, L.P.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
# Bootstrapping Library for Grenade
|
|
# =================================
|
|
#
|
|
# We'd like grenade to largely use DEVSTACK_TARGET functions, but in
|
|
# order to do that we need to get to the point where we have a working
|
|
# DEVSTACK_TARGET. This file includes *just* enough functions to get
|
|
# us a working DEVSTACK_TARGET which then lets us pivot our
|
|
# functionality into there. Most of these will have been manually
|
|
# copied from upstream devstack.
|
|
#
|
|
# The following variables are assumed to be defined by certain functions:
|
|
#
|
|
# - ``GRENADE_DIR``
|
|
# - ``STACK_ROOT``
|
|
# - ``BASE_DEVSTACK_DIR``
|
|
# - ``BASE_DEVSTACK_BRANCH``
|
|
# - ``TARGET_DEVSTACK_DIR``
|
|
# - ``TARGET_DEVSTACK_BRANCH``
|
|
#
|
|
# for git related functions
|
|
#
|
|
# - ``RECLONE``
|
|
# - ``OFFLINE``
|
|
# - ``ERROR_ON_CLONE``
|
|
# - ``GIT_TIMEOUT``
|
|
|
|
|
|
# Echo to summary log file
|
|
#
|
|
# This function should be used to send messages out to the summary log
|
|
# file which makes for good high level progress monitoring.
|
|
function echo_summary {
|
|
local xtrace
|
|
xtrace=$(set +o | grep xtrace)
|
|
set +o xtrace
|
|
|
|
echo $@ >&6
|
|
echo
|
|
echo "#********************************************"
|
|
echo "# " $@
|
|
echo "#********************************************"
|
|
echo
|
|
|
|
$xtrace
|
|
}
|
|
|
|
# Stop at a specific phase
|
|
#
|
|
# Use is DEPRECATED and will be removed once the refactor is done.
|
|
function stop {
|
|
stop=$1
|
|
shift
|
|
if [[ "$@" =~ "$stop" ]]; then
|
|
echo "STOP called for $1"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
# Normalize config values to True or False
|
|
# Accepts as False: 0 no No NO false False FALSE
|
|
# Accepts as True: 1 yes Yes YES true True TRUE
|
|
# VAR=$(trueorfalse default-value test-value)
|
|
function trueorfalse {
|
|
local xtrace=$(set +o | grep xtrace)
|
|
set +o xtrace
|
|
local default=$1
|
|
local literal=$2
|
|
local testval=${!literal:-}
|
|
|
|
[[ -z "$testval" ]] && { echo "$default"; return; }
|
|
[[ "0 no No NO false False FALSE" =~ "$testval" ]] && { echo "False"; return; }
|
|
[[ "1 yes Yes YES true True TRUE" =~ "$testval" ]] && { echo "True"; return; }
|
|
echo "$default"
|
|
$xtrace
|
|
}
|
|
|
|
# git clone only if directory doesn't exist already. Since ``DEST`` might not
|
|
# be owned by the installation user, we create the directory and change the
|
|
# ownership to the proper user.
|
|
# Set global ``RECLONE=yes`` to simulate a clone when dest-dir exists
|
|
# Set global ``ERROR_ON_CLONE=True`` to abort execution with an error if the git repo
|
|
# does not exist (default is False, meaning the repo will be cloned).
|
|
# Uses globals ``ERROR_ON_CLONE``, ``OFFLINE``, ``RECLONE``
|
|
# git_clone remote dest-dir branch
|
|
function git_clone {
|
|
local git_remote=$1
|
|
local git_dest=$2
|
|
local git_ref=$3
|
|
local orig_dir=$(pwd)
|
|
local git_clone_flags=""
|
|
|
|
RECLONE=$(trueorfalse False RECLONE)
|
|
ERROR_ON_CLONE=$(trueorfalse False ERROR_ON_CLONE)
|
|
|
|
if [[ "${GIT_DEPTH}" -gt 0 ]]; then
|
|
git_clone_flags="$git_clone_flags --depth $GIT_DEPTH"
|
|
fi
|
|
|
|
if [[ "$OFFLINE" = "True" ]]; then
|
|
echo "Running in offline mode, clones already exist"
|
|
# print out the results so we know what change was used in the logs
|
|
cd $git_dest
|
|
git show --oneline | head -1
|
|
cd $orig_dir
|
|
return
|
|
fi
|
|
|
|
if echo $git_ref | egrep -q "^refs"; then
|
|
# If our branch name is a gerrit style refs/changes/...
|
|
if [[ ! -d $git_dest ]]; then
|
|
[[ "$ERROR_ON_CLONE" = "True" ]] && \
|
|
die $LINENO "Cloning not allowed in this configuration"
|
|
git_timed clone $git_clone_flags $git_remote $git_dest
|
|
fi
|
|
cd $git_dest
|
|
git_timed fetch $git_remote $git_ref && git checkout FETCH_HEAD
|
|
else
|
|
# do a full clone only if the directory doesn't exist
|
|
if [[ ! -d $git_dest ]]; then
|
|
[[ "$ERROR_ON_CLONE" = "True" ]] && \
|
|
die $LINENO "Cloning not allowed in this configuration"
|
|
git_timed clone $git_clone_flags $git_remote $git_dest
|
|
cd $git_dest
|
|
# This checkout syntax works for both branches and tags
|
|
git checkout $git_ref
|
|
elif [[ "$RECLONE" = "True" ]]; then
|
|
# if it does exist then simulate what clone does if asked to RECLONE
|
|
cd $git_dest
|
|
# set the url to pull from and fetch
|
|
git remote set-url origin $git_remote
|
|
git_timed fetch origin
|
|
# remove the existing ignored files (like pyc) as they cause breakage
|
|
# (due to the py files having older timestamps than our pyc, so python
|
|
# thinks the pyc files are correct using them)
|
|
find $git_dest -name '*.pyc' -delete
|
|
|
|
# handle git_ref accordingly to type (tag, branch)
|
|
if [[ -n "`git show-ref refs/tags/$git_ref`" ]]; then
|
|
git_update_tag $git_ref
|
|
elif [[ -n "`git show-ref refs/heads/$git_ref`" ]]; then
|
|
git_update_branch $git_ref
|
|
elif [[ -n "`git show-ref refs/remotes/origin/$git_ref`" ]]; then
|
|
git_update_remote_branch $git_ref
|
|
else
|
|
die $LINENO "$git_ref is neither branch nor tag"
|
|
fi
|
|
|
|
fi
|
|
fi
|
|
|
|
# print out the results so we know what change was used in the logs
|
|
cd $git_dest
|
|
git show --oneline | head -1
|
|
cd $orig_dir
|
|
}
|
|
|
|
# git can sometimes get itself infinitely stuck with transient network
|
|
# errors or other issues with the remote end. This wraps git in a
|
|
# timeout/retry loop and is intended to watch over non-local git
|
|
# processes that might hang. GIT_TIMEOUT, if set, is passed directly
|
|
# to timeout(1); otherwise the default value of 0 maintains the status
|
|
# quo of waiting forever.
|
|
# usage: git_timed <git-command>
|
|
function git_timed {
|
|
local count=0
|
|
local timeout=0
|
|
|
|
if [[ -n "${GIT_TIMEOUT}" ]]; then
|
|
timeout=${GIT_TIMEOUT}
|
|
fi
|
|
|
|
until timeout -s SIGINT ${timeout} git "$@"; do
|
|
# 124 is timeout(1)'s special return code when it reached the
|
|
# timeout; otherwise assume fatal failure
|
|
if [[ $? -ne 124 ]]; then
|
|
die $LINENO "git call failed: [git $@]"
|
|
fi
|
|
|
|
count=$(($count + 1))
|
|
warn "timeout ${count} for git call: [git $@]"
|
|
if [ $count -eq 3 ]; then
|
|
die $LINENO "Maximum of 3 git retries reached"
|
|
fi
|
|
sleep 5
|
|
done
|
|
}
|
|
|
|
# git update using reference as a branch.
|
|
# git_update_branch ref
|
|
function git_update_branch {
|
|
local git_branch=$1
|
|
|
|
git checkout -f origin/$git_branch
|
|
# a local branch might not exist
|
|
git branch -D $git_branch || true
|
|
git checkout -b $git_branch
|
|
}
|
|
|
|
# git update using reference as a branch.
|
|
# git_update_remote_branch ref
|
|
function git_update_remote_branch {
|
|
local git_branch=$1
|
|
|
|
git checkout -b $git_branch -t origin/$git_branch
|
|
}
|
|
|
|
# git update using reference as a tag. Be careful editing source at that repo
|
|
# as working copy will be in a detached mode
|
|
# git_update_tag ref
|
|
function git_update_tag {
|
|
local git_tag=$1
|
|
|
|
git tag -d $git_tag
|
|
# fetching given tag only
|
|
git_timed fetch origin tag $git_tag
|
|
git checkout -f $git_tag
|
|
}
|
|
|
|
function localrc_path {
|
|
local side=$1
|
|
local path
|
|
if [[ $side == "base" ]]; then
|
|
path=$BASE_DEVSTACK_DIR
|
|
elif [[ $side == "target" ]]; then
|
|
path=$TARGET_DEVSTACK_DIR
|
|
else
|
|
die $LINENO "side must be base or target!"
|
|
fi
|
|
if [[ -e "$path/local.conf" ]]; then
|
|
echo "$path/local.conf"
|
|
else
|
|
echo "$path/localrc"
|
|
fi
|
|
}
|
|
|
|
function dump_local_files {
|
|
local dir=$1
|
|
for i in $dir/local*; do
|
|
echo "_______________________________________________"
|
|
echo "-- $i -- file dump"
|
|
echo "_______________________________________________"
|
|
cat $i
|
|
echo "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"
|
|
done
|
|
}
|
|
|
|
function fetch_devstacks {
|
|
devstacks_clone
|
|
devstacks_setup_environment
|
|
devstacks_setup_settings
|
|
}
|
|
|
|
function devstacks_clone {
|
|
# Fetch Base Devstack
|
|
|
|
# Get DevStack if it doesn't exist
|
|
if [[ ! -d $BASE_DEVSTACK_DIR ]]; then
|
|
git_clone $BASE_DEVSTACK_REPO $BASE_DEVSTACK_DIR $BASE_DEVSTACK_BRANCH
|
|
fi
|
|
# Fetch Target Devstack
|
|
if [[ ! -d $TARGET_DEVSTACK_DIR ]]; then
|
|
git_clone $TARGET_DEVSTACK_REPO $TARGET_DEVSTACK_DIR $TARGET_DEVSTACK_BRANCH
|
|
fi
|
|
}
|
|
|
|
function devstacks_setup_environment {
|
|
# This depends on REQUIREMENTS_DIR being set in grenaderc, which
|
|
# it needs to be to have gotten this far.
|
|
source $TARGET_DEVSTACK_DIR/functions-common
|
|
source $TARGET_DEVSTACK_DIR/inc/python
|
|
|
|
# When Python 3 is supported by an application, adding the specific
|
|
# version of Python 3 to this variable will install the app using that
|
|
# version of the interpreter instead of 2.7.
|
|
_DEFAULT_PYTHON3_VERSION="$(_get_python_version python3)"
|
|
export PYTHON3_VERSION=${PYTHON3_VERSION:-${_DEFAULT_PYTHON3_VERSION:-3.5}}
|
|
|
|
# Load up a copy of the downloaded images if not present
|
|
if [[ -d ${STACK_ROOT}/images ]]; then
|
|
rsync -a ${STACK_ROOT}/images/* $BASE_DEVSTACK_DIR/files
|
|
fi
|
|
}
|
|
|
|
function devstacks_setup_settings {
|
|
# dsconf is required only by this function and devstack_localrc
|
|
# but only when the base is installed by grenade.
|
|
# Moreover, when the deployment of base is not performed by grenade,
|
|
# installing devstack-tools just before creating the resources also
|
|
# updates pbr (at least from rocky->master) leading to an exception
|
|
# in running services using pbr methods (for example nova-conductor)
|
|
# probably because they are still looking for the old version.
|
|
install_devstack_tools
|
|
|
|
# Set up base localrc
|
|
|
|
# if there is a localrc, make that a local.conf.orig and get rid of the artifact
|
|
if [[ -r $BASE_DEVSTACK_DIR/localrc ]]; then
|
|
# Convert to a local.conf
|
|
echo "[[local|localrc]]" > $BASE_DEVSTACK_DIR/local.conf.orig
|
|
cat $BASE_DEVSTACK_DIR/localrc >> $BASE_DEVSTACK_DIR/local.conf.orig
|
|
rm $BASE_DEVSTACK_DIR/localrc
|
|
fi
|
|
|
|
# if a local.conf exists, merge that into local.conf.orig (which we're going to remerge later)
|
|
if [[ -r $BASE_DEVSTACK_DIR/local.conf ]]; then
|
|
dsconf merge_lc $BASE_DEVSTACK_DIR/local.conf.orig $BASE_DEVSTACK_DIR/local.conf
|
|
rm $BASE_DEVSTACK_DIR/local.conf
|
|
fi
|
|
|
|
# put devstack.local.conf.target in place as local.conf
|
|
sed -e "
|
|
s|\@BASE_RELEASE_DIR\@|$BASE_RELEASE_DIR|
|
|
s|\@DATA_DIR@|$DATA_DIR|
|
|
" $GRENADE_DIR/devstack.local.conf.base >$BASE_DEVSTACK_DIR/local.conf
|
|
|
|
# if local.conf.orig or localrc.orig exists, append it to local.conf
|
|
if [[ -r $BASE_DEVSTACK_DIR/local.conf.orig ]]; then
|
|
dsconf merge_lc $BASE_DEVSTACK_DIR/local.conf $BASE_DEVSTACK_DIR/local.conf.orig
|
|
fi
|
|
|
|
# if devstack.local.conf exists append it to locarc
|
|
if [[ -r $GRENADE_DIR/devstack.local.conf ]]; then
|
|
dsconf merge_lc $BASE_DEVSTACK_DIR/local.conf $GRENADE_DIR/devstack.local.conf
|
|
elif [[ -r $GRENADE_DIR/devstack.localrc ]]; then
|
|
echo "[[local|localrc]]" > $GRENADE_DIR/devstack.local.conf
|
|
cat $GRENADE_DIR/devstack.localrc >> $GRENADE_DIR/devstack.local.conf
|
|
dsconf merge_lc $BASE_DEVSTACK_DIR/local.conf $GRENADE_DIR/devstack.local.conf
|
|
fi
|
|
|
|
dump_local_files $BASE_DEVSTACK_DIR
|
|
|
|
# ... time for TARGET
|
|
|
|
# Set up target local.conf
|
|
|
|
# if there is a localrc, make that a local.conf.orig and get rid of the artifact
|
|
if [[ -r $TARGET_DEVSTACK_DIR/localrc ]]; then
|
|
# Convert to a local.conf
|
|
echo "[[local|localrc]]" > $TARGET_DEVSTACK_DIR/local.conf.orig
|
|
cat $TARGET_DEVSTACK_DIR/localrc >> $TARGET_DEVSTACK_DIR/local.conf.orig
|
|
rm $TARGET_DEVSTACK_DIR/localrc
|
|
fi
|
|
|
|
# if a local.conf exists, merge that into local.conf.orig (which we're going to remerge later)
|
|
if [[ -r $TARGET_DEVSTACK_DIR/local.conf ]]; then
|
|
dsconf merge_lc $TARGET_DEVSTACK_DIR/local.conf.orig $TARGET_DEVSTACK_DIR/local.conf
|
|
rm $TARGET_DEVSTACK_DIR/local.conf
|
|
fi
|
|
|
|
# put devstack.local.conf.target in place as local.conf
|
|
sed -e "
|
|
s|\@TARGET_RELEASE_DIR\@|$TARGET_RELEASE_DIR|
|
|
s|\@DATA_DIR@|$DATA_DIR|
|
|
" $GRENADE_DIR/devstack.local.conf.target >$TARGET_DEVSTACK_DIR/local.conf
|
|
|
|
# if local.conf.orig or localrc.orig exists, append it to local.conf
|
|
if [[ -r $TARGET_DEVSTACK_DIR/local.conf.orig ]]; then
|
|
dsconf merge_lc $TARGET_DEVSTACK_DIR/local.conf $TARGET_DEVSTACK_DIR/local.conf.orig
|
|
fi
|
|
|
|
# if devstack.local.conf exists append it to locarc
|
|
if [[ -r $GRENADE_DIR/devstack.local.conf ]]; then
|
|
dsconf merge_lc $TARGET_DEVSTACK_DIR/local.conf $GRENADE_DIR/devstack.local.conf
|
|
elif [[ -r $GRENADE_DIR/devstack.localrc ]]; then
|
|
echo "[[local|localrc]]" > $GRENADE_DIR/devstack.local.conf
|
|
cat $GRENADE_DIR/devstack.localrc >> $GRENADE_DIR/devstack.local.conf
|
|
dsconf merge_lc $TARGET_DEVSTACK_DIR/local.conf $GRENADE_DIR/devstack.local.conf
|
|
fi
|
|
dump_local_files $TARGET_DEVSTACK_DIR
|
|
}
|