diff --git a/scripts/boot-seed-vm b/scripts/boot-seed-vm index e8a1ad34..a0787886 100755 --- a/scripts/boot-seed-vm +++ b/scripts/boot-seed-vm @@ -206,7 +206,7 @@ else fi echo "Waiting for $IMAGE_NAME VM to boot." -wait_for 100 1 poll_vm +wait_for -w 100 --delay 1 -- poll_vm poll_vm echo @@ -215,7 +215,7 @@ echo "Booted. Found IP: $VM_IP." # hostkeys are generated by cloud-init as part of the boot sequence - can # take a few seconds. echo "Waiting for SSH hostkey." -wait_for 30 1 "ssh-keyscan $VM_IP 2>&1 | grep \"$VM_IP.*OpenSSH\"" +wait_for -w 30 --delay 1 -- "ssh-keyscan $VM_IP 2>&1 | grep \"$VM_IP.*OpenSSH\"" # Remove the hostkey, new instance == new key. ssh-keygen -R $(os-apply-config -m $TE_DATAFILE --key baremetal-network.seed.ip --type netaddress --key-default '192.0.2.1') || true diff --git a/scripts/devtest_overcloud.sh b/scripts/devtest_overcloud.sh index 0a80cd71..2e0ec3f2 100755 --- a/scripts/devtest_overcloud.sh +++ b/scripts/devtest_overcloud.sh @@ -289,7 +289,7 @@ fi ## #. Wait for the BM cloud to register BM nodes with the scheduler:: expected_nodes=$(( $OVERCLOUD_COMPUTESCALE + $OVERCLOUD_CONTROLSCALE + $OVERCLOUD_BLOCKSTORAGESCALE )) -wait_for 60 $expected_nodes wait_for_hypervisor_stats $expected_nodes +wait_for -w $((60 * $expected_nodes)) --delay 10 -- wait_for_hypervisor_stats $expected_nodes ## #. Set password for Overcloud SNMPd, same password needs to be set in Undercloud Ceilometer @@ -495,7 +495,7 @@ fi ## :: echo "Waiting for the overcloud stack to be ready" #nodocs -wait_for_stack_ready $(($OVERCLOUD_STACK_TIMEOUT * 60 / 10)) 10 $STACKNAME +wait_for_stack_ready -w $(($OVERCLOUD_STACK_TIMEOUT * 60)) 10 $STACKNAME OVERCLOUD_ENDPOINT=$(heat output-show $STACKNAME KeystoneURL|sed 's/^"\(.*\)"$/\1/') OVERCLOUD_IP=$(echo $OVERCLOUD_ENDPOINT | awk -F '[/:]' '{print $4}') ### --end @@ -599,12 +599,12 @@ fi #nodocs ## #. _`Wait for Nova Compute` ## :: -wait_for 30 10 nova service-list --binary nova-compute 2\>/dev/null \| grep 'enabled.*\ up\ ' +wait_for -w 300 --delay 10 -- nova service-list --binary nova-compute 2\>/dev/null \| grep 'enabled.*\ up\ ' ## #. Wait for L2 Agent On Nova Compute ## :: -wait_for 30 10 neutron agent-list -f csv -c alive -c agent_type -c host \| grep "\":-).*Open vSwitch agent.*-novacompute\"" #nodocs +wait_for -w 300 --delay 10 -- neutron agent-list -f csv -c alive -c agent_type -c host \| grep "\":-).*Open vSwitch agent.*-novacompute\"" #nodocs ## wait_for 30 10 neutron agent-list -f csv -c alive -c agent_type -c host \| grep "\":-).*Open vSwitch agent.*-novacompute\"" ## #. Log in as a user. @@ -627,7 +627,7 @@ if [ "stack-create" = "$HEAT_OP" ] ; then #nodocs ## #. Add an external IP for it. ## :: - wait_for 10 5 neutron port-list -f csv -c id --quote none \| grep id + wait_for -w 50 --delay 5 -- neutron port-list -f csv -c id --quote none \| grep id PORT=$(neutron port-list -f csv -c id --quote none | tail -n1) FLOATINGIP=$(neutron floatingip-create ext-net \ --port-id "${PORT//[[:space:]]/}" \ @@ -654,7 +654,7 @@ fi ## #. After which, you should be able to ping it ## :: -wait_for 30 10 ping -c 1 $FLOATINGIP +wait_for -w 300 --delay 10 -- ping -c 1 $FLOATINGIP ### --end diff --git a/scripts/devtest_seed.sh b/scripts/devtest_seed.sh index 9c2bb05a..5e508660 100755 --- a/scripts/devtest_seed.sh +++ b/scripts/devtest_seed.sh @@ -289,7 +289,7 @@ echo "Waiting for seed node to configure br-ctlplane..." #nodocs timeout 480 sh -c 'printf "HTTP/1.0 200 OK\r\n\r\n\r\n" | nc -l '"$COMP_IP"' '"$SEED_COMP_PORT"' | grep '"$SEED_IMAGE_ID" # Wait for network -wait_for 10 1 ping -c 1 $BM_NETWORK_SEED_IP +wait_for -w 10 --delay 1 -- ping -c 1 $BM_NETWORK_SEED_IP # If ssh-keyscan fails to connect, it returns 0. So grep to see if it succeeded ssh-keyscan -t rsa $BM_NETWORK_SEED_IP | tee -a ~/.ssh/known_hosts | grep -q "^$BM_NETWORK_SEED_IP ssh-rsa " @@ -302,13 +302,13 @@ keystone role-create --name=swiftoperator keystone role-create --name=ResellerAdmin echo "Waiting for nova to initialise..." -wait_for 50 10 nova list +wait_for -w 500 --delay 10 -- nova list user-config echo "Waiting for Nova Compute to be available" -wait_for 30 10 nova service-list --binary nova-compute 2\>/dev/null \| grep 'enabled.*\ up\ ' +wait_for -w 300 --delay 10 -- nova service-list --binary nova-compute 2\>/dev/null \| grep 'enabled.*\ up\ ' echo "Waiting for neutron API and L2 agent to be available" -wait_for 30 10 neutron agent-list -f csv -c alive -c agent_type -c host \| grep "\":-).*Open vSwitch agent.*\"" #nodocs +wait_for -w 300 --delay 10 -- neutron agent-list -f csv -c alive -c agent_type -c host \| grep "\":-).*Open vSwitch agent.*\"" #nodocs BM_NETWORK_SEED_RANGE_START=$(os-apply-config -m $TE_DATAFILE --key baremetal-network.seed.range-start --type raw --key-default '192.0.2.2') BM_NETWORK_SEED_RANGE_END=$(os-apply-config -m $TE_DATAFILE --key baremetal-network.seed.range-end --type raw --key-default '192.0.2.20') diff --git a/scripts/devtest_undercloud.sh b/scripts/devtest_undercloud.sh index d8f34822..916d6b78 100755 --- a/scripts/devtest_undercloud.sh +++ b/scripts/devtest_undercloud.sh @@ -178,7 +178,7 @@ POWER_USER=$(os-apply-config -m $TE_DATAFILE --key ssh-user --type raw) ## #. Wait for the BM cloud to register BM nodes with the scheduler:: -wait_for 60 1 wait_for_hypervisor_stats +wait_for -w 60 --delay 1 -- wait_for_hypervisor_stats ## #. We need an environment file to store the parameters we're going to give @@ -327,7 +327,7 @@ heat $HEAT_OP -e $HEAT_ENV \ echo "Waiting for the undercloud stack to be ready" #nodocs # Make time out 60 mins as like the Heat stack-create default timeout. -wait_for_stack_ready $(($UNDERCLOUD_STACK_TIMEOUT * 60 / 10)) 10 undercloud +wait_for_stack_ready -w $(($UNDERCLOUD_STACK_TIMEOUT * 60 )) 10 undercloud UNDERCLOUD_CTL_IP=$(nova list | grep ctlplane | sed -e "s/.*=\\([0-9.]*\\).*/\1/") ## #. If we're deploying with a public VLAN we must use it, not the control plane diff --git a/scripts/wait_for b/scripts/wait_for index 671aa502..6ad431f3 100755 --- a/scripts/wait_for +++ b/scripts/wait_for @@ -17,47 +17,113 @@ set -e # exit on the first non-zero status set -u # exit on unset variables +set -o pipefail SCRIPT_NAME=$(basename $0) function show_options() { - echo "Usage: $SCRIPT_NAME LOOPS_NUMBER SLEEP_TIME ARGS" + EXITVAL=${1:-1} + echo "Usage: $SCRIPT_NAME [-h] [-w TIMEOUT] [-l LOOP_COUNT] [-f FAIL_MATCH] [-s SUCCESS_MATCH] --delay SLEEP_TIME -- COMMAND" echo - echo "ARGS are read and concatenated together into a single command." - echo "Execute the command in a loop until it succeeds or the number" - echo "of attempts exceeds LOOPS_NUMBER value. After each failure" - echo "pause for SLEEP_TIME seconds." + echo "Waits for a command to fail, succeed, or timeout." echo - echo "An optional FAIL_MATCH_OUTPUT variable may also be set to control " - echo "if the loop exits early if the commands stdout/stderr matches the " - echo "supplied regex string." + echo "Options:" + echo " -h,--help -- this help" + echo " -w,--walltime TIMEOUT -- Timeout after TIMEOUT seconds." + echo " -l,--looptimeout LOOP_COUNT -- Timeout after checking COMMAND LOOP_COUNT times." + echo " -d,--delay SLEEP_TIME -- Seconds to sleep between checks of COMMAND." + echo " -s,--success-match -- Output that indicates a success." + echo " -f,--fail-match -- Output that indicates a short-circuit failure." + echo + echo "Execute the command in a loop until it succeeds, a timeout is reached, or" + echo "a short-circuit failure occurs. Between each check of the command sleep for" + echo "the number of seconds specified by SLEEP_TIME." echo echo "Examples:" - echo " wait_for 30 10 ping -c 1 192.0.2.2" - echo " wait_for 10 1 ls file_we_are_waiting_for" - echo " wait_for 10 3 date \| grep 8" - echo " FAIL_MATCH_OUTPUT=CREATE_FAILED wait_for 30 10 heat stack-show undercloud" - echo " SUCCESSFUL_MATCH_OUTPUT=CREATE_COMPLETE wait_for 30 10 heat stack-show undercloud" - exit 1 + echo " wait_for -w 300 --delay 10 -- ping -c 1 192.0.2.2" + echo " wait_for -w 10 --delay 1 -- ls file_we_are_waiting_for" + echo " wait_for -w 30 --delay 3 -- date \| grep 8" + echo " wait_for -w 300 --delay 10 --fail-match CREATE_FAILED -- heat stack-show undercloud" + echo " wait_for -w 300 --delay 10 --success-match CREATE_COMPLETE -- heat stack-show undercloud" + exit $EXITVAL } +USE_WALLTIME= +TIMEOUT= +DELAY= -LOOPS=${1:-""} -SLEEPTIME=${2:-""} -FAIL_MATCH_OUTPUT=${FAIL_MATCH_OUTPUT:-""} +if [ -n "${SUCCESSFUL_MATCH_OUTPUT:-}" ]; then + echo "DEPRECATION WARNING: Using env vars for specifying SUCCESSFUL_MATCH_OUTPUT is deprecated." +fi SUCCESSFUL_MATCH_OUTPUT=${SUCCESSFUL_MATCH_OUTPUT:-""} -shift 2 || true +if [ -n "${FAIL_MATCH_OUTPUT:-}" ]; then + echo "DEPRECATION WARNING: Using env vars for specifying FAIL_MATCH_OUTPUT is deprecated." +fi +FAIL_MATCH_OUTPUT=${FAIL_MATCH_OUTPUT:-""} + +USE_ARGPARSE=0 +# We have to support positional arguments for backwards compat +if [ -n "$1" -a "${1:0:1}" == "-" ]; then + USE_ARGPARSE=1 +else + echo "DEPRECATION WARNING: Using positional arguments for wait_for is deprecated." +fi + +if [ $USE_ARGPARSE -eq 1 ]; then + set +e + TEMP=$(getopt -o h,w:,l:,d:,s:,f: -l help,walltime:,looptimeout:,delay:,success-match:,fail-match: -n $SCRIPT_NAME -- "$@") + if [ $? != 0 ] ; then show_options ; fi + set -e + + # Note the quotes around `$TEMP': they are essential! + eval set -- "$TEMP" + + while true ; do + case "$1" in + -h) show_options 0;; + --help) show_options 0;; + -w|--walltime) [ -n "$USE_WALLTIME" ] && show_options + USE_WALLTIME=1 + TIMEOUT="$2" + shift 2 + ;; + -l|--looptimeout) [ -n "$USE_WALLTIME" ] && show_options + USE_WALLTIME=0 + TIMEOUT="$2" + shift 2 + ;; + -d|--delay) DELAY="$2"; shift 2;; + -s|--success-match) SUCCESSFUL_MATCH_OUTPUT="$2"; shift 2;; + -f|--fail-match) FAIL_MATCH_OUTPUT="$2"; shift 2;; + --) shift ; break ;; + esac + done +else + TIMEOUT=${1:-""} + DELAY=${2:-""} + USE_WALLTIME=0 + shift 2 || true +fi + COMMAND="$@" -if [ -z "$LOOPS" -o -z "$SLEEPTIME" -o -z "$COMMAND" ]; then +if [ -z "$TIMEOUT" -o -z "$DELAY" -o -z "$COMMAND" ]; then show_options fi -i=0 -while [ $i -lt $LOOPS ]; do - i=$((i + 1)) +ENDTIME=$(($(date +%s) + $TIMEOUT)) +TIME_REMAINING=0 +function update_time_remaining() { + CUR_TIME="$(date +%s)" + TIME_REMAINING=$(($ENDTIME - $CUR_TIME)) +} + + +OUTPUT= + +function check_cmd() { STATUS=0 OUTPUT=$(eval $COMMAND 2>&1) || STATUS=$? if [[ -n "$SUCCESSFUL_MATCH_OUTPUT" ]] \ @@ -72,10 +138,40 @@ while [ $i -lt $LOOPS ]; do # it's output so we have finished waiting. exit 0 fi +} - sleep $SLEEPTIME +i=0 +while [ $USE_WALLTIME -eq 1 -o $i -lt $TIMEOUT ]; do + if [ $USE_WALLTIME -eq 1 ]; then + update_time_remaining + if [ $TIME_REMAINING -le 0 ]; then + break + fi + else + i=$((i + 1)) + fi + + check_cmd + + if [ $USE_WALLTIME -eq 1 ]; then + update_time_remaining + if [ $TIME_REMAINING -lt $DELAY ]; then + if [ $TIME_REMAINING -gt 0 ]; then + sleep $TIME_REMAINING + check_cmd + fi + else + sleep $DELAY + fi + else + sleep $DELAY + fi done -SECONDS=$((LOOPS * SLEEPTIME)) +if [ $USE_WALLTIME -eq 1 ]; then + SECONDS=$TIMEOUT +else + SECONDS=$((TIMEOUT * DELAY)) +fi printf 'Timing out after %d seconds:\nCOMMAND=%s\nOUTPUT=%s\n' \ "$SECONDS" "$COMMAND" "$OUTPUT" exit 1 diff --git a/scripts/wait_for_stack_ready b/scripts/wait_for_stack_ready index e6a63c93..74050aff 100755 --- a/scripts/wait_for_stack_ready +++ b/scripts/wait_for_stack_ready @@ -19,18 +19,25 @@ set -eu SCRIPT_NAME=$(basename $0) +USE_WALLTIME="-l" +if [ -n "$1" -a "$1" = "-w" ]; then + USE_WALLTIME="-w" + shift 1 +fi + LOOPS=${1:-""} SLEEPTIME=${2:-""} STACK_NAME=${3:-""} if [ -z "$LOOPS" -o -z "$SLEEPTIME" -o -z "$STACK_NAME" ]; then - echo "Usage: $SCRIPT_NAME LOOPS_NUMBER SLEEP_TIME STACK_NAME" + echo "Usage: $SCRIPT_NAME [-w] LOOPS_NUMBER SLEEP_TIME STACK_NAME" exit 1 fi SUCCESSFUL_MATCH_OUTPUT="(CREATE|UPDATE)_COMPLETE" FAIL_MATCH_OUTPUT="(CREATE|UPDATE)_FAILED" -SUCCESSFUL_MATCH_OUTPUT=$SUCCESSFUL_MATCH_OUTPUT \ - FAIL_MATCH_OUTPUT=$FAIL_MATCH_OUTPUT \ - wait_for $1 $2 \ - "heat stack-show $STACK_NAME | awk '/stack_status / { print \$4 }'" + +wait_for $USE_WALLTIME $1 --delay $2 \ + --success-match $SUCCESSFUL_MATCH_OUTPUT \ + --fail-match $FAIL_MATCH_OUTPUT -- \ + "heat stack-show $STACK_NAME | awk '/stack_status / { print \$4 }'"