grenade.sh |
|
|---|---|
|
grenade.sh is an OpenStack upgrade test harness to exercise the OpenStack upgrade process. It uses DevStack to perform the initial OpenStack install. # DIVIDER #!/usr/bin/env bash # DIVIDER # DIVIDER # DIVIDER export GRENADE_DIR=$(cd $(dirname "$0") && pwd) # DIVIDER export DSTOOLS_VERSION=${DSTOOLS_VERSION:-0.4.0} source $GRENADE_DIR/grenaderc source $GRENADE_DIR/inc/bootstrap while getopts bqs:t c; do case $c in b) RUN_TARGET=False ;; q) VERBOSE=False ;; s) STOP=$2 ;; t) RUN_BASE=False ;; esac done shift `expr $OPTIND - 1` # DIVIDER sudo mkdir -p $BASE_RELEASE_DIR $TARGET_RELEASE_DIR sudo chown -R `whoami` $STACK_ROOT # DIVIDER # DIVIDER if [[ -n "$LOGFILE" ]]; then LOGDAYS=${LOGDAYS:-7} TIMESTAMP_FORMAT=${TIMESTAMP_FORMAT:-"%F-%H%M%S"} CURRENT_LOG_TIME=$(date "+$TIMESTAMP_FORMAT") # DIVIDER LOGDIR=$(dirname "$LOGFILE") LOGNAME=$(basename "$LOGFILE") echo "Creating $LOGDIR...." sudo mkdir -p $LOGDIR sudo chown -R `whoami` $LOGDIR find $LOGDIR -maxdepth 1 -name $LOGNAME.\* -mtime +$LOGDAYS -exec rm {} \; LOGFILE=$LOGFILE.${CURRENT_LOG_TIME} SUMFILE=$LOGFILE.${CURRENT_LOG_TIME}.summary # DIVIDER exec 3>&1 if [[ "$VERBOSE" == "True" ]]; then echo "Running in verbose mode:" echo " Full logs found at => ${LOGFILE}" echo " Summary logs at => ${SUMFILE}" # DIVIDER exec 1> >( ./tools/outfilter.py -v -o "${LOGFILE}" ) 2>&1 # DIVIDER exec 6> >( ./tools/outfilter.py -o "${SUMFILE}" ) else echo "Running in summary mode:" echo " Full logs found at => ${LOGFILE}" echo " Summary logs at => ${SUMFILE}" # DIVIDER exec 1> >( ./tools/outfilter.py -o "${LOGFILE}") 2>&1 # DIVIDER exec 6> >( ./tools/outfilter.py -v -o "${SUMFILE}" >&3) fi echo_summary "grenade.sh log $LOGFILE" # DIVIDER ln -sf $LOGFILE $LOGDIR/$LOGNAME ln -sf $SUMFILE $LOGDIR/$LOGNAME.summary else # DIVIDER exec 3>&1 if [[ "$VERBOSE" != "yes" ]]; then # DIVIDER exec 1>/dev/null 2>&1 fi # DIVIDER exec 6> >( ./tools/outfilter.py -v -o "${SUMFILE}" >&3) fi # DIVIDER if [[ -n "$SCREEN_LOGDIR" ]]; then # DIVIDER if [[ -d "$SCREEN_LOGDIR" ]]; then # DIVIDER find $SCREEN_LOGDIR -maxdepth 1 -name screen-\*.log -mtime +$LOGDAYS -exec rm {} \; else sudo mkdir -p $SCREEN_LOGDIR sudo chown -R `whoami` $SCREEN_LOGDIR fi fi # DIVIDER set -o errexit # DIVIDER set -o xtrace # DIVIDER if [ "${GRENADE_USE_EXTERNAL_DEVSTACK}" != "True" ]; then # DIVIDER fetch_devstacks else devstacks_setup_environment fi # DIVIDER source $GRENADE_DIR/functions # DIVIDER export PS4='+ $(short_source): ' # DIVIDER export TOP_DIR=$TARGET_DEVSTACK_DIR # DIVIDER source $TARGET_DEVSTACK_DIR/inc/meta-config # DIVIDER extract_localrc_section $BASE_DEVSTACK_DIR/local.conf \ $BASE_DEVSTACK_DIR/localrc \ $BASE_DEVSTACK_DIR/.localrc.auto # DIVIDER ENABLED_SERVICES=$(set +o xtrace && source $BASE_DEVSTACK_DIR/stackrc && echo $ENABLED_SERVICES) # DIVIDER fetch_grenade_plugins # DIVIDER load_settings if [ "${GRENADE_USE_EXTERNAL_DEVSTACK}" != "True" ]; then # DIVIDER devstack_localrc base "CELLSV2_SETUP=singleconductor" devstack_localrc target "CELLSV2_SETUP=singleconductor" fi # DIVIDER extract_localrc_section $TARGET_DEVSTACK_DIR/local.conf \ $TARGET_DEVSTACK_DIR/localrc \ $TARGET_DEVSTACK_DIR/.localrc.auto # DIVIDER if [[ "$RUN_BASE" == "True" ]]; then # DIVIDER init_grenade_db if [ "${GRENADE_USE_EXTERNAL_DEVSTACK}" != "True" ]; then echo_summary "Running base stack.sh" cd $BASE_DEVSTACK_DIR GIT_BASE=$GIT_BASE ./stack.sh stop $STOP stack.sh 10 echo_summary "Running post-stack.sh" if [[ -e $GRENADE_DIR/post-stack.sh ]]; then cd $GRENADE_DIR # DIVIDER export SUB_NODE_ENV_VARS="CELLSV2_SETUP=singleconductor" sed -i 's/stdbuf/$SUB_NODE_ENV_VARS stdbuf/' ./post-stack.sh cat ./post-stack.sh ./post-stack.sh stop $STOP post-stack.sh 15 echo_summary "Completed post-stack.sh" fi fi # DIVIDER echo_summary "Caching downloaded images" mkdir -p $BASE_RELEASE_DIR/images echo "Images: $IMAGE_URLS" for image_url in ${IMAGE_URLS//,/ }; do IMAGE_FNAME=`basename "$image_url"` if [[ -r $BASE_DEVSTACK_DIR/files/$IMAGE_FNAME ]]; then rsync -a $BASE_DEVSTACK_DIR/files/$IMAGE_FNAME $BASE_RELEASE_DIR/images fi done # DIVIDER if [[ -d $BASE_DEVSTACK_DIR/files/images ]] ; then rsync -a $BASE_DEVSTACK_DIR/files/images/ $BASE_RELEASE_DIR/images fi stop $STOP image-cache 20 # DIVIDER # DIVIDER if [[ "$BASE_RUN_SMOKE" == "True" ]]; then echo_summary "Running base smoke test" run_tempest $BASE_RELEASE_DIR if [ "${GRENADE_USE_EXTERNAL_DEVSTACK}" != "True" ]; then # DIVIDER if [[ -e $TARGET_RELEASE_DIR/tempest ]]; then rsync -a $BASE_RELEASE_DIR/tempest/.tox/ $TARGET_RELEASE_DIR/tempest/.tox/ if [[ -d $BASE_RELEASE_DIR/tempest/.testrepository ]]; then rsync -a $BASE_RELEASE_DIR/tempest/.testrepository/ $TARGET_RELEASE_DIR/tempest/.testrepository/ elif [[ -d $BASE_RELEASE_DIR/tempest/.stestr ]]; then rsync -a $BASE_RELEASE_DIR/tempest/.stestr/ $TARGET_RELEASE_DIR/tempest/.stestr/ fi fi fi fi stop $STOP base-smoke 110 # DIVIDER resources early_create # DIVIDER resources create # DIVIDER resources verify pre-upgrade # DIVIDER echo_summary "Saving current state information" $GRENADE_DIR/save-state stop $STOP save-state 130 # DIVIDER echo_summary "Shutting down all services on base devstack..." shutdown_services # DIVIDER resources verify_noapi pre-upgrade fi # DIVIDER if [[ "$RUN_TARGET" == "True" ]]; then # DIVIDER fetch_plugins # DIVIDER echo_summary "Preparing the target devstack environment" $GRENADE_DIR/prep-target # DIVIDER echo "Upgrade projects: $UPGRADE_PROJECTS" for project in $UPGRADE_PROJECTS; do echo "Upgrading project: $project" upgrade_service $project done # DIVIDER if [[ "$ENABLE_TEMPEST" == "True" ]]; then echo_summary "Running upgrade-tempest" $GRENADE_DIR/upgrade-tempest || die $LINENO "Failure in upgrade-tempest" stop $STOP upgrade-tempest 290 fi # DIVIDER # DIVIDER resources verify post-upgrade # DIVIDER if [[ "$TARGET_RUN_SMOKE" == "True" ]]; then echo_summary "Running tempest smoke tests" run_tempest $TARGET_RELEASE_DIR stop $STOP run-smoke 330 fi # DIVIDER save_data $TARGET_RELEASE $TARGET_DEVSTACK_DIR # DIVIDER resources destroy fi # DIVIDER echo_summary "Grenade has completed the pre-programmed upgrade scripts." # DIVIDER echo_summary "grenade.sh completed in $SECONDS seconds." # DIVIDER | |
|
Grenade assumes it is running on the system that will be hosting the upgrade processes | |
|
grenade.sh [-b] [-t] [-s stop-label] [-q] -b Run only the base part -t Run only the target part (assumes a base run is in place) -q Quiet mode -s stop-label is the name of the step after which the script will stop. This is useful for debugging upgrades. | |
|
GRENADE_DIR is set once by the top level grenade.sh and exported so that all subsequent scripts can find their way back to the grenade root directory. No other scripts should set this variable | |
|
Source the bootstrapping facilities Grenade attempts to reuse as much content from devstack on the target side as possible, but we need enough of our own code to get there. grenaderc is a set of X=Y declarations that don't need any of the devstack functions to work inc/bootstrap is the most minimal amount of functions that grenade needs to get going. This includes things like echo functions, trueorfalse, and the functions related to git cloning, so that we can get our devstack trees. | |
|
Create all the base directory structures needed for the rest of the environment to run. This will give you an STACK_ROOT tree that you expect, and that your normal user owns for follow on activities. | |
Logging | |
|
TODO(sdague): should this extract into inc/bootstrap? Set up logging Set LOGFILE to turn on logging Append '.xxxxxxxx' to the given name to maintain history where 'xxxxxxxx' is a representation of the date the file was created | |
|
First clean up old log files. Use the user-specified LOGFILE as the template to search for, appending '.*' to match the date we added on earlier runs. | |
|
Redirect output according to config Copy stdout to fd 3 | |
|
Redirect stdout/stderr to tee to write the log file | |
|
Set up a second fd for output | |
|
Set fd 1 and 2 to primary logfile | |
|
Set fd 6 to summary logfile and stdout | |
|
Specified logfile name always links to the most recent log | |
|
Set up output redirection without log files Copy stdout to fd 3 | |
|
Throw away stdout and stderr | |
|
Always send summary fd to original stdout | |
|
Set up logging of screen windows Set SCREEN_LOGDIR to turn on logging of screen windows to the directory specified in SCREEN_LOGDIR, we will log to the file screen-$SERVICE_NAME-$TIMESTAMP.log in that dir and have a link screen-$SERVICE_NAME.log to the latest log file. Logs are kept for as long specified in LOGDAYS. | |
|
We make sure the directory is created. | |
|
We cleanup the old logs | |
|
This script exits on an error so that errors don't compound and you see only the first error that occurred. | |
|
Print the commands being run so that we can see the command that triggers an error. It is also useful for following allowing as the install occurs. | |
Devstack Phase 2 initializationWe now have enough infrastructure in grenade.sh to go and get both SOURCE and TARGET devstack trees. After which point we 'pivot' onto the TARGET devstack functions file, then source the rest of the grenade settings. This should let us run the bulk of grenade. | |
|
Get both devstack trees, so that BASE_DEVSTACK_DIR, and TARGET_DEVSTACK_DIR are now fully populated. | |
|
Source the rest of the Grenade functions. For convenience $GRENADE_DIR/functions implicitly sources $TARGET_DEVSTACK_DIR/functions. So this line can't happen until we have the devstacks pulled down. | |
|
We now have the 'short_source' function available, so setup our PS4 variable | |
|
Many calls inside of devstack functions reference $TOP_DIR, which is the root of devstack. We export $TOP_DIR to all child processes here to be the TARGET_DEVSTACK_DIR. If you want a script to use functions off of BASE_DEVSTACK_DIR (like the shutdown phase) you must explicitly reset TOP_DIR in those scripts. | |
Install 'Base' Build of OpenStack | |
|
Oh the complexity of bootstrapping. We need to populate enabled services from what devstack-gate might have set it as. | |
|
Collect the ENABLED_SERVICES from the base directory, this is what we are starting with. | |
|
Fetch all the grenade plugins which were registered in pluginrc via the enable_grenade_plugin stanza. This must be done before settings are loaded, but has to be this late to give access to all the devstack functions. | |
|
Load the settings files for all the in tree projects/. This registers all the projects in order that we're going to be upgrading when the time is right. | |
|
Nova should use singleconductor as Grenade doesn't setup multi-cell rabbit for now | |
|
And ensure that we setup the target localrc.auto, because stack.sh isn't run there. This has to be run after load_settings because plugins might change the service list during this phase. | |
|
Run the base install of the environment | |
|
Initialize grenade_db local storage, used for resource tracking | |
|
By the time we get here the sub nodes are already setup with localrc files as those are transferred in devstack-gate even before grenade.sh is called We hack the ./post-stack.sh to inject what we need. if we don't set CELLSV2_SETUP, the default devstack assumes "superconductor" and fails. | |
Cache downloaded instances | |
|
NOTE(sileht): If glance is not enabled the directory cannot exists. | |
Operation | |
|
Validate the install | |
|
once we are done, copy our created artifacts to the target | |
|
Early Create resources, used largely for network setup | |
|
Create resources | |
|
Verify the resources were created | |
|
Save some stuff before we shut that whole thing down | |
|
Shut down running code | |
|
Verify the resources still exist after the shutdown | |
Upgrades | |
|
Clone all devstack plugins on the new side, because we're not running a full stack.sh | |
|
Get target devstack tree ready for services to be run from it, including trying to reuse any existing files we pulled during the base run. | |
|
upgrade all the projects in order | |
|
Upgrade Tempest | |
Upgrade Tests | |
|
Verify the resources still exist after the upgrade | |
|
Validate the upgrade This is used for testing grenade locally, and should not be used in the gate. Instead, grenade.sh runs smoke tests on the old cloud above, but smoke tests are run on the upgraded cloud by the gate script after grenade.sh has finished. If this option is enabled in the gate, tempest will be run twice against the new cloud. | |
Save databases | |
|
Cleanup the resources | |
Fin | |
|
Indicate how long this took to run (bash maintained variable SECONDS) | |