From 6fd09137f8c45ce4790833d7f6b840d885818771 Mon Sep 17 00:00:00 2001 From: Mark McLoughlin Date: Wed, 22 Jan 2014 06:59:30 +0000 Subject: [PATCH] Sync latest config file generator from oslo-incubator Commits included: e3dddd7 generator: use EXTRA_* env vars in the bash script e8e636c generator: add an EXTRA_LIBRARIES env variable 6da13e8 generator: rename EXTRA_MODULES_FILE to RC_FILE 5dce17b Use entry points to discover options in libraries dd9aa2b Remove unused variables ad17a69 Fix filter() usage due to python 3 compability 343686b Add check_uptodate to tools/config The library option discovery support is the main addition we're interested in since it is required for oslo.messaging. Change-Id: I68958ddd1c1d0c02b54002c36b97270efaaaa892 --- nova/openstack/common/config/generator.py | 47 +++++++++++++++++------ tools/config/check_uptodate.sh | 32 +++++++++++---- tools/config/generate_sample.sh | 32 ++++++++++++--- tools/config/oslo.config.generator.rc | 2 +- 4 files changed, 86 insertions(+), 27 deletions(-) diff --git a/nova/openstack/common/config/generator.py b/nova/openstack/common/config/generator.py index bc8772af3505..1cb531234d4f 100644 --- a/nova/openstack/common/config/generator.py +++ b/nova/openstack/common/config/generator.py @@ -18,6 +18,7 @@ from __future__ import print_function +import argparse import imp import os import re @@ -27,6 +28,7 @@ import textwrap from oslo.config import cfg import six +import stevedore.named from nova.openstack.common import gettextutils from nova.openstack.common import importutils @@ -59,34 +61,55 @@ BASEDIR = os.path.abspath(os.path.join(os.path.dirname(__file__), WORDWRAP_WIDTH = 60 -def generate(srcfiles): +def generate(argv): + parser = argparse.ArgumentParser( + description='generate sample configuration file', + ) + parser.add_argument('-m', dest='modules', action='append') + parser.add_argument('-l', dest='libraries', action='append') + parser.add_argument('srcfiles', nargs='*') + parsed_args = parser.parse_args(argv) + mods_by_pkg = dict() - for filepath in srcfiles: + for filepath in parsed_args.srcfiles: pkg_name = filepath.split(os.sep)[1] mod_str = '.'.join(['.'.join(filepath.split(os.sep)[:-1]), os.path.basename(filepath).split('.')[0]]) mods_by_pkg.setdefault(pkg_name, list()).append(mod_str) # NOTE(lzyeval): place top level modules before packages - pkg_names = filter(lambda x: x.endswith(PY_EXT), mods_by_pkg.keys()) - pkg_names.sort() - ext_names = filter(lambda x: x not in pkg_names, mods_by_pkg.keys()) - ext_names.sort() + pkg_names = sorted(pkg for pkg in mods_by_pkg if pkg.endswith(PY_EXT)) + ext_names = sorted(pkg for pkg in mods_by_pkg if pkg not in pkg_names) pkg_names.extend(ext_names) # opts_by_group is a mapping of group name to an options list # The options list is a list of (module, options) tuples opts_by_group = {'DEFAULT': []} - extra_modules = os.getenv("NOVA_CONFIG_GENERATOR_EXTRA_MODULES", "") - if extra_modules: - for module_name in extra_modules.split(','): - module_name = module_name.strip() + if parsed_args.modules: + for module_name in parsed_args.modules: module = _import_module(module_name) if module: for group, opts in _list_opts(module): opts_by_group.setdefault(group, []).append((module_name, opts)) + # Look for entry points defined in libraries (or applications) for + # option discovery, and include their return values in the output. + # + # Each entry point should be a function returning an iterable + # of pairs with the group name (or None for the default group) + # and the list of Opt instances for that group. + if parsed_args.libraries: + loader = stevedore.named.NamedExtensionManager( + 'oslo.config.opts', + names=list(set(parsed_args.libraries)), + invoke_on_load=False, + ) + for ext in loader: + for group, opts in ext.plugin(): + opt_list = opts_by_group.setdefault(group or 'DEFAULT', []) + opt_list.append((ext.name, opts)) + for pkg_name in pkg_names: mods = mods_by_pkg.get(pkg_name) mods.sort() @@ -120,7 +143,7 @@ def _import_module(mod_str): def _is_in_group(opt, group): "Check if opt is in group." - for key, value in group._opts.items(): + for value in group._opts.values(): # NOTE(llu): Temporary workaround for bug #1262148, wait until # newly released oslo.config support '==' operator. if not(value['opt'] != opt): @@ -134,7 +157,7 @@ def _guess_groups(opt, mod_obj): return 'DEFAULT' # what other groups is it in? - for key, value in cfg.CONF.items(): + for value in cfg.CONF.values(): if isinstance(value, cfg.CONF.GroupAttr): if _is_in_group(opt, value._group): return value._group.name diff --git a/tools/config/check_uptodate.sh b/tools/config/check_uptodate.sh index b1f86aaf5fa6..48add47875ef 100755 --- a/tools/config/check_uptodate.sh +++ b/tools/config/check_uptodate.sh @@ -1,9 +1,25 @@ -#!/bin/sh -TEMPDIR=`mktemp -d` -CFGFILE=nova.conf.sample -tools/config/generate_sample.sh -b ./ -p nova -o $TEMPDIR -if ! diff $TEMPDIR/$CFGFILE etc/nova/$CFGFILE -then - echo "E: nova.conf.sample is not up to date, please run tools/config/generate_sample.sh" - exit 42 +#!/usr/bin/env bash + +PROJECT_NAME=${PROJECT_NAME:-nova} +CFGFILE_NAME=${PROJECT_NAME}.conf.sample + +if [ -e etc/${PROJECT_NAME}/${CFGFILE_NAME} ]; then + CFGFILE=etc/${PROJECT_NAME}/${CFGFILE_NAME} +elif [ -e etc/${CFGFILE_NAME} ]; then + CFGFILE=etc/${CFGFILE_NAME} +else + echo "${0##*/}: can not find config file" + exit 1 +fi + +TEMPDIR=`mktemp -d /tmp/${PROJECT_NAME}.XXXXXX` +trap "rm -rf $TEMPDIR" EXIT + +tools/config/generate_sample.sh -b ./ -p ${PROJECT_NAME} -o ${TEMPDIR} + +if ! diff -u ${TEMPDIR}/${CFGFILE_NAME} ${CFGFILE} +then + echo "${0##*/}: ${PROJECT_NAME}.conf.sample is not up to date." + echo "${0##*/}: Please run ${0%%${0##*/}}generate_sample.sh." + exit 1 fi diff --git a/tools/config/generate_sample.sh b/tools/config/generate_sample.sh index 828fbfc7a0e7..94d6f3ec950c 100755 --- a/tools/config/generate_sample.sh +++ b/tools/config/generate_sample.sh @@ -4,8 +4,8 @@ print_hint() { echo "Try \`${0##*/} --help' for more information." >&2 } -PARSED_OPTIONS=$(getopt -n "${0##*/}" -o hb:p:o: \ - --long help,base-dir:,package-name:,output-dir: -- "$@") +PARSED_OPTIONS=$(getopt -n "${0##*/}" -o hb:p:m:l:o: \ + --long help,base-dir:,package-name:,output-dir:,module:,library: -- "$@") if [ $? != 0 ] ; then print_hint ; exit 1 ; fi @@ -21,6 +21,8 @@ while true; do echo "-b, --base-dir=DIR project base directory" echo "-p, --package-name=NAME project package name" echo "-o, --output-dir=DIR file output directory" + echo "-m, --module=MOD extra python module to interrogate for options" + echo "-l, --library=LIB extra library that registers options for discovery" exit 0 ;; -b|--base-dir) @@ -38,6 +40,16 @@ while true; do OUTPUTDIR=`echo $1 | sed -e 's/\/*$//g'` shift ;; + -m|--module) + shift + MODULES="$MODULES -m $1" + shift + ;; + -l|--library) + shift + LIBRARIES="$LIBRARIES -l $1" + shift + ;; --) break ;; @@ -77,12 +89,20 @@ find $TARGETDIR -type f -name "*.pyc" -delete FILES=$(find $TARGETDIR -type f -name "*.py" ! -path "*/tests/*" \ -exec grep -l "Opt(" {} + | sed -e "s/^$BASEDIRESC\///g" | sort -u) -EXTRA_MODULES_FILE="`dirname $0`/oslo.config.generator.rc" -if test -r "$EXTRA_MODULES_FILE" +RC_FILE="`dirname $0`/oslo.config.generator.rc" +if test -r "$RC_FILE" then - source "$EXTRA_MODULES_FILE" + source "$RC_FILE" fi +for mod in ${NOVA_CONFIG_GENERATOR_EXTRA_MODULES}; do + MODULES="$MODULES -m $mod" +done + +for lib in ${NOVA_CONFIG_GENERATOR_EXTRA_LIBRARIES}; do + LIBRARIES="$LIBRARIES -l $lib" +done + export EVENTLET_NO_GREENDNS=yes OS_VARS=$(set | sed -n '/^OS_/s/=[^=]*$//gp' | xargs) @@ -90,7 +110,7 @@ OS_VARS=$(set | sed -n '/^OS_/s/=[^=]*$//gp' | xargs) DEFAULT_MODULEPATH=nova.openstack.common.config.generator MODULEPATH=${MODULEPATH:-$DEFAULT_MODULEPATH} OUTPUTFILE=$OUTPUTDIR/$PACKAGENAME.conf.sample -python -m $MODULEPATH $FILES > $OUTPUTFILE +python -m $MODULEPATH $MODULES $LIBRARIES $FILES > $OUTPUTFILE # Hook to allow projects to append custom config file snippets CONCAT_FILES=$(ls $BASEDIR/tools/config/*.conf.sample 2>/dev/null) diff --git a/tools/config/oslo.config.generator.rc b/tools/config/oslo.config.generator.rc index 3b5c2fd4915b..457d4fc908ad 100644 --- a/tools/config/oslo.config.generator.rc +++ b/tools/config/oslo.config.generator.rc @@ -1 +1 @@ -export NOVA_CONFIG_GENERATOR_EXTRA_MODULES=keystoneclient.middleware.auth_token +NOVA_CONFIG_GENERATOR_EXTRA_MODULES=keystoneclient.middleware.auth_token