From accc46940791989b571d5f10ad46035fb9bc834a Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Mon, 8 Aug 2016 23:23:12 -0500 Subject: [PATCH] Add indexing of individual package keys This change implements indexing of py packags from individual keys so that we can further tune the build process to be even more selective by default. Change-Id: Iad00e4f1268187a29e90d7b8e6afb5abfd1e92a2 Signed-off-by: Kevin Carter --- lookup/py_pkgs.py | 73 ++++++++++++++++++++++++++--------------------- 1 file changed, 41 insertions(+), 32 deletions(-) diff --git a/lookup/py_pkgs.py b/lookup/py_pkgs.py index 734f540..5cfde67 100644 --- a/lookup/py_pkgs.py +++ b/lookup/py_pkgs.py @@ -36,6 +36,7 @@ else: GIT_PACKAGE_DEFAULT_PARTS = dict() +ROLE_BREAKOUT_REQUIREMENTS = dict() ROLE_PACKAGES = dict() ROLE_REQUIREMENTS = dict() @@ -458,7 +459,7 @@ class DependencyFileProcessor(object): ) def _package_build_index(self, packages, role_name, var_name, - pkg_index=ROLE_PACKAGES, project_group='all'): + pkg_index, project_group='all'): self._py_pkg_extend(packages) if role_name: if role_name in pkg_index: @@ -468,19 +469,24 @@ class DependencyFileProcessor(object): role_pkgs['project_group'] = project_group pkgs = role_pkgs.get(var_name, list()) - if 'optional' not in var_name: - pkgs = self._py_pkg_extend(packages, pkgs) - pkg_index[role_name][var_name] = pkgs + pkgs = self._py_pkg_extend(packages, pkgs) + pkg_index[role_name][var_name] = sorted(pkgs) else: for k, v in pkg_index.items(): for item_name in v.keys(): if var_name == item_name: - pkg_index[k][item_name] = self._py_pkg_extend(packages, pkg_index[k][item_name]) + pkg_index[k][item_name] = self._py_pkg_extend( + packages, + pkg_index[k][item_name] + ) def _process_files(self): - """Process files.""" + """Process all of the requirement files.""" + self._process_files_defaults() + self._process_files_requirements() - role_name = None + def _process_files_defaults(self): + """Process files.""" for file_name in self._filter_files(self.file_names, ('yaml', 'yml')): with open(file_name, 'r') as f: # If there is an exception loading the file continue @@ -494,9 +500,11 @@ class DependencyFileProcessor(object): if not loaded_config or not isinstance(loaded_config, dict): continue - if 'roles' in file_name: - _role_name = file_name.split('roles%s' % os.sep)[-1] - role_name = _role_name.split(os.sep)[0] + if 'roles' in file_name: + _role_name = file_name.split('roles%s' % os.sep)[-1] + role_name = _role_name.split(os.sep)[0] + else: + role_name = None for key, value in loaded_config.items(): if key.endswith('role_project_group'): @@ -506,6 +514,7 @@ class DependencyFileProcessor(object): project_group = 'all' for key, values in loaded_config.items(): + key = key.lower() if key.endswith('git_repo'): self._process_git( loaded_yaml=loaded_config, @@ -513,32 +522,30 @@ class DependencyFileProcessor(object): yaml_file_name=file_name ) - if [i for i in BUILT_IN_PIP_PACKAGE_VARS - if i in key - if 'proprietary' not in key]: + if [i for i in BUILT_IN_PIP_PACKAGE_VARS if i in key]: + self._package_build_index( + packages=values, + role_name=role_name, + var_name=key, + pkg_index=ROLE_BREAKOUT_REQUIREMENTS, + project_group=project_group + ) + + if 'optional' in key: + continue + elif 'proprietary' in key: + continue + else: self._package_build_index( packages=values, role_name=role_name, var_name=key, + pkg_index=ROLE_PACKAGES, project_group=project_group ) - for key, values in loaded_config.items(): - if 'proprietary' in key: - proprietary_pkgs = [ - i for i in values - if GIT_PACKAGE_DEFAULT_PARTS.get(i) - ] - if proprietary_pkgs: - self._package_build_index( - packages=proprietary_pkgs, - role_name=role_name, - var_name=key, - project_group=project_group - ) - else: - role_name = None - + def _process_files_requirements(self): + """Process requirements files.""" return_list = self._filter_files(self.file_names, 'txt') for file_name in return_list: base_name = os.path.basename(file_name) @@ -550,12 +557,12 @@ class DependencyFileProcessor(object): for file_name in return_list: if file_name.endswith('other-requirements.txt'): continue - if file_name.endswith('bindep.txt'): + elif file_name.endswith('bindep.txt'): continue - if 'roles' in file_name: + elif 'roles' in file_name: _role_name = file_name.split('roles%s' % os.sep)[-1] role_name = _role_name.split(os.sep)[0] - if not role_name: + else: role_name = 'default' with open(file_name, 'r') as f: packages = [ @@ -648,6 +655,7 @@ class LookupModule(BASECLASS): if isinstance(value, (list, set)): return_data[key] = sorted(value) return_data['role_requirement_files'] = ROLE_REQUIREMENTS + return_data['role_requirements'] = ROLE_BREAKOUT_REQUIREMENTS return [return_data] def run_v1(self, terms, inject=None, **kwargs): @@ -697,6 +705,7 @@ class LookupModule(BASECLASS): if isinstance(value, (list, set)): return_data[key] = sorted(value) return_data['role_requirement_files'] = ROLE_REQUIREMENTS + return_data['role_requirements'] = ROLE_BREAKOUT_REQUIREMENTS return [return_data] # Used for testing and debuging usage: `python plugins/lookups/py_pkgs.py ../`