diff --git a/build-tools/stx/build-image b/build-tools/stx/build-image index 457ece09..396e24b8 100755 --- a/build-tools/stx/build-image +++ b/build-tools/stx/build-image @@ -25,6 +25,7 @@ import time import utils import yaml +REPO_ALL = 'deb-merge-all' REPO_BINARY = 'deb-local-binary' REPO_BUILD = 'deb-local-build' DEB_CONFIG_DIR = 'stx-tools/debian-mirror-tools/config/' @@ -34,6 +35,50 @@ img_pkgs = [] logger = logging.getLogger('build-image') utils.set_logger(logger) + +def merge_local_repos(repomgr): + logger.debug('Calls repo manager to create/udpate the snapshot %s which is merged from local repositories', REPO_ALL) + # REPO_BUILD is higher priority than REPO_BINARY for repomgr to select package + try: + pubname = repomgr.merge(REPO_ALL, ','.join([REPO_BUILD, REPO_BINARY])) + except Exception as e: + logger.error(str(e)) + logger.error('Exception when repo_manager creates/updates snapshot %s', REPO_ALL) + return False + if pubname: + logger.debug('repo manager successfully created/updated snapshot %s', REPO_ALL) + else: + logger.debug('repo manager failed to create/update snapshot %s', REPO_ALL) + return False + return True + + +def update_debootstrap_mirror(img_yaml): + repomgr_url = os.environ.get('REPOMGR_DEPLOY_URL') + if not repomgr_url: + logger.error('REPOMGR_URL is not in current sys ENV') + return False + + repo_all_url = '/'.join([repomgr_url, REPO_ALL]) + try: + with open(img_yaml) as f: + yaml_doc = yaml.safe_load(f) + if not yaml_doc['debootstrap-mirror']: + logger.warning("There is not debootstrap-mirror in %s", img_yaml) + else: + mirror = yaml_doc['debootstrap-mirror'] + if mirror == REPO_ALL: + yaml_doc['debootstrap-mirror'] = repo_all_url + with open(img_yaml, 'w') as f: + yaml.safe_dump(yaml_doc, f, default_flow_style=False, sort_keys=False) + logger.debug('Updating %s, setting debootstrap_mirror to %s', img_yaml, repo_all_url) + return True + except IOError as e: + logger.error(str(e)) + logger.debug('Failed to update %s, could not set debootstrap_mirror to %s', img_yaml, repo_all_url) + return False + + def update_ostree_osname(img_yaml): ostree_osname = os.environ.get('OSTREE_OSNAME') @@ -53,6 +98,7 @@ def update_ostree_osname(img_yaml): logger.debug(' '.join(['Update', img_yaml, 'to update the ostree_osname'])) return True + def feed_lat_src_repos(img_yaml, repo_url): if not os.path.exists(img_yaml): logger.error(' '.join(['LAT yaml file', img_yaml, 'does not exist'])) @@ -97,7 +143,7 @@ def check_base_os_binaries(repomgr): 'does not exist'])) return False - results = query_repo(repomgr, REPO_BINARY, base_bins_list) + results = verify_pkgs_in_repo(repomgr, REPO_BINARY, base_bins_list) if results: logger.error("====OS binaries checking fail:") for deb in results: @@ -117,7 +163,7 @@ def check_stx_binaries(repomgr, btype='std'): # Assume no such list here means ok return True - results = query_repo(repomgr, REPO_BINARY, stx_bins_list) + results = verify_pkgs_in_repo(repomgr, REPO_BINARY, stx_bins_list) if results: logger.error("====STX binaries checking fail:") for deb in results: @@ -136,7 +182,7 @@ def check_stx_patched(repomgr, btype='std'): 'does not exist'])) return False - results = query_repo(repomgr, REPO_BUILD, stx_patched_list) + results = verify_pkgs_in_repo(repomgr, REPO_BUILD, stx_patched_list) if results: logger.error("====STX patched packages checking fail:") for deb in results: @@ -147,7 +193,7 @@ def check_stx_patched(repomgr, btype='std'): return True -def query_repo(repomgr, repo_name, pkg_list_path): +def verify_pkgs_in_repo(repomgr, repo_name, pkg_list_path): failed_pkgs = [] with open(pkg_list_path, 'r') as flist: lines = list(line for line in (lpkg.strip() for lpkg in flist) if line) @@ -169,7 +215,7 @@ def query_repo(repomgr, repo_name, pkg_list_path): 'is missing in local binary repo'])) failed_pkgs.append(pkg_name) else: - if repomgr.search_pkg(repo_name, name): + if repomgr.search_pkg(repo_name, name, None, True): img_pkgs.append(name) logger.debug(''.join(['Found package with name:', name])) else: @@ -205,8 +251,10 @@ if __name__ == "__main__": logger.error(e.msg) usage() + rmg_logger = logging.getLogger('repo_manager') + utils.set_logger(rmg_logger) repo_manager = repo_manage.RepoMgr('aptly', os.environ.get('REPOMGR_URL'), - '/tmp/', logger) + '/tmp/', rmg_logger) repo_manager.upload_pkg(REPO_BUILD, None) repo_manager.upload_pkg(REPO_BINARY, None) @@ -242,9 +290,16 @@ if __name__ == "__main__": shutil.copyfile(base_yaml, lat_yaml) shutil.copyfile(base_initramfs_yaml, lat_initramfs_yaml) except IOError as e: + logger.error(str(e)) logger.error('Fail to copy image yaml files to /localdisk/deploy') sys.exit(1) + if merge_local_repos(repo_manager): + if update_debootstrap_mirror(lat_yaml): + logger.debug("Debootstrap switches to mirror %s in %s", REPO_ALL, lat_yaml) + if update_debootstrap_mirror(lat_initramfs_yaml): + logger.debug("Debootstrap switches to mirror %s in %s", REPO_ALL, lat_initramfs_yaml) + binary_repo_url = ''.join(['deb [trusted=yes] ', os.environ.get('REPOMGR_DEPLOY_URL'), REPO_BINARY, ' bullseye main']) diff --git a/build-tools/stx/build-pkgs b/build-tools/stx/build-pkgs index a90fa1d1..3f1d3dd3 100755 --- a/build-tools/stx/build-pkgs +++ b/build-tools/stx/build-pkgs @@ -450,12 +450,9 @@ class BuildController(): msg = ''.join(['package ', pkg_item[0], '(', pkg_item[1], ')']) logger.info(' '.join(['Searching for binary', msg, 'in repository', REPO_BUILD])) - if self.kits['repo_mgr'].search_pkg(REPO_BUILD, pkg_item[0], - pkg_item[1]): + if self.kits['repo_mgr'].search_pkg(REPO_BUILD, pkg_item[0], None, True): logger.info('Found binary %s in repository %s', msg, REPO_BUILD) - if self.kits['repo_mgr'].delete_pkg(REPO_BUILD, - pkg_item[0], 'binary', - pkg_item[1]): + if self.kits['repo_mgr'].delete_pkg(REPO_BUILD, pkg_item[0], 'binary', None): logger.info('Successfully deleted binary %s from repository %s', msg, REPO_BUILD) else: diff --git a/build-tools/stx/downloader b/build-tools/stx/downloader index 5a6200c2..e981cbfa 100755 --- a/build-tools/stx/downloader +++ b/build-tools/stx/downloader @@ -126,8 +126,10 @@ class BaseDownloader(): self.dl_need = [] self.dl_success = [] self.dl_failed = [] + rlogger = logging.getLogger('repo_manager') + utils.set_logger(rlogger) self.repomgr = repo_manage.RepoMgr('aptly', os.environ.get('REPOMGR_URL'), - '/tmp/', logger) + '/tmp/', rlogger) self.repomgr.upload_pkg(REPO_BIN, None) def clean(self): @@ -177,21 +179,22 @@ class DebDownloader(BaseDownloader): self.repomgr.upload_pkg(REPO_BIN, None) def download(self, _name, _version): - package = self.apt_cache[_name] - candidate = package.versions.get(_version) - if not candidate: - logger.error(' '.join(['Fail to download', _name, - 'with wrong version', _version, '?'])) - logger.error('May need to update the package list file') - return None - - package.candidate = candidate try: + package = self.apt_cache[_name] + candidate = package.versions.get(_version) + if not candidate: + logger.error(' '.join(['Fail to download', _name, + 'with wrong version', _version, '?'])) + logger.error('May need to update the package list file') + return None + + package.candidate = candidate ret = package.candidate.fetch_binary(self.dl_dir) if ret: return ret - except apt.package.FetchError: + except Exception as e: logger.debug("Fail to fetch binray %s_%s", _name, _version) + logger.debug(str(e)) return None def reports(self): @@ -246,7 +249,9 @@ class DebDownloader(BaseDownloader): for deb in self.need_upload: name, ver, arch = deb.split('_') - if not self.repomgr.search_pkg(REPO_BIN, name, ver): + if not self.repomgr.search_pkg(REPO_BIN, name, ver, True): + if name and ver: + logger.debug('Package %s-%s not found in %s', name, ver, REPO_BIN) if self.repomgr.upload_pkg(REPO_BIN, os.path.join(stx_bin_mirror, deb)): logger.info(' '.join([os.path.join(stx_bin_mirror, deb), 'is uploaded to', REPO_BIN]))