From 0142bbe770164a02c3a49693ca7cfdf44d48ff0f Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Date: Tue, 20 Apr 2021 21:30:50 +0500 Subject: [PATCH] Use coloredlogs for logging --- requirements.txt | 1 + scale_build/checkout.py | 2 +- scale_build/iso.py | 3 ++- scale_build/main.py | 3 +++ scale_build/package.py | 7 ++++--- scale_build/packages/package.py | 19 ++++++++++--------- scale_build/update_image.py | 3 ++- setup.py | 8 +++++++- 8 files changed, 30 insertions(+), 16 deletions(-) diff --git a/requirements.txt b/requirements.txt index 590f702..0caf1ac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +coloredlogs toposort psutil pyyaml diff --git a/scale_build/checkout.py b/scale_build/checkout.py index 6d52f6b..a0a17c6 100644 --- a/scale_build/checkout.py +++ b/scale_build/checkout.py @@ -13,7 +13,7 @@ logger = logging.getLogger(__name__) def checkout_sources(): info = retrieve_git_remote_and_sha('.') update_git_manifest(info['url'], info['sha'], 'w') - logger.debug(f'Starting checkout of source') + logger.info('Starting checkout of source') for package in get_packages(): gh_override = BRANCH_OVERRIDES.get(package.name) diff --git a/scale_build/iso.py b/scale_build/iso.py index a1e1a54..a033157 100644 --- a/scale_build/iso.py +++ b/scale_build/iso.py @@ -26,6 +26,7 @@ def build_iso(): def build_impl(): + logger.info('Building TrueNAS SCALE iso') clean_mounts() for f in glob.glob(os.path.join(LOG_DIR, 'cdrom*')): os.unlink(f) @@ -45,4 +46,4 @@ def build_impl(): make_iso_file() umount_tmpfs_and_clean_chroot_dir() - logger.debug('Success! CD/USB: %s/TrueNAS-SCALE-%s.iso', RELEASE_DIR, VERSION) + logger.info('Success! CD/USB: %s/TrueNAS-SCALE-%s.iso', RELEASE_DIR, VERSION) diff --git a/scale_build/main.py b/scale_build/main.py index b8008bb..2fc511a 100644 --- a/scale_build/main.py +++ b/scale_build/main.py @@ -1,4 +1,5 @@ import argparse +import coloredlogs import logging import sys @@ -15,6 +16,8 @@ logger = logging.getLogger(__name__) def setup_logging(): logging.basicConfig(level=logging.DEBUG, format='[%(asctime)s] %(message)s') + if sys.stdout.isatty(): + coloredlogs.install(logging.DEBUG, fmt='[%(asctime)s] %(message)s') handler = logging.StreamHandler(sys.stderr) handler.setLevel(logging.DEBUG) handler.setFormatter(logging.Formatter('[%(asctime)s] %(message)s')) diff --git a/scale_build/package.py b/scale_build/package.py index a26c61e..32aea28 100644 --- a/scale_build/package.py +++ b/scale_build/package.py @@ -69,6 +69,7 @@ def build_package(package_queue, to_build, failed, in_progress, built): shutil.copytree(PKG_DIR, package.dpkg_overlay_packages_path) package._build_impl() except Exception as e: + logger.error('Failed to build %r package', package.name) failed[package.name] = {'package': package, 'exception': e} break else: @@ -80,7 +81,7 @@ def build_package(package_queue, to_build, failed, in_progress, built): ) in_progress.pop(package.name) built[package.name] = package - logger.debug( + logger.info( 'Successfully built %r package (Remaining %d packages)', package.name, len(to_build) + package_queue.qsize() + len(in_progress) ) @@ -96,7 +97,7 @@ def build_packages(): def _build_packages_impl(): - logger.debug('Building packages') + logger.info('Building packages') logger.debug('Setting up bootstrap directory') make_bootstrapdir('package') logger.debug('Successfully setup bootstrap directory') @@ -126,4 +127,4 @@ def _build_packages_impl(): p['package'].delete_overlayfs() logger.error('Failed to build %r package(s)', ', '.join(failed)) else: - logger.debug('Success! Done building packages') + logger.info('Success! Done building packages') diff --git a/scale_build/packages/package.py b/scale_build/packages/package.py index 16a28d9..f808e4f 100644 --- a/scale_build/packages/package.py +++ b/scale_build/packages/package.py @@ -5,8 +5,9 @@ import shutil from scale_build.exceptions import CallError from scale_build.utils.git_utils import retrieve_git_remote_and_sha, retrieve_git_branch, update_git_manifest +from scale_build.utils.logger import get_logger from scale_build.utils.run import run -from scale_build.utils.paths import GIT_LOG_PATH, HASH_DIR, LOG_DIR, PKG_LOG_DIR, SOURCES_DIR +from scale_build.utils.paths import GIT_LOG_PATH, HASH_DIR, PKG_LOG_DIR, SOURCES_DIR from .binary_package import BinaryPackage from .bootstrap import BootstrapMixin @@ -167,17 +168,17 @@ class Package(BootstrapMixin, BuildPackageMixin, BuildCleanMixin, OverlayMixin): def checkout(self, branch_override=None): origin_url = self.retrieve_current_remote_origin_and_sha()['url'] branch = branch_override or self.branch + git_logger = get_logger(f'checkout_{self.name}', 'git-checkout.log', 'w') + git_logger.debug('logged it') if branch == self.existing_branch and self.origin == origin_url: - logger.debug(f'Updating git repo [{self.name}] ({GIT_LOG_PATH})') - with open(GIT_LOG_PATH, 'w') as f: - run(['git', '-C', self.source_path, 'fetch', '--unshallow'], stdout=f, stderr=f, check=False) - run(['git', '-C', self.source_path, 'fetch', 'origin', branch], stdout=f, stderr=f) - run(['git', '-C', self.source_path, 'reset', '--hard', f'origin/{branch}'], stdout=f, stderr=f) + logger.debug('Updating git repo [%s] (%s)', self.name, GIT_LOG_PATH) + run(['git', '-C', self.source_path, 'fetch', '--unshallow'], logger=git_logger, check=False) + run(['git', '-C', self.source_path, 'fetch', 'origin', branch], logger=git_logger) + run(['git', '-C', self.source_path, 'reset', '--hard', f'origin/{branch}'], logger=git_logger) else: - logger.debug(f'Checking out git repo [{self.name}] ({GIT_LOG_PATH})') + logger.debug('Checking out git repo [%s] (%s)', self.name, GIT_LOG_PATH) shutil.rmtree(self.source_path, ignore_errors=True) - with open(GIT_LOG_PATH, 'w') as f: - run(['git', 'clone', '--depth=1', '-b', branch, self.origin, self.source_path], stdout=f, stderr=f) + run(['git', 'clone', '--depth=1', '-b', branch, self.origin, self.source_path], logger=git_logger) self.update_git_manifest() diff --git a/scale_build/update_image.py b/scale_build/update_image.py index 5e3fd78..c0fdf44 100644 --- a/scale_build/update_image.py +++ b/scale_build/update_image.py @@ -24,6 +24,7 @@ def build_update_image(): def build_update_image_impl(): os.makedirs(RELEASE_DIR, exist_ok=True) + logger.info('Building update image') clean_mounts() os.makedirs(CHROOT_BASEDIR) logger.debug('Bootstrapping TrueNAS rootfs [UPDATE] (%s/rootfs-bootstrap.log)', LOG_DIR) @@ -38,4 +39,4 @@ def build_update_image_impl(): build_rootfs_image() umount_tmpfs_and_clean_chroot_dir() - logger.debug('Success! Update image created at: %s', UPDATE_FILE) + logger.info('Success! Update image created at: %s', UPDATE_FILE) diff --git a/setup.py b/setup.py index e557fc4..f43b209 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,13 @@ setup( include_package_data=True, license='BSD', platforms='any', - install_requires=install_requires, + install_requires=[ + 'coloredlogs', + 'toposort', + 'psutil', + 'requests', + 'pyyaml' + ], entry_points={ 'console_scripts': [ 'scale_build = scale_build.main:main',