diff --git a/scale_build/bootstrap/bootstrapdir.py b/scale_build/bootstrap/bootstrapdir.py index efb4871..fdd2d0e 100644 --- a/scale_build/bootstrap/bootstrapdir.py +++ b/scale_build/bootstrap/bootstrapdir.py @@ -37,7 +37,7 @@ class BootstrapDir(CacheMixin, HashMixin): self.add_trusted_apt_key() manifest = get_manifest() apt_repos = manifest['apt-repos'] - self.run( + run( ['debootstrap'] + self.deopts + [ '--keyring', '/etc/apt/trusted.gpg.d/debian-archive-truenas-automatic.gpg', manifest['debian_release'], self.chroot_basedir, apt_repos['url'] @@ -46,7 +46,7 @@ class BootstrapDir(CacheMixin, HashMixin): self.setup_mounts() if self.extra_packages_to_install: - self.run(['chroot', self.chroot_basedir, 'apt', 'install', '-y'] + self.extra_packages_to_install) + run(['chroot', self.chroot_basedir, 'apt', 'install', '-y'] + self.extra_packages_to_install) installed_packages = self.get_packages() @@ -65,7 +65,7 @@ class BootstrapDir(CacheMixin, HashMixin): for repo in apt_repos['additional']: self.logger.debug('Adding additional repo: %r', repo['url']) shutil.copy(os.path.join(BUILDER_DIR, repo['key']), os.path.join(self.chroot_basedir, 'apt.key')) - self.run(['chroot', self.chroot_basedir, 'apt-key', 'add', '/apt.key']) + run(['chroot', self.chroot_basedir, 'apt-key', 'add', '/apt.key']) os.unlink(os.path.join(self.chroot_basedir, 'apt.key')) apt_sources.append(f'deb {repo["url"]} {repo["distribution"]} {repo["component"]}') @@ -73,7 +73,7 @@ class BootstrapDir(CacheMixin, HashMixin): f.write('\n'.join(apt_sources)) # Update apt - self.run(['chroot', self.chroot_basedir, 'apt', 'update']) + run(['chroot', self.chroot_basedir, 'apt', 'update']) # Put our local package up at the top of the food chain apt_sources.insert(0, 'deb [trusted=yes] file:/packages /') @@ -87,14 +87,11 @@ class BootstrapDir(CacheMixin, HashMixin): pass def add_trusted_apt_key(self): - self.run([ + run([ 'apt-key', '--keyring', '/etc/apt/trusted.gpg.d/debian-archive-truenas-automatic.gpg', 'add', os.path.join(BUILDER_DIR, 'keys/truenas.gpg') ]) - def run(self, *args, **kwargs): - return run(*args, logger=self.logger, **kwargs) - @property def extra_packages_to_install(self): raise NotImplementedError @@ -104,8 +101,8 @@ class BootstrapDir(CacheMixin, HashMixin): raise NotImplementedError def setup_mounts(self): - self.run(['mount', 'proc', os.path.join(self.chroot_basedir, 'proc'), '-t', 'proc']) - self.run(['mount', 'sysfs', os.path.join(self.chroot_basedir, 'sys'), '-t', 'sysfs']) + run(['mount', 'proc', os.path.join(self.chroot_basedir, 'proc'), '-t', 'proc']) + run(['mount', 'sysfs', os.path.join(self.chroot_basedir, 'sys'), '-t', 'sysfs']) def clean_mounts(self): for command in ( diff --git a/scale_build/bootstrap/cache.py b/scale_build/bootstrap/cache.py index ec182ce..83b694a 100644 --- a/scale_build/bootstrap/cache.py +++ b/scale_build/bootstrap/cache.py @@ -1,6 +1,7 @@ import os from scale_build.utils.paths import CACHE_DIR +from scale_build.utils.run import run from .hash import get_all_repo_hash @@ -35,7 +36,7 @@ class CacheMixin: def save_build_cache(self, installed_packages): self.logger.debug('Caching CHROOT_BASEDIR for future runs...') - self.run(['mksquashfs', self.chroot_basedir, self.cache_file_path]) + run(['mksquashfs', self.chroot_basedir, self.cache_file_path]) self.update_saved_packages_list(installed_packages) self.update_mirror_cache() @@ -61,4 +62,4 @@ class CacheMixin: return self.installed_packages_in_cache != self.get_packages() def restore_cache(self, chroot_basedir): - self.run(['unsquashfs', '-f', '-d', chroot_basedir, self.cache_file_path]) + run(['unsquashfs', '-f', '-d', chroot_basedir, self.cache_file_path]) diff --git a/scale_build/bootstrap/hash.py b/scale_build/bootstrap/hash.py index 35589da..09654a5 100644 --- a/scale_build/bootstrap/hash.py +++ b/scale_build/bootstrap/hash.py @@ -73,5 +73,5 @@ class HashMixin: e[0]: {'version': e[1], 'architecture': e[2]} for e in INSTALLED_PACKAGES_REGEX.findall(run([ 'chroot', self.chroot_basedir, 'dpkg-query', '-W', '-f', '${Package}\t${Version}\t${Architecture}\n' - ]).stdout) + ], log=False).stdout) } diff --git a/scale_build/main.py b/scale_build/main.py index d842d75..089e1d3 100644 --- a/scale_build/main.py +++ b/scale_build/main.py @@ -22,8 +22,6 @@ logger = logging.getLogger('scale_build') def setup_logging(): logging.basicConfig(level=logging.DEBUG, format='[%(asctime)s] %(message)s', force=True) - if sys.stdout.isatty(): - coloredlogs.install(logging.DEBUG, fmt='[%(asctime)s] %(message)s', logger=logger) handler = logging.StreamHandler(sys.stderr) handler.setLevel(logging.DEBUG) handler.setFormatter(logging.Formatter('[%(asctime)s] %(message)s')) @@ -33,6 +31,8 @@ def setup_logging(): log_handler.setLevel(logging.DEBUG) logger.addHandler(log_handler) logger.propagate = False + if sys.stdout.isatty(): + coloredlogs.install(logging.DEBUG, fmt='[%(asctime)s] %(message)s', logger=logger) def validate_config(): diff --git a/scale_build/package.py b/scale_build/package.py index 321b766..7cb8856 100644 --- a/scale_build/package.py +++ b/scale_build/package.py @@ -11,7 +11,7 @@ from .clean import clean_bootstrap_logs from .config import PARALLEL_BUILD, PKG_DEBUG from .exceptions import CallError from .packages.order import get_initialized_packages, get_to_build_packages -from .utils.logger import LoggingContext, get_logger +from .utils.logger import LoggingContext from .utils.paths import LOG_DIR, PKG_DIR, PKG_LOG_DIR from .utils.run import interactive_run, run @@ -65,24 +65,26 @@ def build_package(package_queue, to_build, failed, in_progress, built): if package: try: logger.debug('Building %r package', package.name) - package.delete_overlayfs() - package.setup_chroot_basedir() - package.make_overlayfs() - with APT_LOCK: - package.clean_previous_packages() - shutil.copytree(PKG_DIR, package.dpkg_overlay_packages_path) - package._build_impl() + with LoggingContext(os.path.join('packages', package.name), 'w'): + package.delete_overlayfs() + package.setup_chroot_basedir() + package.make_overlayfs() + with APT_LOCK: + package.clean_previous_packages() + shutil.copytree(PKG_DIR, package.dpkg_overlay_packages_path) + package._build_impl() except Exception as e: logger.error('Failed to build %r package', package.name) failed[package.name] = {'package': package, 'exception': e} break else: with APT_LOCK: - package.logger.debug('Building local APT repo Packages.gz...') - run( - f'cd {PKG_DIR} && dpkg-scanpackages . /dev/null | gzip -9c > Packages.gz', - shell=True, logger=package.logger - ) + with LoggingContext(os.path.join('packages', package.name)): + logger.debug('Building local APT repo Packages.gz...') + run( + f'cd {PKG_DIR} && dpkg-scanpackages . /dev/null | gzip -9c > Packages.gz', + shell=True + ) in_progress.pop(package.name) built[package.name] = package logger.info( @@ -97,15 +99,15 @@ def build_package(package_queue, to_build, failed, in_progress, built): def build_packages(): clean_bootstrap_logs() - with LoggingContext(get_logger('build_packages', 'build_packages.log', 'w')): - _build_packages_impl() + _build_packages_impl() def _build_packages_impl(): logger.info('Building packages (%s/build_packages.log)', LOG_DIR) logger.debug('Setting up bootstrap directory') - PackageBootstrapDirectory().setup() + with LoggingContext('build_packages', 'w'): + PackageBootstrapDirectory().setup() logger.debug('Successfully setup bootstrap directory') @@ -162,7 +164,8 @@ def _build_packages_impl(): interactive_run(package['package'].debug_command) finally: for p in failed.values(): - p['package'].delete_overlayfs() + with LoggingContext(os.path.join('packages', p.name)): + p['package'].delete_overlayfs() raise CallError(f'{", ".join(failed)!r} Packages failed to build') diff --git a/scale_build/packages/bootstrap.py b/scale_build/packages/bootstrap.py index 2a37970..90ed532 100644 --- a/scale_build/packages/bootstrap.py +++ b/scale_build/packages/bootstrap.py @@ -9,8 +9,5 @@ class BootstrapMixin: self.logger.debug('Restoring CHROOT_BASEDIR for runs...') os.makedirs(self.tmpfs_path, exist_ok=True) if self.tmpfs: - run( - ['mount', '-t', 'tmpfs', '-o', f'size={self.tmpfs_size}G', 'tmpfs', self.tmpfs_path], - logger=self.logger - ) - PackageBootstrapDirectory(self.logger).restore_cache(self.chroot_base_directory) + run(['mount', '-t', 'tmpfs', '-o', f'size={self.tmpfs_size}G', 'tmpfs', self.tmpfs_path]) + PackageBootstrapDirectory().restore_cache(self.chroot_base_directory) diff --git a/scale_build/packages/build.py b/scale_build/packages/build.py index dc0843d..0687324 100644 --- a/scale_build/packages/build.py +++ b/scale_build/packages/build.py @@ -15,8 +15,8 @@ class BuildPackageMixin: def run_in_chroot(self, command, exception_message=None): run( - f'chroot {self.dpkg_overlay} /bin/bash -c "{command}"', shell=True, logger=self.logger, - exception_msg=exception_message, env={ + f'chroot {self.dpkg_overlay} /bin/bash -c "{command}"', shell=True, exception_msg=exception_message, + env={ **os.environ, **APT_ENV, 'CONFIG_DEBUG_INFO': 'Y', # Build kernel with debug symbols diff --git a/scale_build/packages/package.py b/scale_build/packages/package.py index 324921b..48ba192 100644 --- a/scale_build/packages/package.py +++ b/scale_build/packages/package.py @@ -5,7 +5,7 @@ import shutil from scale_build.exceptions import CallError from scale_build.utils.git_utils import retrieve_git_remote_and_sha, retrieve_git_branch, update_git_manifest -from scale_build.utils.logger import get_logger, LoggingContext +from scale_build.utils.logger import LoggingContext from scale_build.utils.run import run from scale_build.utils.paths import GIT_LOG_PATH, HASH_DIR, PKG_LOG_DIR, SOURCES_DIR @@ -50,7 +50,7 @@ class Package(BootstrapMixin, BuildPackageMixin, BuildCleanMixin, OverlayMixin): self.parent_changed = False self._build_time_dependencies = None self.build_stage = None - self.logger = get_logger(f'{self.name}_package', f'packages/{self.name}.log', 'w') + self.logger = logger self.children = set() self.batch_priority = batch_priority @@ -91,7 +91,7 @@ class Package(BootstrapMixin, BuildPackageMixin, BuildCleanMixin, OverlayMixin): self._binary_packages.append(BinaryPackage(self.name, self.build_depends, self.name, self.name, set())) return self._binary_packages - cp = run([DEPENDS_SCRIPT_PATH, self.debian_control_file_path]) + cp = run([DEPENDS_SCRIPT_PATH, self.debian_control_file_path], log=False) info = json.loads(cp.stdout) default_dependencies = {'kernel'} if self.kernel_module else set() self.build_depends = set( @@ -133,14 +133,14 @@ class Package(BootstrapMixin, BuildPackageMixin, BuildCleanMixin, OverlayMixin): existing_hash = f.read().strip() if source_hash == existing_hash: return run( - ['git', '-C', self.source_path, 'diff-files', '--quiet', '--ignore-submodules'], check=False + ['git', '-C', self.source_path, 'diff-files', '--quiet', '--ignore-submodules'], check=False, log=False ).returncode != 0 else: return True @property def source_hash(self): - return run(['git', '-C', self.source_path, 'rev-parse', '--verify', 'HEAD']).stdout.strip() + return run(['git', '-C', self.source_path, 'rev-parse', '--verify', 'HEAD'], log=False).stdout.strip() @property def rebuild(self):