Update package creation target to utilise new logging method

scale_build/bootstrap/bootstrapdir.py

@@ -37,7 +37,7 @@ class BootstrapDir(CacheMixin, HashMixin):
         self.add_trusted_apt_key()
         manifest = get_manifest()
         apt_repos = manifest['apt-repos']
-        self.run(
+        run(
             ['debootstrap'] + self.deopts + [
                 '--keyring', '/etc/apt/trusted.gpg.d/debian-archive-truenas-automatic.gpg', manifest['debian_release'],
                 self.chroot_basedir, apt_repos['url']
@@ -46,7 +46,7 @@ class BootstrapDir(CacheMixin, HashMixin):
         self.setup_mounts()
 
         if self.extra_packages_to_install:
-            self.run(['chroot', self.chroot_basedir, 'apt', 'install', '-y'] + self.extra_packages_to_install)
+            run(['chroot', self.chroot_basedir, 'apt', 'install', '-y'] + self.extra_packages_to_install)
 
         installed_packages = self.get_packages()
 
@@ -65,7 +65,7 @@ class BootstrapDir(CacheMixin, HashMixin):
         for repo in apt_repos['additional']:
             self.logger.debug('Adding additional repo: %r', repo['url'])
             shutil.copy(os.path.join(BUILDER_DIR, repo['key']), os.path.join(self.chroot_basedir, 'apt.key'))
-            self.run(['chroot', self.chroot_basedir, 'apt-key', 'add', '/apt.key'])
+            run(['chroot', self.chroot_basedir, 'apt-key', 'add', '/apt.key'])
             os.unlink(os.path.join(self.chroot_basedir, 'apt.key'))
             apt_sources.append(f'deb {repo["url"]} {repo["distribution"]} {repo["component"]}')
 
@@ -73,7 +73,7 @@ class BootstrapDir(CacheMixin, HashMixin):
             f.write('\n'.join(apt_sources))
 
         # Update apt
-        self.run(['chroot', self.chroot_basedir, 'apt', 'update'])
+        run(['chroot', self.chroot_basedir, 'apt', 'update'])
 
         # Put our local package up at the top of the food chain
         apt_sources.insert(0, 'deb [trusted=yes] file:/packages /')
@@ -87,14 +87,11 @@ class BootstrapDir(CacheMixin, HashMixin):
         pass
 
     def add_trusted_apt_key(self):
-        self.run([
+        run([
             'apt-key', '--keyring', '/etc/apt/trusted.gpg.d/debian-archive-truenas-automatic.gpg', 'add',
             os.path.join(BUILDER_DIR, 'keys/truenas.gpg')
         ])
 
-    def run(self, *args, **kwargs):
-        return run(*args, logger=self.logger, **kwargs)
-
     @property
     def extra_packages_to_install(self):
         raise NotImplementedError
@@ -104,8 +101,8 @@ class BootstrapDir(CacheMixin, HashMixin):
         raise NotImplementedError
 
     def setup_mounts(self):
-        self.run(['mount', 'proc', os.path.join(self.chroot_basedir, 'proc'), '-t', 'proc'])
-        self.run(['mount', 'sysfs', os.path.join(self.chroot_basedir, 'sys'), '-t', 'sysfs'])
+        run(['mount', 'proc', os.path.join(self.chroot_basedir, 'proc'), '-t', 'proc'])
+        run(['mount', 'sysfs', os.path.join(self.chroot_basedir, 'sys'), '-t', 'sysfs'])
 
     def clean_mounts(self):
         for command in (

scale_build/bootstrap/cache.py

@@ -1,6 +1,7 @@
 import os
 
 from scale_build.utils.paths import CACHE_DIR
+from scale_build.utils.run import run
 
 from .hash import get_all_repo_hash
 
@@ -35,7 +36,7 @@ class CacheMixin:
 
     def save_build_cache(self, installed_packages):
         self.logger.debug('Caching CHROOT_BASEDIR for future runs...')
-        self.run(['mksquashfs', self.chroot_basedir, self.cache_file_path])
+        run(['mksquashfs', self.chroot_basedir, self.cache_file_path])
         self.update_saved_packages_list(installed_packages)
         self.update_mirror_cache()
 
@@ -61,4 +62,4 @@ class CacheMixin:
         return self.installed_packages_in_cache != self.get_packages()
 
     def restore_cache(self, chroot_basedir):
-        self.run(['unsquashfs', '-f', '-d', chroot_basedir, self.cache_file_path])
+        run(['unsquashfs', '-f', '-d', chroot_basedir, self.cache_file_path])

scale_build/bootstrap/hash.py

@@ -73,5 +73,5 @@ class HashMixin:
             e[0]: {'version': e[1], 'architecture': e[2]}
             for e in INSTALLED_PACKAGES_REGEX.findall(run([
                 'chroot', self.chroot_basedir, 'dpkg-query', '-W', '-f', '${Package}\t${Version}\t${Architecture}\n'
-            ]).stdout)
+            ], log=False).stdout)
         }

scale_build/main.py

@@ -22,8 +22,6 @@ logger = logging.getLogger('scale_build')
 
 def setup_logging():
     logging.basicConfig(level=logging.DEBUG, format='[%(asctime)s] %(message)s', force=True)
-    if sys.stdout.isatty():
-        coloredlogs.install(logging.DEBUG, fmt='[%(asctime)s] %(message)s', logger=logger)
     handler = logging.StreamHandler(sys.stderr)
     handler.setLevel(logging.DEBUG)
     handler.setFormatter(logging.Formatter('[%(asctime)s] %(message)s'))
@@ -33,6 +31,8 @@ def setup_logging():
     log_handler.setLevel(logging.DEBUG)
     logger.addHandler(log_handler)
     logger.propagate = False
+    if sys.stdout.isatty():
+        coloredlogs.install(logging.DEBUG, fmt='[%(asctime)s] %(message)s', logger=logger)
 
 
 def validate_config():

scale_build/package.py

@@ -11,7 +11,7 @@ from .clean import clean_bootstrap_logs
 from .config import PARALLEL_BUILD, PKG_DEBUG
 from .exceptions import CallError
 from .packages.order import get_initialized_packages, get_to_build_packages
-from .utils.logger import LoggingContext, get_logger
+from .utils.logger import LoggingContext
 from .utils.paths import LOG_DIR, PKG_DIR, PKG_LOG_DIR
 from .utils.run import interactive_run, run
 
@@ -65,24 +65,26 @@ def build_package(package_queue, to_build, failed, in_progress, built):
         if package:
             try:
                 logger.debug('Building %r package', package.name)
-                package.delete_overlayfs()
-                package.setup_chroot_basedir()
-                package.make_overlayfs()
-                with APT_LOCK:
-                    package.clean_previous_packages()
-                    shutil.copytree(PKG_DIR, package.dpkg_overlay_packages_path)
-                package._build_impl()
+                with LoggingContext(os.path.join('packages', package.name), 'w'):
+                    package.delete_overlayfs()
+                    package.setup_chroot_basedir()
+                    package.make_overlayfs()
+                    with APT_LOCK:
+                        package.clean_previous_packages()
+                        shutil.copytree(PKG_DIR, package.dpkg_overlay_packages_path)
+                    package._build_impl()
             except Exception as e:
                 logger.error('Failed to build %r package', package.name)
                 failed[package.name] = {'package': package, 'exception': e}
                 break
             else:
                 with APT_LOCK:
-                    package.logger.debug('Building local APT repo Packages.gz...')
-                    run(
-                        f'cd {PKG_DIR} && dpkg-scanpackages . /dev/null | gzip -9c > Packages.gz',
-                        shell=True, logger=package.logger
-                    )
+                    with LoggingContext(os.path.join('packages', package.name)):
+                        logger.debug('Building local APT repo Packages.gz...')
+                        run(
+                            f'cd {PKG_DIR} && dpkg-scanpackages . /dev/null | gzip -9c > Packages.gz',
+                            shell=True
+                        )
                 in_progress.pop(package.name)
                 built[package.name] = package
                 logger.info(
@@ -97,15 +99,15 @@ def build_package(package_queue, to_build, failed, in_progress, built):
 
 def build_packages():
     clean_bootstrap_logs()
-    with LoggingContext(get_logger('build_packages', 'build_packages.log', 'w')):
-        _build_packages_impl()
+    _build_packages_impl()
 
 
 def _build_packages_impl():
     logger.info('Building packages (%s/build_packages.log)', LOG_DIR)
     logger.debug('Setting up bootstrap directory')
 
-    PackageBootstrapDirectory().setup()
+    with LoggingContext('build_packages', 'w'):
+        PackageBootstrapDirectory().setup()
 
     logger.debug('Successfully setup bootstrap directory')
 
@@ -162,7 +164,8 @@ def _build_packages_impl():
                         interactive_run(package['package'].debug_command)
         finally:
             for p in failed.values():
-                p['package'].delete_overlayfs()
+                with LoggingContext(os.path.join('packages', p.name)):
+                    p['package'].delete_overlayfs()
 
         raise CallError(f'{", ".join(failed)!r} Packages failed to build')
 

scale_build/packages/bootstrap.py

@@ -9,8 +9,5 @@ class BootstrapMixin:
         self.logger.debug('Restoring CHROOT_BASEDIR for runs...')
         os.makedirs(self.tmpfs_path, exist_ok=True)
         if self.tmpfs:
-            run(
-                ['mount', '-t', 'tmpfs', '-o', f'size={self.tmpfs_size}G', 'tmpfs', self.tmpfs_path],
-                logger=self.logger
-            )
-        PackageBootstrapDirectory(self.logger).restore_cache(self.chroot_base_directory)
+            run(['mount', '-t', 'tmpfs', '-o', f'size={self.tmpfs_size}G', 'tmpfs', self.tmpfs_path])
+        PackageBootstrapDirectory().restore_cache(self.chroot_base_directory)

scale_build/packages/build.py

@@ -15,8 +15,8 @@ class BuildPackageMixin:
 
     def run_in_chroot(self, command, exception_message=None):
         run(
-            f'chroot {self.dpkg_overlay} /bin/bash -c "{command}"', shell=True, logger=self.logger,
-            exception_msg=exception_message, env={
+            f'chroot {self.dpkg_overlay} /bin/bash -c "{command}"', shell=True, exception_msg=exception_message,
+            env={
                 **os.environ,
                 **APT_ENV,
                 'CONFIG_DEBUG_INFO': 'Y',  # Build kernel with debug symbols

scale_build/packages/package.py

@@ -5,7 +5,7 @@ import shutil
 
 from scale_build.exceptions import CallError
 from scale_build.utils.git_utils import retrieve_git_remote_and_sha, retrieve_git_branch, update_git_manifest
-from scale_build.utils.logger import get_logger, LoggingContext
+from scale_build.utils.logger import LoggingContext
 from scale_build.utils.run import run
 from scale_build.utils.paths import GIT_LOG_PATH, HASH_DIR, PKG_LOG_DIR, SOURCES_DIR
 
@@ -50,7 +50,7 @@ class Package(BootstrapMixin, BuildPackageMixin, BuildCleanMixin, OverlayMixin):
         self.parent_changed = False
         self._build_time_dependencies = None
         self.build_stage = None
-        self.logger = get_logger(f'{self.name}_package', f'packages/{self.name}.log', 'w')
+        self.logger = logger
         self.children = set()
         self.batch_priority = batch_priority
 
@@ -91,7 +91,7 @@ class Package(BootstrapMixin, BuildPackageMixin, BuildCleanMixin, OverlayMixin):
             self._binary_packages.append(BinaryPackage(self.name, self.build_depends, self.name, self.name, set()))
             return self._binary_packages
 
-        cp = run([DEPENDS_SCRIPT_PATH, self.debian_control_file_path])
+        cp = run([DEPENDS_SCRIPT_PATH, self.debian_control_file_path], log=False)
         info = json.loads(cp.stdout)
         default_dependencies = {'kernel'} if self.kernel_module else set()
         self.build_depends = set(
@@ -133,14 +133,14 @@ class Package(BootstrapMixin, BuildPackageMixin, BuildCleanMixin, OverlayMixin):
                 existing_hash = f.read().strip()
         if source_hash == existing_hash:
             return run(
-                ['git', '-C', self.source_path, 'diff-files', '--quiet', '--ignore-submodules'], check=False
+                ['git', '-C', self.source_path, 'diff-files', '--quiet', '--ignore-submodules'], check=False, log=False
             ).returncode != 0
         else:
             return True
 
     @property
     def source_hash(self):
-        return run(['git', '-C', self.source_path, 'rev-parse', '--verify', 'HEAD']).stdout.strip()
+        return run(['git', '-C', self.source_path, 'rev-parse', '--verify', 'HEAD'], log=False).stdout.strip()
 
     @property
     def rebuild(self):