Build grub with a delay to prioritise other packages

conf/build.manifest

@@ -365,3 +365,4 @@ sources:
   deoptions: nocheck
   generate_version: false
   jobs: 1
+  batch_priority: 150

scale_build/bootstrap/bootstrapdir.py

@@ -28,12 +28,13 @@ class BootstrapDir(CacheMixin, HashMixin):
         self.run(
             ['debootstrap'] + self.deopts + [
                 '--keyring', '/etc/apt/trusted.gpg.d/debian-archive-truenas-automatic.gpg', 'bullseye',
-                CHROOT_BASEDIR, apt_repos['url']
+                self.chroot_basedir, apt_repos['url']
             ]
         )
         self.setup_mounts()
+
         if self.extra_packages_to_install:
-            self.run(['chroot', self.chroot_basedir, 'apt', 'install', 'y'] + self.extra_packages_to_install)
+            self.run(['chroot', self.chroot_basedir, 'apt', 'install', '-y'] + self.extra_packages_to_install)
 
         installed_packages = self.get_packages()
 
@@ -96,18 +97,22 @@ class BootstrapDir(CacheMixin, HashMixin):
 
     def clean_mounts(self):
         for command in (
-            ['umount', '-f', os.path.join(CHROOT_BASEDIR, 'proc')],
-            ['umount', '-f', os.path.join(CHROOT_BASEDIR, 'sys')],
+            ['umount', '-f', os.path.join(self.chroot_basedir, 'proc')],
+            ['umount', '-f', os.path.join(self.chroot_basedir, 'sys')],
         ):
-            self.run(command, check=False)
+            run(command, check=False)
 
-        shutil.rmtree(self.chroot_basedir)
+    def clean_setup(self):
+        self.clean_mounts()
+        shutil.rmtree(self.chroot_basedir, ignore_errors=True)
 
     def __enter__(self):
+        # To ensure we have a clean start
+        self.clean_setup()
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
-        self.clean_mounts()
+        self.clean_setup()
 
 
 class PackageBootstrapDirectory(BootstrapDir):

scale_build/bootstrap/cache.py

@@ -35,7 +35,7 @@ class CacheMixin:
 
     def save_build_cache(self, installed_packages):
         self.logger.debug('Caching CHROOT_BASEDIR for future runs...')
-        self.run(['mksquashfs', self.chroot_basedir, self.cache_filename])
+        self.run(['mksquashfs', self.chroot_basedir, self.cache_file_path])
         self.update_saved_packages_list(installed_packages)
         self.update_mirror_cache()
 

scale_build/bootstrap/hash.py

@@ -75,7 +75,7 @@ class HashMixin:
             return None
 
     def update_saved_packages_list(self, installed_packages):
-        with open(self.packages_file_path, 'w') as f:
+        with open(self.saved_packages_file_path, 'w') as f:
             f.write(json.dumps(installed_packages))
 
     def get_packages(self):
@@ -85,5 +85,3 @@ class HashMixin:
                 'chroot', self.chroot_basedir, 'dpkg-query', '-W', '-f', '${Package}\t${Version}\t${Architecture}\n'
             ]).stdout.decode(errors='ignore'))
         }
-
-

scale_build/checkout.py

@@ -1,7 +1,7 @@
 import logging
-import subprocess
 
 from .config import BRANCH_OVERRIDES, TRY_BRANCH_OVERRIDE
+from .exceptions import CallError
 from .utils.git_utils import branch_exists_in_repository, retrieve_git_remote_and_sha, update_git_manifest
 from .utils.package import get_packages
 
@@ -27,11 +27,11 @@ def checkout_sources():
             retries = 2
             while retries:
                 try:
-                    branch_exists_in_repository(package.origin, TRY_BRANCH_OVERRIDE)
-                except subprocess.CalledProcessError:
+                    if branch_exists_in_repository(package.origin, TRY_BRANCH_OVERRIDE):
+                        gh_override = TRY_BRANCH_OVERRIDE
+                except CallError:
                     retries -= 1
                 else:
-                    gh_override = TRY_BRANCH_OVERRIDE
                     break
 
         package.checkout(gh_override)

scale_build/image/utils.py

@@ -1,6 +1,5 @@
 import os
 
-from scale_build.exceptions import CallError
 from scale_build.utils.environment import APT_ENV
 from scale_build.utils.run import run
 from scale_build.utils.paths import CHROOT_BASEDIR

scale_build/package.py

@@ -9,7 +9,7 @@ from toposort import toposort
 from .bootstrap.bootstrapdir import PackageBootstrapDirectory
 from .clean import clean_bootstrap_logs
 from .config import PARALLEL_BUILD, PKG_DEBUG
-from .packages.order import get_to_build_packages
+from .packages.order import get_initialized_packages, get_to_build_packages
 from .utils.logger import get_logger
 from .utils.paths import LOG_DIR, PKG_DIR, PKG_LOG_DIR
 from .utils.run import interactive_run, run
@@ -112,15 +112,18 @@ def _build_packages_impl():
     shutil.rmtree(PKG_LOG_DIR, ignore_errors=True)
     os.makedirs(PKG_LOG_DIR)
 
-    to_build = get_to_build_packages()
-    logger.debug('Going to build %d packages: %s', len(to_build), ','.join(to_build))
+    all_packages = get_initialized_packages()
+    to_build = get_to_build_packages(all_packages)
     package_queue = queue.Queue()
     in_progress = {}
     failed = {}
-    built = {}
-    update_queue(package_queue, to_build, failed, in_progress, built)
+    built = {p: all_packages[p] for p in set(all_packages) - set(to_build)}
+    if built:
+        logger.debug('%d package(s) do not need to be rebuilt (%s)', len(built), ','.join(built))
+    logger.debug('Going to build %d package(s): %s', len(to_build), ','.join(to_build))
     no_of_tasks = PARALLEL_BUILD if len(to_build) >= PARALLEL_BUILD else len(to_build)
-    logger.debug('Creating %d parallel tasks', no_of_tasks)
+    update_queue(package_queue, to_build, failed, in_progress, built)
+    logger.debug('Creating %d parallel task(s)', no_of_tasks)
     threads = [
         threading.Thread(
             name=f'build_packages_thread_{i + 1}', target=build_package,

scale_build/packages/build.py

@@ -14,7 +14,6 @@ from scale_build.utils.paths import PKG_DIR
 class BuildPackageMixin:
 
     def run_in_chroot(self, command, exception_message=None):
-        exception = CallError if exception_message else None
         run(
             f'chroot {self.dpkg_overlay} /bin/bash -c "{command}"', shell=True, logger=self.logger,
             exception_msg=exception_message, env={

scale_build/packages/order.py

@@ -7,7 +7,7 @@ from scale_build.utils.package import get_packages
 logger = logging.getLogger(__name__)
 
 
-def get_to_build_packages():
+def get_initialized_packages():
     binary_packages = {}
     packages_list = get_packages()
     packages = {}
@@ -27,4 +27,8 @@ def get_to_build_packages():
         for child in package.children:
             packages[child].parent_changed = True
 
-    return {package.name: package for package in packages.values() if package.rebuild}
+    return {package.name: package for package in packages.values()}
+
+
+def get_to_build_packages(packages=None):
+    return {k: v for k, v in (packages or get_initialized_packages()).items() if v.rebuild}

scale_build/packages/package.py

@@ -24,7 +24,7 @@ class Package(BootstrapMixin, BuildPackageMixin, BuildCleanMixin, OverlayMixin):
     def __init__(
         self, name, branch, repo, prebuildcmd=None, kernel_module=False, explicit_deps=None,
         generate_version=True, predepscmd=None, deps_path=None, subdir=None, deoptions=None, jobs=None,
-        buildcmd=None, tmpfs=True, tmpfs_size=12, batch_priority=1000
+        buildcmd=None, tmpfs=True, tmpfs_size=12, batch_priority=100
     ):
         self.name = name
         self.branch = branch
@@ -48,7 +48,7 @@ class Package(BootstrapMixin, BuildPackageMixin, BuildCleanMixin, OverlayMixin):
         self.parent_changed = False
         self._build_time_dependencies = None
         self.build_stage = None
-        self.logger = get_logger(f'{self.name}_package', self.log_file_path, 'w')
+        self.logger = get_logger(f'{self.name}_package', f'packages/{self.name}.log', 'w')
         self.children = set()
         self.batch_priority = batch_priority
 
@@ -166,7 +166,6 @@ class Package(BootstrapMixin, BuildPackageMixin, BuildCleanMixin, OverlayMixin):
         origin_url = self.retrieve_current_remote_origin_and_sha()['url']
         branch = branch_override or self.branch
         git_logger = get_logger(f'checkout_{self.name}', 'git-checkout.log', 'w')
-        git_logger.debug('logged it')
         if branch == self.existing_branch and self.origin == origin_url:
             logger.debug('Updating git repo [%s] (%s)', self.name, GIT_LOG_PATH)
             run(['git', '-C', self.source_path, 'fetch', '--unshallow'], logger=git_logger, check=False)

scale_build/utils/run.py

@@ -27,10 +27,13 @@ def run(*args, **kwargs):
 
     cp = subprocess.CompletedProcess(args, proc.returncode, stdout=stdout, stderr=stderr)
     if check:
-        if cp.returncode and exception_message:
-            raise CallError(f'{exception_message} ({stderr.decode(errors="ignore")}' if stderr else exception_message)
-        else:
-            cp.check_returncode()
+        error_str = exception_message or stderr or ''
+        error_str = error_str.decode(errors='ignore') if isinstance(error_str, bytes) else error_str
+        if cp.returncode:
+            raise CallError(
+                f'Command {" ".join(args) if isinstance(args, list) else args!r} returned exit code '
+                f'{cp.returncode}' + (f' ({error_str})' if error_str else '')
+            )
     return cp
 
 

truenas_install/__main__.py

@@ -52,7 +52,7 @@ def run_command(cmd, **kwargs):
 
 
 def get_partition(disk, partition):
-    paths =[f"/dev/{disk}{partition}", f"/dev/{disk}p{partition}"]
+    paths = [f"/dev/{disk}{partition}", f"/dev/{disk}p{partition}"]
     for path in paths:
         if os.path.exists(path):
             return path