Allow building packages in parallel

scale_build/config.py

@@ -9,6 +9,7 @@ BUILD_TIME_OBJ = datetime.fromtimestamp(BUILD_TIME)
 BUILDER_DIR = os.getenv('BUILDER_DIR', './')
 BRANCH_OVERRIDES = {k[:-(len('_OVERRIDE'))]: v for k, v in os.environ.items() if k.endswith('_OVERRIDE')}
 CODE_NAME = os.getenv('CODENAME', 'Angelfish')
+PARALLEL_BUILD = int(os.getenv('PARALLEL_BUILDS', os.cpu_count() / 4))
 PKG_DEBUG = os.getenv('PKG_DEBUG', False)
 TRAIN = os.getenv('TRUENAS_TRAIN', f'TrueNAS-SCALE-{CODE_NAME}-Nightlies')
 TRY_BRANCH_OVERRIDE = os.getenv('TRY_BRANCH_OVERRIDE')

scale_build/package.py

@@ -1,16 +1,94 @@
 import logging
 import os
+import queue
 import shutil
+import threading
+
+from toposort import toposort
 
 from .bootstrap.configure import make_bootstrapdir
 from .clean import clean_bootstrap_logs
-from .config import PKG_DEBUG
+from .config import PARALLEL_BUILD
 from .packages.order import get_to_build_packages
-from .utils.paths import LOG_DIR, PKG_LOG_DIR
+from .utils.paths import PKG_DIR, PKG_LOG_DIR
+from .utils.run import run
 
 
 logger = logging.getLogger(__name__)
 
+APT_LOCK = threading.Lock()
+PACKAGE_BUILD_LOCK = threading.Lock()
+
+
+def update_queue(package_queue, to_build_orig, failed, in_progress, built):
+    if failed:
+        # If we have failure(s), there is no point in continuing
+        return
+
+    to_build = {k: v for k, v in to_build_orig.items()}
+    to_remove = set()
+    for pkg_name, package in in_progress.items():
+        for child in package.children:
+            to_remove.add(child)
+
+    for rm in to_remove:
+        to_build.pop(rm, None)
+
+    deps_mapping = {
+        p.name: {d for d in p.build_time_dependencies() if d not in built} for p in list(to_build.values())
+    }
+    sorted_ordering = [list(deps) for deps in toposort(deps_mapping)]
+
+    for item in filter(
+        lambda i: i in to_build and i not in in_progress and i not in package_queue.queue and i not in built,
+        sorted_ordering[0] if sorted_ordering else []
+    ):
+        package_queue.put(to_build_orig.pop(item))
+
+
+def build_package(package_queue, to_build, failed, in_progress, built):
+    while True:
+        if not failed and (to_build or package_queue.queue):
+            try:
+                package = package_queue.get(timeout=5)
+            except queue.Empty:
+                package = None
+            else:
+                in_progress[package.name] = package
+        else:
+            break
+
+        if package:
+            try:
+                logger.debug('Building %r package', package.name)
+                package.delete_overlayfs()
+                package.setup_chroot_basedir()
+                package.make_overlayfs()
+                with APT_LOCK:
+                    package.clean_previous_packages()
+                    shutil.copytree(PKG_DIR, package.dpkg_overlay_packages_path)
+                package._build_impl()
+            except Exception as e:
+                failed[package.name] = {'package': package, 'exception': e}
+                break
+            else:
+                with APT_LOCK:
+                    package.logger.debug('Building local APT repo Packages.gz...')
+                    run(
+                        f'cd {PKG_DIR} && dpkg-scanpackages . /dev/null | gzip -9c > Packages.gz',
+                        shell=True, logger=package.logger
+                    )
+                in_progress.pop(package.name)
+                built[package.name] = package
+                logger.debug(
+                    'Successfully built %r package (Remaining %d packages)', package.name,
+                    len(to_build) + package_queue.qsize() + len(in_progress)
+                )
+
+        with PACKAGE_BUILD_LOCK:
+            if not package:
+                update_queue(package_queue, to_build, failed, in_progress, built)
+
 
 def build_packages():
     clean_bootstrap_logs()
@@ -19,19 +97,33 @@ def build_packages():
 
 def _build_packages_impl():
     logger.debug('Building packages')
+    logger.debug('Setting up bootstrap directory')
     make_bootstrapdir('package')
+    logger.debug('Successfully setup bootstrap directory')
 
     shutil.rmtree(PKG_LOG_DIR, ignore_errors=True)
     os.makedirs(PKG_LOG_DIR)
 
-    packages = get_to_build_packages()
+    to_build = get_to_build_packages()
-    for pkg_name, package in packages.items():
+    package_queue = queue.Queue()
-        logger.debug('Building package [%s] (%s/packages/%s.log)', pkg_name, LOG_DIR, pkg_name)
+    in_progress = {}
-        try:
+    failed = {}
-            package.build()
+    built = {}
-        except Exception:
+    update_queue(package_queue, to_build, failed, in_progress, built)
-            logger.error('Failed to build %r package', exc_info=True)
+    threads = [
-            package.delete_overlayfs()
+        threading.Thread(
-            raise
+            name=f'build_packages_thread_{i + 1}', target=build_package,
+            args=(package_queue, to_build, failed, in_progress, built)
+        ) for i in range(PARALLEL_BUILD)
+    ]
+    for thread in threads:
+        thread.start()
+    for thread in threads:
+        thread.join()
 
-    logger.debug('Success! Done building packages')
+    if failed:
+        for p in failed.values():
+            p['package'].delete_overlayfs()
+        logger.error('Failed to build %r package(s)', ', '.join(failed))
+    else:
+        logger.debug('Success! Done building packages')

scale_build/packages/build.py

@@ -51,13 +51,13 @@ class BuildPackageMixin:
         # 10) Generate version
         # 11) Execute relevant building commands
         # 12) Save
-        self._build_impl()
-
-    def _build_impl(self):
         self.delete_overlayfs()
         self.setup_chroot_basedir()
         self.make_overlayfs()
         self.clean_previous_packages()
+        self._build_impl()
+
+    def _build_impl(self):
         shutil.copytree(self.source_path, self.source_in_chroot, dirs_exist_ok=True, symlinks=True)
 
         # TODO: Remove me please
@@ -153,10 +153,6 @@ class BuildPackageMixin:
         with open(self.pkglist_hash_file_path, 'w') as f:
             f.write('\n'.join(built_packages))
 
-        # Update the local APT repo
-        self.logger.debug('Building local APT repo Packages.gz...')
-        self.run_in_chroot('cd /packages && dpkg-scanpackages . /dev/null | gzip -9c > Packages.gz')
-
         with open(self.hash_path, 'w') as f:
             f.write(self.source_hash)
 

scale_build/packages/order.py

@@ -1,7 +1,6 @@
 import errno
 import logging
 
-from collections import defaultdict
 from scale_build.exceptions import CallError
 from scale_build.utils.package import get_packages
 
@@ -23,13 +22,12 @@ def get_to_build_packages():
         for binary_package in package.binary_packages:
             binary_packages[binary_package.name] = binary_package
 
-    parent_mapping = defaultdict(set)
     for pkg_name, package in packages.items():
-        for dep in package.build_time_dependencies(binary_packages):
+        for dep in filter(lambda d: d in packages, package.build_time_dependencies(binary_packages)):
-            parent_mapping[dep].add(pkg_name)
+            packages[dep].children.add(pkg_name)
 
     for pkg_name, package in filter(lambda i: i[1].hash_changed, packages.items()):
-        for child in parent_mapping[pkg_name]:
+        for child in package.children:
             packages[child].parent_changed = True
 
     return {package.name: package for package in packages.values() if package.rebuild}

scale_build/packages/overlay.py

@@ -3,7 +3,7 @@ import shutil
 
 from scale_build.exceptions import CallError
 from scale_build.utils.run import run
-from scale_build.utils.paths import CACHE_DIR, PKG_DIR, TMP_DIR, TMPFS
+from scale_build.utils.paths import TMP_DIR, TMPFS
 
 
 class OverlayMixin:
@@ -48,12 +48,6 @@ class OverlayMixin:
              ], 'Failed overlayfs'),
             (['mount', 'proc', os.path.join(self.dpkg_overlay, 'proc'), '-t', 'proc'], 'Failed mount proc'),
             (['mount', 'sysfs', os.path.join(self.dpkg_overlay, 'sys'), '-t', 'sysfs'], 'Failed mount sysfs'),
-            (['mount', '--bind', PKG_DIR, self.dpkg_overlay_packages_path], 'Failed mount --bind /packages',
-             self.dpkg_overlay_packages_path),
-            (
-                ['mount', '--bind', os.path.join(CACHE_DIR, 'apt'), os.path.join(self.dpkg_overlay, 'var/cache/apt')],
-                'Failed mount --bind /var/cache/apt',
-            ),
             (
                 ['mount', '--bind', self.sources_overlay, self.source_in_chroot],
                 'Failed mount --bind /dpkg-src', self.source_in_chroot
@@ -69,8 +63,6 @@ class OverlayMixin:
 
     def delete_overlayfs(self):
         for command in (
-            ['umount', '-f', os.path.join(self.dpkg_overlay, 'var/cache/apt')],
-            ['umount', '-f', self.dpkg_overlay_packages_path],
             ['umount', '-f', os.path.join(self.dpkg_overlay, 'proc')],
             ['umount', '-f', os.path.join(self.dpkg_overlay, 'sys')],
             ['umount', '-f', self.dpkg_overlay],

scale_build/packages/package.py

@@ -45,13 +45,17 @@ class Package(BootstrapMixin, BuildPackageMixin, BuildCleanMixin, OverlayMixin):
         self.build_depends = set()
         self.source_package = None
         self.parent_changed = False
-        self._build_time_dependencies = set()
+        self._build_time_dependencies = None
         self.build_stage = None
         self.logger = logging.getLogger(f'{self.name}_package')
         self.logger.setLevel('DEBUG')
         self.logger.handlers = []
         self.logger.propagate = False
         self.logger.addHandler(logging.FileHandler(self.log_file_path, mode='w'))
+        self.children = set()
+
+    def __eq__(self, other):
+        return other == self.name if isinstance(other, str) else self.name == other.name
 
     @property
     def log_file_path(self):
@@ -105,7 +109,7 @@ class Package(BootstrapMixin, BuildPackageMixin, BuildCleanMixin, OverlayMixin):
         return self._binary_packages
 
     def build_time_dependencies(self, all_binary_packages=None):
-        if self._build_time_dependencies:
+        if self._build_time_dependencies is not None:
             return self._build_time_dependencies
         elif not all_binary_packages:
             raise CallError('Binary packages must be specified when computing build time dependencies')

scale_build/utils/paths.py

@@ -17,7 +17,7 @@ GIT_MANIFEST_PATH = os.path.join(LOG_DIR, 'GITMANIFEST')
 GIT_LOG_PATH = os.path.join(LOG_DIR, 'git-checkout.log')
 HASH_DIR = os.path.join(TMP_DIR, 'pkghashes')
 MANIFEST = os.path.join(BUILDER_DIR, 'conf/build.manifest')
-PKG_DIR = os.path.join(BUILDER_DIR, 'pkgdir')
+PKG_DIR = os.path.join(TMP_DIR, 'pkgdir')
 PKG_LOG_DIR = os.path.join(LOG_DIR, 'packages')
 RELEASE_DIR = os.path.join(TMP_DIR, 'release')
 REQUIRED_RAM = 16  # GB