From 4b50d1688d2024de439690a63929f3675d89c256 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Date: Tue, 20 Apr 2021 07:11:13 +0500 Subject: [PATCH] Allow building packages in parallel --- scale_build/config.py | 1 + scale_build/package.py | 116 ++++++++++++++++++++++++++++---- scale_build/packages/build.py | 10 +-- scale_build/packages/order.py | 8 +-- scale_build/packages/overlay.py | 10 +-- scale_build/packages/package.py | 8 ++- scale_build/utils/paths.py | 2 +- 7 files changed, 119 insertions(+), 36 deletions(-) diff --git a/scale_build/config.py b/scale_build/config.py index b4e592b..9f3ecd6 100644 --- a/scale_build/config.py +++ b/scale_build/config.py @@ -9,6 +9,7 @@ BUILD_TIME_OBJ = datetime.fromtimestamp(BUILD_TIME) BUILDER_DIR = os.getenv('BUILDER_DIR', './') BRANCH_OVERRIDES = {k[:-(len('_OVERRIDE'))]: v for k, v in os.environ.items() if k.endswith('_OVERRIDE')} CODE_NAME = os.getenv('CODENAME', 'Angelfish') +PARALLEL_BUILD = int(os.getenv('PARALLEL_BUILDS', os.cpu_count() / 4)) PKG_DEBUG = os.getenv('PKG_DEBUG', False) TRAIN = os.getenv('TRUENAS_TRAIN', f'TrueNAS-SCALE-{CODE_NAME}-Nightlies') TRY_BRANCH_OVERRIDE = os.getenv('TRY_BRANCH_OVERRIDE') diff --git a/scale_build/package.py b/scale_build/package.py index a139a0d..a26c61e 100644 --- a/scale_build/package.py +++ b/scale_build/package.py @@ -1,16 +1,94 @@ import logging import os +import queue import shutil +import threading + +from toposort import toposort from .bootstrap.configure import make_bootstrapdir from .clean import clean_bootstrap_logs -from .config import PKG_DEBUG +from .config import PARALLEL_BUILD from .packages.order import get_to_build_packages -from .utils.paths import LOG_DIR, PKG_LOG_DIR +from .utils.paths import PKG_DIR, PKG_LOG_DIR +from .utils.run import run logger = logging.getLogger(__name__) +APT_LOCK = threading.Lock() +PACKAGE_BUILD_LOCK = threading.Lock() + + +def update_queue(package_queue, to_build_orig, failed, in_progress, built): + if failed: + # If we have failure(s), there is no point in continuing + return + + to_build = {k: v for k, v in to_build_orig.items()} + to_remove = set() + for pkg_name, package in in_progress.items(): + for child in package.children: + to_remove.add(child) + + for rm in to_remove: + to_build.pop(rm, None) + + deps_mapping = { + p.name: {d for d in p.build_time_dependencies() if d not in built} for p in list(to_build.values()) + } + sorted_ordering = [list(deps) for deps in toposort(deps_mapping)] + + for item in filter( + lambda i: i in to_build and i not in in_progress and i not in package_queue.queue and i not in built, + sorted_ordering[0] if sorted_ordering else [] + ): + package_queue.put(to_build_orig.pop(item)) + + +def build_package(package_queue, to_build, failed, in_progress, built): + while True: + if not failed and (to_build or package_queue.queue): + try: + package = package_queue.get(timeout=5) + except queue.Empty: + package = None + else: + in_progress[package.name] = package + else: + break + + if package: + try: + logger.debug('Building %r package', package.name) + package.delete_overlayfs() + package.setup_chroot_basedir() + package.make_overlayfs() + with APT_LOCK: + package.clean_previous_packages() + shutil.copytree(PKG_DIR, package.dpkg_overlay_packages_path) + package._build_impl() + except Exception as e: + failed[package.name] = {'package': package, 'exception': e} + break + else: + with APT_LOCK: + package.logger.debug('Building local APT repo Packages.gz...') + run( + f'cd {PKG_DIR} && dpkg-scanpackages . /dev/null | gzip -9c > Packages.gz', + shell=True, logger=package.logger + ) + in_progress.pop(package.name) + built[package.name] = package + logger.debug( + 'Successfully built %r package (Remaining %d packages)', package.name, + len(to_build) + package_queue.qsize() + len(in_progress) + ) + + with PACKAGE_BUILD_LOCK: + if not package: + update_queue(package_queue, to_build, failed, in_progress, built) + def build_packages(): clean_bootstrap_logs() @@ -19,19 +97,33 @@ def build_packages(): def _build_packages_impl(): logger.debug('Building packages') + logger.debug('Setting up bootstrap directory') make_bootstrapdir('package') + logger.debug('Successfully setup bootstrap directory') shutil.rmtree(PKG_LOG_DIR, ignore_errors=True) os.makedirs(PKG_LOG_DIR) - packages = get_to_build_packages() - for pkg_name, package in packages.items(): - logger.debug('Building package [%s] (%s/packages/%s.log)', pkg_name, LOG_DIR, pkg_name) - try: - package.build() - except Exception: - logger.error('Failed to build %r package', exc_info=True) - package.delete_overlayfs() - raise + to_build = get_to_build_packages() + package_queue = queue.Queue() + in_progress = {} + failed = {} + built = {} + update_queue(package_queue, to_build, failed, in_progress, built) + threads = [ + threading.Thread( + name=f'build_packages_thread_{i + 1}', target=build_package, + args=(package_queue, to_build, failed, in_progress, built) + ) for i in range(PARALLEL_BUILD) + ] + for thread in threads: + thread.start() + for thread in threads: + thread.join() - logger.debug('Success! Done building packages') + if failed: + for p in failed.values(): + p['package'].delete_overlayfs() + logger.error('Failed to build %r package(s)', ', '.join(failed)) + else: + logger.debug('Success! Done building packages') diff --git a/scale_build/packages/build.py b/scale_build/packages/build.py index da0f1e2..97971c1 100644 --- a/scale_build/packages/build.py +++ b/scale_build/packages/build.py @@ -51,13 +51,13 @@ class BuildPackageMixin: # 10) Generate version # 11) Execute relevant building commands # 12) Save - self._build_impl() - - def _build_impl(self): self.delete_overlayfs() self.setup_chroot_basedir() self.make_overlayfs() self.clean_previous_packages() + self._build_impl() + + def _build_impl(self): shutil.copytree(self.source_path, self.source_in_chroot, dirs_exist_ok=True, symlinks=True) # TODO: Remove me please @@ -153,10 +153,6 @@ class BuildPackageMixin: with open(self.pkglist_hash_file_path, 'w') as f: f.write('\n'.join(built_packages)) - # Update the local APT repo - self.logger.debug('Building local APT repo Packages.gz...') - self.run_in_chroot('cd /packages && dpkg-scanpackages . /dev/null | gzip -9c > Packages.gz') - with open(self.hash_path, 'w') as f: f.write(self.source_hash) diff --git a/scale_build/packages/order.py b/scale_build/packages/order.py index 49e6c5e..363745b 100644 --- a/scale_build/packages/order.py +++ b/scale_build/packages/order.py @@ -1,7 +1,6 @@ import errno import logging -from collections import defaultdict from scale_build.exceptions import CallError from scale_build.utils.package import get_packages @@ -23,13 +22,12 @@ def get_to_build_packages(): for binary_package in package.binary_packages: binary_packages[binary_package.name] = binary_package - parent_mapping = defaultdict(set) for pkg_name, package in packages.items(): - for dep in package.build_time_dependencies(binary_packages): - parent_mapping[dep].add(pkg_name) + for dep in filter(lambda d: d in packages, package.build_time_dependencies(binary_packages)): + packages[dep].children.add(pkg_name) for pkg_name, package in filter(lambda i: i[1].hash_changed, packages.items()): - for child in parent_mapping[pkg_name]: + for child in package.children: packages[child].parent_changed = True return {package.name: package for package in packages.values() if package.rebuild} diff --git a/scale_build/packages/overlay.py b/scale_build/packages/overlay.py index 6a4aad7..1e67db9 100644 --- a/scale_build/packages/overlay.py +++ b/scale_build/packages/overlay.py @@ -3,7 +3,7 @@ import shutil from scale_build.exceptions import CallError from scale_build.utils.run import run -from scale_build.utils.paths import CACHE_DIR, PKG_DIR, TMP_DIR, TMPFS +from scale_build.utils.paths import TMP_DIR, TMPFS class OverlayMixin: @@ -48,12 +48,6 @@ class OverlayMixin: ], 'Failed overlayfs'), (['mount', 'proc', os.path.join(self.dpkg_overlay, 'proc'), '-t', 'proc'], 'Failed mount proc'), (['mount', 'sysfs', os.path.join(self.dpkg_overlay, 'sys'), '-t', 'sysfs'], 'Failed mount sysfs'), - (['mount', '--bind', PKG_DIR, self.dpkg_overlay_packages_path], 'Failed mount --bind /packages', - self.dpkg_overlay_packages_path), - ( - ['mount', '--bind', os.path.join(CACHE_DIR, 'apt'), os.path.join(self.dpkg_overlay, 'var/cache/apt')], - 'Failed mount --bind /var/cache/apt', - ), ( ['mount', '--bind', self.sources_overlay, self.source_in_chroot], 'Failed mount --bind /dpkg-src', self.source_in_chroot @@ -69,8 +63,6 @@ class OverlayMixin: def delete_overlayfs(self): for command in ( - ['umount', '-f', os.path.join(self.dpkg_overlay, 'var/cache/apt')], - ['umount', '-f', self.dpkg_overlay_packages_path], ['umount', '-f', os.path.join(self.dpkg_overlay, 'proc')], ['umount', '-f', os.path.join(self.dpkg_overlay, 'sys')], ['umount', '-f', self.dpkg_overlay], diff --git a/scale_build/packages/package.py b/scale_build/packages/package.py index e0e3b76..16a28d9 100644 --- a/scale_build/packages/package.py +++ b/scale_build/packages/package.py @@ -45,13 +45,17 @@ class Package(BootstrapMixin, BuildPackageMixin, BuildCleanMixin, OverlayMixin): self.build_depends = set() self.source_package = None self.parent_changed = False - self._build_time_dependencies = set() + self._build_time_dependencies = None self.build_stage = None self.logger = logging.getLogger(f'{self.name}_package') self.logger.setLevel('DEBUG') self.logger.handlers = [] self.logger.propagate = False self.logger.addHandler(logging.FileHandler(self.log_file_path, mode='w')) + self.children = set() + + def __eq__(self, other): + return other == self.name if isinstance(other, str) else self.name == other.name @property def log_file_path(self): @@ -105,7 +109,7 @@ class Package(BootstrapMixin, BuildPackageMixin, BuildCleanMixin, OverlayMixin): return self._binary_packages def build_time_dependencies(self, all_binary_packages=None): - if self._build_time_dependencies: + if self._build_time_dependencies is not None: return self._build_time_dependencies elif not all_binary_packages: raise CallError('Binary packages must be specified when computing build time dependencies') diff --git a/scale_build/utils/paths.py b/scale_build/utils/paths.py index 59a58da..afd8d20 100644 --- a/scale_build/utils/paths.py +++ b/scale_build/utils/paths.py @@ -17,7 +17,7 @@ GIT_MANIFEST_PATH = os.path.join(LOG_DIR, 'GITMANIFEST') GIT_LOG_PATH = os.path.join(LOG_DIR, 'git-checkout.log') HASH_DIR = os.path.join(TMP_DIR, 'pkghashes') MANIFEST = os.path.join(BUILDER_DIR, 'conf/build.manifest') -PKG_DIR = os.path.join(BUILDER_DIR, 'pkgdir') +PKG_DIR = os.path.join(TMP_DIR, 'pkgdir') PKG_LOG_DIR = os.path.join(LOG_DIR, 'packages') RELEASE_DIR = os.path.join(TMP_DIR, 'release') REQUIRED_RAM = 16 # GB