TrueNAS/scale-build
1
0
Fork 0
mirror of https://github.com/truenas/scale-build.git synced 2025-12-24 21:07:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

Allow building packages in parallel

Browse Source
This commit is contained in:
Waqar Ahmed
2021-04-20 07:11:13 +05:00
committed by Waqar Ahmed
parent de59913926
commit 4b50d1688d
7 changed files with 119 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
scale_build/config.py
View File

@@ -9,6 +9,7 @@ BUILD_TIME_OBJ = datetime.fromtimestamp(BUILD_TIME)
BUILDER_DIR = os.getenv('BUILDER_DIR', './')
BRANCH_OVERRIDES = {k[:-(len('_OVERRIDE'))]: v for k, v in os.environ.items() if k.endswith('_OVERRIDE')}
CODE_NAME = os.getenv('CODENAME', 'Angelfish')
PARALLEL_BUILD = int(os.getenv('PARALLEL_BUILDS', os.cpu_count() / 4))
PKG_DEBUG = os.getenv('PKG_DEBUG', False)
TRAIN = os.getenv('TRUENAS_TRAIN', f'TrueNAS-SCALE-{CODE_NAME}-Nightlies')
TRY_BRANCH_OVERRIDE = os.getenv('TRY_BRANCH_OVERRIDE')

116
scale_build/package.py
View File

@@ -1,16 +1,94 @@
import logging
import os
import queue
import shutil
import threading
from toposort import toposort
from .bootstrap.configure import make_bootstrapdir
from .clean import clean_bootstrap_logs
from .config import PKG_DEBUG
from .config import PARALLEL_BUILD
from .packages.order import get_to_build_packages
from .utils.paths import LOG_DIR, PKG_LOG_DIR
from .utils.paths import PKG_DIR, PKG_LOG_DIR
from .utils.run import run
logger = logging.getLogger(__name__)
APT_LOCK = threading.Lock()
PACKAGE_BUILD_LOCK = threading.Lock()
def update_queue(package_queue, to_build_orig, failed, in_progress, built):
if failed:
# If we have failure(s), there is no point in continuing
return
to_build = {k: v for k, v in to_build_orig.items()}
to_remove = set()
for pkg_name, package in in_progress.items():
for child in package.children:
to_remove.add(child)
for rm in to_remove:
to_build.pop(rm, None)
deps_mapping = {
p.name: {d for d in p.build_time_dependencies() if d not in built} for p in list(to_build.values())
}
sorted_ordering = [list(deps) for deps in toposort(deps_mapping)]
for item in filter(
lambda i: i in to_build and i not in in_progress and i not in package_queue.queue and i not in built,
sorted_ordering[0] if sorted_ordering else []
):
package_queue.put(to_build_orig.pop(item))
def build_package(package_queue, to_build, failed, in_progress, built):
while True:
if not failed and (to_build or package_queue.queue):
try:
package = package_queue.get(timeout=5)
except queue.Empty:
package = None
else:
in_progress[package.name] = package
else:
break
if package:
try:
logger.debug('Building %r package', package.name)
package.delete_overlayfs()
package.setup_chroot_basedir()
package.make_overlayfs()
with APT_LOCK:
package.clean_previous_packages()
shutil.copytree(PKG_DIR, package.dpkg_overlay_packages_path)
package._build_impl()
except Exception as e:
failed[package.name] = {'package': package, 'exception': e}
break
else:
with APT_LOCK:
package.logger.debug('Building local APT repo Packages.gz...')
run(
f'cd {PKG_DIR} && dpkg-scanpackages . /dev/null | gzip -9c > Packages.gz',
shell=True, logger=package.logger
)
in_progress.pop(package.name)
built[package.name] = package
logger.debug(
'Successfully built %r package (Remaining %d packages)', package.name,
len(to_build) + package_queue.qsize() + len(in_progress)
)
with PACKAGE_BUILD_LOCK:
if not package:
update_queue(package_queue, to_build, failed, in_progress, built)
def build_packages():
clean_bootstrap_logs()
@@ -19,19 +97,33 @@ def build_packages():
def _build_packages_impl():
logger.debug('Building packages')
logger.debug('Setting up bootstrap directory')
make_bootstrapdir('package')
logger.debug('Successfully setup bootstrap directory')
shutil.rmtree(PKG_LOG_DIR, ignore_errors=True)
os.makedirs(PKG_LOG_DIR)
packages = get_to_build_packages()
for pkg_name, package in packages.items():
logger.debug('Building package [%s] (%s/packages/%s.log)', pkg_name, LOG_DIR, pkg_name)
try:
package.build()
except Exception:
logger.error('Failed to build %r package', exc_info=True)
package.delete_overlayfs()
raise
to_build = get_to_build_packages()
package_queue = queue.Queue()
in_progress = {}
failed = {}
built = {}
update_queue(package_queue, to_build, failed, in_progress, built)
threads = [
threading.Thread(
name=f'build_packages_thread_{i + 1}', target=build_package,
args=(package_queue, to_build, failed, in_progress, built)
) for i in range(PARALLEL_BUILD)
]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
logger.debug('Success! Done building packages')
if failed:
for p in failed.values():
p['package'].delete_overlayfs()
logger.error('Failed to build %r package(s)', ', '.join(failed))
else:
logger.debug('Success! Done building packages')

10
scale_build/packages/build.py
View File

@@ -51,13 +51,13 @@ class BuildPackageMixin:
# 10) Generate version
# 11) Execute relevant building commands
# 12) Save
self._build_impl()
def _build_impl(self):
self.delete_overlayfs()
self.setup_chroot_basedir()
self.make_overlayfs()
self.clean_previous_packages()
self._build_impl()
def _build_impl(self):
shutil.copytree(self.source_path, self.source_in_chroot, dirs_exist_ok=True, symlinks=True)
# TODO: Remove me please
@@ -153,10 +153,6 @@ class BuildPackageMixin:
with open(self.pkglist_hash_file_path, 'w') as f:
f.write('\n'.join(built_packages))
# Update the local APT repo
self.logger.debug('Building local APT repo Packages.gz...')
self.run_in_chroot('cd /packages && dpkg-scanpackages . /dev/null | gzip -9c > Packages.gz')
with open(self.hash_path, 'w') as f:
f.write(self.source_hash)

8
scale_build/packages/order.py
View File

@@ -1,7 +1,6 @@
import errno
import logging
from collections import defaultdict
from scale_build.exceptions import CallError
from scale_build.utils.package import get_packages
@@ -23,13 +22,12 @@ def get_to_build_packages():
for binary_package in package.binary_packages:
binary_packages[binary_package.name] = binary_package
parent_mapping = defaultdict(set)
for pkg_name, package in packages.items():
for dep in package.build_time_dependencies(binary_packages):
parent_mapping[dep].add(pkg_name)
for dep in filter(lambda d: d in packages, package.build_time_dependencies(binary_packages)):
packages[dep].children.add(pkg_name)
for pkg_name, package in filter(lambda i: i[1].hash_changed, packages.items()):
for child in parent_mapping[pkg_name]:
for child in package.children:
packages[child].parent_changed = True
return {package.name: package for package in packages.values() if package.rebuild}

10
scale_build/packages/overlay.py
View File

@@ -3,7 +3,7 @@ import shutil
from scale_build.exceptions import CallError
from scale_build.utils.run import run
from scale_build.utils.paths import CACHE_DIR, PKG_DIR, TMP_DIR, TMPFS
from scale_build.utils.paths import TMP_DIR, TMPFS
class OverlayMixin:
@@ -48,12 +48,6 @@ class OverlayMixin:
], 'Failed overlayfs'),
(['mount', 'proc', os.path.join(self.dpkg_overlay, 'proc'), '-t', 'proc'], 'Failed mount proc'),
(['mount', 'sysfs', os.path.join(self.dpkg_overlay, 'sys'), '-t', 'sysfs'], 'Failed mount sysfs'),
(['mount', '--bind', PKG_DIR, self.dpkg_overlay_packages_path], 'Failed mount --bind /packages',
self.dpkg_overlay_packages_path),
(
['mount', '--bind', os.path.join(CACHE_DIR, 'apt'), os.path.join(self.dpkg_overlay, 'var/cache/apt')],
'Failed mount --bind /var/cache/apt',
),
(
['mount', '--bind', self.sources_overlay, self.source_in_chroot],
'Failed mount --bind /dpkg-src', self.source_in_chroot
@@ -69,8 +63,6 @@ class OverlayMixin:
def delete_overlayfs(self):
for command in (
['umount', '-f', os.path.join(self.dpkg_overlay, 'var/cache/apt')],
['umount', '-f', self.dpkg_overlay_packages_path],
['umount', '-f', os.path.join(self.dpkg_overlay, 'proc')],
['umount', '-f', os.path.join(self.dpkg_overlay, 'sys')],
['umount', '-f', self.dpkg_overlay],

8
scale_build/packages/package.py
View File

@@ -45,13 +45,17 @@ class Package(BootstrapMixin, BuildPackageMixin, BuildCleanMixin, OverlayMixin):
self.build_depends = set()
self.source_package = None
self.parent_changed = False
self._build_time_dependencies = set()
self._build_time_dependencies = None
self.build_stage = None
self.logger = logging.getLogger(f'{self.name}_package')
self.logger.setLevel('DEBUG')
self.logger.handlers = []
self.logger.propagate = False
self.logger.addHandler(logging.FileHandler(self.log_file_path, mode='w'))
self.children = set()
def __eq__(self, other):
return other == self.name if isinstance(other, str) else self.name == other.name
@property
def log_file_path(self):
@@ -105,7 +109,7 @@ class Package(BootstrapMixin, BuildPackageMixin, BuildCleanMixin, OverlayMixin):
return self._binary_packages
def build_time_dependencies(self, all_binary_packages=None):
if self._build_time_dependencies:
if self._build_time_dependencies is not None:
return self._build_time_dependencies
elif not all_binary_packages:
raise CallError('Binary packages must be specified when computing build time dependencies')

2
scale_build/utils/paths.py
View File

@@ -17,7 +17,7 @@ GIT_MANIFEST_PATH = os.path.join(LOG_DIR, 'GITMANIFEST')
GIT_LOG_PATH = os.path.join(LOG_DIR, 'git-checkout.log')
HASH_DIR = os.path.join(TMP_DIR, 'pkghashes')
MANIFEST = os.path.join(BUILDER_DIR, 'conf/build.manifest')
PKG_DIR = os.path.join(BUILDER_DIR, 'pkgdir')
PKG_DIR = os.path.join(TMP_DIR, 'pkgdir')
PKG_LOG_DIR = os.path.join(LOG_DIR, 'packages')
RELEASE_DIR = os.path.join(TMP_DIR, 'release')
REQUIRED_RAM = 16 # GB