Add initial commit outlining scale-build python structure

scale_build/__main__.py

@@ -0,0 +1,9 @@
+import logging
+
+from .main import main
+
+logger = logging.getLogger(__name__)
+
+
+if __name__ == "__main__":
+    main()

scale_build/epoch.py

@@ -0,0 +1,27 @@
+import logging
+import os
+
+from scale_build.utils.manifest import get_manifest
+from .utils.variables import TMP_DIR
+
+
+logger = logging.getLogger(__name__)
+EPOCH_PATH = os.path.join(TMP_DIR, '.buildEpoch')
+
+
+def update_epoch(epoch_value):
+    os.makedirs(TMP_DIR, exist_ok=True)
+    with open(EPOCH_PATH, 'w') as f:
+        f.write(epoch_value)
+
+
+def check_epoch():
+    current_epoch = get_manifest()['build-epoch']
+    if os.path.exists(EPOCH_PATH):
+        with open(EPOCH_PATH, 'r') as f:
+            epoch_num = f.read().strip()
+            if epoch_num != current_epoch:
+                logger.warning('Build epoch changed! Removing temporary files and forcing clean build.')
+                update_epoch(current_epoch)
+    else:
+        update_epoch(current_epoch)

scale_build/exceptions.py

@@ -0,0 +1,26 @@
+import errno
+
+
+class CallError(Exception):
+    def __init__(self, errmsg, errno=errno.EFAULT, extra=None):
+        self.errmsg = errmsg
+        self.errno = errno
+        self.extra = extra
+
+    def __str__(self):
+        return f'[{self.errno}] {self.errmsg}'
+
+
+class MissingManifest(CallError):
+    def __init__(self):
+        super().__init__('Unable to locate manifest file', errno.ENOENT)
+
+
+class InvalidManifest(CallError):
+    def __init__(self):
+        super().__init__('Invalid manifest file found', errno.EINVAL)
+
+
+class MissingPackagesException(CallError):
+    def __init__(self, packages):
+        super().__init__(f'Failed preflight check. Please install {", ".join(packages)!r} packages.', errno.ENOENT)

scale_build/main.py

@@ -0,0 +1,27 @@
+import argparse
+import logging
+import sys
+
+logger = logging.getLogger(__name__)
+
+
+def setup_logging():
+    logging.basicConfig(level=logging.DEBUG, format='[%(asctime)s] %(message)s')
+    handler = logging.StreamHandler(sys.stderr)
+    handler.setLevel(logging.DEBUG)
+    handler.setFormatter(logging.Formatter('[%(asctime)s] %(message)s'))
+    logger.addHandler(handler)
+
+
+def main():
+    setup_logging()
+    parser = argparse.ArgumentParser(prog='scale-build')
+    subparsers = parser.add_subparsers(help='sub-command help', dest='action')
+
+    subparsers.add_parser('checkout', help='Checkout TrueNAS Scale repositories')
+
+    args = parser.parse_args()
+    if args.action == 'checkout':
+        pass
+    else:
+        parser.print_help()

scale_build/preflight.py

@@ -0,0 +1,57 @@
+import logging
+import os
+import shutil
+
+from .exceptions import CallError, MissingPackagesException
+from .utils.manifest import get_manifest
+from .utils.system import has_low_ram
+from .utils.variables import TMP_DIR, HASH_DIR, LOG_DIR, PKG_DIR
+
+
+logger = logging.getLogger(__name__)
+
+WANTED_PACKAGES = {
+    'make',
+    'debootstrap',
+    'git',
+    'xorriso',
+    'grub-mkrescue',
+    'mksquashfs',
+    'unzip',
+}
+
+
+def is_root():
+    return os.geteuid() == 0
+
+
+def retrieve_missing_packages():
+    missing = {pkg for pkg in WANTED_PACKAGES if not shutil.which(pkg)}
+    if not os.path.exists('/lib/grub/x86_64-efi') and not os.path.exists('/usr/lib/grub/x86_64-efi'):
+        missing.add('grub-efi-amd64-bin')
+
+    if not os.path.exists('/lib/grub/i386-pc') and not os.path.exists('/usr/lib/grub/i386-pc'):
+        missing.add('grub-pc-bin')
+
+    return missing
+
+
+def setup_dirs():
+    for d in (TMP_DIR, HASH_DIR, LOG_DIR, PKG_DIR):
+        os.makedirs(d, exist_ok=True)
+
+
+def preflight_check():
+    # TODO: Please see how we should delete overlayfs/bootstrapdir when exceptions are raised
+    if not is_root():
+        raise CallError('Must be run as root (or using sudo)!')
+
+    missing_packages = retrieve_missing_packages()
+    if missing_packages:
+        raise MissingPackagesException(missing_packages)
+
+    if has_low_ram():
+        logging.warning('WARNING: Running with less than 16GB of memory. Build may fail...')
+
+    setup_dirs()
+    get_manifest()

scale_build/utils/manifest.py

@@ -0,0 +1,14 @@
+import yaml
+
+from scale_build.exceptions import MissingManifest, InvalidManifest
+from scale_build.utils.variables import MANIFEST
+
+
+def get_manifest():
+    try:
+        with open(MANIFEST, 'r') as f:
+            return yaml.load(f.read())
+    except FileNotFoundError:
+        raise MissingManifest()
+    except yaml.YAMLError:
+        raise InvalidManifest()

scale_build/utils/run.py

@@ -0,0 +1,15 @@
+import subprocess
+
+
+def run(*args, **kwargs):
+    if isinstance(args[0], list):
+        args = tuple(args[0])
+    kwargs.setdefault('stdout', subprocess.PIPE)
+    kwargs.setdefault('stderr', subprocess.PIPE)
+    check = kwargs.pop('check', True)
+    proc = subprocess.Popen(args, stdout=kwargs['stdout'], stderr=kwargs['stderr'])
+    stdout, stderr = proc.communicate()
+    cp = subprocess.CompletedProcess(args, proc.returncode, stdout=stdout, stderr=stderr)
+    if check:
+        cp.check_returncode()
+    return cp

scale_build/utils/system.py

@@ -0,0 +1,7 @@
+import psutil
+
+from .variables import REQUIRED_RAM
+
+
+def has_low_ram():
+    return psutil.virtual_memory().total < REQUIRED_RAM * 1024 * 1024 * 1024

scale_build/utils/variables.py

@@ -0,0 +1,22 @@
+import os
+
+
+TMPFS = './tmp/tmpfs'
+CACHE_DIR = './tmp/cache'
+CHROOT_BASEDIR = os.path.join(TMPFS, 'chroot')
+CHROOT_OVERLAY = os.path.join(TMPFS, 'chroot-overlay')
+DPKG_OVERLAY = './tmp/dpkg-overlay'
+HASH_DIR = './tmp/pkghashes'
+LOG_DIR = './logs'
+MANIFEST = './conf/build.manifest'
+PARALLEL_BUILDS = int(os.environ.get('PARALLEL_BUILDS') or 4)
+PKG_DEBUG = bool(os.environ.get('PKG_DEBUG'))
+PKG_DIR = './tmp/pkgdir'
+REQUIRED_RAM = 16  # GB
+SOURCES = './sources'
+TMP_DIR = './tmp'
+WORKDIR_OVERLAY = os.path.join(TMPFS, 'workdir-overlay')
+
+
+if PKG_DEBUG:
+    PARALLEL_BUILDS = 1