Allow configuration of env variables

scale_build/checkout.py

@@ -1,7 +1,7 @@
 import logging
-import os
 import re
 
+from .config import BRANCH_OVERRIDES, TRY_BRANCH_OVERRIDE
 from .utils.git_utils import retrieve_git_remote_and_sha, update_git_manifest
 from .utils.package import get_packages
 from .utils.run import run
@@ -14,12 +14,9 @@ def checkout_sources():
     info = retrieve_git_remote_and_sha('.')
     update_git_manifest(info['url'], info['sha'], 'w')
     logger.debug(f'Starting checkout of source')
-    try_branch_override = os.environ.get('TRY_BRANCH_OVERRIDE')
 
     for package in get_packages():
-        gh_override = os.environ.get('TRUENAS_BRANCH_OVERRIDE')
+        gh_override = BRANCH_OVERRIDES.get(package.name)
-        if not gh_override:
-            gh_override = os.environ.get(f'{package.name}_OVERRIDE')
 
         # TRY_BRANCH_OVERRIDE is a special use-case. It allows setting a branch name to be used
         # during the checkout phase, only if it exists on the remote.
@@ -27,9 +24,9 @@ def checkout_sources():
         # This is useful for PR builds and testing where you want to use defaults for most repos
         # but need to test building of a series of repos with the same experimental branch
         #
-        if try_branch_override:
+        if TRY_BRANCH_OVERRIDE:
             cp = run(['git', 'ls-remote', package.origin])
-            if re.findall(fr'/{try_branch_override}$', cp.stdout.decode()):
+            if re.findall(fr'/{TRY_BRANCH_OVERRIDE}$', cp.stdout.decode()):
-                gh_override = try_branch_override
+                gh_override = TRY_BRANCH_OVERRIDE
 
         package.checkout(gh_override)

scale_build/config.py

@@ -0,0 +1,18 @@
+import os
+import time
+
+from datetime import datetime
+
+
+BUILD_TIME = int(time.time())
+BUILD_TIME_OBJ = datetime.fromtimestamp(BUILD_TIME)
+BUILDER_DIR = os.getenv('BUILDER_DIR', './')
+BRANCH_OVERRIDES = {k[:-(len('_OVERRIDE'))]: v for k, v in os.environ.items() if k.endswith('_OVERRIDE')}
+CODE_NAME = os.getenv('CODENAME', 'Angelfish')
+PKG_DEBUG = os.getenv('PKG_DEBUG', False)
+TRAIN = os.getenv('TRUENAS_TRAIN', f'TrueNAS-SCALE-{CODE_NAME}-Nightlies')
+TRY_BRANCH_OVERRIDE = os.getenv('TRY_BRANCH_OVERRIDE')
+if os.getenv('TRUENAS_VERSION'):
+    VERSION = os.getenv('TRUENAS_VERSION')
+else:
+    VERSION = f'{BUILD_TIME_OBJ.strftime("%y.%m")}-MASTER-{BUILD_TIME_OBJ.strftime("%Y%m%d-%H%M%S")}'

scale_build/image/iso.py

@@ -3,6 +3,7 @@ import itertools
 import os
 import shutil
 
+from scale_build.config import VERSION
 from scale_build.utils.manifest import get_manifest
 from scale_build.utils.run import run
 from scale_build.utils.paths import BUILDER_DIR, CD_DIR, CHROOT_BASEDIR, CONF_GRUB, RELEASE_DIR, TMP_DIR
@@ -36,7 +37,7 @@ def make_iso_file():
 
     # Create /etc/version
     with open(os.path.join(CHROOT_BASEDIR, 'etc/version'), 'w') as f:
-        f.write(str(os.environ.get('VERSION')))
+        f.write(VERSION)
 
     # Copy the CD files
     for source, destination in (
@@ -82,15 +83,14 @@ def make_iso_file():
 
     run_in_chroot('apt-get update', logger=iso_logger, check=False)
     run_in_chroot('apt-get install -y grub-efi grub-pc-bin mtools xorriso', logger=iso_logger, check=False)
-    version = str(os.getenv('VERSION'))
     run_in_chroot(
-        f'grub-mkrescue -o {os.path.join(RELEASE_DIR, f"TrueNAS-SCALE-{version}.iso")} {CD_DIR}', logger=iso_logger
+        f'grub-mkrescue -o {os.path.join(RELEASE_DIR, f"TrueNAS-SCALE-{VERSION}.iso")} {CD_DIR}', logger=iso_logger
     )
     run(['umount', '-f', os.path.join(CHROOT_BASEDIR, CD_DIR)])
     run(['umount', '-f', os.path.join(CHROOT_BASEDIR, RELEASE_DIR)])
 
-    with open(os.path.join(RELEASE_DIR, f'TrueNAS-SCALE-{version}.iso.sha256'), 'w') as f:
+    with open(os.path.join(RELEASE_DIR, f'TrueNAS-SCALE-{VERSION}.iso.sha256'), 'w') as f:
-        f.write(run(['sha256sum', os.path.join(RELEASE_DIR, f'TrueNAS-SCALE-{version}.iso')]).stdout.decode().strip())
+        f.write(run(['sha256sum', os.path.join(RELEASE_DIR, f'TrueNAS-SCALE-{VERSION}.iso')]).stdout.decode().strip())
 
 
 def prune_cd_basedir():

scale_build/iso.py

@@ -4,6 +4,7 @@ import logging
 import os
 
 from .bootstrap.configure import make_bootstrapdir
+from .config import VERSION
 from .exceptions import CallError
 from .image.bootstrap import (
     clean_mounts, setup_chroot_basedir, umount_chroot_basedir, umount_tmpfs_and_clean_chroot_dir
@@ -37,4 +38,4 @@ def build_iso():
     make_iso_file()
 
     umount_tmpfs_and_clean_chroot_dir()
-    logger.debug('Success! CD/USB: %s/TrueNAS-SCALE-%s.iso', RELEASE_DIR, os.getenv('VERSION'))
+    logger.debug('Success! CD/USB: %s/TrueNAS-SCALE-%s.iso', RELEASE_DIR, VERSION)

scale_build/packages/build.py

@@ -4,6 +4,7 @@ import os
 import shutil
 
 from datetime import datetime
+from scale_build.config import BUILD_TIME, TRAIN, VERSION
 from scale_build.exceptions import CallError
 from scale_build.utils.environment import APT_ENV
 from scale_build.utils.run import run
@@ -109,14 +110,14 @@ class BuildPackageMixin:
             # FIXME: Please see a good way to have environment variables available
             with open(os.path.join(self.package_source_with_chroot, 'data/manifest.json'), 'w') as f:
                 f.write(json.dumps({
-                    'buildtime': os.environ.get('BUILDTIME'),
+                    'buildtime': BUILD_TIME,
-                    'train': os.environ.get('TRAIN'),
+                    'train': TRAIN,
-                    'version': os.environ.get('VERSION'),
+                    'version': VERSION,
                 }))
             os.makedirs(os.path.join(self.package_source_with_chroot, 'etc'), exist_ok=True)
             with open(os.path.join(self.package_source_with_chroot, 'etc/version'), 'w') as f:
                 # FIXME: Remove string type cast please
-                f.write(str(os.environ.get('VERSION')))
+                f.write(VERSION)
 
         for prebuild_command in self.prebuildcmd:
             self.logger.debug('Running prebuildcmd: %r', prebuild_command)

scale_build/utils/paths.py

@@ -1,31 +1,26 @@
 import os
 
+from scale_build.config import BUILDER_DIR
 
-BUILDER_DIR = './'
+
-TMPFS = './tmp/tmpfs'
+LOG_DIR = os.path.join(BUILDER_DIR, 'logs')
-CACHE_DIR = './tmp/cache'
+TMP_DIR = os.path.join(BUILDER_DIR, 'tmp')
-CD_DIR = './tmp/cdrom'
+TMPFS = os.path.join(TMP_DIR, 'tmpfs')
+CACHE_DIR = os.path.join(BUILDER_DIR, 'cache')
+CD_DIR = os.path.join(BUILDER_DIR, 'cdrom')
 CHROOT_BASEDIR = os.path.join(TMPFS, 'chroot')
 CHROOT_OVERLAY = os.path.join(TMPFS, 'chroot-overlay')
 CONF_SOURCES = os.path.join(BUILDER_DIR, 'conf/sources.list')
 CONF_GRUB = os.path.join(BUILDER_DIR, 'scripts/grub.cfg')
-DPKG_OVERLAY = './tmp/dpkg-overlay'
+DPKG_OVERLAY = os.path.join(TMP_DIR, 'dpkg-overlay')
-GIT_MANIFEST_PATH = './logs/GITMANIFEST'
+GIT_MANIFEST_PATH = os.path.join(LOG_DIR, 'GITMANIFEST')
-GIT_LOG_PATH = './logs/git-checkout.log'
+GIT_LOG_PATH = os.path.join(LOG_DIR, 'git-checkout.log')
-HASH_DIR = './tmp/pkghashes'
+HASH_DIR = os.path.join(TMP_DIR, 'pkghashes')
-LOG_DIR = './logs'
+MANIFEST = os.path.join(BUILDER_DIR, 'conf/build.manifest')
-MANIFEST = './conf/build.manifest'
+PKG_DIR = os.path.join(BUILDER_DIR, 'pkgdir')
-PARALLEL_BUILDS = int(os.environ.get('PARALLEL_BUILDS') or 4)
-PKG_DEBUG = bool(os.environ.get('PKG_DEBUG'))
-PKG_DIR = './tmp/pkgdir'
 PKG_LOG_DIR = os.path.join(LOG_DIR, 'packages')
-RELEASE_DIR = './tmp/release'
+RELEASE_DIR = os.path.join(TMP_DIR, 'release')
 REQUIRED_RAM = 16  # GB
-SOURCES_DIR = './sources'
+SOURCES_DIR = os.path.join(BUILDER_DIR, 'sources')
-TMP_DIR = './tmp'
 UPDATE_DIR = os.path.join(TMP_DIR, 'update')
 WORKDIR_OVERLAY = os.path.join(TMPFS, 'workdir-overlay')
-
-
-if PKG_DEBUG:
-    PARALLEL_BUILDS = 1