NAS-134870 / 25.10 / Allow specifying secret env variables in the build (#842)

* Allow specifying secret_env in build manifest

* Get secret env initialized when initializing package

* Add logic to read secrets file

* Make sure secrets are properly set for package when building the package

* Expose scale release version variable as well

* Fix typo

* Make sure env variables are actually passed to the package itself

* Add secrets yaml file to git ignore

* Do not expose build env variables in ps output

.gitignore

@@ -9,3 +9,4 @@ dist/
 scale_build.egg-info/
 scale_build/__pycache__/
 venv-*
+conf/secrets.yaml

scale_build/packages/build.py

@@ -1,13 +1,14 @@
 import contextlib
 import json
 import os
+import shlex
 import shutil
 
 from datetime import datetime
 from scale_build.config import BUILD_TIME, VERSION
 from scale_build.exceptions import CallError
 from scale_build.utils.environment import APT_ENV
-from scale_build.utils.manifest import get_truenas_train, get_release_code_name
+from scale_build.utils.manifest import get_truenas_train, get_release_code_name, get_secret_env
 from scale_build.utils.run import run
 from scale_build.utils.paths import PKG_DIR
 
@@ -16,8 +17,9 @@ class BuildPackageMixin:
 
     def run_in_chroot(self, command, exception_message=None):
         run(
-            f'chroot {self.dpkg_overlay} /bin/bash -c "{command}"', shell=True, exception_msg=exception_message,
+            f'chroot {self.dpkg_overlay} /bin/bash -c {shlex.quote(command)}', shell=True,
-            env=self._get_build_env()
+            exception_msg=exception_message,
+            env=self._get_build_env() | self._get_chroot_env()
         )
 
     @property
@@ -61,6 +63,15 @@ class BuildPackageMixin:
         env.update(self.ccache_env(env))
         return env
 
+    def _get_chroot_env(self):
+        env = {
+            'RELEASE_VERSION': VERSION,
+        }
+        secrets = get_secret_env()
+        for k in filter(lambda k: k in secrets, self.secret_env):
+            env[k] = secrets[k]
+        return env
+
     def _build_impl(self):
         shutil.copytree(self.source_path, self.source_in_chroot, dirs_exist_ok=True, symlinks=True)
         if os.path.exists(os.path.join(self.dpkg_overlay_packages_path, 'Packages.gz')):
@@ -160,7 +171,8 @@ class BuildPackageMixin:
             return self.buildcmd
         else:
             build_env = f'DEB_BUILD_OPTIONS={self.deoptions} ' if self.deoptions else ''
-            return [f'{build_env} debuild {" ".join(self.deflags)}']
+            env_flags = [f'-e{k}' for k in self._get_chroot_env()]
+            return [f'{build_env} debuild {" ".join(env_flags + self.deflags)}']
 
     @property
     def debug_command(self):

scale_build/packages/package.py

@@ -31,6 +31,7 @@ class Package(BootstrapMixin, BuildPackageMixin, BuildCleanMixin, CCacheMixin, G
         generate_version=True, predepscmd=None, deps_path=None, subdir=None, deoptions=None, jobs=None,
         buildcmd=None, tmpfs=True, tmpfs_size=12, batch_priority=100, env=None, identity_file_path=None,
         build_constraints=None, debian_fork=False, source_name=None, depscmd=None, supports_ccache=False,
+        secret_env=None,
     ):
         self.name = name
         self.source_name = source_name or name
@@ -63,6 +64,7 @@ class Package(BootstrapMixin, BuildPackageMixin, BuildCleanMixin, CCacheMixin, G
         self.children = set()
         self.batch_priority = batch_priority
         self.env = env or {}
+        self.secret_env = secret_env or []
         self.debian_fork = debian_fork
 
         if name in self.explicit_deps:

scale_build/utils/manifest.py

@@ -8,7 +8,7 @@ from urllib.parse import urlparse
 
 from scale_build.config import APT_BASE_CUSTOM, APT_INTERNAL_BUILD, SKIP_SOURCE_REPO_VALIDATION, TRAIN
 from scale_build.exceptions import CallError, MissingManifest
-from scale_build.utils.paths import MANIFEST
+from scale_build.utils.paths import MANIFEST, SECRETS_FILE
 
 
 BRANCH_REGEX = re.compile(r'(branch\s*:\s*)\b[\w/\.-]+\b')
@@ -69,6 +69,7 @@ INDIVIDUAL_REPO_SCHEMA = {
         'jobs': {'type': 'integer'},
         'debian_fork': {'type': 'boolean'},
         'env': {'type': 'object', 'patternProperties': {'^.+$': {'type': 'string'}}},
+        'secret_env': {'type': 'array', 'items': {'type': 'string'}},
     },
     'additionalProperties': False,
 }
@@ -198,6 +199,21 @@ MANIFEST_SCHEMA = {
 }
 
 
+@functools.cache
+def get_secret_env():
+    try:
+        with open(SECRETS_FILE, 'r') as f:
+            secrets = yaml.safe_load(f.read())
+        if not isinstance(secrets, dict):
+            raise CallError('A dictionary containing secrets is expected')
+    except yaml.YAMLError:
+        raise CallError('A valid yaml file is expected for secrets')
+    except FileNotFoundError:
+        return {}
+    else:
+        return secrets
+
+
 def get_manifest_str():
     try:
         with open(MANIFEST, 'r') as f:
@@ -244,6 +260,9 @@ def update_packages_branch(branch_name):
 
 
 def validate_manifest():
+    # We don't consume secrets here but when manifest is being validated, we would like to make sure
+    # if any secret file is present, it gets validated properly and then cached for consumption
+    get_secret_env()
     manifest = get_manifest()
     if SKIP_SOURCE_REPO_VALIDATION:
         return

scale_build/utils/paths.py

@@ -26,6 +26,7 @@ PKG_LOG_DIR = os.path.join(LOG_DIR, 'packages')
 REFERENCE_FILES = ('etc/group', 'etc/passwd')
 REFERENCE_FILES_DIR = os.path.join(BUILDER_DIR, 'conf/reference-files')
 RELEASE_DIR = os.path.join(TMP_DIR, 'release')
+SECRETS_FILE = os.path.join(BUILDER_DIR, 'conf/secrets.yaml')
 SOURCES_DIR = os.path.join(BUILDER_DIR, 'sources')
 UPDATE_DIR = os.path.join(TMP_DIR, 'update')
 WORKDIR_OVERLAY = os.path.join(TMPFS, 'workdir-overlay')