From 76b930d18c1febfa37a45c72f228b3a4f79875e5 Mon Sep 17 00:00:00 2001 From: sonicaj Date: Fri, 18 Apr 2025 20:34:12 +0500 Subject: [PATCH] NAS-134870 / 25.10 / Allow specifying secret env variables in the build (#842) * Allow specifying secret_env in build manifest * Get secret env initialized when initializing package * Add logic to read secrets file * Make sure secrets are properly set for package when building the package * Expose scale release version variable as well * Fix typo * Make sure env variables are actually passed to the package itself * Add secrets yaml file to git ignore * Do not expose build env variables in ps output --- .gitignore | 1 + scale_build/packages/build.py | 20 ++++++++++++++++---- scale_build/packages/package.py | 2 ++ scale_build/utils/manifest.py | 21 ++++++++++++++++++++- scale_build/utils/paths.py | 1 + 5 files changed, 40 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 431e81e..a0b8eb8 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ dist/ scale_build.egg-info/ scale_build/__pycache__/ venv-* +conf/secrets.yaml diff --git a/scale_build/packages/build.py b/scale_build/packages/build.py index 51d944f..da91a6e 100644 --- a/scale_build/packages/build.py +++ b/scale_build/packages/build.py @@ -1,13 +1,14 @@ import contextlib import json import os +import shlex import shutil from datetime import datetime from scale_build.config import BUILD_TIME, VERSION from scale_build.exceptions import CallError from scale_build.utils.environment import APT_ENV -from scale_build.utils.manifest import get_truenas_train, get_release_code_name +from scale_build.utils.manifest import get_truenas_train, get_release_code_name, get_secret_env from scale_build.utils.run import run from scale_build.utils.paths import PKG_DIR @@ -16,8 +17,9 @@ class BuildPackageMixin: def run_in_chroot(self, command, exception_message=None): run( - f'chroot {self.dpkg_overlay} /bin/bash -c "{command}"', shell=True, exception_msg=exception_message, - env=self._get_build_env() + f'chroot {self.dpkg_overlay} /bin/bash -c {shlex.quote(command)}', shell=True, + exception_msg=exception_message, + env=self._get_build_env() | self._get_chroot_env() ) @property @@ -61,6 +63,15 @@ class BuildPackageMixin: env.update(self.ccache_env(env)) return env + def _get_chroot_env(self): + env = { + 'RELEASE_VERSION': VERSION, + } + secrets = get_secret_env() + for k in filter(lambda k: k in secrets, self.secret_env): + env[k] = secrets[k] + return env + def _build_impl(self): shutil.copytree(self.source_path, self.source_in_chroot, dirs_exist_ok=True, symlinks=True) if os.path.exists(os.path.join(self.dpkg_overlay_packages_path, 'Packages.gz')): @@ -160,7 +171,8 @@ class BuildPackageMixin: return self.buildcmd else: build_env = f'DEB_BUILD_OPTIONS={self.deoptions} ' if self.deoptions else '' - return [f'{build_env} debuild {" ".join(self.deflags)}'] + env_flags = [f'-e{k}' for k in self._get_chroot_env()] + return [f'{build_env} debuild {" ".join(env_flags + self.deflags)}'] @property def debug_command(self): diff --git a/scale_build/packages/package.py b/scale_build/packages/package.py index f1e0fc0..d109857 100644 --- a/scale_build/packages/package.py +++ b/scale_build/packages/package.py @@ -31,6 +31,7 @@ class Package(BootstrapMixin, BuildPackageMixin, BuildCleanMixin, CCacheMixin, G generate_version=True, predepscmd=None, deps_path=None, subdir=None, deoptions=None, jobs=None, buildcmd=None, tmpfs=True, tmpfs_size=12, batch_priority=100, env=None, identity_file_path=None, build_constraints=None, debian_fork=False, source_name=None, depscmd=None, supports_ccache=False, + secret_env=None, ): self.name = name self.source_name = source_name or name @@ -63,6 +64,7 @@ class Package(BootstrapMixin, BuildPackageMixin, BuildCleanMixin, CCacheMixin, G self.children = set() self.batch_priority = batch_priority self.env = env or {} + self.secret_env = secret_env or [] self.debian_fork = debian_fork if name in self.explicit_deps: diff --git a/scale_build/utils/manifest.py b/scale_build/utils/manifest.py index 892381e..51aab2a 100644 --- a/scale_build/utils/manifest.py +++ b/scale_build/utils/manifest.py @@ -8,7 +8,7 @@ from urllib.parse import urlparse from scale_build.config import APT_BASE_CUSTOM, APT_INTERNAL_BUILD, SKIP_SOURCE_REPO_VALIDATION, TRAIN from scale_build.exceptions import CallError, MissingManifest -from scale_build.utils.paths import MANIFEST +from scale_build.utils.paths import MANIFEST, SECRETS_FILE BRANCH_REGEX = re.compile(r'(branch\s*:\s*)\b[\w/\.-]+\b') @@ -69,6 +69,7 @@ INDIVIDUAL_REPO_SCHEMA = { 'jobs': {'type': 'integer'}, 'debian_fork': {'type': 'boolean'}, 'env': {'type': 'object', 'patternProperties': {'^.+$': {'type': 'string'}}}, + 'secret_env': {'type': 'array', 'items': {'type': 'string'}}, }, 'additionalProperties': False, } @@ -198,6 +199,21 @@ MANIFEST_SCHEMA = { } +@functools.cache +def get_secret_env(): + try: + with open(SECRETS_FILE, 'r') as f: + secrets = yaml.safe_load(f.read()) + if not isinstance(secrets, dict): + raise CallError('A dictionary containing secrets is expected') + except yaml.YAMLError: + raise CallError('A valid yaml file is expected for secrets') + except FileNotFoundError: + return {} + else: + return secrets + + def get_manifest_str(): try: with open(MANIFEST, 'r') as f: @@ -244,6 +260,9 @@ def update_packages_branch(branch_name): def validate_manifest(): + # We don't consume secrets here but when manifest is being validated, we would like to make sure + # if any secret file is present, it gets validated properly and then cached for consumption + get_secret_env() manifest = get_manifest() if SKIP_SOURCE_REPO_VALIDATION: return diff --git a/scale_build/utils/paths.py b/scale_build/utils/paths.py index b46b1e1..fa953d8 100644 --- a/scale_build/utils/paths.py +++ b/scale_build/utils/paths.py @@ -26,6 +26,7 @@ PKG_LOG_DIR = os.path.join(LOG_DIR, 'packages') REFERENCE_FILES = ('etc/group', 'etc/passwd') REFERENCE_FILES_DIR = os.path.join(BUILDER_DIR, 'conf/reference-files') RELEASE_DIR = os.path.join(TMP_DIR, 'release') +SECRETS_FILE = os.path.join(BUILDER_DIR, 'conf/secrets.yaml') SOURCES_DIR = os.path.join(BUILDER_DIR, 'sources') UPDATE_DIR = os.path.join(TMP_DIR, 'update') WORKDIR_OVERLAY = os.path.join(TMPFS, 'workdir-overlay')