From b38f4516a2580eb9f6b524dc15b4ca09063b8351 Mon Sep 17 00:00:00 2001 From: themylogin Date: Tue, 8 Feb 2022 12:07:03 +0100 Subject: [PATCH] Add version to the `.update` file name --- jenkins/Incremental-Publish | 2 +- jenkins/IncrementalBuild | 2 +- scale_build/image/iso.py | 6 +++--- scale_build/image/manifest.py | 28 +++++++++++++++++++++++----- scale_build/image/update.py | 15 ++++++++------- scale_build/image/utils.py | 22 ---------------------- scale_build/iso.py | 5 ++--- scale_build/update_image.py | 4 ++-- 8 files changed, 40 insertions(+), 44 deletions(-) diff --git a/jenkins/Incremental-Publish b/jenkins/Incremental-Publish index c14ad40..fc4a8c2 100644 --- a/jenkins/Incremental-Publish +++ b/jenkins/Incremental-Publish @@ -13,7 +13,7 @@ pipeline { echo '*** Grabbing artifact from Build - TrueNAS SCALE (Incremental) ***' copyArtifacts filter: '**/*.update', fingerprintArtifacts: true, flatten: true, projectName: 'Build - TrueNAS SCALE (Incremental)', selector: lastSuccessful(), target: 'upload/files' copyArtifacts filter: '**/*.json', fingerprintArtifacts: true, flatten: true, projectName: 'Build - TrueNAS SCALE (Incremental)', selector: lastSuccessful(), target: 'upload/files' - sh 'scp upload/files/manifest.json upload/files/TrueNAS-SCALE.update scale@update-master.freenas.org:/tank/www/scale/TrueNAS-SCALE-Angelfish-Nightlies/' + sh 'scp upload/files/manifest.json upload/files/TrueNAS-SCALE-*.update scale@update-master.freenas.org:/tank/www/scale/TrueNAS-SCALE-Angelfish-Nightlies/' sh 'rm -rf upload/files' } } diff --git a/jenkins/IncrementalBuild b/jenkins/IncrementalBuild index 7a9077e..4d104bf 100644 --- a/jenkins/IncrementalBuild +++ b/jenkins/IncrementalBuild @@ -41,7 +41,7 @@ pipeline { } stage('Upload') { steps { - sh 'scp ${BDIR}/tmp/release/manifest.json ${BDIR}/tmp/release/TrueNAS-SCALE.update scale@update-master.freenas.org:/tank/www/scale/TrueNAS-SCALE-Angelfish-Nightlies/' + sh 'scp ${BDIR}/tmp/release/manifest.json ${BDIR}/tmp/release/TrueNAS-SCALE-*.update scale@update-master.freenas.org:/tank/www/scale/TrueNAS-SCALE-Angelfish-Nightlies/' } } } diff --git a/scale_build/image/iso.py b/scale_build/image/iso.py index 61edded..fcaefcd 100644 --- a/scale_build/image/iso.py +++ b/scale_build/image/iso.py @@ -12,8 +12,8 @@ from scale_build.utils.run import run from scale_build.utils.paths import CD_DIR, CD_FILES_DIR, CHROOT_BASEDIR, CONF_GRUB, PKG_DIR, RELEASE_DIR, TMP_DIR from .bootstrap import umount_chroot_basedir -from .manifest import UPDATE_FILE -from .utils import run_in_chroot, get_image_version +from .manifest import get_image_version, update_file_path +from .utils import run_in_chroot def install_iso_packages(): @@ -77,7 +77,7 @@ def make_iso_file(): ): os.unlink(f) - shutil.copy(UPDATE_FILE, os.path.join(CD_DIR, 'TrueNAS-SCALE.update')) + shutil.copy(update_file_path(), os.path.join(CD_DIR, 'TrueNAS-SCALE.update')) os.makedirs(os.path.join(CHROOT_BASEDIR, RELEASE_DIR), exist_ok=True) os.makedirs(os.path.join(CHROOT_BASEDIR, CD_DIR), exist_ok=True) diff --git a/scale_build/image/manifest.py b/scale_build/image/manifest.py index 14db365..829a94c 100644 --- a/scale_build/image/manifest.py +++ b/scale_build/image/manifest.py @@ -5,11 +5,11 @@ import os import shutil import subprocess +from scale_build.exceptions import CallError from scale_build.utils.paths import BUILDER_DIR, CHROOT_BASEDIR, RELEASE_DIR, UPDATE_DIR -UPDATE_FILE = os.path.join(RELEASE_DIR, 'TrueNAS-SCALE.update') -UPDATE_FILE_HASH = f'{UPDATE_FILE}.sha256' +RELEASE_MANIFEST = os.path.join(RELEASE_DIR, 'manifest.json') def build_manifest(): @@ -46,16 +46,34 @@ def build_manifest(): )[0].split('/')[-1][len('vmlinuz-'):], })) + return version -def build_update_manifest(update_file_checksum): + +def build_release_manifest(update_file, update_file_checksum): with open(os.path.join(UPDATE_DIR, 'manifest.json')) as f: manifest = json.load(f) - with open(os.path.join(RELEASE_DIR, 'manifest.json'), 'w') as f: + with open(RELEASE_MANIFEST, 'w') as f: json.dump({ - 'filename': os.path.basename(UPDATE_FILE), + 'filename': os.path.basename(update_file), 'version': manifest['version'], 'date': manifest['date'], 'changelog': '', 'checksum': update_file_checksum, }, f) + + +def get_image_version(): + if not os.path.exists(RELEASE_MANIFEST): + raise CallError(f'{RELEASE_MANIFEST!r} does not exist') + + with open(RELEASE_MANIFEST) as f: + return json.load(f)['version'] + + +def update_file_path(version=None): + return os.path.join(RELEASE_DIR, f'TrueNAS-SCALE-{version or get_image_version()}.update') + + +def update_file_checksum_path(version=None): + return f'{update_file_path(version)}.sha256' diff --git a/scale_build/image/update.py b/scale_build/image/update.py index 3fe700c..9d89ccd 100644 --- a/scale_build/image/update.py +++ b/scale_build/image/update.py @@ -12,7 +12,7 @@ from scale_build.utils.run import run from scale_build.utils.paths import CHROOT_BASEDIR, CONF_SOURCES, RELEASE_DIR, UPDATE_DIR from .bootstrap import umount_chroot_basedir -from .manifest import build_manifest, build_update_manifest, UPDATE_FILE, UPDATE_FILE_HASH +from .manifest import build_manifest, build_release_manifest, update_file_path, update_file_checksum_path from .utils import run_in_chroot @@ -35,19 +35,20 @@ def build_rootfs_image(): # Create the inner image run(['mksquashfs', CHROOT_BASEDIR, os.path.join(UPDATE_DIR, 'rootfs.squashfs'), '-comp', 'xz']) # Build any MANIFEST information - build_manifest() + version = build_manifest() # Sign the image (if enabled) if SIGNING_KEY and SIGNING_PASSWORD: sign_manifest(SIGNING_KEY, SIGNING_PASSWORD) # Create the outer image now - run(['mksquashfs', UPDATE_DIR, UPDATE_FILE, '-noD']) - update_hash = run(['sha256sum', UPDATE_FILE], log=False).stdout.strip().split()[0] - with open(UPDATE_FILE_HASH, 'w') as f: - f.write(update_hash) + update_file = update_file_path(version) + run(['mksquashfs', UPDATE_DIR, update_file, '-noD']) + update_file_checksum = run(['sha256sum', update_file_path(version)], log=False).stdout.strip().split()[0] + with open(update_file_checksum_path(version), 'w') as f: + f.write(update_file_checksum) - build_update_manifest(update_hash) + build_release_manifest(update_file, update_file_checksum) def sign_manifest(signing_key, signing_pass): diff --git a/scale_build/image/utils.py b/scale_build/image/utils.py index de0ce3b..dc1d84a 100644 --- a/scale_build/image/utils.py +++ b/scale_build/image/utils.py @@ -1,16 +1,9 @@ -import functools -import json import os -import tempfile -from scale_build.exceptions import CallError from scale_build.utils.environment import APT_ENV from scale_build.utils.run import run from scale_build.utils.paths import CHROOT_BASEDIR -from .manifest import UPDATE_FILE - - PACKAGE_PATH = os.path.join(CHROOT_BASEDIR, 'packages') @@ -18,18 +11,3 @@ def run_in_chroot(command, exception_message=None, **kwargs): return run( ['chroot', CHROOT_BASEDIR] + command, exception_msg=exception_message, env={**APT_ENV, **os.environ}, **kwargs ) - - -@functools.cache -def get_image_version(): - if not os.path.exists(UPDATE_FILE): - raise CallError(f'{UPDATE_FILE!r} update file does not exist') - - with tempfile.TemporaryDirectory() as mount_dir: - try: - run(['mount', UPDATE_FILE, mount_dir, '-t', 'squashfs', '-o', 'loop'], log=False) - with open(os.path.join(mount_dir, 'manifest.json'), 'r') as f: - update_manifest = json.loads(f.read()) - return update_manifest['version'] - finally: - run(['umount', '-f', mount_dir], log=False) diff --git a/scale_build/iso.py b/scale_build/iso.py index d881190..a33ad71 100644 --- a/scale_build/iso.py +++ b/scale_build/iso.py @@ -6,8 +6,7 @@ from .bootstrap.bootstrapdir import CdromBootstrapDirectory from .exceptions import CallError from .image.bootstrap import clean_mounts, setup_chroot_basedir, umount_tmpfs_and_clean_chroot_dir from .image.iso import install_iso_packages, make_iso_file -from .image.manifest import UPDATE_FILE -from .image.utils import get_image_version +from .image.manifest import get_image_version, update_file_path from .utils.logger import LoggingContext from .utils.paths import LOG_DIR, RELEASE_DIR @@ -27,7 +26,7 @@ def build_impl(): for f in glob.glob(os.path.join(LOG_DIR, 'cdrom*')): os.unlink(f) - if not os.path.exists(UPDATE_FILE): + if not os.path.exists(update_file_path()): raise CallError('Missing rootfs image. Run \'make update\' first.') logger.debug('Bootstrapping CD chroot [ISO] (%s/cdrom-bootstrap.log)', LOG_DIR) diff --git a/scale_build/update_image.py b/scale_build/update_image.py index 9ffb5bd..fbe8448 100644 --- a/scale_build/update_image.py +++ b/scale_build/update_image.py @@ -3,7 +3,7 @@ import os from .bootstrap.bootstrapdir import PackageBootstrapDirectory from .image.bootstrap import clean_mounts, setup_chroot_basedir, umount_tmpfs_and_clean_chroot_dir -from .image.manifest import UPDATE_FILE +from .image.manifest import update_file_path from .image.update import install_rootfs_packages, build_rootfs_image from .utils.logger import LoggingContext from .utils.paths import CHROOT_BASEDIR, LOG_DIR, RELEASE_DIR @@ -41,4 +41,4 @@ def build_update_image_impl(): finally: umount_tmpfs_and_clean_chroot_dir() - logger.info('Success! Update image created at: %s', UPDATE_FILE) + logger.info('Success! Update image created at: %s', update_file_path())