diff --git a/.github/workflows/manifest.yml b/.github/workflows/manifest.yml index 1984b7d..c11b918 100644 --- a/.github/workflows/manifest.yml +++ b/.github/workflows/manifest.yml @@ -13,9 +13,7 @@ jobs: uses: actions/setup-python@v1 with: python-version: 3.9 - - name: Install dependencies + - name: Validating manifest run: | python -m pip install --upgrade pip - pip install -r requirements.txt - - name: Validating manifest - run: ./scripts/validate_manifest.py validate --path=./conf/build.manifest + make validate_manifest PYTHON=`which python` diff --git a/Makefile b/Makefile index c7b360f..1cc69fd 100644 --- a/Makefile +++ b/Makefile @@ -16,13 +16,17 @@ endif all: checkout packages update iso -clean: check +clean: validate . ./venv-${COMMIT_HASH}/bin/activate && scale_build clean -checkout: check +checkout: validate . ./venv-${COMMIT_HASH}/bin/activate && scale_build checkout -iso: check +iso: validate . ./venv-${COMMIT_HASH}/bin/activate && scale_build iso -packages: check +packages: validate . ./venv-${COMMIT_HASH}/bin/activate && scale_build packages -update: check +update: validate . ./venv-${COMMIT_HASH}/bin/activate && scale_build update +validate_manifest: check + . ./venv-${COMMIT_HASH}/bin/activate && scale_build validate --no-validate-system_state +validate: check + . ./venv-${COMMIT_HASH}/bin/activate && scale_build validate diff --git a/requirements.txt b/requirements.txt index 3af546d..f7476a8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,7 @@ chardet==4.0.0 coloredlogs==15.0 humanfriendly==9.1 idna==2.10 +jsonschema==3.2.0 pexpect==4.8.0 psutil==5.8.0 ptyprocess==0.7.0 @@ -10,4 +11,3 @@ PyYAML==5.4.1 requests==2.25.1 toposort==1.6 urllib3==1.26.4 -jsonschema==3.2.0 diff --git a/scale_build/exceptions.py b/scale_build/exceptions.py index aa17be3..a44741b 100644 --- a/scale_build/exceptions.py +++ b/scale_build/exceptions.py @@ -11,11 +11,6 @@ class MissingManifest(CallError): super().__init__('Unable to locate manifest file') -class InvalidManifest(CallError): - def __init__(self): - super().__init__('Invalid manifest file found') - - class MissingPackagesException(CallError): def __init__(self, packages): super().__init__(f'Failed preflight check. Please install {", ".join(packages)!r} packages.') diff --git a/scale_build/main.py b/scale_build/main.py index 91e85fe..f381012 100644 --- a/scale_build/main.py +++ b/scale_build/main.py @@ -13,6 +13,7 @@ from .package import build_packages from .preflight import preflight_check from .update_image import build_update_image from .utils.manifest import get_manifest +from .validate import validate logger = logging.getLogger(__name__) @@ -49,6 +50,11 @@ def main(): subparsers.add_parser('packages', help='Build TrueNAS Scale packages') subparsers.add_parser('update', help='Create TrueNAS Scale update image') subparsers.add_parser('iso', help='Create TrueNAS Scale iso installation file') + validate_parser = subparsers.add_parser('validate', help='Validate TrueNAS Scale build manifest and system state') + for action in ('manifest', 'system_state'): + validate_parser.add_argument(f'--validate-{action}', dest=action, action='store_true') + validate_parser.add_argument(f'--no-validate-{action}', dest=action, action='store_false') + validate_parser.set_defaults(**{action: True}) args = parser.parse_args() if args.action == 'checkout': @@ -63,5 +69,7 @@ def main(): build_iso() elif args.action == 'clean': complete_cleanup() + elif args.action == 'validate': + validate(args.system_state, args.manifest) else: parser.print_help() diff --git a/scale_build/package.py b/scale_build/package.py index abda302..ef5d6bd 100644 --- a/scale_build/package.py +++ b/scale_build/package.py @@ -9,6 +9,7 @@ from toposort import toposort from .bootstrap.bootstrapdir import PackageBootstrapDirectory from .clean import clean_bootstrap_logs from .config import PARALLEL_BUILD, PKG_DEBUG +from .exceptions import CallError from .packages.order import get_initialized_packages, get_to_build_packages from .utils.logger import get_logger from .utils.paths import LOG_DIR, PKG_DIR, PKG_LOG_DIR @@ -163,5 +164,7 @@ def _build_packages_impl(): for p in failed.values(): p['package'].delete_overlayfs() + raise CallError(f'{", ".join(failed)!r} Packages failed to build') + else: logger.info('Success! Done building packages') diff --git a/scale_build/preflight.py b/scale_build/preflight.py index fad5e62..c5e9bed 100644 --- a/scale_build/preflight.py +++ b/scale_build/preflight.py @@ -1,31 +1,12 @@ import logging import os -import shutil -from .exceptions import CallError, MissingPackagesException -from .utils.manifest import get_manifest from .utils.system import has_low_ram from .utils.paths import CACHE_DIR, HASH_DIR, LOG_DIR, PKG_DIR, PKG_LOG_DIR, SOURCES_DIR, TMP_DIR, TMPFS logger = logging.getLogger(__name__) -WANTED_PACKAGES = { - 'make', - 'debootstrap', - 'git', - 'mksquashfs', - 'unzip', -} - - -def is_root(): - return os.geteuid() == 0 - - -def retrieve_missing_packages(): - return {pkg for pkg in WANTED_PACKAGES if not shutil.which(pkg)} - def setup_dirs(): for d in (CACHE_DIR, TMP_DIR, HASH_DIR, LOG_DIR, PKG_DIR, PKG_LOG_DIR, SOURCES_DIR, TMPFS): @@ -33,16 +14,7 @@ def setup_dirs(): def preflight_check(): - if not is_root(): - raise CallError('Must be run as root (or using sudo)!') - - missing_packages = retrieve_missing_packages() - if missing_packages: - raise MissingPackagesException(missing_packages) - if has_low_ram(): logging.warning('WARNING: Running with less than 16GB of memory. Build may fail...') setup_dirs() - # TODO: Validate contents of manifest like empty string is not provided for source name/repo etc - get_manifest() diff --git a/scale_build/utils/manifest.py b/scale_build/utils/manifest.py index 5314fb7..edf87ec 100644 --- a/scale_build/utils/manifest.py +++ b/scale_build/utils/manifest.py @@ -1,26 +1,136 @@ +import functools +import jsonschema import yaml from scale_build.config import TRAIN -from scale_build.exceptions import MissingManifest, InvalidManifest +from scale_build.exceptions import CallError, MissingManifest from scale_build.utils.paths import MANIFEST -manifest = None +MANIFEST_SCHEMA = { + 'type': 'object', + 'properties': { + 'code_name': {'type': 'string'}, + 'debian_release': {'type': 'string'}, + 'apt-repos': { + 'type': 'object', + 'properties': { + 'url': {'type': 'string'}, + 'distribution': {'type': 'string'}, + 'components': {'type': 'string'}, + 'additional': { + 'type': 'array', + 'items': [{ + 'type': 'object', + 'properties': { + 'url': {'type': 'string'}, + 'distribution': {'type': 'string'}, + 'component': {'type': 'string'}, + 'key': {'type': 'string'}, + }, + 'required': ['url', 'distribution', 'component', 'key'], + }] + } + }, + 'required': ['url', 'distribution', 'components', 'additional'], + }, + 'base-packages': { + 'type': 'array', + 'items': [{'type': 'string'}], + }, + 'base-prune': { + 'type': 'array', + 'items': [{'type': 'string'}], + }, + 'build-epoch': {'type': 'integer'}, + 'apt_preferences': { + 'type': 'array', + 'items': [{ + 'type': 'object', + 'properties': { + 'Package': {'type': 'string'}, + 'Pin': {'type': 'string'}, + 'Pin-Priority': {'type': 'integer'}, + }, + 'required': ['Package', 'Pin', 'Pin-Priority'], + }] + }, + 'additional-packages': { + 'type': 'array', + 'items': [{ + 'type': 'object', + 'properties': { + 'package': {'type': 'string'}, + 'comment': {'type': 'string'}, + }, + 'required': ['package', 'comment'], + }] + }, + 'iso-packages': { + 'type': 'array', + 'items': [{'type': 'string'}], + }, + 'sources': { + 'type': 'array', + 'items': [{ + 'type': 'object', + 'properties': { + 'name': {'type': 'string'}, + 'repo': {'type': 'string'}, + 'branch': {'type': 'string'}, + 'batch_priority': {'type': 'integer'}, + 'predepscmd': { + 'type': 'array', + 'items': [{'type': 'string'}], + }, + 'buildcmd': { + 'type': 'array', + 'items': [{'type': 'string'}], + }, + 'prebuildcmd': { + 'type': 'array', + 'items': [{'type': 'string'}], + }, + 'deps_path': {'type': 'string'}, + 'kernel_module': {'type': 'boolean'}, + 'generate_version': {'type': 'boolean'}, + 'explicit_deps': { + 'type': 'array', + 'items': [{'type': 'string'}], + }, + 'subdir': {'type': 'string'}, + 'deoptions': {'type': 'string'}, + 'jobs': {'type': 'integer'}, + }, + 'required': ['name', 'branch', 'repo'], + }] + }, + }, + 'required': [ + 'code_name', + 'debian_release', + 'apt-repos', + 'base-packages', + 'base-prune', + 'build-epoch', + 'apt_preferences', + 'additional-packages', + 'iso-packages', + 'sources' + ], +} +@functools.cache def get_manifest(): - global manifest - if not manifest: - try: - with open(MANIFEST, 'r') as f: - manifest = yaml.safe_load(f.read()) - return manifest - except FileNotFoundError: - raise MissingManifest() - except yaml.YAMLError: - raise InvalidManifest() - else: - return manifest + try: + with open(MANIFEST, 'r') as f: + manifest = yaml.safe_load(f.read()) + return manifest + except FileNotFoundError: + raise MissingManifest() + except yaml.YAMLError: + raise CallError('Provided manifest has invalid format') def get_release_code_name(): @@ -29,3 +139,10 @@ def get_release_code_name(): def get_truenas_train(): return TRAIN or f'TrueNAS-SCALE-{get_release_code_name()}-Nightlies' + + +def validate_manifest(): + try: + jsonschema.validate(get_manifest(), MANIFEST_SCHEMA) + except jsonschema.ValidationError as e: + raise CallError(f'Provided manifest is invalid: {e}') diff --git a/scale_build/validate.py b/scale_build/validate.py new file mode 100644 index 0000000..6cb3bab --- /dev/null +++ b/scale_build/validate.py @@ -0,0 +1,39 @@ +import os +import logging +import shutil + +from .exceptions import CallError, MissingPackagesException +from .utils.manifest import validate_manifest + + +logger = logging.getLogger(__name__) + +WANTED_PACKAGES = { + 'make', + 'debootstrap', + 'git', + 'mksquashfs', + 'unzip', +} + + +def retrieve_missing_packages(): + return {pkg for pkg in WANTED_PACKAGES if not shutil.which(pkg)} + + +def validate_system_state(): + if os.geteuid() != 0: + raise CallError('Must be run as root (or using sudo)!') + + missing_packages = retrieve_missing_packages() + if missing_packages: + raise MissingPackagesException(missing_packages) + + +def validate(system_state_flag=True, manifest_flag=True): + if system_state_flag: + validate_system_state() + logger.debug('System state Validated') + if manifest_flag: + validate_manifest() + logger.debug('Manifest Validated') diff --git a/scripts/validate_manifest.py b/scripts/validate_manifest.py deleted file mode 100755 index 93ea1f9..0000000 --- a/scripts/validate_manifest.py +++ /dev/null @@ -1,159 +0,0 @@ -#!/usr/bin/env python3 -import argparse -import jsonschema -import yaml - - -def validate(manifest_path): - - error_str = None - try: - with open(manifest_path, 'r') as f: - manifest = yaml.safe_load(f.read()) - except FileNotFoundError: - error_str = f'{manifest_path!r} does not exist' - except yaml.YAMLError: - error_str = f'Unable to read {manifest_path!r} contents. Can you please confirm format is valid ?' - - if error_str: - print(f'[\033[91mFAILED\x1B[0m]\t{error_str}') - exit(1) - - schema = { - 'type': 'object', - 'properties': { - 'code_name': {'type': 'string'}, - 'debian_release': {'type': 'string'}, - 'apt-repos': { - 'type': 'object', - 'properties': { - 'url': {'type': 'string'}, - 'distribution': {'type': 'string'}, - 'components': {'type': 'string'}, - 'additional': { - 'type': 'array', - 'items': [{ - 'type': 'object', - 'properties': { - 'url': {'type': 'string'}, - 'distribution': {'type': 'string'}, - 'component': {'type': 'string'}, - 'key': {'type': 'string'}, - }, - 'required': ['url', 'distribution', 'component', 'key'], - }] - } - }, - 'required': ['url', 'distribution', 'components', 'additional'], - }, - 'base-packages': { - 'type': 'array', - 'items': [{'type': 'string'}], - }, - 'base-prune': { - 'type': 'array', - 'items': [{'type': 'string'}], - }, - 'build-epoch': {'type': 'integer'}, - 'apt_preferences': { - 'type': 'array', - 'items': [{ - 'type': 'object', - 'properties': { - 'Package': {'type': 'string'}, - 'Pin': {'type': 'string'}, - 'Pin-Priority': {'type': 'integer'}, - }, - 'required': ['Package', 'Pin', 'Pin-Priority'], - }] - }, - 'additional-packages': { - 'type': 'array', - 'items': [{ - 'type': 'object', - 'properties': { - 'package': {'type': 'string'}, - 'comment': {'type': 'string'}, - }, - 'required': ['package', 'comment'], - }] - }, - 'iso-packages': { - 'type': 'array', - 'items': [{'type': 'string'}], - }, - 'sources': { - 'type': 'array', - 'items': [{ - 'type': 'object', - 'properties': { - 'name': {'type': 'string'}, - 'repo': {'type': 'string'}, - 'branch': {'type': 'string'}, - 'batch_priority': {'type': 'integer'}, - 'predepscmd': { - 'type': 'array', - 'items': [{'type': 'string'}], - }, - 'buildcmd': { - 'type': 'array', - 'items': [{'type': 'string'}], - }, - 'prebuildcmd': { - 'type': 'array', - 'items': [{'type': 'string'}], - }, - 'deps_path': {'type': 'string'}, - 'kernel_module': {'type': 'boolean'}, - 'generate_version': {'type': 'boolean'}, - 'explicit_deps': { - 'type': 'array', - 'items': [{'type': 'string'}], - }, - 'subdir': {'type': 'string'}, - 'deoptions': {'type': 'string'}, - 'jobs': {'type': 'integer'}, - }, - 'required': ['name', 'branch', 'repo'], - }] - }, - }, - 'required': [ - 'code_name', - 'debian_release', - 'apt-repos', - 'base-packages', - 'base-prune', - 'build-epoch', - 'apt_preferences', - 'additional-packages', - 'iso-packages', - 'sources' - ], - } - - try: - jsonschema.validate(manifest, schema) - except jsonschema.ValidationError as e: - print(f'[\033[91mFAILED\x1B[0m]\tFailed to validate manifest: {e}') - exit(1) - else: - print('[\033[92mOK\x1B[0m]\tManifest validated') - - -def main(): - parser = argparse.ArgumentParser() - subparsers = parser.add_subparsers(help='sub-command help', dest='action') - - parser_setup = subparsers.add_parser('validate', help='Validate TrueNAS Scale build manifest') - parser_setup.add_argument('--path', help='Specify path of build manifest') - - args = parser.parse_args() - if args.action == 'validate': - validate(args.path) - else: - parser.print_help() - - -if __name__ == '__main__': - main()