Validate manifet when executing other targets

.github/workflows/manifest.yml

@@ -13,9 +13,7 @@ jobs:
       uses: actions/setup-python@v1
       with:
         python-version: 3.9
-    - name: Install dependencies
+    - name: Validating manifest
       run: |
         python -m pip install --upgrade pip
-        pip install -r requirements.txt
-    - name: Validating manifest
-      run: ./scripts/validate_manifest.py validate --path=./conf/build.manifest
+        make validate_manifest PYTHON=`which python`

Makefile

@@ -16,13 +16,17 @@ endif
 
 all: checkout packages update iso
 
-clean: check
+clean: validate
 	. ./venv-${COMMIT_HASH}/bin/activate && scale_build clean
-checkout: check
+checkout: validate
 	. ./venv-${COMMIT_HASH}/bin/activate && scale_build checkout
-iso: check
+iso: validate
 	. ./venv-${COMMIT_HASH}/bin/activate && scale_build iso
-packages: check
+packages: validate
 	. ./venv-${COMMIT_HASH}/bin/activate && scale_build packages
-update: check
+update: validate
 	. ./venv-${COMMIT_HASH}/bin/activate && scale_build update
+validate_manifest: check
+	. ./venv-${COMMIT_HASH}/bin/activate && scale_build validate --no-validate-system_state
+validate: check
+	. ./venv-${COMMIT_HASH}/bin/activate && scale_build validate

requirements.txt

@@ -3,6 +3,7 @@ chardet==4.0.0
 coloredlogs==15.0
 humanfriendly==9.1
 idna==2.10
+jsonschema==3.2.0
 pexpect==4.8.0
 psutil==5.8.0
 ptyprocess==0.7.0
@@ -10,4 +11,3 @@ PyYAML==5.4.1
 requests==2.25.1
 toposort==1.6
 urllib3==1.26.4
-jsonschema==3.2.0

scale_build/exceptions.py

@@ -11,11 +11,6 @@ class MissingManifest(CallError):
         super().__init__('Unable to locate manifest file')
 
 
-class InvalidManifest(CallError):
-    def __init__(self):
-        super().__init__('Invalid manifest file found')
-
-
 class MissingPackagesException(CallError):
     def __init__(self, packages):
         super().__init__(f'Failed preflight check. Please install {", ".join(packages)!r} packages.')

scale_build/main.py

@@ -13,6 +13,7 @@ from .package import build_packages
 from .preflight import preflight_check
 from .update_image import build_update_image
 from .utils.manifest import get_manifest
+from .validate import validate
 
 
 logger = logging.getLogger(__name__)
@@ -49,6 +50,11 @@ def main():
     subparsers.add_parser('packages', help='Build TrueNAS Scale packages')
     subparsers.add_parser('update', help='Create TrueNAS Scale update image')
     subparsers.add_parser('iso', help='Create TrueNAS Scale iso installation file')
+    validate_parser = subparsers.add_parser('validate', help='Validate TrueNAS Scale build manifest and system state')
+    for action in ('manifest', 'system_state'):
+        validate_parser.add_argument(f'--validate-{action}', dest=action, action='store_true')
+        validate_parser.add_argument(f'--no-validate-{action}', dest=action, action='store_false')
+        validate_parser.set_defaults(**{action: True})
 
     args = parser.parse_args()
     if args.action == 'checkout':
@@ -63,5 +69,7 @@ def main():
         build_iso()
     elif args.action == 'clean':
         complete_cleanup()
+    elif args.action == 'validate':
+        validate(args.system_state, args.manifest)
     else:
         parser.print_help()

scale_build/package.py

@@ -9,6 +9,7 @@ from toposort import toposort
 from .bootstrap.bootstrapdir import PackageBootstrapDirectory
 from .clean import clean_bootstrap_logs
 from .config import PARALLEL_BUILD, PKG_DEBUG
+from .exceptions import CallError
 from .packages.order import get_initialized_packages, get_to_build_packages
 from .utils.logger import get_logger
 from .utils.paths import LOG_DIR, PKG_DIR, PKG_LOG_DIR
@@ -163,5 +164,7 @@ def _build_packages_impl():
             for p in failed.values():
                 p['package'].delete_overlayfs()
 
+        raise CallError(f'{", ".join(failed)!r} Packages failed to build')
+
     else:
         logger.info('Success! Done building packages')

scale_build/preflight.py

@@ -1,31 +1,12 @@
 import logging
 import os
-import shutil
 
-from .exceptions import CallError, MissingPackagesException
-from .utils.manifest import get_manifest
 from .utils.system import has_low_ram
 from .utils.paths import CACHE_DIR, HASH_DIR, LOG_DIR, PKG_DIR, PKG_LOG_DIR, SOURCES_DIR, TMP_DIR, TMPFS
 
 
 logger = logging.getLogger(__name__)
 
-WANTED_PACKAGES = {
-    'make',
-    'debootstrap',
-    'git',
-    'mksquashfs',
-    'unzip',
-}
-
-
-def is_root():
-    return os.geteuid() == 0
-
-
-def retrieve_missing_packages():
-    return {pkg for pkg in WANTED_PACKAGES if not shutil.which(pkg)}
-
 
 def setup_dirs():
     for d in (CACHE_DIR, TMP_DIR, HASH_DIR, LOG_DIR, PKG_DIR, PKG_LOG_DIR, SOURCES_DIR, TMPFS):
@@ -33,16 +14,7 @@ def setup_dirs():
 
 
 def preflight_check():
-    if not is_root():
-        raise CallError('Must be run as root (or using sudo)!')
-
-    missing_packages = retrieve_missing_packages()
-    if missing_packages:
-        raise MissingPackagesException(missing_packages)
-
     if has_low_ram():
         logging.warning('WARNING: Running with less than 16GB of memory. Build may fail...')
 
     setup_dirs()
-    # TODO: Validate contents of manifest like empty string is not provided for source name/repo etc
-    get_manifest()

scale_build/utils/manifest.py

@@ -1,16 +1,128 @@
+import functools
+import jsonschema
 import yaml
 
 from scale_build.config import TRAIN
-from scale_build.exceptions import MissingManifest, InvalidManifest
+from scale_build.exceptions import CallError, MissingManifest
 from scale_build.utils.paths import MANIFEST
 
 
-manifest = None
+MANIFEST_SCHEMA = {
+    'type': 'object',
+    'properties': {
+        'code_name': {'type': 'string'},
+        'debian_release': {'type': 'string'},
+        'apt-repos': {
+            'type': 'object',
+            'properties': {
+                'url': {'type': 'string'},
+                'distribution': {'type': 'string'},
+                'components': {'type': 'string'},
+                'additional': {
+                    'type': 'array',
+                    'items': [{
+                        'type': 'object',
+                        'properties': {
+                            'url': {'type': 'string'},
+                            'distribution': {'type': 'string'},
+                            'component': {'type': 'string'},
+                            'key': {'type': 'string'},
+                        },
+                        'required': ['url', 'distribution', 'component', 'key'],
+                    }]
+                }
+            },
+            'required': ['url', 'distribution', 'components', 'additional'],
+        },
+        'base-packages': {
+            'type': 'array',
+            'items': [{'type': 'string'}],
+        },
+        'base-prune': {
+            'type': 'array',
+            'items': [{'type': 'string'}],
+        },
+        'build-epoch': {'type': 'integer'},
+        'apt_preferences': {
+            'type': 'array',
+            'items': [{
+                'type': 'object',
+                'properties': {
+                    'Package': {'type': 'string'},
+                    'Pin': {'type': 'string'},
+                    'Pin-Priority': {'type': 'integer'},
+                },
+                'required': ['Package', 'Pin', 'Pin-Priority'],
+            }]
+        },
+        'additional-packages': {
+            'type': 'array',
+            'items': [{
+                'type': 'object',
+                'properties': {
+                    'package': {'type': 'string'},
+                    'comment': {'type': 'string'},
+                },
+                'required': ['package', 'comment'],
+            }]
+        },
+        'iso-packages': {
+            'type': 'array',
+            'items': [{'type': 'string'}],
+        },
+        'sources': {
+            'type': 'array',
+            'items': [{
+                'type': 'object',
+                'properties': {
+                    'name': {'type': 'string'},
+                    'repo': {'type': 'string'},
+                    'branch': {'type': 'string'},
+                    'batch_priority': {'type': 'integer'},
+                    'predepscmd': {
+                        'type': 'array',
+                        'items': [{'type': 'string'}],
+                    },
+                    'buildcmd': {
+                        'type': 'array',
+                        'items': [{'type': 'string'}],
+                    },
+                    'prebuildcmd': {
+                        'type': 'array',
+                        'items': [{'type': 'string'}],
+                    },
+                    'deps_path': {'type': 'string'},
+                    'kernel_module': {'type': 'boolean'},
+                    'generate_version': {'type': 'boolean'},
+                    'explicit_deps': {
+                        'type': 'array',
+                        'items': [{'type': 'string'}],
+                    },
+                    'subdir': {'type': 'string'},
+                    'deoptions': {'type': 'string'},
+                    'jobs': {'type': 'integer'},
+                },
+                'required': ['name', 'branch', 'repo'],
+            }]
+        },
+    },
+    'required': [
+        'code_name',
+        'debian_release',
+        'apt-repos',
+        'base-packages',
+        'base-prune',
+        'build-epoch',
+        'apt_preferences',
+        'additional-packages',
+        'iso-packages',
+        'sources'
+    ],
+}
 
 
+@functools.cache
 def get_manifest():
-    global manifest
-    if not manifest:
     try:
         with open(MANIFEST, 'r') as f:
             manifest = yaml.safe_load(f.read())
@@ -18,9 +130,7 @@ def get_manifest():
     except FileNotFoundError:
         raise MissingManifest()
     except yaml.YAMLError:
-            raise InvalidManifest()
-    else:
-        return manifest
+        raise CallError('Provided manifest has invalid format')
 
 
 def get_release_code_name():
@@ -29,3 +139,10 @@ def get_release_code_name():
 
 def get_truenas_train():
     return TRAIN or f'TrueNAS-SCALE-{get_release_code_name()}-Nightlies'
+
+
+def validate_manifest():
+    try:
+        jsonschema.validate(get_manifest(), MANIFEST_SCHEMA)
+    except jsonschema.ValidationError as e:
+        raise CallError(f'Provided manifest is invalid: {e}')

scale_build/validate.py

@@ -0,0 +1,39 @@
+import os
+import logging
+import shutil
+
+from .exceptions import CallError, MissingPackagesException
+from .utils.manifest import validate_manifest
+
+
+logger = logging.getLogger(__name__)
+
+WANTED_PACKAGES = {
+    'make',
+    'debootstrap',
+    'git',
+    'mksquashfs',
+    'unzip',
+}
+
+
+def retrieve_missing_packages():
+    return {pkg for pkg in WANTED_PACKAGES if not shutil.which(pkg)}
+
+
+def validate_system_state():
+    if os.geteuid() != 0:
+        raise CallError('Must be run as root (or using sudo)!')
+
+    missing_packages = retrieve_missing_packages()
+    if missing_packages:
+        raise MissingPackagesException(missing_packages)
+
+
+def validate(system_state_flag=True, manifest_flag=True):
+    if system_state_flag:
+        validate_system_state()
+        logger.debug('System state Validated')
+    if manifest_flag:
+        validate_manifest()
+        logger.debug('Manifest Validated')

scripts/validate_manifest.py

@@ -1,159 +0,0 @@
-#!/usr/bin/env python3
-import argparse
-import jsonschema
-import yaml
-
-
-def validate(manifest_path):
-
-    error_str = None
-    try:
-        with open(manifest_path, 'r') as f:
-            manifest = yaml.safe_load(f.read())
-    except FileNotFoundError:
-        error_str = f'{manifest_path!r} does not exist'
-    except yaml.YAMLError:
-        error_str = f'Unable to read {manifest_path!r} contents. Can you please confirm format is valid ?'
-
-    if error_str:
-        print(f'[\033[91mFAILED\x1B[0m]\t{error_str}')
-        exit(1)
-
-    schema = {
-        'type': 'object',
-        'properties': {
-            'code_name': {'type': 'string'},
-            'debian_release': {'type': 'string'},
-            'apt-repos': {
-                'type': 'object',
-                'properties': {
-                    'url': {'type': 'string'},
-                    'distribution': {'type': 'string'},
-                    'components': {'type': 'string'},
-                    'additional': {
-                        'type': 'array',
-                        'items': [{
-                            'type': 'object',
-                            'properties': {
-                                'url': {'type': 'string'},
-                                'distribution': {'type': 'string'},
-                                'component': {'type': 'string'},
-                                'key': {'type': 'string'},
-                            },
-                            'required': ['url', 'distribution', 'component', 'key'],
-                        }]
-                    }
-                },
-                'required': ['url', 'distribution', 'components', 'additional'],
-            },
-            'base-packages': {
-                'type': 'array',
-                'items': [{'type': 'string'}],
-            },
-            'base-prune': {
-                'type': 'array',
-                'items': [{'type': 'string'}],
-            },
-            'build-epoch': {'type': 'integer'},
-            'apt_preferences': {
-                'type': 'array',
-                'items': [{
-                    'type': 'object',
-                    'properties': {
-                        'Package': {'type': 'string'},
-                        'Pin': {'type': 'string'},
-                        'Pin-Priority': {'type': 'integer'},
-                    },
-                    'required': ['Package', 'Pin', 'Pin-Priority'],
-                }]
-            },
-            'additional-packages': {
-                'type': 'array',
-                'items': [{
-                    'type': 'object',
-                    'properties': {
-                        'package': {'type': 'string'},
-                        'comment': {'type': 'string'},
-                    },
-                    'required': ['package', 'comment'],
-                }]
-            },
-            'iso-packages': {
-                'type': 'array',
-                'items': [{'type': 'string'}],
-            },
-            'sources': {
-                'type': 'array',
-                'items': [{
-                    'type': 'object',
-                    'properties': {
-                        'name': {'type': 'string'},
-                        'repo': {'type': 'string'},
-                        'branch': {'type': 'string'},
-                        'batch_priority': {'type': 'integer'},
-                        'predepscmd': {
-                            'type': 'array',
-                            'items': [{'type': 'string'}],
-                        },
-                        'buildcmd': {
-                            'type': 'array',
-                            'items': [{'type': 'string'}],
-                        },
-                        'prebuildcmd': {
-                            'type': 'array',
-                            'items': [{'type': 'string'}],
-                        },
-                        'deps_path': {'type': 'string'},
-                        'kernel_module': {'type': 'boolean'},
-                        'generate_version': {'type': 'boolean'},
-                        'explicit_deps': {
-                            'type': 'array',
-                            'items': [{'type': 'string'}],
-                        },
-                        'subdir': {'type': 'string'},
-                        'deoptions': {'type': 'string'},
-                        'jobs': {'type': 'integer'},
-                    },
-                    'required': ['name', 'branch', 'repo'],
-                }]
-            },
-        },
-        'required': [
-            'code_name',
-            'debian_release',
-            'apt-repos',
-            'base-packages',
-            'base-prune',
-            'build-epoch',
-            'apt_preferences',
-            'additional-packages',
-            'iso-packages',
-            'sources'
-        ],
-    }
-
-    try:
-        jsonschema.validate(manifest, schema)
-    except jsonschema.ValidationError as e:
-        print(f'[\033[91mFAILED\x1B[0m]\tFailed to validate manifest: {e}')
-        exit(1)
-    else:
-        print('[\033[92mOK\x1B[0m]\tManifest validated')
-
-
-def main():
-    parser = argparse.ArgumentParser()
-    subparsers = parser.add_subparsers(help='sub-command help', dest='action')
-
-    parser_setup = subparsers.add_parser('validate', help='Validate TrueNAS Scale build manifest')
-    parser_setup.add_argument('--path', help='Specify path of build manifest')
-
-    args = parser.parse_args()
-    if args.action == 'validate':
-        validate(args.path)
-    else:
-        parser.print_help()
-
-
-if __name__ == '__main__':
-    main()