scale-build/scale_build/utils/manifest.py

import functools
import jsonschema
import re
import yaml

from urllib.parse import urlparse

from scale_build.config import SKIP_SOURCE_REPO_VALIDATION, TRAIN
from scale_build.exceptions import CallError, MissingManifest
from scale_build.utils.paths import MANIFEST


BRANCH_REGEX = re.compile(r'(branch\s*:\s*)\b[\w/\.-]+\b')
SSH_SOURCE_REGEX = re.compile(r'^[\w]+@(\w.+):(\w.+)')

MANIFEST_SCHEMA = {
    'type': 'object',
    'properties': {
        'code_name': {'type': 'string'},
        'debian_release': {'type': 'string'},
        'identity_file_path_default': {'type': 'string'},
        'apt-repos': {
            'type': 'object',
            'properties': {
                'url': {'type': 'string'},
                'distribution': {'type': 'string'},
                'components': {'type': 'string'},
                'additional': {
                    'type': 'array',
                    'items': [{
                        'type': 'object',
                        'properties': {
                            'url': {'type': 'string'},
                            'distribution': {'type': 'string'},
                            'component': {'type': 'string'},
                            'key': {'type': 'string'},
                        },
                        'required': ['url', 'distribution', 'component', 'key'],
                    }]
                }
            },
            'required': ['url', 'distribution', 'components', 'additional'],
        },
        'base-packages': {
            'type': 'array',
            'items': [{'type': 'string'}],
        },
        'base-prune': {
            'type': 'array',
            'items': [{'type': 'string'}],
        },
        'build-epoch': {'type': 'integer'},
        'apt_preferences': {
            'type': 'array',
            'items': [{
                'type': 'object',
                'properties': {
                    'Package': {'type': 'string'},
                    'Pin': {'type': 'string'},
                    'Pin-Priority': {'type': 'integer'},
                },
                'required': ['Package', 'Pin', 'Pin-Priority'],
            }]
        },
        'additional-packages': {
            'type': 'array',
            'items': [{
                'type': 'object',
                'properties': {
                    'package': {'type': 'string'},
                    'comment': {'type': 'string'},
                },
                'required': ['package', 'comment'],
            }]
        },
        'iso-packages': {
            'type': 'array',
            'items': [{'type': 'string'}],
        },
        'sources': {
            'type': 'array',
            'items': [{
                'type': 'object',
                'properties': {
                    'name': {'type': 'string'},
                    'repo': {'type': 'string'},
                    'identity_file_path': {'type': 'string'},
                    'branch': {'type': 'string'},
                    'batch_priority': {'type': 'integer'},
                    'predepscmd': {
                        'type': 'array',
                        'items': [{'type': 'string'}],
                    },
                    'build_constraints': {
                        'type': 'array',
                        'items': [{
                            'type': 'object',
                            'properties': {
                                'name': {'type': 'string'},
                                'value': {
                                    'anyOf': [
                                        {'type': 'string'},
                                        {'type': 'integer'},
                                        {'type': 'boolean'},
                                    ],
                                },
                                'type': {
                                    'type': 'string',
                                    'enum': ['boolean', 'string', 'integer'],
                                },
                            },
                            'required': ['name', 'value', 'type'],
                        }],
                    },
                    'buildcmd': {
                        'type': 'array',
                        'items': [{'type': 'string'}],
                    },
                    'prebuildcmd': {
                        'type': 'array',
                        'items': [{'type': 'string'}],
                    },
                    'deps_path': {'type': 'string'},
                    'kernel_module': {'type': 'boolean'},
                    'generate_version': {'type': 'boolean'},
                    'explicit_deps': {
                        'type': 'array',
                        'items': [{'type': 'string'}],
                    },
                    'subdir': {'type': 'string'},
                    'deoptions': {'type': 'string'},
                    'jobs': {'type': 'integer'},
                    'debian_fork': {'type': 'boolean'},
                },
                'required': ['name', 'branch', 'repo'],
                'additionalProperties': False,
            }]
        },
    },
    'required': [
        'code_name',
        'debian_release',
        'apt-repos',
        'base-packages',
        'base-prune',
        'build-epoch',
        'apt_preferences',
        'additional-packages',
        'iso-packages',
        'sources'
    ],
}


def get_manifest_str():
    try:
        with open(MANIFEST, 'r') as f:
            return f.read()
    except FileNotFoundError:
        raise MissingManifest()


def validate_apt_preferences_order(manifest):
    packages = [p['Package'] for p in manifest['apt_preferences']]
    if sorted(packages, key=lambda k: k.strip('*')) != packages:
        raise CallError('Please list down apt preferences in alphabetical order')


@functools.cache
def get_manifest():
    try:
        manifest = yaml.safe_load(get_manifest_str())
        jsonschema.validate(manifest, MANIFEST_SCHEMA)
        validate_apt_preferences_order(manifest)
        return manifest
    except yaml.YAMLError:
        raise CallError('Provided manifest has invalid format')
    except jsonschema.ValidationError as e:
        raise CallError(f'Provided manifest is invalid: {e}')


def get_release_code_name():
    return get_manifest()['code_name']


def get_truenas_train():
    return TRAIN or f'TrueNAS-SCALE-{get_release_code_name()}-Nightlies'


def update_packages_branch(branch_name):
    # We would like to update branches but if we use python module, we would lose the comments which is not desired
    # Let's please use regex and find a better way to do this in the future
    manifest_str = get_manifest_str()
    updated_str = BRANCH_REGEX.sub(fr'\1{branch_name}', manifest_str)

    with open(MANIFEST, 'w') as f:
        f.write(updated_str)


def validate_manifest():
    manifest = get_manifest()
    if SKIP_SOURCE_REPO_VALIDATION:
        return

    # We would like to make sure that each package source we build from is from our fork and not another one
    invalid_packages = []
    for package in manifest['sources']:
        repo_source = package['repo']
        if url := SSH_SOURCE_REGEX.findall(repo_source):
            hostname, repo_path = url[0]
        else:
            url = urlparse(repo_source)
            hostname = url.hostname
            repo_path = url.path

        if hostname not in ['github.com', 'www.github.com'] or not repo_path.lower().strip('/').startswith((
            'truenas/', 'ixsystems/'
        )):
            invalid_packages.append(package['name'])

    if invalid_packages:
        raise CallError(
            f'{",".join(invalid_packages)!r} are using repos from unsupported git upstream. Scale-build only '
            'accepts packages from github.com/truenas organisation (To skip this for dev '
            'purposes, please set "SKIP_SOURCE_REPO_VALIDATION" in your environment).'
        )