From f1b38c78bb01cf3e065327ed9b81e3782701486e Mon Sep 17 00:00:00 2001 From: themylogin Date: Fri, 14 Oct 2022 15:58:20 +0200 Subject: [PATCH] `make check_upstream_package_updates` to check forked Debian repos and `python-truenas-requirements` --- Makefile | 4 +- conf/build.manifest | 4 ++ requirements.txt | 2 + scale_build/main.py | 4 ++ scale_build/packages/package.py | 3 +- scale_build/upstream_package_updates.py | 94 +++++++++++++++++++++++++ scale_build/utils/manifest.py | 2 + 7 files changed, 111 insertions(+), 2 deletions(-) create mode 100644 scale_build/upstream_package_updates.py diff --git a/Makefile b/Makefile index ae1eeb4..5ba71e4 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ check: ifneq ($(REPO_CHANGED),0) @echo "Setting up new virtual environment" @rm -rf venv-* - @${PYTHON} -m pip install -U virtualenv >/dev/null 2>&1 || { echo "Failed to install/upgrade virtualenv package"; exit 1; } + @${PYTHON} -m pip install -U virtualenv >/dev/null || { echo "Failed to install/upgrade virtualenv package"; exit 1; } @${PYTHON} -m venv venv-${COMMIT_HASH} || { echo "Failed to create virutal environment"; exit 1; } @{ . ./venv-${COMMIT_HASH}/bin/activate && \ python3 -m pip install -r requirements.txt >/dev/null 2>&1 && \ @@ -25,6 +25,8 @@ clean: check . ./venv-${COMMIT_HASH}/bin/activate && scale_build clean checkout: check . ./venv-${COMMIT_HASH}/bin/activate && scale_build checkout +check_upstream_package_updates: check + . ./venv-${COMMIT_HASH}/bin/activate && scale_build check_upstream_package_updates iso: check . ./venv-${COMMIT_HASH}/bin/activate && scale_build iso packages: check diff --git a/conf/build.manifest b/conf/build.manifest index 0aca021..334aee0 100644 --- a/conf/build.manifest +++ b/conf/build.manifest @@ -483,6 +483,7 @@ sources: - name: collectd repo: https://github.com/truenas/collectd branch: master + debian_fork: true predepscmd: - "apt install -y wget xz-utils" - "./pull.sh" @@ -491,6 +492,7 @@ sources: - name: grub2 repo: https://github.com/truenas/grub2 branch: master + debian_fork: true predepscmd: - "apt install -y wget xz-utils" - "./pull.sh" @@ -506,6 +508,7 @@ sources: - name: rrdcached repo: https://github.com/truenas/rrdtool branch: master + debian_fork: true predepscmd: - "apt install -y wget xz-utils" - "./pull.sh" @@ -514,6 +517,7 @@ sources: - name: smartmontools repo: https://github.com/truenas/smartmontools branch: master + debian_fork: true predepscmd: - "apt install -y wget xz-utils" - "./pull.sh" diff --git a/requirements.txt b/requirements.txt index 977aecb..3fa1d01 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,9 +4,11 @@ coloredlogs==15.0 humanfriendly==9.1 idna==2.10 jsonschema==3.2.0 +packaging==21.3 pexpect==4.8.0 psutil==5.8.0 ptyprocess==0.7.0 +pyparsing==3.0.9 PyYAML==5.4.1 requests==2.25.1 toposort==1.6 diff --git a/scale_build/main.py b/scale_build/main.py index e0c1ef0..ac2e150 100644 --- a/scale_build/main.py +++ b/scale_build/main.py @@ -7,6 +7,7 @@ from .branch_out import branch_out_repos, validate_branch_out_config from .checkout import checkout_sources from .clean import complete_cleanup from .config import BRANCH_OVERRIDES +from .upstream_package_updates import check_upstream_package_updates from .epoch import check_epoch from .exceptions import CallError from .iso import build_iso @@ -53,6 +54,7 @@ def main(): subparsers = parser.add_subparsers(help='sub-command help', dest='action') subparsers.add_parser('checkout', help='Checkout TrueNAS Scale repositories') + subparsers.add_parser('check_upstream_package_updates', help='Check that all forked Debian packages are up-to-date') subparsers.add_parser('clean', help='Clean build package(s) / cloned source(s) / image(s) of TrueNAS Scale') packages_parser = subparsers.add_parser('packages', help='Build TrueNAS Scale packages') packages_parser.add_argument( @@ -76,6 +78,8 @@ def main(): if args.action == 'checkout': check_epoch() checkout_sources() + if args.action == 'check_upstream_package_updates': + check_upstream_package_updates() elif args.action == 'packages': validate() check_epoch() diff --git a/scale_build/packages/package.py b/scale_build/packages/package.py index 7bed895..6fc31c1 100644 --- a/scale_build/packages/package.py +++ b/scale_build/packages/package.py @@ -26,7 +26,7 @@ class Package(BootstrapMixin, BuildPackageMixin, BuildCleanMixin, GitPackageMixi self, name, branch, repo, prebuildcmd=None, kernel_module=False, explicit_deps=None, generate_version=True, predepscmd=None, deps_path=None, subdir=None, deoptions=None, jobs=None, buildcmd=None, tmpfs=True, tmpfs_size=12, batch_priority=100, env=None, identity_file_path=None, - build_constraints=None, + build_constraints=None, debian_fork=False, ): self.name = name self.branch = branch @@ -57,6 +57,7 @@ class Package(BootstrapMixin, BuildPackageMixin, BuildCleanMixin, GitPackageMixi self.children = set() self.batch_priority = batch_priority self.env = env or {} + self.debian_fork = debian_fork def __eq__(self, other): return other == self.name if isinstance(other, str) else self.name == other.name diff --git a/scale_build/upstream_package_updates.py b/scale_build/upstream_package_updates.py new file mode 100644 index 0000000..d3433da --- /dev/null +++ b/scale_build/upstream_package_updates.py @@ -0,0 +1,94 @@ +import importlib +import logging +import gzip +import os +import re +import sys + +import requests +from packaging import version + +from .utils.manifest import get_manifest +from .utils.package import get_packages + +logger = logging.getLogger(__name__) + + +def get_debian_version(debian_packages, name): + if m := re.search( + rf'Package: {name}\n(' + rf'Version: |' + rf'Source: .+? \(|' + rf'Source: .+?\nVersion: ' + rf')([0-9.\-]+)', + debian_packages + ): + return m.group(2) + + return None + + +def check_python_truenas_requirements(debian_packages, pkg): + sys.path.insert(0, pkg.source_path) + pip_to_debian = importlib.import_module('generate').pip_to_debian + sys.path.remove(pkg.source_path) + + need_update = False + with open(os.path.join(pkg.source_path, 'requirements.txt')) as f: + requirements = f.read().strip().split() + with open(os.path.join(pkg.source_path, 'constraints.txt')) as f: + requirements += f.read().strip().split() + for requirement in requirements: + pip_package_name, requirement_version = requirement.split('#egg=')[-1].split('==') + + debian_package_name = pip_to_debian(pip_package_name) + debian_version = get_debian_version(debian_packages, debian_package_name) + if debian_version is None: + logger.info(f'Debian package {debian_package_name} does not exist') + continue + + if version.parse(debian_version.split('-')[0]) > version.parse(requirement_version): + logger.error(f'Upstream version for python package {pip_package_name} ({debian_version}) is newer than ' + f'local ({requirement_version})') + need_update = True + + return need_update + + +def check_debian_fork(debian_packages, pkg): + with open(os.path.join(pkg.source_path, 'pull.sh')) as f: + pull_sh = f.read() + + local_version = re.search(r'^VERSION=([0-9.]+)$', pull_sh, flags=re.MULTILINE).group(1) + local_version += ('-' + re.search(r'^REVISION=([0-9]+)$', pull_sh, flags=re.MULTILINE).group(1)) + + debian_version = get_debian_version(debian_packages, pkg.name) + if debian_version is None: + raise RuntimeError(f'Unable to find debian package {pkg.name}') + + if version.parse(debian_version) > version.parse(local_version): + logger.error(f'Upstream version for package {pkg.name} ({debian_version}) is newer than local ' + f'({local_version})') + return True + + return False + + +def check_upstream_package_updates(): + manifest = get_manifest() + response = requests.get( + f'https://deb.debian.org/debian/dists/{manifest["debian_release"]}/main/binary-amd64/Packages.gz' + ) + response.raise_for_status() + debian_packages = gzip.decompress(response.content).decode("utf-8") + + need_update = False + for pkg in get_packages(): + if pkg.name == 'python_truenas_requirements': + need_update |= check_python_truenas_requirements(debian_packages, pkg) + + if pkg.debian_fork: + need_update |= check_debian_fork(debian_packages, pkg) + + if need_update: + sys.exit(1) diff --git a/scale_build/utils/manifest.py b/scale_build/utils/manifest.py index 4b2588e..d09bca1 100644 --- a/scale_build/utils/manifest.py +++ b/scale_build/utils/manifest.py @@ -130,8 +130,10 @@ MANIFEST_SCHEMA = { 'subdir': {'type': 'string'}, 'deoptions': {'type': 'string'}, 'jobs': {'type': 'integer'}, + 'debian_fork': {'type': 'boolean'}, }, 'required': ['name', 'branch', 'repo'], + 'additionalProperties': False, }] }, },