make check_upstream_package_updates to check forked Debian repos and python-truenas-requirements

Makefile

@@ -12,7 +12,7 @@ check:
 ifneq ($(REPO_CHANGED),0)
 	@echo "Setting up new virtual environment"
 	@rm -rf venv-*
-	@${PYTHON} -m pip install -U virtualenv >/dev/null 2>&1 || { echo "Failed to install/upgrade virtualenv package"; exit 1; }
+	@${PYTHON} -m pip install -U virtualenv >/dev/null || { echo "Failed to install/upgrade virtualenv package"; exit 1; }
 	@${PYTHON} -m venv venv-${COMMIT_HASH} || { echo "Failed to create virutal environment"; exit 1; }
 	@{ . ./venv-${COMMIT_HASH}/bin/activate && \
 		python3 -m pip install -r requirements.txt >/dev/null 2>&1 && \
@@ -25,6 +25,8 @@ clean: check
 	. ./venv-${COMMIT_HASH}/bin/activate && scale_build clean
 checkout: check
 	. ./venv-${COMMIT_HASH}/bin/activate && scale_build checkout
+check_upstream_package_updates: check
+	. ./venv-${COMMIT_HASH}/bin/activate && scale_build check_upstream_package_updates
 iso: check
 	. ./venv-${COMMIT_HASH}/bin/activate && scale_build iso
 packages: check

conf/build.manifest

@@ -483,6 +483,7 @@ sources:
 - name: collectd
   repo: https://github.com/truenas/collectd
   branch: master
+  debian_fork: true
   predepscmd:
     - "apt install -y wget xz-utils"
     - "./pull.sh"
@@ -491,6 +492,7 @@ sources:
 - name: grub2
   repo: https://github.com/truenas/grub2
   branch: master
+  debian_fork: true
   predepscmd:
     - "apt install -y wget xz-utils"
     - "./pull.sh"
@@ -506,6 +508,7 @@ sources:
 - name: rrdcached
   repo: https://github.com/truenas/rrdtool
   branch: master
+  debian_fork: true
   predepscmd:
     - "apt install -y wget xz-utils"
     - "./pull.sh"
@@ -514,6 +517,7 @@ sources:
 - name: smartmontools
   repo: https://github.com/truenas/smartmontools
   branch: master
+  debian_fork: true
   predepscmd:
     - "apt install -y wget xz-utils"
     - "./pull.sh"

requirements.txt

@@ -4,9 +4,11 @@ coloredlogs==15.0
 humanfriendly==9.1
 idna==2.10
 jsonschema==3.2.0
+packaging==21.3
 pexpect==4.8.0
 psutil==5.8.0
 ptyprocess==0.7.0
+pyparsing==3.0.9
 PyYAML==5.4.1
 requests==2.25.1
 toposort==1.6

scale_build/main.py

@@ -7,6 +7,7 @@ from .branch_out import branch_out_repos, validate_branch_out_config
 from .checkout import checkout_sources
 from .clean import complete_cleanup
 from .config import BRANCH_OVERRIDES
+from .upstream_package_updates import check_upstream_package_updates
 from .epoch import check_epoch
 from .exceptions import CallError
 from .iso import build_iso
@@ -53,6 +54,7 @@ def main():
     subparsers = parser.add_subparsers(help='sub-command help', dest='action')
 
     subparsers.add_parser('checkout', help='Checkout TrueNAS Scale repositories')
+    subparsers.add_parser('check_upstream_package_updates', help='Check that all forked Debian packages are up-to-date')
     subparsers.add_parser('clean', help='Clean build package(s) / cloned source(s) / image(s) of TrueNAS Scale')
     packages_parser = subparsers.add_parser('packages', help='Build TrueNAS Scale packages')
     packages_parser.add_argument(
@@ -76,6 +78,8 @@ def main():
     if args.action == 'checkout':
         check_epoch()
         checkout_sources()
+    if args.action == 'check_upstream_package_updates':
+        check_upstream_package_updates()
     elif args.action == 'packages':
         validate()
         check_epoch()

scale_build/packages/package.py

@@ -26,7 +26,7 @@ class Package(BootstrapMixin, BuildPackageMixin, BuildCleanMixin, GitPackageMixi
         self, name, branch, repo, prebuildcmd=None, kernel_module=False, explicit_deps=None,
         generate_version=True, predepscmd=None, deps_path=None, subdir=None, deoptions=None, jobs=None,
         buildcmd=None, tmpfs=True, tmpfs_size=12, batch_priority=100, env=None, identity_file_path=None,
-        build_constraints=None,
+        build_constraints=None, debian_fork=False,
     ):
         self.name = name
         self.branch = branch
@@ -57,6 +57,7 @@ class Package(BootstrapMixin, BuildPackageMixin, BuildCleanMixin, GitPackageMixi
         self.children = set()
         self.batch_priority = batch_priority
         self.env = env or {}
+        self.debian_fork = debian_fork
 
     def __eq__(self, other):
         return other == self.name if isinstance(other, str) else self.name == other.name

scale_build/upstream_package_updates.py

@@ -0,0 +1,94 @@
+import importlib
+import logging
+import gzip
+import os
+import re
+import sys
+
+import requests
+from packaging import version
+
+from .utils.manifest import get_manifest
+from .utils.package import get_packages
+
+logger = logging.getLogger(__name__)
+
+
+def get_debian_version(debian_packages, name):
+    if m := re.search(
+            rf'Package: {name}\n('
+            rf'Version: |'
+            rf'Source: .+? \(|'
+            rf'Source: .+?\nVersion: '
+            rf')([0-9.\-]+)',
+            debian_packages
+    ):
+        return m.group(2)
+
+    return None
+
+
+def check_python_truenas_requirements(debian_packages, pkg):
+    sys.path.insert(0, pkg.source_path)
+    pip_to_debian = importlib.import_module('generate').pip_to_debian
+    sys.path.remove(pkg.source_path)
+
+    need_update = False
+    with open(os.path.join(pkg.source_path, 'requirements.txt')) as f:
+        requirements = f.read().strip().split()
+    with open(os.path.join(pkg.source_path, 'constraints.txt')) as f:
+        requirements += f.read().strip().split()
+    for requirement in requirements:
+        pip_package_name, requirement_version = requirement.split('#egg=')[-1].split('==')
+
+        debian_package_name = pip_to_debian(pip_package_name)
+        debian_version = get_debian_version(debian_packages, debian_package_name)
+        if debian_version is None:
+            logger.info(f'Debian package {debian_package_name} does not exist')
+            continue
+
+        if version.parse(debian_version.split('-')[0]) > version.parse(requirement_version):
+            logger.error(f'Upstream version for python package {pip_package_name} ({debian_version}) is newer than '
+                         f'local ({requirement_version})')
+            need_update = True
+
+    return need_update
+
+
+def check_debian_fork(debian_packages, pkg):
+    with open(os.path.join(pkg.source_path, 'pull.sh')) as f:
+        pull_sh = f.read()
+
+    local_version = re.search(r'^VERSION=([0-9.]+)$', pull_sh, flags=re.MULTILINE).group(1)
+    local_version += ('-' + re.search(r'^REVISION=([0-9]+)$', pull_sh, flags=re.MULTILINE).group(1))
+
+    debian_version = get_debian_version(debian_packages, pkg.name)
+    if debian_version is None:
+        raise RuntimeError(f'Unable to find debian package {pkg.name}')
+
+    if version.parse(debian_version) > version.parse(local_version):
+        logger.error(f'Upstream version for package {pkg.name} ({debian_version}) is newer than local '
+                     f'({local_version})')
+        return True
+
+    return False
+
+
+def check_upstream_package_updates():
+    manifest = get_manifest()
+    response = requests.get(
+        f'https://deb.debian.org/debian/dists/{manifest["debian_release"]}/main/binary-amd64/Packages.gz'
+    )
+    response.raise_for_status()
+    debian_packages = gzip.decompress(response.content).decode("utf-8")
+
+    need_update = False
+    for pkg in get_packages():
+        if pkg.name == 'python_truenas_requirements':
+            need_update |= check_python_truenas_requirements(debian_packages, pkg)
+
+        if pkg.debian_fork:
+            need_update |= check_debian_fork(debian_packages, pkg)
+
+    if need_update:
+        sys.exit(1)

scale_build/utils/manifest.py

@@ -130,8 +130,10 @@ MANIFEST_SCHEMA = {
                     'subdir': {'type': 'string'},
                     'deoptions': {'type': 'string'},
                     'jobs': {'type': 'integer'},
+                    'debian_fork': {'type': 'boolean'},
                 },
                 'required': ['name', 'branch', 'repo'],
+                'additionalProperties': False,
             }]
         },
     },