Add logic to determine packages which need to be rebuilt

scale_build/packages/binary_package.py

@@ -0,0 +1,13 @@
+class BinaryPackage:
+    def __init__(self, name, build_dependencies, source_package, source_name, install_dependencies):
+        self.name = name
+        self.build_dependencies = build_dependencies
+        self.source_package = source_package
+        self.source_name = source_name
+        self.install_dependencies = install_dependencies
+
+    def __str__(self):
+        return self.name
+
+    def __eq__(self, other):
+        return self.name == other.name

scale_build/packages/order.py

@@ -0,0 +1,35 @@
+import errno
+import logging
+
+from collections import defaultdict
+from scale_build.exceptions import CallError
+from scale_build.utils.manifest import get_packages
+
+
+logger = logging.getLogger(__name__)
+
+
+def get_to_build_packages():
+    binary_packages = {}
+    packages_list = get_packages()
+    packages = {}
+    for package in packages_list:
+        if not package.exists:
+            raise CallError(
+                f'Missing sources for {package.name},  did you forget to run "make checkout" ?', errno=errno.ENOENT
+            )
+
+        packages[package.name] = package
+        for binary_package in package.binary_packages:
+            binary_packages[binary_package.name] = binary_package
+
+    parent_mapping = defaultdict(set)
+    for pkg_name, package in packages.items():
+        for dep in package.build_time_dependencies(binary_packages):
+            parent_mapping[dep].add(pkg_name)
+
+    for pkg_name, package in filter(lambda i: i[1].hash_changed, packages.items()):
+        for child in parent_mapping[pkg_name]:
+            packages[child].parent_changed = True
+
+    return {package.name: package for package in packages.values() if package.rebuild}

scale_build/packages/package.py

@@ -3,11 +3,12 @@ import logging
 import os
 import shutil
 
-from collections import defaultdict
+from scale_build.exceptions import CallError
 from scale_build.utils.git_utils import retrieve_git_remote_and_sha, retrieve_git_branch, update_git_manifest
 from scale_build.utils.run import run
 from scale_build.utils.variables import GIT_LOG_PATH, HASH_DIR, SOURCES_DIR
 
+from .binary_package import BinaryPackage
 from .utils import DEPENDS_SCRIPT_PATH, get_install_deps, normalize_build_depends, normalize_bin_packages_depends
 
 
@@ -24,7 +25,7 @@ class Package:
         self.origin = repo
         self.prebuildcmd = prebuildcmd
         self.kernel_module = kernel_module
-        self.explicit_deps = explicit_deps or set()
+        self.explicit_deps = set(explicit_deps or set())
         self.generate_version = generate_version
         self.predepscmd = predepscmd
         self.deps_path = deps_path
@@ -32,11 +33,11 @@ class Package:
         self.deoptions = deoptions
         self.jobs = jobs
         self.initialized_deps = False
-        self.binary_packages = defaultdict(
+        self._binary_packages = []
-            lambda: {'install_deps': set(), 'source_name': self.name, 'build_deps': set()}
-        )
         self.build_depends = set()
         self.source_package = None
+        self.parent_changed = False
+        self._build_time_dependencies = set()
 
     @property
     def package_path(self):
@@ -57,17 +58,16 @@ class Package:
         return os.path.join(SOURCES_DIR, self.name)
 
     @property
-    def dependencies(self):
+    def binary_packages(self):
-        if self.initialized_deps:
+        if self._binary_packages:
-            return self.binary_packages
+            return self._binary_packages
 
         if self.name == 'kernel' or (self.predepscmd and not self.deps_path):
             # We cannot determine dependency of this package because it does not probably have a control file
             # in it's current state - the only example we have is grub right now. Let's improve this if there are
             # more examples
-            self.binary_packages[self.name].update({
+            self._binary_packages.append(BinaryPackage(self.name, self.build_depends, self.name, self.name, set()))
-                'source_package': self.name,
+            return self._binary_packages
-            })
 
         cp = run([DEPENDS_SCRIPT_PATH, self.debian_control_file_path])
         info = json.loads(cp.stdout)
@@ -77,25 +77,28 @@ class Package:
         ) | default_dependencies
         self.source_package = info['source_package']['name']
         for bin_package in info['binary_packages']:
-            default_dependencies = {'kernel'} if self.kernel_module else set()
+            self._binary_packages.append(BinaryPackage(
-            self.binary_packages[bin_package['name']].update({
+                bin_package['name'], self.build_depends, self.source_package, self.name,
-                'install_deps': set(normalize_bin_packages_depends(bin_package['depends'] or '')),
+                set(normalize_bin_packages_depends(bin_package['depends'] or ''))
-                'build_deps': self.build_depends,
+            ))
-            })
             if self.name == 'truenas':
-                self.binary_packages[bin_package['name']]['build_deps'] |= self.binary_packages[
+                self._binary_packages[-1].build_dependencies |= self._binary_packages[-1].install_dependencies
-                    bin_package['name']]['install_deps']
 
-        self.initialized_deps = True
+        return self._binary_packages
 
-        return self.binary_packages
+    def build_time_dependencies(self, all_binary_packages=None):
+        if self._build_time_dependencies:
+            return self._build_time_dependencies
+        elif not all_binary_packages:
+            raise CallError('Binary packages must be specified when computing build time dependencies')
 
-    def build_time_dependencies(self, all_binary_packages):
+        self._build_time_dependencies = get_install_deps(
-        # Dependencies at build time will be build_depends
+            all_binary_packages, set(), self.build_depends
-        return get_install_deps(all_binary_packages, set(), self.binary_packages[self.name]) | self.explicit_deps
+        ) | self.explicit_deps
+        return self._build_time_dependencies
 
     @property
-    def rebuild(self):
+    def hash_changed(self):
         if self.name == 'truenas':
             # truenas is special and we want to rebuild it always
             # TODO: Do see why that is so
@@ -113,6 +116,10 @@ class Package:
         else:
             return True
 
+    @property
+    def rebuild(self):
+        return self.hash_changed or self.parent_changed
+
     @property
     def hash_path(self):
         return os.path.join(HASH_DIR, f'{self.name}.hash')

scale_build/packages/utils.py

@@ -18,6 +18,8 @@ def normalize_build_depends(build_depends_str):
 
 def get_install_deps(packages, deps, deps_list):
     for dep in filter(lambda p: p in packages, deps_list):
-        deps.add(packages[dep]['source_name'])
+        deps.add(packages[dep].source_name)
-        deps.update(get_install_deps(packages, deps, packages[dep]['install_deps'] | packages[dep]['build_deps']))
+        deps.update(
+            get_install_deps(packages, deps, packages[dep].install_dependencies | packages[dep].build_dependencies)
+        )
     return deps