Cleanup scripts/buildorder.py

scripts/buildorder.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# buildorder.py - script to generate a build order respecting package dependencies
+"Script to generate a build order respecting package dependencies."
 
 import os
 import re
@@ -8,12 +8,12 @@ import sys
 from itertools import filterfalse
 
 
-# https://docs.python.org/3/library/itertools.html#itertools-recipes
-
 def unique_everseen(iterable, key=None):
-    "List unique elements, preserving order. Remember all elements ever seen."
+    """List unique elements, preserving order. Remember all elements ever seen.
-    # unique_everseen('AAAABBBCCDAABBB') --> A B C D
+    See https://docs.python.org/3/library/itertools.html#itertools-recipes
-    # unique_everseen('ABBCcAD', str.lower) --> A B C D
+    Examples:
+    unique_everseen('AAAABBBCCDAABBB') --> A B C D
+    unique_everseen('ABBCcAD', str.lower) --> A B C D"""
     seen = set()
     seen_add = seen.add
     if key is None:
@@ -29,50 +29,44 @@ def unique_everseen(iterable, key=None):
 
 
 def die(msg):
+    "Exit the process with an error message."
     sys.exit('ERROR: ' + msg)
 
-def rchop(thestring, ending):
-    if thestring.endswith(ending):
-        return thestring[:-len(ending)]
-    return thestring
-
-class TermuxBuildFile(object):
-    def __init__(self, path):
-        self.path = path
-
-    def _get_dependencies(self):
-        pkg_dep_prefix = 'TERMUX_PKG_DEPENDS='
-        pkg_build_dep_prefix = 'TERMUX_PKG_BUILD_DEPENDS='
-        subpkg_dep_prefix = 'TERMUX_SUBPKG_DEPENDS='
-        comma_deps = ''
-
-        with open(self.path, encoding="utf-8") as f:
-            prefix = None
-            for line in f:
-                if line.startswith(pkg_dep_prefix):
-                    prefix = pkg_dep_prefix
-                elif line.startswith(pkg_build_dep_prefix):
-                    prefix = pkg_build_dep_prefix
-                elif line.startswith(subpkg_dep_prefix):
-                    prefix = subpkg_dep_prefix
-                else:
-                    continue
 
-                comma_deps += line[len(prefix):].replace('"', '').replace("'", '').replace("\n", ",")
+def parse_build_file_dependencies(path):
+    "Extract the dependencies of a build.sh or *.subpackage.sh file."
+    pkg_dep_prefix = 'TERMUX_PKG_DEPENDS='
+    pkg_build_dep_prefix = 'TERMUX_PKG_BUILD_DEPENDS='
+    subpkg_dep_prefix = 'TERMUX_SUBPKG_DEPENDS='
+    dependencies = []
+
+    with open(path, encoding="utf-8") as build_script:
+        prefix = None
+        for line in build_script:
+            if line.startswith(pkg_dep_prefix):
+                prefix = pkg_dep_prefix
+            elif line.startswith(pkg_build_dep_prefix):
+                prefix = pkg_build_dep_prefix
+            elif line.startswith(subpkg_dep_prefix):
+                prefix = subpkg_dep_prefix
+            else:
+                continue
+
+            dependencies_string = line[len(prefix):]
+            for char in "\"'\n":
+                dependencies_string = dependencies_string.replace(char, '')
 
-        # Remove trailing ',' that is otherwise replacing the final newline
+            for dependency_value in dependencies_string.split(','):
-        comma_deps = comma_deps[:-1]
+                # Replace parenthesis to ignore version qualifiers as in "gcc (>= 5.0)":
-        if not comma_deps:
+                dependency_value = re.sub(r'\(.*?\)', '', dependency_value).strip()
-            # no deps found
+                dependency_value = re.sub('-dev$', '', dependency_value)
-            return set()
+                dependencies.append(dependency_value)
 
-        return set([
+    return set(dependencies)
-            # Replace parenthesis to handle version qualifiers, as in "gcc (>= 5.0)":
-            rchop(re.sub(r'\(.*?\)', '', dep).strip(), '-dev') for dep in comma_deps.split(',')
-        ])
 
 
 class TermuxPackage(object):
+    "A main package definition represented by a directory with a build.sh file."
     def __init__(self, dir_path):
         self.dir = dir_path
         self.name = os.path.basename(self.dir)
@@ -82,8 +76,7 @@ class TermuxPackage(object):
         if not os.path.isfile(build_sh_path):
             raise Exception("build.sh not found for package '" + self.name + "'")
 
-        self.buildfile = TermuxBuildFile(build_sh_path)
+        self.deps = parse_build_file_dependencies(build_sh_path)
-        self.deps = self.buildfile._get_dependencies()
         if 'libandroid-support' not in self.deps and self.name != 'libandroid-support':
             # Every package may depend on libandroid-support without declaring it:
             self.deps.add('libandroid-support')
@@ -92,7 +85,8 @@ class TermuxPackage(object):
         self.subpkgs = []
 
         for filename in os.listdir(self.dir):
-            if not filename.endswith('.subpackage.sh'): continue
+            if not filename.endswith('.subpackage.sh'):
+                continue
             subpkg = TermuxSubPackage(self.dir + '/' + filename, self)
 
             self.subpkgs.append(subpkg)
@@ -103,71 +97,72 @@ class TermuxPackage(object):
         # Do not depend on any sub package
         self.deps.difference_update([subpkg.name for subpkg in self.subpkgs])
 
-        self.needed_by = set()  # to be completed outside, reverse of    deps
+        self.needed_by = set()  # Populated outside constructor, reverse of deps.
 
     def __repr__(self):
         return "<{} '{}'>".format(self.__class__.__name__, self.name)
 
+    def recursive_dependencies(self, pkgs_map):
+        "All the dependencies of the package, both direct and indirect."
+        result = []
+        for dependency_name in sorted(self.deps):
+            dependency_package = pkgs_map[dependency_name]
+            result += dependency_package.recursive_dependencies(pkgs_map)
+            result += [dependency_package]
+        return unique_everseen(result)
 
-class TermuxSubPackage(TermuxPackage):
+
+class TermuxSubPackage:
+    "A sub-package represented by a ${PACKAGE_NAME}.subpackage.sh file."
     def __init__(self, subpackage_file_path, parent):
         if parent is None:
             raise Exception("SubPackages should have a parent")
 
-        self.buildfile = TermuxBuildFile(subpackage_file_path)
         self.name = os.path.basename(subpackage_file_path).split('.subpackage.sh')[0]
         self.parent = parent
-        self.deps = self.buildfile._get_dependencies()
+        self.deps = parse_build_file_dependencies(subpackage_file_path)
 
     def __repr__(self):
         return "<{} '{}' parent='{}'>".format(self.__class__.__name__, self.name, self.parent)
 
 
-# Tracks non-visited deps for each package
+def read_packages_from_directories(directories):
-remaining_deps = {}
+    """Construct a map from package name to TermuxPackage.
-
+    For subpackages this maps from the subpackage name to the parent package."""
-# Mapping from package name to TermuxPackage
+    pkgs_map = {}
-# (if subpackage, mapping from subpackage name to parent package)
-pkgs_map = {}
-
-# Reverse dependencies
-pkg_depends_on = {}
-
-PACKAGES_DIRS = ['packages']
-
-
-def populate():
     all_packages = []
-    for package_dir in PACKAGES_DIRS:
+
+    for package_dir in directories:
         for pkgdir_name in sorted(os.listdir(package_dir)):
             dir_path = package_dir + '/' + pkgdir_name
             if os.path.isfile(dir_path + '/build.sh'):
-                all_packages.append(TermuxPackage(package_dir + '/' + pkgdir_name))
+                new_package = TermuxPackage(package_dir + '/' + pkgdir_name)
-
-    for pkg in all_packages:
-        if pkg.name in pkgs_map: die('Duplicated package: ' + pkg.name)
-        else: pkgs_map[pkg.name] = pkg
-
-        for subpkg in pkg.subpkgs:
-            pkgs_map[subpkg.name] = pkg
-            remaining_deps[subpkg.name] = set(subpkg.deps)
-
-        remaining_deps[pkg.name] = set(pkg.deps)
 
-    all_pkg_names = set(pkgs_map.keys())
+                if new_package.name in pkgs_map:
+                    die('Duplicated package: ' + new_package.name)
+                else:
+                    pkgs_map[new_package.name] = new_package
+                all_packages.append(new_package)
 
-    for name, pkg in pkgs_map.items():
+                for subpkg in new_package.subpkgs:
-        for dep_name in remaining_deps[name]:
+                    if subpkg.name in pkgs_map:
-            if dep_name not in all_pkg_names:
+                        die('Duplicated package: ' + subpkg.name)
-                die('Package %s depends on non-existing package "%s"' % (
+                    else:
-                 name, dep_name
+                        pkgs_map[subpkg.name] = new_package
-                ))
+                    all_packages.append(subpkg)
 
-            dep_pkg = pkgs_map[dep_name]
+    for pkg in all_packages:
-            dep_pkg.needed_by.add(pkg)
+        for dependency_name in pkg.deps:
+            if dependency_name not in pkgs_map:
+                die('Package %s depends on non-existing package "%s"' % (pkg.name, dependency_name))
+            dep_pkg = pkgs_map[dependency_name]
+            if not isinstance(pkg, TermuxSubPackage):
+                dep_pkg.needed_by.add(pkg)
+    return pkgs_map
 
 
-def generate_full_buildorder():
+def generate_full_buildorder(pkgs_map):
+    "Generate a build order for building all packages."
     build_order = []
 
     # List of all TermuxPackages without dependencies
@@ -182,6 +177,13 @@ def generate_full_buildorder():
     # Topological sorting
     visited = set()
 
+    # Tracks non-visited deps for each package
+    remaining_deps = {}
+    for name, pkg in pkgs_map.items():
+        remaining_deps[name] = set(pkg.deps)
+        for subpkg in pkg.subpkgs:
+            remaining_deps[subpkg.name] = set(subpkg.deps)
+
     while pkg_queue:
         pkg = pkg_queue.pop(0)
         if pkg.name in visited:
@@ -212,50 +214,40 @@ def generate_full_buildorder():
 
     return build_order
 
-def deps(pkg):
-    l = []
-    for dep in sorted(pkg.deps):
-        l += deps_then_me(pkgs_map[dep])
-    return l
-
-def deps_then_me(pkg):
-    l = []
-    for dep in sorted(pkg.deps):
-        l += deps_then_me(pkgs_map[dep])
-    l += [pkg]
-    return l
-
-
-def generate_targets_buildorder(target_paths):
-    buildorder = []
 
-    for target_path in target_paths:
+def generate_target_buildorder(target_path, pkgs_map):
-        if target_path.endswith('/'): target_path = target_path[:-1]
+    "Generate a build order for building the dependencies of the specified package."
-        pkgname = os.path.basename(target_path)
+    if target_path.endswith('/'):
-        if not pkgname in pkgs_map:
+        target_path = target_path[:-1]
-            die('Dependencies for ' + pkgname + ' could not be calculated (skip dependency check with -s)')
-        buildorder += deps(pkgs_map[pkgname])
 
-    return unique_everseen(buildorder)
+    package_name = os.path.basename(target_path)
+    package = pkgs_map[package_name]
+    return package.recursive_dependencies(pkgs_map)
 
-if __name__ == '__main__':
+def main():
+    "Generate the build order either for all packages or a specific one."
+    packages_directories = ['packages']
     full_buildorder = len(sys.argv) == 1
     if not full_buildorder:
         packages_real_path = os.path.realpath('packages')
         for path in sys.argv[1:]:
             if not os.path.isdir(path):
                 die('Not a directory: ' + path)
-            if path.endswith('/'): path = path[:-1]
+            if path.endswith('/'):
+                path = path[:-1]
             parent_path = os.path.dirname(path)
             if packages_real_path != os.path.realpath(parent_path):
-                PACKAGES_DIRS.append(parent_path)
+                packages_directories.append(parent_path)
 
-    populate()
+    pkgs_map = read_packages_from_directories(packages_directories)
 
     if full_buildorder:
-        bo = generate_full_buildorder()
+        build_order = generate_full_buildorder(pkgs_map)
     else:
-        bo = generate_targets_buildorder(sys.argv[1:])
+        build_order = generate_target_buildorder(sys.argv[1], pkgs_map)
 
-    for pkg in bo:
+    for pkg in build_order:
         print(pkg.dir)
+
+if __name__ == '__main__':
+    main()