lukechilds
/
termux-packages
mirror of https://github.com/lukechilds/termux-packages.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Cleanup scripts/buildorder.py

android-5
Fredrik Fornwall 7 years ago
parent
4e070076d2
commit
2d24e0586b
1 changed files with 107 additions and 115 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 222
      scripts/buildorder.py

222
scripts/buildorder.py
View File

@ -1,5 +1,5 @@
#!/usr/bin/env python3
# buildorder.py - script to generate a build order respecting package dependencies
"Script to generate a build order respecting package dependencies."
import os
import re
@ -8,12 +8,12 @@ import sys
from itertools import filterfalse
# https://docs.python.org/3/library/itertools.html#itertools-recipes
def unique_everseen(iterable, key=None):
"List unique elements, preserving order. Remember all elements ever seen."
# unique_everseen('AAAABBBCCDAABBB') --> A B C D
# unique_everseen('ABBCcAD', str.lower) --> A B C D
"""List unique elements, preserving order. Remember all elements ever seen.
See https://docs.python.org/3/library/itertools.html#itertools-recipes
Examples:
unique_everseen('AAAABBBCCDAABBB') --> A B C D
unique_everseen('ABBCcAD', str.lower) --> A B C D"""
seen = set()
seen_add = seen.add
if key is None:
@ -29,50 +29,44 @@ def unique_everseen(iterable, key=None):
def die(msg):
"Exit the process with an error message."
sys.exit('ERROR: ' + msg)
def rchop(thestring, ending):
if thestring.endswith(ending):
return thestring[:-len(ending)]
return thestring
class TermuxBuildFile(object):
def __init__(self, path):
self.path = path
def _get_dependencies(self):
pkg_dep_prefix = 'TERMUX_PKG_DEPENDS='
pkg_build_dep_prefix = 'TERMUX_PKG_BUILD_DEPENDS='
subpkg_dep_prefix = 'TERMUX_SUBPKG_DEPENDS='
comma_deps = ''
with open(self.path, encoding="utf-8") as f:
prefix = None
for line in f:
if line.startswith(pkg_dep_prefix):
prefix = pkg_dep_prefix
elif line.startswith(pkg_build_dep_prefix):
prefix = pkg_build_dep_prefix
elif line.startswith(subpkg_dep_prefix):
prefix = subpkg_dep_prefix
else:
continue
comma_deps += line[len(prefix):].replace('"', '').replace("'", '').replace("\n", ",")
def parse_build_file_dependencies(path):
"Extract the dependencies of a build.sh or *.subpackage.sh file."
pkg_dep_prefix = 'TERMUX_PKG_DEPENDS='
pkg_build_dep_prefix = 'TERMUX_PKG_BUILD_DEPENDS='
subpkg_dep_prefix = 'TERMUX_SUBPKG_DEPENDS='
dependencies = []
with open(path, encoding="utf-8") as build_script:
prefix = None
for line in build_script:
if line.startswith(pkg_dep_prefix):
prefix = pkg_dep_prefix
elif line.startswith(pkg_build_dep_prefix):
prefix = pkg_build_dep_prefix
elif line.startswith(subpkg_dep_prefix):
prefix = subpkg_dep_prefix
else:
continue
dependencies_string = line[len(prefix):]
for char in "\"'\n":
dependencies_string = dependencies_string.replace(char, '')
# Remove trailing ',' that is otherwise replacing the final newline
comma_deps = comma_deps[:-1]
if not comma_deps:
# no deps found
return set()
for dependency_value in dependencies_string.split(','):
# Replace parenthesis to ignore version qualifiers as in "gcc (>= 5.0)":
dependency_value = re.sub(r'\(.*?\)', '', dependency_value).strip()
dependency_value = re.sub('-dev$', '', dependency_value)
dependencies.append(dependency_value)
return set([
# Replace parenthesis to handle version qualifiers, as in "gcc (>= 5.0)":
rchop(re.sub(r'\(.*?\)', '', dep).strip(), '-dev') for dep in comma_deps.split(',')
])
return set(dependencies)
class TermuxPackage(object):
"A main package definition represented by a directory with a build.sh file."
def __init__(self, dir_path):
self.dir = dir_path
self.name = os.path.basename(self.dir)
@ -82,8 +76,7 @@ class TermuxPackage(object):
if not os.path.isfile(build_sh_path):
raise Exception("build.sh not found for package '" + self.name + "'")
self.buildfile = TermuxBuildFile(build_sh_path)
self.deps = self.buildfile._get_dependencies()
self.deps = parse_build_file_dependencies(build_sh_path)
if 'libandroid-support' not in self.deps and self.name != 'libandroid-support':
# Every package may depend on libandroid-support without declaring it:
self.deps.add('libandroid-support')
@ -92,7 +85,8 @@ class TermuxPackage(object):
self.subpkgs = []
for filename in os.listdir(self.dir):
if not filename.endswith('.subpackage.sh'): continue
if not filename.endswith('.subpackage.sh'):
continue
subpkg = TermuxSubPackage(self.dir + '/' + filename, self)
self.subpkgs.append(subpkg)
@ -103,71 +97,72 @@ class TermuxPackage(object):
# Do not depend on any sub package
self.deps.difference_update([subpkg.name for subpkg in self.subpkgs])
self.needed_by = set() # to be completed outside, reverse of deps
self.needed_by = set() # Populated outside constructor, reverse of deps.
def __repr__(self):
return "<{} '{}'>".format(self.__class__.__name__, self.name)
def recursive_dependencies(self, pkgs_map):
"All the dependencies of the package, both direct and indirect."
result = []
for dependency_name in sorted(self.deps):
dependency_package = pkgs_map[dependency_name]
result += dependency_package.recursive_dependencies(pkgs_map)
result += [dependency_package]
return unique_everseen(result)
class TermuxSubPackage(TermuxPackage):
class TermuxSubPackage:
"A sub-package represented by a ${PACKAGE_NAME}.subpackage.sh file."
def __init__(self, subpackage_file_path, parent):
if parent is None:
raise Exception("SubPackages should have a parent")
self.buildfile = TermuxBuildFile(subpackage_file_path)
self.name = os.path.basename(subpackage_file_path).split('.subpackage.sh')[0]
self.parent = parent
self.deps = self.buildfile._get_dependencies()
self.deps = parse_build_file_dependencies(subpackage_file_path)
def __repr__(self):
return "<{} '{}' parent='{}'>".format(self.__class__.__name__, self.name, self.parent)
# Tracks non-visited deps for each package
remaining_deps = {}
# Mapping from package name to TermuxPackage
# (if subpackage, mapping from subpackage name to parent package)
pkgs_map = {}
# Reverse dependencies
pkg_depends_on = {}
PACKAGES_DIRS = ['packages']
def populate():
def read_packages_from_directories(directories):
"""Construct a map from package name to TermuxPackage.
For subpackages this maps from the subpackage name to the parent package."""
pkgs_map = {}
all_packages = []
for package_dir in PACKAGES_DIRS:
for package_dir in directories:
for pkgdir_name in sorted(os.listdir(package_dir)):
dir_path = package_dir + '/' + pkgdir_name
if os.path.isfile(dir_path + '/build.sh'):
all_packages.append(TermuxPackage(package_dir + '/' + pkgdir_name))
for pkg in all_packages:
if pkg.name in pkgs_map: die('Duplicated package: ' + pkg.name)
else: pkgs_map[pkg.name] = pkg
for subpkg in pkg.subpkgs:
pkgs_map[subpkg.name] = pkg
remaining_deps[subpkg.name] = set(subpkg.deps)
remaining_deps[pkg.name] = set(pkg.deps)
new_package = TermuxPackage(package_dir + '/' + pkgdir_name)
all_pkg_names = set(pkgs_map.keys())
if new_package.name in pkgs_map:
die('Duplicated package: ' + new_package.name)
else:
pkgs_map[new_package.name] = new_package
all_packages.append(new_package)
for name, pkg in pkgs_map.items():
for dep_name in remaining_deps[name]:
if dep_name not in all_pkg_names:
die('Package %s depends on non-existing package "%s"' % (
name, dep_name
))
for subpkg in new_package.subpkgs:
if subpkg.name in pkgs_map:
die('Duplicated package: ' + subpkg.name)
else:
pkgs_map[subpkg.name] = new_package
all_packages.append(subpkg)
dep_pkg = pkgs_map[dep_name]
dep_pkg.needed_by.add(pkg)
for pkg in all_packages:
for dependency_name in pkg.deps:
if dependency_name not in pkgs_map:
die('Package %s depends on non-existing package "%s"' % (pkg.name, dependency_name))
dep_pkg = pkgs_map[dependency_name]
if not isinstance(pkg, TermuxSubPackage):
dep_pkg.needed_by.add(pkg)
return pkgs_map
def generate_full_buildorder():
def generate_full_buildorder(pkgs_map):
"Generate a build order for building all packages."
build_order = []
# List of all TermuxPackages without dependencies
@ -182,6 +177,13 @@ def generate_full_buildorder():
# Topological sorting
visited = set()
# Tracks non-visited deps for each package
remaining_deps = {}
for name, pkg in pkgs_map.items():
remaining_deps[name] = set(pkg.deps)
for subpkg in pkg.subpkgs:
remaining_deps[subpkg.name] = set(subpkg.deps)
while pkg_queue:
pkg = pkg_queue.pop(0)
if pkg.name in visited:
@ -212,50 +214,40 @@ def generate_full_buildorder():
return build_order
def deps(pkg):
l = []
for dep in sorted(pkg.deps):
l += deps_then_me(pkgs_map[dep])
return l
def deps_then_me(pkg):
l = []
for dep in sorted(pkg.deps):
l += deps_then_me(pkgs_map[dep])
l += [pkg]
return l
def generate_targets_buildorder(target_paths):
buildorder = []
for target_path in target_paths:
if target_path.endswith('/'): target_path = target_path[:-1]
pkgname = os.path.basename(target_path)
if not pkgname in pkgs_map:
die('Dependencies for ' + pkgname + ' could not be calculated (skip dependency check with -s)')
buildorder += deps(pkgs_map[pkgname])
def generate_target_buildorder(target_path, pkgs_map):
"Generate a build order for building the dependencies of the specified package."
if target_path.endswith('/'):
target_path = target_path[:-1]
return unique_everseen(buildorder)
package_name = os.path.basename(target_path)
package = pkgs_map[package_name]
return package.recursive_dependencies(pkgs_map)
if __name__ == '__main__':
def main():
"Generate the build order either for all packages or a specific one."
packages_directories = ['packages']
full_buildorder = len(sys.argv) == 1
if not full_buildorder:
packages_real_path = os.path.realpath('packages')
for path in sys.argv[1:]:
if not os.path.isdir(path):
die('Not a directory: ' + path)
if path.endswith('/'): path = path[:-1]
if path.endswith('/'):
path = path[:-1]
parent_path = os.path.dirname(path)
if packages_real_path != os.path.realpath(parent_path):
PACKAGES_DIRS.append(parent_path)
packages_directories.append(parent_path)
populate()
pkgs_map = read_packages_from_directories(packages_directories)
if full_buildorder:
bo = generate_full_buildorder()
build_order = generate_full_buildorder(pkgs_map)
else:
bo = generate_targets_buildorder(sys.argv[1:])
build_order = generate_target_buildorder(sys.argv[1], pkgs_map)
for pkg in bo:
for pkg in build_order:
print(pkg.dir)
if __name__ == '__main__':
main()

Write Preview
Loading…
Cancel
Save