From 44c34afb8259c1f1c1ee258f69a517c059e29aa6 Mon Sep 17 00:00:00 2001 From: Stephen Groat Date: Mon, 2 Oct 2017 07:54:53 -0700 Subject: [PATCH 01/99] Use apt for dependencies --- .travis.yml | 13 +++++++ Rakefile | 13 ++++--- build-package.sh | 89 ++++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 109 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 691d43524..6ac4908b7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,3 +8,16 @@ env: - TRAVIS_ARCH=arm - TRAVIS_ARCH=i686 - TRAVIS_ARCH=x86_64 +script: bundle exec rake build["${ARGS}"] +jobs: + include: + - stage: Build aarch64 + env: ARGS="-a aarch64" + - env: ARGS="-i -a aarch64" + - stage: Build arm, i686, & x86_64 + env: ARGS="-a arm" + - env: ARGS="-a i686" + - env: ARGS="-a x86_64" + - env: ARGS="-i -a arm" + - env: ARGS="-i -a i686" + - env: ARGS="-i -a x86_64" diff --git a/Rakefile b/Rakefile index bf41db863..08764a36b 100644 --- a/Rakefile +++ b/Rakefile @@ -3,7 +3,7 @@ require 'pty' task default: %w[build] -task :build do +task :build, [:options] do |t, args| repo = Rugged::Repository.new('.') commit = repo.head.target parent = commit.parents.first @@ -20,13 +20,18 @@ task :build do puts "Building #{pkg}" begin # Start blocking build loop - PTY.spawn("./scripts/run-docker.sh ./build-package.sh -a $TRAVIS_ARCH #{pkg}") do |stdout, stdin, pid| + PTY.spawn("./scripts/run-docker.sh ./build-package.sh #{args[:options]} -a $TRAVIS_ARCH #{pkg}") do |stdout, stdin, pid| begin + stdout.sync stdout.each { |line| print line } - rescue Errno::EIO + rescue Errno::EIO => e + puts e + ensure + ::Process.wait pid end end - rescue PTY::ChildExited + rescue PTY::ChildExited => e + puts e puts "Process exited" end # Exit if PTY return a non-zero code diff --git a/build-package.sh b/build-package.sh index 210a6a1ce..33bf93f2e 100755 --- a/build-package.sh +++ b/build-package.sh @@ -248,12 +248,13 @@ termux_setup_cmake() { # First step is to handle command-line arguments. Not to be overridden by packages. termux_step_handle_arguments() { _show_usage () { - echo "Usage: ./build-package.sh [-a ARCH] [-d] [-D] [-f] [-q] [-s] [-o DIR] PACKAGE" + echo "Usage: ./build-package.sh [-a ARCH] [-d] [-D] [-f] [-i] [-q] [-s] [-o DIR] PACKAGE" echo "Build a package by creating a .deb file in the debs/ folder." echo " -a The architecture to build for: aarch64(default), arm, i686, x86_64 or all." echo " -d Build with debug symbols." echo " -D Build a disabled package in disabled-packages/." echo " -f Force build even if package has already been built." + echo " -i Install dependencies." echo " -q Quiet build." echo " -s Skip dependency check." echo " -o Specify deb directory. Default: debs/." @@ -266,6 +267,7 @@ termux_step_handle_arguments() { d) export TERMUX_DEBUG=true;; D) local TERMUX_IS_DISABLED=true;; f) TERMUX_FORCE_BUILD=true;; + i) export TERMUX_INSTALL_DEPS=true;; q) export TERMUX_QUIET_BUILD=true;; s) export TERMUX_SKIP_DEPCHECK=true;; o) TERMUX_DEBDIR="$(realpath -m $OPTARG)";; @@ -438,7 +440,82 @@ termux_step_start_build() { exit 0 fi - if [ -z "${TERMUX_SKIP_DEPCHECK:=""}" ]; then + if [ ! -z ${TERMUX_INSTALL_DEPS+x} ]; then + # Ensure folders present (but not $TERMUX_PKG_SRCDIR, it will be created in build) + mkdir -p "$TERMUX_COMMON_CACHEDIR" \ + "$TERMUX_DEBDIR" \ + "$TERMUX_PKG_BUILDDIR" \ + "$TERMUX_PKG_PACKAGEDIR" \ + "$TERMUX_PKG_TMPDIR" \ + "$TERMUX_PKG_CACHEDIR" \ + "$TERMUX_PKG_MASSAGEDIR" \ + $TERMUX_PREFIX/{bin,etc,lib,libexec,share,tmp,include} + # Setup bootstrap + termux_download https://termux.net/bootstrap/bootstrap-${TERMUX_ARCH}.zip \ + ${TERMUX_COMMON_CACHEDIR}/bootstrap-${TERMUX_ARCH}.zip + unzip ${TERMUX_COMMON_CACHEDIR}/bootstrap-${TERMUX_ARCH}.zip -d $TERMUX_PREFIX + + # TODO move this install to Dockerfile + sudo apt-get update && sudo apt-get -y dist-upgrade && sudo apt-get install -y libcap2-bin gawk tree strace + # set capabilities on dpkg + # Some packages built by uid 1001 (builder is 1000) + # Need capabilities for dpkg to set non-builder uid + sudo /sbin/setcap cap_chown,cap_fowner,cap_dac_override+eip /usr/bin/dpkg + sudo /sbin/setcap cap_chown,cap_fowner,cap_dac_override+eip /usr/bin/apt-get + # install packages that include subpackages + sudo sed -i -e 's/DPkg::Pre-Install-Pkgs/\/\/ DPkg::Pre-Install-Pkgs/' /etc/apt/apt.conf.d/* + sudo sed -i -e 's/DPkg::Post-Invoke/\/\/ DPkg::Post-Invoke/' /etc/apt/apt.conf.d/* + sudo sed -i -e 's/APT::Update::Post-Invoke/\/\/ APT::Update::Post-Invoke/' /etc/apt/apt.conf.d/* + TERMUX_DPKG=" \ + --force-architecture \ + --force-not-root \ + --force-configure-any \ + --force-confdef \ + --force-confold \ + --force-depends \ + --admindir=${TERMUX_PREFIX}/var/lib/dpkg" + export TERMUX_APT=" \ + -o APT::Get::Assume-Yes=true \ + -o APT::Get::ReInstall=true + -o APT::Architecture=${TERMUX_ARCH} \ + -o Dir::Etc=${TERMUX_PREFIX}/etc/apt/ \ + -o Dir::State=${TERMUX_PREFIX}/var/lib/apt \ + -o Dir::State::status=${TERMUX_PREFIX}/var/lib/dpkg/status \ + -o Dir::Cache=${TERMUX_PREFIX}/var/cache/apt \ + -o Dir::Log=${TERMUX_PREFIX}/var/log/apt" + for arg in ${TERMUX_DPKG}; do + TERMUX_APT+=" -o DPkg::Options::=${arg}" + done + export DEBCONF_FRONTEND=noninteractive + apt-get $TERMUX_APT clean && apt-get $TERMUX_APT update && apt-get $TERMUX_APT upgrade + # libandroid-support-dev is build-essential + apt-get $TERMUX_APT install libandroid-support-dev:any + sudo chown -R builder:builder /data + array=( TERMUX_PKG_DEPENDS TERMUX_PKG_BUILD_DEPENDS ) + for i in "${array[@]}"; do + while IFS=',' read -ra PKG; do + for p in "${PKG[@]}"; do + p="$(echo -e "${p}" | tr -d '[:space:]')" + apt-get $TERMUX_APT install "^${p}(-dev)?$":any + sudo chown -R builder:builder /data + done + done <<< "${!i}" + done + for subpkg in $TERMUX_PKG_BUILDER_DIR/*.subpackage.sh; do + test ! -f "$subpkg" && continue + local TERMUX_SUBPKG_DEPENDS="" + source $subpkg + while IFS=',' read -ra PKG; do + for p in "${PKG[@]}"; do + p="$(echo -e "${p}" | tr -d '[:space:]')" + apt-get $TERMUX_APT install "^${p}(-dev)?$":any + sudo chown -R builder:builder /data + done + done <<< $TERMUX_SUBPKG_DEPENDS + done + fi + + if [ -z "${TERMUX_SKIP_DEPCHECK:=""}" ] && [ -z ${TERMUX_INSTALL_DEPS+x} ]; then local p TERMUX_ALL_DEPS TERMUX_ALL_DEPS=$(./scripts/buildorder.py "$TERMUX_PKG_BUILDER_DIR") for p in $TERMUX_ALL_DEPS; do @@ -1302,6 +1379,13 @@ termux_step_create_debfile() { "$TERMUX_PKG_PACKAGEDIR/data.tar.xz" } +termux_step_reverse_depends() { + if [ ! -z ${TERMUX_INSTALL_DEPS+x} ]; then + # TODO build reverse depends with packages + apt-cache $TERMUX_APT rdepends $TERMUX_PKG_NAME + fi +} + # Finish the build. Not to be overridden by package scripts. termux_step_finish_build() { echo "termux - build of '$TERMUX_PKG_NAME' done" @@ -1342,4 +1426,5 @@ cd "$TERMUX_PKG_MASSAGEDIR/$TERMUX_PREFIX" termux_step_post_massage termux_step_create_datatar termux_step_create_debfile +termux_step_reverse_depends termux_step_finish_build From e5acfac1cb97b9f27b27c0fa61846b71b3ec2013 Mon Sep 17 00:00:00 2001 From: Stephen Date: Sun, 8 Oct 2017 13:52:35 -0700 Subject: [PATCH 02/99] Update .travis.yml --- .travis.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6ac4908b7..d7501b49f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,12 +12,12 @@ script: bundle exec rake build["${ARGS}"] jobs: include: - stage: Build aarch64 - env: ARGS="-a aarch64" - - env: ARGS="-i -a aarch64" + env: ARGS="-q -a aarch64" + - env: ARGS="-q -i -a aarch64" - stage: Build arm, i686, & x86_64 - env: ARGS="-a arm" - - env: ARGS="-a i686" - - env: ARGS="-a x86_64" - - env: ARGS="-i -a arm" - - env: ARGS="-i -a i686" - - env: ARGS="-i -a x86_64" + env: ARGS="-q -a arm" + - env: ARGS="-q -a i686" + - env: ARGS="-q -a x86_64" + - env: ARGS="-q -i -a arm" + - env: ARGS="-q -i -a i686" + - env: ARGS="-q -i -a x86_64" From bc776004df0a15d69e0299a4cb37c2f3deb6ffb1 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Fri, 27 Oct 2017 14:58:11 +0200 Subject: [PATCH 03/99] build-package: with -i -q, ignore recommended packages and suggestions (#3) --- build-package.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build-package.sh b/build-package.sh index 33bf93f2e..fa8e02180 100755 --- a/build-package.sh +++ b/build-package.sh @@ -477,6 +477,8 @@ termux_step_start_build() { export TERMUX_APT=" \ -o APT::Get::Assume-Yes=true \ -o APT::Get::ReInstall=true + -o APT::Get::Install-Recommends=false \ + -o APT::Get::Install-Suggests=false \ -o APT::Architecture=${TERMUX_ARCH} \ -o Dir::Etc=${TERMUX_PREFIX}/etc/apt/ \ -o Dir::State=${TERMUX_PREFIX}/var/lib/apt \ From 445e4ca152913e817d4ce711df61f4d56fe79fc5 Mon Sep 17 00:00:00 2001 From: Stephen Date: Wed, 29 Nov 2017 08:18:08 -0800 Subject: [PATCH 04/99] Update build.sh --- packages/libllvm/build.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/libllvm/build.sh b/packages/libllvm/build.sh index a61540978..0a7cb450e 100644 --- a/packages/libllvm/build.sh +++ b/packages/libllvm/build.sh @@ -59,6 +59,9 @@ termux_step_post_extract_package () { } termux_step_host_build () { + if [ ! -z ${TERMUX_INSTALL_DEPS+x} ]; then + apt-get $TERMUX_APT purge ndk-sysroot + fi termux_setup_cmake cmake -G "Unix Makefiles" $TERMUX_PKG_SRCDIR \ -DLLVM_BUILD_TESTS=OFF \ From 865db6dd76f7349fa562daa14fc998ac25099c8a Mon Sep 17 00:00:00 2001 From: Stephen Date: Thu, 30 Nov 2017 10:03:54 -0800 Subject: [PATCH 05/99] Update build.sh --- packages/apache2/build.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/apache2/build.sh b/packages/apache2/build.sh index d9a0bc36e..fbfe7e037 100644 --- a/packages/apache2/build.sh +++ b/packages/apache2/build.sh @@ -71,6 +71,11 @@ TERMUX_PKG_INCLUDE_IN_DEVPACKAGE="share/apache2/build" TERMUX_PKG_EXTRA_MAKE_ARGS="-s" termux_step_pre_configure () { + # address interpreter issues + sed -i 's/\/home\/fornwall\/.termux-build\/apr\/tmp\/libtool/\/usr\/bin\/libtool/' ${TERMUX_PREFIX}/bin/apr-1-config + sed -i 's/\/data\/data\/com.termux\/files\/usr\/bin\/bash/\/bin\/bash/' ${TERMUX_PREFIX}/share/apr-1/build/libtool + sed -i 's/\/home\/fornwall/\/home\/builder/' ${TERMUX_PREFIX}/share/apr-1/build/libtool + # remove old files rm -rf "$TERMUX_PREFIX"/{libexec,share,etc}/apache2 rm -rf "$TERMUX_PREFIX"/lib/cgi-bin From f55b366ed2a8d43885f8e5fb9d67be7ec11c07e2 Mon Sep 17 00:00:00 2001 From: Stephen Date: Thu, 30 Nov 2017 14:03:50 -0800 Subject: [PATCH 06/99] Update build.sh --- packages/php/build.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/php/build.sh b/packages/php/build.sh index 9dde61e2e..8bb400087 100644 --- a/packages/php/build.sh +++ b/packages/php/build.sh @@ -48,6 +48,14 @@ ac_cv_func_res_nsearch=no " termux_step_pre_configure () { + # Replace chrooted perl with system perl + sed -i 's/\/data\/data\/com\.termux\/files\/usr\/bin\/perl/\/usr\/bin\/perl/' $TERMUX_PREFIX/bin/apxs + sed -i 's/\/data\/data\/com\.termux\/files\/usr\/bin\/perl/\/usr\/bin\/perl/' $TERMUX_PREFIX/bin/curl-config + sed -i 's/\/home\/fornwall\/.termux-build\/apr\/tmp/\/home\/builder\/.termux-build\/php\/build/' $TERMUX_PREFIX/bin/apr-1-config + rm $TERMUX_PREFIX/bin/pg_config + + #because the new mariadb hides away all these includes inside server subdir + CFLAGS+=" -I$TERMUX_PREFIX/include/mysql/server -I$TERMUX_PREFIX/include/mysql" LDFLAGS+=" -landroid-glob -llog" export PATH=$PATH:$TERMUX_PKG_HOSTBUILD_DIR/sapi/cli/ From eb85a5b4facae36817e5d23d15b79da0036ca7be Mon Sep 17 00:00:00 2001 From: Stephen Date: Thu, 30 Nov 2017 15:00:30 -0800 Subject: [PATCH 07/99] Update build.sh --- packages/php/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/php/build.sh b/packages/php/build.sh index 8bb400087..fedea2f52 100644 --- a/packages/php/build.sh +++ b/packages/php/build.sh @@ -8,7 +8,7 @@ TERMUX_PKG_SRCURL=https://secure.php.net/distributions/php-${TERMUX_PKG_VERSION} TERMUX_PKG_HOSTBUILD=true # Build the native php without xml support as we only need phar: TERMUX_PKG_EXTRA_HOSTBUILD_CONFIGURE_ARGS="--disable-libxml --disable-dom --disable-simplexml --disable-xml --disable-xmlreader --disable-xmlwriter --without-pear" -TERMUX_PKG_DEPENDS="libandroid-glob, libxml2, liblzma, openssl, pcre, libbz2, libcrypt, libcurl, libgd, readline, freetype" +TERMUX_PKG_DEPENDS="libandroid-glob, libxml2, liblzma, openssl, pcre, libbz2, libcrypt, libcurl, libgd, readline, freetype, apr, apr-util, openssl, libpng, libjpeg-turbo" # mysql modules were initially shared libs TERMUX_PKG_CONFLICTS="php-mysql" TERMUX_PKG_REPLACES="php-mysql" From 8c43a6dcff3002f59f7ed1c969a182567fa3e6ec Mon Sep 17 00:00:00 2001 From: Stephen Date: Mon, 11 Dec 2017 08:10:46 -0800 Subject: [PATCH 08/99] cheap trick to fix + issue in package search --- build-package.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/build-package.sh b/build-package.sh index fa8e02180..d08aa3380 100755 --- a/build-package.sh +++ b/build-package.sh @@ -498,7 +498,8 @@ termux_step_start_build() { while IFS=',' read -ra PKG; do for p in "${PKG[@]}"; do p="$(echo -e "${p}" | tr -d '[:space:]')" - apt-get $TERMUX_APT install "^${p}(-dev)?$":any + # Install packages and dev packages and escape + with \+ to avoid apt regex + apt-get $TERMUX_APT install "^${p//+/\\+}(-dev)?$":any sudo chown -R builder:builder /data done done <<< "${!i}" @@ -510,7 +511,8 @@ termux_step_start_build() { while IFS=',' read -ra PKG; do for p in "${PKG[@]}"; do p="$(echo -e "${p}" | tr -d '[:space:]')" - apt-get $TERMUX_APT install "^${p}(-dev)?$":any + # Install packages and dev packages and escape + with \+ to avoid apt regex + apt-get $TERMUX_APT install "^${p//+/\\+}(-dev)?$":any sudo chown -R builder:builder /data done done <<< $TERMUX_SUBPKG_DEPENDS From 7af73fe3a4e6bbe3f2a841bff63ea7b6c61676ef Mon Sep 17 00:00:00 2001 From: Stephen Date: Thu, 15 Feb 2018 09:18:29 -0800 Subject: [PATCH 09/99] Update build-package.sh --- build-package.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/build-package.sh b/build-package.sh index d08aa3380..d8c506e59 100755 --- a/build-package.sh +++ b/build-package.sh @@ -456,7 +456,7 @@ termux_step_start_build() { unzip ${TERMUX_COMMON_CACHEDIR}/bootstrap-${TERMUX_ARCH}.zip -d $TERMUX_PREFIX # TODO move this install to Dockerfile - sudo apt-get update && sudo apt-get -y dist-upgrade && sudo apt-get install -y libcap2-bin gawk tree strace + sudo apt-get update && sudo apt-get -y dist-upgrade && sudo apt-get install -y libcap2-bin gawk tree strace devscripts # set capabilities on dpkg # Some packages built by uid 1001 (builder is 1000) # Need capabilities for dpkg to set non-builder uid @@ -1387,6 +1387,12 @@ termux_step_reverse_depends() { if [ ! -z ${TERMUX_INSTALL_DEPS+x} ]; then # TODO build reverse depends with packages apt-cache $TERMUX_APT rdepends $TERMUX_PKG_NAME + + # TODO compare package with existing + echo "COMPARING PACKAGES" + apt $TERMUX_APT download $TERMUX_PKG_NAME + debdiff ${TERMUX_PKG_NAME}*.deb ${TERMUX_PKG_DEBFILE} + echo "DONE COMPARE PACKAGES" fi } From bc7037964ee09d96a572cb8c596141d3e6d2cc23 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Wed, 26 Dec 2018 12:21:14 +0100 Subject: [PATCH 10/99] updates --- build-package.sh | 121 ++++++++++++++++++----------------------------- 1 file changed, 47 insertions(+), 74 deletions(-) diff --git a/build-package.sh b/build-package.sh index d8c506e59..e9f2c0a8a 100755 --- a/build-package.sh +++ b/build-package.sh @@ -254,20 +254,20 @@ termux_step_handle_arguments() { echo " -d Build with debug symbols." echo " -D Build a disabled package in disabled-packages/." echo " -f Force build even if package has already been built." - echo " -i Install dependencies." + echo " -i Build dependencies." echo " -q Quiet build." echo " -s Skip dependency check." echo " -o Specify deb directory. Default: debs/." exit 1 } - while getopts :a:hdDfqso: option; do + while getopts :a:hdDfiqso: option; do case "$option" in a) TERMUX_ARCH="$OPTARG";; h) _show_usage;; d) export TERMUX_DEBUG=true;; D) local TERMUX_IS_DISABLED=true;; f) TERMUX_FORCE_BUILD=true;; - i) export TERMUX_INSTALL_DEPS=true;; + i) export TERMUX_BUILD_DEPS=true;; q) export TERMUX_QUIET_BUILD=true;; s) export TERMUX_SKIP_DEPCHECK=true;; o) TERMUX_DEBDIR="$(realpath -m $OPTARG)";; @@ -323,6 +323,7 @@ termux_step_setup_variables() { : "${TERMUX_DEBUG:=""}" : "${TERMUX_PKG_API_LEVEL:="21"}" : "${TERMUX_DEBDIR:="${TERMUX_SCRIPTDIR}/debs"}" + : "${TERMUX_REPO_URL:="https://termux.net/dists/stable/main"}" if [ "x86_64" = "$TERMUX_ARCH" ] || [ "aarch64" = "$TERMUX_ARCH" ]; then TERMUX_ARCH_BITS=64 @@ -403,7 +404,7 @@ termux_step_handle_buildarch() { local TERMUX_PREVIOUS_ARCH TERMUX_PREVIOUS_ARCH=$(cat $TERMUX_ARCH_FILE) if [ "$TERMUX_PREVIOUS_ARCH" != "$TERMUX_ARCH" ]; then - local TERMUX_DATA_BACKUPDIRS=$TERMUX_TOPDIR/_databackups + local TERMUX_DATA_BACKUPDIRS=$TERMUX_TOPDIR/_databackups mkdir -p "$TERMUX_DATA_BACKUPDIRS" local TERMUX_DATA_PREVIOUS_BACKUPDIR=$TERMUX_DATA_BACKUPDIRS/$TERMUX_PREVIOUS_ARCH local TERMUX_DATA_CURRENT_BACKUPDIR=$TERMUX_DATA_BACKUPDIRS/$TERMUX_ARCH @@ -440,10 +441,11 @@ termux_step_start_build() { exit 0 fi - if [ ! -z ${TERMUX_INSTALL_DEPS+x} ]; then + if [ ! ${TERMUX_BUILD_DEPS:=false} = true ]; then # Ensure folders present (but not $TERMUX_PKG_SRCDIR, it will be created in build) mkdir -p "$TERMUX_COMMON_CACHEDIR" \ - "$TERMUX_DEBDIR" \ + "${TERMUX_COMMON_CACHEDIR}-${TERMUX_ARCH}" \ + "$TERMUX_DEBDIR" \ "$TERMUX_PKG_BUILDDIR" \ "$TERMUX_PKG_PACKAGEDIR" \ "$TERMUX_PKG_TMPDIR" \ @@ -452,74 +454,33 @@ termux_step_start_build() { $TERMUX_PREFIX/{bin,etc,lib,libexec,share,tmp,include} # Setup bootstrap termux_download https://termux.net/bootstrap/bootstrap-${TERMUX_ARCH}.zip \ - ${TERMUX_COMMON_CACHEDIR}/bootstrap-${TERMUX_ARCH}.zip - unzip ${TERMUX_COMMON_CACHEDIR}/bootstrap-${TERMUX_ARCH}.zip -d $TERMUX_PREFIX - - # TODO move this install to Dockerfile - sudo apt-get update && sudo apt-get -y dist-upgrade && sudo apt-get install -y libcap2-bin gawk tree strace devscripts - # set capabilities on dpkg - # Some packages built by uid 1001 (builder is 1000) - # Need capabilities for dpkg to set non-builder uid - sudo /sbin/setcap cap_chown,cap_fowner,cap_dac_override+eip /usr/bin/dpkg - sudo /sbin/setcap cap_chown,cap_fowner,cap_dac_override+eip /usr/bin/apt-get - # install packages that include subpackages - sudo sed -i -e 's/DPkg::Pre-Install-Pkgs/\/\/ DPkg::Pre-Install-Pkgs/' /etc/apt/apt.conf.d/* - sudo sed -i -e 's/DPkg::Post-Invoke/\/\/ DPkg::Post-Invoke/' /etc/apt/apt.conf.d/* - sudo sed -i -e 's/APT::Update::Post-Invoke/\/\/ APT::Update::Post-Invoke/' /etc/apt/apt.conf.d/* - TERMUX_DPKG=" \ - --force-architecture \ - --force-not-root \ - --force-configure-any \ - --force-confdef \ - --force-confold \ - --force-depends \ - --admindir=${TERMUX_PREFIX}/var/lib/dpkg" - export TERMUX_APT=" \ - -o APT::Get::Assume-Yes=true \ - -o APT::Get::ReInstall=true - -o APT::Get::Install-Recommends=false \ - -o APT::Get::Install-Suggests=false \ - -o APT::Architecture=${TERMUX_ARCH} \ - -o Dir::Etc=${TERMUX_PREFIX}/etc/apt/ \ - -o Dir::State=${TERMUX_PREFIX}/var/lib/apt \ - -o Dir::State::status=${TERMUX_PREFIX}/var/lib/dpkg/status \ - -o Dir::Cache=${TERMUX_PREFIX}/var/cache/apt \ - -o Dir::Log=${TERMUX_PREFIX}/var/log/apt" - for arg in ${TERMUX_DPKG}; do - TERMUX_APT+=" -o DPkg::Options::=${arg}" - done - export DEBCONF_FRONTEND=noninteractive - apt-get $TERMUX_APT clean && apt-get $TERMUX_APT update && apt-get $TERMUX_APT upgrade - # libandroid-support-dev is build-essential - apt-get $TERMUX_APT install libandroid-support-dev:any - sudo chown -R builder:builder /data - array=( TERMUX_PKG_DEPENDS TERMUX_PKG_BUILD_DEPENDS ) - for i in "${array[@]}"; do - while IFS=',' read -ra PKG; do - for p in "${PKG[@]}"; do - p="$(echo -e "${p}" | tr -d '[:space:]')" - # Install packages and dev packages and escape + with \+ to avoid apt regex - apt-get $TERMUX_APT install "^${p//+/\\+}(-dev)?$":any - sudo chown -R builder:builder /data - done - done <<< "${!i}" - done - for subpkg in $TERMUX_PKG_BUILDER_DIR/*.subpackage.sh; do - test ! -f "$subpkg" && continue - local TERMUX_SUBPKG_DEPENDS="" - source $subpkg - while IFS=',' read -ra PKG; do - for p in "${PKG[@]}"; do - p="$(echo -e "${p}" | tr -d '[:space:]')" - # Install packages and dev packages and escape + with \+ to avoid apt regex - apt-get $TERMUX_APT install "^${p//+/\\+}(-dev)?$":any - sudo chown -R builder:builder /data - done - done <<< $TERMUX_SUBPKG_DEPENDS + ${TERMUX_COMMON_CACHEDIR}/bootstrap-${TERMUX_ARCH}.zip \ + 9f43bfdd23fbc6ae02f1f5a44b8e064374db8291ffe32658e2724d3f3f419981 + unzip -qo ${TERMUX_COMMON_CACHEDIR}/bootstrap-${TERMUX_ARCH}.zip -d $TERMUX_PREFIX + ( + cd $TERMUX_PREFIX + while read line; do + ln -sf ${line/←/ } + done Date: Sat, 29 Dec 2018 11:56:40 +0100 Subject: [PATCH 11/99] build-package: add termux_extract_version function --- build-package.sh | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/build-package.sh b/build-package.sh index e9f2c0a8a..ec48b53d5 100755 --- a/build-package.sh +++ b/build-package.sh @@ -426,6 +426,15 @@ termux_step_handle_buildarch() { echo "$TERMUX_ARCH" > $TERMUX_ARCH_FILE } +# Function to get TERMUX_PKG_VERSION from build.sh +termux_extract_version() { + package=$1 + ( + source $package/build.sh + echo $TERMUX_PKG_VERSION-$TERMUX_PKG_REVISION + ) +} + # Source the package build script and start building. No to be overridden by packages. termux_step_start_build() { # shellcheck source=/dev/null @@ -471,11 +480,8 @@ termux_step_start_build() { TERMUX_ALL_DEPS=$(./scripts/buildorder.py "$TERMUX_PKG_BUILDER_DIR") for p in $TERMUX_ALL_DEPS; do echo "Downloading dependency $(basename $p) if necessary..." - # termux_get_deb $TERMUX_ARCH "$p" - local p_ver=$(termux_extract_version "$p") - echo "hej" - echo "$p_ver" - echo "håå" + local dep_version=$(termux_extract_version "$p") + termux_get_deb $TERMUX_ARCH "$p" done fi From bff8f2c6f6d5acd31404a8b5ed736d95d0584834 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sat, 29 Dec 2018 14:50:27 +0100 Subject: [PATCH 12/99] Working deb_get_info and install_dep_deb functions --- build-package.sh | 101 ++++++++++++++++++++++++++--------------------- 1 file changed, 55 insertions(+), 46 deletions(-) diff --git a/build-package.sh b/build-package.sh index ec48b53d5..6979aea67 100755 --- a/build-package.sh +++ b/build-package.sh @@ -267,7 +267,7 @@ termux_step_handle_arguments() { d) export TERMUX_DEBUG=true;; D) local TERMUX_IS_DISABLED=true;; f) TERMUX_FORCE_BUILD=true;; - i) export TERMUX_BUILD_DEPS=true;; + i) TERMUX_BUILD_DEPS=true;; q) export TERMUX_QUIET_BUILD=true;; s) export TERMUX_SKIP_DEPCHECK=true;; o) TERMUX_DEBDIR="$(realpath -m $OPTARG)";; @@ -323,6 +323,8 @@ termux_step_setup_variables() { : "${TERMUX_DEBUG:=""}" : "${TERMUX_PKG_API_LEVEL:="21"}" : "${TERMUX_DEBDIR:="${TERMUX_SCRIPTDIR}/debs"}" + : "${TERMUX_SKIP_DEPCHECK:="false"}" + : "${TERMUX_BUILD_DEPS:="false"}" : "${TERMUX_REPO_URL:="https://termux.net/dists/stable/main"}" if [ "x86_64" = "$TERMUX_ARCH" ] || [ "aarch64" = "$TERMUX_ARCH" ]; then @@ -427,14 +429,35 @@ termux_step_handle_buildarch() { } # Function to get TERMUX_PKG_VERSION from build.sh -termux_extract_version() { +termux_extract_dep_info() { package=$1 ( source $package/build.sh - echo $TERMUX_PKG_VERSION-$TERMUX_PKG_REVISION + if [ "$TERMUX_PKG_PLATFORM_INDEPENDENT" = "yes" ]; then + TERMUX_ARCH=all + fi + if [ ! "$TERMUX_PKG_REVISION" = 0 ]; then + TERMUX_PKG_VERSION+="-$TERMUX_PKG_REVISION" + fi + echo ${TERMUX_ARCH} ${TERMUX_PKG_VERSION} ) } +termux_install_dep_deb() { + local package=$1 + local package_arch=$2 + local version=$3 + local deb_file=${package}_${version}_${package_arch}.deb + ( + cd ${TERMUX_COMMON_CACHEDIR}-${package_arch} + # TODO: allow for specifying several repos in TERMUX_REPO_URL + curl --fail -LO $TERMUX_REPO_URL/binary-${package_arch}/${deb_file} 2>/dev/null \ + && echo "Extracting $package..." && ar x ${deb_file} data.tar.xz \ + && tar xf data.tar.xz --no-overwrite-dir -C / + ) || ( echo "Download of $package from $TERMUX_REPO_URL failed, building instead" \ + && ./build-package.sh -a $TERMUX_ARCH -s "$package" ) +} + # Source the package build script and start building. No to be overridden by packages. termux_step_start_build() { # shellcheck source=/dev/null @@ -450,10 +473,11 @@ termux_step_start_build() { exit 0 fi - if [ ! ${TERMUX_BUILD_DEPS:=false} = true ]; then + local TERMUX_ALL_DEPS=$(./scripts/buildorder.py "$TERMUX_PKG_BUILDER_DIR") + if [ ! $TERMUX_SKIP_DEPCHECK ] && [ ! $TERMUX_BUILD_DEPS ]; then # Ensure folders present (but not $TERMUX_PKG_SRCDIR, it will be created in build) mkdir -p "$TERMUX_COMMON_CACHEDIR" \ - "${TERMUX_COMMON_CACHEDIR}-${TERMUX_ARCH}" \ + "$TERMUX_COMMON_CACHEDIR-$TERMUX_ARCH" \ "$TERMUX_DEBDIR" \ "$TERMUX_PKG_BUILDDIR" \ "$TERMUX_PKG_PACKAGEDIR" \ @@ -473,39 +497,24 @@ termux_step_start_build() { done Date: Sat, 29 Dec 2018 16:02:39 +0100 Subject: [PATCH 13/99] Revert "Update build.sh" This reverts commit 7668e25134a27e733e8c24a47df2f7456934e586. --- packages/libllvm/build.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/libllvm/build.sh b/packages/libllvm/build.sh index 0a7cb450e..a61540978 100644 --- a/packages/libllvm/build.sh +++ b/packages/libllvm/build.sh @@ -59,9 +59,6 @@ termux_step_post_extract_package () { } termux_step_host_build () { - if [ ! -z ${TERMUX_INSTALL_DEPS+x} ]; then - apt-get $TERMUX_APT purge ndk-sysroot - fi termux_setup_cmake cmake -G "Unix Makefiles" $TERMUX_PKG_SRCDIR \ -DLLVM_BUILD_TESTS=OFF \ From 59da7603949223ad3934c53268fe3c633dd5eb59 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sat, 29 Dec 2018 17:25:52 +0100 Subject: [PATCH 14/99] Change TERMUX_BUILD_DEPS to TERMUX_INSTALL_DEPS again. Fix syntax in if expressions. --- build-package.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/build-package.sh b/build-package.sh index 6979aea67..ca8ce29c6 100755 --- a/build-package.sh +++ b/build-package.sh @@ -267,7 +267,7 @@ termux_step_handle_arguments() { d) export TERMUX_DEBUG=true;; D) local TERMUX_IS_DISABLED=true;; f) TERMUX_FORCE_BUILD=true;; - i) TERMUX_BUILD_DEPS=true;; + i) export TERMUX_INSTALL_DEPS=true;; q) export TERMUX_QUIET_BUILD=true;; s) export TERMUX_SKIP_DEPCHECK=true;; o) TERMUX_DEBDIR="$(realpath -m $OPTARG)";; @@ -324,7 +324,7 @@ termux_step_setup_variables() { : "${TERMUX_PKG_API_LEVEL:="21"}" : "${TERMUX_DEBDIR:="${TERMUX_SCRIPTDIR}/debs"}" : "${TERMUX_SKIP_DEPCHECK:="false"}" - : "${TERMUX_BUILD_DEPS:="false"}" + : "${TERMUX_INSTALL_DEPS:="false"}" : "${TERMUX_REPO_URL:="https://termux.net/dists/stable/main"}" if [ "x86_64" = "$TERMUX_ARCH" ] || [ "aarch64" = "$TERMUX_ARCH" ]; then @@ -474,7 +474,7 @@ termux_step_start_build() { fi local TERMUX_ALL_DEPS=$(./scripts/buildorder.py "$TERMUX_PKG_BUILDER_DIR") - if [ ! $TERMUX_SKIP_DEPCHECK ] && [ ! $TERMUX_BUILD_DEPS ]; then + if [ "$TERMUX_SKIP_DEPCHECK" = false ] && [ "$TERMUX_INSTALL_DEPS" = true ]; then # Ensure folders present (but not $TERMUX_PKG_SRCDIR, it will be created in build) mkdir -p "$TERMUX_COMMON_CACHEDIR" \ "$TERMUX_COMMON_CACHEDIR-$TERMUX_ARCH" \ @@ -505,7 +505,7 @@ termux_step_start_build() { read dep_arch dep_version <<< $(termux_extract_dep_info "$pkg") termux_install_dep_deb $(basename $pkg) $dep_arch $dep_version done - elif [ ! $TERMUX_SKIP_DEPCHECK ] && [ $TERMUX_BUILD_DEPS ]; then + elif [ "$TERMUX_SKIP_DEPCHECK" = false ] && [ "$TERMUX_INSTALL_DEPS" = false ]; then # Build dependencies local pkg for pkg in $TERMUX_ALL_DEPS; do @@ -523,7 +523,7 @@ termux_step_start_build() { TERMUX_PKG_FULLVERSION+="-$TERMUX_PKG_REVISION" fi - if [ "$TERMUX_DEBUG" == "true" ]; then + if [ "$TERMUX_DEBUG" = true ]; then if [ "$TERMUX_PKG_HAS_DEBUG" == "yes" ]; then DEBUG="-dbg" else From e40d87f6248a8b58365e5eabc6ac4ca3c9396c37 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sat, 29 Dec 2018 17:26:35 +0100 Subject: [PATCH 15/99] build-package: add function to compare debs --- build-package.sh | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/build-package.sh b/build-package.sh index ca8ce29c6..b1c712ccc 100755 --- a/build-package.sh +++ b/build-package.sh @@ -478,6 +478,7 @@ termux_step_start_build() { # Ensure folders present (but not $TERMUX_PKG_SRCDIR, it will be created in build) mkdir -p "$TERMUX_COMMON_CACHEDIR" \ "$TERMUX_COMMON_CACHEDIR-$TERMUX_ARCH" \ + "$TERMUX_COMMON_CACHEDIR-all" \ "$TERMUX_DEBDIR" \ "$TERMUX_PKG_BUILDDIR" \ "$TERMUX_PKG_PACKAGEDIR" \ @@ -1371,18 +1372,24 @@ termux_step_create_debfile() { "$TERMUX_PKG_PACKAGEDIR/data.tar.xz" } -# termux_step_reverse_depends() { -# if [ ! ${TERMUX_BUILD_DEPS:=false} = true ]; then -# # TODO build reverse depends with packages -# ### apt-cache $TERMUX_APT rdepends $TERMUX_PKG_NAME -# -# # TODO compare package with existing -# echo "COMPARING PACKAGES" -# ### apt $TERMUX_APT download $TERMUX_PKG_NAME -# debdiff ${TERMUX_PKG_NAME}*.deb ${TERMUX_PKG_DEBFILE} -# echo "DONE COMPARE PACKAGES" -# fi -# } +termux_step_reverse_depends() { + if [ "${TERMUX_INSTALL_DEPS}" = true ]; then + local arch version + echo "COMPARING PACKAGES" + read arch version <<< $(termux_extract_dep_info "$TERMUX_PKG_BUILDER_DIR") + termux_install_dep_deb $(basename $TERMUX_PKG_BUILDER_DIR) $arch $version + deb_file=${TERMUX_PKG_NAME}_${version}_${arch}.deb + + ( + cd ${TERMUX_COMMON_CACHEDIR}-${arch} + # TODO: allow for specifying several repos in TERMUX_REPO_URL + curl --fail -LO $TERMUX_REPO_URL/binary-${arch}/${deb_file} \ + && echo "Extracting ${TERMUX_PKG_NAME}..." + ) + debdiff $TERMUX_DEBDIR/$deb_file $TERMUX_COMMON_CACHEDIR-$arch/$deb_file + echo "DONE COMPARING PACKAGES" + fi +} # Finish the build. Not to be overridden by package scripts. termux_step_finish_build() { @@ -1424,5 +1431,5 @@ cd "$TERMUX_PKG_MASSAGEDIR/$TERMUX_PREFIX" termux_step_post_massage termux_step_create_datatar termux_step_create_debfile -# termux_step_reverse_depends +termux_step_reverse_depends termux_step_finish_build From a0e340779efeb911a360a1a84b78e2f7c5243922 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sat, 29 Dec 2018 17:27:18 +0100 Subject: [PATCH 16/99] build-package: make the bootstrap installtion more intuitive --- build-package.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build-package.sh b/build-package.sh index b1c712ccc..f739d5b82 100755 --- a/build-package.sh +++ b/build-package.sh @@ -493,8 +493,8 @@ termux_step_start_build() { unzip -qo ${TERMUX_COMMON_CACHEDIR}/bootstrap-${TERMUX_ARCH}.zip -d $TERMUX_PREFIX ( cd $TERMUX_PREFIX - while read line; do - ln -sf ${line/←/ } + while read link; do + ln -sf ${link/←/ } done Date: Sun, 30 Dec 2018 00:33:49 +0100 Subject: [PATCH 17/99] termux_extract_dep_info: Fix bug when ARCH=all and REVISION != 0 Also make it less verbose when building with -q and add some TODO notes --- build-package.sh | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/build-package.sh b/build-package.sh index f739d5b82..c7aabc91d 100755 --- a/build-package.sh +++ b/build-package.sh @@ -432,8 +432,14 @@ termux_step_handle_buildarch() { termux_extract_dep_info() { package=$1 ( + # Reset TERMUX_PKG_PLATFORM_INDEPENDENT and TERMUX_PKG_REVISION since these aren't + # mandatory in a build.sh. Otherwise these will equal the main package's values for + # deps that should have the default values + TERMUX_PKG_PLATFORM_INDEPENDENT="" + TERMUX_PKG_REVISION="0" + source $package/build.sh - if [ "$TERMUX_PKG_PLATFORM_INDEPENDENT" = "yes" ]; then + if [ "$TERMUX_PKG_PLATFORM_INDEPENDENT" = yes ]; then TERMUX_ARCH=all fi if [ ! "$TERMUX_PKG_REVISION" = 0 ]; then @@ -448,14 +454,15 @@ termux_install_dep_deb() { local package_arch=$2 local version=$3 local deb_file=${package}_${version}_${package_arch}.deb + # TODO: download InRelease, Packages files and check signature and hash ( cd ${TERMUX_COMMON_CACHEDIR}-${package_arch} # TODO: allow for specifying several repos in TERMUX_REPO_URL curl --fail -LO $TERMUX_REPO_URL/binary-${package_arch}/${deb_file} 2>/dev/null \ - && echo "Extracting $package..." && ar x ${deb_file} data.tar.xz \ - && tar xf data.tar.xz --no-overwrite-dir -C / - ) || ( echo "Download of $package from $TERMUX_REPO_URL failed, building instead" \ - && ./build-package.sh -a $TERMUX_ARCH -s "$package" ) + && if [ ! "$TERMUX_QUIET_BUILD" = true ]; then echo "Extracting $package..."; fi \ + && ar x ${deb_file} data.tar.xz && tar xf data.tar.xz --no-overwrite-dir -C / + # TODO: this implementation is buggy if the `ar x` or `tar xf + ) } # Source the package build script and start building. No to be overridden by packages. From 24a307c64fc3a5a3e73d9cdb1196e77f245cf56d Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sun, 30 Dec 2018 00:37:21 +0100 Subject: [PATCH 18/99] build-package.sh: make dep downloads less verbose for quiet builds Add error messages when download fails though. Try to download -dev packages but ignore errors here --- build-package.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/build-package.sh b/build-package.sh index c7aabc91d..1bfdcb06c 100755 --- a/build-package.sh +++ b/build-package.sh @@ -509,9 +509,17 @@ termux_step_start_build() { # Download dependencies local pkg dep_arch dep_version for pkg in $TERMUX_ALL_DEPS; do - echo "Downloading dependency $(basename $pkg) if necessary..." read dep_arch dep_version <<< $(termux_extract_dep_info "$pkg") - termux_install_dep_deb $(basename $pkg) $dep_arch $dep_version + if [ ! "$TERMUX_QUIET_BUILD" = true ]; then + echo "Downloading dependency $(basename $pkg) $dep_version if necessary..." + fi + termux_install_dep_deb $(basename $pkg) $dep_arch $dep_version \ + || ( echo "Download of $(basename $pkg) $dep_version from $TERMUX_REPO_URL failed, building instead" \ + && ./build-package.sh -a $TERMUX_ARCH -s "$pkg" \ + && continue ) + + termux_install_dep_deb $(basename $pkg)-dev $dep_arch $dep_version || \ + echo "Download of $(basename $pkg)-dev $dep_version from $TERMUX_REPO_URL failed" done elif [ "$TERMUX_SKIP_DEPCHECK" = false ] && [ "$TERMUX_INSTALL_DEPS" = false ]; then # Build dependencies From 04220d9f220632aa447fc1ccb92e059a57c2deec Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sun, 30 Dec 2018 12:22:11 +0100 Subject: [PATCH 19/99] build-package: fix forgotten p->pkg --- build-package.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build-package.sh b/build-package.sh index 1bfdcb06c..d605f2c78 100755 --- a/build-package.sh +++ b/build-package.sh @@ -525,9 +525,9 @@ termux_step_start_build() { # Build dependencies local pkg for pkg in $TERMUX_ALL_DEPS; do - echo "Building dependency $p if necessary..." + echo "Building dependency $pkg if necessary..." # Built dependencies are put in the default TERMUX_DEBDIR instead of the specified one - ./build-package.sh -a $TERMUX_ARCH -s "$p" + ./build-package.sh -a $TERMUX_ARCH -s "$pkg" done else echo "Skipping dependency check" From fbe0de46d2f08a098352ff1f73c8bee5e19e89aa Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sun, 30 Dec 2018 12:22:44 +0100 Subject: [PATCH 20/99] build-package: Fix so that debdiff's exit code doesn't stop build --- build-package.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build-package.sh b/build-package.sh index d605f2c78..ce87e9843 100755 --- a/build-package.sh +++ b/build-package.sh @@ -1401,7 +1401,8 @@ termux_step_reverse_depends() { curl --fail -LO $TERMUX_REPO_URL/binary-${arch}/${deb_file} \ && echo "Extracting ${TERMUX_PKG_NAME}..." ) - debdiff $TERMUX_DEBDIR/$deb_file $TERMUX_COMMON_CACHEDIR-$arch/$deb_file + # `|| true` to prevent debdiff's exit code from stopping build + debdiff $TERMUX_DEBDIR/$deb_file $TERMUX_COMMON_CACHEDIR-$arch/$deb_file || true echo "DONE COMPARING PACKAGES" fi } From 88af24d0fd29258fc94d20edca4d36a67aead419 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sun, 30 Dec 2018 12:23:12 +0100 Subject: [PATCH 21/99] build-package: Set default verbose level --- build-package.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/build-package.sh b/build-package.sh index ce87e9843..fe114b5e1 100755 --- a/build-package.sh +++ b/build-package.sh @@ -322,6 +322,7 @@ termux_step_setup_variables() { : "${TERMUX_ANDROID_HOME:="/data/data/com.termux/files/home"}" : "${TERMUX_DEBUG:=""}" : "${TERMUX_PKG_API_LEVEL:="21"}" + : "${TERMUX_QUIET_BUILD:="false"}" : "${TERMUX_DEBDIR:="${TERMUX_SCRIPTDIR}/debs"}" : "${TERMUX_SKIP_DEPCHECK:="false"}" : "${TERMUX_INSTALL_DEPS:="false"}" From 58c91b51cd4b35a0285d582e3dd47eda491a4731 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sun, 30 Dec 2018 12:24:16 +0100 Subject: [PATCH 22/99] build-package: fix truncated comment --- build-package.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-package.sh b/build-package.sh index fe114b5e1..89fa66e5b 100755 --- a/build-package.sh +++ b/build-package.sh @@ -462,7 +462,7 @@ termux_install_dep_deb() { curl --fail -LO $TERMUX_REPO_URL/binary-${package_arch}/${deb_file} 2>/dev/null \ && if [ ! "$TERMUX_QUIET_BUILD" = true ]; then echo "Extracting $package..."; fi \ && ar x ${deb_file} data.tar.xz && tar xf data.tar.xz --no-overwrite-dir -C / - # TODO: this implementation is buggy if the `ar x` or `tar xf + # TODO: this implementation is buggy if the `ar x` or `tar xf` steps fail ) } From 72402ea1cf3a8bbdc43f82e0f648042ea345b1c9 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Mon, 31 Dec 2018 00:31:11 +0100 Subject: [PATCH 23/99] build-package: Add signature verification and download by hash Also rename termux_install_dep_deb to termux_download_deb. --- build-package.sh | 74 ++++++++++++++++++++++++++++++----------- scripts/get_pkg_hash.py | 20 +++++++++++ 2 files changed, 74 insertions(+), 20 deletions(-) create mode 100755 scripts/get_pkg_hash.py diff --git a/build-package.sh b/build-package.sh index 89fa66e5b..990dc6754 100755 --- a/build-package.sh +++ b/build-package.sh @@ -326,7 +326,9 @@ termux_step_setup_variables() { : "${TERMUX_DEBDIR:="${TERMUX_SCRIPTDIR}/debs"}" : "${TERMUX_SKIP_DEPCHECK:="false"}" : "${TERMUX_INSTALL_DEPS:="false"}" - : "${TERMUX_REPO_URL:="https://termux.net/dists/stable/main"}" + : "${TERMUX_REPO_URL:="https://termux.net/dists"}" + : "${TERMUX_REPO_DISTRIBUTION:="stable"}" + : "${TERMUX_REPO_COMPONENT:="main"}" if [ "x86_64" = "$TERMUX_ARCH" ] || [ "aarch64" = "$TERMUX_ARCH" ]; then TERMUX_ARCH_BITS=64 @@ -450,20 +452,22 @@ termux_extract_dep_info() { ) } -termux_install_dep_deb() { +termux_download_deb() { local package=$1 local package_arch=$2 local version=$3 local deb_file=${package}_${version}_${package_arch}.deb - # TODO: download InRelease, Packages files and check signature and hash - ( - cd ${TERMUX_COMMON_CACHEDIR}-${package_arch} - # TODO: allow for specifying several repos in TERMUX_REPO_URL - curl --fail -LO $TERMUX_REPO_URL/binary-${package_arch}/${deb_file} 2>/dev/null \ - && if [ ! "$TERMUX_QUIET_BUILD" = true ]; then echo "Extracting $package..."; fi \ - && ar x ${deb_file} data.tar.xz && tar xf data.tar.xz --no-overwrite-dir -C / - # TODO: this implementation is buggy if the `ar x` or `tar xf` steps fail - ) + local pkg_hash=$(./scripts/get_pkg_hash.py ${TERMUX_COMMON_CACHEDIR}-${package_arch}/Packages $package) + + if [ "$pkg_hash" = "" ]; then + # No hash found for $package + return 1 + fi + + termux_download $TERMUX_REPO_URL/$TERMUX_REPO_DISTRIBUTION/$TERMUX_REPO_COMPONENT/binary-${package_arch}/${deb_file} \ + $TERMUX_COMMON_CACHEDIR-$package_arch/${deb_file} \ + $pkg_hash + return 0 } # Source the package build script and start building. No to be overridden by packages. @@ -483,6 +487,9 @@ termux_step_start_build() { local TERMUX_ALL_DEPS=$(./scripts/buildorder.py "$TERMUX_PKG_BUILDER_DIR") if [ "$TERMUX_SKIP_DEPCHECK" = false ] && [ "$TERMUX_INSTALL_DEPS" = true ]; then + # Remove all previously extracted/built files from $TERMUX_PREFIX: + rm -r $TERMUX_PREFIX + rm -f /data/data/.built-packages/* # Ensure folders present (but not $TERMUX_PKG_SRCDIR, it will be created in build) mkdir -p "$TERMUX_COMMON_CACHEDIR" \ "$TERMUX_COMMON_CACHEDIR-$TERMUX_ARCH" \ @@ -506,21 +513,50 @@ termux_step_start_build() { done Packages || termux_error_exit "Download of $TERMUX_REPO_URL/$TERMUX_REPO_DISTRIBUTION/$TERMUX_REPO_COMPONENT/Packages.xz failed" + ) # Download dependencies - local pkg dep_arch dep_version + local pkg dep_arch dep_version deb_file for pkg in $TERMUX_ALL_DEPS; do read dep_arch dep_version <<< $(termux_extract_dep_info "$pkg") if [ ! "$TERMUX_QUIET_BUILD" = true ]; then - echo "Downloading dependency $(basename $pkg) $dep_version if necessary..." + echo "Downloading dependency $(basename $pkg)@$dep_version if necessary..." fi - termux_install_dep_deb $(basename $pkg) $dep_arch $dep_version \ - || ( echo "Download of $(basename $pkg) $dep_version from $TERMUX_REPO_URL failed, building instead" \ + termux_download_deb $(basename $pkg) $dep_arch $dep_version \ + || ( echo "Download of $(basename $pkg)@$dep_version from $TERMUX_REPO_URL failed, building instead" \ && ./build-package.sh -a $TERMUX_ARCH -s "$pkg" \ && continue ) - - termux_install_dep_deb $(basename $pkg)-dev $dep_arch $dep_version || \ - echo "Download of $(basename $pkg)-dev $dep_version from $TERMUX_REPO_URL failed" + local deb_file=$(basename $pkg)_${dep_version}_${dep_arch}.deb + if [ ! "$TERMUX_QUIET_BUILD" = true ]; then echo "Extracting $(basename $pkg)..."; fi + ( + cd $TERMUX_COMMON_CACHEDIR-$dep_arch + ar x ${deb_file} data.tar.xz && tar xf data.tar.xz --no-overwrite-dir -C / + ) + + termux_download_deb $(basename $pkg)-dev $dep_arch $dep_version && \ + ( + cd $TERMUX_COMMON_CACHEDIR-$dep_arch + ar x $(basename $pkg)-dev_${dep_version}_${dep_arch}.deb data.tar.xz + tar xf data.tar.xz --no-overwrite-dir -C / + ) || echo "Download of $(basename $pkg)-dev@$dep_version from $TERMUX_REPO_URL failed" + echo "$dep_version" > "/data/data/.built-packages/$(basename $pkg)" done elif [ "$TERMUX_SKIP_DEPCHECK" = false ] && [ "$TERMUX_INSTALL_DEPS" = false ]; then # Build dependencies @@ -530,8 +566,6 @@ termux_step_start_build() { # Built dependencies are put in the default TERMUX_DEBDIR instead of the specified one ./build-package.sh -a $TERMUX_ARCH -s "$pkg" done - else - echo "Skipping dependency check" fi TERMUX_PKG_FULLVERSION=$TERMUX_PKG_VERSION diff --git a/scripts/get_pkg_hash.py b/scripts/get_pkg_hash.py new file mode 100755 index 000000000..668392601 --- /dev/null +++ b/scripts/get_pkg_hash.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python + +import os, sys + +def get_hash_from_Packages(Packages_file, package, hash="SHA256"): + with open(Packages_file, 'r') as Packages: + package_list = Packages.read().split('\n\n') + for pkg in package_list: + if pkg.split('\n')[0] == "Package: "+package: + for line in pkg.split('\n'): + if line.startswith(hash): + print(line.split(" ")[1]) + break + break + +if __name__ == '__main__': + if len(sys.argv) < 2: + print('Too few arguments, I need the path to a Packages file and a package name. Exiting') + sys.exit(1) + get_hash_from_Packages(sys.argv[1], sys.argv[2]) From 4da66550a6fba44869772e38d9d03338a857fac6 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Mon, 31 Dec 2018 00:31:44 +0100 Subject: [PATCH 24/99] build-package: update so that termux_step_compare_debs dl's by hash Changes needed after termux_download_deb function update --- build-package.sh | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/build-package.sh b/build-package.sh index 990dc6754..b77744684 100755 --- a/build-package.sh +++ b/build-package.sh @@ -1422,23 +1422,17 @@ termux_step_create_debfile() { "$TERMUX_PKG_PACKAGEDIR/data.tar.xz" } -termux_step_reverse_depends() { +termux_step_compare_debs() { if [ "${TERMUX_INSTALL_DEPS}" = true ]; then local arch version - echo "COMPARING PACKAGES" - read arch version <<< $(termux_extract_dep_info "$TERMUX_PKG_BUILDER_DIR") - termux_install_dep_deb $(basename $TERMUX_PKG_BUILDER_DIR) $arch $version - deb_file=${TERMUX_PKG_NAME}_${version}_${arch}.deb + if [ ! "$TERMUX_QUIET_BUILD" = true ]; then echo "COMPARING PACKAGES"; fi + cd ${TERMUX_SCRIPTDIR} + termux_download_deb $(basename $TERMUX_PKG_BUILDER_DIR) $TERMUX_ARCH $TERMUX_PKG_FULLVERSION + deb_file=${TERMUX_PKG_NAME}_${TERMUX_PKG_FULLVERSION}_${TERMUX_ARCH}.deb - ( - cd ${TERMUX_COMMON_CACHEDIR}-${arch} - # TODO: allow for specifying several repos in TERMUX_REPO_URL - curl --fail -LO $TERMUX_REPO_URL/binary-${arch}/${deb_file} \ - && echo "Extracting ${TERMUX_PKG_NAME}..." - ) # `|| true` to prevent debdiff's exit code from stopping build - debdiff $TERMUX_DEBDIR/$deb_file $TERMUX_COMMON_CACHEDIR-$arch/$deb_file || true - echo "DONE COMPARING PACKAGES" + debdiff $TERMUX_DEBDIR/$deb_file $TERMUX_COMMON_CACHEDIR-$TERMUX_ARCH/$deb_file || true + if [ ! "$TERMUX_QUIET_BUILD" = true ]; then echo "DONE COMPARING PACKAGES"; fi fi } @@ -1482,5 +1476,5 @@ cd "$TERMUX_PKG_MASSAGEDIR/$TERMUX_PREFIX" termux_step_post_massage termux_step_create_datatar termux_step_create_debfile -termux_step_reverse_depends +termux_step_compare_debs termux_step_finish_build From a1d4f83377daf32b96738fa2a1ad87230fe53151 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Tue, 1 Jan 2019 20:22:32 +0100 Subject: [PATCH 25/99] Rename get_pkg_hash.py->get_hash_from_file.py --- build-package.sh | 22 ++++++++++----------- scripts/get_hash_from_file.py | 37 +++++++++++++++++++++++++++++++++++ scripts/get_pkg_hash.py | 20 ------------------- 3 files changed, 47 insertions(+), 32 deletions(-) create mode 100755 scripts/get_hash_from_file.py delete mode 100755 scripts/get_pkg_hash.py diff --git a/build-package.sh b/build-package.sh index b77744684..16845439b 100755 --- a/build-package.sh +++ b/build-package.sh @@ -326,6 +326,7 @@ termux_step_setup_variables() { : "${TERMUX_DEBDIR:="${TERMUX_SCRIPTDIR}/debs"}" : "${TERMUX_SKIP_DEPCHECK:="false"}" : "${TERMUX_INSTALL_DEPS:="false"}" + : "${TERMUX_REPO_SIGNING_KEY:="2218893D3F679BEFC421FD976700B77E6D8D0AE7"}" : "${TERMUX_REPO_URL:="https://termux.net/dists"}" : "${TERMUX_REPO_DISTRIBUTION:="stable"}" : "${TERMUX_REPO_COMPONENT:="main"}" @@ -457,7 +458,7 @@ termux_download_deb() { local package_arch=$2 local version=$3 local deb_file=${package}_${version}_${package_arch}.deb - local pkg_hash=$(./scripts/get_pkg_hash.py ${TERMUX_COMMON_CACHEDIR}-${package_arch}/Packages $package) + local pkg_hash=$(./scripts/get_hash_from_file.py ${TERMUX_COMMON_CACHEDIR}-${package_arch}/Packages $package) if [ "$pkg_hash" = "" ]; then # No hash found for $package @@ -518,19 +519,16 @@ termux_step_start_build() { curl --fail -LO "$TERMUX_REPO_URL/$TERMUX_REPO_DISTRIBUTION/InRelease" \ || termux_error_exit "Download of $TERMUX_REPO_URL/$TERMUX_REPO_DISTRIBUTION/InRelease failed" # Import Fornwalls key: - gpg --recv 2218893D3F679BEFC421FD976700B77E6D8D0AE7 + gpg --recv $TERMUX_REPO_SIGNING_KEY gpg --verify InRelease - for arch in all $TERMUX_ARCH; do - # A sha256 hashsum has length 64 so grep for hashes that are that long - local packages_hash=$(grep binary-$arch/Packages.xz $TERMUX_COMMON_CACHEDIR/InRelease | awk 'length($1) == 64 {print $1}') - termux_download "$TERMUX_REPO_URL/$TERMUX_REPO_DISTRIBUTION/$TERMUX_REPO_COMPONENT/binary-$arch/Packages.xz" \ - "${TERMUX_COMMON_CACHEDIR}-$arch/Packages.xz" \ - $packages_hash - xz -df "${TERMUX_COMMON_CACHEDIR}-$arch/Packages.xz" - done - # cd ${TERMUX_COMMON_CACHEDIR}-$TERMUX_ARCH - # curl --fail -L "$TERMUX_REPO_URL/$TERMUX_REPO_DISTRIBUTION/$TERMUX_REPO_COMPONENT/binary-${TERMUX_ARCH}/Packages.xz" | xz -d > Packages || termux_error_exit "Download of $TERMUX_REPO_URL/$TERMUX_REPO_DISTRIBUTION/$TERMUX_REPO_COMPONENT/Packages.xz failed" ) + for arch in all $TERMUX_ARCH; do + local packages_hash=$(./scripts/get_hash_from_file.py ${TERMUX_COMMON_CACHEDIR}/InRelease $arch) + termux_download "$TERMUX_REPO_URL/$TERMUX_REPO_DISTRIBUTION/$TERMUX_REPO_COMPONENT/binary-$arch/Packages.xz" \ + "${TERMUX_COMMON_CACHEDIR}-$arch/Packages.xz" \ + $packages_hash + xz -df "${TERMUX_COMMON_CACHEDIR}-$arch/Packages.xz" + done # Download dependencies local pkg dep_arch dep_version deb_file diff --git a/scripts/get_hash_from_file.py b/scripts/get_hash_from_file.py new file mode 100755 index 000000000..4d355832c --- /dev/null +++ b/scripts/get_hash_from_file.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python + +import os, sys + +def get_pkg_hash_from_Packages(Packages_file, package, hash="SHA256"): + with open(Packages_file, 'r') as Packages: + package_list = Packages.read().split('\n\n') + for pkg in package_list: + if pkg.split('\n')[0] == "Package: "+package: + for line in pkg.split('\n'): + if line.startswith(hash): + print(line.split(" ")[1]) + break + break + +def get_Packages_hash_from_InRelease(InRelease_file, arch, hash="SHA256"): + string_to_found = 'binary-'+arch+'/Packages.xz' + with open(InRelease_file, 'r') as InRelease: + hash_list = InRelease.readlines() + for i in range(len(hash_list)): + if hash_list[i].startswith(hash+':'): + break + for j in range(i, len(hash_list)): + if string_to_found in hash_list[j].strip(' '): + print(hash_list[j].strip(' ').split(' ')[0]) + break + +if __name__ == '__main__': + if len(sys.argv) < 2: + sys.exit('Too few arguments, I need the path to a Packages file and a package name. Exiting') + + if sys.argv[1].endswith('Packages'): + get_pkg_hash_from_Packages(sys.argv[1], sys.argv[2]) + elif sys.argv[1].endswith('InRelease'): + get_Packages_hash_from_InRelease(sys.argv[1], sys.argv[2]) + else: + sys.exit(sys.argv[1]+' does not seem to be a path to a Packages or InRelease file') diff --git a/scripts/get_pkg_hash.py b/scripts/get_pkg_hash.py deleted file mode 100755 index 668392601..000000000 --- a/scripts/get_pkg_hash.py +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env python - -import os, sys - -def get_hash_from_Packages(Packages_file, package, hash="SHA256"): - with open(Packages_file, 'r') as Packages: - package_list = Packages.read().split('\n\n') - for pkg in package_list: - if pkg.split('\n')[0] == "Package: "+package: - for line in pkg.split('\n'): - if line.startswith(hash): - print(line.split(" ")[1]) - break - break - -if __name__ == '__main__': - if len(sys.argv) < 2: - print('Too few arguments, I need the path to a Packages file and a package name. Exiting') - sys.exit(1) - get_hash_from_Packages(sys.argv[1], sys.argv[2]) From 11daf710d97bc056790ef1aa4e0da936fe756fb5 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Tue, 1 Jan 2019 21:11:37 +0100 Subject: [PATCH 26/99] build-all.sh: add -i option to build and download dependencies --- build-all.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/build-all.sh b/build-all.sh index 1fd8d64c1..e5fd04943 100755 --- a/build-all.sh +++ b/build-all.sh @@ -8,20 +8,23 @@ test -f $HOME/.termuxrc && . $HOME/.termuxrc : ${TERMUX_TOPDIR:="$HOME/.termux-build"} : ${TERMUX_ARCH:="aarch64"} : ${TERMUX_DEBUG:=""} +: ${TERMUX_INSTALL_DEPS:=""} _show_usage () { - echo "Usage: ./build-all.sh [-a ARCH] [-d] [-o DIR]" + echo "Usage: ./build-all.sh [-a ARCH] [-d] [-i] [-o DIR]" echo "Build all packages." echo " -a The architecture to build for: aarch64(default), arm, i686, x86_64 or all." echo " -d Build with debug symbols." + echo " -i Build dependencies." echo " -o Specify deb directory. Default: debs/." exit 1 } -while getopts :a:hdDso: option; do +while getopts :a:hdio: option; do case "$option" in a) TERMUX_ARCH="$OPTARG";; d) TERMUX_DEBUG='-d';; + i) TERMUX_INSTALL_DEPS='-i';; o) TERMUX_DEBDIR="$(realpath -m $OPTARG)";; h) _show_usage;; esac @@ -63,8 +66,8 @@ for package_path in `cat $BUILDORDER_FILE`; do echo -n "Building $package... " BUILD_START=`date "+%s"` - bash -x $BUILDSCRIPT -a $TERMUX_ARCH -s \ - $TERMUX_DEBUG ${TERMUX_DEBDIR+-o $TERMUX_DEBDIR} $package \ + bash -x $BUILDSCRIPT -a $TERMUX_ARCH -s $TERMUX_DEBUG \ + ${TERMUX_DEBDIR+-o $TERMUX_DEBDIR} $TERMUX_INSTALL_DEPS $package \ > $BUILDALL_DIR/${package}.out 2> $BUILDALL_DIR/${package}.err BUILD_END=`date "+%s"` BUILD_SECONDS=$(( $BUILD_END - $BUILD_START )) From fe2bcd8f52f6abefd3026e4084f1de6eb97c53c7 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Tue, 1 Jan 2019 21:54:05 +0100 Subject: [PATCH 27/99] build-all.sh: don't set -s if -i is set --- build-all.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build-all.sh b/build-all.sh index e5fd04943..0b2cf76ae 100755 --- a/build-all.sh +++ b/build-all.sh @@ -8,7 +8,8 @@ test -f $HOME/.termuxrc && . $HOME/.termuxrc : ${TERMUX_TOPDIR:="$HOME/.termux-build"} : ${TERMUX_ARCH:="aarch64"} : ${TERMUX_DEBUG:=""} -: ${TERMUX_INSTALL_DEPS:=""} +: ${TERMUX_INSTALL_DEPS:="-s"} +# Set TERMUX_INSTALL_DEPS to -s unless set to -i _show_usage () { echo "Usage: ./build-all.sh [-a ARCH] [-d] [-i] [-o DIR]" @@ -66,7 +67,7 @@ for package_path in `cat $BUILDORDER_FILE`; do echo -n "Building $package... " BUILD_START=`date "+%s"` - bash -x $BUILDSCRIPT -a $TERMUX_ARCH -s $TERMUX_DEBUG \ + bash -x $BUILDSCRIPT -a $TERMUX_ARCH $TERMUX_DEBUG \ ${TERMUX_DEBDIR+-o $TERMUX_DEBDIR} $TERMUX_INSTALL_DEPS $package \ > $BUILDALL_DIR/${package}.out 2> $BUILDALL_DIR/${package}.err BUILD_END=`date "+%s"` From 4f6eab13f25d71a55a3c5ad81a2a37833f35e636 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Tue, 1 Jan 2019 21:56:42 +0100 Subject: [PATCH 28/99] build-package.sh: mv download of InRelease and Packages to new function termux_step_get_repo_files --- build-package.sh | 53 +++++++++++++++++++++++++++--------------------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/build-package.sh b/build-package.sh index 16845439b..ae59832a2 100755 --- a/build-package.sh +++ b/build-package.sh @@ -471,25 +471,11 @@ termux_download_deb() { return 0 } -# Source the package build script and start building. No to be overridden by packages. -termux_step_start_build() { - # shellcheck source=/dev/null - source "$TERMUX_PKG_BUILDER_SCRIPT" - - TERMUX_STANDALONE_TOOLCHAIN="$TERMUX_COMMON_CACHEDIR/${TERMUX_NDK_VERSION}-${TERMUX_ARCH}-${TERMUX_PKG_API_LEVEL}" - # Bump the below version if a change is made in toolchain setup to ensure - # that everyone gets an updated toolchain: - TERMUX_STANDALONE_TOOLCHAIN+="-v4" - - if [ -n "${TERMUX_PKG_BLACKLISTED_ARCHES:=""}" ] && [ "$TERMUX_PKG_BLACKLISTED_ARCHES" != "${TERMUX_PKG_BLACKLISTED_ARCHES/$TERMUX_ARCH/}" ]; then - echo "Skipping building $TERMUX_PKG_NAME for arch $TERMUX_ARCH" - exit 0 - fi - - local TERMUX_ALL_DEPS=$(./scripts/buildorder.py "$TERMUX_PKG_BUILDER_DIR") +# Script to download InRelease, verify it's signature and then download Packages.xz by hash +termux_step_get_repo_files() { if [ "$TERMUX_SKIP_DEPCHECK" = false ] && [ "$TERMUX_INSTALL_DEPS" = true ]; then # Remove all previously extracted/built files from $TERMUX_PREFIX: - rm -r $TERMUX_PREFIX + rm -rf $TERMUX_PREFIX rm -f /data/data/.built-packages/* # Ensure folders present (but not $TERMUX_PKG_SRCDIR, it will be created in build) mkdir -p "$TERMUX_COMMON_CACHEDIR" \ @@ -523,13 +509,32 @@ termux_step_start_build() { gpg --verify InRelease ) for arch in all $TERMUX_ARCH; do - local packages_hash=$(./scripts/get_hash_from_file.py ${TERMUX_COMMON_CACHEDIR}/InRelease $arch) - termux_download "$TERMUX_REPO_URL/$TERMUX_REPO_DISTRIBUTION/$TERMUX_REPO_COMPONENT/binary-$arch/Packages.xz" \ - "${TERMUX_COMMON_CACHEDIR}-$arch/Packages.xz" \ - $packages_hash - xz -df "${TERMUX_COMMON_CACHEDIR}-$arch/Packages.xz" + local packages_hash=$(./scripts/get_hash_from_file.py ${TERMUX_COMMON_CACHEDIR}/InRelease $arch) + termux_download "$TERMUX_REPO_URL/$TERMUX_REPO_DISTRIBUTION/$TERMUX_REPO_COMPONENT/binary-$arch/Packages.xz" \ + "${TERMUX_COMMON_CACHEDIR}-$arch/Packages.xz" \ + $packages_hash + xz -df "${TERMUX_COMMON_CACHEDIR}-$arch/Packages.xz" done + fi +} + +# Source the package build script and start building. No to be overridden by packages. +termux_step_start_build() { + # shellcheck source=/dev/null + source "$TERMUX_PKG_BUILDER_SCRIPT" + + TERMUX_STANDALONE_TOOLCHAIN="$TERMUX_COMMON_CACHEDIR/${TERMUX_NDK_VERSION}-${TERMUX_ARCH}-${TERMUX_PKG_API_LEVEL}" + # Bump the below version if a change is made in toolchain setup to ensure + # that everyone gets an updated toolchain: + TERMUX_STANDALONE_TOOLCHAIN+="-v4" + if [ -n "${TERMUX_PKG_BLACKLISTED_ARCHES:=""}" ] && [ "$TERMUX_PKG_BLACKLISTED_ARCHES" != "${TERMUX_PKG_BLACKLISTED_ARCHES/$TERMUX_ARCH/}" ]; then + echo "Skipping building $TERMUX_PKG_NAME for arch $TERMUX_ARCH" + exit 0 + fi + + local TERMUX_ALL_DEPS=$(./scripts/buildorder.py "$TERMUX_PKG_BUILDER_DIR") + if [ "$TERMUX_SKIP_DEPCHECK" = false ] && [ "$TERMUX_INSTALL_DEPS" = true ]; then # Download dependencies local pkg dep_arch dep_version deb_file for pkg in $TERMUX_ALL_DEPS; do @@ -1423,8 +1428,9 @@ termux_step_create_debfile() { termux_step_compare_debs() { if [ "${TERMUX_INSTALL_DEPS}" = true ]; then local arch version - if [ ! "$TERMUX_QUIET_BUILD" = true ]; then echo "COMPARING PACKAGES"; fi cd ${TERMUX_SCRIPTDIR} + if [ ! "$TERMUX_QUIET_BUILD" = true ]; then echo "COMPARING PACKAGES"; fi + termux_download_deb $(basename $TERMUX_PKG_BUILDER_DIR) $TERMUX_ARCH $TERMUX_PKG_FULLVERSION deb_file=${TERMUX_PKG_NAME}_${TERMUX_PKG_FULLVERSION}_${TERMUX_ARCH}.deb @@ -1446,6 +1452,7 @@ termux_step_finish_build() { termux_step_handle_arguments "$@" termux_step_setup_variables termux_step_handle_buildarch +termux_step_get_repo_files termux_step_start_build termux_step_extract_package cd "$TERMUX_PKG_SRCDIR" From 09afc5aca9437139ba1446c768bd26961934d03f Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sun, 20 Jan 2019 17:19:07 +0100 Subject: [PATCH 29/99] build-package.sh: fix termux_extract_dep_info --- build-package.sh | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/build-package.sh b/build-package.sh index ae59832a2..8e89b0099 100755 --- a/build-package.sh +++ b/build-package.sh @@ -435,18 +435,35 @@ termux_step_handle_buildarch() { # Function to get TERMUX_PKG_VERSION from build.sh termux_extract_dep_info() { package=$1 + if [ ! -d packages/$package ] && [ packages/${package} == packages/${package/-dev/} ]; then + # We are probably dealing with a subpackage + TERMUX_ARCH=$( + # set TERMUX_SUBPKG_PLATFORM_INDEPENDENT to mother package's value and override if needed + TERMUX_PKG_PLATFORM_INDEPENDENT="" + source $(dirname $(find packages/ -name "$package.subpackage.sh"))/build.sh + TERMUX_SUBPKG_PLATFORM_INDEPENDENT=$TERMUX_PKG_PLATFORM_INDEPENDENT + source $(find packages/ -name "$package.subpackage.sh") + if [ "$TERMUX_SUBPKG_PLATFORM_INDEPENDENT" = yes ]; then + echo all + else + echo $TERMUX_ARCH + fi + ) + + package=$(basename $(dirname $(find packages/ -name "$package.subpackage.sh"))) + elif [ ! packages/${package} == packages/${package/-dev/} ]; then + # dev package + package=${package/-dev/} + fi ( # Reset TERMUX_PKG_PLATFORM_INDEPENDENT and TERMUX_PKG_REVISION since these aren't # mandatory in a build.sh. Otherwise these will equal the main package's values for # deps that should have the default values TERMUX_PKG_PLATFORM_INDEPENDENT="" TERMUX_PKG_REVISION="0" - - source $package/build.sh - if [ "$TERMUX_PKG_PLATFORM_INDEPENDENT" = yes ]; then - TERMUX_ARCH=all - fi - if [ ! "$TERMUX_PKG_REVISION" = 0 ]; then + source packages/$package/build.sh + if [ "$TERMUX_PKG_PLATFORM_INDEPENDENT" = yes ]; then TERMUX_ARCH=all; fi + if [ "$TERMUX_PKG_REVISION" != "0" ] || [ "$TERMUX_PKG_VERSION" != "${TERMUX_PKG_VERSION/-/}" ]; then TERMUX_PKG_VERSION+="-$TERMUX_PKG_REVISION" fi echo ${TERMUX_ARCH} ${TERMUX_PKG_VERSION} From 3d760868d0e58aba00b8cc9e8da54b0ff861f1b6 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sun, 20 Jan 2019 17:51:21 +0100 Subject: [PATCH 30/99] build-package.sh: return 1 if hashsum doesn't exist for package When downloading depencencies --- build-package.sh | 16 +++++++--------- scripts/get_hash_from_file.py | 14 +++++++++----- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/build-package.sh b/build-package.sh index 8e89b0099..4a088e4bb 100755 --- a/build-package.sh +++ b/build-package.sh @@ -475,17 +475,15 @@ termux_download_deb() { local package_arch=$2 local version=$3 local deb_file=${package}_${version}_${package_arch}.deb - local pkg_hash=$(./scripts/get_hash_from_file.py ${TERMUX_COMMON_CACHEDIR}-${package_arch}/Packages $package) - + local pkg_hash=$(./scripts/get_hash_from_file.py ${TERMUX_COMMON_CACHEDIR}-${package_arch}/Packages $package $version) if [ "$pkg_hash" = "" ]; then - # No hash found for $package return 1 + else + termux_download $TERMUX_REPO_URL/$TERMUX_REPO_DISTRIBUTION/$TERMUX_REPO_COMPONENT/binary-${package_arch}/${deb_file} \ + $TERMUX_COMMON_CACHEDIR-$package_arch/${deb_file} \ + $pkg_hash + return 0 fi - - termux_download $TERMUX_REPO_URL/$TERMUX_REPO_DISTRIBUTION/$TERMUX_REPO_COMPONENT/binary-${package_arch}/${deb_file} \ - $TERMUX_COMMON_CACHEDIR-$package_arch/${deb_file} \ - $pkg_hash - return 0 } # Script to download InRelease, verify it's signature and then download Packages.xz by hash @@ -522,7 +520,7 @@ termux_step_get_repo_files() { curl --fail -LO "$TERMUX_REPO_URL/$TERMUX_REPO_DISTRIBUTION/InRelease" \ || termux_error_exit "Download of $TERMUX_REPO_URL/$TERMUX_REPO_DISTRIBUTION/InRelease failed" # Import Fornwalls key: - gpg --recv $TERMUX_REPO_SIGNING_KEY + gpg -k $TERMUX_REPO_SIGNING_KEY 2>/dev/null || gpg --recv $TERMUX_REPO_SIGNING_KEY gpg --verify InRelease ) for arch in all $TERMUX_ARCH; do diff --git a/scripts/get_hash_from_file.py b/scripts/get_hash_from_file.py index 4d355832c..b34276c7a 100755 --- a/scripts/get_hash_from_file.py +++ b/scripts/get_hash_from_file.py @@ -2,16 +2,20 @@ import os, sys -def get_pkg_hash_from_Packages(Packages_file, package, hash="SHA256"): +def get_pkg_hash_from_Packages(Packages_file, package, version, hash="SHA256"): with open(Packages_file, 'r') as Packages: package_list = Packages.read().split('\n\n') for pkg in package_list: if pkg.split('\n')[0] == "Package: "+package: for line in pkg.split('\n'): - if line.startswith(hash): + if line.startswith('Version:'): + if line != 'Version: '+version: + # Seems the repo contains the wrong version, or several versions + # We can't use this one so continue looking + break + elif line.startswith(hash): print(line.split(" ")[1]) break - break def get_Packages_hash_from_InRelease(InRelease_file, arch, hash="SHA256"): string_to_found = 'binary-'+arch+'/Packages.xz' @@ -27,10 +31,10 @@ def get_Packages_hash_from_InRelease(InRelease_file, arch, hash="SHA256"): if __name__ == '__main__': if len(sys.argv) < 2: - sys.exit('Too few arguments, I need the path to a Packages file and a package name. Exiting') + sys.exit('Too few arguments, I need the path to a Packages file, a package name and a version, or an InRelease file and an architecture. Exiting') if sys.argv[1].endswith('Packages'): - get_pkg_hash_from_Packages(sys.argv[1], sys.argv[2]) + get_pkg_hash_from_Packages(sys.argv[1], sys.argv[2], sys.argv[3]) elif sys.argv[1].endswith('InRelease'): get_Packages_hash_from_InRelease(sys.argv[1], sys.argv[2]) else: From 6c07b40796dac3d2505f554a07313dea52a72470 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sun, 20 Jan 2019 17:52:47 +0100 Subject: [PATCH 31/99] build-package.sh: add sha256sum's for bootstrap zips for all arches --- build-package.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/build-package.sh b/build-package.sh index 4a088e4bb..f746447b5 100755 --- a/build-package.sh +++ b/build-package.sh @@ -504,9 +504,18 @@ termux_step_get_repo_files() { "$TERMUX_PKG_MASSAGEDIR" \ $TERMUX_PREFIX/{bin,etc,lib,libexec,share,tmp,include} # Setup bootstrap + if [ $TERMUX_ARCH == aarch64 ]; then + local bootstrap_sha256=9f43bfdd23fbc6ae02f1f5a44b8e064374db8291ffe32658e2724d3f3f419981 + elif [ $TERMUX_ARCH == i686 ]; then + local bootstrap_sha256=4f30d49f8bdb9a294d6e4e0dea6b5cb7b5cd74163531a04659d043b49dc08737 + elif [ $TERMUX_ARCH == arm ]; then + local bootstrap_sha256=2be38e07f93ca3f960ccb5554e67d0860e921b283ccba0aab5527a53b9e97b10 + elif [ $TERMUX_ARCH == x86_64 ]; then + local bootstrap_sha256=71b334dd3901b8f9c2a8905208912419963700f93b5b7cd68213010d05a662a7 + fi termux_download https://termux.net/bootstrap/bootstrap-${TERMUX_ARCH}.zip \ ${TERMUX_COMMON_CACHEDIR}/bootstrap-${TERMUX_ARCH}.zip \ - 9f43bfdd23fbc6ae02f1f5a44b8e064374db8291ffe32658e2724d3f3f419981 + $bootstrap_sha256 unzip -qo ${TERMUX_COMMON_CACHEDIR}/bootstrap-${TERMUX_ARCH}.zip -d $TERMUX_PREFIX ( cd $TERMUX_PREFIX From 6f9019df526d42a403f3a5664802e91dfa913abb Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sun, 20 Jan 2019 17:56:05 +0100 Subject: [PATCH 32/99] build-package.sh: fix download loop of dependencies when -i building Only download $TERMUX_PKG_DEPENDS and $TERMUX_PKG_BUILD_DEPENDS (and their -dev packages, if they exist), not their dependencies. --- build-package.sh | 56 +++++++++++++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 22 deletions(-) diff --git a/build-package.sh b/build-package.sh index f746447b5..c634bc035 100755 --- a/build-package.sh +++ b/build-package.sh @@ -557,33 +557,45 @@ termux_step_start_build() { exit 0 fi - local TERMUX_ALL_DEPS=$(./scripts/buildorder.py "$TERMUX_PKG_BUILDER_DIR") + local TERMUX_ALL_DEPS=$(./scripts/buildorder.py "$TERMUX_PKG_BUILDER_DIR" | sed 's%packages/%%g') if [ "$TERMUX_SKIP_DEPCHECK" = false ] && [ "$TERMUX_INSTALL_DEPS" = true ]; then # Download dependencies - local pkg dep_arch dep_version deb_file - for pkg in $TERMUX_ALL_DEPS; do + local pkg dep_arch dep_version deb_file _PKG_DEPENDS _PKG_BUILD_DEPENDS + # remove (>= 1.0) and similar version tags with sed: + _PKG_DEPENDS=$(echo ${TERMUX_PKG_DEPENDS//,/ } | sed "s/[(][^)]*[)]//g") + _PKG_BUILD_DEPENDS=${TERMUX_PKG_BUILD_DEPENDS//,/ } + for pkg in $_PKG_DEPENDS $_PKG_BUILD_DEPENDS; do + # llvm doesn't build if ndk-sysroot is installed: + if [ "$pkg" = "ndk-sysroot" ]; then continue; fi read dep_arch dep_version <<< $(termux_extract_dep_info "$pkg") + if [ ! "$TERMUX_QUIET_BUILD" = true ]; then - echo "Downloading dependency $(basename $pkg)@$dep_version if necessary..." + echo "Downloading dependency $pkg@$dep_version if necessary..." + fi + if ! termux_download_deb $pkg $dep_arch $dep_version; then + echo "Download of $pkg@$dep_version from $TERMUX_REPO_URL failed, building instead" + ./build-package.sh -a $TERMUX_ARCH -i "$pkg" + continue + else + if [ ! "$TERMUX_QUIET_BUILD" = true ]; then echo "Extracting $pkg..."; fi + ( + cd $TERMUX_COMMON_CACHEDIR-$dep_arch + ar x ${pkg}_${dep_version}_${dep_arch}.deb data.tar.xz + tar -xf data.tar.xz --no-overwrite-dir -C / + ) + fi + + if termux_download_deb $pkg-dev $dep_arch $dep_version; then + ( + cd $TERMUX_COMMON_CACHEDIR-$dep_arch + ar x $pkg-dev_${dep_version}_${dep_arch}.deb data.tar.xz + tar xf data.tar.xz --no-overwrite-dir -C / + ) + else + echo "Download of $pkg-dev@$dep_version from $TERMUX_REPO_URL failed" fi - termux_download_deb $(basename $pkg) $dep_arch $dep_version \ - || ( echo "Download of $(basename $pkg)@$dep_version from $TERMUX_REPO_URL failed, building instead" \ - && ./build-package.sh -a $TERMUX_ARCH -s "$pkg" \ - && continue ) - local deb_file=$(basename $pkg)_${dep_version}_${dep_arch}.deb - if [ ! "$TERMUX_QUIET_BUILD" = true ]; then echo "Extracting $(basename $pkg)..."; fi - ( - cd $TERMUX_COMMON_CACHEDIR-$dep_arch - ar x ${deb_file} data.tar.xz && tar xf data.tar.xz --no-overwrite-dir -C / - ) - - termux_download_deb $(basename $pkg)-dev $dep_arch $dep_version && \ - ( - cd $TERMUX_COMMON_CACHEDIR-$dep_arch - ar x $(basename $pkg)-dev_${dep_version}_${dep_arch}.deb data.tar.xz - tar xf data.tar.xz --no-overwrite-dir -C / - ) || echo "Download of $(basename $pkg)-dev@$dep_version from $TERMUX_REPO_URL failed" - echo "$dep_version" > "/data/data/.built-packages/$(basename $pkg)" + mkdir -p /data/data/.built-packages + echo "$dep_version" > "/data/data/.built-packages/$pkg" done elif [ "$TERMUX_SKIP_DEPCHECK" = false ] && [ "$TERMUX_INSTALL_DEPS" = false ]; then # Build dependencies From 9e3258d710d9a8895dac2873afb768b18606b1aa Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sun, 20 Jan 2019 17:58:36 +0100 Subject: [PATCH 33/99] build-package.sh: fix termux_step_compare_debs Catch error if termux_download_deb fails --- build-package.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/build-package.sh b/build-package.sh index c634bc035..c3b4b9dc7 100755 --- a/build-package.sh +++ b/build-package.sh @@ -1463,16 +1463,17 @@ termux_step_create_debfile() { termux_step_compare_debs() { if [ "${TERMUX_INSTALL_DEPS}" = true ]; then - local arch version cd ${TERMUX_SCRIPTDIR} if [ ! "$TERMUX_QUIET_BUILD" = true ]; then echo "COMPARING PACKAGES"; fi - termux_download_deb $(basename $TERMUX_PKG_BUILDER_DIR) $TERMUX_ARCH $TERMUX_PKG_FULLVERSION - deb_file=${TERMUX_PKG_NAME}_${TERMUX_PKG_FULLVERSION}_${TERMUX_ARCH}.deb + termux_download_deb $TERMUX_PKG_NAME $TERMUX_ARCH $TERMUX_PKG_FULLVERSION \ + && ( + deb_file=${TERMUX_PKG_NAME}_${TERMUX_PKG_FULLVERSION}_${TERMUX_ARCH}.deb - # `|| true` to prevent debdiff's exit code from stopping build - debdiff $TERMUX_DEBDIR/$deb_file $TERMUX_COMMON_CACHEDIR-$TERMUX_ARCH/$deb_file || true - if [ ! "$TERMUX_QUIET_BUILD" = true ]; then echo "DONE COMPARING PACKAGES"; fi + # `|| true` to prevent debdiff's exit code from stopping build + debdiff $TERMUX_DEBDIR/$deb_file $TERMUX_COMMON_CACHEDIR-$TERMUX_ARCH/$deb_file || true + if [ ! "$TERMUX_QUIET_BUILD" = true ]; then echo "DONE COMPARING PACKAGES"; fi + ) || echo "Download of ${TERMUX_PKG_NAME}@${TERMUX_PKG_FULLVERSION} failed, not comparing debs" fi } From 5611ad0a18a3d3e5a901cd835e778d48a1934d7c Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sun, 20 Jan 2019 18:00:37 +0100 Subject: [PATCH 34/99] build-package.sh: Minor indentation fix --- build-package.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build-package.sh b/build-package.sh index c3b4b9dc7..7c17a1dbf 100755 --- a/build-package.sh +++ b/build-package.sh @@ -410,7 +410,7 @@ termux_step_handle_buildarch() { local TERMUX_PREVIOUS_ARCH TERMUX_PREVIOUS_ARCH=$(cat $TERMUX_ARCH_FILE) if [ "$TERMUX_PREVIOUS_ARCH" != "$TERMUX_ARCH" ]; then - local TERMUX_DATA_BACKUPDIRS=$TERMUX_TOPDIR/_databackups + local TERMUX_DATA_BACKUPDIRS=$TERMUX_TOPDIR/_databackups mkdir -p "$TERMUX_DATA_BACKUPDIRS" local TERMUX_DATA_PREVIOUS_BACKUPDIR=$TERMUX_DATA_BACKUPDIRS/$TERMUX_PREVIOUS_ARCH local TERMUX_DATA_CURRENT_BACKUPDIR=$TERMUX_DATA_BACKUPDIRS/$TERMUX_ARCH @@ -642,7 +642,7 @@ termux_step_start_build() { # Ensure folders present (but not $TERMUX_PKG_SRCDIR, it will be created in build) mkdir -p "$TERMUX_COMMON_CACHEDIR" \ - "$TERMUX_DEBDIR" \ + "$TERMUX_DEBDIR" \ "$TERMUX_PKG_BUILDDIR" \ "$TERMUX_PKG_PACKAGEDIR" \ "$TERMUX_PKG_TMPDIR" \ From 195980e27d998423ad368977f0e2e3d03217c477 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sun, 20 Jan 2019 18:20:56 +0100 Subject: [PATCH 35/99] build-package.sh: fix bug in the case where package name contains (but not ends with) -dev As for icu-devtools --- build-package.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build-package.sh b/build-package.sh index 7c17a1dbf..311d151dc 100755 --- a/build-package.sh +++ b/build-package.sh @@ -435,8 +435,8 @@ termux_step_handle_buildarch() { # Function to get TERMUX_PKG_VERSION from build.sh termux_extract_dep_info() { package=$1 - if [ ! -d packages/$package ] && [ packages/${package} == packages/${package/-dev/} ]; then - # We are probably dealing with a subpackage + if [ ! -d packages/$package ] && [ -f packages/*/${package}.subpackage.sh ]; then + # We are dealing with a subpackage TERMUX_ARCH=$( # set TERMUX_SUBPKG_PLATFORM_INDEPENDENT to mother package's value and override if needed TERMUX_PKG_PLATFORM_INDEPENDENT="" @@ -451,7 +451,7 @@ termux_extract_dep_info() { ) package=$(basename $(dirname $(find packages/ -name "$package.subpackage.sh"))) - elif [ ! packages/${package} == packages/${package/-dev/} ]; then + elif [ "${package/-dev/}-dev" == "${package}" ]; then # dev package package=${package/-dev/} fi From 2157d60f13bc49fcdb0c8f11e48e9aabaa9032f6 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sun, 20 Jan 2019 18:32:49 +0100 Subject: [PATCH 36/99] build-package.sh: improve -i description --- build-package.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-package.sh b/build-package.sh index 311d151dc..63199b1f2 100755 --- a/build-package.sh +++ b/build-package.sh @@ -254,7 +254,7 @@ termux_step_handle_arguments() { echo " -d Build with debug symbols." echo " -D Build a disabled package in disabled-packages/." echo " -f Force build even if package has already been built." - echo " -i Build dependencies." + echo " -i Download and extract dependencies instead of building them." echo " -q Quiet build." echo " -s Skip dependency check." echo " -o Specify deb directory. Default: debs/." From 5d81472249f87511b715591007fbad625d5e020b Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Fri, 25 Jan 2019 22:09:57 +0100 Subject: [PATCH 37/99] build-package.sh: fix `-a all -i` builds Also use `-s -i` for dependencies that have to be built --- build-package.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build-package.sh b/build-package.sh index 63199b1f2..c9e994193 100755 --- a/build-package.sh +++ b/build-package.sh @@ -282,7 +282,7 @@ termux_step_handle_arguments() { # Handle 'all' arch: if [ -n "${TERMUX_ARCH+x}" ] && [ "${TERMUX_ARCH}" = 'all' ]; then for arch in 'aarch64' 'arm' 'i686' 'x86_64'; do - ./build-package.sh ${TERMUX_FORCE_BUILD+-f} -a $arch \ + ./build-package.sh ${TERMUX_FORCE_BUILD+-f} -a $arch ${TERMUX_INSTALL_DEPS+-i} \ ${TERMUX_DEBUG+-d} ${TERMUX_DEBDIR+-o $TERMUX_DEBDIR} "$1" done exit @@ -574,7 +574,7 @@ termux_step_start_build() { fi if ! termux_download_deb $pkg $dep_arch $dep_version; then echo "Download of $pkg@$dep_version from $TERMUX_REPO_URL failed, building instead" - ./build-package.sh -a $TERMUX_ARCH -i "$pkg" + ./build-package.sh -a $TERMUX_ARCH -i -s "$pkg" continue else if [ ! "$TERMUX_QUIET_BUILD" = true ]; then echo "Extracting $pkg..."; fi From 61d8c199452008b757b3f4e15b600e4e9b63cfaa Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Fri, 25 Jan 2019 22:29:51 +0100 Subject: [PATCH 38/99] Revert "Update build.sh" This reverts commit eb85a5b4facae36817e5d23d15b79da0036ca7be. --- packages/php/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/php/build.sh b/packages/php/build.sh index fedea2f52..8bb400087 100644 --- a/packages/php/build.sh +++ b/packages/php/build.sh @@ -8,7 +8,7 @@ TERMUX_PKG_SRCURL=https://secure.php.net/distributions/php-${TERMUX_PKG_VERSION} TERMUX_PKG_HOSTBUILD=true # Build the native php without xml support as we only need phar: TERMUX_PKG_EXTRA_HOSTBUILD_CONFIGURE_ARGS="--disable-libxml --disable-dom --disable-simplexml --disable-xml --disable-xmlreader --disable-xmlwriter --without-pear" -TERMUX_PKG_DEPENDS="libandroid-glob, libxml2, liblzma, openssl, pcre, libbz2, libcrypt, libcurl, libgd, readline, freetype, apr, apr-util, openssl, libpng, libjpeg-turbo" +TERMUX_PKG_DEPENDS="libandroid-glob, libxml2, liblzma, openssl, pcre, libbz2, libcrypt, libcurl, libgd, readline, freetype" # mysql modules were initially shared libs TERMUX_PKG_CONFLICTS="php-mysql" TERMUX_PKG_REPLACES="php-mysql" From 31a68c5bb2dfe5d21512176202f8acee92d3d6d7 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Fri, 25 Jan 2019 22:30:08 +0100 Subject: [PATCH 39/99] Revert "Update build.sh" This reverts commit f55b366ed2a8d43885f8e5fb9d67be7ec11c07e2. --- packages/php/build.sh | 8 -------- 1 file changed, 8 deletions(-) diff --git a/packages/php/build.sh b/packages/php/build.sh index 8bb400087..9dde61e2e 100644 --- a/packages/php/build.sh +++ b/packages/php/build.sh @@ -48,14 +48,6 @@ ac_cv_func_res_nsearch=no " termux_step_pre_configure () { - # Replace chrooted perl with system perl - sed -i 's/\/data\/data\/com\.termux\/files\/usr\/bin\/perl/\/usr\/bin\/perl/' $TERMUX_PREFIX/bin/apxs - sed -i 's/\/data\/data\/com\.termux\/files\/usr\/bin\/perl/\/usr\/bin\/perl/' $TERMUX_PREFIX/bin/curl-config - sed -i 's/\/home\/fornwall\/.termux-build\/apr\/tmp/\/home\/builder\/.termux-build\/php\/build/' $TERMUX_PREFIX/bin/apr-1-config - rm $TERMUX_PREFIX/bin/pg_config - - #because the new mariadb hides away all these includes inside server subdir - CFLAGS+=" -I$TERMUX_PREFIX/include/mysql/server -I$TERMUX_PREFIX/include/mysql" LDFLAGS+=" -landroid-glob -llog" export PATH=$PATH:$TERMUX_PKG_HOSTBUILD_DIR/sapi/cli/ From 362758ae87189f6ccd94b2529c793f892b10006b Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sat, 26 Jan 2019 01:04:30 +0100 Subject: [PATCH 40/99] Revert "Update build.sh" This reverts commit 865db6dd76f7349fa562daa14fc998ac25099c8a. --- packages/apache2/build.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/packages/apache2/build.sh b/packages/apache2/build.sh index fbfe7e037..d9a0bc36e 100644 --- a/packages/apache2/build.sh +++ b/packages/apache2/build.sh @@ -71,11 +71,6 @@ TERMUX_PKG_INCLUDE_IN_DEVPACKAGE="share/apache2/build" TERMUX_PKG_EXTRA_MAKE_ARGS="-s" termux_step_pre_configure () { - # address interpreter issues - sed -i 's/\/home\/fornwall\/.termux-build\/apr\/tmp\/libtool/\/usr\/bin\/libtool/' ${TERMUX_PREFIX}/bin/apr-1-config - sed -i 's/\/data\/data\/com.termux\/files\/usr\/bin\/bash/\/bin\/bash/' ${TERMUX_PREFIX}/share/apr-1/build/libtool - sed -i 's/\/home\/fornwall/\/home\/builder/' ${TERMUX_PREFIX}/share/apr-1/build/libtool - # remove old files rm -rf "$TERMUX_PREFIX"/{libexec,share,etc}/apache2 rm -rf "$TERMUX_PREFIX"/lib/cgi-bin From 720442ba67c9eed9b2c16131b74392924e90af76 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sat, 26 Jan 2019 15:43:14 +0100 Subject: [PATCH 41/99] Add gnupg to docker image --- scripts/setup-ubuntu.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/setup-ubuntu.sh b/scripts/setup-ubuntu.sh index ae4b111ea..1c8102538 100755 --- a/scripts/setup-ubuntu.sh +++ b/scripts/setup-ubuntu.sh @@ -37,6 +37,7 @@ PACKAGES+=" libexpat1-dev" # Needed by ghostscript PACKAGES+=" libjpeg-dev" # Needed by ghostscript PACKAGES+=" gawk" # Needed by apr-util PACKAGES+=" libssl-dev" # Needed to build Rust +PACKAGES+=" gnupg" # Needed to verify downloaded .debs sudo DEBIAN_FRONTEND=noninteractive \ apt-get install -yq --no-install-recommends $PACKAGES From 0a8eaeb77f53f30e9faad64ae8a26305cdd5348f Mon Sep 17 00:00:00 2001 From: Leonid Plyushch Date: Sat, 26 Jan 2019 16:59:34 +0200 Subject: [PATCH 42/99] build-package.sh: use keyserver pool when receive keys Now gpg key retrieving is more reliable. --- build-package.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-package.sh b/build-package.sh index c9e994193..3d39277d5 100755 --- a/build-package.sh +++ b/build-package.sh @@ -529,7 +529,7 @@ termux_step_get_repo_files() { curl --fail -LO "$TERMUX_REPO_URL/$TERMUX_REPO_DISTRIBUTION/InRelease" \ || termux_error_exit "Download of $TERMUX_REPO_URL/$TERMUX_REPO_DISTRIBUTION/InRelease failed" # Import Fornwalls key: - gpg -k $TERMUX_REPO_SIGNING_KEY 2>/dev/null || gpg --recv $TERMUX_REPO_SIGNING_KEY + gpg -k $TERMUX_REPO_SIGNING_KEY 2>/dev/null || gpg --keyserver pool.sks-keyservers.net --recv $TERMUX_REPO_SIGNING_KEY gpg --verify InRelease ) for arch in all $TERMUX_ARCH; do From b978ccd7476338a85fe52016a4e7c38ca35f0eff Mon Sep 17 00:00:00 2001 From: Leonid Plyushch Date: Sat, 26 Jan 2019 17:02:21 +0200 Subject: [PATCH 43/99] setup-ubuntu.sh: install "devscripts" Makes utility "debdiff" available. --- scripts/setup-ubuntu.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/setup-ubuntu.sh b/scripts/setup-ubuntu.sh index 1c8102538..160c92b75 100755 --- a/scripts/setup-ubuntu.sh +++ b/scripts/setup-ubuntu.sh @@ -38,6 +38,7 @@ PACKAGES+=" libjpeg-dev" # Needed by ghostscript PACKAGES+=" gawk" # Needed by apr-util PACKAGES+=" libssl-dev" # Needed to build Rust PACKAGES+=" gnupg" # Needed to verify downloaded .debs +PACKAGES+=" devscripts" # Provides utility "debdiff". sudo DEBIAN_FRONTEND=noninteractive \ apt-get install -yq --no-install-recommends $PACKAGES From 81f1b426296acba94a785bb5208ab5abcb588206 Mon Sep 17 00:00:00 2001 From: Leonid Plyushch Date: Sat, 26 Jan 2019 17:44:49 +0200 Subject: [PATCH 44/99] remove redundant things from CI configuration --- .travis.yml | 21 +++++---------------- Rakefile | 2 +- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/.travis.yml b/.travis.yml index d7501b49f..3a7737342 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,20 +4,9 @@ cache: bundler services: docker before_script: mkdir debs && chmod 777 debs env: - - TRAVIS_ARCH=aarch64 - - TRAVIS_ARCH=arm - - TRAVIS_ARCH=i686 - - TRAVIS_ARCH=x86_64 + matrix: + - ARGS="-i -a aarch64" + - ARGS="-i -a arm" + - ARGS="-i -a i686" + - ARGS="-i -a x86_64" script: bundle exec rake build["${ARGS}"] -jobs: - include: - - stage: Build aarch64 - env: ARGS="-q -a aarch64" - - env: ARGS="-q -i -a aarch64" - - stage: Build arm, i686, & x86_64 - env: ARGS="-q -a arm" - - env: ARGS="-q -a i686" - - env: ARGS="-q -a x86_64" - - env: ARGS="-q -i -a arm" - - env: ARGS="-q -i -a i686" - - env: ARGS="-q -i -a x86_64" diff --git a/Rakefile b/Rakefile index 08764a36b..408d0ece3 100644 --- a/Rakefile +++ b/Rakefile @@ -20,7 +20,7 @@ task :build, [:options] do |t, args| puts "Building #{pkg}" begin # Start blocking build loop - PTY.spawn("./scripts/run-docker.sh ./build-package.sh #{args[:options]} -a $TRAVIS_ARCH #{pkg}") do |stdout, stdin, pid| + PTY.spawn("./scripts/run-docker.sh ./build-package.sh #{args[:options]} #{pkg}") do |stdout, stdin, pid| begin stdout.sync stdout.each { |line| print line } From e0ca532e62ca154ccf30af1f045c0cc1b21eccaf Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sat, 26 Jan 2019 16:37:25 +0000 Subject: [PATCH 45/99] Use apt's trusted.gpg for pkg verification --- build-package.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build-package.sh b/build-package.sh index 3d39277d5..7f5482ffb 100755 --- a/build-package.sh +++ b/build-package.sh @@ -524,12 +524,12 @@ termux_step_get_repo_files() { done/dev/null || gpg --keyserver pool.sks-keyservers.net --recv $TERMUX_REPO_SIGNING_KEY gpg --verify InRelease ) for arch in all $TERMUX_ARCH; do From 81be68dda6f9528ed845bedbf82e7bec7ca3dc7c Mon Sep 17 00:00:00 2001 From: Stephen Groat Date: Mon, 2 Oct 2017 07:54:53 -0700 Subject: [PATCH 46/99] Use apt for dependencies --- .travis.yml | 13 +++++++ Rakefile | 13 ++++--- build-package.sh | 89 ++++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 109 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 691d43524..6ac4908b7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,3 +8,16 @@ env: - TRAVIS_ARCH=arm - TRAVIS_ARCH=i686 - TRAVIS_ARCH=x86_64 +script: bundle exec rake build["${ARGS}"] +jobs: + include: + - stage: Build aarch64 + env: ARGS="-a aarch64" + - env: ARGS="-i -a aarch64" + - stage: Build arm, i686, & x86_64 + env: ARGS="-a arm" + - env: ARGS="-a i686" + - env: ARGS="-a x86_64" + - env: ARGS="-i -a arm" + - env: ARGS="-i -a i686" + - env: ARGS="-i -a x86_64" diff --git a/Rakefile b/Rakefile index bf41db863..08764a36b 100644 --- a/Rakefile +++ b/Rakefile @@ -3,7 +3,7 @@ require 'pty' task default: %w[build] -task :build do +task :build, [:options] do |t, args| repo = Rugged::Repository.new('.') commit = repo.head.target parent = commit.parents.first @@ -20,13 +20,18 @@ task :build do puts "Building #{pkg}" begin # Start blocking build loop - PTY.spawn("./scripts/run-docker.sh ./build-package.sh -a $TRAVIS_ARCH #{pkg}") do |stdout, stdin, pid| + PTY.spawn("./scripts/run-docker.sh ./build-package.sh #{args[:options]} -a $TRAVIS_ARCH #{pkg}") do |stdout, stdin, pid| begin + stdout.sync stdout.each { |line| print line } - rescue Errno::EIO + rescue Errno::EIO => e + puts e + ensure + ::Process.wait pid end end - rescue PTY::ChildExited + rescue PTY::ChildExited => e + puts e puts "Process exited" end # Exit if PTY return a non-zero code diff --git a/build-package.sh b/build-package.sh index 918cac84f..9ea4d67f9 100755 --- a/build-package.sh +++ b/build-package.sh @@ -248,12 +248,13 @@ termux_setup_cmake() { # First step is to handle command-line arguments. Not to be overridden by packages. termux_step_handle_arguments() { _show_usage () { - echo "Usage: ./build-package.sh [-a ARCH] [-d] [-D] [-f] [-q] [-s] [-o DIR] PACKAGE" + echo "Usage: ./build-package.sh [-a ARCH] [-d] [-D] [-f] [-i] [-q] [-s] [-o DIR] PACKAGE" echo "Build a package by creating a .deb file in the debs/ folder." echo " -a The architecture to build for: aarch64(default), arm, i686, x86_64 or all." echo " -d Build with debug symbols." echo " -D Build a disabled package in disabled-packages/." echo " -f Force build even if package has already been built." + echo " -i Install dependencies." echo " -q Quiet build." echo " -s Skip dependency check." echo " -o Specify deb directory. Default: debs/." @@ -266,6 +267,7 @@ termux_step_handle_arguments() { d) export TERMUX_DEBUG=true;; D) local TERMUX_IS_DISABLED=true;; f) TERMUX_FORCE_BUILD=true;; + i) export TERMUX_INSTALL_DEPS=true;; q) export TERMUX_QUIET_BUILD=true;; s) export TERMUX_SKIP_DEPCHECK=true;; o) TERMUX_DEBDIR="$(realpath -m $OPTARG)";; @@ -438,7 +440,82 @@ termux_step_start_build() { exit 0 fi - if [ -z "${TERMUX_SKIP_DEPCHECK:=""}" ]; then + if [ ! -z ${TERMUX_INSTALL_DEPS+x} ]; then + # Ensure folders present (but not $TERMUX_PKG_SRCDIR, it will be created in build) + mkdir -p "$TERMUX_COMMON_CACHEDIR" \ + "$TERMUX_DEBDIR" \ + "$TERMUX_PKG_BUILDDIR" \ + "$TERMUX_PKG_PACKAGEDIR" \ + "$TERMUX_PKG_TMPDIR" \ + "$TERMUX_PKG_CACHEDIR" \ + "$TERMUX_PKG_MASSAGEDIR" \ + $TERMUX_PREFIX/{bin,etc,lib,libexec,share,tmp,include} + # Setup bootstrap + termux_download https://termux.net/bootstrap/bootstrap-${TERMUX_ARCH}.zip \ + ${TERMUX_COMMON_CACHEDIR}/bootstrap-${TERMUX_ARCH}.zip + unzip ${TERMUX_COMMON_CACHEDIR}/bootstrap-${TERMUX_ARCH}.zip -d $TERMUX_PREFIX + + # TODO move this install to Dockerfile + sudo apt-get update && sudo apt-get -y dist-upgrade && sudo apt-get install -y libcap2-bin gawk tree strace + # set capabilities on dpkg + # Some packages built by uid 1001 (builder is 1000) + # Need capabilities for dpkg to set non-builder uid + sudo /sbin/setcap cap_chown,cap_fowner,cap_dac_override+eip /usr/bin/dpkg + sudo /sbin/setcap cap_chown,cap_fowner,cap_dac_override+eip /usr/bin/apt-get + # install packages that include subpackages + sudo sed -i -e 's/DPkg::Pre-Install-Pkgs/\/\/ DPkg::Pre-Install-Pkgs/' /etc/apt/apt.conf.d/* + sudo sed -i -e 's/DPkg::Post-Invoke/\/\/ DPkg::Post-Invoke/' /etc/apt/apt.conf.d/* + sudo sed -i -e 's/APT::Update::Post-Invoke/\/\/ APT::Update::Post-Invoke/' /etc/apt/apt.conf.d/* + TERMUX_DPKG=" \ + --force-architecture \ + --force-not-root \ + --force-configure-any \ + --force-confdef \ + --force-confold \ + --force-depends \ + --admindir=${TERMUX_PREFIX}/var/lib/dpkg" + export TERMUX_APT=" \ + -o APT::Get::Assume-Yes=true \ + -o APT::Get::ReInstall=true + -o APT::Architecture=${TERMUX_ARCH} \ + -o Dir::Etc=${TERMUX_PREFIX}/etc/apt/ \ + -o Dir::State=${TERMUX_PREFIX}/var/lib/apt \ + -o Dir::State::status=${TERMUX_PREFIX}/var/lib/dpkg/status \ + -o Dir::Cache=${TERMUX_PREFIX}/var/cache/apt \ + -o Dir::Log=${TERMUX_PREFIX}/var/log/apt" + for arg in ${TERMUX_DPKG}; do + TERMUX_APT+=" -o DPkg::Options::=${arg}" + done + export DEBCONF_FRONTEND=noninteractive + apt-get $TERMUX_APT clean && apt-get $TERMUX_APT update && apt-get $TERMUX_APT upgrade + # libandroid-support-dev is build-essential + apt-get $TERMUX_APT install libandroid-support-dev:any + sudo chown -R builder:builder /data + array=( TERMUX_PKG_DEPENDS TERMUX_PKG_BUILD_DEPENDS ) + for i in "${array[@]}"; do + while IFS=',' read -ra PKG; do + for p in "${PKG[@]}"; do + p="$(echo -e "${p}" | tr -d '[:space:]')" + apt-get $TERMUX_APT install "^${p}(-dev)?$":any + sudo chown -R builder:builder /data + done + done <<< "${!i}" + done + for subpkg in $TERMUX_PKG_BUILDER_DIR/*.subpackage.sh; do + test ! -f "$subpkg" && continue + local TERMUX_SUBPKG_DEPENDS="" + source $subpkg + while IFS=',' read -ra PKG; do + for p in "${PKG[@]}"; do + p="$(echo -e "${p}" | tr -d '[:space:]')" + apt-get $TERMUX_APT install "^${p}(-dev)?$":any + sudo chown -R builder:builder /data + done + done <<< $TERMUX_SUBPKG_DEPENDS + done + fi + + if [ -z "${TERMUX_SKIP_DEPCHECK:=""}" ] && [ -z ${TERMUX_INSTALL_DEPS+x} ]; then local p TERMUX_ALL_DEPS TERMUX_ALL_DEPS=$(./scripts/buildorder.py "$TERMUX_PKG_BUILDER_DIR") for p in $TERMUX_ALL_DEPS; do @@ -1315,6 +1392,13 @@ termux_step_create_debfile() { "$TERMUX_PKG_PACKAGEDIR/data.tar.xz" } +termux_step_reverse_depends() { + if [ ! -z ${TERMUX_INSTALL_DEPS+x} ]; then + # TODO build reverse depends with packages + apt-cache $TERMUX_APT rdepends $TERMUX_PKG_NAME + fi +} + # Finish the build. Not to be overridden by package scripts. termux_step_finish_build() { echo "termux - build of '$TERMUX_PKG_NAME' done" @@ -1355,4 +1439,5 @@ cd "$TERMUX_PKG_MASSAGEDIR/$TERMUX_PREFIX" termux_step_post_massage termux_step_create_datatar termux_step_create_debfile +termux_step_reverse_depends termux_step_finish_build From 0421842e7d7bc0ba61015c065adf427a0c051124 Mon Sep 17 00:00:00 2001 From: Stephen Date: Sun, 8 Oct 2017 13:52:35 -0700 Subject: [PATCH 47/99] Update .travis.yml --- .travis.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6ac4908b7..d7501b49f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,12 +12,12 @@ script: bundle exec rake build["${ARGS}"] jobs: include: - stage: Build aarch64 - env: ARGS="-a aarch64" - - env: ARGS="-i -a aarch64" + env: ARGS="-q -a aarch64" + - env: ARGS="-q -i -a aarch64" - stage: Build arm, i686, & x86_64 - env: ARGS="-a arm" - - env: ARGS="-a i686" - - env: ARGS="-a x86_64" - - env: ARGS="-i -a arm" - - env: ARGS="-i -a i686" - - env: ARGS="-i -a x86_64" + env: ARGS="-q -a arm" + - env: ARGS="-q -a i686" + - env: ARGS="-q -a x86_64" + - env: ARGS="-q -i -a arm" + - env: ARGS="-q -i -a i686" + - env: ARGS="-q -i -a x86_64" From 723ecc31849eb2ffdf895ab5cdb7669105af528f Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Fri, 27 Oct 2017 14:58:11 +0200 Subject: [PATCH 48/99] build-package: with -i -q, ignore recommended packages and suggestions (#3) --- build-package.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build-package.sh b/build-package.sh index 9ea4d67f9..03e8c4f02 100755 --- a/build-package.sh +++ b/build-package.sh @@ -477,6 +477,8 @@ termux_step_start_build() { export TERMUX_APT=" \ -o APT::Get::Assume-Yes=true \ -o APT::Get::ReInstall=true + -o APT::Get::Install-Recommends=false \ + -o APT::Get::Install-Suggests=false \ -o APT::Architecture=${TERMUX_ARCH} \ -o Dir::Etc=${TERMUX_PREFIX}/etc/apt/ \ -o Dir::State=${TERMUX_PREFIX}/var/lib/apt \ From f2254adde808117ee02e501f3be4c480fac93e5a Mon Sep 17 00:00:00 2001 From: Stephen Date: Wed, 29 Nov 2017 08:18:08 -0800 Subject: [PATCH 49/99] Update build.sh --- packages/libllvm/build.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/libllvm/build.sh b/packages/libllvm/build.sh index a61540978..0a7cb450e 100644 --- a/packages/libllvm/build.sh +++ b/packages/libllvm/build.sh @@ -59,6 +59,9 @@ termux_step_post_extract_package () { } termux_step_host_build () { + if [ ! -z ${TERMUX_INSTALL_DEPS+x} ]; then + apt-get $TERMUX_APT purge ndk-sysroot + fi termux_setup_cmake cmake -G "Unix Makefiles" $TERMUX_PKG_SRCDIR \ -DLLVM_BUILD_TESTS=OFF \ From aab59327269b133b8aeef2eb688f1a48ccfed2b9 Mon Sep 17 00:00:00 2001 From: Stephen Date: Thu, 30 Nov 2017 10:03:54 -0800 Subject: [PATCH 50/99] Update build.sh --- packages/apache2/build.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/apache2/build.sh b/packages/apache2/build.sh index d9a0bc36e..fbfe7e037 100644 --- a/packages/apache2/build.sh +++ b/packages/apache2/build.sh @@ -71,6 +71,11 @@ TERMUX_PKG_INCLUDE_IN_DEVPACKAGE="share/apache2/build" TERMUX_PKG_EXTRA_MAKE_ARGS="-s" termux_step_pre_configure () { + # address interpreter issues + sed -i 's/\/home\/fornwall\/.termux-build\/apr\/tmp\/libtool/\/usr\/bin\/libtool/' ${TERMUX_PREFIX}/bin/apr-1-config + sed -i 's/\/data\/data\/com.termux\/files\/usr\/bin\/bash/\/bin\/bash/' ${TERMUX_PREFIX}/share/apr-1/build/libtool + sed -i 's/\/home\/fornwall/\/home\/builder/' ${TERMUX_PREFIX}/share/apr-1/build/libtool + # remove old files rm -rf "$TERMUX_PREFIX"/{libexec,share,etc}/apache2 rm -rf "$TERMUX_PREFIX"/lib/cgi-bin From d66dab7629165ea52e2cf006320e6f4aba7820c5 Mon Sep 17 00:00:00 2001 From: Stephen Date: Thu, 30 Nov 2017 14:03:50 -0800 Subject: [PATCH 51/99] Update build.sh --- packages/php/build.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/php/build.sh b/packages/php/build.sh index 9dde61e2e..8bb400087 100644 --- a/packages/php/build.sh +++ b/packages/php/build.sh @@ -48,6 +48,14 @@ ac_cv_func_res_nsearch=no " termux_step_pre_configure () { + # Replace chrooted perl with system perl + sed -i 's/\/data\/data\/com\.termux\/files\/usr\/bin\/perl/\/usr\/bin\/perl/' $TERMUX_PREFIX/bin/apxs + sed -i 's/\/data\/data\/com\.termux\/files\/usr\/bin\/perl/\/usr\/bin\/perl/' $TERMUX_PREFIX/bin/curl-config + sed -i 's/\/home\/fornwall\/.termux-build\/apr\/tmp/\/home\/builder\/.termux-build\/php\/build/' $TERMUX_PREFIX/bin/apr-1-config + rm $TERMUX_PREFIX/bin/pg_config + + #because the new mariadb hides away all these includes inside server subdir + CFLAGS+=" -I$TERMUX_PREFIX/include/mysql/server -I$TERMUX_PREFIX/include/mysql" LDFLAGS+=" -landroid-glob -llog" export PATH=$PATH:$TERMUX_PKG_HOSTBUILD_DIR/sapi/cli/ From b2bfc7541c9909dbcc83828a2481eae13e0a2932 Mon Sep 17 00:00:00 2001 From: Stephen Date: Thu, 30 Nov 2017 15:00:30 -0800 Subject: [PATCH 52/99] Update build.sh --- packages/php/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/php/build.sh b/packages/php/build.sh index 8bb400087..fedea2f52 100644 --- a/packages/php/build.sh +++ b/packages/php/build.sh @@ -8,7 +8,7 @@ TERMUX_PKG_SRCURL=https://secure.php.net/distributions/php-${TERMUX_PKG_VERSION} TERMUX_PKG_HOSTBUILD=true # Build the native php without xml support as we only need phar: TERMUX_PKG_EXTRA_HOSTBUILD_CONFIGURE_ARGS="--disable-libxml --disable-dom --disable-simplexml --disable-xml --disable-xmlreader --disable-xmlwriter --without-pear" -TERMUX_PKG_DEPENDS="libandroid-glob, libxml2, liblzma, openssl, pcre, libbz2, libcrypt, libcurl, libgd, readline, freetype" +TERMUX_PKG_DEPENDS="libandroid-glob, libxml2, liblzma, openssl, pcre, libbz2, libcrypt, libcurl, libgd, readline, freetype, apr, apr-util, openssl, libpng, libjpeg-turbo" # mysql modules were initially shared libs TERMUX_PKG_CONFLICTS="php-mysql" TERMUX_PKG_REPLACES="php-mysql" From 1eef3fd12ef00a439fd48cc8efcc0f5f424fe4bd Mon Sep 17 00:00:00 2001 From: Stephen Date: Mon, 11 Dec 2017 08:10:46 -0800 Subject: [PATCH 53/99] cheap trick to fix + issue in package search --- build-package.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/build-package.sh b/build-package.sh index 03e8c4f02..91df9d5ba 100755 --- a/build-package.sh +++ b/build-package.sh @@ -498,7 +498,8 @@ termux_step_start_build() { while IFS=',' read -ra PKG; do for p in "${PKG[@]}"; do p="$(echo -e "${p}" | tr -d '[:space:]')" - apt-get $TERMUX_APT install "^${p}(-dev)?$":any + # Install packages and dev packages and escape + with \+ to avoid apt regex + apt-get $TERMUX_APT install "^${p//+/\\+}(-dev)?$":any sudo chown -R builder:builder /data done done <<< "${!i}" @@ -510,7 +511,8 @@ termux_step_start_build() { while IFS=',' read -ra PKG; do for p in "${PKG[@]}"; do p="$(echo -e "${p}" | tr -d '[:space:]')" - apt-get $TERMUX_APT install "^${p}(-dev)?$":any + # Install packages and dev packages and escape + with \+ to avoid apt regex + apt-get $TERMUX_APT install "^${p//+/\\+}(-dev)?$":any sudo chown -R builder:builder /data done done <<< $TERMUX_SUBPKG_DEPENDS From 2dd7c634365b03ce1a05c083ef3c3f72c1a243c8 Mon Sep 17 00:00:00 2001 From: Stephen Date: Thu, 15 Feb 2018 09:18:29 -0800 Subject: [PATCH 54/99] Update build-package.sh --- build-package.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/build-package.sh b/build-package.sh index 91df9d5ba..fbc6c0c81 100755 --- a/build-package.sh +++ b/build-package.sh @@ -456,7 +456,7 @@ termux_step_start_build() { unzip ${TERMUX_COMMON_CACHEDIR}/bootstrap-${TERMUX_ARCH}.zip -d $TERMUX_PREFIX # TODO move this install to Dockerfile - sudo apt-get update && sudo apt-get -y dist-upgrade && sudo apt-get install -y libcap2-bin gawk tree strace + sudo apt-get update && sudo apt-get -y dist-upgrade && sudo apt-get install -y libcap2-bin gawk tree strace devscripts # set capabilities on dpkg # Some packages built by uid 1001 (builder is 1000) # Need capabilities for dpkg to set non-builder uid @@ -1400,6 +1400,12 @@ termux_step_reverse_depends() { if [ ! -z ${TERMUX_INSTALL_DEPS+x} ]; then # TODO build reverse depends with packages apt-cache $TERMUX_APT rdepends $TERMUX_PKG_NAME + + # TODO compare package with existing + echo "COMPARING PACKAGES" + apt $TERMUX_APT download $TERMUX_PKG_NAME + debdiff ${TERMUX_PKG_NAME}*.deb ${TERMUX_PKG_DEBFILE} + echo "DONE COMPARE PACKAGES" fi } From 6845d6589cf42e6d3c041d044534c87c46909bc0 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Wed, 26 Dec 2018 12:21:14 +0100 Subject: [PATCH 55/99] updates --- build-package.sh | 121 ++++++++++++++++++----------------------------- 1 file changed, 47 insertions(+), 74 deletions(-) diff --git a/build-package.sh b/build-package.sh index fbc6c0c81..978d88c6d 100755 --- a/build-package.sh +++ b/build-package.sh @@ -254,20 +254,20 @@ termux_step_handle_arguments() { echo " -d Build with debug symbols." echo " -D Build a disabled package in disabled-packages/." echo " -f Force build even if package has already been built." - echo " -i Install dependencies." + echo " -i Build dependencies." echo " -q Quiet build." echo " -s Skip dependency check." echo " -o Specify deb directory. Default: debs/." exit 1 } - while getopts :a:hdDfqso: option; do + while getopts :a:hdDfiqso: option; do case "$option" in a) TERMUX_ARCH="$OPTARG";; h) _show_usage;; d) export TERMUX_DEBUG=true;; D) local TERMUX_IS_DISABLED=true;; f) TERMUX_FORCE_BUILD=true;; - i) export TERMUX_INSTALL_DEPS=true;; + i) export TERMUX_BUILD_DEPS=true;; q) export TERMUX_QUIET_BUILD=true;; s) export TERMUX_SKIP_DEPCHECK=true;; o) TERMUX_DEBDIR="$(realpath -m $OPTARG)";; @@ -324,6 +324,7 @@ termux_step_setup_variables() { : "${TERMUX_PKG_API_LEVEL:="21"}" : "${TERMUX_DEBDIR:="${TERMUX_SCRIPTDIR}/debs"}" : "${TERMUX_PKG_MAINTAINER:="Fredrik Fornwall @fornwall"}" + : "${TERMUX_REPO_URL:="https://termux.net/dists/stable/main"}" if [ "x86_64" = "$TERMUX_ARCH" ] || [ "aarch64" = "$TERMUX_ARCH" ]; then TERMUX_ARCH_BITS=64 @@ -403,7 +404,7 @@ termux_step_handle_buildarch() { local TERMUX_PREVIOUS_ARCH TERMUX_PREVIOUS_ARCH=$(cat $TERMUX_ARCH_FILE) if [ "$TERMUX_PREVIOUS_ARCH" != "$TERMUX_ARCH" ]; then - local TERMUX_DATA_BACKUPDIRS=$TERMUX_TOPDIR/_databackups + local TERMUX_DATA_BACKUPDIRS=$TERMUX_TOPDIR/_databackups mkdir -p "$TERMUX_DATA_BACKUPDIRS" local TERMUX_DATA_PREVIOUS_BACKUPDIR=$TERMUX_DATA_BACKUPDIRS/$TERMUX_PREVIOUS_ARCH local TERMUX_DATA_CURRENT_BACKUPDIR=$TERMUX_DATA_BACKUPDIRS/$TERMUX_ARCH @@ -440,10 +441,11 @@ termux_step_start_build() { exit 0 fi - if [ ! -z ${TERMUX_INSTALL_DEPS+x} ]; then + if [ ! ${TERMUX_BUILD_DEPS:=false} = true ]; then # Ensure folders present (but not $TERMUX_PKG_SRCDIR, it will be created in build) mkdir -p "$TERMUX_COMMON_CACHEDIR" \ - "$TERMUX_DEBDIR" \ + "${TERMUX_COMMON_CACHEDIR}-${TERMUX_ARCH}" \ + "$TERMUX_DEBDIR" \ "$TERMUX_PKG_BUILDDIR" \ "$TERMUX_PKG_PACKAGEDIR" \ "$TERMUX_PKG_TMPDIR" \ @@ -452,74 +454,33 @@ termux_step_start_build() { $TERMUX_PREFIX/{bin,etc,lib,libexec,share,tmp,include} # Setup bootstrap termux_download https://termux.net/bootstrap/bootstrap-${TERMUX_ARCH}.zip \ - ${TERMUX_COMMON_CACHEDIR}/bootstrap-${TERMUX_ARCH}.zip - unzip ${TERMUX_COMMON_CACHEDIR}/bootstrap-${TERMUX_ARCH}.zip -d $TERMUX_PREFIX - - # TODO move this install to Dockerfile - sudo apt-get update && sudo apt-get -y dist-upgrade && sudo apt-get install -y libcap2-bin gawk tree strace devscripts - # set capabilities on dpkg - # Some packages built by uid 1001 (builder is 1000) - # Need capabilities for dpkg to set non-builder uid - sudo /sbin/setcap cap_chown,cap_fowner,cap_dac_override+eip /usr/bin/dpkg - sudo /sbin/setcap cap_chown,cap_fowner,cap_dac_override+eip /usr/bin/apt-get - # install packages that include subpackages - sudo sed -i -e 's/DPkg::Pre-Install-Pkgs/\/\/ DPkg::Pre-Install-Pkgs/' /etc/apt/apt.conf.d/* - sudo sed -i -e 's/DPkg::Post-Invoke/\/\/ DPkg::Post-Invoke/' /etc/apt/apt.conf.d/* - sudo sed -i -e 's/APT::Update::Post-Invoke/\/\/ APT::Update::Post-Invoke/' /etc/apt/apt.conf.d/* - TERMUX_DPKG=" \ - --force-architecture \ - --force-not-root \ - --force-configure-any \ - --force-confdef \ - --force-confold \ - --force-depends \ - --admindir=${TERMUX_PREFIX}/var/lib/dpkg" - export TERMUX_APT=" \ - -o APT::Get::Assume-Yes=true \ - -o APT::Get::ReInstall=true - -o APT::Get::Install-Recommends=false \ - -o APT::Get::Install-Suggests=false \ - -o APT::Architecture=${TERMUX_ARCH} \ - -o Dir::Etc=${TERMUX_PREFIX}/etc/apt/ \ - -o Dir::State=${TERMUX_PREFIX}/var/lib/apt \ - -o Dir::State::status=${TERMUX_PREFIX}/var/lib/dpkg/status \ - -o Dir::Cache=${TERMUX_PREFIX}/var/cache/apt \ - -o Dir::Log=${TERMUX_PREFIX}/var/log/apt" - for arg in ${TERMUX_DPKG}; do - TERMUX_APT+=" -o DPkg::Options::=${arg}" - done - export DEBCONF_FRONTEND=noninteractive - apt-get $TERMUX_APT clean && apt-get $TERMUX_APT update && apt-get $TERMUX_APT upgrade - # libandroid-support-dev is build-essential - apt-get $TERMUX_APT install libandroid-support-dev:any - sudo chown -R builder:builder /data - array=( TERMUX_PKG_DEPENDS TERMUX_PKG_BUILD_DEPENDS ) - for i in "${array[@]}"; do - while IFS=',' read -ra PKG; do - for p in "${PKG[@]}"; do - p="$(echo -e "${p}" | tr -d '[:space:]')" - # Install packages and dev packages and escape + with \+ to avoid apt regex - apt-get $TERMUX_APT install "^${p//+/\\+}(-dev)?$":any - sudo chown -R builder:builder /data - done - done <<< "${!i}" - done - for subpkg in $TERMUX_PKG_BUILDER_DIR/*.subpackage.sh; do - test ! -f "$subpkg" && continue - local TERMUX_SUBPKG_DEPENDS="" - source $subpkg - while IFS=',' read -ra PKG; do - for p in "${PKG[@]}"; do - p="$(echo -e "${p}" | tr -d '[:space:]')" - # Install packages and dev packages and escape + with \+ to avoid apt regex - apt-get $TERMUX_APT install "^${p//+/\\+}(-dev)?$":any - sudo chown -R builder:builder /data - done - done <<< $TERMUX_SUBPKG_DEPENDS + ${TERMUX_COMMON_CACHEDIR}/bootstrap-${TERMUX_ARCH}.zip \ + 9f43bfdd23fbc6ae02f1f5a44b8e064374db8291ffe32658e2724d3f3f419981 + unzip -qo ${TERMUX_COMMON_CACHEDIR}/bootstrap-${TERMUX_ARCH}.zip -d $TERMUX_PREFIX + ( + cd $TERMUX_PREFIX + while read line; do + ln -sf ${line/←/ } + done Date: Sat, 29 Dec 2018 11:56:40 +0100 Subject: [PATCH 56/99] build-package: add termux_extract_version function --- build-package.sh | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/build-package.sh b/build-package.sh index 978d88c6d..f726c4753 100755 --- a/build-package.sh +++ b/build-package.sh @@ -426,6 +426,15 @@ termux_step_handle_buildarch() { echo "$TERMUX_ARCH" > $TERMUX_ARCH_FILE } +# Function to get TERMUX_PKG_VERSION from build.sh +termux_extract_version() { + package=$1 + ( + source $package/build.sh + echo $TERMUX_PKG_VERSION-$TERMUX_PKG_REVISION + ) +} + # Source the package build script and start building. No to be overridden by packages. termux_step_start_build() { # shellcheck source=/dev/null @@ -471,11 +480,8 @@ termux_step_start_build() { TERMUX_ALL_DEPS=$(./scripts/buildorder.py "$TERMUX_PKG_BUILDER_DIR") for p in $TERMUX_ALL_DEPS; do echo "Downloading dependency $(basename $p) if necessary..." - # termux_get_deb $TERMUX_ARCH "$p" - local p_ver=$(termux_extract_version "$p") - echo "hej" - echo "$p_ver" - echo "håå" + local dep_version=$(termux_extract_version "$p") + termux_get_deb $TERMUX_ARCH "$p" done fi From 795cc15f5fe1102cc456fc54ce678cb8d59d1619 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sat, 29 Dec 2018 14:50:27 +0100 Subject: [PATCH 57/99] Working deb_get_info and install_dep_deb functions --- build-package.sh | 101 ++++++++++++++++++++++++++--------------------- 1 file changed, 55 insertions(+), 46 deletions(-) diff --git a/build-package.sh b/build-package.sh index f726c4753..67137c4ce 100755 --- a/build-package.sh +++ b/build-package.sh @@ -267,7 +267,7 @@ termux_step_handle_arguments() { d) export TERMUX_DEBUG=true;; D) local TERMUX_IS_DISABLED=true;; f) TERMUX_FORCE_BUILD=true;; - i) export TERMUX_BUILD_DEPS=true;; + i) TERMUX_BUILD_DEPS=true;; q) export TERMUX_QUIET_BUILD=true;; s) export TERMUX_SKIP_DEPCHECK=true;; o) TERMUX_DEBDIR="$(realpath -m $OPTARG)";; @@ -324,6 +324,8 @@ termux_step_setup_variables() { : "${TERMUX_PKG_API_LEVEL:="21"}" : "${TERMUX_DEBDIR:="${TERMUX_SCRIPTDIR}/debs"}" : "${TERMUX_PKG_MAINTAINER:="Fredrik Fornwall @fornwall"}" + : "${TERMUX_SKIP_DEPCHECK:="false"}" + : "${TERMUX_BUILD_DEPS:="false"}" : "${TERMUX_REPO_URL:="https://termux.net/dists/stable/main"}" if [ "x86_64" = "$TERMUX_ARCH" ] || [ "aarch64" = "$TERMUX_ARCH" ]; then @@ -427,14 +429,35 @@ termux_step_handle_buildarch() { } # Function to get TERMUX_PKG_VERSION from build.sh -termux_extract_version() { +termux_extract_dep_info() { package=$1 ( source $package/build.sh - echo $TERMUX_PKG_VERSION-$TERMUX_PKG_REVISION + if [ "$TERMUX_PKG_PLATFORM_INDEPENDENT" = "yes" ]; then + TERMUX_ARCH=all + fi + if [ ! "$TERMUX_PKG_REVISION" = 0 ]; then + TERMUX_PKG_VERSION+="-$TERMUX_PKG_REVISION" + fi + echo ${TERMUX_ARCH} ${TERMUX_PKG_VERSION} ) } +termux_install_dep_deb() { + local package=$1 + local package_arch=$2 + local version=$3 + local deb_file=${package}_${version}_${package_arch}.deb + ( + cd ${TERMUX_COMMON_CACHEDIR}-${package_arch} + # TODO: allow for specifying several repos in TERMUX_REPO_URL + curl --fail -LO $TERMUX_REPO_URL/binary-${package_arch}/${deb_file} 2>/dev/null \ + && echo "Extracting $package..." && ar x ${deb_file} data.tar.xz \ + && tar xf data.tar.xz --no-overwrite-dir -C / + ) || ( echo "Download of $package from $TERMUX_REPO_URL failed, building instead" \ + && ./build-package.sh -a $TERMUX_ARCH -s "$package" ) +} + # Source the package build script and start building. No to be overridden by packages. termux_step_start_build() { # shellcheck source=/dev/null @@ -450,10 +473,11 @@ termux_step_start_build() { exit 0 fi - if [ ! ${TERMUX_BUILD_DEPS:=false} = true ]; then + local TERMUX_ALL_DEPS=$(./scripts/buildorder.py "$TERMUX_PKG_BUILDER_DIR") + if [ ! $TERMUX_SKIP_DEPCHECK ] && [ ! $TERMUX_BUILD_DEPS ]; then # Ensure folders present (but not $TERMUX_PKG_SRCDIR, it will be created in build) mkdir -p "$TERMUX_COMMON_CACHEDIR" \ - "${TERMUX_COMMON_CACHEDIR}-${TERMUX_ARCH}" \ + "$TERMUX_COMMON_CACHEDIR-$TERMUX_ARCH" \ "$TERMUX_DEBDIR" \ "$TERMUX_PKG_BUILDDIR" \ "$TERMUX_PKG_PACKAGEDIR" \ @@ -473,39 +497,24 @@ termux_step_start_build() { done Date: Sat, 29 Dec 2018 16:02:39 +0100 Subject: [PATCH 58/99] Revert "Update build.sh" This reverts commit 7668e25134a27e733e8c24a47df2f7456934e586. --- packages/libllvm/build.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/libllvm/build.sh b/packages/libllvm/build.sh index 0a7cb450e..a61540978 100644 --- a/packages/libllvm/build.sh +++ b/packages/libllvm/build.sh @@ -59,9 +59,6 @@ termux_step_post_extract_package () { } termux_step_host_build () { - if [ ! -z ${TERMUX_INSTALL_DEPS+x} ]; then - apt-get $TERMUX_APT purge ndk-sysroot - fi termux_setup_cmake cmake -G "Unix Makefiles" $TERMUX_PKG_SRCDIR \ -DLLVM_BUILD_TESTS=OFF \ From 32acdf3861e1ebeeb9cfe1e83b403f28fff94af1 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sat, 29 Dec 2018 17:25:52 +0100 Subject: [PATCH 59/99] Change TERMUX_BUILD_DEPS to TERMUX_INSTALL_DEPS again. Fix syntax in if expressions. --- build-package.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/build-package.sh b/build-package.sh index 67137c4ce..d232b0a44 100755 --- a/build-package.sh +++ b/build-package.sh @@ -267,7 +267,7 @@ termux_step_handle_arguments() { d) export TERMUX_DEBUG=true;; D) local TERMUX_IS_DISABLED=true;; f) TERMUX_FORCE_BUILD=true;; - i) TERMUX_BUILD_DEPS=true;; + i) export TERMUX_INSTALL_DEPS=true;; q) export TERMUX_QUIET_BUILD=true;; s) export TERMUX_SKIP_DEPCHECK=true;; o) TERMUX_DEBDIR="$(realpath -m $OPTARG)";; @@ -325,7 +325,7 @@ termux_step_setup_variables() { : "${TERMUX_DEBDIR:="${TERMUX_SCRIPTDIR}/debs"}" : "${TERMUX_PKG_MAINTAINER:="Fredrik Fornwall @fornwall"}" : "${TERMUX_SKIP_DEPCHECK:="false"}" - : "${TERMUX_BUILD_DEPS:="false"}" + : "${TERMUX_INSTALL_DEPS:="false"}" : "${TERMUX_REPO_URL:="https://termux.net/dists/stable/main"}" if [ "x86_64" = "$TERMUX_ARCH" ] || [ "aarch64" = "$TERMUX_ARCH" ]; then @@ -474,7 +474,7 @@ termux_step_start_build() { fi local TERMUX_ALL_DEPS=$(./scripts/buildorder.py "$TERMUX_PKG_BUILDER_DIR") - if [ ! $TERMUX_SKIP_DEPCHECK ] && [ ! $TERMUX_BUILD_DEPS ]; then + if [ "$TERMUX_SKIP_DEPCHECK" = false ] && [ "$TERMUX_INSTALL_DEPS" = true ]; then # Ensure folders present (but not $TERMUX_PKG_SRCDIR, it will be created in build) mkdir -p "$TERMUX_COMMON_CACHEDIR" \ "$TERMUX_COMMON_CACHEDIR-$TERMUX_ARCH" \ @@ -505,7 +505,7 @@ termux_step_start_build() { read dep_arch dep_version <<< $(termux_extract_dep_info "$pkg") termux_install_dep_deb $(basename $pkg) $dep_arch $dep_version done - elif [ ! $TERMUX_SKIP_DEPCHECK ] && [ $TERMUX_BUILD_DEPS ]; then + elif [ "$TERMUX_SKIP_DEPCHECK" = false ] && [ "$TERMUX_INSTALL_DEPS" = false ]; then # Build dependencies local pkg for pkg in $TERMUX_ALL_DEPS; do @@ -523,7 +523,7 @@ termux_step_start_build() { TERMUX_PKG_FULLVERSION+="-$TERMUX_PKG_REVISION" fi - if [ "$TERMUX_DEBUG" == "true" ]; then + if [ "$TERMUX_DEBUG" = true ]; then if [ "$TERMUX_PKG_HAS_DEBUG" == "yes" ]; then DEBUG="-dbg" else From 40da58977ec363771741ff2648b54e946a89ada7 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sat, 29 Dec 2018 17:26:35 +0100 Subject: [PATCH 60/99] build-package: add function to compare debs --- build-package.sh | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/build-package.sh b/build-package.sh index d232b0a44..83676e579 100755 --- a/build-package.sh +++ b/build-package.sh @@ -478,6 +478,7 @@ termux_step_start_build() { # Ensure folders present (but not $TERMUX_PKG_SRCDIR, it will be created in build) mkdir -p "$TERMUX_COMMON_CACHEDIR" \ "$TERMUX_COMMON_CACHEDIR-$TERMUX_ARCH" \ + "$TERMUX_COMMON_CACHEDIR-all" \ "$TERMUX_DEBDIR" \ "$TERMUX_PKG_BUILDDIR" \ "$TERMUX_PKG_PACKAGEDIR" \ @@ -1384,18 +1385,24 @@ termux_step_create_debfile() { "$TERMUX_PKG_PACKAGEDIR/data.tar.xz" } -# termux_step_reverse_depends() { -# if [ ! ${TERMUX_BUILD_DEPS:=false} = true ]; then -# # TODO build reverse depends with packages -# ### apt-cache $TERMUX_APT rdepends $TERMUX_PKG_NAME -# -# # TODO compare package with existing -# echo "COMPARING PACKAGES" -# ### apt $TERMUX_APT download $TERMUX_PKG_NAME -# debdiff ${TERMUX_PKG_NAME}*.deb ${TERMUX_PKG_DEBFILE} -# echo "DONE COMPARE PACKAGES" -# fi -# } +termux_step_reverse_depends() { + if [ "${TERMUX_INSTALL_DEPS}" = true ]; then + local arch version + echo "COMPARING PACKAGES" + read arch version <<< $(termux_extract_dep_info "$TERMUX_PKG_BUILDER_DIR") + termux_install_dep_deb $(basename $TERMUX_PKG_BUILDER_DIR) $arch $version + deb_file=${TERMUX_PKG_NAME}_${version}_${arch}.deb + + ( + cd ${TERMUX_COMMON_CACHEDIR}-${arch} + # TODO: allow for specifying several repos in TERMUX_REPO_URL + curl --fail -LO $TERMUX_REPO_URL/binary-${arch}/${deb_file} \ + && echo "Extracting ${TERMUX_PKG_NAME}..." + ) + debdiff $TERMUX_DEBDIR/$deb_file $TERMUX_COMMON_CACHEDIR-$arch/$deb_file + echo "DONE COMPARING PACKAGES" + fi +} # Finish the build. Not to be overridden by package scripts. termux_step_finish_build() { @@ -1437,5 +1444,5 @@ cd "$TERMUX_PKG_MASSAGEDIR/$TERMUX_PREFIX" termux_step_post_massage termux_step_create_datatar termux_step_create_debfile -# termux_step_reverse_depends +termux_step_reverse_depends termux_step_finish_build From c61c8f8947a582609aef7304461d1135f6e73270 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sat, 29 Dec 2018 17:27:18 +0100 Subject: [PATCH 61/99] build-package: make the bootstrap installtion more intuitive --- build-package.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build-package.sh b/build-package.sh index 83676e579..6bbb1d2ab 100755 --- a/build-package.sh +++ b/build-package.sh @@ -493,8 +493,8 @@ termux_step_start_build() { unzip -qo ${TERMUX_COMMON_CACHEDIR}/bootstrap-${TERMUX_ARCH}.zip -d $TERMUX_PREFIX ( cd $TERMUX_PREFIX - while read line; do - ln -sf ${line/←/ } + while read link; do + ln -sf ${link/←/ } done Date: Sun, 30 Dec 2018 00:33:49 +0100 Subject: [PATCH 62/99] termux_extract_dep_info: Fix bug when ARCH=all and REVISION != 0 Also make it less verbose when building with -q and add some TODO notes --- build-package.sh | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/build-package.sh b/build-package.sh index 6bbb1d2ab..7150136b6 100755 --- a/build-package.sh +++ b/build-package.sh @@ -432,8 +432,14 @@ termux_step_handle_buildarch() { termux_extract_dep_info() { package=$1 ( + # Reset TERMUX_PKG_PLATFORM_INDEPENDENT and TERMUX_PKG_REVISION since these aren't + # mandatory in a build.sh. Otherwise these will equal the main package's values for + # deps that should have the default values + TERMUX_PKG_PLATFORM_INDEPENDENT="" + TERMUX_PKG_REVISION="0" + source $package/build.sh - if [ "$TERMUX_PKG_PLATFORM_INDEPENDENT" = "yes" ]; then + if [ "$TERMUX_PKG_PLATFORM_INDEPENDENT" = yes ]; then TERMUX_ARCH=all fi if [ ! "$TERMUX_PKG_REVISION" = 0 ]; then @@ -448,14 +454,15 @@ termux_install_dep_deb() { local package_arch=$2 local version=$3 local deb_file=${package}_${version}_${package_arch}.deb + # TODO: download InRelease, Packages files and check signature and hash ( cd ${TERMUX_COMMON_CACHEDIR}-${package_arch} # TODO: allow for specifying several repos in TERMUX_REPO_URL curl --fail -LO $TERMUX_REPO_URL/binary-${package_arch}/${deb_file} 2>/dev/null \ - && echo "Extracting $package..." && ar x ${deb_file} data.tar.xz \ - && tar xf data.tar.xz --no-overwrite-dir -C / - ) || ( echo "Download of $package from $TERMUX_REPO_URL failed, building instead" \ - && ./build-package.sh -a $TERMUX_ARCH -s "$package" ) + && if [ ! "$TERMUX_QUIET_BUILD" = true ]; then echo "Extracting $package..."; fi \ + && ar x ${deb_file} data.tar.xz && tar xf data.tar.xz --no-overwrite-dir -C / + # TODO: this implementation is buggy if the `ar x` or `tar xf + ) } # Source the package build script and start building. No to be overridden by packages. From 4b9fcfeece9b575ef92a0fed40154d48a9b4c41e Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sun, 30 Dec 2018 00:37:21 +0100 Subject: [PATCH 63/99] build-package.sh: make dep downloads less verbose for quiet builds Add error messages when download fails though. Try to download -dev packages but ignore errors here --- build-package.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/build-package.sh b/build-package.sh index 7150136b6..c366a24fa 100755 --- a/build-package.sh +++ b/build-package.sh @@ -509,9 +509,17 @@ termux_step_start_build() { # Download dependencies local pkg dep_arch dep_version for pkg in $TERMUX_ALL_DEPS; do - echo "Downloading dependency $(basename $pkg) if necessary..." read dep_arch dep_version <<< $(termux_extract_dep_info "$pkg") - termux_install_dep_deb $(basename $pkg) $dep_arch $dep_version + if [ ! "$TERMUX_QUIET_BUILD" = true ]; then + echo "Downloading dependency $(basename $pkg) $dep_version if necessary..." + fi + termux_install_dep_deb $(basename $pkg) $dep_arch $dep_version \ + || ( echo "Download of $(basename $pkg) $dep_version from $TERMUX_REPO_URL failed, building instead" \ + && ./build-package.sh -a $TERMUX_ARCH -s "$pkg" \ + && continue ) + + termux_install_dep_deb $(basename $pkg)-dev $dep_arch $dep_version || \ + echo "Download of $(basename $pkg)-dev $dep_version from $TERMUX_REPO_URL failed" done elif [ "$TERMUX_SKIP_DEPCHECK" = false ] && [ "$TERMUX_INSTALL_DEPS" = false ]; then # Build dependencies From fdd700ce4fef6f2140bb28c26b1af5a9d136e0bd Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sun, 30 Dec 2018 12:22:11 +0100 Subject: [PATCH 64/99] build-package: fix forgotten p->pkg --- build-package.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build-package.sh b/build-package.sh index c366a24fa..fc8e65e77 100755 --- a/build-package.sh +++ b/build-package.sh @@ -525,9 +525,9 @@ termux_step_start_build() { # Build dependencies local pkg for pkg in $TERMUX_ALL_DEPS; do - echo "Building dependency $p if necessary..." + echo "Building dependency $pkg if necessary..." # Built dependencies are put in the default TERMUX_DEBDIR instead of the specified one - ./build-package.sh -a $TERMUX_ARCH -s "$p" + ./build-package.sh -a $TERMUX_ARCH -s "$pkg" done else echo "Skipping dependency check" From 3e60a99eec06a859f9a0cc27499914ae6f2630f7 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sun, 30 Dec 2018 12:22:44 +0100 Subject: [PATCH 65/99] build-package: Fix so that debdiff's exit code doesn't stop build --- build-package.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build-package.sh b/build-package.sh index fc8e65e77..6cba76178 100755 --- a/build-package.sh +++ b/build-package.sh @@ -1414,7 +1414,8 @@ termux_step_reverse_depends() { curl --fail -LO $TERMUX_REPO_URL/binary-${arch}/${deb_file} \ && echo "Extracting ${TERMUX_PKG_NAME}..." ) - debdiff $TERMUX_DEBDIR/$deb_file $TERMUX_COMMON_CACHEDIR-$arch/$deb_file + # `|| true` to prevent debdiff's exit code from stopping build + debdiff $TERMUX_DEBDIR/$deb_file $TERMUX_COMMON_CACHEDIR-$arch/$deb_file || true echo "DONE COMPARING PACKAGES" fi } From 88f241b5533653b2c9f8a6e658e97d82302fb9fe Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sun, 30 Dec 2018 12:23:12 +0100 Subject: [PATCH 66/99] build-package: Set default verbose level --- build-package.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/build-package.sh b/build-package.sh index 6cba76178..ab6b4e446 100755 --- a/build-package.sh +++ b/build-package.sh @@ -322,6 +322,7 @@ termux_step_setup_variables() { : "${TERMUX_ANDROID_HOME:="/data/data/com.termux/files/home"}" : "${TERMUX_DEBUG:=""}" : "${TERMUX_PKG_API_LEVEL:="21"}" + : "${TERMUX_QUIET_BUILD:="false"}" : "${TERMUX_DEBDIR:="${TERMUX_SCRIPTDIR}/debs"}" : "${TERMUX_PKG_MAINTAINER:="Fredrik Fornwall @fornwall"}" : "${TERMUX_SKIP_DEPCHECK:="false"}" From 0a25f20b22e51efe3f4e499037ec4c3ee3fe76ff Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sun, 30 Dec 2018 12:24:16 +0100 Subject: [PATCH 67/99] build-package: fix truncated comment --- build-package.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-package.sh b/build-package.sh index ab6b4e446..bfef23fc2 100755 --- a/build-package.sh +++ b/build-package.sh @@ -462,7 +462,7 @@ termux_install_dep_deb() { curl --fail -LO $TERMUX_REPO_URL/binary-${package_arch}/${deb_file} 2>/dev/null \ && if [ ! "$TERMUX_QUIET_BUILD" = true ]; then echo "Extracting $package..."; fi \ && ar x ${deb_file} data.tar.xz && tar xf data.tar.xz --no-overwrite-dir -C / - # TODO: this implementation is buggy if the `ar x` or `tar xf + # TODO: this implementation is buggy if the `ar x` or `tar xf` steps fail ) } From cc39d8c9b8556e83ffd3199eacc493fb1d514a94 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Mon, 31 Dec 2018 00:31:11 +0100 Subject: [PATCH 68/99] build-package: Add signature verification and download by hash Also rename termux_install_dep_deb to termux_download_deb. --- build-package.sh | 74 ++++++++++++++++++++++++++++++----------- scripts/get_pkg_hash.py | 20 +++++++++++ 2 files changed, 74 insertions(+), 20 deletions(-) create mode 100755 scripts/get_pkg_hash.py diff --git a/build-package.sh b/build-package.sh index bfef23fc2..54342f5f2 100755 --- a/build-package.sh +++ b/build-package.sh @@ -327,7 +327,9 @@ termux_step_setup_variables() { : "${TERMUX_PKG_MAINTAINER:="Fredrik Fornwall @fornwall"}" : "${TERMUX_SKIP_DEPCHECK:="false"}" : "${TERMUX_INSTALL_DEPS:="false"}" - : "${TERMUX_REPO_URL:="https://termux.net/dists/stable/main"}" + : "${TERMUX_REPO_URL:="https://termux.net/dists"}" + : "${TERMUX_REPO_DISTRIBUTION:="stable"}" + : "${TERMUX_REPO_COMPONENT:="main"}" if [ "x86_64" = "$TERMUX_ARCH" ] || [ "aarch64" = "$TERMUX_ARCH" ]; then TERMUX_ARCH_BITS=64 @@ -450,20 +452,22 @@ termux_extract_dep_info() { ) } -termux_install_dep_deb() { +termux_download_deb() { local package=$1 local package_arch=$2 local version=$3 local deb_file=${package}_${version}_${package_arch}.deb - # TODO: download InRelease, Packages files and check signature and hash - ( - cd ${TERMUX_COMMON_CACHEDIR}-${package_arch} - # TODO: allow for specifying several repos in TERMUX_REPO_URL - curl --fail -LO $TERMUX_REPO_URL/binary-${package_arch}/${deb_file} 2>/dev/null \ - && if [ ! "$TERMUX_QUIET_BUILD" = true ]; then echo "Extracting $package..."; fi \ - && ar x ${deb_file} data.tar.xz && tar xf data.tar.xz --no-overwrite-dir -C / - # TODO: this implementation is buggy if the `ar x` or `tar xf` steps fail - ) + local pkg_hash=$(./scripts/get_pkg_hash.py ${TERMUX_COMMON_CACHEDIR}-${package_arch}/Packages $package) + + if [ "$pkg_hash" = "" ]; then + # No hash found for $package + return 1 + fi + + termux_download $TERMUX_REPO_URL/$TERMUX_REPO_DISTRIBUTION/$TERMUX_REPO_COMPONENT/binary-${package_arch}/${deb_file} \ + $TERMUX_COMMON_CACHEDIR-$package_arch/${deb_file} \ + $pkg_hash + return 0 } # Source the package build script and start building. No to be overridden by packages. @@ -483,6 +487,9 @@ termux_step_start_build() { local TERMUX_ALL_DEPS=$(./scripts/buildorder.py "$TERMUX_PKG_BUILDER_DIR") if [ "$TERMUX_SKIP_DEPCHECK" = false ] && [ "$TERMUX_INSTALL_DEPS" = true ]; then + # Remove all previously extracted/built files from $TERMUX_PREFIX: + rm -r $TERMUX_PREFIX + rm -f /data/data/.built-packages/* # Ensure folders present (but not $TERMUX_PKG_SRCDIR, it will be created in build) mkdir -p "$TERMUX_COMMON_CACHEDIR" \ "$TERMUX_COMMON_CACHEDIR-$TERMUX_ARCH" \ @@ -506,21 +513,50 @@ termux_step_start_build() { done Packages || termux_error_exit "Download of $TERMUX_REPO_URL/$TERMUX_REPO_DISTRIBUTION/$TERMUX_REPO_COMPONENT/Packages.xz failed" + ) # Download dependencies - local pkg dep_arch dep_version + local pkg dep_arch dep_version deb_file for pkg in $TERMUX_ALL_DEPS; do read dep_arch dep_version <<< $(termux_extract_dep_info "$pkg") if [ ! "$TERMUX_QUIET_BUILD" = true ]; then - echo "Downloading dependency $(basename $pkg) $dep_version if necessary..." + echo "Downloading dependency $(basename $pkg)@$dep_version if necessary..." fi - termux_install_dep_deb $(basename $pkg) $dep_arch $dep_version \ - || ( echo "Download of $(basename $pkg) $dep_version from $TERMUX_REPO_URL failed, building instead" \ + termux_download_deb $(basename $pkg) $dep_arch $dep_version \ + || ( echo "Download of $(basename $pkg)@$dep_version from $TERMUX_REPO_URL failed, building instead" \ && ./build-package.sh -a $TERMUX_ARCH -s "$pkg" \ && continue ) - - termux_install_dep_deb $(basename $pkg)-dev $dep_arch $dep_version || \ - echo "Download of $(basename $pkg)-dev $dep_version from $TERMUX_REPO_URL failed" + local deb_file=$(basename $pkg)_${dep_version}_${dep_arch}.deb + if [ ! "$TERMUX_QUIET_BUILD" = true ]; then echo "Extracting $(basename $pkg)..."; fi + ( + cd $TERMUX_COMMON_CACHEDIR-$dep_arch + ar x ${deb_file} data.tar.xz && tar xf data.tar.xz --no-overwrite-dir -C / + ) + + termux_download_deb $(basename $pkg)-dev $dep_arch $dep_version && \ + ( + cd $TERMUX_COMMON_CACHEDIR-$dep_arch + ar x $(basename $pkg)-dev_${dep_version}_${dep_arch}.deb data.tar.xz + tar xf data.tar.xz --no-overwrite-dir -C / + ) || echo "Download of $(basename $pkg)-dev@$dep_version from $TERMUX_REPO_URL failed" + echo "$dep_version" > "/data/data/.built-packages/$(basename $pkg)" done elif [ "$TERMUX_SKIP_DEPCHECK" = false ] && [ "$TERMUX_INSTALL_DEPS" = false ]; then # Build dependencies @@ -530,8 +566,6 @@ termux_step_start_build() { # Built dependencies are put in the default TERMUX_DEBDIR instead of the specified one ./build-package.sh -a $TERMUX_ARCH -s "$pkg" done - else - echo "Skipping dependency check" fi TERMUX_PKG_FULLVERSION=$TERMUX_PKG_VERSION diff --git a/scripts/get_pkg_hash.py b/scripts/get_pkg_hash.py new file mode 100755 index 000000000..668392601 --- /dev/null +++ b/scripts/get_pkg_hash.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python + +import os, sys + +def get_hash_from_Packages(Packages_file, package, hash="SHA256"): + with open(Packages_file, 'r') as Packages: + package_list = Packages.read().split('\n\n') + for pkg in package_list: + if pkg.split('\n')[0] == "Package: "+package: + for line in pkg.split('\n'): + if line.startswith(hash): + print(line.split(" ")[1]) + break + break + +if __name__ == '__main__': + if len(sys.argv) < 2: + print('Too few arguments, I need the path to a Packages file and a package name. Exiting') + sys.exit(1) + get_hash_from_Packages(sys.argv[1], sys.argv[2]) From 368421c2815229e3305911fc549d906eb71622c3 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Mon, 31 Dec 2018 00:31:44 +0100 Subject: [PATCH 69/99] build-package: update so that termux_step_compare_debs dl's by hash Changes needed after termux_download_deb function update --- build-package.sh | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/build-package.sh b/build-package.sh index 54342f5f2..1700de750 100755 --- a/build-package.sh +++ b/build-package.sh @@ -1435,23 +1435,17 @@ termux_step_create_debfile() { "$TERMUX_PKG_PACKAGEDIR/data.tar.xz" } -termux_step_reverse_depends() { +termux_step_compare_debs() { if [ "${TERMUX_INSTALL_DEPS}" = true ]; then local arch version - echo "COMPARING PACKAGES" - read arch version <<< $(termux_extract_dep_info "$TERMUX_PKG_BUILDER_DIR") - termux_install_dep_deb $(basename $TERMUX_PKG_BUILDER_DIR) $arch $version - deb_file=${TERMUX_PKG_NAME}_${version}_${arch}.deb + if [ ! "$TERMUX_QUIET_BUILD" = true ]; then echo "COMPARING PACKAGES"; fi + cd ${TERMUX_SCRIPTDIR} + termux_download_deb $(basename $TERMUX_PKG_BUILDER_DIR) $TERMUX_ARCH $TERMUX_PKG_FULLVERSION + deb_file=${TERMUX_PKG_NAME}_${TERMUX_PKG_FULLVERSION}_${TERMUX_ARCH}.deb - ( - cd ${TERMUX_COMMON_CACHEDIR}-${arch} - # TODO: allow for specifying several repos in TERMUX_REPO_URL - curl --fail -LO $TERMUX_REPO_URL/binary-${arch}/${deb_file} \ - && echo "Extracting ${TERMUX_PKG_NAME}..." - ) # `|| true` to prevent debdiff's exit code from stopping build - debdiff $TERMUX_DEBDIR/$deb_file $TERMUX_COMMON_CACHEDIR-$arch/$deb_file || true - echo "DONE COMPARING PACKAGES" + debdiff $TERMUX_DEBDIR/$deb_file $TERMUX_COMMON_CACHEDIR-$TERMUX_ARCH/$deb_file || true + if [ ! "$TERMUX_QUIET_BUILD" = true ]; then echo "DONE COMPARING PACKAGES"; fi fi } @@ -1495,5 +1489,5 @@ cd "$TERMUX_PKG_MASSAGEDIR/$TERMUX_PREFIX" termux_step_post_massage termux_step_create_datatar termux_step_create_debfile -termux_step_reverse_depends +termux_step_compare_debs termux_step_finish_build From 2103c04b0bf54c46d6d633de7b7e3ea047021f28 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Tue, 1 Jan 2019 20:22:32 +0100 Subject: [PATCH 70/99] Rename get_pkg_hash.py->get_hash_from_file.py --- build-package.sh | 22 ++++++++++----------- scripts/get_hash_from_file.py | 37 +++++++++++++++++++++++++++++++++++ scripts/get_pkg_hash.py | 20 ------------------- 3 files changed, 47 insertions(+), 32 deletions(-) create mode 100755 scripts/get_hash_from_file.py delete mode 100755 scripts/get_pkg_hash.py diff --git a/build-package.sh b/build-package.sh index 1700de750..656eb64d1 100755 --- a/build-package.sh +++ b/build-package.sh @@ -327,6 +327,7 @@ termux_step_setup_variables() { : "${TERMUX_PKG_MAINTAINER:="Fredrik Fornwall @fornwall"}" : "${TERMUX_SKIP_DEPCHECK:="false"}" : "${TERMUX_INSTALL_DEPS:="false"}" + : "${TERMUX_REPO_SIGNING_KEY:="2218893D3F679BEFC421FD976700B77E6D8D0AE7"}" : "${TERMUX_REPO_URL:="https://termux.net/dists"}" : "${TERMUX_REPO_DISTRIBUTION:="stable"}" : "${TERMUX_REPO_COMPONENT:="main"}" @@ -457,7 +458,7 @@ termux_download_deb() { local package_arch=$2 local version=$3 local deb_file=${package}_${version}_${package_arch}.deb - local pkg_hash=$(./scripts/get_pkg_hash.py ${TERMUX_COMMON_CACHEDIR}-${package_arch}/Packages $package) + local pkg_hash=$(./scripts/get_hash_from_file.py ${TERMUX_COMMON_CACHEDIR}-${package_arch}/Packages $package) if [ "$pkg_hash" = "" ]; then # No hash found for $package @@ -518,19 +519,16 @@ termux_step_start_build() { curl --fail -LO "$TERMUX_REPO_URL/$TERMUX_REPO_DISTRIBUTION/InRelease" \ || termux_error_exit "Download of $TERMUX_REPO_URL/$TERMUX_REPO_DISTRIBUTION/InRelease failed" # Import Fornwalls key: - gpg --recv 2218893D3F679BEFC421FD976700B77E6D8D0AE7 + gpg --recv $TERMUX_REPO_SIGNING_KEY gpg --verify InRelease - for arch in all $TERMUX_ARCH; do - # A sha256 hashsum has length 64 so grep for hashes that are that long - local packages_hash=$(grep binary-$arch/Packages.xz $TERMUX_COMMON_CACHEDIR/InRelease | awk 'length($1) == 64 {print $1}') - termux_download "$TERMUX_REPO_URL/$TERMUX_REPO_DISTRIBUTION/$TERMUX_REPO_COMPONENT/binary-$arch/Packages.xz" \ - "${TERMUX_COMMON_CACHEDIR}-$arch/Packages.xz" \ - $packages_hash - xz -df "${TERMUX_COMMON_CACHEDIR}-$arch/Packages.xz" - done - # cd ${TERMUX_COMMON_CACHEDIR}-$TERMUX_ARCH - # curl --fail -L "$TERMUX_REPO_URL/$TERMUX_REPO_DISTRIBUTION/$TERMUX_REPO_COMPONENT/binary-${TERMUX_ARCH}/Packages.xz" | xz -d > Packages || termux_error_exit "Download of $TERMUX_REPO_URL/$TERMUX_REPO_DISTRIBUTION/$TERMUX_REPO_COMPONENT/Packages.xz failed" ) + for arch in all $TERMUX_ARCH; do + local packages_hash=$(./scripts/get_hash_from_file.py ${TERMUX_COMMON_CACHEDIR}/InRelease $arch) + termux_download "$TERMUX_REPO_URL/$TERMUX_REPO_DISTRIBUTION/$TERMUX_REPO_COMPONENT/binary-$arch/Packages.xz" \ + "${TERMUX_COMMON_CACHEDIR}-$arch/Packages.xz" \ + $packages_hash + xz -df "${TERMUX_COMMON_CACHEDIR}-$arch/Packages.xz" + done # Download dependencies local pkg dep_arch dep_version deb_file diff --git a/scripts/get_hash_from_file.py b/scripts/get_hash_from_file.py new file mode 100755 index 000000000..4d355832c --- /dev/null +++ b/scripts/get_hash_from_file.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python + +import os, sys + +def get_pkg_hash_from_Packages(Packages_file, package, hash="SHA256"): + with open(Packages_file, 'r') as Packages: + package_list = Packages.read().split('\n\n') + for pkg in package_list: + if pkg.split('\n')[0] == "Package: "+package: + for line in pkg.split('\n'): + if line.startswith(hash): + print(line.split(" ")[1]) + break + break + +def get_Packages_hash_from_InRelease(InRelease_file, arch, hash="SHA256"): + string_to_found = 'binary-'+arch+'/Packages.xz' + with open(InRelease_file, 'r') as InRelease: + hash_list = InRelease.readlines() + for i in range(len(hash_list)): + if hash_list[i].startswith(hash+':'): + break + for j in range(i, len(hash_list)): + if string_to_found in hash_list[j].strip(' '): + print(hash_list[j].strip(' ').split(' ')[0]) + break + +if __name__ == '__main__': + if len(sys.argv) < 2: + sys.exit('Too few arguments, I need the path to a Packages file and a package name. Exiting') + + if sys.argv[1].endswith('Packages'): + get_pkg_hash_from_Packages(sys.argv[1], sys.argv[2]) + elif sys.argv[1].endswith('InRelease'): + get_Packages_hash_from_InRelease(sys.argv[1], sys.argv[2]) + else: + sys.exit(sys.argv[1]+' does not seem to be a path to a Packages or InRelease file') diff --git a/scripts/get_pkg_hash.py b/scripts/get_pkg_hash.py deleted file mode 100755 index 668392601..000000000 --- a/scripts/get_pkg_hash.py +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env python - -import os, sys - -def get_hash_from_Packages(Packages_file, package, hash="SHA256"): - with open(Packages_file, 'r') as Packages: - package_list = Packages.read().split('\n\n') - for pkg in package_list: - if pkg.split('\n')[0] == "Package: "+package: - for line in pkg.split('\n'): - if line.startswith(hash): - print(line.split(" ")[1]) - break - break - -if __name__ == '__main__': - if len(sys.argv) < 2: - print('Too few arguments, I need the path to a Packages file and a package name. Exiting') - sys.exit(1) - get_hash_from_Packages(sys.argv[1], sys.argv[2]) From cad78d6094379b96da976fd05f4371a228a71042 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Tue, 1 Jan 2019 21:11:37 +0100 Subject: [PATCH 71/99] build-all.sh: add -i option to build and download dependencies --- build-all.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/build-all.sh b/build-all.sh index 1fd8d64c1..e5fd04943 100755 --- a/build-all.sh +++ b/build-all.sh @@ -8,20 +8,23 @@ test -f $HOME/.termuxrc && . $HOME/.termuxrc : ${TERMUX_TOPDIR:="$HOME/.termux-build"} : ${TERMUX_ARCH:="aarch64"} : ${TERMUX_DEBUG:=""} +: ${TERMUX_INSTALL_DEPS:=""} _show_usage () { - echo "Usage: ./build-all.sh [-a ARCH] [-d] [-o DIR]" + echo "Usage: ./build-all.sh [-a ARCH] [-d] [-i] [-o DIR]" echo "Build all packages." echo " -a The architecture to build for: aarch64(default), arm, i686, x86_64 or all." echo " -d Build with debug symbols." + echo " -i Build dependencies." echo " -o Specify deb directory. Default: debs/." exit 1 } -while getopts :a:hdDso: option; do +while getopts :a:hdio: option; do case "$option" in a) TERMUX_ARCH="$OPTARG";; d) TERMUX_DEBUG='-d';; + i) TERMUX_INSTALL_DEPS='-i';; o) TERMUX_DEBDIR="$(realpath -m $OPTARG)";; h) _show_usage;; esac @@ -63,8 +66,8 @@ for package_path in `cat $BUILDORDER_FILE`; do echo -n "Building $package... " BUILD_START=`date "+%s"` - bash -x $BUILDSCRIPT -a $TERMUX_ARCH -s \ - $TERMUX_DEBUG ${TERMUX_DEBDIR+-o $TERMUX_DEBDIR} $package \ + bash -x $BUILDSCRIPT -a $TERMUX_ARCH -s $TERMUX_DEBUG \ + ${TERMUX_DEBDIR+-o $TERMUX_DEBDIR} $TERMUX_INSTALL_DEPS $package \ > $BUILDALL_DIR/${package}.out 2> $BUILDALL_DIR/${package}.err BUILD_END=`date "+%s"` BUILD_SECONDS=$(( $BUILD_END - $BUILD_START )) From bdf42387bcf640e8659cad6070e747e65da43d83 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Tue, 1 Jan 2019 21:54:05 +0100 Subject: [PATCH 72/99] build-all.sh: don't set -s if -i is set --- build-all.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build-all.sh b/build-all.sh index e5fd04943..0b2cf76ae 100755 --- a/build-all.sh +++ b/build-all.sh @@ -8,7 +8,8 @@ test -f $HOME/.termuxrc && . $HOME/.termuxrc : ${TERMUX_TOPDIR:="$HOME/.termux-build"} : ${TERMUX_ARCH:="aarch64"} : ${TERMUX_DEBUG:=""} -: ${TERMUX_INSTALL_DEPS:=""} +: ${TERMUX_INSTALL_DEPS:="-s"} +# Set TERMUX_INSTALL_DEPS to -s unless set to -i _show_usage () { echo "Usage: ./build-all.sh [-a ARCH] [-d] [-i] [-o DIR]" @@ -66,7 +67,7 @@ for package_path in `cat $BUILDORDER_FILE`; do echo -n "Building $package... " BUILD_START=`date "+%s"` - bash -x $BUILDSCRIPT -a $TERMUX_ARCH -s $TERMUX_DEBUG \ + bash -x $BUILDSCRIPT -a $TERMUX_ARCH $TERMUX_DEBUG \ ${TERMUX_DEBDIR+-o $TERMUX_DEBDIR} $TERMUX_INSTALL_DEPS $package \ > $BUILDALL_DIR/${package}.out 2> $BUILDALL_DIR/${package}.err BUILD_END=`date "+%s"` From 2a9a8ab4f93484d1c5b90a2280e87d82016617bb Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Tue, 1 Jan 2019 21:56:42 +0100 Subject: [PATCH 73/99] build-package.sh: mv download of InRelease and Packages to new function termux_step_get_repo_files --- build-package.sh | 53 +++++++++++++++++++++++++++--------------------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/build-package.sh b/build-package.sh index 656eb64d1..5ee02aecb 100755 --- a/build-package.sh +++ b/build-package.sh @@ -471,25 +471,11 @@ termux_download_deb() { return 0 } -# Source the package build script and start building. No to be overridden by packages. -termux_step_start_build() { - # shellcheck source=/dev/null - source "$TERMUX_PKG_BUILDER_SCRIPT" - - TERMUX_STANDALONE_TOOLCHAIN="$TERMUX_COMMON_CACHEDIR/${TERMUX_NDK_VERSION}-${TERMUX_ARCH}-${TERMUX_PKG_API_LEVEL}" - # Bump the below version if a change is made in toolchain setup to ensure - # that everyone gets an updated toolchain: - TERMUX_STANDALONE_TOOLCHAIN+="-v4" - - if [ -n "${TERMUX_PKG_BLACKLISTED_ARCHES:=""}" ] && [ "$TERMUX_PKG_BLACKLISTED_ARCHES" != "${TERMUX_PKG_BLACKLISTED_ARCHES/$TERMUX_ARCH/}" ]; then - echo "Skipping building $TERMUX_PKG_NAME for arch $TERMUX_ARCH" - exit 0 - fi - - local TERMUX_ALL_DEPS=$(./scripts/buildorder.py "$TERMUX_PKG_BUILDER_DIR") +# Script to download InRelease, verify it's signature and then download Packages.xz by hash +termux_step_get_repo_files() { if [ "$TERMUX_SKIP_DEPCHECK" = false ] && [ "$TERMUX_INSTALL_DEPS" = true ]; then # Remove all previously extracted/built files from $TERMUX_PREFIX: - rm -r $TERMUX_PREFIX + rm -rf $TERMUX_PREFIX rm -f /data/data/.built-packages/* # Ensure folders present (but not $TERMUX_PKG_SRCDIR, it will be created in build) mkdir -p "$TERMUX_COMMON_CACHEDIR" \ @@ -523,13 +509,32 @@ termux_step_start_build() { gpg --verify InRelease ) for arch in all $TERMUX_ARCH; do - local packages_hash=$(./scripts/get_hash_from_file.py ${TERMUX_COMMON_CACHEDIR}/InRelease $arch) - termux_download "$TERMUX_REPO_URL/$TERMUX_REPO_DISTRIBUTION/$TERMUX_REPO_COMPONENT/binary-$arch/Packages.xz" \ - "${TERMUX_COMMON_CACHEDIR}-$arch/Packages.xz" \ - $packages_hash - xz -df "${TERMUX_COMMON_CACHEDIR}-$arch/Packages.xz" + local packages_hash=$(./scripts/get_hash_from_file.py ${TERMUX_COMMON_CACHEDIR}/InRelease $arch) + termux_download "$TERMUX_REPO_URL/$TERMUX_REPO_DISTRIBUTION/$TERMUX_REPO_COMPONENT/binary-$arch/Packages.xz" \ + "${TERMUX_COMMON_CACHEDIR}-$arch/Packages.xz" \ + $packages_hash + xz -df "${TERMUX_COMMON_CACHEDIR}-$arch/Packages.xz" done + fi +} + +# Source the package build script and start building. No to be overridden by packages. +termux_step_start_build() { + # shellcheck source=/dev/null + source "$TERMUX_PKG_BUILDER_SCRIPT" + + TERMUX_STANDALONE_TOOLCHAIN="$TERMUX_COMMON_CACHEDIR/${TERMUX_NDK_VERSION}-${TERMUX_ARCH}-${TERMUX_PKG_API_LEVEL}" + # Bump the below version if a change is made in toolchain setup to ensure + # that everyone gets an updated toolchain: + TERMUX_STANDALONE_TOOLCHAIN+="-v4" + if [ -n "${TERMUX_PKG_BLACKLISTED_ARCHES:=""}" ] && [ "$TERMUX_PKG_BLACKLISTED_ARCHES" != "${TERMUX_PKG_BLACKLISTED_ARCHES/$TERMUX_ARCH/}" ]; then + echo "Skipping building $TERMUX_PKG_NAME for arch $TERMUX_ARCH" + exit 0 + fi + + local TERMUX_ALL_DEPS=$(./scripts/buildorder.py "$TERMUX_PKG_BUILDER_DIR") + if [ "$TERMUX_SKIP_DEPCHECK" = false ] && [ "$TERMUX_INSTALL_DEPS" = true ]; then # Download dependencies local pkg dep_arch dep_version deb_file for pkg in $TERMUX_ALL_DEPS; do @@ -1436,8 +1441,9 @@ termux_step_create_debfile() { termux_step_compare_debs() { if [ "${TERMUX_INSTALL_DEPS}" = true ]; then local arch version - if [ ! "$TERMUX_QUIET_BUILD" = true ]; then echo "COMPARING PACKAGES"; fi cd ${TERMUX_SCRIPTDIR} + if [ ! "$TERMUX_QUIET_BUILD" = true ]; then echo "COMPARING PACKAGES"; fi + termux_download_deb $(basename $TERMUX_PKG_BUILDER_DIR) $TERMUX_ARCH $TERMUX_PKG_FULLVERSION deb_file=${TERMUX_PKG_NAME}_${TERMUX_PKG_FULLVERSION}_${TERMUX_ARCH}.deb @@ -1459,6 +1465,7 @@ termux_step_finish_build() { termux_step_handle_arguments "$@" termux_step_setup_variables termux_step_handle_buildarch +termux_step_get_repo_files termux_step_start_build termux_step_extract_package cd "$TERMUX_PKG_SRCDIR" From cefd4e472be2469147a70ee0e533c80829159617 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sun, 20 Jan 2019 17:19:07 +0100 Subject: [PATCH 74/99] build-package.sh: fix termux_extract_dep_info --- build-package.sh | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/build-package.sh b/build-package.sh index 5ee02aecb..baa6c8116 100755 --- a/build-package.sh +++ b/build-package.sh @@ -435,18 +435,35 @@ termux_step_handle_buildarch() { # Function to get TERMUX_PKG_VERSION from build.sh termux_extract_dep_info() { package=$1 + if [ ! -d packages/$package ] && [ packages/${package} == packages/${package/-dev/} ]; then + # We are probably dealing with a subpackage + TERMUX_ARCH=$( + # set TERMUX_SUBPKG_PLATFORM_INDEPENDENT to mother package's value and override if needed + TERMUX_PKG_PLATFORM_INDEPENDENT="" + source $(dirname $(find packages/ -name "$package.subpackage.sh"))/build.sh + TERMUX_SUBPKG_PLATFORM_INDEPENDENT=$TERMUX_PKG_PLATFORM_INDEPENDENT + source $(find packages/ -name "$package.subpackage.sh") + if [ "$TERMUX_SUBPKG_PLATFORM_INDEPENDENT" = yes ]; then + echo all + else + echo $TERMUX_ARCH + fi + ) + + package=$(basename $(dirname $(find packages/ -name "$package.subpackage.sh"))) + elif [ ! packages/${package} == packages/${package/-dev/} ]; then + # dev package + package=${package/-dev/} + fi ( # Reset TERMUX_PKG_PLATFORM_INDEPENDENT and TERMUX_PKG_REVISION since these aren't # mandatory in a build.sh. Otherwise these will equal the main package's values for # deps that should have the default values TERMUX_PKG_PLATFORM_INDEPENDENT="" TERMUX_PKG_REVISION="0" - - source $package/build.sh - if [ "$TERMUX_PKG_PLATFORM_INDEPENDENT" = yes ]; then - TERMUX_ARCH=all - fi - if [ ! "$TERMUX_PKG_REVISION" = 0 ]; then + source packages/$package/build.sh + if [ "$TERMUX_PKG_PLATFORM_INDEPENDENT" = yes ]; then TERMUX_ARCH=all; fi + if [ "$TERMUX_PKG_REVISION" != "0" ] || [ "$TERMUX_PKG_VERSION" != "${TERMUX_PKG_VERSION/-/}" ]; then TERMUX_PKG_VERSION+="-$TERMUX_PKG_REVISION" fi echo ${TERMUX_ARCH} ${TERMUX_PKG_VERSION} From 23a21c9ed4df0f9f16f836bd32767daa16728738 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sun, 20 Jan 2019 17:51:21 +0100 Subject: [PATCH 75/99] build-package.sh: return 1 if hashsum doesn't exist for package When downloading depencencies --- build-package.sh | 16 +++++++--------- scripts/get_hash_from_file.py | 14 +++++++++----- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/build-package.sh b/build-package.sh index baa6c8116..55e4cad2c 100755 --- a/build-package.sh +++ b/build-package.sh @@ -475,17 +475,15 @@ termux_download_deb() { local package_arch=$2 local version=$3 local deb_file=${package}_${version}_${package_arch}.deb - local pkg_hash=$(./scripts/get_hash_from_file.py ${TERMUX_COMMON_CACHEDIR}-${package_arch}/Packages $package) - + local pkg_hash=$(./scripts/get_hash_from_file.py ${TERMUX_COMMON_CACHEDIR}-${package_arch}/Packages $package $version) if [ "$pkg_hash" = "" ]; then - # No hash found for $package return 1 + else + termux_download $TERMUX_REPO_URL/$TERMUX_REPO_DISTRIBUTION/$TERMUX_REPO_COMPONENT/binary-${package_arch}/${deb_file} \ + $TERMUX_COMMON_CACHEDIR-$package_arch/${deb_file} \ + $pkg_hash + return 0 fi - - termux_download $TERMUX_REPO_URL/$TERMUX_REPO_DISTRIBUTION/$TERMUX_REPO_COMPONENT/binary-${package_arch}/${deb_file} \ - $TERMUX_COMMON_CACHEDIR-$package_arch/${deb_file} \ - $pkg_hash - return 0 } # Script to download InRelease, verify it's signature and then download Packages.xz by hash @@ -522,7 +520,7 @@ termux_step_get_repo_files() { curl --fail -LO "$TERMUX_REPO_URL/$TERMUX_REPO_DISTRIBUTION/InRelease" \ || termux_error_exit "Download of $TERMUX_REPO_URL/$TERMUX_REPO_DISTRIBUTION/InRelease failed" # Import Fornwalls key: - gpg --recv $TERMUX_REPO_SIGNING_KEY + gpg -k $TERMUX_REPO_SIGNING_KEY 2>/dev/null || gpg --recv $TERMUX_REPO_SIGNING_KEY gpg --verify InRelease ) for arch in all $TERMUX_ARCH; do diff --git a/scripts/get_hash_from_file.py b/scripts/get_hash_from_file.py index 4d355832c..b34276c7a 100755 --- a/scripts/get_hash_from_file.py +++ b/scripts/get_hash_from_file.py @@ -2,16 +2,20 @@ import os, sys -def get_pkg_hash_from_Packages(Packages_file, package, hash="SHA256"): +def get_pkg_hash_from_Packages(Packages_file, package, version, hash="SHA256"): with open(Packages_file, 'r') as Packages: package_list = Packages.read().split('\n\n') for pkg in package_list: if pkg.split('\n')[0] == "Package: "+package: for line in pkg.split('\n'): - if line.startswith(hash): + if line.startswith('Version:'): + if line != 'Version: '+version: + # Seems the repo contains the wrong version, or several versions + # We can't use this one so continue looking + break + elif line.startswith(hash): print(line.split(" ")[1]) break - break def get_Packages_hash_from_InRelease(InRelease_file, arch, hash="SHA256"): string_to_found = 'binary-'+arch+'/Packages.xz' @@ -27,10 +31,10 @@ def get_Packages_hash_from_InRelease(InRelease_file, arch, hash="SHA256"): if __name__ == '__main__': if len(sys.argv) < 2: - sys.exit('Too few arguments, I need the path to a Packages file and a package name. Exiting') + sys.exit('Too few arguments, I need the path to a Packages file, a package name and a version, or an InRelease file and an architecture. Exiting') if sys.argv[1].endswith('Packages'): - get_pkg_hash_from_Packages(sys.argv[1], sys.argv[2]) + get_pkg_hash_from_Packages(sys.argv[1], sys.argv[2], sys.argv[3]) elif sys.argv[1].endswith('InRelease'): get_Packages_hash_from_InRelease(sys.argv[1], sys.argv[2]) else: From 11be84612f154ec965f756debab5ffc44dd5ca11 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sun, 20 Jan 2019 17:52:47 +0100 Subject: [PATCH 76/99] build-package.sh: add sha256sum's for bootstrap zips for all arches --- build-package.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/build-package.sh b/build-package.sh index 55e4cad2c..2f736460c 100755 --- a/build-package.sh +++ b/build-package.sh @@ -504,9 +504,18 @@ termux_step_get_repo_files() { "$TERMUX_PKG_MASSAGEDIR" \ $TERMUX_PREFIX/{bin,etc,lib,libexec,share,tmp,include} # Setup bootstrap + if [ $TERMUX_ARCH == aarch64 ]; then + local bootstrap_sha256=9f43bfdd23fbc6ae02f1f5a44b8e064374db8291ffe32658e2724d3f3f419981 + elif [ $TERMUX_ARCH == i686 ]; then + local bootstrap_sha256=4f30d49f8bdb9a294d6e4e0dea6b5cb7b5cd74163531a04659d043b49dc08737 + elif [ $TERMUX_ARCH == arm ]; then + local bootstrap_sha256=2be38e07f93ca3f960ccb5554e67d0860e921b283ccba0aab5527a53b9e97b10 + elif [ $TERMUX_ARCH == x86_64 ]; then + local bootstrap_sha256=71b334dd3901b8f9c2a8905208912419963700f93b5b7cd68213010d05a662a7 + fi termux_download https://termux.net/bootstrap/bootstrap-${TERMUX_ARCH}.zip \ ${TERMUX_COMMON_CACHEDIR}/bootstrap-${TERMUX_ARCH}.zip \ - 9f43bfdd23fbc6ae02f1f5a44b8e064374db8291ffe32658e2724d3f3f419981 + $bootstrap_sha256 unzip -qo ${TERMUX_COMMON_CACHEDIR}/bootstrap-${TERMUX_ARCH}.zip -d $TERMUX_PREFIX ( cd $TERMUX_PREFIX From 51891e235b45254eb281b2a6f8ea19de2f4ac2f6 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sun, 20 Jan 2019 17:56:05 +0100 Subject: [PATCH 77/99] build-package.sh: fix download loop of dependencies when -i building Only download $TERMUX_PKG_DEPENDS and $TERMUX_PKG_BUILD_DEPENDS (and their -dev packages, if they exist), not their dependencies. --- build-package.sh | 56 +++++++++++++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 22 deletions(-) diff --git a/build-package.sh b/build-package.sh index 2f736460c..1719cd27f 100755 --- a/build-package.sh +++ b/build-package.sh @@ -557,33 +557,45 @@ termux_step_start_build() { exit 0 fi - local TERMUX_ALL_DEPS=$(./scripts/buildorder.py "$TERMUX_PKG_BUILDER_DIR") + local TERMUX_ALL_DEPS=$(./scripts/buildorder.py "$TERMUX_PKG_BUILDER_DIR" | sed 's%packages/%%g') if [ "$TERMUX_SKIP_DEPCHECK" = false ] && [ "$TERMUX_INSTALL_DEPS" = true ]; then # Download dependencies - local pkg dep_arch dep_version deb_file - for pkg in $TERMUX_ALL_DEPS; do + local pkg dep_arch dep_version deb_file _PKG_DEPENDS _PKG_BUILD_DEPENDS + # remove (>= 1.0) and similar version tags with sed: + _PKG_DEPENDS=$(echo ${TERMUX_PKG_DEPENDS//,/ } | sed "s/[(][^)]*[)]//g") + _PKG_BUILD_DEPENDS=${TERMUX_PKG_BUILD_DEPENDS//,/ } + for pkg in $_PKG_DEPENDS $_PKG_BUILD_DEPENDS; do + # llvm doesn't build if ndk-sysroot is installed: + if [ "$pkg" = "ndk-sysroot" ]; then continue; fi read dep_arch dep_version <<< $(termux_extract_dep_info "$pkg") + if [ ! "$TERMUX_QUIET_BUILD" = true ]; then - echo "Downloading dependency $(basename $pkg)@$dep_version if necessary..." + echo "Downloading dependency $pkg@$dep_version if necessary..." + fi + if ! termux_download_deb $pkg $dep_arch $dep_version; then + echo "Download of $pkg@$dep_version from $TERMUX_REPO_URL failed, building instead" + ./build-package.sh -a $TERMUX_ARCH -i "$pkg" + continue + else + if [ ! "$TERMUX_QUIET_BUILD" = true ]; then echo "Extracting $pkg..."; fi + ( + cd $TERMUX_COMMON_CACHEDIR-$dep_arch + ar x ${pkg}_${dep_version}_${dep_arch}.deb data.tar.xz + tar -xf data.tar.xz --no-overwrite-dir -C / + ) + fi + + if termux_download_deb $pkg-dev $dep_arch $dep_version; then + ( + cd $TERMUX_COMMON_CACHEDIR-$dep_arch + ar x $pkg-dev_${dep_version}_${dep_arch}.deb data.tar.xz + tar xf data.tar.xz --no-overwrite-dir -C / + ) + else + echo "Download of $pkg-dev@$dep_version from $TERMUX_REPO_URL failed" fi - termux_download_deb $(basename $pkg) $dep_arch $dep_version \ - || ( echo "Download of $(basename $pkg)@$dep_version from $TERMUX_REPO_URL failed, building instead" \ - && ./build-package.sh -a $TERMUX_ARCH -s "$pkg" \ - && continue ) - local deb_file=$(basename $pkg)_${dep_version}_${dep_arch}.deb - if [ ! "$TERMUX_QUIET_BUILD" = true ]; then echo "Extracting $(basename $pkg)..."; fi - ( - cd $TERMUX_COMMON_CACHEDIR-$dep_arch - ar x ${deb_file} data.tar.xz && tar xf data.tar.xz --no-overwrite-dir -C / - ) - - termux_download_deb $(basename $pkg)-dev $dep_arch $dep_version && \ - ( - cd $TERMUX_COMMON_CACHEDIR-$dep_arch - ar x $(basename $pkg)-dev_${dep_version}_${dep_arch}.deb data.tar.xz - tar xf data.tar.xz --no-overwrite-dir -C / - ) || echo "Download of $(basename $pkg)-dev@$dep_version from $TERMUX_REPO_URL failed" - echo "$dep_version" > "/data/data/.built-packages/$(basename $pkg)" + mkdir -p /data/data/.built-packages + echo "$dep_version" > "/data/data/.built-packages/$pkg" done elif [ "$TERMUX_SKIP_DEPCHECK" = false ] && [ "$TERMUX_INSTALL_DEPS" = false ]; then # Build dependencies From 26048535039d065d365789c2cb47d0ae3b844a80 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sun, 20 Jan 2019 17:58:36 +0100 Subject: [PATCH 78/99] build-package.sh: fix termux_step_compare_debs Catch error if termux_download_deb fails --- build-package.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/build-package.sh b/build-package.sh index 1719cd27f..9ee240ec2 100755 --- a/build-package.sh +++ b/build-package.sh @@ -1476,16 +1476,17 @@ termux_step_create_debfile() { termux_step_compare_debs() { if [ "${TERMUX_INSTALL_DEPS}" = true ]; then - local arch version cd ${TERMUX_SCRIPTDIR} if [ ! "$TERMUX_QUIET_BUILD" = true ]; then echo "COMPARING PACKAGES"; fi - termux_download_deb $(basename $TERMUX_PKG_BUILDER_DIR) $TERMUX_ARCH $TERMUX_PKG_FULLVERSION - deb_file=${TERMUX_PKG_NAME}_${TERMUX_PKG_FULLVERSION}_${TERMUX_ARCH}.deb + termux_download_deb $TERMUX_PKG_NAME $TERMUX_ARCH $TERMUX_PKG_FULLVERSION \ + && ( + deb_file=${TERMUX_PKG_NAME}_${TERMUX_PKG_FULLVERSION}_${TERMUX_ARCH}.deb - # `|| true` to prevent debdiff's exit code from stopping build - debdiff $TERMUX_DEBDIR/$deb_file $TERMUX_COMMON_CACHEDIR-$TERMUX_ARCH/$deb_file || true - if [ ! "$TERMUX_QUIET_BUILD" = true ]; then echo "DONE COMPARING PACKAGES"; fi + # `|| true` to prevent debdiff's exit code from stopping build + debdiff $TERMUX_DEBDIR/$deb_file $TERMUX_COMMON_CACHEDIR-$TERMUX_ARCH/$deb_file || true + if [ ! "$TERMUX_QUIET_BUILD" = true ]; then echo "DONE COMPARING PACKAGES"; fi + ) || echo "Download of ${TERMUX_PKG_NAME}@${TERMUX_PKG_FULLVERSION} failed, not comparing debs" fi } From 47356f9601555672421ad123acfb22c65f643aba Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sun, 20 Jan 2019 18:00:37 +0100 Subject: [PATCH 79/99] build-package.sh: Minor indentation fix --- build-package.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build-package.sh b/build-package.sh index 9ee240ec2..b1c541702 100755 --- a/build-package.sh +++ b/build-package.sh @@ -410,7 +410,7 @@ termux_step_handle_buildarch() { local TERMUX_PREVIOUS_ARCH TERMUX_PREVIOUS_ARCH=$(cat $TERMUX_ARCH_FILE) if [ "$TERMUX_PREVIOUS_ARCH" != "$TERMUX_ARCH" ]; then - local TERMUX_DATA_BACKUPDIRS=$TERMUX_TOPDIR/_databackups + local TERMUX_DATA_BACKUPDIRS=$TERMUX_TOPDIR/_databackups mkdir -p "$TERMUX_DATA_BACKUPDIRS" local TERMUX_DATA_PREVIOUS_BACKUPDIR=$TERMUX_DATA_BACKUPDIRS/$TERMUX_PREVIOUS_ARCH local TERMUX_DATA_CURRENT_BACKUPDIR=$TERMUX_DATA_BACKUPDIRS/$TERMUX_ARCH @@ -642,7 +642,7 @@ termux_step_start_build() { # Ensure folders present (but not $TERMUX_PKG_SRCDIR, it will be created in build) mkdir -p "$TERMUX_COMMON_CACHEDIR" \ - "$TERMUX_DEBDIR" \ + "$TERMUX_DEBDIR" \ "$TERMUX_PKG_BUILDDIR" \ "$TERMUX_PKG_PACKAGEDIR" \ "$TERMUX_PKG_TMPDIR" \ From 5b863102c3fd33c62f55465d613dafc6c824459a Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sun, 20 Jan 2019 18:20:56 +0100 Subject: [PATCH 80/99] build-package.sh: fix bug in the case where package name contains (but not ends with) -dev As for icu-devtools --- build-package.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build-package.sh b/build-package.sh index b1c541702..30cc2f134 100755 --- a/build-package.sh +++ b/build-package.sh @@ -435,8 +435,8 @@ termux_step_handle_buildarch() { # Function to get TERMUX_PKG_VERSION from build.sh termux_extract_dep_info() { package=$1 - if [ ! -d packages/$package ] && [ packages/${package} == packages/${package/-dev/} ]; then - # We are probably dealing with a subpackage + if [ ! -d packages/$package ] && [ -f packages/*/${package}.subpackage.sh ]; then + # We are dealing with a subpackage TERMUX_ARCH=$( # set TERMUX_SUBPKG_PLATFORM_INDEPENDENT to mother package's value and override if needed TERMUX_PKG_PLATFORM_INDEPENDENT="" @@ -451,7 +451,7 @@ termux_extract_dep_info() { ) package=$(basename $(dirname $(find packages/ -name "$package.subpackage.sh"))) - elif [ ! packages/${package} == packages/${package/-dev/} ]; then + elif [ "${package/-dev/}-dev" == "${package}" ]; then # dev package package=${package/-dev/} fi From 34ca9dce21fdd5cc25aee5bf0d382c432d05fd64 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sun, 20 Jan 2019 18:32:49 +0100 Subject: [PATCH 81/99] build-package.sh: improve -i description --- build-package.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-package.sh b/build-package.sh index 30cc2f134..011a2e264 100755 --- a/build-package.sh +++ b/build-package.sh @@ -254,7 +254,7 @@ termux_step_handle_arguments() { echo " -d Build with debug symbols." echo " -D Build a disabled package in disabled-packages/." echo " -f Force build even if package has already been built." - echo " -i Build dependencies." + echo " -i Download and extract dependencies instead of building them." echo " -q Quiet build." echo " -s Skip dependency check." echo " -o Specify deb directory. Default: debs/." From 33643d4bc8cc0cc49ab9621abe5d074d72d24436 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Fri, 25 Jan 2019 22:09:57 +0100 Subject: [PATCH 82/99] build-package.sh: fix `-a all -i` builds Also use `-s -i` for dependencies that have to be built --- build-package.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build-package.sh b/build-package.sh index 011a2e264..09a360c1f 100755 --- a/build-package.sh +++ b/build-package.sh @@ -282,7 +282,7 @@ termux_step_handle_arguments() { # Handle 'all' arch: if [ -n "${TERMUX_ARCH+x}" ] && [ "${TERMUX_ARCH}" = 'all' ]; then for arch in 'aarch64' 'arm' 'i686' 'x86_64'; do - ./build-package.sh ${TERMUX_FORCE_BUILD+-f} -a $arch \ + ./build-package.sh ${TERMUX_FORCE_BUILD+-f} -a $arch ${TERMUX_INSTALL_DEPS+-i} \ ${TERMUX_DEBUG+-d} ${TERMUX_DEBDIR+-o $TERMUX_DEBDIR} "$1" done exit @@ -574,7 +574,7 @@ termux_step_start_build() { fi if ! termux_download_deb $pkg $dep_arch $dep_version; then echo "Download of $pkg@$dep_version from $TERMUX_REPO_URL failed, building instead" - ./build-package.sh -a $TERMUX_ARCH -i "$pkg" + ./build-package.sh -a $TERMUX_ARCH -i -s "$pkg" continue else if [ ! "$TERMUX_QUIET_BUILD" = true ]; then echo "Extracting $pkg..."; fi From 350ce1fb06302d9b495e9d0bf2b1a97d1a1c218e Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Fri, 25 Jan 2019 22:29:51 +0100 Subject: [PATCH 83/99] Revert "Update build.sh" This reverts commit eb85a5b4facae36817e5d23d15b79da0036ca7be. --- packages/php/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/php/build.sh b/packages/php/build.sh index fedea2f52..8bb400087 100644 --- a/packages/php/build.sh +++ b/packages/php/build.sh @@ -8,7 +8,7 @@ TERMUX_PKG_SRCURL=https://secure.php.net/distributions/php-${TERMUX_PKG_VERSION} TERMUX_PKG_HOSTBUILD=true # Build the native php without xml support as we only need phar: TERMUX_PKG_EXTRA_HOSTBUILD_CONFIGURE_ARGS="--disable-libxml --disable-dom --disable-simplexml --disable-xml --disable-xmlreader --disable-xmlwriter --without-pear" -TERMUX_PKG_DEPENDS="libandroid-glob, libxml2, liblzma, openssl, pcre, libbz2, libcrypt, libcurl, libgd, readline, freetype, apr, apr-util, openssl, libpng, libjpeg-turbo" +TERMUX_PKG_DEPENDS="libandroid-glob, libxml2, liblzma, openssl, pcre, libbz2, libcrypt, libcurl, libgd, readline, freetype" # mysql modules were initially shared libs TERMUX_PKG_CONFLICTS="php-mysql" TERMUX_PKG_REPLACES="php-mysql" From 7649877f9db88a9ff57563ab7478355b60156edf Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Fri, 25 Jan 2019 22:30:08 +0100 Subject: [PATCH 84/99] Revert "Update build.sh" This reverts commit f55b366ed2a8d43885f8e5fb9d67be7ec11c07e2. --- packages/php/build.sh | 8 -------- 1 file changed, 8 deletions(-) diff --git a/packages/php/build.sh b/packages/php/build.sh index 8bb400087..9dde61e2e 100644 --- a/packages/php/build.sh +++ b/packages/php/build.sh @@ -48,14 +48,6 @@ ac_cv_func_res_nsearch=no " termux_step_pre_configure () { - # Replace chrooted perl with system perl - sed -i 's/\/data\/data\/com\.termux\/files\/usr\/bin\/perl/\/usr\/bin\/perl/' $TERMUX_PREFIX/bin/apxs - sed -i 's/\/data\/data\/com\.termux\/files\/usr\/bin\/perl/\/usr\/bin\/perl/' $TERMUX_PREFIX/bin/curl-config - sed -i 's/\/home\/fornwall\/.termux-build\/apr\/tmp/\/home\/builder\/.termux-build\/php\/build/' $TERMUX_PREFIX/bin/apr-1-config - rm $TERMUX_PREFIX/bin/pg_config - - #because the new mariadb hides away all these includes inside server subdir - CFLAGS+=" -I$TERMUX_PREFIX/include/mysql/server -I$TERMUX_PREFIX/include/mysql" LDFLAGS+=" -landroid-glob -llog" export PATH=$PATH:$TERMUX_PKG_HOSTBUILD_DIR/sapi/cli/ From 7bf043aa86c3c1e2dbc7e5947c513e648206ffaa Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sat, 26 Jan 2019 01:04:30 +0100 Subject: [PATCH 85/99] Revert "Update build.sh" This reverts commit 865db6dd76f7349fa562daa14fc998ac25099c8a. --- packages/apache2/build.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/packages/apache2/build.sh b/packages/apache2/build.sh index fbfe7e037..d9a0bc36e 100644 --- a/packages/apache2/build.sh +++ b/packages/apache2/build.sh @@ -71,11 +71,6 @@ TERMUX_PKG_INCLUDE_IN_DEVPACKAGE="share/apache2/build" TERMUX_PKG_EXTRA_MAKE_ARGS="-s" termux_step_pre_configure () { - # address interpreter issues - sed -i 's/\/home\/fornwall\/.termux-build\/apr\/tmp\/libtool/\/usr\/bin\/libtool/' ${TERMUX_PREFIX}/bin/apr-1-config - sed -i 's/\/data\/data\/com.termux\/files\/usr\/bin\/bash/\/bin\/bash/' ${TERMUX_PREFIX}/share/apr-1/build/libtool - sed -i 's/\/home\/fornwall/\/home\/builder/' ${TERMUX_PREFIX}/share/apr-1/build/libtool - # remove old files rm -rf "$TERMUX_PREFIX"/{libexec,share,etc}/apache2 rm -rf "$TERMUX_PREFIX"/lib/cgi-bin From f32d3cb6488f20493fa16506bb04bc577c0fa18a Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sat, 26 Jan 2019 15:43:14 +0100 Subject: [PATCH 86/99] Add gnupg to docker image --- scripts/setup-ubuntu.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/setup-ubuntu.sh b/scripts/setup-ubuntu.sh index ae4b111ea..1c8102538 100755 --- a/scripts/setup-ubuntu.sh +++ b/scripts/setup-ubuntu.sh @@ -37,6 +37,7 @@ PACKAGES+=" libexpat1-dev" # Needed by ghostscript PACKAGES+=" libjpeg-dev" # Needed by ghostscript PACKAGES+=" gawk" # Needed by apr-util PACKAGES+=" libssl-dev" # Needed to build Rust +PACKAGES+=" gnupg" # Needed to verify downloaded .debs sudo DEBIAN_FRONTEND=noninteractive \ apt-get install -yq --no-install-recommends $PACKAGES From 4b6ffa7952d61e160473758eb20791b0d329b424 Mon Sep 17 00:00:00 2001 From: Leonid Plyushch Date: Sat, 26 Jan 2019 16:59:34 +0200 Subject: [PATCH 87/99] build-package.sh: use keyserver pool when receive keys Now gpg key retrieving is more reliable. --- build-package.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-package.sh b/build-package.sh index 09a360c1f..7edaf99ee 100755 --- a/build-package.sh +++ b/build-package.sh @@ -529,7 +529,7 @@ termux_step_get_repo_files() { curl --fail -LO "$TERMUX_REPO_URL/$TERMUX_REPO_DISTRIBUTION/InRelease" \ || termux_error_exit "Download of $TERMUX_REPO_URL/$TERMUX_REPO_DISTRIBUTION/InRelease failed" # Import Fornwalls key: - gpg -k $TERMUX_REPO_SIGNING_KEY 2>/dev/null || gpg --recv $TERMUX_REPO_SIGNING_KEY + gpg -k $TERMUX_REPO_SIGNING_KEY 2>/dev/null || gpg --keyserver pool.sks-keyservers.net --recv $TERMUX_REPO_SIGNING_KEY gpg --verify InRelease ) for arch in all $TERMUX_ARCH; do From f2b51dcfd98efd9e541a0b6c3f75269e8ccba676 Mon Sep 17 00:00:00 2001 From: Leonid Plyushch Date: Sat, 26 Jan 2019 17:02:21 +0200 Subject: [PATCH 88/99] setup-ubuntu.sh: install "devscripts" Makes utility "debdiff" available. --- scripts/setup-ubuntu.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/setup-ubuntu.sh b/scripts/setup-ubuntu.sh index 1c8102538..160c92b75 100755 --- a/scripts/setup-ubuntu.sh +++ b/scripts/setup-ubuntu.sh @@ -38,6 +38,7 @@ PACKAGES+=" libjpeg-dev" # Needed by ghostscript PACKAGES+=" gawk" # Needed by apr-util PACKAGES+=" libssl-dev" # Needed to build Rust PACKAGES+=" gnupg" # Needed to verify downloaded .debs +PACKAGES+=" devscripts" # Provides utility "debdiff". sudo DEBIAN_FRONTEND=noninteractive \ apt-get install -yq --no-install-recommends $PACKAGES From 159dcec267e6dafa998e449ce6faced4f2e0452f Mon Sep 17 00:00:00 2001 From: Leonid Plyushch Date: Sat, 26 Jan 2019 17:44:49 +0200 Subject: [PATCH 89/99] remove redundant things from CI configuration --- .travis.yml | 21 +++++---------------- Rakefile | 2 +- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/.travis.yml b/.travis.yml index d7501b49f..3a7737342 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,20 +4,9 @@ cache: bundler services: docker before_script: mkdir debs && chmod 777 debs env: - - TRAVIS_ARCH=aarch64 - - TRAVIS_ARCH=arm - - TRAVIS_ARCH=i686 - - TRAVIS_ARCH=x86_64 + matrix: + - ARGS="-i -a aarch64" + - ARGS="-i -a arm" + - ARGS="-i -a i686" + - ARGS="-i -a x86_64" script: bundle exec rake build["${ARGS}"] -jobs: - include: - - stage: Build aarch64 - env: ARGS="-q -a aarch64" - - env: ARGS="-q -i -a aarch64" - - stage: Build arm, i686, & x86_64 - env: ARGS="-q -a arm" - - env: ARGS="-q -a i686" - - env: ARGS="-q -a x86_64" - - env: ARGS="-q -i -a arm" - - env: ARGS="-q -i -a i686" - - env: ARGS="-q -i -a x86_64" diff --git a/Rakefile b/Rakefile index 08764a36b..408d0ece3 100644 --- a/Rakefile +++ b/Rakefile @@ -20,7 +20,7 @@ task :build, [:options] do |t, args| puts "Building #{pkg}" begin # Start blocking build loop - PTY.spawn("./scripts/run-docker.sh ./build-package.sh #{args[:options]} -a $TRAVIS_ARCH #{pkg}") do |stdout, stdin, pid| + PTY.spawn("./scripts/run-docker.sh ./build-package.sh #{args[:options]} #{pkg}") do |stdout, stdin, pid| begin stdout.sync stdout.each { |line| print line } From 3b34603b566a73c049d089beef8106829328caed Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sat, 26 Jan 2019 16:37:25 +0000 Subject: [PATCH 90/99] Use apt's trusted.gpg for pkg verification --- build-package.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build-package.sh b/build-package.sh index 7edaf99ee..545baa6c6 100755 --- a/build-package.sh +++ b/build-package.sh @@ -524,12 +524,12 @@ termux_step_get_repo_files() { done/dev/null || gpg --keyserver pool.sks-keyservers.net --recv $TERMUX_REPO_SIGNING_KEY gpg --verify InRelease ) for arch in all $TERMUX_ARCH; do From bfedc132eec42c6e6f79583ee7c2dcd440e719b6 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sun, 3 Feb 2019 15:02:06 +0100 Subject: [PATCH 91/99] Move termux_dep_extract_info to another function Can be used in bintray upload script to extract pkg version --- build-package.sh | 37 +-------------------------- scripts/termux_extract_dep_info.sh | 41 ++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 36 deletions(-) create mode 100644 scripts/termux_extract_dep_info.sh diff --git a/build-package.sh b/build-package.sh index 545baa6c6..006c76ec8 100755 --- a/build-package.sh +++ b/build-package.sh @@ -433,42 +433,7 @@ termux_step_handle_buildarch() { } # Function to get TERMUX_PKG_VERSION from build.sh -termux_extract_dep_info() { - package=$1 - if [ ! -d packages/$package ] && [ -f packages/*/${package}.subpackage.sh ]; then - # We are dealing with a subpackage - TERMUX_ARCH=$( - # set TERMUX_SUBPKG_PLATFORM_INDEPENDENT to mother package's value and override if needed - TERMUX_PKG_PLATFORM_INDEPENDENT="" - source $(dirname $(find packages/ -name "$package.subpackage.sh"))/build.sh - TERMUX_SUBPKG_PLATFORM_INDEPENDENT=$TERMUX_PKG_PLATFORM_INDEPENDENT - source $(find packages/ -name "$package.subpackage.sh") - if [ "$TERMUX_SUBPKG_PLATFORM_INDEPENDENT" = yes ]; then - echo all - else - echo $TERMUX_ARCH - fi - ) - - package=$(basename $(dirname $(find packages/ -name "$package.subpackage.sh"))) - elif [ "${package/-dev/}-dev" == "${package}" ]; then - # dev package - package=${package/-dev/} - fi - ( - # Reset TERMUX_PKG_PLATFORM_INDEPENDENT and TERMUX_PKG_REVISION since these aren't - # mandatory in a build.sh. Otherwise these will equal the main package's values for - # deps that should have the default values - TERMUX_PKG_PLATFORM_INDEPENDENT="" - TERMUX_PKG_REVISION="0" - source packages/$package/build.sh - if [ "$TERMUX_PKG_PLATFORM_INDEPENDENT" = yes ]; then TERMUX_ARCH=all; fi - if [ "$TERMUX_PKG_REVISION" != "0" ] || [ "$TERMUX_PKG_VERSION" != "${TERMUX_PKG_VERSION/-/}" ]; then - TERMUX_PKG_VERSION+="-$TERMUX_PKG_REVISION" - fi - echo ${TERMUX_ARCH} ${TERMUX_PKG_VERSION} - ) -} +source scripts/termux_extract_dep_info.sh termux_download_deb() { local package=$1 diff --git a/scripts/termux_extract_dep_info.sh b/scripts/termux_extract_dep_info.sh new file mode 100644 index 000000000..91b99c301 --- /dev/null +++ b/scripts/termux_extract_dep_info.sh @@ -0,0 +1,41 @@ +termux_extract_dep_info() { + package=$1 + if [ ! -d packages/$package ] && [ -f packages/*/${package}.subpackage.sh ]; then + # We are dealing with a subpackage + TERMUX_ARCH=$( + # set TERMUX_SUBPKG_PLATFORM_INDEPENDENT to mother package's value and override if needed + TERMUX_PKG_PLATFORM_INDEPENDENT="" + source $(dirname $(find packages/ -name "$package.subpackage.sh"))/build.sh + TERMUX_SUBPKG_PLATFORM_INDEPENDENT=$TERMUX_PKG_PLATFORM_INDEPENDENT + source $(find packages/ -name "$package.subpackage.sh") + if [ "$TERMUX_SUBPKG_PLATFORM_INDEPENDENT" = yes ]; then + echo all + else + echo $TERMUX_ARCH + fi + ) + + package=$(basename $(dirname $(find packages/ -name "$package.subpackage.sh"))) + elif [ "${package/-dev/}-dev" == "${package}" ]; then + # dev package + package=${package/-dev/} + fi + ( + # Reset TERMUX_PKG_PLATFORM_INDEPENDENT and TERMUX_PKG_REVISION since these aren't + # mandatory in a build.sh. Otherwise these will equal the main package's values for + # deps that should have the default values + TERMUX_PKG_PLATFORM_INDEPENDENT="" + TERMUX_PKG_REVISION="0" + source packages/$package/build.sh + if [ "$TERMUX_PKG_PLATFORM_INDEPENDENT" = yes ]; then TERMUX_ARCH=all; fi + if [ "$TERMUX_PKG_REVISION" != "0" ] || [ "$TERMUX_PKG_VERSION" != "${TERMUX_PKG_VERSION/-/}" ]; then + TERMUX_PKG_VERSION+="-$TERMUX_PKG_REVISION" + fi + echo ${TERMUX_ARCH} ${TERMUX_PKG_VERSION} + ) +} + +# Make script standalone executable as well as sourceable +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + termux_extract_dep_info "$@" +fi From 3bbfb7fce4002d0239ca7290905da28cd9d9401d Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Fri, 8 Feb 2019 20:15:34 +0000 Subject: [PATCH 92/99] Update bootstrap hashsums --- build-package.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build-package.sh b/build-package.sh index 9f129cdce..a305a5f81 100755 --- a/build-package.sh +++ b/build-package.sh @@ -476,13 +476,13 @@ termux_step_get_repo_files() { $TERMUX_PREFIX/{bin,etc,lib,libexec,share,tmp,include} # Setup bootstrap if [ $TERMUX_ARCH == aarch64 ]; then - local bootstrap_sha256=9f43bfdd23fbc6ae02f1f5a44b8e064374db8291ffe32658e2724d3f3f419981 + local bootstrap_sha256=2944ad699814329007d1f9c056e7c8323243c8b4a257cbd05904216f89fc3746 elif [ $TERMUX_ARCH == i686 ]; then - local bootstrap_sha256=4f30d49f8bdb9a294d6e4e0dea6b5cb7b5cd74163531a04659d043b49dc08737 + local bootstrap_sha256=8f4dee0b1e161689b60f330ac0cc813b56ab479f2cd789eb8459165a3be13bdb elif [ $TERMUX_ARCH == arm ]; then - local bootstrap_sha256=2be38e07f93ca3f960ccb5554e67d0860e921b283ccba0aab5527a53b9e97b10 + local bootstrap_sha256=f471c0af326677d87ca4926d54860d10d751dd4f8d615d5b1de902841601b41e elif [ $TERMUX_ARCH == x86_64 ]; then - local bootstrap_sha256=71b334dd3901b8f9c2a8905208912419963700f93b5b7cd68213010d05a662a7 + local bootstrap_sha256=93384f0343c13f604dbacd069276291bd7042fc6d42c6d7514c7e573d968c614 fi termux_download https://termux.net/bootstrap/bootstrap-${TERMUX_ARCH}.zip \ ${TERMUX_COMMON_CACHEDIR}/bootstrap-${TERMUX_ARCH}.zip \ From 71b37a31771413fadeec9a80d3787a11aace89ab Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sat, 9 Feb 2019 12:57:33 +0100 Subject: [PATCH 93/99] Fix so that depdencies can be built correctly with -i option --- build-package.sh | 79 +++++++++++++++++++++++++----------------------- 1 file changed, 42 insertions(+), 37 deletions(-) diff --git a/build-package.sh b/build-package.sh index a305a5f81..4252fcbd8 100755 --- a/build-package.sh +++ b/build-package.sh @@ -255,12 +255,13 @@ termux_step_handle_arguments() { echo " -D Build a disabled package in disabled-packages/." echo " -f Force build even if package has already been built." echo " -i Download and extract dependencies instead of building them." + echo " -I Download and extract dependencies instead of building them, keep existing files." echo " -q Quiet build." echo " -s Skip dependency check." echo " -o Specify deb directory. Default: debs/." exit 1 } - while getopts :a:hdDfiqso: option; do + while getopts :a:hdDfiIqso: option; do case "$option" in a) TERMUX_ARCH="$OPTARG";; h) _show_usage;; @@ -268,6 +269,7 @@ termux_step_handle_arguments() { D) local TERMUX_IS_DISABLED=true;; f) TERMUX_FORCE_BUILD=true;; i) export TERMUX_INSTALL_DEPS=true;; + I) export TERMUX_INSTALL_DEPS=true && export TERMUX_NO_CLEAN=true;; q) export TERMUX_QUIET_BUILD=true;; s) export TERMUX_SKIP_DEPCHECK=true;; o) TERMUX_DEBDIR="$(realpath -m $OPTARG)";; @@ -322,6 +324,7 @@ termux_step_setup_variables() { : "${TERMUX_ANDROID_HOME:="/data/data/com.termux/files/home"}" : "${TERMUX_DEBUG:=""}" : "${TERMUX_PKG_API_LEVEL:="21"}" + : "${TERMUX_NO_CLEAN:="true"}" : "${TERMUX_QUIET_BUILD:="false"}" : "${TERMUX_DEBDIR:="${TERMUX_SCRIPTDIR}/debs"}" : "${TERMUX_SKIP_DEPCHECK:="false"}" @@ -459,42 +462,44 @@ termux_download_deb() { # Script to download InRelease, verify it's signature and then download Packages.xz by hash termux_step_get_repo_files() { - if [ "$TERMUX_SKIP_DEPCHECK" = false ] && [ "$TERMUX_INSTALL_DEPS" = true ]; then - # Remove all previously extracted/built files from $TERMUX_PREFIX: - rm -rf $TERMUX_PREFIX - rm -f /data/data/.built-packages/* - # Ensure folders present (but not $TERMUX_PKG_SRCDIR, it will be created in build) - mkdir -p "$TERMUX_COMMON_CACHEDIR" \ - "$TERMUX_COMMON_CACHEDIR-$TERMUX_ARCH" \ - "$TERMUX_COMMON_CACHEDIR-all" \ - "$TERMUX_DEBDIR" \ - "$TERMUX_PKG_BUILDDIR" \ - "$TERMUX_PKG_PACKAGEDIR" \ - "$TERMUX_PKG_TMPDIR" \ - "$TERMUX_PKG_CACHEDIR" \ - "$TERMUX_PKG_MASSAGEDIR" \ - $TERMUX_PREFIX/{bin,etc,lib,libexec,share,tmp,include} - # Setup bootstrap - if [ $TERMUX_ARCH == aarch64 ]; then - local bootstrap_sha256=2944ad699814329007d1f9c056e7c8323243c8b4a257cbd05904216f89fc3746 - elif [ $TERMUX_ARCH == i686 ]; then - local bootstrap_sha256=8f4dee0b1e161689b60f330ac0cc813b56ab479f2cd789eb8459165a3be13bdb - elif [ $TERMUX_ARCH == arm ]; then - local bootstrap_sha256=f471c0af326677d87ca4926d54860d10d751dd4f8d615d5b1de902841601b41e - elif [ $TERMUX_ARCH == x86_64 ]; then - local bootstrap_sha256=93384f0343c13f604dbacd069276291bd7042fc6d42c6d7514c7e573d968c614 + if [ "$TERMUX_INSTALL_DEPS" = true ]; then + if [ "$TERMUX_NO_CLEAN" = false ]; then + # Remove all previously extracted/built files from $TERMUX_PREFIX: + rm -rf $TERMUX_PREFIX + rm -f /data/data/.built-packages/* + # Ensure folders present (but not $TERMUX_PKG_SRCDIR, it will be created in build) + mkdir -p "$TERMUX_COMMON_CACHEDIR" \ + "$TERMUX_COMMON_CACHEDIR-$TERMUX_ARCH" \ + "$TERMUX_COMMON_CACHEDIR-all" \ + "$TERMUX_DEBDIR" \ + "$TERMUX_PKG_BUILDDIR" \ + "$TERMUX_PKG_PACKAGEDIR" \ + "$TERMUX_PKG_TMPDIR" \ + "$TERMUX_PKG_CACHEDIR" \ + "$TERMUX_PKG_MASSAGEDIR" \ + $TERMUX_PREFIX/{bin,etc,lib,libexec,share,tmp,include} + # Setup bootstrap + if [ $TERMUX_ARCH == aarch64 ]; then + local bootstrap_sha256=2944ad699814329007d1f9c056e7c8323243c8b4a257cbd05904216f89fc3746 + elif [ $TERMUX_ARCH == i686 ]; then + local bootstrap_sha256=8f4dee0b1e161689b60f330ac0cc813b56ab479f2cd789eb8459165a3be13bdb + elif [ $TERMUX_ARCH == arm ]; then + local bootstrap_sha256=f471c0af326677d87ca4926d54860d10d751dd4f8d615d5b1de902841601b41e + elif [ $TERMUX_ARCH == x86_64 ]; then + local bootstrap_sha256=93384f0343c13f604dbacd069276291bd7042fc6d42c6d7514c7e573d968c614 + fi + termux_download https://termux.net/bootstrap/bootstrap-${TERMUX_ARCH}.zip \ + ${TERMUX_COMMON_CACHEDIR}/bootstrap-${TERMUX_ARCH}.zip \ + $bootstrap_sha256 + unzip -qo ${TERMUX_COMMON_CACHEDIR}/bootstrap-${TERMUX_ARCH}.zip -d $TERMUX_PREFIX + ( + cd $TERMUX_PREFIX + while read link; do + ln -sf ${link/←/ } + done Date: Sun, 10 Feb 2019 09:13:33 +0100 Subject: [PATCH 94/99] build-package.sh: always create directories, if necessary --- build-package.sh | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/build-package.sh b/build-package.sh index 4252fcbd8..6c16d9eff 100755 --- a/build-package.sh +++ b/build-package.sh @@ -462,22 +462,22 @@ termux_download_deb() { # Script to download InRelease, verify it's signature and then download Packages.xz by hash termux_step_get_repo_files() { + # Ensure folders present (but not $TERMUX_PKG_SRCDIR, it will be created in build) + mkdir -p "$TERMUX_COMMON_CACHEDIR" \ + "$TERMUX_COMMON_CACHEDIR-$TERMUX_ARCH" \ + "$TERMUX_COMMON_CACHEDIR-all" \ + "$TERMUX_DEBDIR" \ + "$TERMUX_PKG_BUILDDIR" \ + "$TERMUX_PKG_PACKAGEDIR" \ + "$TERMUX_PKG_TMPDIR" \ + "$TERMUX_PKG_CACHEDIR" \ + "$TERMUX_PKG_MASSAGEDIR" \ + $TERMUX_PREFIX/{bin,etc,lib,libexec,share,tmp,include} if [ "$TERMUX_INSTALL_DEPS" = true ]; then if [ "$TERMUX_NO_CLEAN" = false ]; then # Remove all previously extracted/built files from $TERMUX_PREFIX: rm -rf $TERMUX_PREFIX rm -f /data/data/.built-packages/* - # Ensure folders present (but not $TERMUX_PKG_SRCDIR, it will be created in build) - mkdir -p "$TERMUX_COMMON_CACHEDIR" \ - "$TERMUX_COMMON_CACHEDIR-$TERMUX_ARCH" \ - "$TERMUX_COMMON_CACHEDIR-all" \ - "$TERMUX_DEBDIR" \ - "$TERMUX_PKG_BUILDDIR" \ - "$TERMUX_PKG_PACKAGEDIR" \ - "$TERMUX_PKG_TMPDIR" \ - "$TERMUX_PKG_CACHEDIR" \ - "$TERMUX_PKG_MASSAGEDIR" \ - $TERMUX_PREFIX/{bin,etc,lib,libexec,share,tmp,include} # Setup bootstrap if [ $TERMUX_ARCH == aarch64 ]; then local bootstrap_sha256=2944ad699814329007d1f9c056e7c8323243c8b4a257cbd05904216f89fc3746 From ba248d8e9f287b92b0dcd3e759fa6fd183f774aa Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sun, 10 Feb 2019 09:27:03 +0100 Subject: [PATCH 95/99] build-package.sh: make -i builds less verbose --- build-package.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build-package.sh b/build-package.sh index 6c16d9eff..7218c5a45 100755 --- a/build-package.sh +++ b/build-package.sh @@ -501,7 +501,7 @@ termux_step_get_repo_files() { ) fi # Import Fornwalls key: - gpg --import packages/apt/trusted.gpg + gpg --quiet --import packages/apt/trusted.gpg ( cd ${TERMUX_COMMON_CACHEDIR} curl --fail -LO "$TERMUX_REPO_URL/$TERMUX_REPO_DISTRIBUTION/InRelease" \ @@ -513,7 +513,7 @@ termux_step_get_repo_files() { termux_download "$TERMUX_REPO_URL/$TERMUX_REPO_DISTRIBUTION/$TERMUX_REPO_COMPONENT/binary-$arch/Packages.xz" \ "${TERMUX_COMMON_CACHEDIR}-$arch/Packages.xz" \ $packages_hash - xz -df "${TERMUX_COMMON_CACHEDIR}-$arch/Packages.xz" + xz --keep -df "${TERMUX_COMMON_CACHEDIR}-$arch/Packages.xz" done fi } From de8c36b63975dc6b1d4dcdb554625a38e49a7843 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sun, 10 Feb 2019 09:31:07 +0100 Subject: [PATCH 96/99] build-packages.sh: improve -I help message --- build-package.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build-package.sh b/build-package.sh index d59d860af..9260c708d 100755 --- a/build-package.sh +++ b/build-package.sh @@ -248,14 +248,14 @@ termux_setup_cmake() { # First step is to handle command-line arguments. Not to be overridden by packages. termux_step_handle_arguments() { _show_usage () { - echo "Usage: ./build-package.sh [-a ARCH] [-d] [-D] [-f] [-i] [-q] [-s] [-o DIR] PACKAGE" + echo "Usage: ./build-package.sh [-a ARCH] [-d] [-D] [-f] [-i] [-I] [-q] [-s] [-o DIR] PACKAGE" echo "Build a package by creating a .deb file in the debs/ folder." echo " -a The architecture to build for: aarch64(default), arm, i686, x86_64 or all." echo " -d Build with debug symbols." echo " -D Build a disabled package in disabled-packages/." echo " -f Force build even if package has already been built." echo " -i Download and extract dependencies instead of building them." - echo " -I Download and extract dependencies instead of building them, keep existing files." + echo " -I Download and extract dependencies instead of building them, keep existing /data/data/com.termux files." echo " -q Quiet build." echo " -s Skip dependency check." echo " -o Specify deb directory. Default: debs/." From e5c0651d17f003add90e0058dc0507c80ce53167 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sun, 10 Feb 2019 12:26:32 +0100 Subject: [PATCH 97/99] Make it possible to download dependencies from several repositories Only thing left to fix is so that it finds build scripts outside of packages/. --- build-package.sh | 75 +++++++++++++++++++++++------------ scripts/get_hash_from_file.py | 10 ++--- 2 files changed, 54 insertions(+), 31 deletions(-) diff --git a/build-package.sh b/build-package.sh index 9260c708d..e6f9be2c5 100755 --- a/build-package.sh +++ b/build-package.sh @@ -329,17 +329,23 @@ termux_step_setup_variables() { : "${TERMUX_DEBDIR:="${TERMUX_SCRIPTDIR}/debs"}" : "${TERMUX_SKIP_DEPCHECK:="false"}" : "${TERMUX_INSTALL_DEPS:="false"}" - : "${TERMUX_REPO_SIGNING_KEY:="2218893D3F679BEFC421FD976700B77E6D8D0AE7"}" - : "${TERMUX_REPO_URL:="https://termux.net/dists"}" - : "${TERMUX_REPO_DISTRIBUTION:="stable"}" - : "${TERMUX_REPO_COMPONENT:="main"}" + : "${TERMUX_REPO_SIGNING_KEYS:="packages/apt/trusted.gpg packages/termux-keyring/grimler.gpg packages/termux-keyring/xeffyr.gpg"}" : "${TERMUX_PKG_MAINTAINER:="Fredrik Fornwall @fornwall"}" : "${TERMUX_SKIP_DEPCHECK:="false"}" : "${TERMUX_INSTALL_DEPS:="false"}" - : "${TERMUX_REPO_SIGNING_KEY:="2218893D3F679BEFC421FD976700B77E6D8D0AE7"}" - : "${TERMUX_REPO_URL:="https://termux.net/dists"}" - : "${TERMUX_REPO_DISTRIBUTION:="stable"}" - : "${TERMUX_REPO_COMPONENT:="main"}" + + if [ -z ${TERMUX_REPO_URL+x} ]; then + TERMUX_REPO_URL=(https://termux.net/dists) + # TERMUX_REPO_URL=(https://termux.net/dists https://grimler.se/dists https://dl.bintray.com/xeffyr/unstable-packages/dists) + fi + if [ -z ${TERMUX_REPO_DISTRIBUTION+x} ]; then + TERMUX_REPO_DISTRIBUTION=(stable) + # TERMUX_REPO_DISTRIBUTION=(stable root unstable) + fi + if [ -z ${TERMUX_REPO_COMPONENT+x} ]; then + TERMUX_REPO_COMPONENT=(main) + # TERMUX_REPO_COMPONENT=(main stable main) + fi if [ "x86_64" = "$TERMUX_ARCH" ] || [ "aarch64" = "$TERMUX_ARCH" ]; then TERMUX_ARCH_BITS=64 @@ -449,13 +455,23 @@ termux_download_deb() { local package_arch=$2 local version=$3 local deb_file=${package}_${version}_${package_arch}.deb - local pkg_hash=$(./scripts/get_hash_from_file.py ${TERMUX_COMMON_CACHEDIR}-${package_arch}/Packages $package $version) - if [ "$pkg_hash" = "" ]; then + for idx in $(seq ${#TERMUX_REPO_URL[@]}); do + local TERMUX_REPO_NAME=$(echo ${TERMUX_REPO_URL[$idx-1]} | sed -e 's%https://%%g' -e 's%http://%%g' -e 's%/dists%%g' -e 's%/%-%g') + local PACKAGE_FILE_PATH="${TERMUX_REPO_NAME}-${TERMUX_REPO_DISTRIBUTION[$idx-1]}-${TERMUX_REPO_COMPONENT[$idx-1]}-Packages" + PKG_HASH=$(./scripts/get_hash_from_file.py "${TERMUX_COMMON_CACHEDIR}-$arch/$PACKAGE_FILE_PATH" $package $version) + if ! [ "$PKG_HASH" = "" ]; then + if [ ! "$TERMUX_QUIET_BUILD" = true ]; then + echo "Found $package in ${TERMUX_REPO_URL[$idx-1]}" + fi + break + fi + done + if [ "$PKG_HASH" = "" ]; then return 1 else - termux_download $TERMUX_REPO_URL/$TERMUX_REPO_DISTRIBUTION/$TERMUX_REPO_COMPONENT/binary-${package_arch}/${deb_file} \ + termux_download ${TERMUX_REPO_URL[$idx-1]}/${TERMUX_REPO_DISTRIBUTION[$idx-1]}/${TERMUX_REPO_COMPONENT[$idx-1]}/binary-${package_arch}/${deb_file} \ $TERMUX_COMMON_CACHEDIR-$package_arch/${deb_file} \ - $pkg_hash + $PKG_HASH return 0 fi } @@ -500,20 +516,27 @@ termux_step_get_repo_files() { rm SYMLINKS.txt ) fi - # Import Fornwalls key: - gpg --quiet --import packages/apt/trusted.gpg - ( - cd ${TERMUX_COMMON_CACHEDIR} - curl --fail -LO "$TERMUX_REPO_URL/$TERMUX_REPO_DISTRIBUTION/InRelease" \ - || termux_error_exit "Download of $TERMUX_REPO_URL/$TERMUX_REPO_DISTRIBUTION/InRelease failed" - gpg --verify InRelease - ) - for arch in all $TERMUX_ARCH; do - local packages_hash=$(./scripts/get_hash_from_file.py ${TERMUX_COMMON_CACHEDIR}/InRelease $arch) - termux_download "$TERMUX_REPO_URL/$TERMUX_REPO_DISTRIBUTION/$TERMUX_REPO_COMPONENT/binary-$arch/Packages.xz" \ - "${TERMUX_COMMON_CACHEDIR}-$arch/Packages.xz" \ - $packages_hash - xz --keep -df "${TERMUX_COMMON_CACHEDIR}-$arch/Packages.xz" + # Import signing keys from files + gpg --import ${TERMUX_REPO_SIGNING_KEYS} + + for idx in $(seq ${#TERMUX_REPO_URL[@]}); do + local TERMUX_REPO_NAME=$(echo ${TERMUX_REPO_URL[$idx-1]} | sed -e 's%https://%%g' -e 's%http://%%g' -e 's%/dists%%g' -e 's%/%-%g') + curl --fail -L "${TERMUX_REPO_URL[$idx-1]}/${TERMUX_REPO_DISTRIBUTION[$idx-1]}/InRelease" \ + -o ${TERMUX_COMMON_CACHEDIR}/${TERMUX_REPO_NAME}-${TERMUX_REPO_DISTRIBUTION[$idx-1]}-InRelease \ + || termux_error_exit "Download of ${TERMUX_REPO_URL[$idx-1]}/${TERMUX_REPO_DISTRIBUTION[$idx-1]}/InRelease failed" + + gpg --verify ${TERMUX_COMMON_CACHEDIR}/${TERMUX_REPO_NAME}-${TERMUX_REPO_DISTRIBUTION[$idx-1]}-InRelease + for arch in all $TERMUX_ARCH; do + local packages_hash=$(./scripts/get_hash_from_file.py ${TERMUX_COMMON_CACHEDIR}/${TERMUX_REPO_NAME}-${TERMUX_REPO_DISTRIBUTION[$idx-1]}-InRelease \ + $arch ${TERMUX_REPO_COMPONENT[$idx-1]}) + # If packages_hash = "" then the repo probably doesn't contain debs for $arch + if ! [ "$packages_hash" = "" ]; then + termux_download "${TERMUX_REPO_URL[$idx-1]}/${TERMUX_REPO_DISTRIBUTION[$idx-1]}/${TERMUX_REPO_COMPONENT[$idx-1]}/binary-$arch/Packages.xz" \ + "${TERMUX_COMMON_CACHEDIR}-$arch/${TERMUX_REPO_NAME}-${TERMUX_REPO_DISTRIBUTION[$idx-1]}-${TERMUX_REPO_COMPONENT[$idx-1]}-Packages.xz" \ + $packages_hash + xz --keep -df "${TERMUX_COMMON_CACHEDIR}-$arch/${TERMUX_REPO_NAME}-${TERMUX_REPO_DISTRIBUTION[$idx-1]}-${TERMUX_REPO_COMPONENT[$idx-1]}-Packages.xz" + fi + done done fi } diff --git a/scripts/get_hash_from_file.py b/scripts/get_hash_from_file.py index b34276c7a..bf7dfd009 100755 --- a/scripts/get_hash_from_file.py +++ b/scripts/get_hash_from_file.py @@ -17,25 +17,25 @@ def get_pkg_hash_from_Packages(Packages_file, package, version, hash="SHA256"): print(line.split(" ")[1]) break -def get_Packages_hash_from_InRelease(InRelease_file, arch, hash="SHA256"): - string_to_found = 'binary-'+arch+'/Packages.xz' +def get_Packages_hash_from_InRelease(InRelease_file, arch, component, hash="SHA256"): + string_to_find = component+'/binary-'+arch+'/Packages.xz' with open(InRelease_file, 'r') as InRelease: hash_list = InRelease.readlines() for i in range(len(hash_list)): if hash_list[i].startswith(hash+':'): break for j in range(i, len(hash_list)): - if string_to_found in hash_list[j].strip(' '): + if string_to_find in hash_list[j].strip(' '): print(hash_list[j].strip(' ').split(' ')[0]) break if __name__ == '__main__': if len(sys.argv) < 2: - sys.exit('Too few arguments, I need the path to a Packages file, a package name and a version, or an InRelease file and an architecture. Exiting') + sys.exit('Too few arguments, I need the path to a Packages file, a package name and a version, or an InRelease file, an architecture and a component name. Exiting') if sys.argv[1].endswith('Packages'): get_pkg_hash_from_Packages(sys.argv[1], sys.argv[2], sys.argv[3]) elif sys.argv[1].endswith('InRelease'): - get_Packages_hash_from_InRelease(sys.argv[1], sys.argv[2]) + get_Packages_hash_from_InRelease(sys.argv[1], sys.argv[2], sys.argv[3]) else: sys.exit(sys.argv[1]+' does not seem to be a path to a Packages or InRelease file') From c931f222167685125bf1099294fdaf1e70b0bfcf Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sun, 10 Feb 2019 12:27:48 +0100 Subject: [PATCH 98/99] Revert changes to TERMUX_ALL_DEPS for non -i builds --- build-package.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build-package.sh b/build-package.sh index e6f9be2c5..dd5f0340a 100755 --- a/build-package.sh +++ b/build-package.sh @@ -557,6 +557,8 @@ termux_step_start_build() { fi local TERMUX_ALL_DEPS=$(./scripts/buildorder.py "$TERMUX_PKG_BUILDER_DIR" | sed 's%packages/%%g') + local TERMUX_ALL_DEPS=$(./scripts/buildorder.py "$TERMUX_PKG_BUILDER_DIR") + if [ "$TERMUX_SKIP_DEPCHECK" = false ] && [ "$TERMUX_INSTALL_DEPS" = true ]; then # Download dependencies local pkg dep_arch dep_version deb_file _PKG_DEPENDS _PKG_BUILD_DEPENDS From f50865f84ffd9e136a47a5a151148d8226c4d76f Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sun, 10 Feb 2019 20:41:35 +0100 Subject: [PATCH 99/99] build-package.sh: minor cleanup --- build-package.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/build-package.sh b/build-package.sh index dd5f0340a..3692aabeb 100755 --- a/build-package.sh +++ b/build-package.sh @@ -331,8 +331,6 @@ termux_step_setup_variables() { : "${TERMUX_INSTALL_DEPS:="false"}" : "${TERMUX_REPO_SIGNING_KEYS:="packages/apt/trusted.gpg packages/termux-keyring/grimler.gpg packages/termux-keyring/xeffyr.gpg"}" : "${TERMUX_PKG_MAINTAINER:="Fredrik Fornwall @fornwall"}" - : "${TERMUX_SKIP_DEPCHECK:="false"}" - : "${TERMUX_INSTALL_DEPS:="false"}" if [ -z ${TERMUX_REPO_URL+x} ]; then TERMUX_REPO_URL=(https://termux.net/dists) @@ -556,7 +554,6 @@ termux_step_start_build() { exit 0 fi - local TERMUX_ALL_DEPS=$(./scripts/buildorder.py "$TERMUX_PKG_BUILDER_DIR" | sed 's%packages/%%g') local TERMUX_ALL_DEPS=$(./scripts/buildorder.py "$TERMUX_PKG_BUILDER_DIR") if [ "$TERMUX_SKIP_DEPCHECK" = false ] && [ "$TERMUX_INSTALL_DEPS" = true ]; then