Fast build fixes (#3696)

* Do not re=download release files.

* Efficiently handle dependencies: do not try to download \*.deb files when they already downloaded, do not try to extract them more than one time.

.gitlab-ci.yml

@@ -18,7 +18,7 @@ stages:
     - unset BINTRAY_GPG_PASSPHRASE
     - |
       for package in $(./scripts/build/ci/determine_git_changes.sh); do
-          ./build-package.sh -q -i -a "$TERMUX_ARCH" "$package" || exit 1
+          ./build-package.sh -q -I -a "$TERMUX_ARCH" "$package" || exit 1
       done
   retry:
     max: 2

scripts/build/termux_get_repo_files.sh

@@ -21,7 +21,10 @@ termux_get_repo_files() {
 			local TERMUX_REPO_NAME=$(echo ${TERMUX_REPO_URL[$idx-1]} | sed -e 's%https://%%g' -e 's%http://%%g' -e 's%/%-%g')
 			local RELEASE_FILE=${TERMUX_COMMON_CACHEDIR}/${TERMUX_REPO_NAME}-${TERMUX_REPO_DISTRIBUTION[$idx-1]}-Release
 
-			termux_download "${TERMUX_REPO_URL[$idx-1]}/dists/${TERMUX_REPO_DISTRIBUTION[$idx-1]}/Release" $RELEASE_FILE "SKIP_CHECKSUM"
+			if [ ! -e "$RELEASE_FILE" ]; then
+				termux_download "${TERMUX_REPO_URL[$idx-1]}/dists/${TERMUX_REPO_DISTRIBUTION[$idx-1]}/Release" \
+					"$RELEASE_FILE" SKIP_CHECKSUM
+			fi
 
 			for arch in all $TERMUX_ARCH; do
 				local PACKAGES_HASH=$(./scripts/get_hash_from_file.py ${RELEASE_FILE} $arch ${TERMUX_REPO_COMPONENT[$idx-1]})

scripts/build/termux_step_start_build.sh

@@ -12,6 +12,32 @@ termux_step_start_build() {
 		exit 0
 	fi
 
+	TERMUX_PKG_FULLVERSION=$TERMUX_PKG_VERSION
+	if [ "$TERMUX_PKG_REVISION" != "0" ] || [ "$TERMUX_PKG_FULLVERSION" != "${TERMUX_PKG_FULLVERSION/-/}" ]; then
+		# "0" is the default revision, so only include it if the upstream versions contains "-" itself
+		TERMUX_PKG_FULLVERSION+="-$TERMUX_PKG_REVISION"
+	fi
+
+	if [ "$TERMUX_DEBUG" = true ]; then
+		if [ "$TERMUX_PKG_HAS_DEBUG" == "yes" ]; then
+			DEBUG="-dbg"
+		else
+			echo "Skipping building debug build for $TERMUX_PKG_NAME"
+			exit 0
+		fi
+	else
+		DEBUG=""
+	fi
+
+	if [ -z "$TERMUX_DEBUG" ] &&
+	   [ -z "${TERMUX_FORCE_BUILD+x}" ] &&
+	   [ -e "/data/data/.built-packages/$TERMUX_PKG_NAME" ]; then
+		if [ "$(cat "/data/data/.built-packages/$TERMUX_PKG_NAME")" = "$TERMUX_PKG_FULLVERSION" ]; then
+			echo "$TERMUX_PKG_NAME@$TERMUX_PKG_FULLVERSION built - skipping (rm /data/data/.built-packages/$TERMUX_PKG_NAME to force rebuild)"
+			exit 0
+		fi
+	fi
+
 	if [ "$TERMUX_SKIP_DEPCHECK" = false ] && [ "$TERMUX_INSTALL_DEPS" = true ]; then
 		# Download repo files
 		termux_get_repo_files
@@ -29,6 +55,13 @@ termux_step_start_build() {
 			if [ ! "$TERMUX_QUIET_BUILD" = true ]; then
 				echo "Downloading dependency $PKG@$DEP_VERSION if necessary..."
 			fi
+
+			if [ -e "/data/data/.built-packages/$PKG" ]; then
+				if [ "$(cat "/data/data/.built-packages/$PKG")" = "$DEP_VERSION" ]; then
+					continue
+				fi
+			fi
+
 			if ! termux_download_deb $PKG $DEP_ARCH $DEP_VERSION; then
 				echo "Download of $PKG@$DEP_VERSION from $TERMUX_REPO_URL failed, building instead"
 				TERMUX_BUILD_IGNORE_LOCK=true ./build-package.sh -a $TERMUX_ARCH -I "${PKG_DIR}"
@@ -59,32 +92,6 @@ termux_step_start_build() {
 		done<<<$(./scripts/buildorder.py "$TERMUX_PKG_BUILDER_DIR" $TERMUX_PACKAGES_DIRECTORIES || echo "ERROR")
 	fi
 
-	TERMUX_PKG_FULLVERSION=$TERMUX_PKG_VERSION
-	if [ "$TERMUX_PKG_REVISION" != "0" ] || [ "$TERMUX_PKG_FULLVERSION" != "${TERMUX_PKG_FULLVERSION/-/}" ]; then
-		# "0" is the default revision, so only include it if the upstream versions contains "-" itself
-		TERMUX_PKG_FULLVERSION+="-$TERMUX_PKG_REVISION"
-	fi
-
-	if [ "$TERMUX_DEBUG" = true ]; then
-		if [ "$TERMUX_PKG_HAS_DEBUG" == "yes" ]; then
-			DEBUG="-dbg"
-		else
-			echo "Skipping building debug build for $TERMUX_PKG_NAME"
-			exit 0
-		fi
-	else
-		DEBUG=""
-	fi
-
-	if [ -z "$TERMUX_DEBUG" ] &&
-	   [ -z "${TERMUX_FORCE_BUILD+x}" ] &&
-	   [ -e "/data/data/.built-packages/$TERMUX_PKG_NAME" ]; then
-		if [ "$(cat "/data/data/.built-packages/$TERMUX_PKG_NAME")" = "$TERMUX_PKG_FULLVERSION" ]; then
-			echo "$TERMUX_PKG_NAME@$TERMUX_PKG_FULLVERSION built - skipping (rm /data/data/.built-packages/$TERMUX_PKG_NAME to force rebuild)"
-			exit 0
-		fi
-	fi
-
 	# Cleanup old state:
 	rm -Rf "$TERMUX_PKG_BUILDDIR" \
 		"$TERMUX_PKG_PACKAGEDIR" \