CI: restore old way for building & uploading packages

Do not use pipes for building & uploading packages because:

 * They spawn more jobs on Bintray for metadata generation - each upload is
   done in a separate session.

 * Bintray doesn't handle well concurrent uploads and metadata can be corrupted.
   This happened in all recent builds of platform independent packages - in
   each upload task file had a same name and that triggered problem with metadata.

 * Using delays for workaround for a problem described above works, but isn't
   safe.

.cirrus.yml

@@ -1,80 +1,42 @@
 container:
+  image: termux/package-builder:latest
   cpu: 8
   memory: 16
 
-##
+task:
-## * Maximal timeout is 120 minutes.
+  name: Build
-## * Each architecture require a separate pipe configuration.
-## * Test builds (i.e. non-uploadable) are done via separate task.
-##
-
-build_pipe_template: &BUILD_PIPE_TEMPLATE
-  only_if: $CIRRUS_BRANCH == 'master'
 
+  # 2 hours is a maximal timeout available for free use.
   timeout_in: 120m
 
   environment:
-    BINTRAY_USERNAME: xeffyr
+    matrix:
-    BINTRAY_API_KEY: ENCRYPTED[c056d345d7a15f90b818dd9382129c27ce4ce2b802831ec890786414fc214c4203714c353a8fef3a6b9b1b1d68e4685e]
-    BINTRAY_GPG_SUBJECT: termux
-    BINTRAY_GPG_PASSPHRASE: ENCRYPTED[854c09c78b3318a30d3c9094b1010dfebd4c71ad9173b64479eaafaa7e941b050afaa936ad635af3709382492828316f]
-
-  steps:
-    - image: termux/package-builder:latest
-      build_script: bash ./scripts/build/ci/cirrus-ci_dispatcher.sh
-    - image: termux/package-builder:latest
-      upload_script: bash ./scripts/build/ci/cirrus-ci_dispatcher.sh --upload
-      output_artifacts:
-        path: "./debs/*.deb"
-
-
-pipe:
-  << : *BUILD_PIPE_TEMPLATE
-  name: Build and Upload - aarch64
-  environment:
       TERMUX_ARCH: aarch64
-    UPLOAD_DELAY: 0
-
-pipe:
-  << : *BUILD_PIPE_TEMPLATE
-  name: Build and Upload - arm
-  environment:
       TERMUX_ARCH: arm
-    UPLOAD_DELAY: 20
-
-pipe:
-  << : *BUILD_PIPE_TEMPLATE
-  name: Build and Upload - i686
-  environment:
       TERMUX_ARCH: i686
-    UPLOAD_DELAY: 40
-
-pipe:
-  << : *BUILD_PIPE_TEMPLATE
-  name: Build and Upload - x86_64
-  environment:
       TERMUX_ARCH: x86_64
-    UPLOAD_DELAY: 60
 
+  build_script: |
+    bash ./scripts/build/ci/cirrus-ci_dispatcher.sh
+
+  output_artifacts:
+    path: "./debs/*.deb"
 
 task:
-  name: Test build
+  name: Publish
 
-  only_if: $CIRRUS_BRANCH != 'master'
+  only_if: $CIRRUS_BRANCH == 'master'
 
-  timeout_in: 120m
+  timeout_in: 60m
 
-  container:
+  depends_on:
-    image: termux/package-builder:latest
+    - Build
 
   environment:
-    matrix:
+    BINTRAY_USERNAME: xeffyr
-      TERMUX_ARCH: aarch64
+    BINTRAY_API_KEY: ENCRYPTED[c056d345d7a15f90b818dd9382129c27ce4ce2b802831ec890786414fc214c4203714c353a8fef3a6b9b1b1d68e4685e]
-      TERMUX_ARCH: arm
+    BINTRAY_GPG_SUBJECT: termux
-      TERMUX_ARCH: i686
+    BINTRAY_GPG_PASSPHRASE: ENCRYPTED[854c09c78b3318a30d3c9094b1010dfebd4c71ad9173b64479eaafaa7e941b050afaa936ad635af3709382492828316f]
-      TERMUX_ARCH: x86_64
-
-  build_script: bash ./scripts/build/ci/cirrus-ci_dispatcher.sh
 
-  output_artifacts:
+  upload_script: |
-    path: "./debs/*.deb"
+    bash ./scripts/build/ci/cirrus-ci_dispatcher.sh --upload

scripts/build/ci/cirrus-ci_dispatcher.sh

@@ -5,6 +5,9 @@
 
 set -e
 
+## Some packages should be excluded from auto builds.
+EXCLUDED_PACKAGES="rust texlive"
+
 ###############################################################################
 ##
 ##  Preparation.
@@ -60,42 +63,42 @@ if grep -qiP '^\s*%ci:reset-backlog\s*$' <(git log --format="%B" -n 1 "$CIRRUS_C
 fi
 
 if [ -z "$CIRRUS_PR" ]; then
+	# Changes determined from the last commit where CI finished with status
+	# 'passed' (green) and the top commit.
 	if [ -z "$CIRRUS_LAST_GREEN_CHANGE" ]; then
-		UPDATED_FILES=$(git diff-tree --no-commit-id --name-only -r "$CIRRUS_CHANGE_IN_REPO" 2>/dev/null | grep -P "packages/")
+		GIT_CHANGES="$CIRRUS_CHANGE_IN_REPO"
 	else
-		UPDATED_FILES=$(git diff-tree --no-commit-id --name-only -r "${CIRRUS_LAST_GREEN_CHANGE}..${CIRRUS_CHANGE_IN_REPO}" 2>/dev/null | grep -P "packages/")
+		GIT_CHANGES="${CIRRUS_LAST_GREEN_CHANGE}..${CIRRUS_CHANGE_IN_REPO}"
 	fi
+	echo "[*] Changes: $GIT_CHANGES"
 else
-	# Pull requests are handled in a bit different way.
+	# Changes in pull request are determined from commits between the
-	UPDATED_FILES=$(git diff-tree --no-commit-id --name-only -r "${CIRRUS_BASE_SHA}..${CIRRUS_CHANGE_IN_REPO}" 2>/dev/null | grep -P "packages/")
+	# top commit of base branch and latest commit of PR's branch.
+	GIT_CHANGES="${CIRRUS_BASE_SHA}..${CIRRUS_CHANGE_IN_REPO}"
+	echo "[*] Pull request: https://github.com/termux/termux-packages/pull/${CIRRUS_PR}"
 fi
 
-## Determine modified packages.
+# Determine changes from commit range.
-existing_dirs=""
+PACKAGE_NAMES=$(git diff-tree --no-commit-id --name-only -r "$GIT_CHANGES" packages/ 2>/dev/null | sed -E 's@^packages/([^/]*)/build.sh@\1@')
-for dir in $(echo "$UPDATED_FILES" | grep -oP "packages/[a-z0-9+._-]+" | sort | uniq); do
+
-	if [ -d "${REPO_DIR}/${dir}" ]; then
+## Filter deleted packages.
-		existing_dirs+=" $dir"
+for pkg in $PACKAGE_NAMES; do
+	if [ ! -d "${REPO_DIR}/packages/${pkg}" ]; then
+		PACKAGE_NAMES=$(sed "s/\<${pkg}\>//g" <<< "$PACKAGE_NAMES")
 	fi
 done
-PACKAGE_DIRS="$existing_dirs"
-unset dir existing_dirs
 
-## Get names of modified packages.
+## Filter excluded packages.
-PACKAGE_NAMES=$(echo "$PACKAGE_DIRS" | sed 's/packages\///g')
+for pkg in $EXCLUDED_PACKAGES; do
+	PACKAGE_NAMES=$(sed "s/\<${pkg}\>//g" <<< "$PACKAGE_NAMES")
+done
+unset pkg
+
 if [ -z "$PACKAGE_NAMES" ]; then
-	echo "[*] No modified packages found." >&2
+	echo "[*] No modified packages found."
 	exit 0
 fi
 
-
-## Some packages should be excluded from auto builds.
-EXCLUDED_PACKAGES="rust texlive"
-
-for excluded_pkg in $EXCLUDED_PACKAGES; do
-	PACKAGE_NAMES=$(echo "$PACKAGE_NAMES" | sed "s/\<${excluded_pkg}\>//g")
-done
-unset excluded_pkg
-
 set -e
 
 ###############################################################################
@@ -106,17 +109,29 @@ set -e
 
 if ! $DO_UPLOAD; then
 	echo "[*] Building packages: $PACKAGE_NAMES"
-	if [ -n "$CIRRUS_PR" ]; then
+	./build-package.sh -a "$TERMUX_ARCH" -I $PACKAGE_NAMES
-		echo "[*] Pull request: https://github.com/termux/termux-packages/pull/${CIRRUS_PR}"
-	else
-		if [ -n "$CIRRUS_LAST_GREEN_CHANGE" ]; then
-			echo "[*] Changes: ${CIRRUS_LAST_GREEN_CHANGE}..${CIRRUS_CHANGE_IN_REPO}"
-		else
-			echo "[*] Changes: ${CIRRUS_CHANGE_IN_REPO}"
-		fi
 fi
 
-	./build-package.sh -a "$TERMUX_ARCH" -I $PACKAGE_NAMES
+###############################################################################
+##
+##  Storing packages in cache // retrieving and uploading to Bintray.
+##
+###############################################################################
+
+if [ "$CIRRUS_BRANCH" = "master" ]; then
+	if ! $DO_UPLOAD; then
+		ARCHIVE_NAME="debs-${TERMUX_ARCH}-${CIRRUS_CHANGE_IN_REPO}.tar.gz"
+
+		if [ -d "${REPO_DIR}/debs" ]; then
+			echo "[*] Archiving packages into '${ARCHIVE_NAME}'."
+			tar zcf "$ARCHIVE_NAME" debs
+
+			echo "[*] Uploading '${ARCHIVE_NAME}' to cache:"
+			echo
+			curl --upload-file "$ARCHIVE_NAME" \
+				"http://${CIRRUS_HTTP_CACHE_HOST}/${ARCHIVE_NAME}"
+			echo
+		fi
 	else
 		if [ -z "$BINTRAY_API_KEY" ]; then
 			echo "[!] Can't upload without Bintray API key."
@@ -128,25 +143,27 @@ else
 			exit 1
 		fi
 
-	# Workaround for concurrent uploads.
+		for arch in aarch64 arm i686 x86_64; do
-	if [ "$UPLOAD_DELAY" != "0" ]; then
+			ARCHIVE_NAME="debs-${arch}-${CIRRUS_CHANGE_IN_REPO}.tar.gz"
-		echo "[!] Using workaround for Bintray issue with concurrent uploads."
-		echo "[!] Delaying upload by ${UPLOAD_DELAY} seconds."
-		sleep "$UPLOAD_DELAY"
-	fi
 
-	for attempt in 1 2 3; do
+			echo "[*] Downloading '${ARCHIVE_NAME}' from cache:"
-		echo "[*] Uploading packages to Bintray:"
+			echo
-		if ./scripts/package_uploader.sh -p "${PWD}/debs" $PACKAGE_NAMES; then
+			curl --output "/tmp/${ARCHIVE_NAME}" \
-			break
+				"http://${CIRRUS_HTTP_CACHE_HOST}/${ARCHIVE_NAME}"
-		else
+			echo
-			if [ "$attempt" = "3" ]; then
+
-				echo "[!] Went through 3 attempts of upload, giving up..."
+			if [ -s "/tmp/${ARCHIVE_NAME}" ]; then
-				exit 1
+				echo "[*] Unpacking '/tmp/${ARCHIVE_NAME}':"
+				echo
+				tar xvf "/tmp/${ARCHIVE_NAME}"
+				echo
 			else
-				echo "[!] Failure, retrying in 30 seconds..."
+				echo "[!] Empty archive '/tmp/${ARCHIVE_NAME}'."
-				sleep 30
-			fi
 			fi
 		done
+
+		echo "[*] Uploading packages to Bintray:"
+		echo
+		"${REPO_DIR}/scripts/package_uploader.sh" -p "${PWD}/debs" $PACKAGE_NAMES
+	fi
 fi