From d17827275f3387b63d4e74598d4d00df055d4a33 Mon Sep 17 00:00:00 2001 From: Leonid Plyushch Date: Sat, 20 Jul 2019 13:16:52 +0300 Subject: [PATCH] update CI setup * Move package building implementation from config to a separate script. * Implement new tag '%ci:no-build' to allow immediately stop build with status 'passed'. --- .cirrus.yml | 25 ++---- scripts/build/ci/cirrus-ci_dispatcher.sh | 104 ++++++++++++++++++++++ scripts/build/ci/determine_git_changes.sh | 98 -------------------- 3 files changed, 110 insertions(+), 117 deletions(-) create mode 100755 scripts/build/ci/cirrus-ci_dispatcher.sh delete mode 100755 scripts/build/ci/determine_git_changes.sh diff --git a/.cirrus.yml b/.cirrus.yml index a38fd1926..86863cb7f 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -15,8 +15,8 @@ build_task: TERMUX_ARCH: i686 TERMUX_ARCH: x86_64 - # Do not use built-in git client provided by Cirrus as it - # causes problems when determining changed files. + # Do not use built-in git client provided by Cirrus as may + # cause problems when determining changed files. clone_script: | if [[ -z "$CIRRUS_PR" ]]; then git clone --recursive --branch="$CIRRUS_BRANCH" "https://x-access-token:${CIRRUS_REPO_CLONE_TOKEN}@github.com/${CIRRUS_REPO_FULL_NAME}.git" "$CIRRUS_WORKING_DIR" @@ -27,24 +27,11 @@ build_task: git reset --hard "$CIRRUS_CHANGE_IN_REPO" fi - # Determine changes in repository and build modified packages. + # Following script will determine changes via Git and build + # modified packages. build_script: | - if grep -qiP '^\s*%ci:reset-backlog\s*$' <(git log --format="%B" -n 1 "$CIRRUS_CHANGE_IN_REPO"); then - # If commit message contains line '%ci:reset-backlog', builds will be done - # only for current commit. - unset CIRRUS_LAST_GREEN_CHANGE - unset CIRRUS_BASE_SHA - fi - MODIFIED_PACKAGES=$(./scripts/build/ci/determine_git_changes.sh) - for package in $MODIFIED_PACKAGES; do - if [ -n "$CIRRUS_PR" ]; then - # Perform full builds for PR. - ./build-package.sh -a "$TERMUX_ARCH" "$package" - else - ./build-package.sh -I -a "$TERMUX_ARCH" "$package" - fi - done + bash ./scripts/build/ci/cirrus-ci_dispatcher.sh - # Also make them downloadable from the UI. + # Make built packages downloadable from web UI. output_artifacts: path: "./debs/*.deb" diff --git a/scripts/build/ci/cirrus-ci_dispatcher.sh b/scripts/build/ci/cirrus-ci_dispatcher.sh new file mode 100755 index 000000000..334309b95 --- /dev/null +++ b/scripts/build/ci/cirrus-ci_dispatcher.sh @@ -0,0 +1,104 @@ +#!/bin/bash +## +## Determine modified packages and build/upload them. +## + +set -e + +############################################################################### +## +## Preparation. +## +############################################################################### + +REPO_DIR=$(realpath "$(dirname "$(realpath "$0")")/../../../") +cd "$REPO_DIR" || { + echo "[!] Failed to cd into '$REPO_DIR'." + exit 1 +} + +############################################################################### +## +## Determining changes. +## +############################################################################### + +set +e + +# Process tag '%ci:no-build' that may be added as line to commit message. +# Will force CI to exit with status 'passed' without performing build. +if grep -qiP '^\s*%ci:no-build\s*$' <(git log --format="%B" -n 1 "$CIRRUS_CHANGE_IN_REPO"); then + echo "[*] Exiting with status 'passed' (tag '%ci:no-build' applied)." + exit 0 +fi + +# Process tag '%ci:reset-backlog' that may be added as line to commit message. +# Will force CI to build changes only for the current commit. +if grep -qiP '^\s*%ci:reset-backlog\s*$' <(git log --format="%B" -n 1 "$CIRRUS_CHANGE_IN_REPO"); then + echo "[*] Building only last pushed commit (tag '%ci:reset-backlog' applied)." + unset CIRRUS_LAST_GREEN_CHANGE + unset CIRRUS_BASE_SHA +fi + +if [ -z "$CIRRUS_PR" ]; then + if [ -z "$CIRRUS_LAST_GREEN_CHANGE" ]; then + UPDATED_FILES=$(git diff-tree --no-commit-id --name-only -r "$CIRRUS_CHANGE_IN_REPO" 2>/dev/null | grep -P "packages/") + else + UPDATED_FILES=$(git diff-tree --no-commit-id --name-only -r "${CIRRUS_LAST_GREEN_CHANGE}..${CIRRUS_CHANGE_IN_REPO}" 2>/dev/null | grep -P "packages/") + fi +else + # Pull requests are handled in a bit different way. + UPDATED_FILES=$(git diff-tree --no-commit-id --name-only -r "${CIRRUS_BASE_SHA}..${CIRRUS_CHANGE_IN_REPO}" 2>/dev/null | grep -P "packages/") +fi + +## Determine modified packages. +existing_dirs="" +for dir in $(echo "$UPDATED_FILES" | grep -oP "packages/[a-z0-9+._-]+" | sort | uniq); do + if [ -d "${REPO_DIR}/${dir}" ]; then + existing_dirs+=" $dir" + fi +done +PACKAGE_DIRS="$existing_dirs" +unset dir existing_dirs + +## Get names of modified packages. +PACKAGE_NAMES=$(echo "$PACKAGE_DIRS" | sed 's/packages\///g') +if [ -z "$PACKAGE_NAMES" ]; then + echo "[*] No modified packages found." >&2 + exit 0 +fi + + +## Some packages should be excluded from auto builds. +EXCLUDED_PACKAGES="rust texlive" + +for excluded_pkg in $EXCLUDED_PACKAGES; do + PACKAGE_NAMES=$(echo "$PACKAGE_NAMES" | sed "s/\<${excluded_pkg}\>//g") +done +unset excluded_pkg + +set -e + +############################################################################### +## +## Building packages. +## +############################################################################### + +echo "[*] Building packages: $PACKAGE_NAMES" +echo +if [ -n "$CIRRUS_PR" ]; then + echo " Pull request: https://github.com/termux/unstable-packages/pull/${CIRRUS_PR}" +else + if [ -n "$CIRRUS_LAST_GREEN_CHANGE" ]; then + echo " Changes: ${CIRRUS_LAST_GREEN_CHANGE}..${CIRRUS_CHANGE_IN_REPO}" + else + echo " Changes: ${CIRRUS_CHANGE_IN_REPO}" + fi +fi + +echo +for pkg in $PACKAGE_NAMES; do + ./build-package.sh -a "$TERMUX_ARCH" -I "$pkg" +done +echo diff --git a/scripts/build/ci/determine_git_changes.sh b/scripts/build/ci/determine_git_changes.sh deleted file mode 100755 index 09f6240d7..000000000 --- a/scripts/build/ci/determine_git_changes.sh +++ /dev/null @@ -1,98 +0,0 @@ -#!/bin/bash -## -## Script for detecting modified packages. -## Designed for use with Cirrus, Gitlab or Travis CI. -## -## Leonid Plyushch (C) 2019 -## -## This program is free software: you can redistribute it and/or modify -## it under the terms of the GNU General Public License as published by -## the Free Software Foundation, either version 3 of the License, or -## (at your option) any later version. -## -## This program is distributed in the hope that it will be useful, -## but WITHOUT ANY WARRANTY; without even the implied warranty of -## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -## GNU General Public License for more details. -## -## You should have received a copy of the GNU General Public License -## along with this program. If not, see . -## - -REPO_DIR=$(realpath "$(dirname "$(realpath "$0")")/../../../") -cd "$REPO_DIR" || { - echo "[!] Failed to cd into '$REPO_DIR'." >&2 - exit 1 -} - -if [ -n "$TRAVIS_COMMIT_RANGE" ]; then - # We are on Travis CI. - UPDATED_FILES=$(git diff-tree --no-commit-id --name-only -r "${TRAVIS_COMMIT_RANGE//.../..}" 2>/dev/null | grep -P "packages/") -elif [ -n "$CI_COMMIT_SHA" ]; then - # We are on Gitlab CI. - - # Make sure that we can use commit range. - if [ -z "$CI_COMMIT_BEFORE_SHA" ]; then - echo "[!] CI_COMMIT_BEFORE_SHA is not set." >&2 - exit 1 - fi - - if [ "$CI_COMMIT_BEFORE_SHA" = "0000000000000000000000000000000000000000" ]; then - UPDATED_FILES=$(git diff-tree --no-commit-id --name-only -r "$CI_COMMIT_SHA" 2>/dev/null | grep -P "packages/") - else - UPDATED_FILES=$(git diff-tree --no-commit-id --name-only -r "${CI_COMMIT_BEFORE_SHA}..${CI_COMMIT_SHA}" 2>/dev/null | grep -P "packages/") - fi -elif [ -n "$CIRRUS_CI" ]; then - # We are on Cirrus CI. - if [ -z "$CIRRUS_PR" ]; then - if [ -z "$CIRRUS_LAST_GREEN_CHANGE" ]; then - UPDATED_FILES=$(git diff-tree --no-commit-id --name-only -r "$CIRRUS_CHANGE_IN_REPO" 2>/dev/null | grep -P "packages/") - else - UPDATED_FILES=$(git diff-tree --no-commit-id --name-only -r "${CIRRUS_LAST_GREEN_CHANGE}..${CIRRUS_CHANGE_IN_REPO}" 2>/dev/null | grep -P "packages/") - fi - else - # Pull requests are handled in a bit different way. - UPDATED_FILES=$(git diff-tree --no-commit-id --name-only -r "${CIRRUS_BASE_SHA}..${CIRRUS_CHANGE_IN_REPO}" 2>/dev/null | grep -P "packages/") - fi -else - # Something wrong. - echo "[!] Cannot determine git commit range." >&2 - echo " Did you executed this script under CI ?" >&2 - exit 1 -fi - -## Determine modified packages. -existing_dirs="" -for dir in $(echo "$UPDATED_FILES" | grep -oP "packages/[a-z0-9+._-]+" | sort | uniq); do - if [ -d "$REPO_DIR/$dir" ]; then - existing_dirs+=" $dir" - fi -done -PACKAGE_DIRS="$existing_dirs" -unset dir existing_dirs - -## Get names of modified packages. -PACKAGE_NAMES=$(echo "$PACKAGE_DIRS" | sed 's/packages\///g') -if [ -z "$PACKAGE_NAMES" ]; then - echo "[*] No modified packages found." >&2 - exit 0 -fi - -## Print names of modified packages. -for pkg in $PACKAGE_NAMES; do - case "$pkg" in - # Skip packages that known to have long build time. - rust|texlive) - { - echo - echo "Package '$pkg' cannot be built via CI because it has" - echo "long build time." - echo - } >&2 - continue - ;; - *) - echo "$pkg" - ;; - esac -done