From 6f311c8930a6ec33694a1d1d9c24c21c42acbb61 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Fri, 4 Nov 2022 05:20:25 +0000 Subject: [PATCH] appimage build: better caching, both locally and on CI --- .cirrus.yml | 10 ++++-- contrib/build-linux/appimage/make_appimage.sh | 31 ++++++++++++++----- contrib/make_libsecp256k1.sh | 2 +- contrib/make_zbar.sh | 2 +- 4 files changed, 34 insertions(+), 11 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 99752d176..168dbab52 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -273,12 +273,18 @@ task: cpu: 2 memory: 1G pip_cache: - folder: contrib/build-linux/appimage/.cache/appimage/pip_cache + folder: contrib/build-linux/appimage/.cache/pip_cache fingerprint_script: - echo $CIRRUS_TASK_NAME - find contrib/deterministic-build/*.txt -type f -print0 | sort -z | xargs -0 sha256sum | sha256sum - find contrib/build-linux/appimage/ -type f -print0 | sort -z | xargs -0 sha256sum | sha256sum - populate_script: mkdir -p contrib/build-linux/appimage/.cache/appimage/pip_cache + populate_script: mkdir -p contrib/build-linux/appimage/.cache/pip_cache + build2_cache: + folder: contrib/build-linux/appimage/.cache/appimage + fingerprint_script: + - echo $CIRRUS_TASK_NAME + - cat contrib/make_libsecp256k1.sh | sha256sum + - find contrib/build-linux/appimage/ -type f -print0 | sort -z | xargs -0 sha256sum | sha256sum build_script: - ./contrib/build-linux/appimage/make_appimage.sh binaries_artifacts: diff --git a/contrib/build-linux/appimage/make_appimage.sh b/contrib/build-linux/appimage/make_appimage.sh index bf62279e3..a188ee5ff 100755 --- a/contrib/build-linux/appimage/make_appimage.sh +++ b/contrib/build-linux/appimage/make_appimage.sh @@ -9,7 +9,8 @@ DISTDIR="$PROJECT_ROOT/dist" BUILDDIR="$CONTRIB_APPIMAGE/build/appimage" APPDIR="$BUILDDIR/electrum.AppDir" CACHEDIR="$CONTRIB_APPIMAGE/.cache/appimage" -PIP_CACHE_DIR="$CACHEDIR/pip_cache" +export DLL_TARGET_DIR="$CACHEDIR/dlls" +PIP_CACHE_DIR="$CONTRIB_APPIMAGE/.cache/pip_cache" export GCC_STRIP_BINARIES="1" @@ -24,7 +25,7 @@ APPIMAGE="$DISTDIR/electrum-$VERSION-x86_64.AppImage" . "$CONTRIB"/build_tools_util.sh rm -rf "$BUILDDIR" -mkdir -p "$APPDIR" "$CACHEDIR" "$PIP_CACHE_DIR" "$DISTDIR" +mkdir -p "$APPDIR" "$CACHEDIR" "$PIP_CACHE_DIR" "$DISTDIR" "$DLL_TARGET_DIR" # potential leftover from setuptools that might make pip put garbage in binary rm -rf "$PROJECT_ROOT/build" @@ -43,9 +44,13 @@ verify_hash "$CACHEDIR/Python-$PYTHON_VERSION.tar.xz" "66767a35309d724f370df9e50 info "building python." -tar xf "$CACHEDIR/Python-$PYTHON_VERSION.tar.xz" -C "$BUILDDIR" +tar xf "$CACHEDIR/Python-$PYTHON_VERSION.tar.xz" -C "$CACHEDIR" ( - cd "$BUILDDIR/Python-$PYTHON_VERSION" + if [ -f "$CACHEDIR/Python-$PYTHON_VERSION/python" ]; then + info "python already built, skipping" + exit 0 + fi + cd "$CACHEDIR/Python-$PYTHON_VERSION" LC_ALL=C export BUILD_DATE=$(date -u -d "@$SOURCE_DATE_EPOCH" "+%b %d %Y") LC_ALL=C export BUILD_TIME=$(date -u -d "@$SOURCE_DATE_EPOCH" "+%H:%M:%S") # Patch taken from Ubuntu http://archive.ubuntu.com/ubuntu/pool/main/p/python3.9/python3.9_3.9.5-3~21.04.debian.tar.xz @@ -57,6 +62,10 @@ tar xf "$CACHEDIR/Python-$PYTHON_VERSION.tar.xz" -C "$BUILDDIR" --enable-shared \ -q make -j4 -s || fail "Could not build Python" +) +info "installing python." +( + cd "$CACHEDIR/Python-$PYTHON_VERSION" make -s install > /dev/null || fail "Could not install Python" # When building in docker on macOS, python builds with .exe extension because the # case insensitive file system of macOS leaks into docker. This causes the build @@ -67,8 +76,12 @@ tar xf "$CACHEDIR/Python-$PYTHON_VERSION.tar.xz" -C "$BUILDDIR" ) -"$CONTRIB"/make_libsecp256k1.sh || fail "Could not build libsecp" -cp -f "$PROJECT_ROOT/electrum/libsecp256k1.so.0" "$APPDIR/usr/lib/libsecp256k1.so.0" || fail "Could not copy libsecp to its destination" +if [ -f "$DLL_TARGET_DIR/libsecp256k1.so.0" ]; then + info "libsecp256k1 already built, skipping" +else + "$CONTRIB"/make_libsecp256k1.sh || fail "Could not build libsecp" +fi +cp -f "$DLL_TARGET_DIR/libsecp256k1.so.0" "$APPDIR/usr/lib/libsecp256k1.so.0" || fail "Could not copy libsecp to its destination" # note: libxcb-util1 is not available in debian 10 (buster), only libxcb-util0. So we build it ourselves. @@ -77,6 +90,10 @@ info "building libxcb-util1." XCB_UTIL_VERSION="acf790d7752f36e450d476ad79807d4012ec863b" # ^ git tag 0.4.0 ( + if [ -f "$CACHEDIR/libxcb-util1/util/src/.libs/libxcb-util.so.1" ]; then + info "libxcb-util1 already built, skipping" + exit 0 + fi cd "$CACHEDIR" mkdir "libxcb-util1" cd "libxcb-util1" @@ -95,8 +112,8 @@ XCB_UTIL_VERSION="acf790d7752f36e450d476ad79807d4012ec863b" ./autogen.sh ./configure --enable-shared make -j4 -s || fail "Could not build libxcb-util1" - cp "$CACHEDIR/libxcb-util1/util/src/.libs/libxcb-util.so.1" "$APPDIR/usr/lib/libxcb-util.so.1" ) || fail "Could build libxcb-util1" +cp "$CACHEDIR/libxcb-util1/util/src/.libs/libxcb-util.so.1" "$APPDIR/usr/lib/libxcb-util.so.1" appdir_python() { diff --git a/contrib/make_libsecp256k1.sh b/contrib/make_libsecp256k1.sh index 275221049..213602891 100755 --- a/contrib/make_libsecp256k1.sh +++ b/contrib/make_libsecp256k1.sh @@ -66,6 +66,6 @@ info "Building $pkgname..." cp -fpv "$here/$pkgname/dist/lib/$dlname" "$PROJECT_ROOT/electrum" || fail "Could not copy the $pkgname binary to its destination" info "$dlname has been placed in the inner 'electrum' folder." if [ -n "$DLL_TARGET_DIR" ] ; then - cp -fpv "$here/$pkgname/dist/lib/$dlname" "$DLL_TARGET_DIR" || fail "Could not copy the $pkgname binary to DLL_TARGET_DIR" + cp -fpv "$here/$pkgname/dist/lib/$dlname" "$DLL_TARGET_DIR/" || fail "Could not copy the $pkgname binary to DLL_TARGET_DIR" fi ) diff --git a/contrib/make_zbar.sh b/contrib/make_zbar.sh index 7546def65..dde7a755c 100755 --- a/contrib/make_zbar.sh +++ b/contrib/make_zbar.sh @@ -88,6 +88,6 @@ info "Building $pkgname..." cp -fpv "$here/$pkgname/dist/lib/$dlname" "$PROJECT_ROOT/electrum" || fail "Could not copy the $pkgname binary to its destination" info "$dlname has been placed in the inner 'electrum' folder." if [ -n "$DLL_TARGET_DIR" ] ; then - cp -fpv "$here/$pkgname/dist/lib/$dlname" "$DLL_TARGET_DIR" || fail "Could not copy the $pkgname binary to DLL_TARGET_DIR" + cp -fpv "$here/$pkgname/dist/lib/$dlname" "$DLL_TARGET_DIR/" || fail "Could not copy the $pkgname binary to DLL_TARGET_DIR" fi )