diff --git a/.travis.yml b/.travis.yml index 5907a4632..fef9701bc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -58,10 +58,8 @@ jobs: - sudo apt-get update - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce - docker version - install: - - sudo docker build --no-cache -t electrum-wine-builder-img ./contrib/build-wine/ script: - - sudo docker run --name electrum-wine-builder-cont -v $PWD:/opt/wine64/drive_c/electrum --rm --workdir /opt/wine64/drive_c/electrum/contrib/build-wine electrum-wine-builder-img ./build.sh + - ELECBUILD_COMMIT=HEAD ./contrib/build-wine/build.sh after_success: true - if: (branch = master) OR (tag IS present) name: "Android build" @@ -69,17 +67,15 @@ jobs: python: 3.7 services: - docker - install: - - ./contrib/android/build_docker_image.sh script: - sudo chown -R 1000:1000 . # Output something every minute or Travis kills the job - while sleep 60; do echo "=====[ $SECONDS seconds still running ]====="; done & - - sudo docker run -it -u 1000:1000 --rm --name electrum-android-builder-cont --env CI=true -v $PWD:/home/user/wspace/electrum --workdir /home/user/wspace/electrum electrum-android-builder-img ./contrib/android/make_apk + - ELECBUILD_COMMIT=HEAD ./contrib/android/build.sh # kill background sleep loop - kill %1 - - ls -la bin - - if [ $(ls bin | grep -c Electrum-*) -eq 0 ]; then exit 1; fi + - ls -la dist + - if [ $(ls dist | grep -c Electrum-*) -eq 0 ]; then exit 1; fi after_success: true # disabled for now as travis started to always time out: - if: false AND ((branch = master) OR (tag IS present)) @@ -100,10 +96,8 @@ jobs: python: false services: - docker - install: - - sudo docker build --no-cache -t electrum-appimage-builder-img ./contrib/build-linux/appimage/ script: - - sudo docker run --name electrum-appimage-builder-cont -v $PWD:/opt/electrum --rm --workdir /opt/electrum/contrib/build-linux/appimage electrum-appimage-builder-img ./build.sh + - ELECBUILD_COMMIT=HEAD ./contrib/build-linux/appimage/build.sh after_success: true - if: (branch = master) OR (tag IS present) name: "tarball build" @@ -111,19 +105,8 @@ jobs: python: false services: - docker - before_install: - # hack: travis already cloned the repo, but we re-clone now, as we need to have umask set BEFORE cloning - - umask 0022 - - mkdir fresh_clone && cd fresh_clone - - git clone https://github.com/$TRAVIS_REPO_SLUG.git && cd electrum - - if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then git fetch origin pull/$TRAVIS_PULL_REQUEST/merge; fi - - git checkout $TRAVIS_COMMIT - - echo "Second git clone ready at $PWD" - install: - - sudo docker build --no-cache -t electrum-sdist-builder-img ./contrib/build-linux/sdist/ script: - - echo "Building sdist at $PWD" - - sudo docker run --name electrum-sdist-builder-cont -v $PWD:/opt/electrum --rm --workdir /opt/electrum/contrib/build-linux/sdist electrum-sdist-builder-img ./build.sh + - ELECBUILD_COMMIT=HEAD ./contrib/build-linux/sdist/build.sh after_success: true - stage: release check install: diff --git a/contrib/android/Readme.md b/contrib/android/Readme.md index 5281cfbf8..9ecaebd26 100644 --- a/contrib/android/Readme.md +++ b/contrib/android/Readme.md @@ -9,8 +9,7 @@ To generate an APK file, follow these instructions. binaries that match the official releases._ This assumes an Ubuntu (x86_64) host, but it should not be too hard to adapt to another -similar system. The docker commands should be executed in the project's root -folder. +similar system. 1. Install Docker @@ -21,49 +20,20 @@ folder. $ sudo apt-get install -y docker-ce ``` -2. Build image +2. Build binaries ``` - $ ./contrib/android/build_docker_image.sh + $ ./build.sh ``` - -3. Build binaries - - It's recommended to build from a fresh clone - (but you can skip this if reproducibility is not necessary). - + If you want reproducibility, try instead e.g.: ``` - $ FRESH_CLONE=contrib/android/fresh_clone && \ - sudo rm -rf $FRESH_CLONE && \ - umask 0022 && \ - mkdir -p $FRESH_CLONE && \ - cd $FRESH_CLONE && \ - git clone https://github.com/spesmilo/electrum.git && \ - cd electrum + $ ELECBUILD_COMMIT=HEAD ELECBUILD_NOCACHE=1 ./build.sh release-unsigned ``` - - And then build from this directory: - ``` - $ git checkout $REV - $ mkdir --parents $PWD/.buildozer/.gradle - $ sudo docker run -it --rm \ - --name electrum-android-builder-cont \ - -v $PWD:/home/user/wspace/electrum \ - -v $PWD/.buildozer/.gradle:/home/user/.gradle \ - -v ~/.keystore:/home/user/.keystore \ - --workdir /home/user/wspace/electrum \ - electrum-android-builder-img \ - ./contrib/android/make_apk - ``` - - Note: this builds a debug apk. `make_apk` takes an optional parameter - which can be either `release` or `release-unsigned`. - This mounts the project dir inside the container, - and so the modifications will affect it, e.g. `.buildozer` folder - will be created. + Note: `build.sh` takes an optional parameter which can be + `release`, `release-unsigned`, or `debug` (default). -5. The generated binary is in `./bin`. +3. The generated binary is in `./dist`. ## Verifying reproducibility and comparing against official binary @@ -95,7 +65,7 @@ You probably need to clear the cache: `rm -rf .buildozer/android/platform/build- ### How do I deploy on connected phone for quick testing? Assuming `adb` is installed: ``` -$ adb -d install -r bin/Electrum-*-arm64-v8a-debug.apk +$ adb -d install -r dist/Electrum-*-arm64-v8a-debug.apk $ adb shell monkey -p org.electrum.electrum 1 ``` @@ -154,7 +124,7 @@ $ run-as org.electrum.electrum cp /data/data/org.electrum.electrum/files/data/wa ### How to investigate diff between binaries if reproducibility fails? ``` -cd bin/ +cd dist/ unzip Electrum-*.apk1 -d apk1 mkdir apk1/assets/private_mp3/ tar -xzvf apk1/assets/private.mp3 --directory apk1/assets/private_mp3/ diff --git a/contrib/android/build.sh b/contrib/android/build.sh new file mode 100755 index 000000000..f199eb533 --- /dev/null +++ b/contrib/android/build.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# +# env vars: +# - ELECBUILD_NOCACHE: if set, forces rebuild of docker image +# - ELECBUILD_COMMIT: if set, do a fresh clone and git checkout + +set -e + +PROJECT_ROOT="$(dirname "$(readlink -e "$0")")/../.." +PROJECT_ROOT_OR_FRESHCLONE_ROOT="$PROJECT_ROOT" +CONTRIB="$PROJECT_ROOT/contrib" +CONTRIB_ANDROID="$CONTRIB/android" +DISTDIR="$PROJECT_ROOT/dist" + +. "$CONTRIB"/build_tools_util.sh + + +DOCKER_BUILD_FLAGS="" +if [ ! -z "$ELECBUILD_NOCACHE" ] ; then + info "ELECBUILD_NOCACHE is set. forcing rebuild of docker image." + DOCKER_BUILD_FLAGS="--pull --no-cache" +fi + +info "building docker image." +cp "$CONTRIB/deterministic-build/requirements-build-android.txt" "$CONTRIB_ANDROID/requirements-build-android.txt" +sudo docker build \ + $DOCKER_BUILD_FLAGS \ + -t electrum-android-builder-img \ + "$CONTRIB_ANDROID" +rm "$CONTRIB_ANDROID/requirements-build-android.txt" + + +# maybe do fresh clone +if [ ! -z "$ELECBUILD_COMMIT" ] ; then + info "ELECBUILD_COMMIT=$ELECBUILD_COMMIT. doing fresh clone and git checkout." + FRESH_CLONE="$CONTRIB_ANDROID/fresh_clone/electrum" && \ + sudo rm -rf "$FRESH_CLONE" && \ + umask 0022 && \ + git clone "$PROJECT_ROOT" "$FRESH_CLONE" && \ + cd "$FRESH_CLONE" + git checkout "$ELECBUILD_COMMIT" + PROJECT_ROOT_OR_FRESHCLONE_ROOT="$FRESH_CLONE" +else + info "not doing fresh clone." +fi + +DOCKER_RUN_FLAGS="" +if [[ -n "$1" && "$1" == "release" ]] ; then + info "'release' mode selected. mounting ~/.keystore inside container." + DOCKER_RUN_FLAGS="-v $HOME/.keystore:/home/user/.keystore" +fi + +info "building binary..." +mkdir --parents "$PROJECT_ROOT_OR_FRESHCLONE_ROOT"/.buildozer/.gradle +sudo docker run -it --rm \ + --name electrum-android-builder-cont \ + -v "$PROJECT_ROOT_OR_FRESHCLONE_ROOT":/home/user/wspace/electrum \ + -v "$PROJECT_ROOT_OR_FRESHCLONE_ROOT"/.buildozer/.gradle:/home/user/.gradle \ + $DOCKER_RUN_FLAGS \ + --workdir /home/user/wspace/electrum \ + electrum-android-builder-img \ + ./contrib/android/make_apk "$@" + +# make sure resulting binary location is independent of fresh_clone +if [ ! -z "$ELECBUILD_COMMIT" ] ; then + mkdir --parents "$DISTDIR/" + sudo cp -f "$FRESH_CLONE/dist"/* "$DISTDIR/" +fi diff --git a/contrib/android/build_docker_image.sh b/contrib/android/build_docker_image.sh deleted file mode 100755 index 3eae956c5..000000000 --- a/contrib/android/build_docker_image.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -set -e - -CONTRIB_ANDROID="$(dirname "$(readlink -e "$0")")" -CONTRIB="$CONTRIB_ANDROID"/.. - -cp "$CONTRIB/deterministic-build/requirements-build-android.txt" "$CONTRIB_ANDROID/requirements-build-android.txt" -sudo docker build -t electrum-android-builder-img "$CONTRIB_ANDROID" -rm "$CONTRIB_ANDROID/requirements-build-android.txt" diff --git a/contrib/android/buildozer.spec b/contrib/android/buildozer.spec index 43d505352..aadb5c5b5 100644 --- a/contrib/android/buildozer.spec +++ b/contrib/android/buildozer.spec @@ -206,6 +206,9 @@ p4a.local_recipes = %(source.dir)s/contrib/android/p4a_recipes/ # (int) Log level (0 = error only, 1 = info, 2 = debug (with command output)) log_level = 1 +# (str) Path to build output (i.e. .apk, .ipa) storage +bin_dir = ./dist + # ----------------------------------------------------------------------------- # List as sections diff --git a/contrib/android/make_apk b/contrib/android/make_apk index 655d9da42..3248c3769 100755 --- a/contrib/android/make_apk +++ b/contrib/android/make_apk @@ -82,5 +82,5 @@ popd info "done." -ls -la "$PROJECT_ROOT/bin" -sha256sum "$PROJECT_ROOT/bin"/* +ls -la "$PROJECT_ROOT/dist" +sha256sum "$PROJECT_ROOT/dist"/* diff --git a/contrib/build-linux/appimage/README.md b/contrib/build-linux/appimage/README.md index 0b50ec7e3..838777b87 100644 --- a/contrib/build-linux/appimage/README.md +++ b/contrib/build-linux/appimage/README.md @@ -6,7 +6,6 @@ AppImage binary for Electrum This assumes an Ubuntu host, but it should not be too hard to adapt to another similar system. The host architecture should be x86_64 (amd64). -The docker commands should be executed in the project's root folder. We currently only build a single AppImage, for x86_64 architecture. Help to adapt these scripts to build for (some flavor of) ARM would be welcome, @@ -22,25 +21,17 @@ see [issue #5159](https://github.com/spesmilo/electrum/issues/5159). $ sudo apt-get install -y docker-ce ``` -2. Build image +2. Build binary ``` - $ sudo docker build -t electrum-appimage-builder-img contrib/build-linux/appimage + $ ./build.sh ``` - -3. Build binary - + If you want reproducibility, try instead e.g.: ``` - $ sudo docker run -it \ - --name electrum-appimage-builder-cont \ - -v $PWD:/opt/electrum \ - --rm \ - --workdir /opt/electrum/contrib/build-linux/appimage \ - electrum-appimage-builder-img \ - ./build.sh + $ ELECBUILD_COMMIT=HEAD ELECBUILD_NOCACHE=1 ./build.sh ``` -4. The generated binary is in `./dist`. +3. The generated binary is in `./dist`. ## FAQ diff --git a/contrib/build-linux/appimage/build.sh b/contrib/build-linux/appimage/build.sh index cf078bb90..65b361081 100755 --- a/contrib/build-linux/appimage/build.sh +++ b/contrib/build-linux/appimage/build.sh @@ -1,263 +1,57 @@ #!/bin/bash +# +# env vars: +# - ELECBUILD_NOCACHE: if set, forces rebuild of docker image +# - ELECBUILD_COMMIT: if set, do a fresh clone and git checkout set -e PROJECT_ROOT="$(dirname "$(readlink -e "$0")")/../../.." +PROJECT_ROOT_OR_FRESHCLONE_ROOT="$PROJECT_ROOT" CONTRIB="$PROJECT_ROOT/contrib" CONTRIB_APPIMAGE="$CONTRIB/build-linux/appimage" DISTDIR="$PROJECT_ROOT/dist" -BUILDDIR="$CONTRIB_APPIMAGE/build/appimage" -APPDIR="$BUILDDIR/electrum.AppDir" -CACHEDIR="$CONTRIB_APPIMAGE/.cache/appimage" -PIP_CACHE_DIR="$CACHEDIR/pip_cache" - -export GCC_STRIP_BINARIES="1" - -# pinned versions -# note: compiling python 3.8.x requires at least glibc 2.27, -# which is first available on ubuntu 18.04 -PYTHON_VERSION=3.7.10 -PKG2APPIMAGE_COMMIT="eb8f3acdd9f11ab19b78f5cb15daa772367daf15" -SQUASHFSKIT_COMMIT="ae0d656efa2d0df2fcac795b6823b44462f19386" - - -VERSION=`git describe --tags --dirty --always` -APPIMAGE="$DISTDIR/electrum-$VERSION-x86_64.AppImage" . "$CONTRIB"/build_tools_util.sh -rm -rf "$BUILDDIR" -mkdir -p "$APPDIR" "$CACHEDIR" "$PIP_CACHE_DIR" "$DISTDIR" - -# potential leftover from setuptools that might make pip put garbage in binary -rm -rf "$PROJECT_ROOT/build" - - -info "downloading some dependencies." -download_if_not_exist "$CACHEDIR/functions.sh" "https://raw.githubusercontent.com/AppImage/pkg2appimage/$PKG2APPIMAGE_COMMIT/functions.sh" -verify_hash "$CACHEDIR/functions.sh" "78b7ee5a04ffb84ee1c93f0cb2900123773bc6709e5d1e43c37519f590f86918" - -download_if_not_exist "$CACHEDIR/appimagetool" "https://github.com/AppImage/AppImageKit/releases/download/12/appimagetool-x86_64.AppImage" -verify_hash "$CACHEDIR/appimagetool" "d918b4df547b388ef253f3c9e7f6529ca81a885395c31f619d9aaf7030499a13" - -download_if_not_exist "$CACHEDIR/Python-$PYTHON_VERSION.tar.xz" "https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tar.xz" -verify_hash "$CACHEDIR/Python-$PYTHON_VERSION.tar.xz" "f8d82e7572c86ec9d55c8627aae5040124fd2203af400c383c821b980306ee6b" - - - -info "building python." -tar xf "$CACHEDIR/Python-$PYTHON_VERSION.tar.xz" -C "$BUILDDIR" -( - cd "$BUILDDIR/Python-$PYTHON_VERSION" - LC_ALL=C export BUILD_DATE=$(date -u -d "@$SOURCE_DATE_EPOCH" "+%b %d %Y") - LC_ALL=C export BUILD_TIME=$(date -u -d "@$SOURCE_DATE_EPOCH" "+%H:%M:%S") - # Patch taken from Ubuntu http://archive.ubuntu.com/ubuntu/pool/main/p/python3.7/python3.7_3.7.6-1.debian.tar.xz - patch -p1 < "$CONTRIB_APPIMAGE/patches/python-3.7-reproducible-buildinfo.diff" - ./configure \ - --cache-file="$CACHEDIR/python.config.cache" \ - --prefix="$APPDIR/usr" \ - --enable-ipv6 \ - --enable-shared \ - -q - make -j4 -s || fail "Could not build Python" - make -s install > /dev/null || fail "Could not install Python" - # When building in docker on macOS, python builds with .exe extension because the - # case insensitive file system of macOS leaks into docker. This causes the build - # to result in a different output on macOS compared to Linux. We simply patch - # sysconfigdata to remove the extension. - # Some more info: https://bugs.python.org/issue27631 - sed -i -e 's/\.exe//g' "$APPDIR"/usr/lib/python3.7/_sysconfigdata* -) - - -info "Building squashfskit" -git clone "https://github.com/squashfskit/squashfskit.git" "$BUILDDIR/squashfskit" -( - cd "$BUILDDIR/squashfskit" - git checkout "${SQUASHFSKIT_COMMIT}^{commit}" - make -C squashfs-tools mksquashfs || fail "Could not build squashfskit" -) -MKSQUASHFS="$BUILDDIR/squashfskit/squashfs-tools/mksquashfs" - - -"$CONTRIB"/make_libsecp256k1.sh || fail "Could not build libsecp" -cp -f "$PROJECT_ROOT/electrum/libsecp256k1.so.0" "$APPDIR/usr/lib/libsecp256k1.so.0" || fail "Could not copy libsecp to its destination" - - -appdir_python() { - env \ - PYTHONNOUSERSITE=1 \ - LD_LIBRARY_PATH="$APPDIR/usr/lib:$APPDIR/usr/lib/x86_64-linux-gnu${LD_LIBRARY_PATH+:$LD_LIBRARY_PATH}" \ - "$APPDIR/usr/bin/python3.7" "$@" -} - -python='appdir_python' - - -info "installing pip." -"$python" -m ensurepip - -break_legacy_easy_install - - -info "preparing electrum-locale." -( - cd "$PROJECT_ROOT" - git submodule update --init - - pushd "$CONTRIB"/deterministic-build/electrum-locale - if ! which msgfmt > /dev/null 2>&1; then - fail "Please install gettext" - fi - # we want the binary to have only compiled (.mo) locale files; not source (.po) files - rm -rf "$PROJECT_ROOT/electrum/locale/" - for i in ./locale/*; do - dir="$PROJECT_ROOT/electrum/$i/LC_MESSAGES" - mkdir -p $dir - msgfmt --output-file="$dir/electrum.mo" "$i/electrum.po" || true - done - popd -) - - -info "Installing build dependencies." -"$python" -m pip install --no-dependencies --no-binary :all: --no-warn-script-location \ - --cache-dir "$PIP_CACHE_DIR" -r "$CONTRIB/deterministic-build/requirements-build-appimage.txt" - -info "installing electrum and its dependencies." -# note: we prefer compiling C extensions ourselves, instead of using binary wheels, -# hence "--no-binary :all:" flags. However, we specifically allow -# - PyQt5, as it's harder to build from source -# - cryptography, as building it would need openssl 1.1, not available on ubuntu 16.04 -"$python" -m pip install --no-dependencies --no-binary :all: --no-warn-script-location \ - --cache-dir "$PIP_CACHE_DIR" -r "$CONTRIB/deterministic-build/requirements.txt" -"$python" -m pip install --no-dependencies --no-binary :all: --only-binary PyQt5,PyQt5-Qt5,cryptography --no-warn-script-location \ - --cache-dir "$PIP_CACHE_DIR" -r "$CONTRIB/deterministic-build/requirements-binaries.txt" -"$python" -m pip install --no-dependencies --no-binary :all: --no-warn-script-location \ - --cache-dir "$PIP_CACHE_DIR" -r "$CONTRIB/deterministic-build/requirements-hw.txt" - -"$python" -m pip install --no-dependencies --no-warn-script-location \ - --cache-dir "$PIP_CACHE_DIR" "$PROJECT_ROOT" - -# was only needed during build time, not runtime -"$python" -m pip uninstall -y Cython - - -info "copying zbar" -cp "/usr/lib/x86_64-linux-gnu/libzbar.so.0" "$APPDIR/usr/lib/libzbar.so.0" - - -info "desktop integration." -cp "$PROJECT_ROOT/electrum.desktop" "$APPDIR/electrum.desktop" -cp "$PROJECT_ROOT/electrum/gui/icons/electrum.png" "$APPDIR/electrum.png" - - -# add launcher -cp "$CONTRIB_APPIMAGE/apprun.sh" "$APPDIR/AppRun" - -info "finalizing AppDir." -( - export PKG2AICOMMIT="$PKG2APPIMAGE_COMMIT" - . "$CACHEDIR/functions.sh" - - cd "$APPDIR" - # copy system dependencies - copy_deps; copy_deps; copy_deps - move_lib - - # apply global appimage blacklist to exclude stuff - # move usr/include out of the way to preserve usr/include/python3.7m. - mv usr/include usr/include.tmp - delete_blacklisted - mv usr/include.tmp usr/include -) || fail "Could not finalize AppDir" - -info "Copying additional libraries" -( - # On some systems it can cause problems to use the system libusb (on AppImage excludelist) - cp -f /usr/lib/x86_64-linux-gnu/libusb-1.0.so "$APPDIR/usr/lib/libusb-1.0.so" || fail "Could not copy libusb" - # some distros lack libxkbcommon-x11 - cp -f /usr/lib/x86_64-linux-gnu/libxkbcommon-x11.so.0 "$APPDIR"/usr/lib/x86_64-linux-gnu || fail "Could not copy libxkbcommon-x11" - # some distros lack some libxcb libraries (see https://github.com/Electron-Cash/Electron-Cash/issues/2196) - cp -f /usr/lib/x86_64-linux-gnu/libxcb-* "$APPDIR"/usr/lib/x86_64-linux-gnu || fail "Could not copy libxcb" -) - -info "stripping binaries from debug symbols." -# "-R .note.gnu.build-id" also strips the build id -# "-R .comment" also strips the GCC version information -strip_binaries() -{ - chmod u+w -R "$APPDIR" - { - printf '%s\0' "$APPDIR/usr/bin/python3.7" - find "$APPDIR" -type f -regex '.*\.so\(\.[0-9.]+\)?$' -print0 - } | xargs -0 --no-run-if-empty --verbose strip -R .note.gnu.build-id -R .comment -} -strip_binaries - -remove_emptydirs() -{ - find "$APPDIR" -type d -empty -print0 | xargs -0 --no-run-if-empty rmdir -vp --ignore-fail-on-non-empty -} -remove_emptydirs - - -info "removing some unneeded stuff to decrease binary size." -rm -rf "$APPDIR"/usr/{share,include} -PYDIR="$APPDIR"/usr/lib/python3.7 -rm -rf "$PYDIR"/{test,ensurepip,lib2to3,idlelib,turtledemo} -rm -rf "$PYDIR"/{ctypes,sqlite3,tkinter,unittest}/test -rm -rf "$PYDIR"/distutils/{command,tests} -rm -rf "$PYDIR"/config-3.7m-x86_64-linux-gnu -rm -rf "$PYDIR"/site-packages/{opt,pip,setuptools,wheel} -rm -rf "$PYDIR"/site-packages/Cryptodome/SelfTest -rm -rf "$PYDIR"/site-packages/{psutil,qrcode,websocket}/tests -# rm lots of unused parts of Qt/PyQt. (assuming PyQt 5.15.3+ layout) -for component in connectivity declarative help location multimedia quickcontrols2 serialport webengine websockets xmlpatterns ; do - rm -rf "$PYDIR"/site-packages/PyQt5/Qt5/translations/qt${component}_* - rm -rf "$PYDIR"/site-packages/PyQt5/Qt5/resources/qt${component}_* -done -rm -rf "$PYDIR"/site-packages/PyQt5/Qt5/{qml,libexec} -rm -rf "$PYDIR"/site-packages/PyQt5/{pyrcc*.so,pylupdate*.so,uic} -rm -rf "$PYDIR"/site-packages/PyQt5/Qt5/plugins/{bearer,gamepads,geometryloaders,geoservices,playlistformats,position,renderplugins,sceneparsers,sensors,sqldrivers,texttospeech,webview} -for component in Bluetooth Concurrent Designer Help Location NetworkAuth Nfc Positioning PositioningQuick Qml Quick Sensors SerialPort Sql Test Web Xml ; do - rm -rf "$PYDIR"/site-packages/PyQt5/Qt5/lib/libQt5${component}* - rm -rf "$PYDIR"/site-packages/PyQt5/Qt${component}* -done -rm -rf "$PYDIR"/site-packages/PyQt5/Qt.so - -# these are deleted as they were not deterministic; and are not needed anyway -find "$APPDIR" -path '*/__pycache__*' -delete -# note that *.dist-info is needed by certain packages. -# e.g. see https://gitlab.com/python-devs/importlib_metadata/issues/71 -for f in "$PYDIR"/site-packages/importlib_metadata-*.dist-info; do mv "$f" "$(echo "$f" | sed s/\.dist-info/\.dist-info2/)"; done -rm -rf "$PYDIR"/site-packages/*.dist-info/ -rm -rf "$PYDIR"/site-packages/*.egg-info/ -for f in "$PYDIR"/site-packages/importlib_metadata-*.dist-info2; do mv "$f" "$(echo "$f" | sed s/\.dist-info2/\.dist-info/)"; done - - -find -exec touch -h -d '2000-11-11T11:11:11+00:00' {} + - - -info "creating the AppImage." -( - cd "$BUILDDIR" - cp "$CACHEDIR/appimagetool" "$CACHEDIR/appimagetool_copy" - # zero out "appimage" magic bytes, as on some systems they confuse the linker - sed -i 's|AI\x02|\x00\x00\x00|' "$CACHEDIR/appimagetool_copy" - chmod +x "$CACHEDIR/appimagetool_copy" - "$CACHEDIR/appimagetool_copy" --appimage-extract - # We build a small wrapper for mksquashfs that removes the -mkfs-fixed-time option - # that mksquashfs from squashfskit does not support. It is not needed for squashfskit. - cat > ./squashfs-root/usr/lib/appimagekit/mksquashfs << EOF -#!/bin/sh -args=\$(echo "\$@" | sed -e 's/-mkfs-fixed-time 0//') -"$MKSQUASHFS" \$args -EOF - env VERSION="$VERSION" ARCH=x86_64 SOURCE_DATE_EPOCH=1530212462 ./squashfs-root/AppRun --no-appstream --verbose "$APPDIR" "$APPIMAGE" -) - -info "done." -ls -la "$DISTDIR" -sha256sum "$DISTDIR"/* +DOCKER_BUILD_FLAGS="" +if [ ! -z "$ELECBUILD_NOCACHE" ] ; then + info "ELECBUILD_NOCACHE is set. forcing rebuild of docker image." + DOCKER_BUILD_FLAGS="--pull --no-cache" +fi + +info "building docker image." +sudo docker build \ + $DOCKER_BUILD_FLAGS \ + -t electrum-appimage-builder-img \ + "$CONTRIB_APPIMAGE" + +# maybe do fresh clone +if [ ! -z "$ELECBUILD_COMMIT" ] ; then + info "ELECBUILD_COMMIT=$ELECBUILD_COMMIT. doing fresh clone and git checkout." + FRESH_CLONE="$CONTRIB_APPIMAGE/fresh_clone/electrum" && \ + sudo rm -rf "$FRESH_CLONE" && \ + umask 0022 && \ + git clone "$PROJECT_ROOT" "$FRESH_CLONE" && \ + cd "$FRESH_CLONE" + git checkout "$ELECBUILD_COMMIT" + PROJECT_ROOT_OR_FRESHCLONE_ROOT="$FRESH_CLONE" +else + info "not doing fresh clone." +fi + +info "building binary..." +sudo docker run -it \ + --name electrum-appimage-builder-cont \ + -v "$PROJECT_ROOT_OR_FRESHCLONE_ROOT":/opt/electrum \ + --rm \ + --workdir /opt/electrum/contrib/build-linux/appimage \ + electrum-appimage-builder-img \ + ./make_appimage.sh + +# make sure resulting binary location is independent of fresh_clone +if [ ! -z "$ELECBUILD_COMMIT" ] ; then + mkdir --parents "$DISTDIR/" + sudo cp -f "$FRESH_CLONE/dist"/* "$DISTDIR/" +fi diff --git a/contrib/build-linux/appimage/make_appimage.sh b/contrib/build-linux/appimage/make_appimage.sh new file mode 100755 index 000000000..cf078bb90 --- /dev/null +++ b/contrib/build-linux/appimage/make_appimage.sh @@ -0,0 +1,263 @@ +#!/bin/bash + +set -e + +PROJECT_ROOT="$(dirname "$(readlink -e "$0")")/../../.." +CONTRIB="$PROJECT_ROOT/contrib" +CONTRIB_APPIMAGE="$CONTRIB/build-linux/appimage" +DISTDIR="$PROJECT_ROOT/dist" +BUILDDIR="$CONTRIB_APPIMAGE/build/appimage" +APPDIR="$BUILDDIR/electrum.AppDir" +CACHEDIR="$CONTRIB_APPIMAGE/.cache/appimage" +PIP_CACHE_DIR="$CACHEDIR/pip_cache" + +export GCC_STRIP_BINARIES="1" + +# pinned versions +# note: compiling python 3.8.x requires at least glibc 2.27, +# which is first available on ubuntu 18.04 +PYTHON_VERSION=3.7.10 +PKG2APPIMAGE_COMMIT="eb8f3acdd9f11ab19b78f5cb15daa772367daf15" +SQUASHFSKIT_COMMIT="ae0d656efa2d0df2fcac795b6823b44462f19386" + + +VERSION=`git describe --tags --dirty --always` +APPIMAGE="$DISTDIR/electrum-$VERSION-x86_64.AppImage" + +. "$CONTRIB"/build_tools_util.sh + +rm -rf "$BUILDDIR" +mkdir -p "$APPDIR" "$CACHEDIR" "$PIP_CACHE_DIR" "$DISTDIR" + +# potential leftover from setuptools that might make pip put garbage in binary +rm -rf "$PROJECT_ROOT/build" + + +info "downloading some dependencies." +download_if_not_exist "$CACHEDIR/functions.sh" "https://raw.githubusercontent.com/AppImage/pkg2appimage/$PKG2APPIMAGE_COMMIT/functions.sh" +verify_hash "$CACHEDIR/functions.sh" "78b7ee5a04ffb84ee1c93f0cb2900123773bc6709e5d1e43c37519f590f86918" + +download_if_not_exist "$CACHEDIR/appimagetool" "https://github.com/AppImage/AppImageKit/releases/download/12/appimagetool-x86_64.AppImage" +verify_hash "$CACHEDIR/appimagetool" "d918b4df547b388ef253f3c9e7f6529ca81a885395c31f619d9aaf7030499a13" + +download_if_not_exist "$CACHEDIR/Python-$PYTHON_VERSION.tar.xz" "https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tar.xz" +verify_hash "$CACHEDIR/Python-$PYTHON_VERSION.tar.xz" "f8d82e7572c86ec9d55c8627aae5040124fd2203af400c383c821b980306ee6b" + + + +info "building python." +tar xf "$CACHEDIR/Python-$PYTHON_VERSION.tar.xz" -C "$BUILDDIR" +( + cd "$BUILDDIR/Python-$PYTHON_VERSION" + LC_ALL=C export BUILD_DATE=$(date -u -d "@$SOURCE_DATE_EPOCH" "+%b %d %Y") + LC_ALL=C export BUILD_TIME=$(date -u -d "@$SOURCE_DATE_EPOCH" "+%H:%M:%S") + # Patch taken from Ubuntu http://archive.ubuntu.com/ubuntu/pool/main/p/python3.7/python3.7_3.7.6-1.debian.tar.xz + patch -p1 < "$CONTRIB_APPIMAGE/patches/python-3.7-reproducible-buildinfo.diff" + ./configure \ + --cache-file="$CACHEDIR/python.config.cache" \ + --prefix="$APPDIR/usr" \ + --enable-ipv6 \ + --enable-shared \ + -q + make -j4 -s || fail "Could not build Python" + make -s install > /dev/null || fail "Could not install Python" + # When building in docker on macOS, python builds with .exe extension because the + # case insensitive file system of macOS leaks into docker. This causes the build + # to result in a different output on macOS compared to Linux. We simply patch + # sysconfigdata to remove the extension. + # Some more info: https://bugs.python.org/issue27631 + sed -i -e 's/\.exe//g' "$APPDIR"/usr/lib/python3.7/_sysconfigdata* +) + + +info "Building squashfskit" +git clone "https://github.com/squashfskit/squashfskit.git" "$BUILDDIR/squashfskit" +( + cd "$BUILDDIR/squashfskit" + git checkout "${SQUASHFSKIT_COMMIT}^{commit}" + make -C squashfs-tools mksquashfs || fail "Could not build squashfskit" +) +MKSQUASHFS="$BUILDDIR/squashfskit/squashfs-tools/mksquashfs" + + +"$CONTRIB"/make_libsecp256k1.sh || fail "Could not build libsecp" +cp -f "$PROJECT_ROOT/electrum/libsecp256k1.so.0" "$APPDIR/usr/lib/libsecp256k1.so.0" || fail "Could not copy libsecp to its destination" + + +appdir_python() { + env \ + PYTHONNOUSERSITE=1 \ + LD_LIBRARY_PATH="$APPDIR/usr/lib:$APPDIR/usr/lib/x86_64-linux-gnu${LD_LIBRARY_PATH+:$LD_LIBRARY_PATH}" \ + "$APPDIR/usr/bin/python3.7" "$@" +} + +python='appdir_python' + + +info "installing pip." +"$python" -m ensurepip + +break_legacy_easy_install + + +info "preparing electrum-locale." +( + cd "$PROJECT_ROOT" + git submodule update --init + + pushd "$CONTRIB"/deterministic-build/electrum-locale + if ! which msgfmt > /dev/null 2>&1; then + fail "Please install gettext" + fi + # we want the binary to have only compiled (.mo) locale files; not source (.po) files + rm -rf "$PROJECT_ROOT/electrum/locale/" + for i in ./locale/*; do + dir="$PROJECT_ROOT/electrum/$i/LC_MESSAGES" + mkdir -p $dir + msgfmt --output-file="$dir/electrum.mo" "$i/electrum.po" || true + done + popd +) + + +info "Installing build dependencies." +"$python" -m pip install --no-dependencies --no-binary :all: --no-warn-script-location \ + --cache-dir "$PIP_CACHE_DIR" -r "$CONTRIB/deterministic-build/requirements-build-appimage.txt" + +info "installing electrum and its dependencies." +# note: we prefer compiling C extensions ourselves, instead of using binary wheels, +# hence "--no-binary :all:" flags. However, we specifically allow +# - PyQt5, as it's harder to build from source +# - cryptography, as building it would need openssl 1.1, not available on ubuntu 16.04 +"$python" -m pip install --no-dependencies --no-binary :all: --no-warn-script-location \ + --cache-dir "$PIP_CACHE_DIR" -r "$CONTRIB/deterministic-build/requirements.txt" +"$python" -m pip install --no-dependencies --no-binary :all: --only-binary PyQt5,PyQt5-Qt5,cryptography --no-warn-script-location \ + --cache-dir "$PIP_CACHE_DIR" -r "$CONTRIB/deterministic-build/requirements-binaries.txt" +"$python" -m pip install --no-dependencies --no-binary :all: --no-warn-script-location \ + --cache-dir "$PIP_CACHE_DIR" -r "$CONTRIB/deterministic-build/requirements-hw.txt" + +"$python" -m pip install --no-dependencies --no-warn-script-location \ + --cache-dir "$PIP_CACHE_DIR" "$PROJECT_ROOT" + +# was only needed during build time, not runtime +"$python" -m pip uninstall -y Cython + + +info "copying zbar" +cp "/usr/lib/x86_64-linux-gnu/libzbar.so.0" "$APPDIR/usr/lib/libzbar.so.0" + + +info "desktop integration." +cp "$PROJECT_ROOT/electrum.desktop" "$APPDIR/electrum.desktop" +cp "$PROJECT_ROOT/electrum/gui/icons/electrum.png" "$APPDIR/electrum.png" + + +# add launcher +cp "$CONTRIB_APPIMAGE/apprun.sh" "$APPDIR/AppRun" + +info "finalizing AppDir." +( + export PKG2AICOMMIT="$PKG2APPIMAGE_COMMIT" + . "$CACHEDIR/functions.sh" + + cd "$APPDIR" + # copy system dependencies + copy_deps; copy_deps; copy_deps + move_lib + + # apply global appimage blacklist to exclude stuff + # move usr/include out of the way to preserve usr/include/python3.7m. + mv usr/include usr/include.tmp + delete_blacklisted + mv usr/include.tmp usr/include +) || fail "Could not finalize AppDir" + +info "Copying additional libraries" +( + # On some systems it can cause problems to use the system libusb (on AppImage excludelist) + cp -f /usr/lib/x86_64-linux-gnu/libusb-1.0.so "$APPDIR/usr/lib/libusb-1.0.so" || fail "Could not copy libusb" + # some distros lack libxkbcommon-x11 + cp -f /usr/lib/x86_64-linux-gnu/libxkbcommon-x11.so.0 "$APPDIR"/usr/lib/x86_64-linux-gnu || fail "Could not copy libxkbcommon-x11" + # some distros lack some libxcb libraries (see https://github.com/Electron-Cash/Electron-Cash/issues/2196) + cp -f /usr/lib/x86_64-linux-gnu/libxcb-* "$APPDIR"/usr/lib/x86_64-linux-gnu || fail "Could not copy libxcb" +) + +info "stripping binaries from debug symbols." +# "-R .note.gnu.build-id" also strips the build id +# "-R .comment" also strips the GCC version information +strip_binaries() +{ + chmod u+w -R "$APPDIR" + { + printf '%s\0' "$APPDIR/usr/bin/python3.7" + find "$APPDIR" -type f -regex '.*\.so\(\.[0-9.]+\)?$' -print0 + } | xargs -0 --no-run-if-empty --verbose strip -R .note.gnu.build-id -R .comment +} +strip_binaries + +remove_emptydirs() +{ + find "$APPDIR" -type d -empty -print0 | xargs -0 --no-run-if-empty rmdir -vp --ignore-fail-on-non-empty +} +remove_emptydirs + + +info "removing some unneeded stuff to decrease binary size." +rm -rf "$APPDIR"/usr/{share,include} +PYDIR="$APPDIR"/usr/lib/python3.7 +rm -rf "$PYDIR"/{test,ensurepip,lib2to3,idlelib,turtledemo} +rm -rf "$PYDIR"/{ctypes,sqlite3,tkinter,unittest}/test +rm -rf "$PYDIR"/distutils/{command,tests} +rm -rf "$PYDIR"/config-3.7m-x86_64-linux-gnu +rm -rf "$PYDIR"/site-packages/{opt,pip,setuptools,wheel} +rm -rf "$PYDIR"/site-packages/Cryptodome/SelfTest +rm -rf "$PYDIR"/site-packages/{psutil,qrcode,websocket}/tests +# rm lots of unused parts of Qt/PyQt. (assuming PyQt 5.15.3+ layout) +for component in connectivity declarative help location multimedia quickcontrols2 serialport webengine websockets xmlpatterns ; do + rm -rf "$PYDIR"/site-packages/PyQt5/Qt5/translations/qt${component}_* + rm -rf "$PYDIR"/site-packages/PyQt5/Qt5/resources/qt${component}_* +done +rm -rf "$PYDIR"/site-packages/PyQt5/Qt5/{qml,libexec} +rm -rf "$PYDIR"/site-packages/PyQt5/{pyrcc*.so,pylupdate*.so,uic} +rm -rf "$PYDIR"/site-packages/PyQt5/Qt5/plugins/{bearer,gamepads,geometryloaders,geoservices,playlistformats,position,renderplugins,sceneparsers,sensors,sqldrivers,texttospeech,webview} +for component in Bluetooth Concurrent Designer Help Location NetworkAuth Nfc Positioning PositioningQuick Qml Quick Sensors SerialPort Sql Test Web Xml ; do + rm -rf "$PYDIR"/site-packages/PyQt5/Qt5/lib/libQt5${component}* + rm -rf "$PYDIR"/site-packages/PyQt5/Qt${component}* +done +rm -rf "$PYDIR"/site-packages/PyQt5/Qt.so + +# these are deleted as they were not deterministic; and are not needed anyway +find "$APPDIR" -path '*/__pycache__*' -delete +# note that *.dist-info is needed by certain packages. +# e.g. see https://gitlab.com/python-devs/importlib_metadata/issues/71 +for f in "$PYDIR"/site-packages/importlib_metadata-*.dist-info; do mv "$f" "$(echo "$f" | sed s/\.dist-info/\.dist-info2/)"; done +rm -rf "$PYDIR"/site-packages/*.dist-info/ +rm -rf "$PYDIR"/site-packages/*.egg-info/ +for f in "$PYDIR"/site-packages/importlib_metadata-*.dist-info2; do mv "$f" "$(echo "$f" | sed s/\.dist-info2/\.dist-info/)"; done + + +find -exec touch -h -d '2000-11-11T11:11:11+00:00' {} + + + +info "creating the AppImage." +( + cd "$BUILDDIR" + cp "$CACHEDIR/appimagetool" "$CACHEDIR/appimagetool_copy" + # zero out "appimage" magic bytes, as on some systems they confuse the linker + sed -i 's|AI\x02|\x00\x00\x00|' "$CACHEDIR/appimagetool_copy" + chmod +x "$CACHEDIR/appimagetool_copy" + "$CACHEDIR/appimagetool_copy" --appimage-extract + # We build a small wrapper for mksquashfs that removes the -mkfs-fixed-time option + # that mksquashfs from squashfskit does not support. It is not needed for squashfskit. + cat > ./squashfs-root/usr/lib/appimagekit/mksquashfs << EOF +#!/bin/sh +args=\$(echo "\$@" | sed -e 's/-mkfs-fixed-time 0//') +"$MKSQUASHFS" \$args +EOF + env VERSION="$VERSION" ARCH=x86_64 SOURCE_DATE_EPOCH=1530212462 ./squashfs-root/AppRun --no-appstream --verbose "$APPDIR" "$APPIMAGE" +) + + +info "done." +ls -la "$DISTDIR" +sha256sum "$DISTDIR"/* diff --git a/contrib/build-linux/sdist/README.md b/contrib/build-linux/sdist/README.md index 20aef2b56..2ff3ed011 100644 --- a/contrib/build-linux/sdist/README.md +++ b/contrib/build-linux/sdist/README.md @@ -5,8 +5,7 @@ Source tarballs distributables that match the official releases._ This assumes an Ubuntu (x86_64) host, but it should not be too hard to adapt to another -similar system. The docker commands should be executed in the project's root -folder. +similar system. 1. Install Docker @@ -17,36 +16,14 @@ folder. $ sudo apt-get install -y docker-ce ``` -2. Build image +2. Build source tarball ``` - $ sudo docker build -t electrum-sdist-builder-img contrib/build-linux/sdist + $ ./build.sh ``` - -3. Build source tarballs - - It's recommended to build from a fresh clone - (but you can skip this if reproducibility is not necessary). - + If you want reproducibility, try instead e.g.: ``` - $ FRESH_CLONE=contrib/build-linux/sdist/fresh_clone && \ - sudo rm -rf $FRESH_CLONE && \ - umask 0022 && \ - mkdir -p $FRESH_CLONE && \ - cd $FRESH_CLONE && \ - git clone https://github.com/spesmilo/electrum.git && \ - cd electrum + $ ELECBUILD_COMMIT=HEAD ELECBUILD_NOCACHE=1 ./build.sh ``` - And then build from this directory: - ``` - $ git checkout $REV - $ sudo docker run -it \ - --name electrum-sdist-builder-cont \ - -v $PWD:/opt/electrum \ - --rm \ - --workdir /opt/electrum/contrib/build-linux/sdist \ - electrum-sdist-builder-img \ - ./build.sh - ``` -4. The generated distributables are in `./dist`. +3. The generated distributables are in `./dist`. diff --git a/contrib/build-linux/sdist/build.sh b/contrib/build-linux/sdist/build.sh index 16d2bcd28..5f64c160e 100755 --- a/contrib/build-linux/sdist/build.sh +++ b/contrib/build-linux/sdist/build.sh @@ -1,32 +1,57 @@ #!/bin/bash +# +# env vars: +# - ELECBUILD_NOCACHE: if set, forces rebuild of docker image +# - ELECBUILD_COMMIT: if set, do a fresh clone and git checkout set -e PROJECT_ROOT="$(dirname "$(readlink -e "$0")")/../../.." +PROJECT_ROOT_OR_FRESHCLONE_ROOT="$PROJECT_ROOT" CONTRIB="$PROJECT_ROOT/contrib" CONTRIB_SDIST="$CONTRIB/build-linux/sdist" DISTDIR="$PROJECT_ROOT/dist" . "$CONTRIB"/build_tools_util.sh -# note that at least py3.7 is needed, to have https://bugs.python.org/issue30693 -python3 --version || fail "python interpreter not found" -break_legacy_easy_install - -# upgrade to modern pip so that it knows the flags we need. -# we will then install a pinned version of pip as part of requirements-build-sdist -python3 -m pip install --upgrade pip - -info "Installing pinned requirements." -python3 -m pip install --no-dependencies --no-warn-script-location -r "$CONTRIB"/deterministic-build/requirements-build-sdist.txt - - -"$CONTRIB"/make_packages || fail "make_packages failed" - -"$CONTRIB_SDIST"/make_tgz || fail "make_tgz failed" - - -info "done." -ls -la "$DISTDIR" -sha256sum "$DISTDIR"/* +DOCKER_BUILD_FLAGS="" +if [ ! -z "$ELECBUILD_NOCACHE" ] ; then + info "ELECBUILD_NOCACHE is set. forcing rebuild of docker image." + DOCKER_BUILD_FLAGS="--pull --no-cache" +fi + +info "building docker image." +sudo docker build \ + $DOCKER_BUILD_FLAGS \ + -t electrum-sdist-builder-img \ + "$CONTRIB_SDIST" + +# maybe do fresh clone +if [ ! -z "$ELECBUILD_COMMIT" ] ; then + info "ELECBUILD_COMMIT=$ELECBUILD_COMMIT. doing fresh clone and git checkout." + FRESH_CLONE="$CONTRIB_SDIST/fresh_clone/electrum" && \ + sudo rm -rf "$FRESH_CLONE" && \ + umask 0022 && \ + git clone "$PROJECT_ROOT" "$FRESH_CLONE" && \ + cd "$FRESH_CLONE" + git checkout "$ELECBUILD_COMMIT" + PROJECT_ROOT_OR_FRESHCLONE_ROOT="$FRESH_CLONE" +else + info "not doing fresh clone." +fi + +info "building binary..." +sudo docker run -it \ + --name electrum-sdist-builder-cont \ + -v "$PROJECT_ROOT_OR_FRESHCLONE_ROOT":/opt/electrum \ + --rm \ + --workdir /opt/electrum/contrib/build-linux/sdist \ + electrum-sdist-builder-img \ + ./make_sdist.sh + +# make sure resulting binary location is independent of fresh_clone +if [ ! -z "$ELECBUILD_COMMIT" ] ; then + mkdir --parents "$DISTDIR/" + sudo cp -f "$FRESH_CLONE/dist"/* "$DISTDIR/" +fi diff --git a/contrib/build-linux/sdist/make_sdist.sh b/contrib/build-linux/sdist/make_sdist.sh new file mode 100755 index 000000000..7aae3f423 --- /dev/null +++ b/contrib/build-linux/sdist/make_sdist.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +set -e + +PROJECT_ROOT="$(dirname "$(readlink -e "$0")")/../../.." +CONTRIB="$PROJECT_ROOT/contrib" +CONTRIB_SDIST="$CONTRIB/build-linux/sdist" +DISTDIR="$PROJECT_ROOT/dist" +LOCALE="$PROJECT_ROOT/electrum/locale/" + +. "$CONTRIB"/build_tools_util.sh + +# note that at least py3.7 is needed, to have https://bugs.python.org/issue30693 +python3 --version || fail "python interpreter not found" + +break_legacy_easy_install + +# upgrade to modern pip so that it knows the flags we need. +# we will then install a pinned version of pip as part of requirements-build-sdist +python3 -m pip install --upgrade pip + +info "Installing pinned requirements." +python3 -m pip install --no-dependencies --no-warn-script-location -r "$CONTRIB"/deterministic-build/requirements-build-sdist.txt + + +"$CONTRIB"/make_packages || fail "make_packages failed" + +git submodule update --init + +( + cd "$CONTRIB/deterministic-build/electrum-locale/" + if ! which msgfmt > /dev/null 2>&1; then + echo "Please install gettext" + exit 1 + fi + # We include both source (.po) and compiled (.mo) locale files in the source dist. + # Maybe we should exclude the compiled locale files? see https://askubuntu.com/a/144139 + # (also see MANIFEST.in) + rm -rf "$LOCALE" + for i in ./locale/*; do + dir="$PROJECT_ROOT/electrum/$i/LC_MESSAGES" + mkdir -p "$dir" + msgfmt --output-file="$dir/electrum.mo" "$i/electrum.po" || true + cp $i/electrum.po "$PROJECT_ROOT/electrum/$i/electrum.po" + done +) + +( + cd "$PROJECT_ROOT" + + find -exec touch -h -d '2000-11-11T11:11:11+00:00' {} + + + # note: .zip sdists would not be reproducible due to https://bugs.python.org/issue40963 + TZ=UTC faketime -f '2000-11-11 11:11:11' python3 setup.py --quiet sdist --format=gztar +) + + +info "done." +ls -la "$DISTDIR" +sha256sum "$DISTDIR"/* diff --git a/contrib/build-linux/sdist/make_tgz b/contrib/build-linux/sdist/make_tgz deleted file mode 100755 index b61bbbbbc..000000000 --- a/contrib/build-linux/sdist/make_tgz +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash - -set -e - -CONTRIB_SDIST="$(dirname "$(readlink -e "$0")")" -CONTRIB="$CONTRIB_SDIST"/../.. -ROOT_FOLDER="$CONTRIB"/.. -PACKAGES="$ROOT_FOLDER"/packages/ -LOCALE="$ROOT_FOLDER"/electrum/locale/ - -if [ ! -d "$PACKAGES" ]; then - echo "Run make_packages first!" - exit 1 -fi - -git submodule update --init - -( - cd "$CONTRIB/deterministic-build/electrum-locale/" - if ! which msgfmt > /dev/null 2>&1; then - echo "Please install gettext" - exit 1 - fi - # We include both source (.po) and compiled (.mo) locale files in the source dist. - # Maybe we should exclude the compiled locale files? see https://askubuntu.com/a/144139 - # (also see MANIFEST.in) - rm -rf "$LOCALE" - for i in ./locale/*; do - dir="$ROOT_FOLDER/electrum/$i/LC_MESSAGES" - mkdir -p "$dir" - msgfmt --output-file="$dir/electrum.mo" "$i/electrum.po" || true - cp $i/electrum.po "$ROOT_FOLDER/electrum/$i/electrum.po" - done -) - -( - cd "$ROOT_FOLDER" - - find -exec touch -h -d '2000-11-11T11:11:11+00:00' {} + - - # note: .zip sdists would not be reproducible due to https://bugs.python.org/issue40963 - TZ=UTC faketime -f '2000-11-11 11:11:11' python3 setup.py --quiet sdist --format=gztar -) diff --git a/contrib/build-wine/README.md b/contrib/build-wine/README.md index 41bd906cd..61fdf205d 100644 --- a/contrib/build-wine/README.md +++ b/contrib/build-wine/README.md @@ -5,8 +5,7 @@ Windows binaries binaries that match the official releases._ This assumes an Ubuntu (x86_64) host, but it should not be too hard to adapt to another -similar system. The docker commands should be executed in the project's root -folder. +similar system. 1. Install Docker @@ -21,40 +20,17 @@ folder. (see [#6971](https://github.com/spesmilo/electrum/issues/6971)). If having problems, try to upgrade to at least `docker 20.10`. -2. Build image +2. Build Windows binaries ``` - $ sudo docker build -t electrum-wine-builder-img contrib/build-wine + $ ./build.sh ``` - - Note: see [this](https://stackoverflow.com/a/40516974/7499128) if having dns problems - -3. Build Windows binaries - - It's recommended to build from a fresh clone - (but you can skip this if reproducibility is not necessary). - + If you want reproducibility, try instead e.g.: ``` - $ FRESH_CLONE=contrib/build-wine/fresh_clone && \ - sudo rm -rf $FRESH_CLONE && \ - mkdir -p $FRESH_CLONE && \ - cd $FRESH_CLONE && \ - git clone https://github.com/spesmilo/electrum.git && \ - cd electrum + $ ELECBUILD_COMMIT=HEAD ELECBUILD_NOCACHE=1 ./build.sh ``` - And then build from this directory: - ``` - $ git checkout $REV - $ sudo docker run -it \ - --name electrum-wine-builder-cont \ - -v $PWD:/opt/wine64/drive_c/electrum \ - --rm \ - --workdir /opt/wine64/drive_c/electrum/contrib/build-wine \ - electrum-wine-builder-img \ - ./build.sh - ``` -4. The generated binaries are in `./contrib/build-wine/dist`. +3. The generated binaries are in `./contrib/build-wine/dist`. @@ -71,18 +47,18 @@ The release signing procedure involves a signer (the holder of the certificate/key) and one or multiple trusted verifiers: -| Signer | Verifier | -|-----------------------------------------------------------|-----------------------------------| -| Build .exe files using `build.sh` | | -| Sign .exe with `./sign.sh` | | -| Upload signed files to download server | | -| | Build .exe files using `build.sh` | -| | Compare files using `unsign.sh` | -| | Sign .exe file using `gpg -b` | +| Signer | Verifier | +|-----------------------------------------------------------|--------------------------------------| +| Build .exe files using `make_win.sh` | | +| Sign .exe with `./sign.sh` | | +| Upload signed files to download server | | +| | Build .exe files using `make_win.sh` | +| | Compare files using `unsign.sh` | +| | Sign .exe file using `gpg -b` | -| Signer and verifiers: | -|-----------------------------------------------------------------------------------------------| -| Upload signatures to 'electrum-signatures' repo, as `$version/$filename.$builder.asc` | +| Signer and verifiers: | +|--------------------------------------------------------------------------------------------------| +| Upload signatures to 'electrum-signatures' repo, as `$version/$filename.$builder.asc` | diff --git a/contrib/build-wine/build.sh b/contrib/build-wine/build.sh index fee22bed9..ff0c062da 100755 --- a/contrib/build-wine/build.sh +++ b/contrib/build-wine/build.sh @@ -1,67 +1,56 @@ #!/bin/bash +# +# env vars: +# - ELECBUILD_NOCACHE: if set, forces rebuild of docker image +# - ELECBUILD_COMMIT: if set, do a fresh clone and git checkout set -e -here="$(dirname "$(readlink -e "$0")")" -test -n "$here" -a -d "$here" || exit - -if [ -z "$WIN_ARCH" ] ; then - export WIN_ARCH="win32" # default -fi -if [ "$WIN_ARCH" = "win32" ] ; then - export GCC_TRIPLET_HOST="i686-w64-mingw32" -elif [ "$WIN_ARCH" = "win64" ] ; then - export GCC_TRIPLET_HOST="x86_64-w64-mingw32" -else - echo "unexpected WIN_ARCH: $WIN_ARCH" - exit 1 -fi - -export BUILD_TYPE="wine" -export GCC_TRIPLET_BUILD="x86_64-pc-linux-gnu" -export GCC_STRIP_BINARIES="1" - -export CONTRIB="$here/.." -export PROJECT_ROOT="$CONTRIB/.." -export CACHEDIR="$here/.cache/$WIN_ARCH" -export PIP_CACHE_DIR="$CACHEDIR/wine_pip_cache" -export WINE_PIP_CACHE_DIR="c:/electrum/contrib/build-wine/.cache/$WIN_ARCH/wine_pip_cache" -export DLL_TARGET_DIR="$CACHEDIR/dlls" - -export WINEPREFIX="/opt/wine64" -export WINEDEBUG=-all -export WINE_PYHOME="c:/python3" -export WINE_PYTHON="wine $WINE_PYHOME/python.exe -OO -B" +PROJECT_ROOT="$(dirname "$(readlink -e "$0")")/../.." +PROJECT_ROOT_OR_FRESHCLONE_ROOT="$PROJECT_ROOT" +CONTRIB="$PROJECT_ROOT/contrib" +CONTRIB_WINE="$CONTRIB/build-wine" . "$CONTRIB"/build_tools_util.sh -info "Clearing $here/build and $here/dist..." -rm "$here"/build/* -rf -rm "$here"/dist/* -rf -mkdir -p "$CACHEDIR" "$DLL_TARGET_DIR" "$PIP_CACHE_DIR" - -if [ -f "$DLL_TARGET_DIR/libsecp256k1-0.dll" ]; then - info "libsecp256k1 already built, skipping" -else - "$CONTRIB"/make_libsecp256k1.sh || fail "Could not build libsecp" +DOCKER_BUILD_FLAGS="" +if [ ! -z "$ELECBUILD_NOCACHE" ] ; then + info "ELECBUILD_NOCACHE is set. forcing rebuild of docker image." + DOCKER_BUILD_FLAGS="--pull --no-cache" fi -if [ -f "$DLL_TARGET_DIR/libzbar-0.dll" ]; then - info "libzbar already built, skipping" +info "building docker image." +sudo docker build \ + $DOCKER_BUILD_FLAGS \ + -t electrum-wine-builder-img \ + "$CONTRIB_WINE" + +# maybe do fresh clone +if [ ! -z "$ELECBUILD_COMMIT" ] ; then + info "ELECBUILD_COMMIT=$ELECBUILD_COMMIT. doing fresh clone and git checkout." + FRESH_CLONE="$CONTRIB_WINE/fresh_clone/electrum" && \ + sudo rm -rf "$FRESH_CLONE" && \ + umask 0022 && \ + git clone "$PROJECT_ROOT" "$FRESH_CLONE" && \ + cd "$FRESH_CLONE" + git checkout "$ELECBUILD_COMMIT" + PROJECT_ROOT_OR_FRESHCLONE_ROOT="$FRESH_CLONE" else - "$CONTRIB"/make_zbar.sh || fail "Could not build zbar" + info "not doing fresh clone." fi -$here/prepare-wine.sh || fail "prepare-wine failed" - -info "Resetting modification time in C:\Python..." -# (Because of some bugs in pyinstaller) -pushd /opt/wine64/drive_c/python* -find -exec touch -d '2000-11-11T11:11:11+00:00' {} + -popd -ls -l /opt/wine64/drive_c/python* - -$here/build-electrum-git.sh || fail "build-electrum-git failed" - -info "Done." +info "building binary..." +sudo docker run -it \ + --name electrum-wine-builder-cont \ + -v "$PROJECT_ROOT_OR_FRESHCLONE_ROOT":/opt/wine64/drive_c/electrum \ + --rm \ + --workdir /opt/wine64/drive_c/electrum/contrib/build-wine \ + electrum-wine-builder-img \ + ./make_win.sh + +# make sure resulting binary location is independent of fresh_clone +if [ ! -z "$ELECBUILD_COMMIT" ] ; then + mkdir --parents "$PROJECT_ROOT/contrib/build-wine/dist/" + sudo cp -f "$FRESH_CLONE/contrib/build-wine/dist"/* "$PROJECT_ROOT/contrib/build-wine/dist/" +fi diff --git a/contrib/build-wine/make_win.sh b/contrib/build-wine/make_win.sh new file mode 100755 index 000000000..5ee803e6a --- /dev/null +++ b/contrib/build-wine/make_win.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +set -e + +here="$(dirname "$(readlink -e "$0")")" +test -n "$here" -a -d "$here" || exit + +if [ -z "$WIN_ARCH" ] ; then + export WIN_ARCH="win32" # default +fi +if [ "$WIN_ARCH" = "win32" ] ; then + export GCC_TRIPLET_HOST="i686-w64-mingw32" +elif [ "$WIN_ARCH" = "win64" ] ; then + export GCC_TRIPLET_HOST="x86_64-w64-mingw32" +else + echo "unexpected WIN_ARCH: $WIN_ARCH" + exit 1 +fi + +export BUILD_TYPE="wine" +export GCC_TRIPLET_BUILD="x86_64-pc-linux-gnu" +export GCC_STRIP_BINARIES="1" + +export CONTRIB="$here/.." +export PROJECT_ROOT="$CONTRIB/.." +export CACHEDIR="$here/.cache/$WIN_ARCH" +export PIP_CACHE_DIR="$CACHEDIR/wine_pip_cache" +export WINE_PIP_CACHE_DIR="c:/electrum/contrib/build-wine/.cache/$WIN_ARCH/wine_pip_cache" +export DLL_TARGET_DIR="$CACHEDIR/dlls" + +export WINEPREFIX="/opt/wine64" +export WINEDEBUG=-all +export WINE_PYHOME="c:/python3" +export WINE_PYTHON="wine $WINE_PYHOME/python.exe -OO -B" + +. "$CONTRIB"/build_tools_util.sh + +info "Clearing $here/build and $here/dist..." +rm "$here"/build/* -rf +rm "$here"/dist/* -rf + +mkdir -p "$CACHEDIR" "$DLL_TARGET_DIR" "$PIP_CACHE_DIR" + +if [ -f "$DLL_TARGET_DIR/libsecp256k1-0.dll" ]; then + info "libsecp256k1 already built, skipping" +else + "$CONTRIB"/make_libsecp256k1.sh || fail "Could not build libsecp" +fi + +if [ -f "$DLL_TARGET_DIR/libzbar-0.dll" ]; then + info "libzbar already built, skipping" +else + "$CONTRIB"/make_zbar.sh || fail "Could not build zbar" +fi + +"$here/prepare-wine.sh" || fail "prepare-wine failed" + +info "Resetting modification time in C:\Python..." +# (Because of some bugs in pyinstaller) +pushd /opt/wine64/drive_c/python* +find -exec touch -d '2000-11-11T11:11:11+00:00' {} + +popd +ls -l /opt/wine64/drive_c/python* + +"$here/build-electrum-git.sh" || fail "build-electrum-git failed" + +info "Done." diff --git a/contrib/release.sh b/contrib/release.sh index fe8d898e1..f91945562 100755 --- a/contrib/release.sh +++ b/contrib/release.sh @@ -1,10 +1,14 @@ -# -# Note: update locale before: -# 1. cd /opt/electrum-locale && ./update && push -# 2. cd to the submodule dir, and git pull -# 3. cd .. && git push #!/bin/bash +# Note: steps before doing a new release: +# +# - update locale: +# 1. cd /opt/electrum-locale && ./update && push +# 2. cd to the submodule dir, and git pull +# 3. cd .. && git push +# - update RELEASE-NOTES and version.py +# - git tag + ELECTRUM_DIR=/opt/electrum WWW_DIR=/opt/electrum-web @@ -20,6 +24,9 @@ REV=`git describe --tags` echo "REV: $REV" COMMIT=$(git rev-parse HEAD) +export ELECBUILD_COMMIT="${COMMIT}^{commit}" +#export ELECBUILD_NOCACHE=1 + git_status=$(git status --porcelain) if [ ! -z "$git_status" ]; then @@ -35,25 +42,7 @@ target=Electrum-$VERSION.tar.gz if test -f dist/$target; then echo "file exists: $target" else - pushd . - sudo docker build -t electrum-sdist-builder-img contrib/build-linux/sdist - FRESH_CLONE=contrib/build-linux/sdist/fresh_clone && \ - sudo rm -rf $FRESH_CLONE && \ - umask 0022 && \ - mkdir -p $FRESH_CLONE && \ - cd $FRESH_CLONE && \ - git clone https://github.com/spesmilo/electrum.git &&\ - cd electrum - git checkout "${COMMIT}^{commit}" - sudo docker run -it \ - --name electrum-sdist-builder-cont \ - -v $PWD:/opt/electrum \ - --rm \ - --workdir /opt/electrum/contrib/build-linux/sdist \ - electrum-sdist-builder-img \ - ./build.sh - popd - cp /opt/electrum/contrib/build-linux/sdist/fresh_clone/electrum/dist/$target dist/ + ./contrib/build-linux/sdist/build.sh fi # appimage @@ -66,14 +55,7 @@ fi if test -f dist/$target; then echo "file exists: $target" else - sudo docker build -t electrum-appimage-builder-img contrib/build-linux/appimage - sudo docker run -it \ - --name electrum-appimage-builder-cont \ - -v $PWD:/opt/electrum \ - --rm \ - --workdir /opt/electrum/contrib/build-linux/appimage \ - electrum-appimage-builder-img \ - ./build.sh + ./contrib/build-linux/appimage/build.sh fi @@ -83,21 +65,7 @@ if test -f dist/$target; then echo "file exists: $target" else pushd . - FRESH_CLONE=contrib/build-wine/fresh_clone && \ - sudo rm -rf $FRESH_CLONE && \ - mkdir -p $FRESH_CLONE && \ - cd $FRESH_CLONE && \ - git clone https://github.com/spesmilo/electrum.git && \ - cd electrum - git checkout "${COMMIT}^{commit}" - sudo docker run -it \ - --name electrum-wine-builder-cont \ - -v $PWD:/opt/wine64/drive_c/electrum \ - --rm \ - --workdir /opt/wine64/drive_c/electrum/contrib/build-wine \ - electrum-wine-builder-img \ - ./build.sh - # do this in the fresh clone directory! + ./contrib/build-wine/build.sh cd contrib/build-wine/ ./sign.sh cp ./signed/*.exe /opt/electrum/dist/ @@ -111,40 +79,17 @@ target2=Electrum-$VERSION.0-arm64-v8a-release.apk if test -f dist/$target1; then echo "file exists: $target1" else - pushd . - ./contrib/android/build_docker_image.sh - FRESH_CLONE=contrib/android/fresh_clone && \ - sudo rm -rf $FRESH_CLONE && \ - umask 0022 && \ - mkdir -p $FRESH_CLONE && \ - cd $FRESH_CLONE && \ - git clone https://github.com/spesmilo/electrum.git && \ - cd electrum - git checkout "${COMMIT}^{commit}" - mkdir --parents $PWD/.buildozer/.gradle - sudo docker run -it --rm \ - --name electrum-android-builder-cont \ - -v $PWD:/home/user/wspace/electrum \ - -v $PWD/.buildozer/.gradle:/home/user/.gradle \ - -v ~/.keystore:/home/user/.keystore \ - --workdir /home/user/wspace/electrum \ - electrum-android-builder-img \ - ./contrib/android/make_apk release - popd - - cp contrib/android/fresh_clone/electrum/bin/$target1 dist/ - cp contrib/android/fresh_clone/electrum/bin/$target2 dist/ - + ./contrib/android/build.sh release fi # wait for dmg before signing if test -f dist/electrum-$VERSION.dmg; then if test -f dist/electrum-$VERSION.dmg.asc; then - echo "packages are already signed" + echo "packages are already signed" else - echo "signing packages" - ./contrib/sign_packages ThomasV + echo "signing packages" + ./contrib/sign_packages ThomasV fi else echo "dmg is missing, aborting" @@ -154,7 +99,7 @@ fi echo "build complete" sha256sum dist/*.tar.gz sha256sum dist/*.AppImage -sha256sum contrib/build-wine/fresh_clone/electrum/contrib/build-wine/dist/*.exe +sha256sum contrib/build-wine/dist/*.exe echo -n "proceed (y/n)? " read answer