diff --git a/.travis.yml b/.travis.yml index 89e905c41..7c90bb85c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -99,6 +99,26 @@ jobs: script: - sudo docker run --name electrum-appimage-builder-cont -v $PWD:/opt/electrum --rm --workdir /opt/electrum/contrib/build-linux/appimage electrum-appimage-builder-img ./build.sh after_success: true + - if: branch = master + name: "tarball build" + language: c + python: false + services: + - docker + before_install: + # hack: travis already cloned the repo, but we re-clone now, as we need to have umask set BEFORE cloning + - umask 0022 + - mkdir fresh_clone && cd fresh_clone + - git clone https://github.com/$TRAVIS_REPO_SLUG.git && cd electrum + - if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then git fetch origin pull/$TRAVIS_PULL_REQUEST/merge; fi + - git checkout $TRAVIS_COMMIT + - echo "Second git clone ready at $PWD" + install: + - sudo docker build --no-cache -t electrum-sdist-builder-img ./contrib/build-linux/sdist/ + script: + - echo "Building sdist at $PWD" + - sudo docker run --name electrum-sdist-builder-cont -v $PWD:/opt/electrum --rm --workdir /opt/electrum/contrib/build-linux/sdist electrum-sdist-builder-img ./build.sh + after_success: true - stage: release check install: - git fetch --all --tags diff --git a/README.rst b/README.rst index 545486012..3938642bc 100644 --- a/README.rst +++ b/README.rst @@ -121,7 +121,7 @@ Creating Binaries Linux (tarball) --------------- -See :code:`contrib/build-linux/README.md`. +See :code:`contrib/build-linux/sdist/README.md`. Linux (AppImage) diff --git a/contrib/build-linux/README.md b/contrib/build-linux/README.md deleted file mode 100644 index 2bbe4c320..000000000 --- a/contrib/build-linux/README.md +++ /dev/null @@ -1,16 +0,0 @@ -Source tarballs -=============== - -✗ _This script does not produce reproducible output (yet!)._ - -1. Prepare python dependencies used by Electrum. - - ``` - contrib/make_packages - ``` - -2. Create source tarball. - - ``` - contrib/make_tgz - ``` diff --git a/contrib/build-linux/appimage/README.md b/contrib/build-linux/appimage/README.md index cccc884da..0b50ec7e3 100644 --- a/contrib/build-linux/appimage/README.md +++ b/contrib/build-linux/appimage/README.md @@ -61,6 +61,11 @@ diff sha256sum1 sha256sum2 > d cat d ``` +For file metadata, e.g. timestamps: +``` +rsync -n -a -i --delete squashfs-root1/ squashfs-root2/ +``` + Useful binary comparison tools: - vbindiff - diffoscope diff --git a/contrib/build-linux/sdist/Dockerfile b/contrib/build-linux/sdist/Dockerfile new file mode 100644 index 000000000..b7391f379 --- /dev/null +++ b/contrib/build-linux/sdist/Dockerfile @@ -0,0 +1,17 @@ +FROM ubuntu:20.04@sha256:5747316366b8cc9e3021cd7286f42b2d6d81e3d743e2ab571f55bcd5df788cc8 + +ENV LC_ALL=C.UTF-8 LANG=C.UTF-8 +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update -q && \ + apt-get install -qy \ + git \ + gettext \ + python3 \ + python3-pip \ + python3-setuptools \ + faketime \ + && \ + rm -rf /var/lib/apt/lists/* && \ + apt-get autoremove -y && \ + apt-get clean diff --git a/contrib/build-linux/sdist/README.md b/contrib/build-linux/sdist/README.md new file mode 100644 index 000000000..20aef2b56 --- /dev/null +++ b/contrib/build-linux/sdist/README.md @@ -0,0 +1,52 @@ +Source tarballs +=============== + +✓ _This file should be reproducible, meaning you should be able to generate + distributables that match the official releases._ + +This assumes an Ubuntu (x86_64) host, but it should not be too hard to adapt to another +similar system. The docker commands should be executed in the project's root +folder. + +1. Install Docker + + ``` + $ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + $ sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + $ sudo apt-get update + $ sudo apt-get install -y docker-ce + ``` + +2. Build image + + ``` + $ sudo docker build -t electrum-sdist-builder-img contrib/build-linux/sdist + ``` + +3. Build source tarballs + + It's recommended to build from a fresh clone + (but you can skip this if reproducibility is not necessary). + + ``` + $ FRESH_CLONE=contrib/build-linux/sdist/fresh_clone && \ + sudo rm -rf $FRESH_CLONE && \ + umask 0022 && \ + mkdir -p $FRESH_CLONE && \ + cd $FRESH_CLONE && \ + git clone https://github.com/spesmilo/electrum.git && \ + cd electrum + ``` + + And then build from this directory: + ``` + $ git checkout $REV + $ sudo docker run -it \ + --name electrum-sdist-builder-cont \ + -v $PWD:/opt/electrum \ + --rm \ + --workdir /opt/electrum/contrib/build-linux/sdist \ + electrum-sdist-builder-img \ + ./build.sh + ``` +4. The generated distributables are in `./dist`. diff --git a/contrib/build-linux/sdist/build.sh b/contrib/build-linux/sdist/build.sh new file mode 100755 index 000000000..7e7109192 --- /dev/null +++ b/contrib/build-linux/sdist/build.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +set -e + +PROJECT_ROOT="$(dirname "$(readlink -e "$0")")/../../.." +CONTRIB="$PROJECT_ROOT/contrib" +CONTRIB_SDIST="$CONTRIB/build-linux/sdist" +DISTDIR="$PROJECT_ROOT/dist" + +. "$CONTRIB"/build_tools_util.sh + +# note that at least py3.7 is needed, to have https://bugs.python.org/issue30693 +python3 --version || fail "python interpreter not found" + +# upgrade to modern pip so that it knows the flags we need. +# we will then install a pinned version of pip as part of requirements-sdist-build +python3 -m pip install --upgrade pip + +info "Installing pinned requirements." +python3 -m pip install --no-dependencies --no-warn-script-location -r "$CONTRIB"/deterministic-build/requirements-sdist-build.txt + + +"$CONTRIB"/make_packages || fail "make_packages failed" + +"$CONTRIB_SDIST"/make_tgz || fail "make_tgz failed" + + +info "done." +ls -la "$DISTDIR" +sha256sum "$DISTDIR"/* diff --git a/contrib/make_tgz b/contrib/build-linux/sdist/make_tgz similarity index 74% rename from contrib/make_tgz rename to contrib/build-linux/sdist/make_tgz index 4505d2c2e..28032fe47 100755 --- a/contrib/make_tgz +++ b/contrib/build-linux/sdist/make_tgz @@ -2,7 +2,8 @@ set -e -CONTRIB="$(dirname "$(readlink -e "$0")")" +CONTRIB_SDIST="$(dirname "$(readlink -e "$0")")" +CONTRIB="$CONTRIB_SDIST"/../.. ROOT_FOLDER="$CONTRIB"/.. PACKAGES="$ROOT_FOLDER"/packages/ LOCALE="$ROOT_FOLDER"/electrum/locale/ @@ -39,5 +40,8 @@ git submodule update --init # we could build the kivy atlas potentially? #(cd electrum/gui/kivy/; make theming) || echo "building kivy atlas failed! skipping." - python3 setup.py --quiet sdist --format=zip,gztar + find -exec touch -h -d '2000-11-11T11:11:11+00:00' {} + + + # note: .zip sdists would not be reproducible due to https://bugs.python.org/issue40963 + TZ=UTC faketime -f '2000-11-11 11:11:11' python3 setup.py --quiet sdist --format=gztar ) diff --git a/contrib/deterministic-build/requirements-sdist-build.txt b/contrib/deterministic-build/requirements-sdist-build.txt new file mode 100644 index 000000000..7fa59d2bb --- /dev/null +++ b/contrib/deterministic-build/requirements-sdist-build.txt @@ -0,0 +1,9 @@ +pip==20.1.1 \ + --hash=sha256:27f8dc29387dd83249e06e681ce087e6061826582198a425085e0bf4c1cf3a55 \ + --hash=sha256:b27c4dedae8c41aa59108f2fa38bf78e0890e590545bc8ece7cdceb4ba60f6e4 +setuptools==46.4.0 \ + --hash=sha256:4334fc63121aafb1cc98fd5ae5dd47ea8ad4a38ad638b47af03a686deb14ef5b \ + --hash=sha256:d05c2c47bbef97fd58632b63dd2b83426db38af18f65c180b2423fea4b67e6b8 +wheel==0.34.2 \ + --hash=sha256:8788e9155fe14f54164c1b9eb0a319d98ef02c160725587ad60f14ddc57b6f96 \ + --hash=sha256:df277cb51e61359aba502208d680f90c0493adec6f0e848af94948778aed386e \ No newline at end of file diff --git a/contrib/freeze_packages.sh b/contrib/freeze_packages.sh index 64f063327..46578fe50 100755 --- a/contrib/freeze_packages.sh +++ b/contrib/freeze_packages.sh @@ -20,7 +20,7 @@ which virtualenv > /dev/null 2>&1 || { echo "Please install virtualenv" && exit ${SYSTEM_PYTHON} -m hashin -h > /dev/null 2>&1 || { ${SYSTEM_PYTHON} -m pip install hashin; } -for i in '' '-hw' '-binaries' '-wine-build' '-mac-build'; do +for i in '' '-hw' '-binaries' '-wine-build' '-mac-build' '-sdist-build'; do rm -rf "$venv_dir" virtualenv -p ${SYSTEM_PYTHON} $venv_dir diff --git a/contrib/make_packages b/contrib/make_packages index 56098d337..26441447d 100755 --- a/contrib/make_packages +++ b/contrib/make_packages @@ -6,5 +6,6 @@ test -n "$CONTRIB" -a -d "$CONTRIB" || exit rm "$CONTRIB"/../packages/ -r #Install pure python modules in electrum directory -python3 -m pip install -r "$CONTRIB"/deterministic-build/requirements.txt -t "$CONTRIB"/../packages +python3 -m pip install --no-dependencies --no-binary :all: \ + -r "$CONTRIB"/deterministic-build/requirements.txt -t "$CONTRIB"/../packages diff --git a/contrib/requirements/requirements-sdist-build.txt b/contrib/requirements/requirements-sdist-build.txt new file mode 100644 index 000000000..c367eb47a --- /dev/null +++ b/contrib/requirements/requirements-sdist-build.txt @@ -0,0 +1,3 @@ +# need modern versions of pip (and maybe other build tools), the one in apt had issues +pip +setuptools