From e12bc4817a2e18bf564879e171f3e02d2359b0e9 Mon Sep 17 00:00:00 2001
From: SomberNight <somber.night@protonmail.com>
Date: Thu, 11 Jun 2020 03:50:32 +0200
Subject: [PATCH 1/8] attempt at reproducible tarballs (sdist)

---
 .travis.yml                              | 11 ++++++
 README.rst                               |  2 +-
 contrib/build-linux/README.md            | 16 --------
 contrib/build-linux/appimage/README.md   |  5 +++
 contrib/build-linux/sdist/Dockerfile     | 16 ++++++++
 contrib/build-linux/sdist/README.md      | 50 ++++++++++++++++++++++++
 contrib/build-linux/sdist/build.sh       | 20 ++++++++++
 contrib/{ => build-linux/sdist}/make_tgz |  7 +++-
 contrib/make_packages                    |  3 +-
 9 files changed, 110 insertions(+), 20 deletions(-)
 delete mode 100644 contrib/build-linux/README.md
 create mode 100644 contrib/build-linux/sdist/Dockerfile
 create mode 100644 contrib/build-linux/sdist/README.md
 create mode 100755 contrib/build-linux/sdist/build.sh
 rename contrib/{ => build-linux/sdist}/make_tgz (79%)

diff --git a/.travis.yml b/.travis.yml
index 078e57638..020b381ee 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -99,6 +99,17 @@ jobs:
       script:
         - sudo docker run --name electrum-appimage-builder-cont -v $PWD:/opt/electrum --rm --workdir /opt/electrum/contrib/build-linux/appimage electrum-appimage-builder-img ./build.sh
       after_success: true
+    - if: branch = master
+      name: "tarball build"
+      language: c
+      python: false
+      services:
+        - docker
+      install:
+        - sudo docker build --no-cache -t electrum-sdist-builder-img ./contrib/build-linux/sdist/
+      script:
+        - sudo docker run --name electrum-sdist-builder-cont -v $PWD:/opt/electrum --rm --workdir /opt/electrum/contrib/build-linux/sdist electrum-sdist-builder-img ./build.sh
+      after_success: true
     - stage: release check
       install:
           - git fetch --all --tags
diff --git a/README.rst b/README.rst
index 545486012..3938642bc 100644
--- a/README.rst
+++ b/README.rst
@@ -121,7 +121,7 @@ Creating Binaries
 Linux (tarball)
 ---------------
 
-See :code:`contrib/build-linux/README.md`.
+See :code:`contrib/build-linux/sdist/README.md`.
 
 
 Linux (AppImage)
diff --git a/contrib/build-linux/README.md b/contrib/build-linux/README.md
deleted file mode 100644
index 2bbe4c320..000000000
--- a/contrib/build-linux/README.md
+++ /dev/null
@@ -1,16 +0,0 @@
-Source tarballs
-===============
-
-✗ _This script does not produce reproducible output (yet!)._
-
-1. Prepare python dependencies used by Electrum.
-
-    ```
-    contrib/make_packages
-    ```
-
-2. Create source tarball.
-
-    ```
-    contrib/make_tgz
-    ```
diff --git a/contrib/build-linux/appimage/README.md b/contrib/build-linux/appimage/README.md
index cccc884da..0b50ec7e3 100644
--- a/contrib/build-linux/appimage/README.md
+++ b/contrib/build-linux/appimage/README.md
@@ -61,6 +61,11 @@ diff sha256sum1 sha256sum2 > d
 cat d
 ```
 
+For file metadata, e.g. timestamps:
+```
+rsync -n -a -i --delete squashfs-root1/ squashfs-root2/
+```
+
 Useful binary comparison tools:
 - vbindiff
 - diffoscope
diff --git a/contrib/build-linux/sdist/Dockerfile b/contrib/build-linux/sdist/Dockerfile
new file mode 100644
index 000000000..a215ed9a0
--- /dev/null
+++ b/contrib/build-linux/sdist/Dockerfile
@@ -0,0 +1,16 @@
+FROM ubuntu:18.04@sha256:b58746c8a89938b8c9f5b77de3b8cf1fe78210c696ab03a1442e235eea65d84f
+
+ENV LC_ALL=C.UTF-8 LANG=C.UTF-8
+
+RUN apt-get update -q && \
+    apt-get install -qy \
+        git \
+        gettext \
+        python3 \
+        python3-pip \
+        python3-setuptools \
+        faketime \
+        && \
+    rm -rf /var/lib/apt/lists/* && \
+    apt-get autoremove -y && \
+    apt-get clean
diff --git a/contrib/build-linux/sdist/README.md b/contrib/build-linux/sdist/README.md
new file mode 100644
index 000000000..ae6460312
--- /dev/null
+++ b/contrib/build-linux/sdist/README.md
@@ -0,0 +1,50 @@
+Source tarballs
+===============
+
+✗ _This script does not produce reproducible output (yet!)._
+
+This assumes an Ubuntu (x86_64) host, but it should not be too hard to adapt to another
+similar system. The docker commands should be executed in the project's root
+folder.
+
+1. Install Docker
+
+    ```
+    $ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+    $ sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+    $ sudo apt-get update
+    $ sudo apt-get install -y docker-ce
+    ```
+
+2. Build image
+
+    ```
+    $ sudo docker build -t electrum-sdist-builder-img contrib/build-linux/sdist
+    ```
+
+3. Build source tarballs
+
+    It's recommended to build from a fresh clone
+    (but you can skip this if reproducibility is not necessary).
+
+    ```
+    $ FRESH_CLONE=contrib/build-linux/sdist/fresh_clone && \
+        sudo rm -rf $FRESH_CLONE && \
+        mkdir -p $FRESH_CLONE && \
+        cd $FRESH_CLONE  && \
+        git clone https://github.com/spesmilo/electrum.git && \
+        cd electrum
+    ```
+
+    And then build from this directory:
+    ```
+    $ git checkout $REV
+    $ sudo docker run -it \
+        --name electrum-sdist-builder-cont \
+        -v $PWD:/opt/electrum \
+        --rm \
+        --workdir /opt/electrum/contrib/build-linux/sdist \
+        electrum-sdist-builder-img \
+        ./build.sh
+    ```
+4. The generated distributables are in `./dist`.
diff --git a/contrib/build-linux/sdist/build.sh b/contrib/build-linux/sdist/build.sh
new file mode 100755
index 000000000..300c05455
--- /dev/null
+++ b/contrib/build-linux/sdist/build.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -e
+
+PROJECT_ROOT="$(dirname "$(readlink -e "$0")")/../../.."
+CONTRIB="$PROJECT_ROOT/contrib"
+CONTRIB_SDIST="$CONTRIB/build-linux/sdist"
+DISTDIR="$PROJECT_ROOT/dist"
+
+. "$CONTRIB"/build_tools_util.sh
+
+
+"$CONTRIB"/make_packages || fail "make_packages failed"
+
+"$CONTRIB_SDIST"/make_tgz || fail "make_tgz failed"
+
+
+info "done."
+ls -la "$DISTDIR"
+sha256sum "$DISTDIR"/*
diff --git a/contrib/make_tgz b/contrib/build-linux/sdist/make_tgz
similarity index 79%
rename from contrib/make_tgz
rename to contrib/build-linux/sdist/make_tgz
index 4505d2c2e..6bc7c4b65 100755
--- a/contrib/make_tgz
+++ b/contrib/build-linux/sdist/make_tgz
@@ -2,7 +2,8 @@
 
 set -e
 
-CONTRIB="$(dirname "$(readlink -e "$0")")"
+CONTRIB_SDIST="$(dirname "$(readlink -e "$0")")"
+CONTRIB="$CONTRIB_SDIST"/../..
 ROOT_FOLDER="$CONTRIB"/..
 PACKAGES="$ROOT_FOLDER"/packages/
 LOCALE="$ROOT_FOLDER"/electrum/locale/
@@ -39,5 +40,7 @@ git submodule update --init
     # we could build the kivy atlas potentially?
     #(cd electrum/gui/kivy/; make theming) || echo "building kivy atlas failed! skipping."
 
-    python3 setup.py --quiet sdist --format=zip,gztar
+    find -exec touch -h -d '2000-11-11T11:11:11+00:00' {} +
+
+    TZ=UTC faketime -f '2000-11-11 11:11:11' python3 setup.py --quiet sdist --format=zip,gztar
 )
diff --git a/contrib/make_packages b/contrib/make_packages
index 56098d337..26441447d 100755
--- a/contrib/make_packages
+++ b/contrib/make_packages
@@ -6,5 +6,6 @@ test -n "$CONTRIB" -a -d "$CONTRIB" || exit
 rm "$CONTRIB"/../packages/ -r
 
 #Install pure python modules in electrum directory
-python3 -m pip install -r "$CONTRIB"/deterministic-build/requirements.txt -t "$CONTRIB"/../packages
+python3 -m pip install --no-dependencies --no-binary :all: \
+    -r "$CONTRIB"/deterministic-build/requirements.txt -t "$CONTRIB"/../packages
 

From 24a007840f0172ccc978fbb9a6b921f0ea616e00 Mon Sep 17 00:00:00 2001
From: SomberNight <somber.night@protonmail.com>
Date: Thu, 11 Jun 2020 20:19:28 +0200
Subject: [PATCH 2/8] sdist build: use modern pip
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

the one in apt refused to install certain package versions (that were pinned by hash!!)
and installed different versions instead... e.g.:

Collecting wheel==0.34.2 (from -r /opt/electrum/contrib/build-linux/sdist/../../../contrib/deterministic-build/requirements.txt (line 112))
  Downloading https://files.pythonhosted.org/packages/75/28/521c6dc7fef23a68368efefdcd682f5b3d1d58c2b90b06dc1d0b805b51ae/wheel-0.34.2.tar.gz (58kB)
    100% |████████████████████████████████| 61kB 3.8MB/s
  Requested wheel==0.34.2 from https://files.pythonhosted.org/packages/75/28/521c6dc7fef23a68368efefdcd682f5b3d1d58c2b90b06dc1d0b805b51ae/wheel-0.34.2.tar.gz#sha256=8788e9155fe14f54164c1b9eb0a319d98ef02c160725587ad60f14ddc57b6f96 (from -r /opt/electrum/contrib/build-linux/sdist/../../../contrib/deterministic-build/requirements.txt (line 112)), but installing version 0.30.0
---
 contrib/build-linux/sdist/build.sh                       | 9 +++++++++
 contrib/deterministic-build/requirements-sdist-build.txt | 9 +++++++++
 contrib/freeze_packages.sh                               | 2 +-
 contrib/requirements/requirements-sdist-build.txt        | 3 +++
 4 files changed, 22 insertions(+), 1 deletion(-)
 create mode 100644 contrib/deterministic-build/requirements-sdist-build.txt
 create mode 100644 contrib/requirements/requirements-sdist-build.txt

diff --git a/contrib/build-linux/sdist/build.sh b/contrib/build-linux/sdist/build.sh
index 300c05455..62cf2c553 100755
--- a/contrib/build-linux/sdist/build.sh
+++ b/contrib/build-linux/sdist/build.sh
@@ -9,6 +9,15 @@ DISTDIR="$PROJECT_ROOT/dist"
 
 . "$CONTRIB"/build_tools_util.sh
 
+python3 --version || fail "python interpreter not found"
+
+# upgrade to modern pip so that it knows the flags we need.
+# we will then install a pinned version of pip as part of requirements-sdist-build
+python3 -m pip install --upgrade pip
+
+info "Installing pinned requirements."
+python3 -m pip install --no-dependencies --no-warn-script-location -r "$CONTRIB"/deterministic-build/requirements-sdist-build.txt
+
 
 "$CONTRIB"/make_packages || fail "make_packages failed"
 
diff --git a/contrib/deterministic-build/requirements-sdist-build.txt b/contrib/deterministic-build/requirements-sdist-build.txt
new file mode 100644
index 000000000..7fa59d2bb
--- /dev/null
+++ b/contrib/deterministic-build/requirements-sdist-build.txt
@@ -0,0 +1,9 @@
+pip==20.1.1 \
+    --hash=sha256:27f8dc29387dd83249e06e681ce087e6061826582198a425085e0bf4c1cf3a55 \
+    --hash=sha256:b27c4dedae8c41aa59108f2fa38bf78e0890e590545bc8ece7cdceb4ba60f6e4
+setuptools==46.4.0 \
+    --hash=sha256:4334fc63121aafb1cc98fd5ae5dd47ea8ad4a38ad638b47af03a686deb14ef5b \
+    --hash=sha256:d05c2c47bbef97fd58632b63dd2b83426db38af18f65c180b2423fea4b67e6b8
+wheel==0.34.2 \
+    --hash=sha256:8788e9155fe14f54164c1b9eb0a319d98ef02c160725587ad60f14ddc57b6f96 \
+    --hash=sha256:df277cb51e61359aba502208d680f90c0493adec6f0e848af94948778aed386e
\ No newline at end of file
diff --git a/contrib/freeze_packages.sh b/contrib/freeze_packages.sh
index 64f063327..46578fe50 100755
--- a/contrib/freeze_packages.sh
+++ b/contrib/freeze_packages.sh
@@ -20,7 +20,7 @@ which virtualenv > /dev/null 2>&1 || { echo "Please install virtualenv" && exit
 
 ${SYSTEM_PYTHON} -m hashin -h > /dev/null 2>&1 || { ${SYSTEM_PYTHON} -m pip install hashin; }
 
-for i in '' '-hw' '-binaries' '-wine-build' '-mac-build'; do
+for i in '' '-hw' '-binaries' '-wine-build' '-mac-build' '-sdist-build'; do
     rm -rf "$venv_dir"
     virtualenv -p ${SYSTEM_PYTHON} $venv_dir
 
diff --git a/contrib/requirements/requirements-sdist-build.txt b/contrib/requirements/requirements-sdist-build.txt
new file mode 100644
index 000000000..c367eb47a
--- /dev/null
+++ b/contrib/requirements/requirements-sdist-build.txt
@@ -0,0 +1,3 @@
+# need modern versions of pip (and maybe other build tools), the one in apt had issues
+pip
+setuptools

From 891390f9a128b54c7a909a77f06008bbced55c6e Mon Sep 17 00:00:00 2001
From: SomberNight <somber.night@protonmail.com>
Date: Thu, 11 Jun 2020 21:46:50 +0200
Subject: [PATCH 3/8] sdist build: umask should be specified for git clone

(not nice to change umask of host :/)
---
 contrib/build-linux/sdist/README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/contrib/build-linux/sdist/README.md b/contrib/build-linux/sdist/README.md
index ae6460312..1a603682c 100644
--- a/contrib/build-linux/sdist/README.md
+++ b/contrib/build-linux/sdist/README.md
@@ -30,6 +30,7 @@ folder.
     ```
     $ FRESH_CLONE=contrib/build-linux/sdist/fresh_clone && \
         sudo rm -rf $FRESH_CLONE && \
+        umask 0022 && \
         mkdir -p $FRESH_CLONE && \
         cd $FRESH_CLONE  && \
         git clone https://github.com/spesmilo/electrum.git && \

From a06f5da7c201c4b904391ecc8bc42a0468379111 Mon Sep 17 00:00:00 2001
From: SomberNight <somber.night@protonmail.com>
Date: Thu, 11 Jun 2020 22:38:03 +0200
Subject: [PATCH 4/8] sdist build: bump base image to ubuntu 20.04

---
 contrib/build-linux/sdist/Dockerfile | 2 +-
 contrib/build-linux/sdist/build.sh   | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/contrib/build-linux/sdist/Dockerfile b/contrib/build-linux/sdist/Dockerfile
index a215ed9a0..ea1cb01da 100644
--- a/contrib/build-linux/sdist/Dockerfile
+++ b/contrib/build-linux/sdist/Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:18.04@sha256:b58746c8a89938b8c9f5b77de3b8cf1fe78210c696ab03a1442e235eea65d84f
+FROM ubuntu:20.04@sha256:5747316366b8cc9e3021cd7286f42b2d6d81e3d743e2ab571f55bcd5df788cc8
 
 ENV LC_ALL=C.UTF-8 LANG=C.UTF-8
 
diff --git a/contrib/build-linux/sdist/build.sh b/contrib/build-linux/sdist/build.sh
index 62cf2c553..7e7109192 100755
--- a/contrib/build-linux/sdist/build.sh
+++ b/contrib/build-linux/sdist/build.sh
@@ -9,6 +9,7 @@ DISTDIR="$PROJECT_ROOT/dist"
 
 . "$CONTRIB"/build_tools_util.sh
 
+# note that at least py3.7 is needed, to have https://bugs.python.org/issue30693
 python3 --version || fail "python interpreter not found"
 
 # upgrade to modern pip so that it knows the flags we need.

From 901a900ec54d410fa9f2e820e9acbd9f6f3470b7 Mon Sep 17 00:00:00 2001
From: SomberNight <somber.night@protonmail.com>
Date: Thu, 11 Jun 2020 22:44:56 +0200
Subject: [PATCH 5/8] sdist build: when building docker image, no interactive
 prompts!

see https://askubuntu.com/questions/909277/
---
 contrib/build-linux/sdist/Dockerfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/contrib/build-linux/sdist/Dockerfile b/contrib/build-linux/sdist/Dockerfile
index ea1cb01da..b7391f379 100644
--- a/contrib/build-linux/sdist/Dockerfile
+++ b/contrib/build-linux/sdist/Dockerfile
@@ -1,6 +1,7 @@
 FROM ubuntu:20.04@sha256:5747316366b8cc9e3021cd7286f42b2d6d81e3d743e2ab571f55bcd5df788cc8
 
 ENV LC_ALL=C.UTF-8 LANG=C.UTF-8
+ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update -q && \
     apt-get install -qy \

From c5c8ea15bbb2df18f9ac600a9f64bd43f05f81af Mon Sep 17 00:00:00 2001
From: SomberNight <somber.night@protonmail.com>
Date: Fri, 12 Jun 2020 19:43:47 +0200
Subject: [PATCH 6/8] sdist build: stop making .zip distributables as they are
 not deterministic

see https://bugs.python.org/issue40963
---
 contrib/build-linux/sdist/make_tgz | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/contrib/build-linux/sdist/make_tgz b/contrib/build-linux/sdist/make_tgz
index 6bc7c4b65..28032fe47 100755
--- a/contrib/build-linux/sdist/make_tgz
+++ b/contrib/build-linux/sdist/make_tgz
@@ -42,5 +42,6 @@ git submodule update --init
 
     find -exec touch -h -d '2000-11-11T11:11:11+00:00' {} +
 
-    TZ=UTC faketime -f '2000-11-11 11:11:11' python3 setup.py --quiet sdist --format=zip,gztar
+    # note: .zip sdists would not be reproducible due to https://bugs.python.org/issue40963
+    TZ=UTC faketime -f '2000-11-11 11:11:11' python3 setup.py --quiet sdist --format=gztar
 )

From 612259f70f000fc01b3825e889a30cef0ce31388 Mon Sep 17 00:00:00 2001
From: SomberNight <somber.night@protonmail.com>
Date: Fri, 12 Jun 2020 20:01:19 +0200
Subject: [PATCH 7/8] travis sdist: git clone repo a second time, to properly
 set umask

see https://stackoverflow.com/questions/32580821/
---
 .travis.yml | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index 020b381ee..a19674c48 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -105,9 +105,18 @@ jobs:
       python: false
       services:
         - docker
+      before_install:
+        # hack: travis already cloned the repo, but we re-clone now, as we need to have umask set BEFORE cloning
+        - umask 0022
+        - mkdir fresh_clone && cd fresh_clone
+        - git clone https://github.com/$TRAVIS_REPO_SLUG.git && cd electrum
+        - if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then git fetch origin pull/$TRAVIS_PULL_REQUEST/merge; fi
+        - git checkout $TRAVIS_COMMIT
+        - echo "Second git clone ready at $PWD"
       install:
         - sudo docker build --no-cache -t electrum-sdist-builder-img ./contrib/build-linux/sdist/
       script:
+        - echo "Building sdist at $PWD"
         - sudo docker run --name electrum-sdist-builder-cont -v $PWD:/opt/electrum --rm --workdir /opt/electrum/contrib/build-linux/sdist electrum-sdist-builder-img ./build.sh
       after_success: true
     - stage: release check

From dacc61a41d40c6edc5ee0c7c6f9c6a1dd689dd62 Mon Sep 17 00:00:00 2001
From: SomberNight <somber.night@protonmail.com>
Date: Fri, 12 Jun 2020 20:54:18 +0200
Subject: [PATCH 8/8] sdist build: update message about reproducibility

---
 contrib/build-linux/sdist/README.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/contrib/build-linux/sdist/README.md b/contrib/build-linux/sdist/README.md
index 1a603682c..20aef2b56 100644
--- a/contrib/build-linux/sdist/README.md
+++ b/contrib/build-linux/sdist/README.md
@@ -1,7 +1,8 @@
 Source tarballs
 ===============
 
-✗ _This script does not produce reproducible output (yet!)._
+✓ _This file should be reproducible, meaning you should be able to generate
+   distributables that match the official releases._
 
 This assumes an Ubuntu (x86_64) host, but it should not be too hard to adapt to another
 similar system. The docker commands should be executed in the project's root