attempt at reproducible tarballs (sdist)

.travis.yml

@@ -99,6 +99,17 @@ jobs:
       script:
         - sudo docker run --name electrum-appimage-builder-cont -v $PWD:/opt/electrum --rm --workdir /opt/electrum/contrib/build-linux/appimage electrum-appimage-builder-img ./build.sh
       after_success: true
+    - if: branch = master
+      name: "tarball build"
+      language: c
+      python: false
+      services:
+        - docker
+      install:
+        - sudo docker build --no-cache -t electrum-sdist-builder-img ./contrib/build-linux/sdist/
+      script:
+        - sudo docker run --name electrum-sdist-builder-cont -v $PWD:/opt/electrum --rm --workdir /opt/electrum/contrib/build-linux/sdist electrum-sdist-builder-img ./build.sh
+      after_success: true
     - stage: release check
       install:
           - git fetch --all --tags

README.rst

@@ -121,7 +121,7 @@ Creating Binaries
 Linux (tarball)
 ---------------
 
-See :code:`contrib/build-linux/README.md`.
+See :code:`contrib/build-linux/sdist/README.md`.
 
 
 Linux (AppImage)

contrib/build-linux/README.md

@@ -1,16 +0,0 @@
-Source tarballs
-===============
-
-✗ _This script does not produce reproducible output (yet!)._
-
-1. Prepare python dependencies used by Electrum.
-
-    ```
-    contrib/make_packages
-    ```
-
-2. Create source tarball.
-
-    ```
-    contrib/make_tgz
-    ```

contrib/build-linux/appimage/README.md

@@ -61,6 +61,11 @@ diff sha256sum1 sha256sum2 > d
 cat d
 ```
 
+For file metadata, e.g. timestamps:
+```
+rsync -n -a -i --delete squashfs-root1/ squashfs-root2/
+```
+
 Useful binary comparison tools:
 - vbindiff
 - diffoscope

contrib/build-linux/sdist/Dockerfile

@@ -0,0 +1,16 @@
+FROM ubuntu:18.04@sha256:b58746c8a89938b8c9f5b77de3b8cf1fe78210c696ab03a1442e235eea65d84f
+
+ENV LC_ALL=C.UTF-8 LANG=C.UTF-8
+
+RUN apt-get update -q && \
+    apt-get install -qy \
+        git \
+        gettext \
+        python3 \
+        python3-pip \
+        python3-setuptools \
+        faketime \
+        && \
+    rm -rf /var/lib/apt/lists/* && \
+    apt-get autoremove -y && \
+    apt-get clean

contrib/build-linux/sdist/README.md

@@ -0,0 +1,50 @@
+Source tarballs
+===============
+
+✗ _This script does not produce reproducible output (yet!)._
+
+This assumes an Ubuntu (x86_64) host, but it should not be too hard to adapt to another
+similar system. The docker commands should be executed in the project's root
+folder.
+
+1. Install Docker
+
+    ```
+    $ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+    $ sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+    $ sudo apt-get update
+    $ sudo apt-get install -y docker-ce
+    ```
+
+2. Build image
+
+    ```
+    $ sudo docker build -t electrum-sdist-builder-img contrib/build-linux/sdist
+    ```
+
+3. Build source tarballs
+
+    It's recommended to build from a fresh clone
+    (but you can skip this if reproducibility is not necessary).
+
+    ```
+    $ FRESH_CLONE=contrib/build-linux/sdist/fresh_clone && \
+        sudo rm -rf $FRESH_CLONE && \
+        mkdir -p $FRESH_CLONE && \
+        cd $FRESH_CLONE  && \
+        git clone https://github.com/spesmilo/electrum.git && \
+        cd electrum
+    ```
+
+    And then build from this directory:
+    ```
+    $ git checkout $REV
+    $ sudo docker run -it \
+        --name electrum-sdist-builder-cont \
+        -v $PWD:/opt/electrum \
+        --rm \
+        --workdir /opt/electrum/contrib/build-linux/sdist \
+        electrum-sdist-builder-img \
+        ./build.sh
+    ```
+4. The generated distributables are in `./dist`.

contrib/build-linux/sdist/build.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -e
+
+PROJECT_ROOT="$(dirname "$(readlink -e "$0")")/../../.."
+CONTRIB="$PROJECT_ROOT/contrib"
+CONTRIB_SDIST="$CONTRIB/build-linux/sdist"
+DISTDIR="$PROJECT_ROOT/dist"
+
+. "$CONTRIB"/build_tools_util.sh
+
+
+"$CONTRIB"/make_packages || fail "make_packages failed"
+
+"$CONTRIB_SDIST"/make_tgz || fail "make_tgz failed"
+
+
+info "done."
+ls -la "$DISTDIR"
+sha256sum "$DISTDIR"/*

contrib/make_tgz → contrib/build-linux/sdist/make_tgz

@@ -2,7 +2,8 @@
 
 set -e
 
-CONTRIB="$(dirname "$(readlink -e "$0")")"
+CONTRIB_SDIST="$(dirname "$(readlink -e "$0")")"
+CONTRIB="$CONTRIB_SDIST"/../..
 ROOT_FOLDER="$CONTRIB"/..
 PACKAGES="$ROOT_FOLDER"/packages/
 LOCALE="$ROOT_FOLDER"/electrum/locale/
@@ -39,5 +40,7 @@ git submodule update --init
     # we could build the kivy atlas potentially?
     #(cd electrum/gui/kivy/; make theming) || echo "building kivy atlas failed! skipping."
 
-    python3 setup.py --quiet sdist --format=zip,gztar
+    find -exec touch -h -d '2000-11-11T11:11:11+00:00' {} +
+
+    TZ=UTC faketime -f '2000-11-11 11:11:11' python3 setup.py --quiet sdist --format=zip,gztar
 )

contrib/make_packages

@@ -6,5 +6,6 @@ test -n "$CONTRIB" -a -d "$CONTRIB" || exit
 rm "$CONTRIB"/../packages/ -r
 
 #Install pure python modules in electrum directory
-python3 -m pip install -r "$CONTRIB"/deterministic-build/requirements.txt -t "$CONTRIB"/../packages
+python3 -m pip install --no-dependencies --no-binary :all: \
+    -r "$CONTRIB"/deterministic-build/requirements.txt -t "$CONTRIB"/../packages