Browse Source

Merge pull request #6228 from SomberNight/202006_reproducible_targz

make targz sdist reproducible, rm zip
bip39-recovery
ghost43 5 years ago
committed by GitHub
parent
commit
9cfb954a72
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
  1. 20
      .travis.yml
  2. 2
      README.rst
  3. 16
      contrib/build-linux/README.md
  4. 5
      contrib/build-linux/appimage/README.md
  5. 17
      contrib/build-linux/sdist/Dockerfile
  6. 52
      contrib/build-linux/sdist/README.md
  7. 30
      contrib/build-linux/sdist/build.sh
  8. 8
      contrib/build-linux/sdist/make_tgz
  9. 9
      contrib/deterministic-build/requirements-sdist-build.txt
  10. 2
      contrib/freeze_packages.sh
  11. 3
      contrib/make_packages
  12. 3
      contrib/requirements/requirements-sdist-build.txt

20
.travis.yml

@ -99,6 +99,26 @@ jobs:
script:
- sudo docker run --name electrum-appimage-builder-cont -v $PWD:/opt/electrum --rm --workdir /opt/electrum/contrib/build-linux/appimage electrum-appimage-builder-img ./build.sh
after_success: true
- if: branch = master
name: "tarball build"
language: c
python: false
services:
- docker
before_install:
# hack: travis already cloned the repo, but we re-clone now, as we need to have umask set BEFORE cloning
- umask 0022
- mkdir fresh_clone && cd fresh_clone
- git clone https://github.com/$TRAVIS_REPO_SLUG.git && cd electrum
- if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then git fetch origin pull/$TRAVIS_PULL_REQUEST/merge; fi
- git checkout $TRAVIS_COMMIT
- echo "Second git clone ready at $PWD"
install:
- sudo docker build --no-cache -t electrum-sdist-builder-img ./contrib/build-linux/sdist/
script:
- echo "Building sdist at $PWD"
- sudo docker run --name electrum-sdist-builder-cont -v $PWD:/opt/electrum --rm --workdir /opt/electrum/contrib/build-linux/sdist electrum-sdist-builder-img ./build.sh
after_success: true
- stage: release check
install:
- git fetch --all --tags

2
README.rst

@ -121,7 +121,7 @@ Creating Binaries
Linux (tarball)
---------------
See :code:`contrib/build-linux/README.md`.
See :code:`contrib/build-linux/sdist/README.md`.
Linux (AppImage)

16
contrib/build-linux/README.md

@ -1,16 +0,0 @@
Source tarballs
===============
✗ _This script does not produce reproducible output (yet!)._
1. Prepare python dependencies used by Electrum.
```
contrib/make_packages
```
2. Create source tarball.
```
contrib/make_tgz
```

5
contrib/build-linux/appimage/README.md

@ -61,6 +61,11 @@ diff sha256sum1 sha256sum2 > d
cat d
```
For file metadata, e.g. timestamps:
```
rsync -n -a -i --delete squashfs-root1/ squashfs-root2/
```
Useful binary comparison tools:
- vbindiff
- diffoscope

17
contrib/build-linux/sdist/Dockerfile

@ -0,0 +1,17 @@
FROM ubuntu:20.04@sha256:5747316366b8cc9e3021cd7286f42b2d6d81e3d743e2ab571f55bcd5df788cc8
ENV LC_ALL=C.UTF-8 LANG=C.UTF-8
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update -q && \
apt-get install -qy \
git \
gettext \
python3 \
python3-pip \
python3-setuptools \
faketime \
&& \
rm -rf /var/lib/apt/lists/* && \
apt-get autoremove -y && \
apt-get clean

52
contrib/build-linux/sdist/README.md

@ -0,0 +1,52 @@
Source tarballs
===============
✓ _This file should be reproducible, meaning you should be able to generate
distributables that match the official releases._
This assumes an Ubuntu (x86_64) host, but it should not be too hard to adapt to another
similar system. The docker commands should be executed in the project's root
folder.
1. Install Docker
```
$ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
$ sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
$ sudo apt-get update
$ sudo apt-get install -y docker-ce
```
2. Build image
```
$ sudo docker build -t electrum-sdist-builder-img contrib/build-linux/sdist
```
3. Build source tarballs
It's recommended to build from a fresh clone
(but you can skip this if reproducibility is not necessary).
```
$ FRESH_CLONE=contrib/build-linux/sdist/fresh_clone && \
sudo rm -rf $FRESH_CLONE && \
umask 0022 && \
mkdir -p $FRESH_CLONE && \
cd $FRESH_CLONE && \
git clone https://github.com/spesmilo/electrum.git && \
cd electrum
```
And then build from this directory:
```
$ git checkout $REV
$ sudo docker run -it \
--name electrum-sdist-builder-cont \
-v $PWD:/opt/electrum \
--rm \
--workdir /opt/electrum/contrib/build-linux/sdist \
electrum-sdist-builder-img \
./build.sh
```
4. The generated distributables are in `./dist`.

30
contrib/build-linux/sdist/build.sh

@ -0,0 +1,30 @@
#!/bin/bash
set -e
PROJECT_ROOT="$(dirname "$(readlink -e "$0")")/../../.."
CONTRIB="$PROJECT_ROOT/contrib"
CONTRIB_SDIST="$CONTRIB/build-linux/sdist"
DISTDIR="$PROJECT_ROOT/dist"
. "$CONTRIB"/build_tools_util.sh
# note that at least py3.7 is needed, to have https://bugs.python.org/issue30693
python3 --version || fail "python interpreter not found"
# upgrade to modern pip so that it knows the flags we need.
# we will then install a pinned version of pip as part of requirements-sdist-build
python3 -m pip install --upgrade pip
info "Installing pinned requirements."
python3 -m pip install --no-dependencies --no-warn-script-location -r "$CONTRIB"/deterministic-build/requirements-sdist-build.txt
"$CONTRIB"/make_packages || fail "make_packages failed"
"$CONTRIB_SDIST"/make_tgz || fail "make_tgz failed"
info "done."
ls -la "$DISTDIR"
sha256sum "$DISTDIR"/*

8
contrib/make_tgz → contrib/build-linux/sdist/make_tgz

@ -2,7 +2,8 @@
set -e
CONTRIB="$(dirname "$(readlink -e "$0")")"
CONTRIB_SDIST="$(dirname "$(readlink -e "$0")")"
CONTRIB="$CONTRIB_SDIST"/../..
ROOT_FOLDER="$CONTRIB"/..
PACKAGES="$ROOT_FOLDER"/packages/
LOCALE="$ROOT_FOLDER"/electrum/locale/
@ -39,5 +40,8 @@ git submodule update --init
# we could build the kivy atlas potentially?
#(cd electrum/gui/kivy/; make theming) || echo "building kivy atlas failed! skipping."
python3 setup.py --quiet sdist --format=zip,gztar
find -exec touch -h -d '2000-11-11T11:11:11+00:00' {} +
# note: .zip sdists would not be reproducible due to https://bugs.python.org/issue40963
TZ=UTC faketime -f '2000-11-11 11:11:11' python3 setup.py --quiet sdist --format=gztar
)

9
contrib/deterministic-build/requirements-sdist-build.txt

@ -0,0 +1,9 @@
pip==20.1.1 \
--hash=sha256:27f8dc29387dd83249e06e681ce087e6061826582198a425085e0bf4c1cf3a55 \
--hash=sha256:b27c4dedae8c41aa59108f2fa38bf78e0890e590545bc8ece7cdceb4ba60f6e4
setuptools==46.4.0 \
--hash=sha256:4334fc63121aafb1cc98fd5ae5dd47ea8ad4a38ad638b47af03a686deb14ef5b \
--hash=sha256:d05c2c47bbef97fd58632b63dd2b83426db38af18f65c180b2423fea4b67e6b8
wheel==0.34.2 \
--hash=sha256:8788e9155fe14f54164c1b9eb0a319d98ef02c160725587ad60f14ddc57b6f96 \
--hash=sha256:df277cb51e61359aba502208d680f90c0493adec6f0e848af94948778aed386e

2
contrib/freeze_packages.sh

@ -20,7 +20,7 @@ which virtualenv > /dev/null 2>&1 || { echo "Please install virtualenv" && exit
${SYSTEM_PYTHON} -m hashin -h > /dev/null 2>&1 || { ${SYSTEM_PYTHON} -m pip install hashin; }
for i in '' '-hw' '-binaries' '-wine-build' '-mac-build'; do
for i in '' '-hw' '-binaries' '-wine-build' '-mac-build' '-sdist-build'; do
rm -rf "$venv_dir"
virtualenv -p ${SYSTEM_PYTHON} $venv_dir

3
contrib/make_packages

@ -6,5 +6,6 @@ test -n "$CONTRIB" -a -d "$CONTRIB" || exit
rm "$CONTRIB"/../packages/ -r
#Install pure python modules in electrum directory
python3 -m pip install -r "$CONTRIB"/deterministic-build/requirements.txt -t "$CONTRIB"/../packages
python3 -m pip install --no-dependencies --no-binary :all: \
-r "$CONTRIB"/deterministic-build/requirements.txt -t "$CONTRIB"/../packages

3
contrib/requirements/requirements-sdist-build.txt

@ -0,0 +1,3 @@
# need modern versions of pip (and maybe other build tools), the one in apt had issues
pip
setuptools
Loading…
Cancel
Save