build: also build a "source-only" linux-packager-friendly tarball

We now distribute two tarballs, a "normal" one (the default, recommended for users),
and a new strictly source-only one (for Linux distro packagers).
The normal tarball, in addition to including everything from
the source-only one, also includes:
- compiled (`.mo`) locale files (in addition to source `.po` locale files)
- compiled (`_pb2.py`) protobuf files (in addition to source `.proto` files)
- the `packages/` folder containing source-only pure-python runtime dependencies

merges https://github.com/spesmilo/electrum/pull/7594
with follow-ups

.cirrus.yml

@@ -208,7 +208,6 @@ task:
     path: "dist/*"
 
 task:
-  name: tarball build
   container:
     dockerfile: contrib/build-linux/sdist/Dockerfile
     cpu: 1
@@ -217,6 +216,11 @@ task:
     - ./contrib/build-linux/sdist/make_sdist.sh
   binaries_artifacts:
     path: "dist/*"
+  matrix:
+    - name: tarball build
+    - name: source-only tarball build
+      env:
+        OMIT_UNCLEAN_FILES: 1
 
 task:
   name: Submodules

MANIFEST.in

@@ -8,8 +8,7 @@ recursive-include packages cacert.pem
 
 include contrib/requirements/requirements*.txt
 include contrib/deterministic-build/requirements*.txt
-include contrib/make_libsecp256k1.sh
-include contrib/build_tools_util.sh
+include contrib/*.sh
 
 graft electrum
 prune electrum/tests

contrib/build-linux/sdist/README.md

@@ -1,5 +1,4 @@
-Source tarballs
-===============
+# Source tarballs
 
 ✓ _This file should be reproducible, meaning you should be able to generate
    distributables that match the official releases._
@@ -7,18 +6,31 @@ Source tarballs
 This assumes an Ubuntu (x86_64) host, but it should not be too hard to adapt to another
 similar system.
 
+We distribute two tarballs, a "normal" one (the default, recommended for users),
+and a strictly source-only one (for Linux distro packagers).
+The normal tarball, in addition to including everything from
+the source-only one, also includes:
+- compiled (`.mo`) locale files (in addition to source `.po` locale files)
+- compiled (`_pb2.py`) protobuf files (in addition to source `.proto` files)
+- the `packages/` folder containing source-only pure-python runtime dependencies
+
+
+## Build steps
+
 1. Install Docker
 
     See `contrib/docker_notes.md`.
 
-2. Build source tarball
+2. Build tarball
 
+    (set envvar `OMIT_UNCLEAN_FILES=1` to build the "source-only" tarball)
     ```
     $ ./build.sh
     ```
     If you want reproducibility, try instead e.g.:
     ```
     $ ELECBUILD_COMMIT=HEAD ELECBUILD_NOCACHE=1 ./build.sh
+    $ ELECBUILD_COMMIT=HEAD ELECBUILD_NOCACHE=1 OMIT_UNCLEAN_FILES=1 ./build.sh
     ```
 
 3. The generated distributables are in `./dist`.

contrib/build-linux/sdist/build.sh

@@ -47,6 +47,7 @@ docker run -it \
     -v "$PROJECT_ROOT_OR_FRESHCLONE_ROOT":/opt/electrum \
     --rm \
     --workdir /opt/electrum/contrib/build-linux/sdist \
+    --env OMIT_UNCLEAN_FILES \
     electrum-sdist-builder-img \
     ./make_sdist.sh
 

contrib/build-linux/sdist/make_sdist.sh

@@ -6,7 +6,7 @@ PROJECT_ROOT="$(dirname "$(readlink -e "$0")")/../../.."
 CONTRIB="$PROJECT_ROOT/contrib"
 CONTRIB_SDIST="$CONTRIB/build-linux/sdist"
 DISTDIR="$PROJECT_ROOT/dist"
-LOCALE="$PROJECT_ROOT/electrum/locale/"
+LOCALE="$PROJECT_ROOT/electrum/locale"
 
 . "$CONTRIB"/build_tools_util.sh
 
@@ -19,35 +19,55 @@ break_legacy_easy_install
 # (make_packages will later install a pinned version of pip in a venv)
 python3 -m pip install --upgrade pip
 
-"$CONTRIB"/make_packages || fail "make_packages failed"
+rm -rf "$PROJECT_ROOT/packages/"
+if ([ "$OMIT_UNCLEAN_FILES" != 1 ]); then
+  "$CONTRIB"/make_packages || fail "make_packages failed"
+fi
 
 git submodule update --init
 
 (
-    cd "$CONTRIB/deterministic-build/electrum-locale/"
-    if ! which msgfmt > /dev/null 2>&1; then
-        echo "Please install gettext"
-        exit 1
-    fi
-    # We include both source (.po) and compiled (.mo) locale files in the source dist.
-    # Maybe we should exclude the compiled locale files? see https://askubuntu.com/a/144139
-    # (also see MANIFEST.in)
+    # By default, include both source (.po) and compiled (.mo) locale files in the source dist.
+    # Set option OMIT_UNCLEAN_FILES=1 to exclude the compiled locale files
+    # see https://askubuntu.com/a/144139 (also see MANIFEST.in)
     rm -rf "$LOCALE"
-    for i in ./locale/*; do
-        dir="$PROJECT_ROOT/electrum/$i/LC_MESSAGES"
-        mkdir -p "$dir"
-        msgfmt --output-file="$dir/electrum.mo" "$i/electrum.po" || true
-        cp $i/electrum.po "$PROJECT_ROOT/electrum/$i/electrum.po"
-    done
+    cp -r "$CONTRIB/deterministic-build/electrum-locale/locale/" "$LOCALE/"
+    if ([ "$OMIT_UNCLEAN_FILES" != 1 ]); then
+      "$CONTRIB/build_locale.sh" "$LOCALE"
+    fi
 )
 
+if ([ "$OMIT_UNCLEAN_FILES" = 1 ]); then
+    # FIXME side-effecting repo... though in practice, this script probably runs in fresh_clone
+    rm -f "$PROJECT_ROOT/electrum/paymentrequest_pb2.py"
+fi
+
 (
     cd "$PROJECT_ROOT"
 
     find -exec touch -h -d '2000-11-11T11:11:11+00:00' {} +
 
     # note: .zip sdists would not be reproducible due to https://bugs.python.org/issue40963
-    TZ=UTC faketime -f '2000-11-11 11:11:11' python3 setup.py --quiet sdist --format=gztar
+    if ([ "$OMIT_UNCLEAN_FILES" = 1 ])
+        then PY_DISTDIR="dist/_sourceonly" # The DISTDIR variable of this script is only used to find where the output is *finally* placed.
+        else PY_DISTDIR="dist"
+    fi
+    TZ=UTC faketime -f '2000-11-11 11:11:11' python3 setup.py --quiet sdist --format=gztar --dist-dir="$PY_DISTDIR"
+    if ([ "$OMIT_UNCLEAN_FILES" = 1 ]); then
+        python3 <<EOF
+import importlib.util
+import os
+
+# load version.py; needlessly complicated alternative to "imp.load_source":
+version_spec = importlib.util.spec_from_file_location('version', 'electrum/version.py')
+version_module = importlib.util.module_from_spec(version_spec)
+version_spec.loader.exec_module(version_module)
+
+VER = version_module.ELECTRUM_VERSION
+os.rename(f"dist/_sourceonly/Electrum-{VER}.tar.gz", f"dist/Electrum-sourceonly-{VER}.tar.gz")
+EOF
+        rmdir "$PY_DISTDIR"
+    fi
 )
 
 

contrib/build_locale.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+if [ ! -d "$1" ]; then
+    echo "usage: $0 path/to/locale"
+    exit 1
+fi
+
+if ! which msgfmt > /dev/null 2>&1; then
+    echo "Please install gettext"
+    exit 1
+fi
+
+for i in "$1/"*; do
+  mkdir -p "$i/LC_MESSAGES"
+  (msgfmt --output-file="$i/LC_MESSAGES/electrum.mo" "$i/electrum.po" || true)
+done

contrib/generate_payreqpb2.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+# Generates the file paymentrequest_pb2.py
+
+CONTRIB="$(dirname "$(readlink -e "$0")")"
+EL="$CONTRIB"/../electrum
+
+if ! which protoc > /dev/null 2>&1; then
+    echo "Please install 'protoc'"
+    echo "If you're on Debian, try 'sudo apt install protobuf-compiler'?"
+    exit 1
+fi
+
+protoc --proto_path="$EL" --python_out="$EL" "$EL"/paymentrequest.proto

contrib/release.sh

@@ -88,6 +88,14 @@ else
    ./contrib/build-linux/sdist/build.sh
 fi
 
+# create source-only tarball
+srctarball="Electrum-sourceonly-$VERSION.tar.gz"
+if test -f "dist/$srctarball"; then
+    info "file exists: $srctarball"
+else
+   OMIT_UNCLEAN_FILES=1 ./contrib/build-linux/sdist/build.sh
+fi
+
 # appimage
 appimage="electrum-$REV-x86_64.AppImage"
 if test -f "dist/$appimage"; then
@@ -187,6 +195,7 @@ if [ -z "$RELEASEMANAGER" ] ; then
 !
     # check we have each binary
     test -f "$tarball"    || fail "tarball not found among sftp downloads"
+    test -f "$srctarball" || fail "srctarball not found among sftp downloads"
     test -f "$appimage"   || fail "appimage not found among sftp downloads"
     test -f "$win1"       || fail "win1 not found among sftp downloads"
     test -f "$win2"       || fail "win2 not found among sftp downloads"
@@ -195,6 +204,7 @@ if [ -z "$RELEASEMANAGER" ] ; then
     test -f "$apk2"       || fail "apk2 not found among sftp downloads"
     test -f "$dmg"        || fail "dmg not found among sftp downloads"
     test -f "$PROJECT_ROOT/dist/$tarball"    || fail "tarball not found among built files"
+    test -f "$PROJECT_ROOT/dist/$srctarball" || fail "srctarball not found among built files"
     test -f "$PROJECT_ROOT/dist/$appimage"   || fail "appimage not found among built files"
     test -f "$CONTRIB/build-wine/dist/$win1" || fail "win1 not found among built files"
     test -f "$CONTRIB/build-wine/dist/$win2" || fail "win2 not found among built files"
@@ -204,6 +214,7 @@ if [ -z "$RELEASEMANAGER" ] ; then
     test -f "$PROJECT_ROOT/dist/$dmg"        || fail "dmg not found among built files"
     # compare downloaded binaries against ones we built
     cmp --silent "$tarball"    "$PROJECT_ROOT/dist/$tarball"    || fail "files are different. tarball."
+    cmp --silent "$srctarball" "$PROJECT_ROOT/dist/$srctarball" || fail "files are different. srctarball."
     cmp --silent "$appimage"   "$PROJECT_ROOT/dist/$appimage"   || fail "files are different. appimage."
     rm -rf "$CONTRIB/build-wine/signed/" && mkdir --parents "$CONTRIB/build-wine/signed/"
     cp -f "$win1" "$win2" "$win3" "$CONTRIB/build-wine/signed/"
@@ -214,7 +225,7 @@ if [ -z "$RELEASEMANAGER" ] ; then
     # all files matched. sign them.
     rm -rf "$PROJECT_ROOT/dist/sigs/"
     mkdir --parents "$PROJECT_ROOT/dist/sigs/"
-    for fname in "$tarball" "$appimage" "$win1" "$win2" "$win3" "$apk1" "$apk2" "$dmg" ; do
+    for fname in "$tarball" "$srctarball" "$appimage" "$win1" "$win2" "$win3" "$apk1" "$apk2" "$dmg" ; do
         signame="$fname.$GPGUSER.asc"
         gpg --sign --armor --detach $PUBKEY --output "$PROJECT_ROOT/dist/sigs/$signame" "$fname"
     done

electrum/paymentrequest.py

@@ -36,8 +36,7 @@ import aiohttp
 try:
     from . import paymentrequest_pb2 as pb2
 except ImportError:
-    # sudo apt-get install protobuf-compiler
-    sys.exit("Error: could not find paymentrequest_pb2.py. Create it with 'protoc --proto_path=electrum/ --python_out=electrum/ electrum/paymentrequest.proto'")
+    sys.exit("Error: could not find paymentrequest_pb2.py. Create it with 'contrib/generate_payreqpb2.sh'")
 
 from . import bitcoin, constants, ecc, util, transaction, x509, rsakey
 from .util import bh2u, bfh, make_aiohttp_session