From 98d32f41d5ed0b29a9d45d773bfc97f6ad392670 Mon Sep 17 00:00:00 2001
From: SomberNight <somber.night@protonmail.com>
Date: Sat, 26 Mar 2022 02:12:12 +0100
Subject: [PATCH] build: (reproducibility) always uses "pip install" with
 "--no-build-isolation"

fixes https://github.com/spesmilo/electrum/issues/7737
fixes https://github.com/spesmilo/electrum/issues/7736
related https://github.com/spesmilo/electrum/commit/8559d1eb723c8c462c3d2a7365967ada46ef21c5
---
 contrib/android/Dockerfile                    |  9 ++++++---
 contrib/build-linux/appimage/make_appimage.sh | 12 ++++++-----
 contrib/build-wine/build-electrum-git.sh      |  8 ++++----
 contrib/build-wine/prepare-wine.sh            |  6 ++++--
 ...ckages.txt => requirements-build-base.txt} |  0
 contrib/freeze_packages.sh                    |  2 +-
 contrib/make_packages                         |  3 ++-
 contrib/osx/make_osx                          | 20 +++++++++++++------
 ...ckages.txt => requirements-build-base.txt} |  3 ++-
 9 files changed, 40 insertions(+), 23 deletions(-)
 rename contrib/deterministic-build/{requirements-build-makepackages.txt => requirements-build-base.txt} (100%)
 rename contrib/requirements/{requirements-build-makepackages.txt => requirements-build-base.txt} (88%)

diff --git a/contrib/android/Dockerfile b/contrib/android/Dockerfile
index 8e56fd2b2..88f4f0813 100644
--- a/contrib/android/Dockerfile
+++ b/contrib/android/Dockerfile
@@ -145,8 +145,11 @@ RUN chown ${USER} /opt
 USER ${USER}
 
 
+COPY contrib/deterministic-build/requirements-build-base.txt /opt/deterministic-build/
 COPY contrib/deterministic-build/requirements-build-android.txt /opt/deterministic-build/
-RUN python3 -m pip install --no-dependencies --user \
+RUN python3 -m pip install --no-build-isolation --no-dependencies --user \
+    -r /opt/deterministic-build/requirements-build-base.txt
+RUN python3 -m pip install --no-build-isolation --no-dependencies --user \
     -r /opt/deterministic-build/requirements-build-android.txt
 
 # install buildozer
@@ -157,7 +160,7 @@ RUN cd /opt \
     && git fetch --all \
     # commit: from branch sombernight/electrum_20210421
     && git checkout "d570116e88184b0eca0c6b59a25edd49d977da23^{commit}" \
-    && python3 -m pip install --no-dependencies --user -e .
+    && python3 -m pip install --no-build-isolation --no-dependencies --user -e .
 
 # install python-for-android
 RUN cd /opt \
@@ -168,7 +171,7 @@ RUN cd /opt \
     && git fetch --all \
     # commit: from branch accumulator/qt5-wip
     && git checkout "ebbe8dcc271e36468666feb98f936d0a96936cf2^{commit}" \
-    && python3 -m pip install --no-dependencies --user -e .
+    && python3 -m pip install --no-build-isolation --no-dependencies --user -e .
 
 # build env vars
 ENV USE_SDK_WRAPPER=1
diff --git a/contrib/build-linux/appimage/make_appimage.sh b/contrib/build-linux/appimage/make_appimage.sh
index acf2bff68..b7048e39d 100755
--- a/contrib/build-linux/appimage/make_appimage.sh
+++ b/contrib/build-linux/appimage/make_appimage.sh
@@ -108,7 +108,9 @@ info "preparing electrum-locale."
 
 
 info "Installing build dependencies."
-"$python" -m pip install --no-dependencies --no-binary :all: --no-warn-script-location \
+"$python" -m pip install --no-build-isolation --no-dependencies --no-warn-script-location \
+    --cache-dir "$PIP_CACHE_DIR" -r "$CONTRIB/deterministic-build/requirements-build-base.txt"
+"$python" -m pip install --no-build-isolation --no-dependencies --no-binary :all: --no-warn-script-location \
     --cache-dir "$PIP_CACHE_DIR" -r "$CONTRIB/deterministic-build/requirements-build-appimage.txt"
 
 info "installing electrum and its dependencies."
@@ -116,14 +118,14 @@ info "installing electrum and its dependencies."
 #       hence "--no-binary :all:" flags. However, we specifically allow
 #       - PyQt5, as it's harder to build from source
 #       - cryptography, as building it would need openssl 1.1, not available on ubuntu 16.04
-"$python" -m pip install --no-dependencies --no-binary :all: --no-warn-script-location \
+"$python" -m pip install --no-build-isolation --no-dependencies --no-binary :all: --no-warn-script-location \
     --cache-dir "$PIP_CACHE_DIR" -r "$CONTRIB/deterministic-build/requirements.txt"
-"$python" -m pip install --no-dependencies --no-binary :all: --only-binary PyQt5,PyQt5-Qt5,cryptography --no-warn-script-location \
+"$python" -m pip install --no-build-isolation --no-dependencies --no-binary :all: --only-binary PyQt5,PyQt5-Qt5,cryptography --no-warn-script-location \
     --cache-dir "$PIP_CACHE_DIR" -r "$CONTRIB/deterministic-build/requirements-binaries.txt"
-"$python" -m pip install --no-dependencies --no-binary :all: --no-warn-script-location \
+"$python" -m pip install --no-build-isolation --no-dependencies --no-binary :all: --no-warn-script-location \
     --cache-dir "$PIP_CACHE_DIR" -r "$CONTRIB/deterministic-build/requirements-hw.txt"
 
-"$python" -m pip install --no-dependencies --no-warn-script-location \
+"$python" -m pip install --no-build-isolation --no-dependencies --no-warn-script-location \
     --cache-dir "$PIP_CACHE_DIR" "$PROJECT_ROOT"
 
 # was only needed during build time, not runtime
diff --git a/contrib/build-wine/build-electrum-git.sh b/contrib/build-wine/build-electrum-git.sh
index 3bfcbbd76..86aa6a910 100755
--- a/contrib/build-wine/build-electrum-git.sh
+++ b/contrib/build-wine/build-electrum-git.sh
@@ -36,19 +36,19 @@ popd
 
 
 # Install frozen dependencies
-$WINE_PYTHON -m pip install --no-dependencies --no-warn-script-location \
+$WINE_PYTHON -m pip install --no-build-isolation --no-dependencies --no-warn-script-location \
     --cache-dir "$WINE_PIP_CACHE_DIR" -r "$CONTRIB"/deterministic-build/requirements.txt
 
-$WINE_PYTHON -m pip install --no-dependencies --no-warn-script-location \
+$WINE_PYTHON -m pip install --no-build-isolation --no-dependencies --no-warn-script-location \
     --cache-dir "$WINE_PIP_CACHE_DIR" -r "$CONTRIB"/deterministic-build/requirements-binaries.txt
 
-$WINE_PYTHON -m pip install --no-dependencies --no-warn-script-location \
+$WINE_PYTHON -m pip install --no-build-isolation --no-dependencies --no-warn-script-location \
     --cache-dir "$WINE_PIP_CACHE_DIR" -r "$CONTRIB"/deterministic-build/requirements-hw.txt
 
 pushd $WINEPREFIX/drive_c/electrum
 # see https://github.com/pypa/pip/issues/2195 -- pip makes a copy of the entire directory
 info "Pip installing Electrum. This might take a long time if the project folder is large."
-$WINE_PYTHON -m pip install --no-dependencies --no-warn-script-location .
+$WINE_PYTHON -m pip install --no-build-isolation --no-dependencies --no-warn-script-location .
 popd
 
 
diff --git a/contrib/build-wine/prepare-wine.sh b/contrib/build-wine/prepare-wine.sh
index 1b55d7181..186ffd6ae 100755
--- a/contrib/build-wine/prepare-wine.sh
+++ b/contrib/build-wine/prepare-wine.sh
@@ -51,7 +51,9 @@ done
 break_legacy_easy_install
 
 info "Installing build dependencies."
-$WINE_PYTHON -m pip install --no-dependencies --no-warn-script-location \
+$WINE_PYTHON -m pip install --no-build-isolation --no-dependencies --no-warn-script-location \
+    --cache-dir "$WINE_PIP_CACHE_DIR" -r "$CONTRIB"/deterministic-build/requirements-build-base.txt
+$WINE_PYTHON -m pip install --no-build-isolation --no-dependencies --no-warn-script-location \
     --cache-dir "$WINE_PIP_CACHE_DIR" -r "$CONTRIB"/deterministic-build/requirements-build-wine.txt
 
 info "Installing NSIS."
@@ -105,6 +107,6 @@ info "Building PyInstaller."
     [[ -e "PyInstaller/bootloader/Windows-$PYINST_ARCH/runw.exe" ]] || fail "Could not find runw.exe in target dir!"
 ) || fail "PyInstaller build failed"
 info "Installing PyInstaller."
-$WINE_PYTHON -m pip install --no-dependencies --no-build-isolation --no-warn-script-location ./pyinstaller
+$WINE_PYTHON -m pip install --no-build-isolation --no-dependencies --no-warn-script-location ./pyinstaller
 
 info "Wine is configured."
diff --git a/contrib/deterministic-build/requirements-build-makepackages.txt b/contrib/deterministic-build/requirements-build-base.txt
similarity index 100%
rename from contrib/deterministic-build/requirements-build-makepackages.txt
rename to contrib/deterministic-build/requirements-build-base.txt
diff --git a/contrib/freeze_packages.sh b/contrib/freeze_packages.sh
index d6e91bf55..0f9a159b3 100755
--- a/contrib/freeze_packages.sh
+++ b/contrib/freeze_packages.sh
@@ -20,7 +20,7 @@ which virtualenv > /dev/null 2>&1 || { echo "Please install virtualenv" && exit
 
 ${SYSTEM_PYTHON} -m hashin -h > /dev/null 2>&1 || { ${SYSTEM_PYTHON} -m pip install hashin; }
 
-for i in '' '-hw' '-binaries' '-binaries-mac' '-build-wine' '-build-mac' '-build-makepackages' '-build-appimage' '-build-android'; do
+for i in '' '-hw' '-binaries' '-binaries-mac' '-build-wine' '-build-mac' '-build-base' '-build-appimage' '-build-android'; do
     rm -rf "$venv_dir"
     virtualenv -p ${SYSTEM_PYTHON} $venv_dir
 
diff --git a/contrib/make_packages b/contrib/make_packages
index 40e4bf0be..4fa7e4615 100755
--- a/contrib/make_packages
+++ b/contrib/make_packages
@@ -21,7 +21,8 @@ python3 -m venv "$venv_dir"
 source "$venv_dir"/bin/activate
 
 # installing pinned build-time requirements, such as pip/wheel/setuptools
-python -m pip install --no-dependencies --no-warn-script-location -r "$CONTRIB"/deterministic-build/requirements-build-makepackages.txt
+python -m pip install --no-build-isolation --no-dependencies --no-warn-script-location \
+    -r "$CONTRIB"/deterministic-build/requirements-build-base.txt
 
 # opt out of compiling C extensions
 # FIXME aiohttp opt-out is not released yet: https://github.com/aio-libs/aiohttp/pull/3828
diff --git a/contrib/osx/make_osx b/contrib/osx/make_osx
index 4dbf4d0e4..c85904c5d 100755
--- a/contrib/osx/make_osx
+++ b/contrib/osx/make_osx
@@ -93,8 +93,12 @@ python3 -m venv $VENV_DIR
 source $VENV_DIR/bin/activate
 
 info "Installing build dependencies"
-python3 -m pip install --no-dependencies --no-warn-script-location -Ir ./contrib/deterministic-build/requirements-build-mac.txt \
-    || fail "Could not install build dependencies"
+python3 -m pip install --no-build-isolation --no-dependencies --no-warn-script-location \
+    -Ir ./contrib/deterministic-build/requirements-build-base.txt \
+    || fail "Could not install build dependencies (base)"
+python3 -m pip install --no-build-isolation --no-dependencies --no-warn-script-location \
+    -Ir ./contrib/deterministic-build/requirements-build-mac.txt \
+    || fail "Could not install build dependencies (mac)"
 
 info "Using these versions for building $PACKAGE:"
 sw_vers
@@ -155,19 +159,23 @@ cp "$PROJECT_ROOT"/electrum/libusb-1.0.dylib "$CONTRIB"/osx
 
 
 info "Installing requirements..."
-python3 -m pip install --no-dependencies --no-warn-script-location -Ir ./contrib/deterministic-build/requirements.txt \
+python3 -m pip install --no-build-isolation --no-dependencies --no-warn-script-location \
+    -Ir ./contrib/deterministic-build/requirements.txt \
     || fail "Could not install requirements"
 
 info "Installing hardware wallet requirements..."
-python3 -m pip install --no-dependencies --no-warn-script-location -Ir ./contrib/deterministic-build/requirements-hw.txt \
+python3 -m pip install --no-build-isolation --no-dependencies --no-warn-script-location \
+    -Ir ./contrib/deterministic-build/requirements-hw.txt \
     || fail "Could not install hardware wallet requirements"
 
 info "Installing dependencies specific to binaries..."
-python3 -m pip install --no-dependencies --no-warn-script-location -Ir ./contrib/deterministic-build/requirements-binaries-mac.txt \
+python3 -m pip install --no-build-isolation --no-dependencies --no-warn-script-location \
+    -Ir ./contrib/deterministic-build/requirements-binaries-mac.txt \
     || fail "Could not install dependencies specific to binaries"
 
 info "Building $PACKAGE..."
-python3 -m pip install --no-dependencies --no-warn-script-location . > /dev/null || fail "Could not build $PACKAGE"
+python3 -m pip install --no-build-isolation --no-dependencies \
+    --no-warn-script-location . > /dev/null || fail "Could not build $PACKAGE"
 
 info "Faking timestamps..."
 find . -exec touch -t '200101220000' {} + || true
diff --git a/contrib/requirements/requirements-build-makepackages.txt b/contrib/requirements/requirements-build-base.txt
similarity index 88%
rename from contrib/requirements/requirements-build-makepackages.txt
rename to contrib/requirements/requirements-build-base.txt
index 7a80817ef..55717c8e2 100644
--- a/contrib/requirements/requirements-build-makepackages.txt
+++ b/contrib/requirements/requirements-build-base.txt
@@ -1,4 +1,5 @@
-# This file contains build-time dependencies needed to be able to build our pure python dependencies.
+# This file contains build-time dependencies needed to build other higher level build-time dependencies
+# and runtime dependencies.
 # For reproducibility, some build-time deps, most notably "wheel", need to be pinned. (see #7640)
 # By default, when doing e.g. "pip install", pip downloads the latest version of wheel (and setuptools, etc),
 # regardless whether a sufficiently recent version of wheel is already installed locally...