Merge pull request #7741 from SomberNight/202203_build_pip_install_noisolation

build: (reproducibility) always uses `pip install` with `--no-build-isolation`

contrib/android/Dockerfile

@@ -145,8 +145,11 @@ RUN chown ${USER} /opt
 USER ${USER}
 
 
+COPY contrib/deterministic-build/requirements-build-base.txt /opt/deterministic-build/
 COPY contrib/deterministic-build/requirements-build-android.txt /opt/deterministic-build/
-RUN python3 -m pip install --no-dependencies --user \
+RUN python3 -m pip install --no-build-isolation --no-dependencies --user \
+    -r /opt/deterministic-build/requirements-build-base.txt
+RUN python3 -m pip install --no-build-isolation --no-dependencies --user \
     -r /opt/deterministic-build/requirements-build-android.txt
 
 # install buildozer
@@ -157,7 +160,7 @@ RUN cd /opt \
     && git fetch --all \
     # commit: from branch sombernight/electrum_20210421
     && git checkout "d570116e88184b0eca0c6b59a25edd49d977da23^{commit}" \
-    && python3 -m pip install --no-dependencies --user -e .
+    && python3 -m pip install --no-build-isolation --no-dependencies --user -e .
 
 # install python-for-android
 RUN cd /opt \
@@ -168,7 +171,7 @@ RUN cd /opt \
     && git fetch --all \
     # commit: from branch accumulator/qt5-wip
     && git checkout "ebbe8dcc271e36468666feb98f936d0a96936cf2^{commit}" \
-    && python3 -m pip install --no-dependencies --user -e .
+    && python3 -m pip install --no-build-isolation --no-dependencies --user -e .
 
 # build env vars
 ENV USE_SDK_WRAPPER=1

contrib/build-linux/appimage/make_appimage.sh

@@ -108,7 +108,9 @@ info "preparing electrum-locale."
 
 
 info "Installing build dependencies."
-"$python" -m pip install --no-dependencies --no-binary :all: --no-warn-script-location \
+"$python" -m pip install --no-build-isolation --no-dependencies --no-warn-script-location \
+    --cache-dir "$PIP_CACHE_DIR" -r "$CONTRIB/deterministic-build/requirements-build-base.txt"
+"$python" -m pip install --no-build-isolation --no-dependencies --no-binary :all: --no-warn-script-location \
     --cache-dir "$PIP_CACHE_DIR" -r "$CONTRIB/deterministic-build/requirements-build-appimage.txt"
 
 info "installing electrum and its dependencies."
@@ -116,14 +118,14 @@ info "installing electrum and its dependencies."
 #       hence "--no-binary :all:" flags. However, we specifically allow
 #       - PyQt5, as it's harder to build from source
 #       - cryptography, as building it would need openssl 1.1, not available on ubuntu 16.04
-"$python" -m pip install --no-dependencies --no-binary :all: --no-warn-script-location \
+"$python" -m pip install --no-build-isolation --no-dependencies --no-binary :all: --no-warn-script-location \
     --cache-dir "$PIP_CACHE_DIR" -r "$CONTRIB/deterministic-build/requirements.txt"
-"$python" -m pip install --no-dependencies --no-binary :all: --only-binary PyQt5,PyQt5-Qt5,cryptography --no-warn-script-location \
+"$python" -m pip install --no-build-isolation --no-dependencies --no-binary :all: --only-binary PyQt5,PyQt5-Qt5,cryptography --no-warn-script-location \
     --cache-dir "$PIP_CACHE_DIR" -r "$CONTRIB/deterministic-build/requirements-binaries.txt"
-"$python" -m pip install --no-dependencies --no-binary :all: --no-warn-script-location \
+"$python" -m pip install --no-build-isolation --no-dependencies --no-binary :all: --no-warn-script-location \
     --cache-dir "$PIP_CACHE_DIR" -r "$CONTRIB/deterministic-build/requirements-hw.txt"
 
-"$python" -m pip install --no-dependencies --no-warn-script-location \
+"$python" -m pip install --no-build-isolation --no-dependencies --no-warn-script-location \
     --cache-dir "$PIP_CACHE_DIR" "$PROJECT_ROOT"
 
 # was only needed during build time, not runtime

contrib/build-wine/build-electrum-git.sh

@@ -36,19 +36,19 @@ popd
 
 
 # Install frozen dependencies
-$WINE_PYTHON -m pip install --no-dependencies --no-warn-script-location \
+$WINE_PYTHON -m pip install --no-build-isolation --no-dependencies --no-warn-script-location \
     --cache-dir "$WINE_PIP_CACHE_DIR" -r "$CONTRIB"/deterministic-build/requirements.txt
 
-$WINE_PYTHON -m pip install --no-dependencies --no-warn-script-location \
+$WINE_PYTHON -m pip install --no-build-isolation --no-dependencies --no-warn-script-location \
     --cache-dir "$WINE_PIP_CACHE_DIR" -r "$CONTRIB"/deterministic-build/requirements-binaries.txt
 
-$WINE_PYTHON -m pip install --no-dependencies --no-warn-script-location \
+$WINE_PYTHON -m pip install --no-build-isolation --no-dependencies --no-warn-script-location \
     --cache-dir "$WINE_PIP_CACHE_DIR" -r "$CONTRIB"/deterministic-build/requirements-hw.txt
 
 pushd $WINEPREFIX/drive_c/electrum
 # see https://github.com/pypa/pip/issues/2195 -- pip makes a copy of the entire directory
 info "Pip installing Electrum. This might take a long time if the project folder is large."
-$WINE_PYTHON -m pip install --no-dependencies --no-warn-script-location .
+$WINE_PYTHON -m pip install --no-build-isolation --no-dependencies --no-warn-script-location .
 popd
 
 

contrib/build-wine/prepare-wine.sh

@@ -51,7 +51,9 @@ done
 break_legacy_easy_install
 
 info "Installing build dependencies."
-$WINE_PYTHON -m pip install --no-dependencies --no-warn-script-location \
+$WINE_PYTHON -m pip install --no-build-isolation --no-dependencies --no-warn-script-location \
+    --cache-dir "$WINE_PIP_CACHE_DIR" -r "$CONTRIB"/deterministic-build/requirements-build-base.txt
+$WINE_PYTHON -m pip install --no-build-isolation --no-dependencies --no-warn-script-location \
     --cache-dir "$WINE_PIP_CACHE_DIR" -r "$CONTRIB"/deterministic-build/requirements-build-wine.txt
 
 info "Installing NSIS."
@@ -105,6 +107,6 @@ info "Building PyInstaller."
     [[ -e "PyInstaller/bootloader/Windows-$PYINST_ARCH/runw.exe" ]] || fail "Could not find runw.exe in target dir!"
 ) || fail "PyInstaller build failed"
 info "Installing PyInstaller."
-$WINE_PYTHON -m pip install --no-dependencies --no-build-isolation --no-warn-script-location ./pyinstaller
+$WINE_PYTHON -m pip install --no-build-isolation --no-dependencies --no-warn-script-location ./pyinstaller
 
 info "Wine is configured."

contrib/freeze_packages.sh

@@ -20,7 +20,7 @@ which virtualenv > /dev/null 2>&1 || { echo "Please install virtualenv" && exit
 
 ${SYSTEM_PYTHON} -m hashin -h > /dev/null 2>&1 || { ${SYSTEM_PYTHON} -m pip install hashin; }
 
-for i in '' '-hw' '-binaries' '-binaries-mac' '-build-wine' '-build-mac' '-build-makepackages' '-build-appimage' '-build-android'; do
+for i in '' '-hw' '-binaries' '-binaries-mac' '-build-wine' '-build-mac' '-build-base' '-build-appimage' '-build-android'; do
     rm -rf "$venv_dir"
     virtualenv -p ${SYSTEM_PYTHON} $venv_dir
 

contrib/make_packages

@@ -21,7 +21,8 @@ python3 -m venv "$venv_dir"
 source "$venv_dir"/bin/activate
 
 # installing pinned build-time requirements, such as pip/wheel/setuptools
-python -m pip install --no-dependencies --no-warn-script-location -r "$CONTRIB"/deterministic-build/requirements-build-makepackages.txt
+python -m pip install --no-build-isolation --no-dependencies --no-warn-script-location \
+    -r "$CONTRIB"/deterministic-build/requirements-build-base.txt
 
 # opt out of compiling C extensions
 # FIXME aiohttp opt-out is not released yet: https://github.com/aio-libs/aiohttp/pull/3828

contrib/osx/make_osx

@@ -93,8 +93,12 @@ python3 -m venv $VENV_DIR
 source $VENV_DIR/bin/activate
 
 info "Installing build dependencies"
-python3 -m pip install --no-dependencies --no-warn-script-location -Ir ./contrib/deterministic-build/requirements-build-mac.txt \
-    || fail "Could not install build dependencies"
+python3 -m pip install --no-build-isolation --no-dependencies --no-warn-script-location \
+    -Ir ./contrib/deterministic-build/requirements-build-base.txt \
+    || fail "Could not install build dependencies (base)"
+python3 -m pip install --no-build-isolation --no-dependencies --no-warn-script-location \
+    -Ir ./contrib/deterministic-build/requirements-build-mac.txt \
+    || fail "Could not install build dependencies (mac)"
 
 info "Using these versions for building $PACKAGE:"
 sw_vers
@@ -155,19 +159,23 @@ cp "$PROJECT_ROOT"/electrum/libusb-1.0.dylib "$CONTRIB"/osx
 
 
 info "Installing requirements..."
-python3 -m pip install --no-dependencies --no-warn-script-location -Ir ./contrib/deterministic-build/requirements.txt \
+python3 -m pip install --no-build-isolation --no-dependencies --no-warn-script-location \
+    -Ir ./contrib/deterministic-build/requirements.txt \
     || fail "Could not install requirements"
 
 info "Installing hardware wallet requirements..."
-python3 -m pip install --no-dependencies --no-warn-script-location -Ir ./contrib/deterministic-build/requirements-hw.txt \
+python3 -m pip install --no-build-isolation --no-dependencies --no-warn-script-location \
+    -Ir ./contrib/deterministic-build/requirements-hw.txt \
     || fail "Could not install hardware wallet requirements"
 
 info "Installing dependencies specific to binaries..."
-python3 -m pip install --no-dependencies --no-warn-script-location -Ir ./contrib/deterministic-build/requirements-binaries-mac.txt \
+python3 -m pip install --no-build-isolation --no-dependencies --no-warn-script-location \
+    -Ir ./contrib/deterministic-build/requirements-binaries-mac.txt \
     || fail "Could not install dependencies specific to binaries"
 
 info "Building $PACKAGE..."
-python3 -m pip install --no-dependencies --no-warn-script-location . > /dev/null || fail "Could not build $PACKAGE"
+python3 -m pip install --no-build-isolation --no-dependencies \
+    --no-warn-script-location . > /dev/null || fail "Could not build $PACKAGE"
 
 info "Faking timestamps..."
 find . -exec touch -t '200101220000' {} + || true

contrib/requirements/requirements-build-makepackages.txt → contrib/requirements/requirements-build-base.txt

@@ -1,4 +1,5 @@
-# This file contains build-time dependencies needed to be able to build our pure python dependencies.
+# This file contains build-time dependencies needed to build other higher level build-time dependencies
+# and runtime dependencies.
 # For reproducibility, some build-time deps, most notably "wheel", need to be pinned. (see #7640)
 # By default, when doing e.g. "pip install", pip downloads the latest version of wheel (and setuptools, etc),
 # regardless whether a sufficiently recent version of wheel is already installed locally...