From 8559d1eb723c8c462c3d2a7365967ada46ef21c5 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Fri, 21 Jan 2022 18:02:37 +0100 Subject: [PATCH] build: android reprod: "pip install" needs "--no-build-isolation" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit maybe fixes https://github.com/spesmilo/electrum/issues/7640 Looks like by default pip is ignoring the locally available setuptools and wheel, and downloading the latest ones from the internet at build time... https://pip.pypa.io/en/stable/reference/build-system/pyproject-toml/?highlight=no-build-isolation#disabling-build-isolation https://stackoverflow.com/a/62889268 > When making build requirements available, pip does so in an isolated environment. That is, pip does not install those requirements into the user’s site-packages, but rather installs them in a temporary directory which it adds to the user’s sys.path for the duration of the build. This ensures that build requirements are handled independently of the user’s runtime environment. For example, a project that needs a recent version of setuptools to build can still be installed, even if the user has an older version installed (and without silently replacing that version). > > In certain cases, projects (or redistributors) may have workflows that explicitly manage the build environment. For such workflows, build isolation can be problematic. If this is the case, pip provides a --no-build-isolation flag to disable build isolation. Users supplying this flag are responsible for ensuring the build environment is managed appropriately (including ensuring that all required build dependencies are installed). If only it were that easy! If we add the "--no-build-isolation" flag, it becomes our responsibility to install *all* build time deps, hence we now have "requirements-build-makepackages.txt". --- .gitignore | 1 + contrib/android/Dockerfile | 1 + contrib/build-linux/sdist/Dockerfile | 1 + contrib/build-linux/sdist/make_sdist.sh | 6 +----- .../requirements-build-makepackages.txt | 21 +++++++++++++++++++ .../requirements-build-sdist.txt | 9 -------- contrib/freeze_packages.sh | 2 +- contrib/make_packages | 15 +++++++++++-- .../requirements-build-makepackages.txt | 14 +++++++++++++ .../requirements/requirements-build-sdist.txt | 4 ---- 10 files changed, 53 insertions(+), 21 deletions(-) create mode 100644 contrib/deterministic-build/requirements-build-makepackages.txt delete mode 100644 contrib/deterministic-build/requirements-build-sdist.txt create mode 100644 contrib/requirements/requirements-build-makepackages.txt delete mode 100644 contrib/requirements/requirements-build-sdist.txt diff --git a/.gitignore b/.gitignore index 5b2da718c..193ffea3a 100644 --- a/.gitignore +++ b/.gitignore @@ -44,6 +44,7 @@ contrib/android/android_debug.keystore contrib/secp256k1/ contrib/zbar/ contrib/libusb/ +contrib/.venv_make_packages/ # shared objects electrum/*.so diff --git a/contrib/android/Dockerfile b/contrib/android/Dockerfile index 9f8fbd855..71ac55172 100644 --- a/contrib/android/Dockerfile +++ b/contrib/android/Dockerfile @@ -108,6 +108,7 @@ RUN apt -y update -qq \ python3-dev \ python3-pip \ python3-setuptools \ + python3-venv \ wget \ lbzip2 \ patch \ diff --git a/contrib/build-linux/sdist/Dockerfile b/contrib/build-linux/sdist/Dockerfile index b7391f379..bf7d03850 100644 --- a/contrib/build-linux/sdist/Dockerfile +++ b/contrib/build-linux/sdist/Dockerfile @@ -10,6 +10,7 @@ RUN apt-get update -q && \ python3 \ python3-pip \ python3-setuptools \ + python3-venv \ faketime \ && \ rm -rf /var/lib/apt/lists/* && \ diff --git a/contrib/build-linux/sdist/make_sdist.sh b/contrib/build-linux/sdist/make_sdist.sh index 7aae3f423..5429ec74e 100755 --- a/contrib/build-linux/sdist/make_sdist.sh +++ b/contrib/build-linux/sdist/make_sdist.sh @@ -16,13 +16,9 @@ python3 --version || fail "python interpreter not found" break_legacy_easy_install # upgrade to modern pip so that it knows the flags we need. -# we will then install a pinned version of pip as part of requirements-build-sdist +# (make_packages will later install a pinned version of pip in a venv) python3 -m pip install --upgrade pip -info "Installing pinned requirements." -python3 -m pip install --no-dependencies --no-warn-script-location -r "$CONTRIB"/deterministic-build/requirements-build-sdist.txt - - "$CONTRIB"/make_packages || fail "make_packages failed" git submodule update --init diff --git a/contrib/deterministic-build/requirements-build-makepackages.txt b/contrib/deterministic-build/requirements-build-makepackages.txt new file mode 100644 index 000000000..147da8d6b --- /dev/null +++ b/contrib/deterministic-build/requirements-build-makepackages.txt @@ -0,0 +1,21 @@ +packaging==21.3 \ + --hash=sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb \ + --hash=sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522 +pip==21.3.1 \ + --hash=sha256:deaf32dcd9ab821e359cd8330786bcd077604b5c5730c0b096eda46f95c24a2d \ + --hash=sha256:fd11ba3d0fdb4c07fbc5ecbba0b1b719809420f25038f8ee3cd913d3faa3033a +pyparsing==3.0.7 \ + --hash=sha256:18ee9022775d270c55187733956460083db60b37d0d0fb357445f3094eed3eea \ + --hash=sha256:a6c06a88f252e6c322f65faf8f418b16213b51bdfaece0524c1c1bc30c63c484 +setuptools==59.6.0 \ + --hash=sha256:22c7348c6d2976a52632c67f7ab0cdf40147db7789f9aed18734643fe9cf3373 \ + --hash=sha256:4ce92f1e1f8f01233ee9952c04f6b81d1e02939d6e1b488428154974a4d0783e +setuptools-scm==6.4.2 \ + --hash=sha256:6833ac65c6ed9711a4d5d2266f8024cfa07c533a0e55f4c12f6eff280a5a9e30 \ + --hash=sha256:acea13255093849de7ccb11af9e1fb8bde7067783450cee9ef7a93139bddf6d4 +tomli==1.2.3 \ + --hash=sha256:05b6166bff487dc068d322585c7ea4ef78deed501cc124060e0f238e89a9231f \ + --hash=sha256:e3069e4be3ead9668e21cb9b074cd948f7b3113fd9c8bba083f48247aab8b11c +wheel==0.37.1 \ + --hash=sha256:4bdcd7d840138086126cd09254dc6195fb4fc6f01c050a1d7236f2630db1d22a \ + --hash=sha256:e9a504e793efbca1b8e0e9cb979a249cf4a0a7b5b8c9e8b65a5e39d49529c1c4 \ No newline at end of file diff --git a/contrib/deterministic-build/requirements-build-sdist.txt b/contrib/deterministic-build/requirements-build-sdist.txt deleted file mode 100644 index f19e9db07..000000000 --- a/contrib/deterministic-build/requirements-build-sdist.txt +++ /dev/null @@ -1,9 +0,0 @@ -pip==21.3.1 \ - --hash=sha256:deaf32dcd9ab821e359cd8330786bcd077604b5c5730c0b096eda46f95c24a2d \ - --hash=sha256:fd11ba3d0fdb4c07fbc5ecbba0b1b719809420f25038f8ee3cd913d3faa3033a -setuptools==59.6.0 \ - --hash=sha256:22c7348c6d2976a52632c67f7ab0cdf40147db7789f9aed18734643fe9cf3373 \ - --hash=sha256:4ce92f1e1f8f01233ee9952c04f6b81d1e02939d6e1b488428154974a4d0783e -wheel==0.37.1 \ - --hash=sha256:4bdcd7d840138086126cd09254dc6195fb4fc6f01c050a1d7236f2630db1d22a \ - --hash=sha256:e9a504e793efbca1b8e0e9cb979a249cf4a0a7b5b8c9e8b65a5e39d49529c1c4 \ No newline at end of file diff --git a/contrib/freeze_packages.sh b/contrib/freeze_packages.sh index c265345f9..3455771fd 100755 --- a/contrib/freeze_packages.sh +++ b/contrib/freeze_packages.sh @@ -20,7 +20,7 @@ which virtualenv > /dev/null 2>&1 || { echo "Please install virtualenv" && exit ${SYSTEM_PYTHON} -m hashin -h > /dev/null 2>&1 || { ${SYSTEM_PYTHON} -m pip install hashin; } -for i in '' '-hw' '-binaries' '-binaries-mac' '-build-wine' '-build-mac' '-build-sdist' '-build-appimage' '-build-android'; do +for i in '' '-hw' '-binaries' '-binaries-mac' '-build-wine' '-build-mac' '-build-makepackages' '-build-appimage' '-build-android'; do rm -rf "$venv_dir" virtualenv -p ${SYSTEM_PYTHON} $venv_dir diff --git a/contrib/make_packages b/contrib/make_packages index 6d29044fe..40e4bf0be 100755 --- a/contrib/make_packages +++ b/contrib/make_packages @@ -13,6 +13,16 @@ if [ -d "$PACKAGES" ]; then rm -r "$PACKAGES" fi +# create virtualenv +# note: venv path needs to be deterministic as some produced files will contain it +venv_dir="$CONTRIB/.venv_make_packages/" +rm -rf "$venv_dir" +python3 -m venv "$venv_dir" +source "$venv_dir"/bin/activate + +# installing pinned build-time requirements, such as pip/wheel/setuptools +python -m pip install --no-dependencies --no-warn-script-location -r "$CONTRIB"/deterministic-build/requirements-build-makepackages.txt + # opt out of compiling C extensions # FIXME aiohttp opt-out is not released yet: https://github.com/aio-libs/aiohttp/pull/3828 export AIOHTTP_NO_EXTENSIONS=1 @@ -34,6 +44,7 @@ export BUILD_TIME="$(LC_ALL=C TZ=UTC date +'%H:%M:%S' -d @$SOURCE_DATE_EPOCH)" # Hence "aiohttp-*.dist-info/" is not reproducible either. # All this means that downstream users of this script, such as the sdist build # and the android apk build need to make sure these files get excluded. -python3 -m pip install --no-compile --no-dependencies --no-binary :all: \ +# note: --no-build-isolation is needed so that pip uses the locally available setuptools and wheel, +# instead of downloading the latest ones +python3 -m pip install --no-build-isolation --no-compile --no-dependencies --no-binary :all: \ -r "$CONTRIB"/deterministic-build/requirements.txt -t "$CONTRIB"/../packages - diff --git a/contrib/requirements/requirements-build-makepackages.txt b/contrib/requirements/requirements-build-makepackages.txt new file mode 100644 index 000000000..ed3876c5c --- /dev/null +++ b/contrib/requirements/requirements-build-makepackages.txt @@ -0,0 +1,14 @@ +# This file contains build-time dependencies needed to be able to build our pure python dependencies. +# For reproducibility, some build-time deps, most notably "wheel", need to be pinned. (see #7640) +# By default, when doing e.g. "pip install", pip downloads the latest version of wheel (and setuptools, etc), +# regardless whether a sufficiently recent version of wheel is already installed locally... +# The only way I have found to avoid this, is to use the "--no-build-isolation" flag, +# in which case it becomes our responsibility to install *all* build time deps... + +pip +setuptools +wheel + +# importlib_metadata also needs: +# https://github.com/python/importlib_metadata/blob/1e2381fe101fd70742a0171e51c1be82aedf519b/pyproject.toml#L2 +setuptools_scm[toml]>=3.4.1 diff --git a/contrib/requirements/requirements-build-sdist.txt b/contrib/requirements/requirements-build-sdist.txt deleted file mode 100644 index 347c999b6..000000000 --- a/contrib/requirements/requirements-build-sdist.txt +++ /dev/null @@ -1,4 +0,0 @@ -# need modern versions of pip (and maybe other build tools), the one in apt had issues -pip -setuptools -wheel