diff --git a/.gitignore b/.gitignore index 5b2da718c..193ffea3a 100644 --- a/.gitignore +++ b/.gitignore @@ -44,6 +44,7 @@ contrib/android/android_debug.keystore contrib/secp256k1/ contrib/zbar/ contrib/libusb/ +contrib/.venv_make_packages/ # shared objects electrum/*.so diff --git a/contrib/android/Dockerfile b/contrib/android/Dockerfile index 9f8fbd855..71ac55172 100644 --- a/contrib/android/Dockerfile +++ b/contrib/android/Dockerfile @@ -108,6 +108,7 @@ RUN apt -y update -qq \ python3-dev \ python3-pip \ python3-setuptools \ + python3-venv \ wget \ lbzip2 \ patch \ diff --git a/contrib/build-linux/sdist/Dockerfile b/contrib/build-linux/sdist/Dockerfile index b7391f379..bf7d03850 100644 --- a/contrib/build-linux/sdist/Dockerfile +++ b/contrib/build-linux/sdist/Dockerfile @@ -10,6 +10,7 @@ RUN apt-get update -q && \ python3 \ python3-pip \ python3-setuptools \ + python3-venv \ faketime \ && \ rm -rf /var/lib/apt/lists/* && \ diff --git a/contrib/build-linux/sdist/make_sdist.sh b/contrib/build-linux/sdist/make_sdist.sh index 7aae3f423..5429ec74e 100755 --- a/contrib/build-linux/sdist/make_sdist.sh +++ b/contrib/build-linux/sdist/make_sdist.sh @@ -16,13 +16,9 @@ python3 --version || fail "python interpreter not found" break_legacy_easy_install # upgrade to modern pip so that it knows the flags we need. -# we will then install a pinned version of pip as part of requirements-build-sdist +# (make_packages will later install a pinned version of pip in a venv) python3 -m pip install --upgrade pip -info "Installing pinned requirements." -python3 -m pip install --no-dependencies --no-warn-script-location -r "$CONTRIB"/deterministic-build/requirements-build-sdist.txt - - "$CONTRIB"/make_packages || fail "make_packages failed" git submodule update --init diff --git a/contrib/deterministic-build/requirements-build-makepackages.txt b/contrib/deterministic-build/requirements-build-makepackages.txt new file mode 100644 index 000000000..147da8d6b --- /dev/null +++ b/contrib/deterministic-build/requirements-build-makepackages.txt @@ -0,0 +1,21 @@ +packaging==21.3 \ + --hash=sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb \ + --hash=sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522 +pip==21.3.1 \ + --hash=sha256:deaf32dcd9ab821e359cd8330786bcd077604b5c5730c0b096eda46f95c24a2d \ + --hash=sha256:fd11ba3d0fdb4c07fbc5ecbba0b1b719809420f25038f8ee3cd913d3faa3033a +pyparsing==3.0.7 \ + --hash=sha256:18ee9022775d270c55187733956460083db60b37d0d0fb357445f3094eed3eea \ + --hash=sha256:a6c06a88f252e6c322f65faf8f418b16213b51bdfaece0524c1c1bc30c63c484 +setuptools==59.6.0 \ + --hash=sha256:22c7348c6d2976a52632c67f7ab0cdf40147db7789f9aed18734643fe9cf3373 \ + --hash=sha256:4ce92f1e1f8f01233ee9952c04f6b81d1e02939d6e1b488428154974a4d0783e +setuptools-scm==6.4.2 \ + --hash=sha256:6833ac65c6ed9711a4d5d2266f8024cfa07c533a0e55f4c12f6eff280a5a9e30 \ + --hash=sha256:acea13255093849de7ccb11af9e1fb8bde7067783450cee9ef7a93139bddf6d4 +tomli==1.2.3 \ + --hash=sha256:05b6166bff487dc068d322585c7ea4ef78deed501cc124060e0f238e89a9231f \ + --hash=sha256:e3069e4be3ead9668e21cb9b074cd948f7b3113fd9c8bba083f48247aab8b11c +wheel==0.37.1 \ + --hash=sha256:4bdcd7d840138086126cd09254dc6195fb4fc6f01c050a1d7236f2630db1d22a \ + --hash=sha256:e9a504e793efbca1b8e0e9cb979a249cf4a0a7b5b8c9e8b65a5e39d49529c1c4 \ No newline at end of file diff --git a/contrib/deterministic-build/requirements-build-sdist.txt b/contrib/deterministic-build/requirements-build-sdist.txt deleted file mode 100644 index f19e9db07..000000000 --- a/contrib/deterministic-build/requirements-build-sdist.txt +++ /dev/null @@ -1,9 +0,0 @@ -pip==21.3.1 \ - --hash=sha256:deaf32dcd9ab821e359cd8330786bcd077604b5c5730c0b096eda46f95c24a2d \ - --hash=sha256:fd11ba3d0fdb4c07fbc5ecbba0b1b719809420f25038f8ee3cd913d3faa3033a -setuptools==59.6.0 \ - --hash=sha256:22c7348c6d2976a52632c67f7ab0cdf40147db7789f9aed18734643fe9cf3373 \ - --hash=sha256:4ce92f1e1f8f01233ee9952c04f6b81d1e02939d6e1b488428154974a4d0783e -wheel==0.37.1 \ - --hash=sha256:4bdcd7d840138086126cd09254dc6195fb4fc6f01c050a1d7236f2630db1d22a \ - --hash=sha256:e9a504e793efbca1b8e0e9cb979a249cf4a0a7b5b8c9e8b65a5e39d49529c1c4 \ No newline at end of file diff --git a/contrib/freeze_packages.sh b/contrib/freeze_packages.sh index c265345f9..3455771fd 100755 --- a/contrib/freeze_packages.sh +++ b/contrib/freeze_packages.sh @@ -20,7 +20,7 @@ which virtualenv > /dev/null 2>&1 || { echo "Please install virtualenv" && exit ${SYSTEM_PYTHON} -m hashin -h > /dev/null 2>&1 || { ${SYSTEM_PYTHON} -m pip install hashin; } -for i in '' '-hw' '-binaries' '-binaries-mac' '-build-wine' '-build-mac' '-build-sdist' '-build-appimage' '-build-android'; do +for i in '' '-hw' '-binaries' '-binaries-mac' '-build-wine' '-build-mac' '-build-makepackages' '-build-appimage' '-build-android'; do rm -rf "$venv_dir" virtualenv -p ${SYSTEM_PYTHON} $venv_dir diff --git a/contrib/make_packages b/contrib/make_packages index 6d29044fe..40e4bf0be 100755 --- a/contrib/make_packages +++ b/contrib/make_packages @@ -13,6 +13,16 @@ if [ -d "$PACKAGES" ]; then rm -r "$PACKAGES" fi +# create virtualenv +# note: venv path needs to be deterministic as some produced files will contain it +venv_dir="$CONTRIB/.venv_make_packages/" +rm -rf "$venv_dir" +python3 -m venv "$venv_dir" +source "$venv_dir"/bin/activate + +# installing pinned build-time requirements, such as pip/wheel/setuptools +python -m pip install --no-dependencies --no-warn-script-location -r "$CONTRIB"/deterministic-build/requirements-build-makepackages.txt + # opt out of compiling C extensions # FIXME aiohttp opt-out is not released yet: https://github.com/aio-libs/aiohttp/pull/3828 export AIOHTTP_NO_EXTENSIONS=1 @@ -34,6 +44,7 @@ export BUILD_TIME="$(LC_ALL=C TZ=UTC date +'%H:%M:%S' -d @$SOURCE_DATE_EPOCH)" # Hence "aiohttp-*.dist-info/" is not reproducible either. # All this means that downstream users of this script, such as the sdist build # and the android apk build need to make sure these files get excluded. -python3 -m pip install --no-compile --no-dependencies --no-binary :all: \ +# note: --no-build-isolation is needed so that pip uses the locally available setuptools and wheel, +# instead of downloading the latest ones +python3 -m pip install --no-build-isolation --no-compile --no-dependencies --no-binary :all: \ -r "$CONTRIB"/deterministic-build/requirements.txt -t "$CONTRIB"/../packages - diff --git a/contrib/requirements/requirements-build-makepackages.txt b/contrib/requirements/requirements-build-makepackages.txt new file mode 100644 index 000000000..ed3876c5c --- /dev/null +++ b/contrib/requirements/requirements-build-makepackages.txt @@ -0,0 +1,14 @@ +# This file contains build-time dependencies needed to be able to build our pure python dependencies. +# For reproducibility, some build-time deps, most notably "wheel", need to be pinned. (see #7640) +# By default, when doing e.g. "pip install", pip downloads the latest version of wheel (and setuptools, etc), +# regardless whether a sufficiently recent version of wheel is already installed locally... +# The only way I have found to avoid this, is to use the "--no-build-isolation" flag, +# in which case it becomes our responsibility to install *all* build time deps... + +pip +setuptools +wheel + +# importlib_metadata also needs: +# https://github.com/python/importlib_metadata/blob/1e2381fe101fd70742a0171e51c1be82aedf519b/pyproject.toml#L2 +setuptools_scm[toml]>=3.4.1 diff --git a/contrib/requirements/requirements-build-sdist.txt b/contrib/requirements/requirements-build-sdist.txt deleted file mode 100644 index 347c999b6..000000000 --- a/contrib/requirements/requirements-build-sdist.txt +++ /dev/null @@ -1,4 +0,0 @@ -# need modern versions of pip (and maybe other build tools), the one in apt had issues -pip -setuptools -wheel