Merge remote-tracking branch 'remotes/spesmilo/appimage-debian-base'

Change Docker base images from Ubuntu to Debian, and use `snapshot.debian.org` as apt source list. Ubuntu occasionally removes version-pinned packages from apt (see #7484), which - breaks historical reproducible builds - introduces maintenance burden as we have to update the version pins Hopefully this change fixes both issues. merges https://github.com/spesmilo/electrum/pull/7926 closes https://github.com/spesmilo/electrum/issues/7484
2 years ago · 4f574afe5a
12 changed files with 183 additions and 78 deletions
--- a/.cirrus.yml
+++ b/.cirrus.yml
@ -168,6 +168,7 @@ task:
    path: "contrib/build-wine/dist/*"
  env:
    CIRRUS_WORKING_DIR: /opt/wine64/drive_c/electrum
+    CIRRUS_DOCKER_CONTEXT: contrib/build-wine

 task:
  name: Android build
@ -206,6 +207,8 @@ task:
    - ./contrib/build-linux/appimage/make_appimage.sh
  binaries_artifacts:
    path: "dist/*"
+  env:
+    CIRRUS_DOCKER_CONTEXT: contrib/build-linux/appimage

 task:
  container:
--- a/contrib/android/Dockerfile
+++ b/contrib/android/Dockerfile
@ -1,11 +1,19 @@
 # based on https://github.com/kivy/python-for-android/blob/master/Dockerfile

-FROM ubuntu:20.04@sha256:86ac87f73641c920fb42cc9612d4fb57b5626b56ea2a19b894d0673fd5b4f2e9
+FROM debian:bullseye@sha256:82bab30ed448b8e2509aabe21f40f0607d905b7fd0dec72802627a20274eba55

 ENV DEBIAN_FRONTEND=noninteractive

 ENV ANDROID_HOME="/opt/android"

+# need ca-certificates before using snapshot packages
+RUN apt update -qq > /dev/null && apt install -qq --yes --no-install-recommends \
+    ca-certificates
+
+# pin the distro packages.
+COPY contrib/android/apt.sources.list /etc/apt/sources.list
+COPY contrib/android/apt.preferences /etc/apt/preferences.d/snapshot
+
 # configure locale
 RUN apt update -qq > /dev/null && apt install -qq --yes --no-install-recommends \
    locales && \
@ -96,7 +104,6 @@ RUN curl --location --progress-bar \

 # install system/build dependencies
 # https://github.com/kivy/buildozer/blob/master/docs/source/installation.rst#android-on-ubuntu-2004-64bit
-# TODO probably need to pin versions of at least some of these for over-time reproducibility?
 RUN apt -y update -qq \
    && apt -y install -qq --no-install-recommends \
        python3 \
--- a/contrib/android/apt.preferences
+++ b/contrib/android/apt.preferences
@ -0,0 +1,3 @@
+Package: *
+Pin: origin "snapshot.debian.org"
+Pin-Priority: 1001
--- a/contrib/android/apt.sources.list
+++ b/contrib/android/apt.sources.list
@ -0,0 +1,2 @@
+deb https://snapshot.debian.org/archive/debian/20220811T031049Z/ bullseye main non-free contrib
+deb-src https://snapshot.debian.org/archive/debian/20220811T031049Z/ bullseye main non-free contrib
--- a/contrib/build-linux/appimage/Dockerfile
+++ b/contrib/build-linux/appimage/Dockerfile
@ -1,57 +1,66 @@
-# Note: we deliberately use an old Ubuntu LTS as base image.
+# Note: we deliberately use an old Debian stable as base image.
 # from https://docs.appimage.org/introduction/concepts.html :
 # "[AppImages] should be built on the oldest possible system, allowing them to run on newer system[s]"
-FROM ubuntu:18.04@sha256:9bc830af2bef73276515a29aa896eedfa7bdf4bdbc5c1063b4c457a4bbb8cd79
+
+FROM debian:buster@sha256:fb9654aac57319592f1d51497c62001e7033eddf059355408a0b53f7c71f8d5f

 ENV LC_ALL=C.UTF-8 LANG=C.UTF-8
 ENV DEBIAN_FRONTEND=noninteractive

+# need ca-certificates before using snapshot packages
+RUN apt update -qq > /dev/null && apt install -qq --yes --no-install-recommends \
+    ca-certificates
+
+# pin the distro packages
+COPY apt.sources.list /etc/apt/sources.list
+COPY apt.preferences /etc/apt/preferences.d/snapshot
+
 RUN apt-get update -q && \
-    apt-get install -qy \
-        git=1:2.17.1-1ubuntu0.12 \
-        wget=1.19.4-1ubuntu2.2 \
-        make=4.1-9.1ubuntu1 \
-        autotools-dev=20180224.1 \
-        autoconf=2.69-11 \
-        libtool=2.4.6-2 \
-        autopoint=0.19.8.1-6ubuntu0.3 \
-        xz-utils=5.2.2-1.3 \
-        libssl-dev=1.1.1-1ubuntu2.1~18.04.20 \
-        libssl1.1=1.1.1-1ubuntu2.1~18.04.20 \
-        openssl=1.1.1-1ubuntu2.1~18.04.20 \
-        zlib1g-dev=1:1.2.11.dfsg-0ubuntu2 \
-        libffi-dev=3.2.1-8 \
-        libncurses5-dev=6.1-1ubuntu1.18.04 \
-        libncurses5=6.1-1ubuntu1.18.04 \
-        libtinfo-dev=6.1-1ubuntu1.18.04 \
-        libtinfo5=6.1-1ubuntu1.18.04 \
-        libsqlite3-dev=3.22.0-1ubuntu0.5 \
-        libusb-1.0-0-dev=2:1.0.21-2 \
-        libudev-dev=237-3ubuntu10.53 \
-        libudev1=237-3ubuntu10.53 \
-        gettext=0.19.8.1-6ubuntu0.3 \
-        libzbar0=0.10+doc-10.1build2  \
-        libdbus-1-3=1.12.2-1ubuntu1.3 \
-        libxkbcommon0=0.8.0-1ubuntu0.1 \
-        libxkbcommon-x11-0=0.8.0-1ubuntu0.1 \
-        libxcb1=1.13-1 \
-        libxcb-xinerama0=1.13-1 \
-        libxcb-randr0=1.13-1 \
-        libxcb-render0=1.13-1 \
-        libxcb-shm0=1.13-1 \
-        libxcb-shape0=1.13-1 \
-        libxcb-sync1=1.13-1 \
-        libxcb-xfixes0=1.13-1 \
-        libxcb-xkb1=1.13-1 \
-        libxcb-icccm4=0.4.1-1ubuntu1 \
-        libxcb-image0=0.4.0-1build1 \
-        libxcb-keysyms1=0.4.0-1 \
-        libxcb-util1=0.4.0-0ubuntu3 \
-        libxcb-render-util0=0.3.9-1 \
-        libx11-xcb1=2:1.6.4-3ubuntu0.4 \
-        libc6-dev=2.27-3ubuntu1.5 \
-        libc6=2.27-3ubuntu1.5 \
-        libc-dev-bin=2.27-3ubuntu1.5 \
+    apt-get install -qy --allow-downgrades \
+        git \
+        wget \
+        make \
+        autotools-dev \
+        autoconf \
+        libtool \
+        autopoint \
+        xz-utils \
+        libssl-dev \
+        libssl1.1 \
+        openssl \
+        zlib1g-dev \
+        libffi-dev \
+        libncurses5-dev \
+        libncurses5 \
+        libtinfo-dev \
+        libtinfo5 \
+        libsqlite3-dev \
+        libusb-1.0-0-dev \
+        libudev-dev \
+        libudev1 \
+        gettext \
+        libzbar0 \
+        libdbus-1-3 \
+        libxkbcommon0 \
+        libxkbcommon-x11-0 \
+        libxcb1 \
+        libxcb-xinerama0 \
+        libxcb-randr0 \
+        libxcb-render0 \
+        libxcb-shm0 \
+        libxcb-shape0 \
+        libxcb-sync1 \
+        libxcb-xfixes0 \
+        libxcb-xkb1 \
+        libxcb-icccm4 \
+        libxcb-image0 \
+        libxcb-keysyms1 \
+        libxcb-util0 \
+        libxcb-render-util0 \
+        libx11-xcb1 \
+        libc6-dev \
+        libc6 \
+        libc-dev-bin \
        && \
    rm -rf /var/lib/apt/lists/* && \
    apt-get autoremove -y && \
--- a/contrib/build-linux/appimage/apt.preferences
+++ b/contrib/build-linux/appimage/apt.preferences
@ -0,0 +1,3 @@
+Package: *
+Pin: origin "snapshot.debian.org"
+Pin-Priority: 1001
--- a/contrib/build-linux/appimage/apt.sources.list
+++ b/contrib/build-linux/appimage/apt.sources.list
@ -0,0 +1,2 @@
+deb https://snapshot.debian.org/archive/debian/20220811T031049Z/ buster main non-free contrib
+deb-src https://snapshot.debian.org/archive/debian/20220811T031049Z/ buster main non-free contrib
--- a/contrib/build-wine/Dockerfile
+++ b/contrib/build-wine/Dockerfile
@ -1,4 +1,12 @@
-FROM ubuntu:20.04@sha256:c95a8e48bf88e9849f3e0f723d9f49fa12c5a00cfc6e60d2bc99d87555295e4c
+FROM debian:bullseye@sha256:82bab30ed448b8e2509aabe21f40f0607d905b7fd0dec72802627a20274eba55
+
+# need ca-certificates before using snapshot packages
+RUN apt update -qq > /dev/null && apt install -qq --yes --no-install-recommends \
+    ca-certificates
+
+# pin the distro packages.
+COPY apt.sources.list /etc/apt/sources.list
+COPY apt.preferences /etc/apt/preferences.d/snapshot

 ENV LC_ALL=C.UTF-8 LANG=C.UTF-8
 ENV DEBIAN_FRONTEND=noninteractive
@ -6,29 +14,22 @@ ENV DEBIAN_FRONTEND=noninteractive
 RUN dpkg --add-architecture i386 && \
    apt-get update -q && \
    apt-get install -qy \
-        wget=1.20.3-1ubuntu1 \
-        gnupg2=2.2.19-3ubuntu2.2 \
-        dirmngr=2.2.19-3ubuntu2.2 \
-        python3-software-properties=0.98.9.2 \
-        software-properties-common=0.98.9.2 \
-        && \
-    rm -rf /var/lib/apt/lists/* && \
-    apt-get autoremove -y && \
-    apt-get clean
-
-RUN apt-get update -q && \
-    apt-get install -qy \
-        git=1:2.25.1-1ubuntu3 \
-        p7zip-full=16.02+dfsg-7build1 \
-        make=4.2.1-1.2 \
-        mingw-w64=7.0.0-2 \
-        mingw-w64-tools=7.0.0-2 \
-        win-iconv-mingw-w64-dev=0.0.8-3 \
-        autotools-dev=20180224.1 \
-        autoconf=2.69-11.1 \
-        autopoint=0.19.8.1-10build1 \
-        libtool=2.4.6-14 \
-        gettext=0.19.8.1-10build1 \
+        wget \
+        gnupg2 \
+        dirmngr \
+        python3-software-properties \
+        software-properties-common \
+        git \
+        p7zip-full \
+        make \
+        mingw-w64 \
+        mingw-w64-tools \
+        autotools-dev \
+        autoconf \
+        autopoint \
+        libtool \
+        gettext \
+        sudo \
        && \
    rm -rf /var/lib/apt/lists/* && \
    apt-get autoremove -y && \
@ -42,13 +43,15 @@ RUN wget -nc https://dl.winehq.org/wine-builds/Release.key && \
        echo "78b185fabdb323971d13bd329fefc8038e08559aa51c4996de18db0639a51df6 winehq.key" | sha256sum -c - && \
        apt-key add winehq.key && \
        rm winehq.key && \
-    apt-add-repository https://dl.winehq.org/wine-builds/ubuntu/ && \
+    apt-add-repository https://dl.winehq.org/wine-builds/debian/ && \
    apt-get update -q && \
    apt-get install -qy \
-        wine-stable-amd64:amd64=7.0.0.0~focal-1 \
-        wine-stable-i386:i386=7.0.0.0~focal-1 \
-        wine-stable:amd64=7.0.0.0~focal-1 \
-        winehq-stable:amd64=7.0.0.0~focal-1 \
+        wine-stable-amd64:amd64=7.0.0.0~bullseye-1 \
+        wine-stable-i386:i386=7.0.0.0~bullseye-1 \
+        wine-stable:amd64=7.0.0.0~bullseye-1 \
+        winehq-stable:amd64=7.0.0.0~bullseye-1 \
+        libvkd3d1:amd64=1.2~bullseye-1 \
+        libvkd3d1:i386=1.2~bullseye-1 \
        && \
    rm -rf /var/lib/apt/lists/* && \
    apt-get autoremove -y && \
--- a/contrib/build-wine/apt.preferences
+++ b/contrib/build-wine/apt.preferences
@ -0,0 +1,3 @@
+Package: *
+Pin: origin "snapshot.debian.org"
+Pin-Priority: 1001
--- a/contrib/build-wine/apt.sources.list
+++ b/contrib/build-wine/apt.sources.list
@ -0,0 +1,2 @@
+deb https://snapshot.debian.org/archive/debian/20220811T031049Z/ bullseye main non-free contrib
+deb-src https://snapshot.debian.org/archive/debian/20220811T031049Z/ bullseye main non-free contrib
--- a/contrib/build-wine/make_win.sh
+++ b/contrib/build-wine/make_win.sh
@ -50,6 +50,28 @@ fi
 if [ -f "$DLL_TARGET_DIR/libzbar-0.dll" ]; then
    info "libzbar already built, skipping"
 else
+    (
+        # As debian bullseye doesn't provide win-iconv-mingw-w64-dev, we need to build it:
+        WIN_ICONV_COMMIT="c9df88a284d448da5434c6ad2737b54a907f888c"
+        # ^ tag "v0.0.8"
+        info "Building win-iconv..."
+        cd "$CACHEDIR"
+        if [ ! -d win-iconv ]; then
+            git clone https://github.com/win-iconv/win-iconv.git
+        fi
+        cd win-iconv
+        if ! $(git cat-file -e ${WIN_ICONV_COMMIT}) ; then
+            info "Could not find requested version $WIN_ICONV_COMMIT in local clone; fetching..."
+            git fetch --all
+        fi
+        git reset --hard
+        git clean -dfxq
+        git checkout "${WIN_ICONV_COMMIT}^{commit}"
+
+        CC="${GCC_TRIPLET_HOST}-gcc" make -j4 || fail "Could not build win-iconv"
+        # FIXME avoid using sudo
+        sudo make install prefix="/usr/${GCC_TRIPLET_HOST}"  || fail "Could not install win-iconv"
+    )
    "$CONTRIB"/make_zbar.sh || fail "Could not build zbar"
 fi

--- a/contrib/freeze_containers_distro.sh
+++ b/contrib/freeze_containers_distro.sh
@ -0,0 +1,46 @@
+#!/bin/sh
+
+# Run this after a new release to update pin for build container distro packages
+
+set -e
+
+DEBIAN_SNAPSHOT_BASE="https://snapshot.debian.org/archive/debian/"
+DEBIAN_APPIMAGE_DISTRO="buster" # should match build-linux/appimage Dockerfile base
+DEBIAN_WINE_DISTRO="bullseye" # should match build-wine Dockerfile base
+DEBIAN_ANDROID_DISTRO="bullseye" # should match android Dockerfile base
+
+contrib=$(dirname "$0")
+
+
+if [ ! -x /bin/wget ]; then
+  echo "no wget"
+  exit 1
+fi
+
+DEBIAN_SNAPSHOT_LATEST=$(wget -O- ${DEBIAN_SNAPSHOT_BASE}$(date +"?year=%Y&month=%m") 2>/dev/null|grep "^<a href=\"20"|tail -1|sed -e 's#[^"]*"\(.\{17,17\}\).*#\1#')
+
+if [ "${DEBIAN_SNAPSHOT_LATEST}x" = "x" ]; then
+  echo "could not find timestamp for debian packages"
+  exit 1
+fi
+
+DEBIAN_SNAPSHOT=${DEBIAN_SNAPSHOT_BASE}${DEBIAN_SNAPSHOT_LATEST}
+
+echo "Checking if URL valid.."
+wget -O /dev/null ${DEBIAN_SNAPSHOT} 2>/dev/null
+
+echo "Valid!"
+
+# build-linux
+echo "deb ${DEBIAN_SNAPSHOT} ${DEBIAN_APPIMAGE_DISTRO} main non-free contrib" >$contrib/build-linux/appimage/apt.sources.list
+echo "deb-src ${DEBIAN_SNAPSHOT} ${DEBIAN_APPIMAGE_DISTRO} main non-free contrib" >>$contrib/build-linux/appimage/apt.sources.list
+
+# build-wine
+echo "deb ${DEBIAN_SNAPSHOT} ${DEBIAN_WINE_DISTRO} main non-free contrib" >$contrib/build-wine/apt.sources.list
+echo "deb-src ${DEBIAN_SNAPSHOT} ${DEBIAN_WINE_DISTRO} main non-free contrib" >>$contrib/build-wine/apt.sources.list
+
+# android
+echo "deb ${DEBIAN_SNAPSHOT} ${DEBIAN_ANDROID_DISTRO} main non-free contrib" >$contrib/android/apt.sources.list
+echo "deb-src ${DEBIAN_SNAPSHOT} ${DEBIAN_ANDROID_DISTRO} main non-free contrib" >>$contrib/android/apt.sources.list
+
+echo "updated APT sources to ${DEBIAN_SNAPSHOT}"