Switch mender-client download to APT repository

Add a deb module with reverse engineered logic to download a given deb
package from an APT repository, and modify mender-client download to use
this method instead of the legacy wget direct download.

Changelog: None

Signed-off-by: Lluis Campos <lluis.campos@northern.tech>

Dockerfile

@@ -33,7 +33,7 @@ RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install -y \
     sudo \
 # mkfs.vfat (required for boot partition)
     dosfstools \
-# to download mender-artifact
+# to download Mender binaries
     wget \
 # to download mender-grub-env
     git \

configs/mender_convert_config

@@ -111,12 +111,18 @@ MENDER_PARTITION_ALIGNMENT="8388608"
 # Mender client version
 #
 # This is used to fetch the correct binaries
+#
+# Valid values are "latest" (default), "master" or a specific version
 MENDER_CLIENT_VERSION="latest"
 
 # File storage, containing binary files, do not modify this unless you know
 # what you are doing.
 MENDER_STORAGE_URL="https://downloads.mender.io"
 
+# Mender APT repo, containing binary files, do not modify this unless you know
+# what you are doing.
+MENDER_APT_REPO_URL="${MENDER_STORAGE_URL}/repos/debian"
+
 # Mender GitHub organization URL prefix
 MENDER_GITHUB_ORG="https://github.com/mendersoftware"
 

mender-convert-modify

@@ -49,6 +49,7 @@ echo "Running $(basename $0): $@"
 source modules/bootstrap.sh
 source modules/disk.sh
 source modules/probe.sh
+source modules/deb.sh
 
 # The mender_convert_config is always used and provides all the defaults
 declare -a configs=("configs/mender_convert_config")
@@ -95,24 +96,27 @@ mkdir -p work/rootfs
 sudo mount ${boot_part} work/boot
 sudo mount ${root_part} work/rootfs
 
-mkdir -p work/mender-deb/files
+mkdir -p work/mender-client-deb/files
 
 log_info "Installing Mender client and related files"
 
 deb_arch=$(probe_debian_arch_name)
-DEBIAN_REVISION="-1"
+
 if [ "${MENDER_CLIENT_VERSION}" = "latest" ]; then
-  DEBIAN_REVISION=""
+  deb_name=$(deb_from_repo_get "work/mender-client-deb" ${MENDER_APT_REPO_URL} ${deb_arch} "stable" "mender-client" "latest")
+elif [ "${MENDER_CLIENT_VERSION}" = "master" ]; then
+  deb_name=$(deb_from_repo_get "work/mender-client-deb" ${MENDER_APT_REPO_URL} ${deb_arch} "experimental" "mender-client" "latest")
+else
+  DEBIAN_REVISION="-1"
+  deb_name=$(deb_from_repo_get "work/mender-client-deb" ${MENDER_APT_REPO_URL} ${deb_arch} "experimental" "mender-client" "${MENDER_CLIENT_VERSION}${DEBIAN_REVISION}")
 fi
-deb_name="mender-client_${MENDER_CLIENT_VERSION}${DEBIAN_REVISION}_${deb_arch}.deb"
-run_and_log_cmd "wget -Nq ${MENDER_STORAGE_URL}/${MENDER_CLIENT_VERSION}/dist-packages/debian/${deb_arch}/${deb_name} -P work/mender-deb"
 
-cd work/mender-deb
+cd work/mender-client-deb
 run_and_log_cmd "ar -xv ${deb_name}"
 run_and_log_cmd "sudo tar xJf data.tar.xz -C files"
 cd - > /dev/null 2>&1
 
-run_and_log_cmd "sudo rsync --archive --keep-dirlinks --verbose work/mender-deb/files/ work/rootfs/"
+run_and_log_cmd "sudo rsync --archive --keep-dirlinks --verbose work/mender-client-deb/files/ work/rootfs/"
 
 if [ "${MENDER_ENABLE_SYSTEMD}" == "y" ]; then
   run_and_log_cmd "sudo ln -sf /lib/systemd/system/mender-client.service \

modules/deb.sh

@@ -0,0 +1,64 @@
+#
+# Copyright 2020 Northern.tech AS
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+source modules/log.sh
+
+# Direct download of a deb package from an APT repository
+#
+#  $1 - Download directory
+#  $2 - APT repository url
+#  $3 - Debian architecture
+#  $4 - Debian Distribution
+#  $5 - Package name
+#  $6 - Package version (optional, default "latest")
+#  $7 - Component (optional, default "main")
+#
+# @return - Filename of the downloaded package
+#
+function deb_from_repo_get () {
+  if [[ $# -lt 5 || $# -gt 7 ]]; then
+    log_fatal "deb_from_repo_get() requires 5 or 6 arguments"
+  fi
+  local -r download_dir="${1}"
+  local -r repo_url="${2}"
+  local -r architecture="${3}"
+  local -r distribution="${4}"
+  local -r package="${5}"
+  local -r version="${6:-latest}"
+  local -r component="${7:-main}"
+
+  local deb_package_path=""
+  if [ "${version}" = "latest" ]; then
+    # If latest, fetch and parse the packages list of the given distribution to find the latest version
+    local -r packages_url="${repo_url}/dists/${distribution}/${component}/binary-${architecture}/Packages"
+    run_and_log_cmd "wget -Nq ${packages_url} -P /tmp"
+
+    deb_package_path=$(grep Filename /tmp/Packages | grep ${package}_ | grep ${architecture} | tail -n1 | sed 's/Filename: //')
+    if [ -z "${deb_package_path}" ]; then
+      log_fatal "Couldn't fine package ${package} in ${packages_url}"
+    fi
+  else
+    # Else, try to download the specified version (ignoring distribution)
+    local -l initial="$(echo $package | head -c 1)"
+    deb_package_path="pool/${component}/${initial}/${package}/${package}_${version}_${architecture}.deb"
+  fi
+
+  local -r filename=$(basename $deb_package_path)
+  run_and_log_cmd "wget -Nq ${repo_url}/${deb_package_path} -P ${download_dir}"
+
+  rm -f /tmp/Packages
+  log_info "Successfully downloaded ${filename}"
+  echo ${filename}
+}