Merge pull request #425 from mendersoftware/cherry-2.5.x-MEN-5410-debian-and-ubuntu-repos

[Cherry 2.5.x]: MEN-5410: Modify Debian packages installation for new repositories

LICENSE

@@ -1,4 +1,4 @@
-Copyright 2021 Northern.tech AS
+Copyright 2022 Northern.tech AS
 
 All content in this project is licensed under the Apache License v2, unless
 indicated otherwise.

LIC_FILES_CHKSUM.sha256

@@ -1 +1 @@
-b4acfcfa2a0ba1a8c82ec3965fbcee886cff8394ca4214e0ddac0a36beb1e05a  LICENSE
+1033348db7606a7e61b6484f293847cf8d7a35766efebb97e304d4bd5d7f3f6b  LICENSE

configs/mender_convert_config

@@ -146,10 +146,14 @@ MENDER_ADDON_CONFIGURE_VERSION="latest"
 # what you are doing.
 MENDER_STORAGE_URL="https://downloads.mender.io"
 
-# Mender APT repo, containing binary files, do not modify this unless you know
+# Mender APT repo url, containing binary files, do not modify this unless you know
 # what you are doing.
 MENDER_APT_REPO_URL="${MENDER_STORAGE_URL}/repos/debian"
 
+# Mender APT repo available distributions, do not modify this unless you know
+# what you are doing.
+MENDER_APT_REPO_DISTS="debian/buster debian/bullseye ubuntu/bionic ubuntu/focal"
+
 # Mender GitHub organization URL prefix
 MENDER_GITHUB_ORG="https://github.com/mendersoftware"
 

mender-convert-modify

@@ -97,30 +97,14 @@ mkdir -p work/rootfs
 sudo mount ${boot_part} work/boot
 sudo mount ${root_part} work/rootfs
 
-mkdir -p work/deb-packages
-
 log_info "Installing Mender client and related files"
 
-deb_arch=$(probe_debian_arch_name)
-
 if [ "${MENDER_CLIENT_INSTALL}" = "y" ]; then
-
     log_info "Installing Mender client version ${MENDER_CLIENT_VERSION}"
-
+    deb_get_and_install_pacakge mender-client "${MENDER_CLIENT_VERSION}"
-    if [ "${MENDER_CLIENT_VERSION}" = "latest" ]; then
-        deb_name=$(deb_from_repo_dist_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "stable" "mender-client")
-    elif [ "${MENDER_CLIENT_VERSION}" = "master" ]; then
-        deb_name=$(deb_from_repo_dist_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "experimental" "mender-client")
-    else
-        DEBIAN_REVISION="-1"
-        deb_name=$(deb_from_repo_pool_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "mender-client" "${MENDER_CLIENT_VERSION}${DEBIAN_REVISION}")
-    fi
-
-    deb_extract_package "work/deb-packages/${deb_name}" "work/rootfs/"
 
     # Save installed client version for tests in Yocto variable format
-    testscfg_add "PREFERRED_VERSION_mender-client" "$(echo ${deb_name} | sed -r 's/.*_([0-9]+\.[0-9]+\.[0-9]+).*/\1/')"
+    testscfg_add "PREFERRED_VERSION_mender-client" "$(echo ${DEB_NAME} | sed -r 's/.*_([0-9]+\.[0-9]+\.[0-9]+).*/\1/')"
-
 fi
 
 if [ "${MENDER_ENABLE_SYSTEMD}" == "y" ]; then
@@ -129,38 +113,16 @@ if [ "${MENDER_ENABLE_SYSTEMD}" == "y" ]; then
 fi
 
 if [ "${MENDER_ADDON_CONNECT_INSTALL}" = "y" ]; then
-
     log_info "Installing Mender Connect addon"
-
+    deb_get_and_install_pacakge mender-connect "${MENDER_ADDON_CONNECT_VERSION}"
-    if [ "${MENDER_ADDON_CONNECT_VERSION}" = "latest" ]; then
-        deb_name=$(deb_from_repo_dist_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "stable" "mender-connect")
-    elif [ "${MENDER_ADDON_CONNECT_VERSION}" = "master" ]; then
-        deb_name=$(deb_from_repo_dist_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "experimental" "mender-connect")
-    else
-        DEBIAN_REVISION="-1"
-        deb_name=$(deb_from_repo_pool_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "mender-connect" "${MENDER_ADDON_CONNECT_VERSION}${DEBIAN_REVISION}")
-    fi
-
-    deb_extract_package "work/deb-packages/${deb_name}" "work/rootfs/"
 
     run_and_log_cmd "sudo ln -sf /lib/systemd/system/mender-connect.service \
         work/rootfs/etc/systemd/system/multi-user.target.wants/mender-connect.service"
 fi
 
 if [ "${MENDER_ADDON_CONFIGURE_INSTALL}" = "y" ]; then
-
     log_info "Installing Mender Configure addon"
-
+    deb_get_and_install_pacakge mender-configure "${MENDER_ADDON_CONFIGURE_VERSION}" "true"
-    if [ "${MENDER_ADDON_CONFIGURE_VERSION}" = "latest" ]; then
-        deb_name=$(deb_from_repo_dist_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "stable" "mender-configure")
-    elif [ "${MENDER_ADDON_CONFIGURE_VERSION}" = "master" ]; then
-        deb_name=$(deb_from_repo_dist_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "experimental" "mender-configure")
-    else
-        DEBIAN_REVISION="-1"
-        deb_name=$(deb_from_repo_pool_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "mender-configure" "${MENDER_ADDON_CONFIGURE_VERSION}${DEBIAN_REVISION}")
-    fi
-
-    deb_extract_package "work/deb-packages/${deb_name}" "work/rootfs/"
 fi
 
 # Do this unconditionally even if not installing mender-configure. The reason is

modules/deb.sh

@@ -1,5 +1,5 @@
 #
-# Copyright 2020 Northern.tech AS
+# Copyright 2022 Northern.tech AS
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
@@ -14,6 +14,7 @@
 #  limitations under the License.
 
 source modules/log.sh
+source modules/probe.sh .
 
 # Download of latest deb package for the given distribution of an APT repository
 #
@@ -81,11 +82,18 @@ function deb_from_repo_pool_get()  {
     local -r deb_package_path="pool/${component}/${initial}/${package}/${package}_${version}_${architecture}.deb"
 
     local -r filename=$(basename $deb_package_path)
-    run_and_log_cmd "wget -Nq ${repo_url}/${deb_package_path} -P ${download_dir}"
+    local -r deb_package_url=$(echo ${repo_url}/${deb_package_path} | sed 's/+/%2B/g')
+    run_and_log_cmd_noexit "wget -Nq ${deb_package_url} -P ${download_dir}"
+    local exit_code=$?
 
     rm -f /tmp/Packages
-    log_info "Successfully downloaded ${filename}"
+    if [[ ${exit_code} -ne 0 ]]; then
-    echo ${filename}
+        log_warn "Could not download ${filename}"
+        echo ""
+    else
+        log_info "Successfully downloaded ${filename}"
+        echo ${filename}
+    fi
 }
 
 # Extract the binary files of a deb package into a directory
@@ -111,3 +119,56 @@ function deb_extract_package()  {
 
     log_info "Successfully installed $(basename ${deb_package}) into ${dest_dir}"
 }
+
+# Download and install the binary files of a deb package into work/deb-packages
+# This is the main entry point of deb.sh
+# Defines variable DEB_NAME with the actual filename installed
+#
+#  $1 - Package name
+#  $2 - Package version
+#  $3 - Arch independent (optional, default "false")
+#
+function deb_get_and_install_pacakge() {
+    if ! [[ $# -eq 2 || $# -eq 3 ]]; then
+        log_fatal "deb_get_and_install_pacakge() requires 2 or 3 arguments"
+    fi
+    local package="$1"
+    local version="$2"
+    local arch_indep="${3:-false}"
+
+    mkdir -p work/deb-packages
+
+    local deb_arch=$(probe_debian_arch_name)
+    local deb_distro=$(probe_debian_distro_name)
+    local deb_codename=$(probe_debian_distro_codename)
+    if ! [[ "$MENDER_APT_REPO_DISTS" == *"${deb_distro}/${deb_codename}"* ]]; then
+        deb_distro="debian"
+        deb_codename="buster"
+    fi
+
+    DEB_NAME=""
+    if [ "${version}" = "latest" ]; then
+        DEB_NAME=$(deb_from_repo_dist_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "${deb_distro}/${deb_codename}/stable" "${package}")
+    elif [ "${version}" = "master" ]; then
+        DEB_NAME=$(deb_from_repo_dist_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${deb_arch} "${deb_distro}/${deb_codename}/experimental" "${package}")
+    else
+        # On direct downloads, the architecture suffix will be "all" for arch independent packages
+        local pool_arch=""
+        if [[ "$arch_indep" == "true" ]]; then
+            pool_arch="all"
+        else
+            pool_arch="$deb_arch"
+        fi
+
+        local debian_version="-1+${deb_distro}+${deb_codename}"
+        DEB_NAME=$(deb_from_repo_pool_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${pool_arch} "${package}" "${version}${debian_version}")
+        if [[ -z "${DEB_NAME}" ]]; then
+            local debian_version_fallback="-1"
+            DEB_NAME=$(deb_from_repo_pool_get "work/deb-packages" ${MENDER_APT_REPO_URL} ${pool_arch} "${package}" "${version}${debian_version_fallback}")
+            if [[ -z "${DEB_NAME}" ]]; then
+                log_fatal "Specified version for ${package} cannot be found, tried ${version}${debian_version} and ${version}${debian_version_fallback}"
+            fi
+        fi
+    fi
+    deb_extract_package "work/deb-packages/${DEB_NAME}" "work/rootfs/"
+}

modules/probe.sh

@@ -97,6 +97,29 @@ probe_debian_arch_name() {
     echo "${deb_arch}"
 }
 
+# Prints Debian distro name based on ID from /etc/os-release
+#
+# Special handling for raspbian, where ID_LIKE is used instead
+#
+# No input parameters and these work on the assumption that boot and root parts
+# are mounted at work/boot and work/rootfs
+probe_debian_distro_name() {
+    distro_name="$(. work/rootfs/etc/os-release && echo "$ID")"
+    if [[ "$distro_name" == "raspbian" ]]; then
+        distro_name="$(. work/rootfs/etc/os-release && echo "$ID_LIKE")"
+    fi
+    echo "${distro_name}"
+}
+
+# Prints Debian distro codename based on VERSION_CODENAME from /etc/os-release
+#
+# No input parameters and these work on the assumption that boot and root parts
+# are mounted at work/boot and work/rootfs
+probe_debian_distro_codename() {
+    distro_codename="$(. work/rootfs/etc/os-release && echo "$VERSION_CODENAME")"
+    echo "${distro_codename}"
+}
+
 # Prints GRUB EFI target name depending on target architecture
 #
 # This is what the file name should be when put on target boot part.

modules/run.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 #
-# Copyright 2019 Northern.tech AS
+# Copyright 2022 Northern.tech AS
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
@@ -14,7 +14,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-# Run a command, capture output and log it
+# Run a command, capture and log output, and exit on non-zero return code
 #
 #  $1 - command to run
 function run_and_log_cmd() {
@@ -31,3 +31,19 @@ function run_and_log_cmd() {
         exit ${exit_code}
     fi
 }
+
+# Run a command, capture and log output, and return the command's return code
+#
+#  $1 - command to run
+function run_and_log_cmd_noexit() {
+    local -r cmd="${1}"
+    local -r position="(${BASH_SOURCE[1]}:${BASH_LINENO[0]}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }"
+    local exit_code=0
+    output="$({ eval ${cmd}; } 2>&1)" || exit_code=$?
+    local log_msg="Running: ${position} \n\r\n\r\t${cmd}"
+    if [[ "${output}" != "" ]]; then
+        log_msg="${log_msg}\n\t${output}\n"
+    fi
+    log_debug "${log_msg}"
+    return ${exit_code}
+}