Merge pull request #382 from lluiscampos/update-ubuntu-image

Update Ubuntu image and pipeline fixes

.gitlab-ci.yml

@@ -161,6 +161,9 @@ convert_raspbian_raspberrypi4:
     - curl -f -O https://mender.s3.amazonaws.com/mender-artifact/$MENDER_ARTIFACT_VERSION/linux/mender-artifact
     - chmod ugo+x mender-artifact
     - mv mender-artifact /usr/bin/
+    - echo "MENDER_CLIENT_VERSION=${MENDER_CLIENT_VERSION}" > versions_override_config
+    - echo "MENDER_ADDON_CONNECT_VERSION=${MENDER_ADDON_CONNECT_VERSION}" >> versions_override_config
+    - echo "MENDER_ADDON_CONFIGURE_VERSION=${MENDER_ADDON_CONFIGURE_VERSION}" >> versions_override_config
     # Get Update Modules Artifact generators for the tests
     - if [ "${MENDER_CLIENT_VERSION}" = "latest" ]; then
     -   mender_version=master
@@ -220,22 +223,22 @@ test_acceptance_prebuilt_raspberrypi4:
 test_acceptance_qemux86_64:
   <<: *test_acceptance
   script:
-    - ./scripts/test/run-tests.sh --only qemux86_64
+    - ./scripts/test/run-tests.sh --config versions_override_config --only qemux86_64
 
 test_acceptance_raspberrypi:
   <<: *test_acceptance
   script:
-    - ./scripts/test/run-tests.sh --only raspberrypi3
+    - ./scripts/test/run-tests.sh --config versions_override_config --only raspberrypi3
 
 test_acceptance_beaglebone:
   <<: *test_acceptance
   script:
-    - ./scripts/test/run-tests.sh --only beaglebone
+    - ./scripts/test/run-tests.sh --config versions_override_config --only beaglebone
 
 test_acceptance_ubuntu:
   <<: *test_acceptance
   script:
-    - ./scripts/test/run-tests.sh --only ubuntu
+    - ./scripts/test/run-tests.sh --config versions_override_config --only ubuntu
 
 publish:s3:
   stage: publish

configs/qemux86-64_config

@@ -1,16 +1,19 @@
 # This configuration can be used to run on a qemux86-64 machine.
 #
-# This has been tested on images generated with the following command:
+# The image is generated with the following command:
 #
-#    mkosi -d ubuntu -r bionic -t gpt_ext4 -b --checksum --password password --package=openssh-server,dhcpcd5 -o image.raw
+#    mkosi --root-size=2G --distribution=ubuntu --release=focal --format=gpt_ext4 --bootable --checksum
+#          --password password --package=openssh-server,dhcpcd5 --output=Ubuntu-Focal-x86-64.img build
 #
-# Converted with the following command:
+# Then manually uploaded to Mender's AWS S3 bucket for CI testing
 #
-#    MENDER_ARTIFACT_NAME=release-1 ./mender-convert --disk-image input/image.raw  --overlay rootfs_overlay_demo --config configs/qemux86-64_config
+# Locally, converted with the following command:
+#
+#    MENDER_ARTIFACT_NAME=release-1 ./docker-mender-convert --disk-image input/Ubuntu-Focal-x86-64.img --overlay rootfs_overlay_demo --config configs/qemux86-64_config
 #
 # and qemu is executed with the following command:
 #
-#    qemu-system-x86_64 -enable-kvm -m 512 -smp 2 -bios /usr/share/ovmf/x64/OVMF_CODE.fd -drive format=raw,file=qemux86_64-release-1.sdimg
+#    qemu-system-x86_64 -enable-kvm -m 512 -smp 2 -bios /usr/share/OVMF/OVMF_CODE.fd -drive format=raw,file=deploy/Ubuntu-Focal-x86-64-qemux86_64-mender.img
 
 MENDER_STORAGE_DEVICE_BASE=/dev/sda
 MENDER_DEVICE_TYPE="qemux86_64"

modules/deb.sh

@@ -47,7 +47,8 @@ function deb_from_repo_dist_get()  {
     fi
 
     local -r filename=$(basename $deb_package_path)
-    run_and_log_cmd "wget -Nq ${repo_url}/${deb_package_path} -P ${download_dir}"
+    local -r deb_package_url=$(echo ${repo_url}/${deb_package_path} | sed 's/+/%2B/g')
+    run_and_log_cmd "wget -Nq ${deb_package_url} -P ${download_dir}"
 
     rm -f /tmp/Packages
     log_info "Successfully downloaded ${filename}"

scripts/test/run-tests.sh

@@ -3,7 +3,7 @@
 set -e
 
 usage() {
-  echo "$0 <--all | --only DEVICE_TYPE | --prebuilt-image DEVICE_TYPE IMAGE_NAME>"
+  echo "$0 [--config EXTRA_CONFIG_FILE] <--all | --only DEVICE_TYPE | --prebuilt-image DEVICE_TYPE IMAGE_NAME>"
   exit 1
 }
 
@@ -26,7 +26,7 @@ BBB_DEBIAN_EMMC_IMAGE_URL="https://rcn-ee.com/rootfs/bb.org/testing/2021-09-01/b
 ## Auto-update
 RASPBIAN_IMAGE_URL="http://downloads.raspberrypi.org/raspbian_lite/images/raspbian_lite-2020-02-14/2020-02-13-raspbian-buster-lite.zip"
 
-UBUNTU_IMAGE_URL="https://downloads.mender.io/mender-convert/images/Ubuntu-Bionic-x86-64.img.gz"
+UBUNTU_IMAGE_URL="https://downloads.mender.io/mender-convert/images/Ubuntu-Focal-x86-64.img.gz"
 
 ## Auto-update
 UBUNTU_SERVER_RPI_IMAGE_URL="http://cdimage.ubuntu.com/ubuntu/releases/20.04/release/ubuntu-20.04.3-preinstalled-server-armhf+raspi.img.xz"
@@ -44,54 +44,83 @@ get_pytest_files
 
 prepare_ssh_keys
 
-if ! [ "$1" == "--all" -o "$1" == "--only" -a -n "$2" -o "$1" == "--prebuilt-image" -a -n "$3" ]; then
+usage_if_empty() {
-  usage
+  if [ -z "$1" ]; then
-fi
+    usage
+  fi
+}
+
+PREBUILT_IMAGE=
+TEST_PLATFORM=
+TEST_ALL=0
+EXTRA_CONFIG=
+while [ -n "$1" ]; do
+  case "$1" in
+    --prebuilt-image)
+      usage_if_empty "$3"
+      PREBUILT_IMAGE="$2 $3"
+      ;;
+    --all)
+      TEST_ALL=1
+      ;;
+    --only)
+      usage_if_empty "$2"
+      TEST_PLATFORM="$2"
+      shift
+      ;;
+    --config)
+      usage_if_empty "$2"
+      EXTRA_CONFIG="$EXTRA_CONFIG --config $2"
+      shift
+      ;;
+  esac
+  shift
+done
 
 test_result=0
 
-if [ "$1" == "--prebuilt-image" ]; then
+if [ -n "$PREBUILT_IMAGE" ]; then
-  run_tests "$2" "$3" \
+  run_tests $PREBUILT_IMAGE \
             "-k" "'not test_update'" \
             || test_result=$?
   exit $test_result
 
 else
-  if [ "$1" == "--all" -o "$1" == "--only" -a "$2" == "qemux86_64" ]; then
+  if [ "$TEST_ALL" == "1" -o "$TEST_PLATFORM" == "qemux86_64" ]; then
     wget --progress=dot:giga -N ${UBUNTU_IMAGE_URL} -P input/
     convert_and_test "qemux86_64" \
                      "release-1" \
-                     "input/Ubuntu-Bionic-x86-64.img.gz" \
+                     "input/Ubuntu-Focal-x86-64.img.gz" \
                      "--overlay tests/ssh-public-key-overlay" \
-                     "--config configs/qemux86-64_config" \
+                     "--config configs/qemux86-64_config $EXTRA_CONFIG" \
                      || test_result=$?
 
     echo >&2 "----------------------------------------"
     echo >&2 "Running the uncompressed test"
     echo >&2 "----------------------------------------"
     rm -rf deploy
-    gunzip --force "input/Ubuntu-Bionic-x86-64.img.gz"
+    gunzip --force "input/Ubuntu-Focal-x86-64.img.gz"
     run_convert "release-2" \
-                "input/Ubuntu-Bionic-x86-64.img" \
+                "input/Ubuntu-Focal-x86-64.img" \
-                "--config configs/qemux86-64_config" || test_result=$?
+                "--config configs/qemux86-64_config $EXTRA_CONFIG" || test_result=$?
     ret=0
-    test -f deploy/Ubuntu-Bionic-x86-64-qemux86_64-mender.img || ret=$?
+    test -f deploy/Ubuntu-Focal-x86-64-qemux86_64-mender.img || ret=$?
-    assert "${ret}" "0" "Expected uncompressed file deploy/Ubuntu-Bionic-x86-64-qemux86_64-mender.img"
+    assert "${ret}" "0" "Expected uncompressed file deploy/Ubuntu-Focal-x86-64-qemux86_64-mender.img"
   fi
 
-  if [ "$1" == "--all" -o "$1" == "--only" -a "$2" == "raspberrypi3" ]; then
+  if [ "$TEST_ALL" == "1" -o "$TEST_PLATFORM" == "raspberrypi3" ]; then
     wget --progress=dot:giga -N ${RASPBIAN_IMAGE_URL} -P input/
     RASPBIAN_IMAGE="${RASPBIAN_IMAGE_URL##*/}"
     convert_and_test "raspberrypi3" \
                      "release-1" \
                      "input/${RASPBIAN_IMAGE}" \
-                     "--config configs/raspberrypi3_config" \
+                     "--config configs/raspberrypi3_config $EXTRA_CONFIG" \
                      -- \
                      "-k" "'not test_update'" \
                      || test_result=$?
   fi
 
-  if [ "$1" == "--all" -o "$1" == "--only" -a "$2" == "beaglebone" ]; then
+  if [ "$TEST_ALL" == "1" -o "$TEST_PLATFORM" == "beaglebone" ]; then
     wget --progress=dot:giga -N ${BBB_DEBIAN_SDCARD_IMAGE_URL} -P input/
     BBB_DEBIAN_SDCARD_IMAGE_COMPRESSED="${BBB_DEBIAN_SDCARD_IMAGE_URL##*/}"
     BBB_DEBIAN_SDCARD_IMAGE_UNCOMPRESSED="${BBB_DEBIAN_SDCARD_IMAGE_COMPRESSED%.xz}"
@@ -100,7 +129,7 @@ else
     convert_and_test "beaglebone-sdcard" \
                      "release-1" \
                      "input/${BBB_DEBIAN_SDCARD_IMAGE_UNCOMPRESSED}" \
-                     "--config configs/beaglebone_black_debian_sdcard_config" \
+                     "--config configs/beaglebone_black_debian_sdcard_config $EXTRA_CONFIG" \
                      -- \
                      "-k" "'not test_update'" \
                      || test_result=$?
@@ -113,19 +142,19 @@ else
     convert_and_test "beaglebone-emmc" \
                      "release-1" \
                      "input/${BBB_DEBIAN_EMMC_IMAGE_UNCOMPRESSED}" \
-                     "--config configs/beaglebone_black_debian_emmc_config" \
+                     "--config configs/beaglebone_black_debian_emmc_config $EXTRA_CONFIG" \
                      -- \
                      "-k" "'not test_update'" \
                      || test_result=$?
   fi
 
-  if [ "$1" == "--all" -o "$1" == "--only" -a "$2" == "ubuntu" ]; then
+  if [ "$TEST_ALL" == "1" -o "$TEST_PLATFORM" == "ubuntu" ]; then
     wget --progress=dot:giga -N ${UBUNTU_SERVER_RPI_IMAGE_URL} -P input/
     UBUNTU_SERVER_RPI_IMAGE_COMPRESSED="${UBUNTU_SERVER_RPI_IMAGE_URL##*/}"
     convert_and_test "raspberrypi3" \
                      "release-1" \
                      "input/${UBUNTU_SERVER_RPI_IMAGE_COMPRESSED}" \
-                     "--config configs/raspberrypi3_config" \
+                     "--config configs/raspberrypi3_config $EXTRA_CONFIG" \
                      -- \
                      "-k" "'not test_update'" \
                      || test_result=$?