From 0aa0a7342c599f8751bfd33a77ca63a34fb6664b Mon Sep 17 00:00:00 2001 From: Andrey Basov Date: Sat, 18 Sep 2021 17:39:29 +0300 Subject: [PATCH 1/5] Closing curly brace added in bootstrap-rootfs-overlay-production-server.sh Changelog: Title Signed-off-by: Andrey Basov Signed-off-by: Kristian Amlie (cherry picked from commit 8f2e198ce6e8d7bb7702dad9b28e3b6be1bf0a08) --- scripts/bootstrap-rootfs-overlay-production-server.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/bootstrap-rootfs-overlay-production-server.sh b/scripts/bootstrap-rootfs-overlay-production-server.sh index deb28d0..a60bbb0 100755 --- a/scripts/bootstrap-rootfs-overlay-production-server.sh +++ b/scripts/bootstrap-rootfs-overlay-production-server.sh @@ -75,6 +75,10 @@ EOF cp -f "${server_cert}" ${output_dir}/etc/mender/server.crt fi +cat <<- EOF >> ${root_dir}/resources/mender.conf +} +EOF + sudo chown -R 0 ${output_dir} sudo chgrp -R 0 ${output_dir} From bab46c1d610ac256fb01e2acdd64d2a3eb2ed57d Mon Sep 17 00:00:00 2001 From: Kristian Amlie Date: Tue, 7 Sep 2021 16:53:56 +0200 Subject: [PATCH 2/5] Make sure version overrides are used in tests as well as production. Changelog: None Signed-off-by: Kristian Amlie (cherry picked from commit 6580e27f9274185fac8691c32f93ed1d10011985) --- .gitlab-ci.yml | 11 ++++--- scripts/test/run-tests.sh | 61 +++++++++++++++++++++++++++++---------- 2 files changed, 52 insertions(+), 20 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2101caa..0033631 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -161,6 +161,9 @@ convert_raspbian_raspberrypi4: - curl -f -O https://mender.s3.amazonaws.com/mender-artifact/$MENDER_ARTIFACT_VERSION/linux/mender-artifact - chmod ugo+x mender-artifact - mv mender-artifact /usr/bin/ + - echo "MENDER_CLIENT_VERSION=${MENDER_CLIENT_VERSION}" > versions_override_config + - echo "MENDER_ADDON_CONNECT_VERSION=${MENDER_ADDON_CONNECT_VERSION}" >> versions_override_config + - echo "MENDER_ADDON_CONFIGURE_VERSION=${MENDER_ADDON_CONFIGURE_VERSION}" >> versions_override_config # Get Update Modules Artifact generators for the tests - if [ "${MENDER_CLIENT_VERSION}" = "latest" ]; then - mender_version=master @@ -220,22 +223,22 @@ test_acceptance_prebuilt_raspberrypi4: test_acceptance_qemux86_64: <<: *test_acceptance script: - - ./scripts/test/run-tests.sh --only qemux86_64 + - ./scripts/test/run-tests.sh --config versions_override_config --only qemux86_64 test_acceptance_raspberrypi: <<: *test_acceptance script: - - ./scripts/test/run-tests.sh --only raspberrypi3 + - ./scripts/test/run-tests.sh --config versions_override_config --only raspberrypi3 test_acceptance_beaglebone: <<: *test_acceptance script: - - ./scripts/test/run-tests.sh --only beaglebone + - ./scripts/test/run-tests.sh --config versions_override_config --only beaglebone test_acceptance_ubuntu: <<: *test_acceptance script: - - ./scripts/test/run-tests.sh --only ubuntu + - ./scripts/test/run-tests.sh --config versions_override_config --only ubuntu publish:s3: stage: publish diff --git a/scripts/test/run-tests.sh b/scripts/test/run-tests.sh index 9c2d4b9..916aaee 100755 --- a/scripts/test/run-tests.sh +++ b/scripts/test/run-tests.sh @@ -3,7 +3,7 @@ set -e usage() { - echo "$0 <--all | --only DEVICE_TYPE | --prebuilt-image DEVICE_TYPE IMAGE_NAME>" + echo "$0 [--config EXTRA_CONFIG_FILE] <--all | --only DEVICE_TYPE | --prebuilt-image DEVICE_TYPE IMAGE_NAME>" exit 1 } @@ -44,26 +44,55 @@ get_pytest_files prepare_ssh_keys -if ! [ "$1" == "--all" -o "$1" == "--only" -a -n "$2" -o "$1" == "--prebuilt-image" -a -n "$3" ]; then - usage -fi +usage_if_empty() { + if [ -z "$1" ]; then + usage + fi +} + +PREBUILT_IMAGE= +TEST_PLATFORM= +TEST_ALL=0 +EXTRA_CONFIG= +while [ -n "$1" ]; do + case "$1" in + --prebuilt-image) + usage_if_empty "$3" + PREBUILT_IMAGE="$2 $3" + ;; + --all) + TEST_ALL=1 + ;; + --only) + usage_if_empty "$2" + TEST_PLATFORM="$2" + shift + ;; + --config) + usage_if_empty "$2" + EXTRA_CONFIG="$EXTRA_CONFIG --config $2" + shift + ;; + esac + shift +done test_result=0 -if [ "$1" == "--prebuilt-image" ]; then - run_tests "$2" "$3" \ +if [ -n "$PREBUILT_IMAGE" ]; then + run_tests $PREBUILT_IMAGE \ "-k" "'not test_update'" \ || test_result=$? exit $test_result else - if [ "$1" == "--all" -o "$1" == "--only" -a "$2" == "qemux86_64" ]; then + if [ "$TEST_ALL" == "1" -o "$TEST_PLATFORM" == "qemux86_64" ]; then wget --progress=dot:giga -N ${UBUNTU_IMAGE_URL} -P input/ convert_and_test "qemux86_64" \ "release-1" \ "input/Ubuntu-Bionic-x86-64.img.gz" \ "--overlay tests/ssh-public-key-overlay" \ - "--config configs/qemux86-64_config" \ + "--config configs/qemux86-64_config $EXTRA_CONFIG" \ || test_result=$? echo >&2 "----------------------------------------" @@ -73,25 +102,25 @@ else gunzip --force "input/Ubuntu-Bionic-x86-64.img.gz" run_convert "release-2" \ "input/Ubuntu-Bionic-x86-64.img" \ - "--config configs/qemux86-64_config" || test_result=$? + "--config configs/qemux86-64_config $EXTRA_CONFIG" || test_result=$? ret=0 test -f deploy/Ubuntu-Bionic-x86-64-qemux86_64-mender.img || ret=$? assert "${ret}" "0" "Expected uncompressed file deploy/Ubuntu-Bionic-x86-64-qemux86_64-mender.img" fi - if [ "$1" == "--all" -o "$1" == "--only" -a "$2" == "raspberrypi3" ]; then + if [ "$TEST_ALL" == "1" -o "$TEST_PLATFORM" == "raspberrypi3" ]; then wget --progress=dot:giga -N ${RASPBIAN_IMAGE_URL} -P input/ RASPBIAN_IMAGE="${RASPBIAN_IMAGE_URL##*/}" convert_and_test "raspberrypi3" \ "release-1" \ "input/${RASPBIAN_IMAGE}" \ - "--config configs/raspberrypi3_config" \ + "--config configs/raspberrypi3_config $EXTRA_CONFIG" \ -- \ "-k" "'not test_update'" \ || test_result=$? fi - if [ "$1" == "--all" -o "$1" == "--only" -a "$2" == "beaglebone" ]; then + if [ "$TEST_ALL" == "1" -o "$TEST_PLATFORM" == "beaglebone" ]; then wget --progress=dot:giga -N ${BBB_DEBIAN_SDCARD_IMAGE_URL} -P input/ BBB_DEBIAN_SDCARD_IMAGE_COMPRESSED="${BBB_DEBIAN_SDCARD_IMAGE_URL##*/}" BBB_DEBIAN_SDCARD_IMAGE_UNCOMPRESSED="${BBB_DEBIAN_SDCARD_IMAGE_COMPRESSED%.xz}" @@ -100,7 +129,7 @@ else convert_and_test "beaglebone-sdcard" \ "release-1" \ "input/${BBB_DEBIAN_SDCARD_IMAGE_UNCOMPRESSED}" \ - "--config configs/beaglebone_black_debian_sdcard_config" \ + "--config configs/beaglebone_black_debian_sdcard_config $EXTRA_CONFIG" \ -- \ "-k" "'not test_update'" \ || test_result=$? @@ -113,19 +142,19 @@ else convert_and_test "beaglebone-emmc" \ "release-1" \ "input/${BBB_DEBIAN_EMMC_IMAGE_UNCOMPRESSED}" \ - "--config configs/beaglebone_black_debian_emmc_config" \ + "--config configs/beaglebone_black_debian_emmc_config $EXTRA_CONFIG" \ -- \ "-k" "'not test_update'" \ || test_result=$? fi - if [ "$1" == "--all" -o "$1" == "--only" -a "$2" == "ubuntu" ]; then + if [ "$TEST_ALL" == "1" -o "$TEST_PLATFORM" == "ubuntu" ]; then wget --progress=dot:giga -N ${UBUNTU_SERVER_RPI_IMAGE_URL} -P input/ UBUNTU_SERVER_RPI_IMAGE_COMPRESSED="${UBUNTU_SERVER_RPI_IMAGE_URL##*/}" convert_and_test "raspberrypi3" \ "release-1" \ "input/${UBUNTU_SERVER_RPI_IMAGE_COMPRESSED}" \ - "--config configs/raspberrypi3_config" \ + "--config configs/raspberrypi3_config $EXTRA_CONFIG" \ -- \ "-k" "'not test_update'" \ || test_result=$? From 7981e223a8065c5db1b1a4dacb4b719e23f02f12 Mon Sep 17 00:00:00 2001 From: Kristian Amlie Date: Wed, 8 Sep 2021 08:49:10 +0200 Subject: [PATCH 3/5] URL encode '+' characters in package names, or else we get Forbidden. Changelog: Fix broken download of packages when `*_VERSION` are set to `master`. Signed-off-by: Kristian Amlie (cherry picked from commit ed5da386679c2995c18c522d4e2abff3614f4139) --- modules/deb.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/deb.sh b/modules/deb.sh index e8686ca..5729962 100644 --- a/modules/deb.sh +++ b/modules/deb.sh @@ -47,7 +47,8 @@ function deb_from_repo_dist_get() { fi local -r filename=$(basename $deb_package_path) - run_and_log_cmd "wget -Nq ${repo_url}/${deb_package_path} -P ${download_dir}" + local -r deb_package_url=$(echo ${repo_url}/${deb_package_path} | sed 's/+/%2B/g') + run_and_log_cmd "wget -Nq ${deb_package_url} -P ${download_dir}" rm -f /tmp/Packages log_info "Successfully downloaded ${filename}" From 8f6adb384a843284c363ceb76df21e49318aaf92 Mon Sep 17 00:00:00 2001 From: Lluis Campos Date: Wed, 15 Sep 2021 14:49:31 +0200 Subject: [PATCH 4/5] Update testing image Ubuntu x86-64 to Focal release Changelog: None Signed-off-by: Lluis Campos (cherry picked from commit 50da54d8bf5899448c3c95d0aa646b76bbde2e88) --- configs/qemux86-64_config | 13 ++++++++----- scripts/test/run-tests.sh | 12 ++++++------ 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/configs/qemux86-64_config b/configs/qemux86-64_config index 6b0599d..2f21b14 100644 --- a/configs/qemux86-64_config +++ b/configs/qemux86-64_config @@ -1,16 +1,19 @@ # This configuration can be used to run on a qemux86-64 machine. # -# This has been tested on images generated with the following command: +# The image is generated with the following command: # -# mkosi -d ubuntu -r bionic -t gpt_ext4 -b --checksum --password password --package=openssh-server,dhcpcd5 -o image.raw +# mkosi --root-size=2G --distribution=ubuntu --release=focal --format=gpt_ext4 --bootable --checksum +# --password password --package=openssh-server,dhcpcd5 --output=Ubuntu-Focal-x86-64.img build # -# Converted with the following command: +# Then manually uploaded to Mender's AWS S3 bucket for CI testing # -# MENDER_ARTIFACT_NAME=release-1 ./mender-convert --disk-image input/image.raw --overlay rootfs_overlay_demo --config configs/qemux86-64_config +# Locally, converted with the following command: +# +# MENDER_ARTIFACT_NAME=release-1 ./docker-mender-convert --disk-image input/Ubuntu-Focal-x86-64.img --overlay rootfs_overlay_demo --config configs/qemux86-64_config # # and qemu is executed with the following command: # -# qemu-system-x86_64 -enable-kvm -m 512 -smp 2 -bios /usr/share/ovmf/x64/OVMF_CODE.fd -drive format=raw,file=qemux86_64-release-1.sdimg +# qemu-system-x86_64 -enable-kvm -m 512 -smp 2 -bios /usr/share/OVMF/OVMF_CODE.fd -drive format=raw,file=deploy/Ubuntu-Focal-x86-64-qemux86_64-mender.img MENDER_STORAGE_DEVICE_BASE=/dev/sda MENDER_DEVICE_TYPE="qemux86_64" diff --git a/scripts/test/run-tests.sh b/scripts/test/run-tests.sh index 916aaee..84a6cdc 100755 --- a/scripts/test/run-tests.sh +++ b/scripts/test/run-tests.sh @@ -26,7 +26,7 @@ BBB_DEBIAN_EMMC_IMAGE_URL="https://rcn-ee.com/rootfs/bb.org/testing/2021-05-28/b ## Auto-update RASPBIAN_IMAGE_URL="http://downloads.raspberrypi.org/raspbian_lite/images/raspbian_lite-2020-02-14/2020-02-13-raspbian-buster-lite.zip" -UBUNTU_IMAGE_URL="https://downloads.mender.io/mender-convert/images/Ubuntu-Bionic-x86-64.img.gz" +UBUNTU_IMAGE_URL="https://downloads.mender.io/mender-convert/images/Ubuntu-Focal-x86-64.img.gz" ## Auto-update UBUNTU_SERVER_RPI_IMAGE_URL="http://cdimage.ubuntu.com/ubuntu/releases/20.04.2/release/ubuntu-20.04.2-preinstalled-server-armhf+raspi.img.xz" @@ -90,7 +90,7 @@ else wget --progress=dot:giga -N ${UBUNTU_IMAGE_URL} -P input/ convert_and_test "qemux86_64" \ "release-1" \ - "input/Ubuntu-Bionic-x86-64.img.gz" \ + "input/Ubuntu-Focal-x86-64.img.gz" \ "--overlay tests/ssh-public-key-overlay" \ "--config configs/qemux86-64_config $EXTRA_CONFIG" \ || test_result=$? @@ -99,13 +99,13 @@ else echo >&2 "Running the uncompressed test" echo >&2 "----------------------------------------" rm -rf deploy - gunzip --force "input/Ubuntu-Bionic-x86-64.img.gz" + gunzip --force "input/Ubuntu-Focal-x86-64.img.gz" run_convert "release-2" \ - "input/Ubuntu-Bionic-x86-64.img" \ + "input/Ubuntu-Focal-x86-64.img" \ "--config configs/qemux86-64_config $EXTRA_CONFIG" || test_result=$? ret=0 - test -f deploy/Ubuntu-Bionic-x86-64-qemux86_64-mender.img || ret=$? - assert "${ret}" "0" "Expected uncompressed file deploy/Ubuntu-Bionic-x86-64-qemux86_64-mender.img" + test -f deploy/Ubuntu-Focal-x86-64-qemux86_64-mender.img || ret=$? + assert "${ret}" "0" "Expected uncompressed file deploy/Ubuntu-Focal-x86-64-qemux86_64-mender.img" fi if [ "$TEST_ALL" == "1" -o "$TEST_PLATFORM" == "raspberrypi3" ]; then From 937c2bdc6aba685bfc46608a1da7c904e6ad052c Mon Sep 17 00:00:00 2001 From: mender-test-bot Date: Mon, 30 Aug 2021 03:16:57 +0000 Subject: [PATCH 5/5] Image-Bot - Image updates Changelog: None Signed-off-by: mender-test-bot (cherry picked from commit 4ec9e26055a0c1080f3afe6c335a9600b47a6958) --- scripts/test/run-tests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/test/run-tests.sh b/scripts/test/run-tests.sh index 84a6cdc..68f525f 100755 --- a/scripts/test/run-tests.sh +++ b/scripts/test/run-tests.sh @@ -21,7 +21,7 @@ BBB_DEBIAN_SDCARD_IMAGE_URL="https://debian.beagleboard.org/images/bone-debian-1 # Not on official home page, but found via https://elinux.org/Beagleboard:BeagleBoneBlack_Debian: ## Auto-update -BBB_DEBIAN_EMMC_IMAGE_URL="https://rcn-ee.com/rootfs/bb.org/testing/2021-05-28/buster-console/bone-debian-10.9-console-armhf-2021-05-28-1gb.img.xz" +BBB_DEBIAN_EMMC_IMAGE_URL="https://rcn-ee.com/rootfs/bb.org/testing/2021-08-23/buster-console/bone-debian-10.10-console-armhf-2021-08-23-1gb.img.xz" ## Auto-update RASPBIAN_IMAGE_URL="http://downloads.raspberrypi.org/raspbian_lite/images/raspbian_lite-2020-02-14/2020-02-13-raspbian-buster-lite.zip" @@ -29,7 +29,7 @@ RASPBIAN_IMAGE_URL="http://downloads.raspberrypi.org/raspbian_lite/images/raspbi UBUNTU_IMAGE_URL="https://downloads.mender.io/mender-convert/images/Ubuntu-Focal-x86-64.img.gz" ## Auto-update -UBUNTU_SERVER_RPI_IMAGE_URL="http://cdimage.ubuntu.com/ubuntu/releases/20.04.2/release/ubuntu-20.04.2-preinstalled-server-armhf+raspi.img.xz" +UBUNTU_SERVER_RPI_IMAGE_URL="http://cdimage.ubuntu.com/ubuntu/releases/20.04/release/ubuntu-20.04.3-preinstalled-server-armhf+raspi.img.xz" # Keep common function declarations in separate utils script UTILS_PATH=${0/$(basename $0)/test-utils.sh}