From 73d0625a38b1630b0dde930844abef124caab0c9 Mon Sep 17 00:00:00 2001
From: Kristian Amlie <kristian.amlie@northern.tech>
Date: Wed, 4 Dec 2019 09:53:46 +0100
Subject: [PATCH 01/16] Make sure artifacts are owned by same user as the
 launch directory.

Changelog: Title

Signed-off-by: Kristian Amlie <kristian.amlie@northern.tech>
---
 docker-entrypoint.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh
index bfc703d..092334d 100755
--- a/docker-entrypoint.sh
+++ b/docker-entrypoint.sh
@@ -23,3 +23,6 @@ cd /mender-convert
 echo "Running mender-convert "$@""
 
 ./mender-convert "$@"
+
+# Set owner and group to same as launch directory.
+[ -d deploy ] && chown -R --reference=. deploy

From 3812d85dbe820b76d93d44cc39fca9013cc0b035 Mon Sep 17 00:00:00 2001
From: Kristian Amlie <kristian.amlie@northern.tech>
Date: Thu, 5 Dec 2019 13:46:54 +0100
Subject: [PATCH 02/16] Don't use --apparent-size to calculate rootfs total
 size.

Not using this option produces a more realistic result, because block
padding has to be taken into account when calculating the totals. It
is not a perfect method however, since the calculation becomes
dependent on the block size of the filesystem on the build machine.
4096 is a *very* common size though, and almost certainly will be the
size used both on the build host and in the resulting filesystem.

Changelog: Changed the calculation of occupied space on the rootfs
partition to a more accurate formula.

Signed-off-by: Kristian Amlie <kristian.amlie@northern.tech>
---
 mender-convert-package | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mender-convert-package b/mender-convert-package
index 1109a45..1a78a20 100755
--- a/mender-convert-package
+++ b/mender-convert-package
@@ -140,7 +140,7 @@ artifact_name=$(cat work/rootfs/etc/mender/artifact_info | sed 's/[^=]*=//')
 
 image_name="${device_type}-${artifact_name}"
 
-actual_rootfs_size=$(sudo du --apparent-size -s --block-size=512 work/rootfs | cut -f 1)
+actual_rootfs_size=$(sudo du -s --block-size=512 work/rootfs | cut -f 1)
 
 # 50 % free space, not to be confused with rootfs_part_sectors
 rootfs_image_sectors=$(awk -v r1="$actual_rootfs_size" 'BEGIN{printf "%.0f", r1 * 1.50}')

From 79ceadb81ff5eddefe215c6e3f445a4f86fde254 Mon Sep 17 00:00:00 2001
From: Kristian Amlie <kristian.amlie@northern.tech>
Date: Thu, 5 Dec 2019 14:26:02 +0100
Subject: [PATCH 03/16] Add support for controlling rootfs size with `IMAGE_*`
 variables.

* `IMAGE_ROOTFS_SIZE` - The base size of the rootfs. The other two
  variables modify this value

* `IMAGE_ROOTFS_EXTRA_SIZE` - The amount of free space to add to the
  base size of the rootfs

* `IMAGE_OVERHEAD_FACTOR` - Factor determining the amount of free
  space to add to the base size of the rootfs

The final size will be the largest of the two calculations. Please see
the `mender_convert_config` file comments for more information.

Changelog: Commit

Signed-off-by: Kristian Amlie <kristian.amlie@northern.tech>
---
 configs/mender_convert_config | 35 +++++++++++++++++++++++++++++++++++
 mender-convert-package        | 23 ++++++++++++++++++++---
 2 files changed, 55 insertions(+), 3 deletions(-)

diff --git a/configs/mender_convert_config b/configs/mender_convert_config
index b056b74..b4929da 100644
--- a/configs/mender_convert_config
+++ b/configs/mender_convert_config
@@ -127,6 +127,41 @@ MENDER_USE_BMAP="n"
 # In most cases you would like this enabled.
 MENDER_COPY_BOOT_GAP="y"
 
+# The size of each of the two rootfs filesystems, in KiB. If this is 0,
+# mender-convert will use the size of the filesystem content as a basis. If the
+# value is -1, mender-convert will use the maximum size that will fit inside the
+# created partition. The size is further modified by IMAGE_ROOTFS_EXTRA_SPACE
+# and IMAGE_OVERHEAD_FACTOR.
+#
+# This variable directly mirrors the variable from the Yocto Project, which is
+# why it is missing a "MENDER_" prefix.
+IMAGE_ROOTFS_SIZE="0"
+
+# The amount of extra free space requested on the rootfs, in KiB. This is added
+# to the value of IMAGE_ROOTFS_SIZE. The size is further modified by
+# IMAGE_OVERHEAD_FACTOR.
+#
+# Note that due to reserved space for the root user on the filesystem, "df" may
+# report a significantly lower number than requested. A more accurate number can
+# be fetched using for example "dumpe2fs" and looking for the "Free blocks"
+# field, but even this value is usually going to be lower than requested due to
+# meta data on the filesystem.
+#
+# This variable directly mirrors the variable from the Yocto Project, which is
+# why it is missing a "MENDER_" prefix.
+IMAGE_ROOTFS_EXTRA_SPACE="0"
+
+# This factor is multiplied by the used space value for the generated rootfs,
+# and if the result is larger than IMAGE_ROOTFS_SIZE + IMAGE_ROOTFS_EXTRA_SPACE,
+# it will be used as the size of the rootfs instead of the other two variables.
+#
+# The actual free space will usually be lower than requested. See comment for
+# IMAGE_ROOTFS_EXTRA_SPACE.
+#
+# This variable directly mirrors the variable from the Yocto Project, which is
+# why it is missing a "MENDER_" prefix.
+IMAGE_OVERHEAD_FACTOR="1.5"
+
 source configs/mender_grub_config
 
 # Function to create Mender Artifact
diff --git a/mender-convert-package b/mender-convert-package
index 1a78a20..cf48049 100755
--- a/mender-convert-package
+++ b/mender-convert-package
@@ -142,15 +142,32 @@ image_name="${device_type}-${artifact_name}"
 
 actual_rootfs_size=$(sudo du -s --block-size=512 work/rootfs | cut -f 1)
 
-# 50 % free space, not to be confused with rootfs_part_sectors
-rootfs_image_sectors=$(awk -v r1="$actual_rootfs_size" 'BEGIN{printf "%.0f", r1 * 1.50}')
+# KiB -> 512 sectors
+image_rootfs_size_sectors=$((${IMAGE_ROOTFS_SIZE} * 2))
+
+if [ "${IMAGE_ROOTFS_SIZE}" -eq -1 ]; then
+  rootfs_image_sectors="${rootfs_part_sectors}"
+elif [ "${image_rootfs_size_sectors}" -lt "${actual_rootfs_size}" ]; then
+  rootfs_image_sectors="${actual_rootfs_size}"
+else
+  rootfs_image_sectors="${image_rootfs_size_sectors}"
+fi
+
+rootfs_image_sectors=$((${rootfs_image_sectors} + ${IMAGE_ROOTFS_EXTRA_SPACE} * 2))
+
+rootfs_image_sectors_overhead=$(awk -v r1="$actual_rootfs_size" "BEGIN{printf \"%.0f\", r1 * ${IMAGE_OVERHEAD_FACTOR}}")
+if [ "${rootfs_image_sectors_overhead}" -gt "${rootfs_image_sectors}" ]; then
+  rootfs_image_sectors="${rootfs_image_sectors_overhead}"
+fi
 
 if [ ${rootfs_image_sectors} -gt ${rootfs_part_sectors} ]; then
   log_warn "The calculated rootfs partition size $(disk_sectors_to_mb ${rootfs_part_sectors}) MiB is too small."
   log_warn "The actual rootfs image size is $(disk_sectors_to_mb ${rootfs_image_sectors}) MiB"
-  log_fatal "You can try adjusting the MENDER_STORAGE_TOTAL_SIZE_MB variable to increase available space"
+  log_fatal "You can try adjusting the MENDER_STORAGE_TOTAL_SIZE_MB variable to increase available space, or modify one of the variables IMAGE_ROOTFS_SIZE, IMAGE_ROOTFS_EXTRA_SPACE or IMAGE_OVERHEAD_FACTOR to reduce the size of the root filesystem."
 fi
 
+log_info "Rootfs filesystem size will be $(disk_sectors_to_mb ${rootfs_image_sectors}) MiB"
+
 # Extract file-system type from rootfs
 if file ${root_part} | grep -q ext4; then
   image_fs_type="ext4"

From af17516372b45cd9a3dbaa96c285bdefd3dcf09a Mon Sep 17 00:00:00 2001
From: Kristian Amlie <kristian.amlie@northern.tech>
Date: Thu, 5 Dec 2019 14:47:29 +0100
Subject: [PATCH 04/16] Some cosmetic changes in the partition size logging.

Make it clear that these are partition sizes, not filesystem sizes
(mainly important for the rootfs), and fix the ambiguous placement of
"x 2" (the existing placement made it look like there were 3892
partitions of 2 MiB each).

Changelog: None

Signed-off-by: Kristian Amlie <kristian.amlie@northern.tech>
---
 mender-convert-package | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/mender-convert-package b/mender-convert-package
index cf48049..05ece85 100755
--- a/mender-convert-package
+++ b/mender-convert-package
@@ -197,9 +197,9 @@ log_info "Creating Mender compatible disk-image"
 sdimg_path=deploy/${image_name}.sdimg
 
 log_info "Total disk size: $(disk_sectors_to_mb ${disk_image_total_sectors}) MiB"
-log_info "  Boot partition $(disk_sectors_to_mb ${boot_part_sectors}) MiB"
-log_info "  RootFS         $(disk_sectors_to_mb ${rootfs_part_sectors}) x 2 MiB"
-log_info "  Data           $(disk_sectors_to_mb ${data_part_sectors}) MiB"
+log_info "  Boot partition    $(disk_sectors_to_mb ${boot_part_sectors}) MiB"
+log_info "  RootFS partitions $(disk_sectors_to_mb ${rootfs_part_sectors}) MiB x 2"
+log_info "  Data partition    $(disk_sectors_to_mb ${data_part_sectors}) MiB"
 
 # Initialize sdcard image file
 run_and_log_cmd \

From ac82202afd3750e585da14f9eccf62bbe50f899d Mon Sep 17 00:00:00 2001
From: Kristian Amlie <kristian.amlie@northern.tech>
Date: Fri, 6 Dec 2019 10:47:34 +0100
Subject: [PATCH 05/16] Add lzma compression option for
 `MENDER_COMPRESS_DISK_IMAGE`.

It uses the parallel pxz compression tool.

Changelog: Title

Signed-off-by: Kristian Amlie <kristian.amlie@northern.tech>
---
 Dockerfile                    | 17 ++++++++++++++++-
 configs/mender_convert_config | 10 ++++++++--
 mender-convert-package        | 20 ++++++++++++++++----
 3 files changed, 40 insertions(+), 7 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index b19e133..6a2529d 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,3 +1,14 @@
+# Build pxz in separate image to avoid big image size
+FROM ubuntu:19.04 AS build
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    git \
+    liblzma-dev
+
+# Parallel xz (LZMA) compression
+RUN git clone https://github.com/jnovy/pxz.git /root/pxz
+RUN cd /root/pxz && make
+
 FROM ubuntu:19.04
 
 ARG MENDER_ARTIFACT_VERSION=3.1.0
@@ -30,7 +41,11 @@ RUN apt-get update && apt-get install -y \
 # to get rid of 'sh: 1: udevadm: not found' errors triggered by parted
     udev \
 # to create bmap index file (MENDER_USE_BMAP)
-    bmap-tools
+    bmap-tools \
+# needed to run pxz
+    libgomp1
+
+COPY --from=build /root/pxz/pxz /usr/bin/pxz
 
 RUN wget -q -O /usr/bin/mender-artifact https://d1b0l86ne08fsf.cloudfront.net/mender-artifact/$MENDER_ARTIFACT_VERSION/linux/mender-artifact \
     && chmod +x /usr/bin/mender-artifact
diff --git a/configs/mender_convert_config b/configs/mender_convert_config
index b4929da..74f5615 100644
--- a/configs/mender_convert_config
+++ b/configs/mender_convert_config
@@ -4,12 +4,18 @@
 #
 # NOTE! This file will always be sourced.
 
-# Compress generated disk image using gzip
+# Compress generated disk image
 #
 # This is useful when you have large disk images, compressing them
 # makes it easier to transfer them between e.g an build server and a local
 # machine, and obviously saves space.
-MENDER_COMPRESS_DISK_IMAGE=y
+#
+# Possible values are:
+# 'n'    - No compression
+# 'gzip' - Compress with gzip
+# 'lzma' - Compress with xz (LZMA)
+# 'y'    - Alias for 'gzip'
+MENDER_COMPRESS_DISK_IMAGE=gzip
 
 # Compression algorithm for Mender Artifact
 #
diff --git a/mender-convert-package b/mender-convert-package
index 05ece85..344d54b 100755
--- a/mender-convert-package
+++ b/mender-convert-package
@@ -251,10 +251,22 @@ if [ "${MENDER_USE_BMAP}" == "y" ]; then
   run_and_log_cmd "${BMAP_TOOL} create ${sdimg_path} > ${sdimg_path}.bmap"
 fi
 
-if [ "${MENDER_COMPRESS_DISK_IMAGE}" == "y" ]; then
-  log_info "Compressing ${sdimg_path}.gz"
-  run_and_log_cmd "pigz --best --force ${sdimg_path}"
-fi
+case "${MENDER_COMPRESS_DISK_IMAGE}" in
+  y | gzip)
+    log_info "Compressing ${sdimg_path}.gz"
+    run_and_log_cmd "pigz --best --force ${sdimg_path}"
+    ;;
+  lzma)
+    log_info "Compressing ${sdimg_path}.xz"
+    run_and_log_cmd "pxz --best --force ${sdimg_path}"
+    ;;
+  n)
+    :
+    ;;
+  *)
+    log_fatal "Unknown MENDER_COMPRESS_DISK_IMAGE value: ${MENDER_COMPRESS_DISK_IMAGE}"
+    ;;
+esac
 
 log_info "Conversion has completed! \o/"
 

From ba4ed6dfc57d4683ea6a2fb16cf24657c468fed2 Mon Sep 17 00:00:00 2001
From: Kristian Amlie <kristian.amlie@northern.tech>
Date: Fri, 6 Dec 2019 11:00:31 +0100
Subject: [PATCH 06/16] Bump mender-artifact version to 3.2.1.

Changelog: Title

Signed-off-by: Kristian Amlie <kristian.amlie@northern.tech>
---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 6a2529d..3002abb 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -11,7 +11,7 @@ RUN cd /root/pxz && make
 
 FROM ubuntu:19.04
 
-ARG MENDER_ARTIFACT_VERSION=3.1.0
+ARG MENDER_ARTIFACT_VERSION=3.2.1
 
 RUN apt-get update && apt-get install -y \
 # For 'ar' command to unpack .deb

From 44fa719103af70721ac58775de540fe4a0a1fefa Mon Sep 17 00:00:00 2001
From: Kristian Amlie <kristian.amlie@northern.tech>
Date: Mon, 9 Dec 2019 14:58:32 +0100
Subject: [PATCH 07/16] Add conversion and publication stages for Raspbian
 images.

Changelog: None

Signed-off-by: Kristian Amlie <kristian.amlie@northern.tech>
---
 .gitlab-ci.yml                              | 58 ++++++++++++++++++++-
 configs/images/raspberrypi3_raspbian_config | 14 +++++
 docker-build                                |  2 +-
 3 files changed, 72 insertions(+), 2 deletions(-)
 create mode 100644 configs/images/raspberrypi3_raspbian_config

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a6ab48f..a6271e2 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -2,6 +2,11 @@ image: docker:git
 
 variables:
   DOCKER_REPOSITORY: mendersoftware/mender-convert
+  S3_BUCKET_NAME: mender-convert-images
+  MENDER_ARTIFACT_VERSION: master
+  MENDER_CLIENT_VERSION: master
+  RASPBIAN_URL: http://downloads.raspberrypi.org/raspbian_lite/images/raspbian_lite-2019-09-30/2019-09-26-raspbian-buster-lite.zip
+  RASPBIAN_NAME: 2019-09-26-raspbian-buster-lite
 
 include:
   - project: 'Northern.tech/Mender/mendertesting'
@@ -13,13 +18,15 @@ stages:
   - test
   - build
   - test_acceptance
+  - convert
+  - publish
 
 build:
   stage: build
   services:
     - docker:dind
   script:
-    - IMAGE_NAME=$DOCKER_REPOSITORY:pr ./docker-build
+    - IMAGE_NAME=$DOCKER_REPOSITORY:pr ./docker-build --build-arg MENDER_ARTIFACT_VERSION=${MENDER_ARTIFACT_VERSION}
     - docker save $DOCKER_REPOSITORY:pr > image.tar
   artifacts:
     expire_in: 2w
@@ -57,3 +64,52 @@ test_acceptance:
       - report_*.html
     reports:
       junit: results_*.xml
+
+convert_raspbian:
+  stage: convert
+  image: teracy/ubuntu:18.04-dind-18.09.9
+  services:
+    - docker:18-dind
+  tags:
+    - mender-qa-slave
+  dependencies:
+    - build
+  before_script:
+    - apt update && apt install -yy bash wget unzip
+
+    - export IMAGE_NAME=$DOCKER_REPOSITORY:pr
+    - docker load -i image.tar
+
+    - wget -q ${RASPBIAN_URL}
+    - unzip ${RASPBIAN_NAME}.zip
+
+  script:
+    - echo "MENDER_CLIENT_VERSION=${MENDER_CLIENT_VERSION}" > mender_client_version_config
+    - env MENDER_ARTIFACT_NAME=mender-raspbian
+      ./docker-mender-convert -d ${RASPBIAN_NAME}.img
+      -c configs/raspberrypi3_config
+      -c configs/images/raspberrypi3_raspbian_config
+      -c mender_client_version_config
+    - mv deploy/*-mender-raspbian.sdimg.xz deploy/${RASPBIAN_NAME}-mender-${MENDER_CLIENT_VERSION}.img.xz
+
+  after_script:
+    - mkdir -p output
+    - mv deploy/*.img.xz output
+
+  artifacts:
+    paths:
+      - output
+    expire_in: 2 weeks
+
+publish:s3:
+  when: manual
+  stage: publish
+  image: debian:buster
+  before_script:
+    - apt update && apt install -yyq awscli
+  script:
+    - echo "Publishing ${RASPBIAN_NAME}-mender-${MENDER_CLIENT_VERSION}.img.xz version to S3"
+    - aws s3 cp output/${RASPBIAN_NAME}-mender-${MENDER_CLIENT_VERSION}.img.xz
+        s3://$S3_BUCKET_NAME/${RASPBIAN_NAME}/arm/${RASPBIAN_NAME}-mender-${MENDER_CLIENT_VERSION}.img.xz
+    - aws s3api put-object-acl --acl public-read --bucket $S3_BUCKET_NAME
+        --key ${RASPBIAN_NAME}/arm/${RASPBIAN_NAME}-mender-${MENDER_CLIENT_VERSION}.img.xz
diff --git a/configs/images/raspberrypi3_raspbian_config b/configs/images/raspberrypi3_raspbian_config
new file mode 100644
index 0000000..f8d42d8
--- /dev/null
+++ b/configs/images/raspberrypi3_raspbian_config
@@ -0,0 +1,14 @@
+# Real life SD cards typically have less than they advertise. First off, they
+# often use a base of 1000 instead of 1024, and even then they are often smaller
+# than advertised. The number below is based on a conservative target of 3.9GB
+# (that's mathematical GB, base 1000), converted to MiB, rounding down.
+MENDER_STORAGE_TOTAL_SIZE_MB=3719
+
+# Use all there is, which gets us almost, but not quite, to 500MiB free space
+# (about 480MiB at the time of writing).
+IMAGE_ROOTFS_SIZE=-1
+IMAGE_ROOTFS_EXTRA_SPACE=0
+IMAGE_OVERHEAD_FACTOR=1.0
+
+# Best compression there is!
+MENDER_COMPRESS_DISK_IMAGE=lzma
diff --git a/docker-build b/docker-build
index e0b5e03..8aa5f25 100755
--- a/docker-build
+++ b/docker-build
@@ -18,4 +18,4 @@ set -e
 
 IMAGE_NAME=${IMAGE_NAME:-mender-convert}
 
-eval docker build . -t ${IMAGE_NAME}
+eval docker build . -t ${IMAGE_NAME} "$@"

From f52fdfa8cf38ebb5c68370152aa34b43de0d6f11 Mon Sep 17 00:00:00 2001
From: Alf-Rune Siqveland <alf.rune@northern.tech>
Date: Tue, 10 Dec 2019 09:08:17 +0100
Subject: [PATCH 08/16] Split run-tests.sh to export common shell functions in
 a utility script

changelog: none

Modified-by: Kristian Amlie <kristian.amlie@northern.tech>

Signed-off-by: Alf-Rune Siqveland <alf.rune@northern.tech>
Signed-off-by: Kristian Amlie <kristian.amlie@northern.tech>
---
 .gitlab-ci.yml                               |  8 +-
 scripts/test/run-tests.sh                    | 83 +++++++++++++++++
 scripts/{run-tests.sh => test/test-utils.sh} | 94 +++-----------------
 3 files changed, 103 insertions(+), 82 deletions(-)
 create mode 100755 scripts/test/run-tests.sh
 rename scripts/{run-tests.sh => test/test-utils.sh} (57%)
 mode change 100755 => 100644

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a6271e2..ec46d65 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -55,7 +55,7 @@ test_acceptance:
     - export IMAGE_NAME=$DOCKER_REPOSITORY:pr
     - docker load -i image.tar
   script:
-    - ./scripts/run-tests.sh
+    - ./scripts/test/run-tests.sh
   artifacts:
     expire_in: 2w
     when: always
@@ -90,7 +90,11 @@ convert_raspbian:
       -c configs/raspberrypi3_config
       -c configs/images/raspberrypi3_raspbian_config
       -c mender_client_version_config
-    - mv deploy/*-mender-raspbian.sdimg.xz deploy/${RASPBIAN_NAME}-mender-${MENDER_CLIENT_VERSION}.img.xz
+    - cp deploy/raspberrypi-mender-raspbian.sdimg.xz deploy/${RASPBIAN_NAME}-mender-${MENDER_CLIENT_VERSION}.img.xz
+    - unxz deploy/raspberrypi-mender-raspbian.sdimg.xz
+      # Verify converted image
+    - . ./scripts/test/test-utils.sh
+    - run_tests raspbian mender-raspbian "-k TestMenderInstalled"
 
   after_script:
     - mkdir -p output
diff --git a/scripts/test/run-tests.sh b/scripts/test/run-tests.sh
new file mode 100755
index 0000000..197f8c1
--- /dev/null
+++ b/scripts/test/run-tests.sh
@@ -0,0 +1,83 @@
+#!/bin/bash
+
+set -e
+
+root_dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )/../../" && pwd )
+if [ "${root_dir}" != "${PWD}" ]; then
+  echo "You must execute $(basename $0) from the root directory: ${root_dir}"
+  exit 1
+fi
+
+WORKSPACE=./tests
+
+BBB_DEBIAN_IMAGE="bone-debian-9.5-iot-armhf-2018-10-07-4gb.img"
+BBB_DEBIAN_IMAGE_URL="http://debian.beagleboard.org/images/${BBB_DEBIAN_IMAGE}.xz"
+
+RASPBIAN_IMAGE="2019-04-08-raspbian-stretch-lite"
+RASPBIAN_IMAGE_URL="https://downloads.raspberrypi.org/raspbian_lite/images/raspbian_lite-2019-04-09/${RASPBIAN_IMAGE}.zip"
+
+TINKER_IMAGE="20170417-tinker-board-linaro-stretch-alip-v1.8"
+TINKER_IMAGE_URL="http://dlcdnet.asus.com/pub/ASUS/mb/Linux/Tinker_Board_2GB/${TINKER_IMAGE}.zip"
+
+UBUNTU_IMAGE="Ubuntu-Bionic-x86-64.img"
+UBUNTU_IMAGE_URL="https://d1b0l86ne08fsf.cloudfront.net/mender-convert/images/${UBUNTU_IMAGE}.gz"
+
+UBUNTU_SERVER_RPI_IMAGE="ubuntu-18.04.3-preinstalled-server-armhf+raspi3.img"
+UBUNTU_SERVER_RPI_IMAGE_URL="http://cdimage.ubuntu.com/ubuntu/releases/bionic/release/${UBUNTU_SERVER_RPI_IMAGE}.xz"
+
+# Keep common function declarations in separate utils script
+UTILS_PATH=${0/$(basename $0)/test-utils.sh}
+source $UTILS_PATH
+
+# Some distros do not have /sbin in path for "normal users"
+export PATH="${PATH}:/sbin"
+
+if [ ! -d ${WORKSPACE}/mender-image-tests ]; then
+  git clone https://github.com/mendersoftware/mender-image-tests ${WORKSPACE}/mender-image-tests
+else
+  cd ${WORKSPACE}/mender-image-tests
+  git pull
+  cd -
+fi
+
+mkdir -p ${WORKSPACE}
+
+get_pytest_files
+
+test_result=0
+
+convert_and_test "qemux86_64" \
+  "release-1" \
+  "${UBUNTU_IMAGE_URL}" \
+  "${UBUNTU_IMAGE}" \
+  "${UBUNTU_IMAGE}.gz" \
+  "configs/qemux86-64_config" || test_result=$?
+
+convert_and_test "raspberrypi" \
+  "release-1" \
+  "${RASPBIAN_IMAGE_URL}" \
+  "${RASPBIAN_IMAGE}.img" \
+  "${RASPBIAN_IMAGE}.zip" \
+  "configs/raspberrypi3_config" || test_result=$?
+
+# MEN-2809: Disabled due broken download link
+#convert_and_test "linaro-alip" \
+#                 "release-1" \
+#                 "${TINKER_IMAGE_URL}" \
+#                 "${TINKER_IMAGE}.img" \
+#                 "${TINKER_IMAGE}.zip" || test_result=$?
+
+convert_and_test "beaglebone" \
+  "release-1" \
+  "${BBB_DEBIAN_IMAGE_URL}" \
+  "${BBB_DEBIAN_IMAGE}" \
+  "${BBB_DEBIAN_IMAGE}.xz" || test_result=$?
+
+convert_and_test "ubuntu" \
+  "release-1" \
+  "${UBUNTU_SERVER_RPI_IMAGE_URL}" \
+  "${UBUNTU_SERVER_RPI_IMAGE}" \
+  "${UBUNTU_SERVER_RPI_IMAGE}.xz" \
+  "configs/raspberrypi3_config" || test_result=$?
+
+exit $test_result
diff --git a/scripts/run-tests.sh b/scripts/test/test-utils.sh
old mode 100755
new mode 100644
similarity index 57%
rename from scripts/run-tests.sh
rename to scripts/test/test-utils.sh
index cdef884..042f467
--- a/scripts/run-tests.sh
+++ b/scripts/test/test-utils.sh
@@ -1,38 +1,10 @@
-#!/bin/bash
-
-set -e
-
-root_dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )/../" && pwd )
-if [ "${root_dir}" != "${PWD}" ]; then
-  echo "You must execute $(basename $0) from the root directory: ${root_dir}"
-  exit 1
-fi
+MENDER_ACCEPTANCE_URL="https://raw.githubusercontent.com/mendersoftware/meta-mender/master/tests/acceptance"
 
-WORKSPACE=./tests
+WORKSPACE=${WORKSPACE:-./tests}
 
 # Relative to where scripts are executed (${WORKSPACE}/mender-image-tests)
 MENDER_CONVERT_DIR=../../
 
-BBB_DEBIAN_IMAGE="bone-debian-9.5-iot-armhf-2018-10-07-4gb.img"
-BBB_DEBIAN_IMAGE_URL="http://debian.beagleboard.org/images/${BBB_DEBIAN_IMAGE}.xz"
-
-RASPBIAN_IMAGE="2019-04-08-raspbian-stretch-lite"
-RASPBIAN_IMAGE_URL="https://downloads.raspberrypi.org/raspbian_lite/images/raspbian_lite-2019-04-09/${RASPBIAN_IMAGE}.zip"
-
-TINKER_IMAGE="20170417-tinker-board-linaro-stretch-alip-v1.8"
-TINKER_IMAGE_URL="http://dlcdnet.asus.com/pub/ASUS/mb/Linux/Tinker_Board_2GB/${TINKER_IMAGE}.zip"
-
-UBUNTU_IMAGE="Ubuntu-Bionic-x86-64.img"
-UBUNTU_IMAGE_URL="https://d1b0l86ne08fsf.cloudfront.net/mender-convert/images/${UBUNTU_IMAGE}.gz"
-
-UBUNTU_SERVER_RPI_IMAGE="ubuntu-18.04.3-preinstalled-server-armhf+raspi3.img"
-UBUNTU_SERVER_RPI_IMAGE_URL="http://cdimage.ubuntu.com/ubuntu/releases/bionic/release/${UBUNTU_SERVER_RPI_IMAGE}.xz"
-
-MENDER_ACCEPTANCE_URL="https://raw.githubusercontent.com/mendersoftware/meta-mender/master/tests/acceptance"
-
-# Some distros do not have /sbin in path for "normal users"
-export PATH="${PATH}:/sbin"
-
 convert_and_test() {
   device_type=$1
   artifact_name=$2
@@ -85,6 +57,16 @@ convert_and_test() {
     --disk-image input/${image_name} \
     ${MENDER_CONVERT_EXTRA_ARGS}
 
+  run_tests "${device_type}" "${artifact_name}"
+  return $?
+}
+
+run_tests() {
+  device_type=$1
+  artifact_name=$2
+  shift 2
+  pytest_args_extra=$@
+
   if pip3 list | grep -q -e pytest-html; then
     html_report_args="--html=${MENDER_CONVERT_DIR}/report_${device_type}.html --self-contained-html"
   fi
@@ -111,7 +93,8 @@ convert_and_test() {
     --board-type="${device_type}" \
     --mender-image=${device_type}-${artifact_name}.sdimg \
     --sdimg-location="${MENDER_CONVERT_DIR}/deploy" \
-    tests_${device_type}
+    tests_${device_type} \
+    ${pytest_args_extra}
   exitcode=$?
 
   cd -
@@ -126,52 +109,3 @@ get_pytest_files() {
   wget -N ${MENDER_ACCEPTANCE_URL}/fixtures.py -P $WORKSPACE/mender-image-tests
 }
 
-if [ ! -d ${WORKSPACE}/mender-image-tests ]; then
-  git clone https://github.com/mendersoftware/mender-image-tests ${WORKSPACE}/mender-image-tests
-else
-  cd ${WORKSPACE}/mender-image-tests
-  git pull
-  cd -
-fi
-
-mkdir -p ${WORKSPACE}
-
-get_pytest_files
-
-test_result=0
-
-convert_and_test "qemux86_64" \
-  "release-1" \
-  "${UBUNTU_IMAGE_URL}" \
-  "${UBUNTU_IMAGE}" \
-  "${UBUNTU_IMAGE}.gz" \
-  "configs/qemux86-64_config" || test_result=$?
-
-convert_and_test "raspberrypi" \
-  "release-1" \
-  "${RASPBIAN_IMAGE_URL}" \
-  "${RASPBIAN_IMAGE}.img" \
-  "${RASPBIAN_IMAGE}.zip" \
-  "configs/raspberrypi3_config" || test_result=$?
-
-# MEN-2809: Disabled due broken download link
-#convert_and_test "linaro-alip" \
-#                 "release-1" \
-#                 "${TINKER_IMAGE_URL}" \
-#                 "${TINKER_IMAGE}.img" \
-#                 "${TINKER_IMAGE}.zip" || test_result=$?
-
-convert_and_test "beaglebone" \
-  "release-1" \
-  "${BBB_DEBIAN_IMAGE_URL}" \
-  "${BBB_DEBIAN_IMAGE}" \
-  "${BBB_DEBIAN_IMAGE}.xz" || test_result=$?
-
-convert_and_test "ubuntu" \
-  "release-1" \
-  "${UBUNTU_SERVER_RPI_IMAGE_URL}" \
-  "${UBUNTU_SERVER_RPI_IMAGE}" \
-  "${UBUNTU_SERVER_RPI_IMAGE}.xz" \
-  "configs/raspberrypi3_config" || test_result=$?
-
-exit $test_result

From ab1ec219b5ce83055f9267e2618f273715c0d50b Mon Sep 17 00:00:00 2001
From: Kristian Amlie <kristian.amlie@northern.tech>
Date: Tue, 10 Dec 2019 13:12:08 +0100
Subject: [PATCH 09/16] tests: Make path definition absolute (a bit more
 predictable).

Changelog: None

Signed-off-by: Kristian Amlie <kristian.amlie@northern.tech>
---
 scripts/test/test-utils.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scripts/test/test-utils.sh b/scripts/test/test-utils.sh
index 042f467..72857ef 100644
--- a/scripts/test/test-utils.sh
+++ b/scripts/test/test-utils.sh
@@ -2,8 +2,7 @@ MENDER_ACCEPTANCE_URL="https://raw.githubusercontent.com/mendersoftware/meta-men
 
 WORKSPACE=${WORKSPACE:-./tests}
 
-# Relative to where scripts are executed (${WORKSPACE}/mender-image-tests)
-MENDER_CONVERT_DIR=../../
+MENDER_CONVERT_DIR=$PWD
 
 convert_and_test() {
   device_type=$1

From 0903a31e78b458ebd096b94fd5c90a93a6eefe37 Mon Sep 17 00:00:00 2001
From: Kristian Amlie <kristian.amlie@northern.tech>
Date: Wed, 11 Dec 2019 14:37:00 +0100
Subject: [PATCH 10/16] Move all tests into test_acceptance stage and use
 pre-converted image.

This also means using the new mender-gitlab-tmp-storage bucket to
transfer artifacts between stages, because the built in artifact
mechanism has a size cap that we are exceeding.

Changelog: None

Signed-off-by: Kristian Amlie <kristian.amlie@northern.tech>
---
 .gitlab-ci.yml            | 50 ++++++++++++++++++++-------------------
 scripts/test/run-tests.sh | 15 +++++-------
 2 files changed, 32 insertions(+), 33 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ec46d65..51de1b4 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -8,6 +8,8 @@ variables:
   RASPBIAN_URL: http://downloads.raspberrypi.org/raspbian_lite/images/raspbian_lite-2019-09-30/2019-09-26-raspbian-buster-lite.zip
   RASPBIAN_NAME: 2019-09-26-raspbian-buster-lite
 
+  DEBIAN_FRONTEND: noninteractive
+
 include:
   - project: 'Northern.tech/Mender/mendertesting'
     file: '.gitlab-ci-check-commits.yml'
@@ -17,8 +19,8 @@ include:
 stages:
   - test
   - build
-  - test_acceptance
   - convert
+  - test_acceptance
   - publish
 
 build:
@@ -35,25 +37,30 @@ build:
 
 test_acceptance:
   stage: test_acceptance
-  image: docker:18-dind
+  image: teracy/ubuntu:18.04-dind-18.09.9
+  services:
+    - docker:18-dind
   tags:
     - mender-qa-slave
   dependencies:
     - build
+    - convert_raspbian
   before_script:
-    # Start up Docker (DonD)
-    - /usr/local/bin/dockerd-entrypoint.sh &
-    - sleep 10
-    - export DOCKER_HOST="unix:///var/run/docker.sock"
-    - docker version
     # Install dependencies
-    - apk --update --no-cache add bash wget git util-linux mtools python3 py3-pip
-      gcc python3-dev libffi-dev lzo-dev libc-dev openssl-dev make sudo
+    - apt update
+    - apt install -qyy bash wget git util-linux mtools python3 python3-pip
+      gcc python3-dev libffi-dev liblzo2-dev libc-dev libssl-dev make sudo
+      awscli unzip
     # Python3 dependencies
     - pip3 install -r https://raw.githubusercontent.com/mendersoftware/meta-mender/master/tests/acceptance/requirements_py3.txt
     # Load image under test
     - export IMAGE_NAME=$DOCKER_REPOSITORY:pr
     - docker load -i image.tar
+    # Fetch artifacts from temporary S3 bucket
+    - aws s3 cp s3://mender-gitlab-tmp-storage/$CI_PROJECT_NAME/$CI_PIPELINE_ID/deploy.tar.gz deploy.tar.gz
+    - tar xzf deploy.tar.gz
+    # Extract converted Raspbian artifacts
+    - unxz deploy/raspberrypi-mender-raspbian.sdimg.xz
   script:
     - ./scripts/test/run-tests.sh
   artifacts:
@@ -75,7 +82,7 @@ convert_raspbian:
   dependencies:
     - build
   before_script:
-    - apt update && apt install -yy bash wget unzip
+    - apt update && apt install -yy bash wget unzip awscli
 
     - export IMAGE_NAME=$DOCKER_REPOSITORY:pr
     - docker load -i image.tar
@@ -90,20 +97,10 @@ convert_raspbian:
       -c configs/raspberrypi3_config
       -c configs/images/raspberrypi3_raspbian_config
       -c mender_client_version_config
-    - cp deploy/raspberrypi-mender-raspbian.sdimg.xz deploy/${RASPBIAN_NAME}-mender-${MENDER_CLIENT_VERSION}.img.xz
-    - unxz deploy/raspberrypi-mender-raspbian.sdimg.xz
-      # Verify converted image
-    - . ./scripts/test/test-utils.sh
-    - run_tests raspbian mender-raspbian "-k TestMenderInstalled"
-
-  after_script:
-    - mkdir -p output
-    - mv deploy/*.img.xz output
 
-  artifacts:
-    paths:
-      - output
-    expire_in: 2 weeks
+    # Upload to temporary S3 bucket
+    - tar czf deploy.tar.gz deploy
+    - aws s3 cp deploy.tar.gz s3://mender-gitlab-tmp-storage/$CI_PROJECT_NAME/$CI_PIPELINE_ID/deploy.tar.gz
 
 publish:s3:
   when: manual
@@ -111,9 +108,14 @@ publish:s3:
   image: debian:buster
   before_script:
     - apt update && apt install -yyq awscli
+
+    # Fetch artifacts from temporary S3 bucket
+    - aws s3 cp s3://mender-gitlab-tmp-storage/$CI_PROJECT_NAME/$CI_PIPELINE_ID/deploy.tar.gz deploy.tar.gz
+    - tar xzf deploy.tar.gz
+
   script:
     - echo "Publishing ${RASPBIAN_NAME}-mender-${MENDER_CLIENT_VERSION}.img.xz version to S3"
-    - aws s3 cp output/${RASPBIAN_NAME}-mender-${MENDER_CLIENT_VERSION}.img.xz
+    - aws s3 cp deploy/raspberrypi-mender-raspbian.sdimg.xz
         s3://$S3_BUCKET_NAME/${RASPBIAN_NAME}/arm/${RASPBIAN_NAME}-mender-${MENDER_CLIENT_VERSION}.img.xz
     - aws s3api put-object-acl --acl public-read --bucket $S3_BUCKET_NAME
         --key ${RASPBIAN_NAME}/arm/${RASPBIAN_NAME}-mender-${MENDER_CLIENT_VERSION}.img.xz
diff --git a/scripts/test/run-tests.sh b/scripts/test/run-tests.sh
index 197f8c1..30aaaca 100755
--- a/scripts/test/run-tests.sh
+++ b/scripts/test/run-tests.sh
@@ -13,9 +13,6 @@ WORKSPACE=./tests
 BBB_DEBIAN_IMAGE="bone-debian-9.5-iot-armhf-2018-10-07-4gb.img"
 BBB_DEBIAN_IMAGE_URL="http://debian.beagleboard.org/images/${BBB_DEBIAN_IMAGE}.xz"
 
-RASPBIAN_IMAGE="2019-04-08-raspbian-stretch-lite"
-RASPBIAN_IMAGE_URL="https://downloads.raspberrypi.org/raspbian_lite/images/raspbian_lite-2019-04-09/${RASPBIAN_IMAGE}.zip"
-
 TINKER_IMAGE="20170417-tinker-board-linaro-stretch-alip-v1.8"
 TINKER_IMAGE_URL="http://dlcdnet.asus.com/pub/ASUS/mb/Linux/Tinker_Board_2GB/${TINKER_IMAGE}.zip"
 
@@ -53,12 +50,12 @@ convert_and_test "qemux86_64" \
   "${UBUNTU_IMAGE}.gz" \
   "configs/qemux86-64_config" || test_result=$?
 
-convert_and_test "raspberrypi" \
-  "release-1" \
-  "${RASPBIAN_IMAGE_URL}" \
-  "${RASPBIAN_IMAGE}.img" \
-  "${RASPBIAN_IMAGE}.zip" \
-  "configs/raspberrypi3_config" || test_result=$?
+if [ -f deploy/raspberrypi-mender-raspbian.sdimg ]; then
+  run_tests "raspberrypi" "mender-raspbian" || test_result=$?
+else
+  echo "FAILED! This test needs a pre-converted Raspbian image. See the convert_raspbian job in the .gitlab-ci.yml file for how to generate, and then put the *uncompressed* image in the deploy folder"
+  test_result=1
+fi
 
 # MEN-2809: Disabled due broken download link
 #convert_and_test "linaro-alip" \

From 25e49cc8900091ec44f5d1c068b57edab6fb641e Mon Sep 17 00:00:00 2001
From: Kristian Amlie <kristian.amlie@northern.tech>
Date: Wed, 11 Dec 2019 16:03:06 +0100
Subject: [PATCH 11/16] Increase testing timeout to battle slow I/O.

Changelog: None

Signed-off-by: Kristian Amlie <kristian.amlie@northern.tech>
---
 .gitlab-ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 51de1b4..fc4af4a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -45,6 +45,7 @@ test_acceptance:
   dependencies:
     - build
     - convert_raspbian
+  timeout: 2h
   before_script:
     # Install dependencies
     - apt update

From 85774e88e4539f38e2f48e9aebb90c64de9d5851 Mon Sep 17 00:00:00 2001
From: Kristian Amlie <kristian.amlie@northern.tech>
Date: Wed, 11 Dec 2019 16:03:31 +0100
Subject: [PATCH 12/16] Delete converted images after testing to cut down on
 space usage.

Changelog: None

Signed-off-by: Kristian Amlie <kristian.amlie@northern.tech>
---
 scripts/test/test-utils.sh | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/scripts/test/test-utils.sh b/scripts/test/test-utils.sh
index 72857ef..4f31803 100644
--- a/scripts/test/test-utils.sh
+++ b/scripts/test/test-utils.sh
@@ -56,8 +56,12 @@ convert_and_test() {
     --disk-image input/${image_name} \
     ${MENDER_CONVERT_EXTRA_ARGS}
 
-  run_tests "${device_type}" "${artifact_name}"
-  return $?
+  local ret=0
+  run_tests "${device_type}" "${artifact_name}" || ret=$?
+
+  rm -f deploy/${device_type}-${artifact_name}.*
+
+  return $ret
 }
 
 run_tests() {

From 901e40944f7b565cace3c1a2a9ce0519cba35690 Mon Sep 17 00:00:00 2001
From: Kristian Amlie <kristian.amlie@northern.tech>
Date: Thu, 12 Dec 2019 10:06:18 +0100
Subject: [PATCH 13/16] Make MENDER_COMPRESS_DISK_IMAGE have same args as
 MENDER_ARTIFACT_COMPRESSION

Changelog: None

Signed-off-by: Kristian Amlie <kristian.amlie@northern.tech>
---
 configs/mender_convert_config | 3 +--
 mender-convert-package        | 4 ++--
 scripts/test/test-utils.sh    | 6 +++---
 3 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/configs/mender_convert_config b/configs/mender_convert_config
index 74f5615..9071ad1 100644
--- a/configs/mender_convert_config
+++ b/configs/mender_convert_config
@@ -11,10 +11,9 @@
 # machine, and obviously saves space.
 #
 # Possible values are:
-# 'n'    - No compression
+# 'none' - No compression
 # 'gzip' - Compress with gzip
 # 'lzma' - Compress with xz (LZMA)
-# 'y'    - Alias for 'gzip'
 MENDER_COMPRESS_DISK_IMAGE=gzip
 
 # Compression algorithm for Mender Artifact
diff --git a/mender-convert-package b/mender-convert-package
index 344d54b..f52e668 100755
--- a/mender-convert-package
+++ b/mender-convert-package
@@ -252,7 +252,7 @@ if [ "${MENDER_USE_BMAP}" == "y" ]; then
 fi
 
 case "${MENDER_COMPRESS_DISK_IMAGE}" in
-  y | gzip)
+  gzip)
     log_info "Compressing ${sdimg_path}.gz"
     run_and_log_cmd "pigz --best --force ${sdimg_path}"
     ;;
@@ -260,7 +260,7 @@ case "${MENDER_COMPRESS_DISK_IMAGE}" in
     log_info "Compressing ${sdimg_path}.xz"
     run_and_log_cmd "pxz --best --force ${sdimg_path}"
     ;;
-  n)
+  none)
     :
     ;;
   *)
diff --git a/scripts/test/test-utils.sh b/scripts/test/test-utils.sh
index 4f31803..bbc0ddc 100644
--- a/scripts/test/test-utils.sh
+++ b/scripts/test/test-utils.sh
@@ -43,12 +43,12 @@ convert_and_test() {
   # - test providing multiple '--config' options
   #
   # - (when no platform configuration is provided) test conversion without
-  #   '--config' and with MENDER_COMPRESS_DISK_IMAGE=y. Compressed disk
+  #   '--config' and with MENDER_COMPRESS_DISK_IMAGE=gzip. Compressed disk
   #   images is the default user facing option and we need to ensure that we
   #   cover this in the tests.
   if [ -n "${config}" ]; then
     echo "Will disable MENDER_COMPRESS_DISK_IMAGE for this image"
-    echo "MENDER_COMPRESS_DISK_IMAGE=n" > ${WORKSPACE}/test_config
+    echo "MENDER_COMPRESS_DISK_IMAGE=none" > ${WORKSPACE}/test_config
     local MENDER_CONVERT_EXTRA_ARGS="--config ${config} --config ${WORKSPACE}/test_config"
   fi
 
@@ -74,7 +74,7 @@ run_tests() {
     html_report_args="--html=${MENDER_CONVERT_DIR}/report_${device_type}.html --self-contained-html"
   fi
 
-  # Need to decompress images built with MENDER_COMPRESS_DISK_IMAGE=y before
+  # Need to decompress images built with MENDER_COMPRESS_DISK_IMAGE=gzip before
   # running tests.
   if [ -f deploy/${device_type}-${artifact_name}.sdimg.gz ]; then
     # sudo is needed because the image is created using docker-mender-convert

From 3cb78bf17cf852469c44aa9c8dfa7fd9dedd314b Mon Sep 17 00:00:00 2001
From: Kristian Amlie <kristian.amlie@northern.tech>
Date: Thu, 12 Dec 2019 13:54:32 +0100
Subject: [PATCH 14/16] Switch run-test to accept --prebuilt-image parameters.

This avoid problems when running the tests locally. Technically it
will run the Raspberry Pi test twice on Gitlab, once with the prebuilt
image, and once with a test image, but the two configurations are not
identical, so I reckon that's alright.

Changelog: None

Signed-off-by: Kristian Amlie <kristian.amlie@northern.tech>
---
 .gitlab-ci.yml            |  9 ++--
 scripts/test/run-tests.sh | 96 +++++++++++++++++++++++++--------------
 2 files changed, 66 insertions(+), 39 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index fc4af4a..3b7db07 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -61,9 +61,10 @@ test_acceptance:
     - aws s3 cp s3://mender-gitlab-tmp-storage/$CI_PROJECT_NAME/$CI_PIPELINE_ID/deploy.tar.gz deploy.tar.gz
     - tar xzf deploy.tar.gz
     # Extract converted Raspbian artifacts
-    - unxz deploy/raspberrypi-mender-raspbian.sdimg.xz
+    - unxz deploy/raspberrypi-${RASPBIAN_NAME}-mender-${MENDER_CLIENT_VERSION}.sdimg.xz
   script:
-    - ./scripts/test/run-tests.sh
+    - ./scripts/test/run-tests.sh --prebuilt-image raspberrypi ${RASPBIAN_NAME}-mender-${MENDER_CLIENT_VERSION}
+    - ./scripts/test/run-tests.sh --all
   artifacts:
     expire_in: 2w
     when: always
@@ -93,7 +94,7 @@ convert_raspbian:
 
   script:
     - echo "MENDER_CLIENT_VERSION=${MENDER_CLIENT_VERSION}" > mender_client_version_config
-    - env MENDER_ARTIFACT_NAME=mender-raspbian
+    - env MENDER_ARTIFACT_NAME=${RASPBIAN_NAME}-mender-${MENDER_CLIENT_VERSION}
       ./docker-mender-convert -d ${RASPBIAN_NAME}.img
       -c configs/raspberrypi3_config
       -c configs/images/raspberrypi3_raspbian_config
@@ -116,7 +117,7 @@ publish:s3:
 
   script:
     - echo "Publishing ${RASPBIAN_NAME}-mender-${MENDER_CLIENT_VERSION}.img.xz version to S3"
-    - aws s3 cp deploy/raspberrypi-mender-raspbian.sdimg.xz
+    - aws s3 cp deploy/raspberrypi-${RASPBIAN_NAME}-mender-${MENDER_CLIENT_VERSION}.sdimg.xz
         s3://$S3_BUCKET_NAME/${RASPBIAN_NAME}/arm/${RASPBIAN_NAME}-mender-${MENDER_CLIENT_VERSION}.img.xz
     - aws s3api put-object-acl --acl public-read --bucket $S3_BUCKET_NAME
         --key ${RASPBIAN_NAME}/arm/${RASPBIAN_NAME}-mender-${MENDER_CLIENT_VERSION}.img.xz
diff --git a/scripts/test/run-tests.sh b/scripts/test/run-tests.sh
index 30aaaca..0f493f6 100755
--- a/scripts/test/run-tests.sh
+++ b/scripts/test/run-tests.sh
@@ -2,6 +2,11 @@
 
 set -e
 
+usage() {
+  echo "$0 <--all | --prebuilt-image DEVICE_TYPE IMAGE_NAME>"
+  exit 1
+}
+
 root_dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )/../../" && pwd )
 if [ "${root_dir}" != "${PWD}" ]; then
   echo "You must execute $(basename $0) from the root directory: ${root_dir}"
@@ -13,6 +18,9 @@ WORKSPACE=./tests
 BBB_DEBIAN_IMAGE="bone-debian-9.5-iot-armhf-2018-10-07-4gb.img"
 BBB_DEBIAN_IMAGE_URL="http://debian.beagleboard.org/images/${BBB_DEBIAN_IMAGE}.xz"
 
+RASPBIAN_IMAGE="2019-09-26-raspbian-buster-lite"
+RASPBIAN_IMAGE_URL="http://downloads.raspberrypi.org/raspbian_lite/images/raspbian_lite-2019-09-30/2019-09-26-raspbian-buster-lite.zip"
+
 TINKER_IMAGE="20170417-tinker-board-linaro-stretch-alip-v1.8"
 TINKER_IMAGE_URL="http://dlcdnet.asus.com/pub/ASUS/mb/Linux/Tinker_Board_2GB/${TINKER_IMAGE}.zip"
 
@@ -43,38 +51,56 @@ get_pytest_files
 
 test_result=0
 
-convert_and_test "qemux86_64" \
-  "release-1" \
-  "${UBUNTU_IMAGE_URL}" \
-  "${UBUNTU_IMAGE}" \
-  "${UBUNTU_IMAGE}.gz" \
-  "configs/qemux86-64_config" || test_result=$?
-
-if [ -f deploy/raspberrypi-mender-raspbian.sdimg ]; then
-  run_tests "raspberrypi" "mender-raspbian" || test_result=$?
-else
-  echo "FAILED! This test needs a pre-converted Raspbian image. See the convert_raspbian job in the .gitlab-ci.yml file for how to generate, and then put the *uncompressed* image in the deploy folder"
-  test_result=1
-fi
-
-# MEN-2809: Disabled due broken download link
-#convert_and_test "linaro-alip" \
-#                 "release-1" \
-#                 "${TINKER_IMAGE_URL}" \
-#                 "${TINKER_IMAGE}.img" \
-#                 "${TINKER_IMAGE}.zip" || test_result=$?
-
-convert_and_test "beaglebone" \
-  "release-1" \
-  "${BBB_DEBIAN_IMAGE_URL}" \
-  "${BBB_DEBIAN_IMAGE}" \
-  "${BBB_DEBIAN_IMAGE}.xz" || test_result=$?
-
-convert_and_test "ubuntu" \
-  "release-1" \
-  "${UBUNTU_SERVER_RPI_IMAGE_URL}" \
-  "${UBUNTU_SERVER_RPI_IMAGE}" \
-  "${UBUNTU_SERVER_RPI_IMAGE}.xz" \
-  "configs/raspberrypi3_config" || test_result=$?
-
-exit $test_result
+case "$1" in
+  --prebuilt-image)
+    if [ -z "$3" ]; then
+      echo "Both DEVICE_TYPE and IMAGE_NAME must be specified"
+      exit 1
+    fi
+    test_result=0
+    run_tests "$2" "$3" || test_result=$?
+    exit $test_result
+    ;;
+
+  --all)
+    convert_and_test "qemux86_64" \
+                     "release-1" \
+                     "${UBUNTU_IMAGE_URL}" \
+                     "${UBUNTU_IMAGE}" \
+                     "${UBUNTU_IMAGE}.gz" \
+                     "configs/qemux86-64_config" || test_result=$?
+
+    convert_and_test "raspberrypi" \
+                     "release-1" \
+                     "${RASPBIAN_IMAGE_URL}" \
+                     "${RASPBIAN_IMAGE}.img" \
+                     "${RASPBIAN_IMAGE}.zip" \
+                     "configs/raspberrypi3_config" || test_result=$?
+
+    # MEN-2809: Disabled due broken download link
+    #convert_and_test "linaro-alip" \
+    #                 "release-1" \
+    #                 "${TINKER_IMAGE_URL}" \
+    #                 "${TINKER_IMAGE}.img" \
+    #                 "${TINKER_IMAGE}.zip" || test_result=$?
+
+    convert_and_test "beaglebone" \
+                     "release-1" \
+                     "${BBB_DEBIAN_IMAGE_URL}" \
+                     "${BBB_DEBIAN_IMAGE}" \
+                     "${BBB_DEBIAN_IMAGE}.xz" || test_result=$?
+
+    convert_and_test "ubuntu" \
+                     "release-1" \
+                     "${UBUNTU_SERVER_RPI_IMAGE_URL}" \
+                     "${UBUNTU_SERVER_RPI_IMAGE}" \
+                     "${UBUNTU_SERVER_RPI_IMAGE}.xz" \
+                     "configs/raspberrypi3_config" || test_result=$?
+
+    exit $test_result
+    ;;
+
+  *)
+    usage
+    ;;
+esac

From 351973f0b8589297cabe5db4d93ddf69efb2e79c Mon Sep 17 00:00:00 2001
From: Kristian Amlie <kristian.amlie@northern.tech>
Date: Thu, 12 Dec 2019 15:39:17 +0100
Subject: [PATCH 15/16] Make test XML results unique by putting in different
 folders.

Changelog: None

Signed-off-by: Kristian Amlie <kristian.amlie@northern.tech>
---
 scripts/test/test-utils.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/test/test-utils.sh b/scripts/test/test-utils.sh
index bbc0ddc..f4f323d 100644
--- a/scripts/test/test-utils.sh
+++ b/scripts/test/test-utils.sh
@@ -86,7 +86,7 @@ run_tests() {
 
   # This is a trick to make pytest generate different junit reports
   # for different runs: renaming the tests folder to tests_<testsuite>
-  cp -r tests tests_${device_type}
+  cp -r tests tests_${device_type}_${artifact_name}
 
   python3 -m pytest --verbose \
     --junit-xml="${MENDER_CONVERT_DIR}/results_${device_type}.xml" \
@@ -96,7 +96,7 @@ run_tests() {
     --board-type="${device_type}" \
     --mender-image=${device_type}-${artifact_name}.sdimg \
     --sdimg-location="${MENDER_CONVERT_DIR}/deploy" \
-    tests_${device_type} \
+    tests_${device_type}_${artifact_name} \
     ${pytest_args_extra}
   exitcode=$?
 

From 72cb5e227df247b4fd771f65f78e9f8ab3d734be Mon Sep 17 00:00:00 2001
From: Kristian Amlie <kristian.amlie@northern.tech>
Date: Thu, 12 Dec 2019 17:01:27 +0100
Subject: [PATCH 16/16] Use lower privilege AWS keys for the tmp storage.

Changelog: None

Signed-off-by: Kristian Amlie <kristian.amlie@northern.tech>
---
 .gitlab-ci.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 3b7db07..210ced1 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -117,6 +117,9 @@ publish:s3:
 
   script:
     - echo "Publishing ${RASPBIAN_NAME}-mender-${MENDER_CLIENT_VERSION}.img.xz version to S3"
+    # Prepare high privilege S3 keys (the base keys are for the tmp storage only)
+    - export AWS_ACCESS_KEY_ID=$PUBLISH_AWS_ACCESS_KEY_ID
+    - export AWS_SECRET_ACCESS_KEY=$PUBLISH_AWS_SECRET_ACCESS_KEY
     - aws s3 cp deploy/raspberrypi-${RASPBIAN_NAME}-mender-${MENDER_CLIENT_VERSION}.sdimg.xz
         s3://$S3_BUCKET_NAME/${RASPBIAN_NAME}/arm/${RASPBIAN_NAME}-mender-${MENDER_CLIENT_VERSION}.img.xz
     - aws s3api put-object-acl --acl public-read --bucket $S3_BUCKET_NAME