test: Make tests parallelizable and update to parallel image-tests.

Ticket: QA-499

Signed-off-by: Kristian Amlie <kristian.amlie@northern.tech>

.gitlab-ci.yml

@@ -25,6 +25,8 @@ variables:
   # Whether to publish packages automatically - they can always be published manually
   PUBLISH_MENDER_CONVERT_AUTOMATIC: "false"
 
+  TESTS_IN_PARALLEL_CLIENT_ACCEPTANCE: "2"
+
   DEBIAN_FRONTEND: noninteractive
 
   # Docker dind configuration.
@@ -229,7 +231,11 @@ convert_raspbian_raspberrypi4:
     - mv ${RASPBERRYPI_PLATFORM} deploy
     # Extract converted Raspbian artifacts
     - unxz deploy/${RASPBIAN_NAME}-${RASPBERRYPI_PLATFORM}-mender.img.xz
-    - ./scripts/test/run-tests.sh --prebuilt-image raspberrypi ${RASPBIAN_NAME}-${RASPBERRYPI_PLATFORM}-mender
+    - ./scripts/test/run-tests.sh
+      --prebuilt-image raspberrypi
+      ${RASPBIAN_NAME}-${RASPBERRYPI_PLATFORM}-mender
+      --
+      -n $TESTS_IN_PARALLEL_CLIENT_ACCEPTANCE
 
 test_acceptance_prebuilt_raspberrypi3:
   <<: *test_acceptance_prebuilt_raspberrypi
@@ -256,42 +262,66 @@ test_acceptance_ubuntu_qemux86_64:
   script:
     - mkdir -p input/config
     - cp versions_override_config input/config/versions_override_config
-    - ./scripts/test/run-tests.sh --config input/config/versions_override_config --only ubuntu-qemux86-64
+    - ./scripts/test/run-tests.sh
+      --config input/config/versions_override_config
+      --only ubuntu-qemux86-64
+      --
+      -n $TESTS_IN_PARALLEL_CLIENT_ACCEPTANCE
 
 test_acceptance_ubuntu_qemux86_64_no_grub_d:
   <<: *test_acceptance
   script:
     - mkdir -p input/config
     - cp versions_override_config input/config/versions_override_config
-    - ./scripts/test/run-tests.sh --config input/config/versions_override_config --only ubuntu-qemux86-64-no-grub-d
+    - ./scripts/test/run-tests.sh
+      --config input/config/versions_override_config
+      --only ubuntu-qemux86-64-no-grub-d
+      --
+      -n $TESTS_IN_PARALLEL_CLIENT_ACCEPTANCE
 
 test_acceptance_debian_qemux86_64:
   <<: *test_acceptance
   script:
     - mkdir -p input/config
     - cp versions_override_config input/config/versions_override_config
-    - ./scripts/test/run-tests.sh --config input/config/versions_override_config --only debian-qemux86-64
+    - ./scripts/test/run-tests.sh
+      --config input/config/versions_override_config
+      --only debian-qemux86-64
+      --
+      -n $TESTS_IN_PARALLEL_CLIENT_ACCEPTANCE
 
 test_acceptance_raspberrypi:
   <<: *test_acceptance
   script:
     - mkdir -p input/config
     - cp versions_override_config input/config/versions_override_config
-    - ./scripts/test/run-tests.sh --config input/config/versions_override_config --only raspberrypi3
+    - ./scripts/test/run-tests.sh
+      --config input/config/versions_override_config
+      --only raspberrypi3
+      --
+      -n $TESTS_IN_PARALLEL_CLIENT_ACCEPTANCE
 
 test_acceptance_beaglebone:
   <<: *test_acceptance
   script:
     - mkdir -p input/config
     - cp versions_override_config input/config/versions_override_config
-    - ./scripts/test/run-tests.sh --config input/config/versions_override_config --only beaglebone
+    - ./scripts/test/run-tests.sh
+      --config input/config/versions_override_config
+      --only beaglebone
+      --
+      -n $TESTS_IN_PARALLEL_CLIENT_ACCEPTANCE
 
 test_acceptance_ubuntu_raspberrypi3:
   <<: *test_acceptance
   script:
     - mkdir -p input/config
     - cp versions_override_config input/config/versions_override_config
-    - ./scripts/test/run-tests.sh --config input/config/versions_override_config --only ubuntu-raspberrypi3
+    - ./scripts/test/run-tests.sh
+      --config input/config/versions_override_config
+      --only ubuntu-raspberrypi3
+      --
+      -n $TESTS_IN_PARALLEL_CLIENT_ACCEPTANCE
 
 .template:publish:s3:
   stage: publish
@@ -432,5 +462,7 @@ test:hardware:acceptance:
       --prebuilt-image
       raspberrypi4
       "${RASPBIAN_NAME}-${RASPBERRYPI_PLATFORM}-mender
+      --
       --hardware-testing
       --host ${SSH_JUMP_HOST_IP}:${SSH_JUMP_HOST_PORT} mender-image-tests -k 'not test_network_based_image_update and not test_image_update_broken_kernel'"
+      -n 1

scripts/test/mender-convert-qemu

@@ -1,5 +1,5 @@
 #!/bin/sh
-# Copyright 2022 Northern.tech AS
+# Copyright 2023 Northern.tech AS
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -35,7 +35,7 @@ qemu-system-x86_64 \
     -enable-kvm \
     -nographic \
     -m 512 \
-    -net user,hostfwd=tcp::8822-:22 \
+    -net user,hostfwd=tcp::${PORT_NUMBER:-8822}-:22 \
     -net nic,macaddr=52:54:00$(od -txC -An -N3 /dev/urandom|tr \  :) \
     -drive file=${ovmf_file},if=pflash,format=raw,unit=0,readonly=on \
     $OVMF_VARS \

scripts/test/test-utils.sh

@@ -1,4 +1,4 @@
-# Copyright 2022 Northern.tech AS
+# Copyright 2023 Northern.tech AS
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -155,8 +155,8 @@ run_tests() {
     --board-type="${device_type}" \
     --mender-image="${converted_image_name}.sdimg" \
     --sdimg-location="${MENDER_CONVERT_DIR}/deploy" \
-    --ssh-priv-key="./ssh-priv-key/key" \
-    --qemu-wrapper="../scripts/test/mender-convert-qemu" \
+    --ssh-priv-key="${MENDER_CONVERT_DIR}/tests/ssh-priv-key/key" \
+    --qemu-wrapper="${MENDER_CONVERT_DIR}/scripts/test/mender-convert-qemu" \
     ${pytest_extra_args}
 
   exitcode=$?

tests/mender-image-tests

@@ -1 +1 @@
-Subproject commit e40f05676df9b0cbe82026e2768ee6d1fe961ed2
+Subproject commit 85a184c913c0bed74e8b23a079edb21995a76881

tests/pytest.ini

@@ -1,2 +0,0 @@
-[pytest]
-addopts = --capture=no

tests/requirements_py3.txt

@@ -19,6 +19,7 @@ pyparsing==3.0.9
 pytest==7.2.0
 pytest-html==3.2.0
 pytest-metadata==2.0.4
+pytest-xdist==2.5.0
 python-lzo==1.14
 PyYAML==6.0
 redo==2.0.4

tests/test_grub_integration.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# Copyright 2022 Northern.tech AS
+# Copyright 2023 Northern.tech AS
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -21,6 +21,7 @@ import subprocess
 from utils.common import (
     extract_partition,
     get_no_sftp,
+    make_tempdir,
 )
 
 
@@ -93,57 +94,40 @@ class TestGrubIntegration:
         partition."""
 
         # First, check that the offline generated scripts don't have any.
-        extract_partition(latest_part_image, 1)
-        try:
+        with make_tempdir() as tmpdir:
+            extract_partition(latest_part_image, 1, tmpdir)
             subprocess.check_call(
-                ["mcopy", "-i", "img1.fs", "::/grub-mender-grubenv/grub.cfg", "."]
+                ["mcopy", "-i", f"{tmpdir}/img1.fs", "::/grub-mender-grubenv/grub.cfg", tmpdir]
             )
-            check_all_root_occurrences_valid("grub.cfg")
-        finally:
-            os.remove("img1.fs")
-            os.remove("grub.cfg")
+            check_all_root_occurrences_valid(f"{tmpdir}/grub.cfg")
 
-        extract_partition(latest_part_image, 2)
-        try:
+            extract_partition(latest_part_image, 2, tmpdir)
             subprocess.check_call(
                 [
                     "debugfs",
                     "-R",
-                    "dump -p /boot/grub-mender-grubenv.cfg grub-mender-grubenv.cfg",
-                    "img2.fs",
+                    f"dump -p /boot/grub-mender-grubenv.cfg {tmpdir}/grub-mender-grubenv.cfg",
+                    f"{tmpdir}/img2.fs",
                 ]
             )
-            check_all_root_occurrences_valid("grub-mender-grubenv.cfg")
-        finally:
-            os.remove("img2.fs")
-            os.remove("grub-mender-grubenv.cfg")
+            check_all_root_occurrences_valid(f"{tmpdir}/grub-mender-grubenv.cfg")
 
         # Then, check that the runtime generated scripts don't have any.
-        get_no_sftp("/boot/grub/grub.cfg", connection)
-        try:
-            check_all_root_occurrences_valid("grub.cfg")
-        finally:
-            os.remove("grub.cfg")
+        with make_tempdir() as tmpdir:
+            get_no_sftp("/boot/grub/grub.cfg", connection, local=tmpdir)
+            check_all_root_occurrences_valid(f"{tmpdir}/grub.cfg")
 
-        get_no_sftp("/boot/grub-mender-grubenv.cfg", connection)
-        try:
-            check_all_root_occurrences_valid("grub-mender-grubenv.cfg")
-        finally:
-            os.remove("grub-mender-grubenv.cfg")
+            get_no_sftp("/boot/grub-mender-grubenv.cfg", connection, local=tmpdir)
+            check_all_root_occurrences_valid(f"{tmpdir}/grub-mender-grubenv.cfg")
 
         # Check again after running `update-grub`.
         connection.run("grub-install && update-grub")
-        get_no_sftp("/boot/grub/grub.cfg", connection)
-        try:
-            check_all_root_occurrences_valid("grub.cfg")
-        finally:
-            os.remove("grub.cfg")
+        with make_tempdir() as tmpdir:
+            get_no_sftp("/boot/grub/grub.cfg", connection, local=tmpdir)
+            check_all_root_occurrences_valid(f"{tmpdir}/grub.cfg")
 
-        get_no_sftp("/boot/grub-mender-grubenv.cfg", connection)
-        try:
-            check_all_root_occurrences_valid("grub-mender-grubenv.cfg")
-        finally:
-            os.remove("grub-mender-grubenv.cfg")
+            get_no_sftp("/boot/grub-mender-grubenv.cfg", connection, local=tmpdir)
+            check_all_root_occurrences_valid(f"{tmpdir}/grub-mender-grubenv.cfg")
 
     @pytest.mark.min_mender_version("1.0.0")
     def test_offline_and_runtime_boot_scripts_identical(self, connection):