Merge pull request #530 from kacf/parallel_testing

test: Make tests parallelizable and update to parallel image-tests.

.gitlab-ci.yml

@@ -25,6 +25,8 @@ variables:
   # Whether to publish packages automatically - they can always be published manually
   PUBLISH_MENDER_CONVERT_AUTOMATIC: "false"
 
+  TESTS_IN_PARALLEL_CLIENT_ACCEPTANCE: "4"
+
   DEBIAN_FRONTEND: noninteractive
 
   # Docker dind configuration.
@@ -229,7 +231,11 @@ convert_raspbian_raspberrypi4:
     - mv ${RASPBERRYPI_PLATFORM} deploy
     # Extract converted Raspbian artifacts
     - unxz deploy/${RASPBIAN_NAME}-${RASPBERRYPI_PLATFORM}-mender.img.xz
-    - ./scripts/test/run-tests.sh --prebuilt-image raspberrypi ${RASPBIAN_NAME}-${RASPBERRYPI_PLATFORM}-mender
+    - ./scripts/test/run-tests.sh
+      --prebuilt-image raspberrypi
+      ${RASPBIAN_NAME}-${RASPBERRYPI_PLATFORM}-mender
+      --
+      -n $TESTS_IN_PARALLEL_CLIENT_ACCEPTANCE
 
 test_acceptance_prebuilt_raspberrypi3:
   <<: *test_acceptance_prebuilt_raspberrypi
@@ -256,42 +262,66 @@ test_acceptance_ubuntu_qemux86_64:
   script:
     - mkdir -p input/config
     - cp versions_override_config input/config/versions_override_config
-    - ./scripts/test/run-tests.sh --config input/config/versions_override_config --only ubuntu-qemux86-64
+    - ./scripts/test/run-tests.sh
+      --config input/config/versions_override_config
+      --only ubuntu-qemux86-64
+      --
+      -n $TESTS_IN_PARALLEL_CLIENT_ACCEPTANCE
 
 test_acceptance_ubuntu_qemux86_64_no_grub_d:
   <<: *test_acceptance
   script:
     - mkdir -p input/config
     - cp versions_override_config input/config/versions_override_config
-    - ./scripts/test/run-tests.sh --config input/config/versions_override_config --only ubuntu-qemux86-64-no-grub-d
+    - ./scripts/test/run-tests.sh
+      --config input/config/versions_override_config
+      --only ubuntu-qemux86-64-no-grub-d
+      --
+      -n $TESTS_IN_PARALLEL_CLIENT_ACCEPTANCE
 
 test_acceptance_debian_qemux86_64:
   <<: *test_acceptance
   script:
     - mkdir -p input/config
     - cp versions_override_config input/config/versions_override_config
-    - ./scripts/test/run-tests.sh --config input/config/versions_override_config --only debian-qemux86-64
+    - ./scripts/test/run-tests.sh
+      --config input/config/versions_override_config
+      --only debian-qemux86-64
+      --
+      -n $TESTS_IN_PARALLEL_CLIENT_ACCEPTANCE
 
 test_acceptance_raspberrypi:
   <<: *test_acceptance
   script:
     - mkdir -p input/config
     - cp versions_override_config input/config/versions_override_config
-    - ./scripts/test/run-tests.sh --config input/config/versions_override_config --only raspberrypi3
+    - ./scripts/test/run-tests.sh
+      --config input/config/versions_override_config
+      --only raspberrypi3
+      --
+      -n $TESTS_IN_PARALLEL_CLIENT_ACCEPTANCE
 
 test_acceptance_beaglebone:
   <<: *test_acceptance
   script:
     - mkdir -p input/config
     - cp versions_override_config input/config/versions_override_config
-    - ./scripts/test/run-tests.sh --config input/config/versions_override_config --only beaglebone
+    - ./scripts/test/run-tests.sh
+      --config input/config/versions_override_config
+      --only beaglebone
+      --
+      -n $TESTS_IN_PARALLEL_CLIENT_ACCEPTANCE
 
 test_acceptance_ubuntu_raspberrypi3:
   <<: *test_acceptance
   script:
     - mkdir -p input/config
     - cp versions_override_config input/config/versions_override_config
-    - ./scripts/test/run-tests.sh --config input/config/versions_override_config --only ubuntu-raspberrypi3
+    - ./scripts/test/run-tests.sh
+      --config input/config/versions_override_config
+      --only ubuntu-raspberrypi3
+      --
+      -n $TESTS_IN_PARALLEL_CLIENT_ACCEPTANCE
 
 .template:publish:s3:
   stage: publish
@@ -432,5 +462,7 @@ test:hardware:acceptance:
       --prebuilt-image
       raspberrypi4
       "${RASPBIAN_NAME}-${RASPBERRYPI_PLATFORM}-mender
+      --
       --hardware-testing
       --host ${SSH_JUMP_HOST_IP}:${SSH_JUMP_HOST_PORT} mender-image-tests -k 'not test_network_based_image_update and not test_image_update_broken_kernel'"
+      -n 1

scripts/test/mender-convert-qemu

@@ -1,5 +1,5 @@
 #!/bin/sh
-# Copyright 2022 Northern.tech AS
+# Copyright 2023 Northern.tech AS
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -35,7 +35,7 @@ qemu-system-x86_64 \
     -enable-kvm \
     -nographic \
     -m 512 \
-    -net user,hostfwd=tcp::8822-:22 \
+    -net user,hostfwd=tcp::${PORT_NUMBER:-8822}-:22 \
     -net nic,macaddr=52:54:00$(od -txC -An -N3 /dev/urandom|tr \  :) \
     -drive file=${ovmf_file},if=pflash,format=raw,unit=0,readonly=on \
     $OVMF_VARS \

scripts/test/run-tests.sh

@@ -1,4 +1,4 @@
-# Copyright 2022 Northern.tech AS
+# Copyright 2023 Northern.tech AS
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -15,7 +15,7 @@
 set -e
 
 usage() {
-  echo "$0 [--config EXTRA_CONFIG_FILE] <--all | --only DEVICE_TYPE | --prebuilt-image DEVICE_TYPE IMAGE_NAME>"
+  echo "$0 [--config EXTRA_CONFIG_FILE] <--all | --only DEVICE_TYPE | --prebuilt-image DEVICE_TYPE IMAGE_NAME> [-- <pytest-options>]"
   exit 1
 }
 
@@ -87,6 +87,10 @@ while [ -n "$1" ]; do
       EXTRA_CONFIG="$EXTRA_CONFIG --config $2"
       shift
       ;;
+    --)
+      shift
+      break
+      ;;
   esac
   shift
 done
@@ -94,7 +98,7 @@ done
 test_result=0
 
 if [ -n "$PREBUILT_IMAGE" ]; then
-  run_tests $PREBUILT_IMAGE \
+  run_tests $PREBUILT_IMAGE "$@" \
             || test_result=$?
   exit $test_result
 
@@ -108,6 +112,8 @@ else
                      "input/image/Ubuntu-Focal-x86-64.img.gz" \
                      "--overlay input/tests/ssh-public-key-overlay" \
                      "--config configs/ubuntu-qemux86-64_config $EXTRA_CONFIG" \
+                     "--" \
+                     "$@" \
                      || test_result=$?
 
     echo >&2 "----------------------------------------"
@@ -135,6 +141,8 @@ else
                      "--overlay input/tests/ssh-public-key-overlay" \
                      "--config configs/ubuntu-qemux86-64_config" \
                      "--config configs/testing/no-grub.d_config $EXTRA_CONFIG" \
+                     "--" \
+                     "$@" \
                      || test_result=$?
   fi
 
@@ -145,6 +153,8 @@ else
                      "release-1" \
                      "input/image/${RASPBIAN_IMAGE}" \
                      "--config configs/raspberrypi3_config $EXTRA_CONFIG" \
+                     "--" \
+                     "$@" \
                      || test_result=$?
   fi
 
@@ -158,6 +168,8 @@ else
                      "release-1" \
                      "input/image/${BBB_DEBIAN_SDCARD_IMAGE_UNCOMPRESSED}" \
                      "--config configs/beaglebone_black_debian_sdcard_config $EXTRA_CONFIG" \
+                     "--" \
+                     "$@" \
                      || test_result=$?
 
     rm -rf deploy
@@ -169,6 +181,8 @@ else
                      "release-1" \
                      "input/image/${BBB_DEBIAN_EMMC_IMAGE_UNCOMPRESSED}" \
                      "--config configs/beaglebone_black_debian_emmc_config $EXTRA_CONFIG" \
+                     "--" \
+                     "$@" \
                      || test_result=$?
   fi
 
@@ -179,6 +193,8 @@ else
                      "release-1" \
                      "input/image/${UBUNTU_SERVER_RPI_IMAGE_COMPRESSED}" \
                      "--config configs/raspberrypi3_config $EXTRA_CONFIG" \
+                     "--" \
+                     "$@" \
                      || test_result=$?
   fi
 
@@ -191,6 +207,8 @@ else
                      "input/image/Debian-11-x86-64.img.gz" \
                      "--overlay input/tests/ssh-public-key-overlay" \
                      "--config configs/debian-qemux86-64_config $EXTRA_CONFIG" \
+                     "--" \
+                     "$@" \
                      || test_result=$?
   fi
 

scripts/test/test-utils.sh

@@ -1,4 +1,4 @@
-# Copyright 2022 Northern.tech AS
+# Copyright 2023 Northern.tech AS
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -155,8 +155,8 @@ run_tests() {
     --board-type="${device_type}" \
     --mender-image="${converted_image_name}.sdimg" \
     --sdimg-location="${MENDER_CONVERT_DIR}/deploy" \
-    --ssh-priv-key="./ssh-priv-key/key" \
+    --ssh-priv-key="${MENDER_CONVERT_DIR}/tests/ssh-priv-key/key" \
-    --qemu-wrapper="../scripts/test/mender-convert-qemu" \
+    --qemu-wrapper="${MENDER_CONVERT_DIR}/scripts/test/mender-convert-qemu" \
     ${pytest_extra_args}
 
   exitcode=$?

tests/mender-image-tests

@@ -1 +1 @@
-Subproject commit e40f05676df9b0cbe82026e2768ee6d1fe961ed2
+Subproject commit 85a184c913c0bed74e8b23a079edb21995a76881

tests/pytest.ini

@@ -1,2 +0,0 @@
-[pytest]
-addopts = --capture=no

tests/requirements_py3.txt

@@ -19,6 +19,7 @@ pyparsing==3.0.9
 pytest==7.2.0
 pytest-html==3.2.0
 pytest-metadata==2.0.4
+pytest-xdist==2.5.0
 python-lzo==1.14
 PyYAML==6.0
 redo==2.0.4

tests/test_grub_integration.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# Copyright 2022 Northern.tech AS
+# Copyright 2023 Northern.tech AS
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -21,6 +21,7 @@ import subprocess
 from utils.common import (
     extract_partition,
     get_no_sftp,
+    make_tempdir,
 )
 
 
@@ -93,57 +94,40 @@ class TestGrubIntegration:
         partition."""
 
         # First, check that the offline generated scripts don't have any.
-        extract_partition(latest_part_image, 1)
+        with make_tempdir() as tmpdir:
-        try:
+            extract_partition(latest_part_image, 1, tmpdir)
             subprocess.check_call(
-                ["mcopy", "-i", "img1.fs", "::/grub-mender-grubenv/grub.cfg", "."]
+                ["mcopy", "-i", f"{tmpdir}/img1.fs", "::/grub-mender-grubenv/grub.cfg", tmpdir]
             )
-            check_all_root_occurrences_valid("grub.cfg")
+            check_all_root_occurrences_valid(f"{tmpdir}/grub.cfg")
-        finally:
-            os.remove("img1.fs")
-            os.remove("grub.cfg")
 
-        extract_partition(latest_part_image, 2)
+            extract_partition(latest_part_image, 2, tmpdir)
-        try:
             subprocess.check_call(
                 [
                     "debugfs",
                     "-R",
-                    "dump -p /boot/grub-mender-grubenv.cfg grub-mender-grubenv.cfg",
+                    f"dump -p /boot/grub-mender-grubenv.cfg {tmpdir}/grub-mender-grubenv.cfg",
-                    "img2.fs",
+                    f"{tmpdir}/img2.fs",
                 ]
             )
-            check_all_root_occurrences_valid("grub-mender-grubenv.cfg")
+            check_all_root_occurrences_valid(f"{tmpdir}/grub-mender-grubenv.cfg")
-        finally:
-            os.remove("img2.fs")
-            os.remove("grub-mender-grubenv.cfg")
 
         # Then, check that the runtime generated scripts don't have any.
-        get_no_sftp("/boot/grub/grub.cfg", connection)
+        with make_tempdir() as tmpdir:
-        try:
+            get_no_sftp("/boot/grub/grub.cfg", connection, local=tmpdir)
-            check_all_root_occurrences_valid("grub.cfg")
+            check_all_root_occurrences_valid(f"{tmpdir}/grub.cfg")
-        finally:
-            os.remove("grub.cfg")
 
-        get_no_sftp("/boot/grub-mender-grubenv.cfg", connection)
+            get_no_sftp("/boot/grub-mender-grubenv.cfg", connection, local=tmpdir)
-        try:
+            check_all_root_occurrences_valid(f"{tmpdir}/grub-mender-grubenv.cfg")
-            check_all_root_occurrences_valid("grub-mender-grubenv.cfg")
-        finally:
-            os.remove("grub-mender-grubenv.cfg")
 
         # Check again after running `update-grub`.
         connection.run("grub-install && update-grub")
-        get_no_sftp("/boot/grub/grub.cfg", connection)
+        with make_tempdir() as tmpdir:
-        try:
+            get_no_sftp("/boot/grub/grub.cfg", connection, local=tmpdir)
-            check_all_root_occurrences_valid("grub.cfg")
+            check_all_root_occurrences_valid(f"{tmpdir}/grub.cfg")
-        finally:
-            os.remove("grub.cfg")
 
-        get_no_sftp("/boot/grub-mender-grubenv.cfg", connection)
+            get_no_sftp("/boot/grub-mender-grubenv.cfg", connection, local=tmpdir)
-        try:
+            check_all_root_occurrences_valid(f"{tmpdir}/grub-mender-grubenv.cfg")
-            check_all_root_occurrences_valid("grub-mender-grubenv.cfg")
-        finally:
-            os.remove("grub-mender-grubenv.cfg")
 
     @pytest.mark.min_mender_version("1.0.0")
     def test_offline_and_runtime_boot_scripts_identical(self, connection):
@@ -172,13 +156,16 @@ class TestGrubIntegration:
         # * `root` variable is not set in offline copy.
         # * `fwsetup` is added somewhat randomly depending on availability both
         #   on build host and device.
+        # * locale, lang and gettext settings and module may or may not be
+        #   present depending on test host.
         try:
             connection.run("cp /data/grub-main.cfg /data/old-grub-modified.cfg")
             connection.run("cp /boot/grub/grub.cfg /data/new-grub-modified.cfg")
             connection.run(
                 r"sed -i -En -e '/\bsearch\b/{s/ --hint[^ ]*//g;}' "
                 "-e \"/^set root='hd0,gpt1'$/d\" "
-                r"-e '\,### BEGIN /etc/grub.d/30_uefi-firmware ###,{p; n; :loop; \,### END /etc/grub.d/30_uefi-firmware ###,b end; n; b loop; :end;}' "
+                r"-e '\,### BEGIN /etc/grub.d/30_uefi-firmware ###,{p; n; :uefi_loop; \,### END /etc/grub.d/30_uefi-firmware ###,b uefi_end; n; b uefi_loop; :uefi_end;}' "
+                r"-e ':locale_loop; /^\s*(set (locale_dir|lang)=|insmod gettext)/{n; b locale_loop;}' "
                 "-e p "
                 "/data/old-grub-modified.cfg /data/new-grub-modified.cfg"
             )
@@ -193,7 +180,8 @@ class TestGrubIntegration:
             connection.run(
                 r"sed -i -En -e '/\bsearch\b/{s/ --hint[^ ]*//g;}' "
                 "-e \"/^set root='hd0,gpt1'$/d\" "
-                r"-e '\,### BEGIN /etc/grub.d/30_uefi-firmware ###,{p; n; :loop; \,### END /etc/grub.d/30_uefi-firmware ###,b end; n; b loop; :end;}' "
+                r"-e '\,### BEGIN /etc/grub.d/30_uefi-firmware ###,{p; n; :uefi_loop; \,### END /etc/grub.d/30_uefi-firmware ###,b uefi_end; n; b uefi_loop; :uefi_end;}' "
+                r"-e ':locale_loop; /^\s*(set (locale_dir|lang)=|insmod gettext)/{n; b locale_loop;}' "
                 "-e p "
                 "/data/old-grub-mender-grubenv-modified.cfg /data/new-grub-mender-grubenv-modified.cfg"
             )