Merge pull request #154 from lluiscampos/parallel-acceptance-test-jobs

Set the acceptance test jobs to run in parallel on different machines

.gitlab-ci.yml

@@ -36,7 +36,7 @@ build:
     paths:
       - image.tar
 
-test_acceptance:
+.template_test_acceptance: &test_acceptance
   stage: test_acceptance
   image: teracy/ubuntu:18.04-dind-18.09.9
   services:
@@ -58,16 +58,8 @@ test_acceptance:
     # Load image under test
     - export IMAGE_NAME=$DOCKER_REPOSITORY:pr
     - docker load -i image.tar
-    # Fetch artifacts from temporary S3 bucket
-    - aws s3 cp s3://mender-gitlab-tmp-storage/$CI_PROJECT_NAME/$CI_PIPELINE_ID/deploy.tar.gz deploy.tar.gz
-    - tar xzf deploy.tar.gz
-    # Extract converted Raspbian artifacts
-    - unxz deploy/raspberrypi-${RASPBIAN_NAME}-mender-${MENDER_CLIENT_VERSION}.sdimg.xz
     # Set submodule to correct version
     - ( cd tests/mender-image-tests && git submodule update --init --remote && git checkout origin/${MENDER_IMAGE_TESTS_REV} )
-  script:
-    - ./scripts/test/run-tests.sh --no-pull --prebuilt-image raspberrypi ${RASPBIAN_NAME}-mender-${MENDER_CLIENT_VERSION}
-    - ./scripts/test/run-tests.sh --no-pull --all
   artifacts:
     expire_in: 2w
     when: always
@@ -77,6 +69,41 @@ test_acceptance:
     reports:
       junit: results_*.xml
 
+test_acceptance_prebuilt_raspberrypi:
+  <<: *test_acceptance
+  script:
+    # Fetch artifacts from temporary S3 bucket
+    - aws s3 cp s3://mender-gitlab-tmp-storage/$CI_PROJECT_NAME/$CI_PIPELINE_ID/deploy.tar.gz deploy.tar.gz
+    - tar xzf deploy.tar.gz
+    # Extract converted Raspbian artifacts
+    - unxz deploy/raspberrypi-${RASPBIAN_NAME}-mender-${MENDER_CLIENT_VERSION}.sdimg.xz
+    - ./scripts/test/run-tests.sh --no-pull --prebuilt-image raspberrypi ${RASPBIAN_NAME}-mender-${MENDER_CLIENT_VERSION}
+
+test_acceptance_qemux86_64:
+  <<: *test_acceptance
+  script:
+    - ./scripts/test/run-tests.sh --no-pull --only qemux86_64
+
+test_acceptance_raspberrypi:
+  <<: *test_acceptance
+  script:
+    - ./scripts/test/run-tests.sh --no-pull --only raspberrypi
+
+test_acceptance_linaro-alip:
+  <<: *test_acceptance
+  script:
+    - ./scripts/test/run-tests.sh --no-pull --only linaro-alip
+
+test_acceptance_beaglebone:
+  <<: *test_acceptance
+  script:
+    - ./scripts/test/run-tests.sh --no-pull --only beaglebone
+
+test_acceptance_ubuntu:
+  <<: *test_acceptance
+  script:
+    - ./scripts/test/run-tests.sh --no-pull --only ubuntu
+
 convert_raspbian:
   stage: convert
   image: teracy/ubuntu:18.04-dind-18.09.9

scripts/test/run-tests.sh

@@ -3,7 +3,7 @@
 set -e
 
 usage() {
-  echo "$0 [--no-pull] <--all | --prebuilt-image DEVICE_TYPE IMAGE_NAME>"
+  echo "$0 [--no-pull] <--all | --only DEVICE_TYPE | --prebuilt-image DEVICE_TYPE IMAGE_NAME>"
   exit 1
 }
 
@@ -48,58 +48,61 @@ mkdir -p ${WORKSPACE}
 
 get_pytest_files
 
+if ! [ "$1" == "--all" -o "$1" == "--only" -a -n "$2" -o "$1" == "--prebuilt-image" -a -n "$3" ]; then
+  usage
+fi
+
 test_result=0
 
-case "$1" in
-  --prebuilt-image)
-    if [ -z "$3" ]; then
-      echo "Both DEVICE_TYPE and IMAGE_NAME must be specified"
-      exit 1
-    fi
-    test_result=0
-    run_tests "$2" "$3" || test_result=$?
-    exit $test_result
-    ;;
-
-  --all)
+if [ "$1" == "--prebuilt-image" ]; then
+  run_tests "$2" "$3" || test_result=$?
+  exit $test_result
+
+else
+  if [ "$1" == "--all" -o "$1" == "--only" -a "$2" == "qemux86_64" ]; then
     convert_and_test "qemux86_64" \
                      "release-1" \
                      "${UBUNTU_IMAGE_URL}" \
                      "${UBUNTU_IMAGE}" \
                      "${UBUNTU_IMAGE}.gz" \
                      "configs/qemux86-64_config" || test_result=$?
+  fi
 
+  if [ "$1" == "--all" -o "$1" == "--only" -a "$2" == "raspberrypi" ]; then
     convert_and_test "raspberrypi" \
                      "release-1" \
                      "${RASPBIAN_IMAGE_URL}" \
                      "${RASPBIAN_IMAGE}.img" \
                      "${RASPBIAN_IMAGE}.zip" \
                      "configs/raspberrypi3_config" || test_result=$?
+  fi
 
+  if [ "$1" == "--all" -o "$1" == "--only" -a "$2" == "linaro-alip" ]; then
     # MEN-2809: Disabled due broken download link
     #convert_and_test "linaro-alip" \
     #                 "release-1" \
     #                 "${TINKER_IMAGE_URL}" \
     #                 "${TINKER_IMAGE}.img" \
     #                 "${TINKER_IMAGE}.zip" || test_result=$?
+    true
+  fi
 
+  if [ "$1" == "--all" -o "$1" == "--only" -a "$2" == "beaglebone" ]; then
     convert_and_test "beaglebone" \
                      "release-1" \
                      "${BBB_DEBIAN_IMAGE_URL}" \
                      "${BBB_DEBIAN_IMAGE}" \
                      "${BBB_DEBIAN_IMAGE}.xz" || test_result=$?
+  fi
 
+  if [ "$1" == "--all" -o "$1" == "--only" -a "$2" == "ubuntu" ]; then
     convert_and_test "ubuntu" \
                      "release-1" \
                      "${UBUNTU_SERVER_RPI_IMAGE_URL}" \
                      "${UBUNTU_SERVER_RPI_IMAGE}" \
                      "${UBUNTU_SERVER_RPI_IMAGE}.xz" \
                      "configs/raspberrypi3_config" || test_result=$?
+  fi
 
-    exit $test_result
-    ;;
-
-  *)
-    usage
-    ;;
-esac
+  exit $test_result
+fi

scripts/test/test-utils.sh

@@ -84,10 +84,6 @@ run_tests() {
 
   cd ${WORKSPACE}/mender-image-tests
 
-  # This is a trick to make pytest generate different junit reports
-  # for different runs: renaming the tests folder to tests_<testsuite>
-  cp -r tests tests_${device_type}_${artifact_name}
-
   python3 -m pytest --verbose \
     --junit-xml="${MENDER_CONVERT_DIR}/results_${device_type}.xml" \
     ${html_report_args} \
@@ -96,7 +92,7 @@ run_tests() {
     --board-type="${device_type}" \
     --mender-image=${device_type}-${artifact_name}.sdimg \
     --sdimg-location="${MENDER_CONVERT_DIR}/deploy" \
-    tests_${device_type}_${artifact_name} \
+    tests \
     ${pytest_args_extra}
   exitcode=$?