From f8c21024317d39f4b362f231a40a36e68fe3aab3 Mon Sep 17 00:00:00 2001 From: Lluis Campos Date: Tue, 1 Oct 2019 13:50:55 +0200 Subject: [PATCH 1/7] Remove unnecessary check for mender-artifact The tests do not use the tool. Also, github_PR_status is undefined. Changelog: None Signed-off-by: Lluis Campos --- scripts/run-tests.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/scripts/run-tests.sh b/scripts/run-tests.sh index d431344..d4a5647 100755 --- a/scripts/run-tests.sh +++ b/scripts/run-tests.sh @@ -103,12 +103,6 @@ else cd - fi -if ! [ -x "$(command -v mender-artifact)" ]; then - echo "mender-artifact: not found in PATH." - github_PR_status "failure" "mender-artifact: not found in PATH." - exit 1 -fi - mkdir -p ${WORKSPACE} get_pytest_files From 5c2da454c8cac45645ef31f59628be1e5c8170f4 Mon Sep 17 00:00:00 2001 From: Lluis Campos Date: Tue, 1 Oct 2019 13:52:08 +0200 Subject: [PATCH 2/7] Use volume rather than mount on Docker invocation The reason for having a bind mount is unknown. In order to be able to run this container fom inside another, the directory must be a volume. Changelog: None Signed-off-by: Lluis Campos --- docker-mender-convert | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-mender-convert b/docker-mender-convert index 51e3bd0..59195f8 100755 --- a/docker-mender-convert +++ b/docker-mender-convert @@ -21,7 +21,7 @@ IMAGE_NAME=${IMAGE_NAME:-mender-convert} MENDER_CONVERT_DIR="$(pwd)" docker run \ - --mount type=bind,source="$MENDER_CONVERT_DIR,target=/mender-convert" \ + -v $MENDER_CONVERT_DIR:/mender-convert \ --privileged=true \ --cap-add=SYS_MODULE \ -v /dev:/dev \ From 2e5e37659e3ee1aed8bcd63d619f0793ba1d4c02 Mon Sep 17 00:00:00 2001 From: Lluis Campos Date: Tue, 1 Oct 2019 13:54:56 +0200 Subject: [PATCH 3/7] Avoid using the py.test wrapper and directly call python2 -m pytest As it might not be installed in some systems. Changelog: None Signed-off-by: Lluis Campos --- scripts/run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/run-tests.sh b/scripts/run-tests.sh index d4a5647..3828835 100755 --- a/scripts/run-tests.sh +++ b/scripts/run-tests.sh @@ -77,7 +77,7 @@ convert_and_test() { cd ${WORKSPACE}/mender-image-tests - py.test --verbose \ + python2 -m pytest --verbose \ --junit-xml="${WORKSPACE}/results.xml" \ --test-conversion \ --test-variables="${MENDER_CONVERT_DIR}/deploy/${device_type}-${artifact_name}.cfg" \ From 5c97d08a0e3b2442f78b2f4ee49dad00e36927b8 Mon Sep 17 00:00:00 2001 From: Lluis Campos Date: Tue, 1 Oct 2019 13:57:12 +0200 Subject: [PATCH 4/7] MEN-2785: Add acceptance tests to GitLab pipeline Created a new test_acceptance stage that will execute in our private runners. This stage will execute scripts/run-tests.sh and collect the now produced xml/html reports for each device type. Removed the build from the test script, so that we make sure that CI tests the exact same image build in "build" stage. Changelog: None Signed-off-by: Lluis Campos --- .gitlab-ci.yml | 29 +++++++++++++++++++++++++++++ scripts/run-tests.sh | 9 ++++++--- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 478b1b3..5c6a5e1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,6 +6,7 @@ variables: stages: - test - build + - test_acceptance test: stage: test @@ -39,3 +40,31 @@ build: paths: - image.tar +test_acceptance: + stage: test_acceptance + image: docker:18-dind + tags: + - mender-qa-slave + dependencies: + - build + before_script: + # Start up Docker (DonD) + - /usr/local/bin/dockerd-entrypoint.sh & + - sleep 10 + - export DOCKER_HOST="unix:///var/run/docker.sock" + - docker version + # Install dependencies + - apk --update --no-cache add bash wget git util-linux mtools python + py-pip gcc python2-dev libffi-dev libc-dev openssl-dev make + - pip install "pytest>=3.0" "Fabric>=1.13.0, <2" pytest-html + # Load image under test + - export IMAGE_NAME=$DOCKER_REPOSITORY:pr + - docker load -i image.tar + script: + - ./scripts/run-tests.sh + artifacts: + expire_in: 2w + when: always + paths: + - results_*.xml + - report_*.html diff --git a/scripts/run-tests.sh b/scripts/run-tests.sh index 3828835..e1bbdda 100755 --- a/scripts/run-tests.sh +++ b/scripts/run-tests.sh @@ -75,10 +75,15 @@ convert_and_test() { --disk-image input/${image_name} \ --config ${WORKSPACE}/test_config + if pip list | grep -q -e pytest-html; then + html_report_args="--html=${MENDER_CONVERT_DIR}/report_${device_type}.html --self-contained-html" + fi + cd ${WORKSPACE}/mender-image-tests python2 -m pytest --verbose \ - --junit-xml="${WORKSPACE}/results.xml" \ + --junit-xml="${MENDER_CONVERT_DIR}/results_${device_type}.xml" \ + ${html_report_args} \ --test-conversion \ --test-variables="${MENDER_CONVERT_DIR}/deploy/${device_type}-${artifact_name}.cfg" \ --board-type="${device_type}" \ @@ -107,8 +112,6 @@ mkdir -p ${WORKSPACE} get_pytest_files -./docker-build - convert_and_test "qemux86_64" \ "release-1" \ "${UBUNTU_IMAGE_URL}" \ From 9ea8cd0ff37a0728c95c26d21424d11b82f9cfab Mon Sep 17 00:00:00 2001 From: Lluis Campos Date: Wed, 2 Oct 2019 10:11:58 +0200 Subject: [PATCH 5/7] Minimize the verbosity of the donwload progress in wget The images are very big, and the default donwload withouht TTY (used by CI) adds a lot of noise in the logs. Changelog: None Signed-off-by: Lluis Campos --- scripts/run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/run-tests.sh b/scripts/run-tests.sh index e1bbdda..65b3f48 100755 --- a/scripts/run-tests.sh +++ b/scripts/run-tests.sh @@ -38,7 +38,7 @@ convert_and_test() { image_name_compressed=$5 config=$6 # Optional - wget -N ${image_url} -P input/ + wget --progress=dot:giga -N ${image_url} -P input/ echo "Extracting: ${image_name_compressed}" case "${image_name_compressed}" in From ba83abd3255106cbbcbd40d791a43bb3a73a150e Mon Sep 17 00:00:00 2001 From: Lluis Campos Date: Wed, 2 Oct 2019 10:13:08 +0200 Subject: [PATCH 6/7] Continue executing tests regardless of previous failures The final exit code will represent if any of the cases have failed. Changelog: None Signed-off-by: Lluis Campos --- scripts/run-tests.sh | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/scripts/run-tests.sh b/scripts/run-tests.sh index 65b3f48..3179055 100755 --- a/scripts/run-tests.sh +++ b/scripts/run-tests.sh @@ -89,8 +89,11 @@ convert_and_test() { --board-type="${device_type}" \ --mender-image=${device_type}-${artifact_name}.sdimg \ --sdimg-location="${MENDER_CONVERT_DIR}/deploy" + exitcode=$? cd - + + return $exitcode } get_pytest_files() { @@ -112,29 +115,32 @@ mkdir -p ${WORKSPACE} get_pytest_files +test_result=0 + convert_and_test "qemux86_64" \ "release-1" \ "${UBUNTU_IMAGE_URL}" \ "${UBUNTU_IMAGE}" \ "${UBUNTU_IMAGE}.gz" \ - "configs/qemux86-64_config" - + "configs/qemux86-64_config" || test_result=$? convert_and_test "raspberrypi" \ "release-1" \ "${RASPBIAN_IMAGE_URL}" \ "${RASPBIAN_IMAGE}.img" \ "${RASPBIAN_IMAGE}.zip" \ - "configs/raspberrypi3_config" + "configs/raspberrypi3_config" || test_result=$? convert_and_test "linaro-alip" \ "release-1" \ "${TINKER_IMAGE_URL}" \ "${TINKER_IMAGE}.img" \ - "${TINKER_IMAGE}.zip" + "${TINKER_IMAGE}.zip" || test_result=$? convert_and_test "beaglebone" \ "release-1" \ "${BBB_DEBIAN_IMAGE_URL}" \ "${BBB_DEBIAN_IMAGE}" \ - "${BBB_DEBIAN_IMAGE}.xz" + "${BBB_DEBIAN_IMAGE}.xz" || test_result=$? + +exit $test_result From 8a30bf4ecc6d3d99bb221e2246cc290d45651898 Mon Sep 17 00:00:00 2001 From: Lluis Campos Date: Wed, 2 Oct 2019 12:44:00 +0200 Subject: [PATCH 7/7] Disable acceptance test for Tinker OS image. See MEN-2785 Changelog: None Signed-off-by: Lluis Campos --- scripts/run-tests.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/scripts/run-tests.sh b/scripts/run-tests.sh index 3179055..4864e27 100755 --- a/scripts/run-tests.sh +++ b/scripts/run-tests.sh @@ -131,11 +131,12 @@ convert_and_test "raspberrypi" \ "${RASPBIAN_IMAGE}.zip" \ "configs/raspberrypi3_config" || test_result=$? -convert_and_test "linaro-alip" \ - "release-1" \ - "${TINKER_IMAGE_URL}" \ - "${TINKER_IMAGE}.img" \ - "${TINKER_IMAGE}.zip" || test_result=$? +# MEN-2809: Disabled due broken download link +#convert_and_test "linaro-alip" \ +# "release-1" \ +# "${TINKER_IMAGE_URL}" \ +# "${TINKER_IMAGE}.img" \ +# "${TINKER_IMAGE}.zip" || test_result=$? convert_and_test "beaglebone" \ "release-1" \