From 88fe7558c94c61c855f52e5b49bf74aaf9b4b8c8 Mon Sep 17 00:00:00 2001 From: Mirza Krak Date: Wed, 2 Oct 2019 13:09:18 +0000 Subject: [PATCH 1/5] add the default configuration (mender_convert_config) to the config array at creation Current logic actually fails when no '-c/--config' options are provided, with this error: $ MENDER_ARTIFACT_NAME="test-1" ./mender-convert --disk-image input/Armbian_5.95_Tinkerboard_Debian_buster_next_4.19.69_minimal.img Running mender-convert-extract: --disk-image input/Armbian_5.95_Tinkerboard_Debian_buster_next_4.19.69_minimal.img 2019-10-02 12:46:39 [INFO] [mender-convert-extract] Using configuration file: configs/mender_convert_config 2019-10-02 12:46:39 [INFO] [mender-convert-extract] Using configuration file: '' modules/config.sh: line 28: '': No such file or directory The 'configs' array is actually not empty when passed to modules/config.sh as it will contain "''", which will be translated as a array element and hence the above failure. This commit ensures that we set a valid default value to 'configs', which is the 'mender_convert_config'. This is moved from modules/config.sh, to the calling scripts that define the array of configs. Fixes: 9b944615a8a ("configs: Support multiple config files on command line.") Changelog: None Signed-off-by: Mirza Krak --- mender-convert-extract | 3 ++- mender-convert-modify | 3 ++- mender-convert-package | 3 ++- modules/config.sh | 1 - 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/mender-convert-extract b/mender-convert-extract index 7713b41..d50f8a4 100755 --- a/mender-convert-extract +++ b/mender-convert-extract @@ -19,7 +19,8 @@ echo "Running $(basename $0): $@" source modules/bootstrap.sh source modules/disk.sh -declare -a configs +# The mender_convert_config is always used and provides all the defaults +declare -a configs=("configs/mender_convert_config") disk_image="" while (( "$#" )); do diff --git a/mender-convert-modify b/mender-convert-modify index b352e81..48b5250 100755 --- a/mender-convert-modify +++ b/mender-convert-modify @@ -39,7 +39,8 @@ source modules/bootstrap.sh source modules/disk.sh source modules/probe.sh -declare -a configs +# The mender_convert_config is always used and provides all the defaults +declare -a configs=("configs/mender_convert_config") while (( "$#" )); do case "$1" in diff --git a/mender-convert-package b/mender-convert-package index 9d3489f..14214bf 100755 --- a/mender-convert-package +++ b/mender-convert-package @@ -38,7 +38,8 @@ echo "Running $(basename $0): $@" source modules/bootstrap.sh source modules/disk.sh -declare -a configs +# The mender_convert_config is always used and provides all the defaults +declare -a configs=("configs/mender_convert_config") while (( "$#" )); do case "$1" in diff --git a/modules/config.sh b/modules/config.sh index ea653cb..e2dafea 100644 --- a/modules/config.sh +++ b/modules/config.sh @@ -16,7 +16,6 @@ # Read in the array of config files to process read -a configs <<< "${@}" -configs=( "configs/mender_convert_config" "${configs[@]}" ) for config in "${configs[@]}"; do log_info "Using configuration file: ${config}" source "${config}" From ee58c51596e37735e89ef37384aa6e9051b3c15c Mon Sep 17 00:00:00 2001 From: Mirza Krak Date: Wed, 2 Oct 2019 14:26:13 +0000 Subject: [PATCH 2/5] scripts: run-tests: increase coverage With these changes we will additionally cover test with: - MENDER_COMPRESS_DISK_IMAGE set to 'y' and 'n' - multiple --config files - no external configuration files (no '--config' options) Changelog: None Signed-off-by: Mirza Krak --- scripts/run-tests.sh | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/scripts/run-tests.sh b/scripts/run-tests.sh index 4864e27..452725f 100755 --- a/scripts/run-tests.sh +++ b/scripts/run-tests.sh @@ -60,25 +60,40 @@ convert_and_test() { rm -f ${WORKSPACE}/test_config + + # Two motives for the following statement + # + # - speed up tests by avoiding decompression on all images (majority of images + # we test have a platform specific configuration) + # + # - test providing multiple '--config' options + # + # - (when no platform configuration is provided) test conversion without + # '--config' and with MENDER_COMPRESS_DISK_IMAGE=y. Compressed disk + # images is the default user facing option and we need to ensure that we + # cover this in the tests. if [ -n "${config}" ]; then - cp ${config} ${WORKSPACE}/test_config + echo "Will disable MENDER_COMPRESS_DISK_IMAGE for this image" + echo "MENDER_COMPRESS_DISK_IMAGE=n" > ${WORKSPACE}/test_config + local MENDER_CONVERT_EXTRA_ARGS="--config ${config} --config ${WORKSPACE}/test_config" fi - # Disable compression of disk image when testing, otherwise we need to - # unpack each image we test which is time consuming - echo "MENDER_COMPRESS_DISK_IMAGE=n" >> ${WORKSPACE}/test_config - - echo "Configuration used:" - cat ${WORKSPACE}/test_config - MENDER_ARTIFACT_NAME=${artifact_name} ./docker-mender-convert \ --disk-image input/${image_name} \ - --config ${WORKSPACE}/test_config + ${MENDER_CONVERT_EXTRA_ARGS} if pip list | grep -q -e pytest-html; then html_report_args="--html=${MENDER_CONVERT_DIR}/report_${device_type}.html --self-contained-html" fi + # Need to decompress images built with MENDER_COMPRESS_DISK_IMAGE=y before + # running tests. + if [ -f deploy/${device_type}-${artifact_name}.sdimg.gz ]; then + # sudo is needed because the image is created using docker-mender-convert + # which sets root permissions on the image + sudo gunzip --force deploy/${device_type}-${artifact_name}.sdimg.gz + fi + cd ${WORKSPACE}/mender-image-tests python2 -m pytest --verbose \ From e0e82b22244815bc38093fd6a34db7aa710d971a Mon Sep 17 00:00:00 2001 From: Mirza Krak Date: Wed, 2 Oct 2019 14:31:57 +0000 Subject: [PATCH 3/5] add *.xml and *.html files to .gitignore These are created by scripts/run-tests.sh Changelog: None Signed-off-by: Mirza Krak --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 8b2889a..77a99bc 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ work rootfs_overlay_demo/* tests mender_local_config +*.xml +*.html From 5d268286534ef3ebf69a2fc28feea8159fa91713 Mon Sep 17 00:00:00 2001 From: Mirza Krak Date: Wed, 2 Oct 2019 16:40:44 +0000 Subject: [PATCH 4/5] README-run-tests.md: update instructions The run-tests.sh script utilizes 'docker-mender-convert`. Changelog: None Signed-off-by: Mirza Krak --- scripts/README-run-tests.md | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/scripts/README-run-tests.md b/scripts/README-run-tests.md index f6b5219..541080e 100644 --- a/scripts/README-run-tests.md +++ b/scripts/README-run-tests.md @@ -1,13 +1,16 @@ # Run tests -Run the following commands to install test dependencies (assumes that all mender-convert dependencies are already installed): +The tests utilize the provided Docker container in this repository and conversion +is done using the `docker-mender-convert` command. For this reason we first need +to build the container (commands must be run from the root directory of +`mender-convert`): - sudo apt-get update e2fsprogs=1.44.1-1 - sudo apt-get -qy --force-yes install python-pip - sudo pip2 install pytest --upgrade - sudo pip2 install pytest-xdist --upgrade - sudo pip2 install pytest-html --upgrade +```bash +./docker-build +``` Run tests: - ./scripts/run-tests.sh +```bash +./scripts/run-tests.sh +``` From f65c39311bb0da4f7f011772f4bedd9e85dd50d7 Mon Sep 17 00:00:00 2001 From: Mirza Krak Date: Thu, 3 Oct 2019 14:49:47 +0000 Subject: [PATCH 5/5] gitlab-ci: install sudo Required by the scripts/run-script.sh Changelog: None Signed-off-by: Mirza Krak --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5c6a5e1..46863d6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -55,7 +55,7 @@ test_acceptance: - docker version # Install dependencies - apk --update --no-cache add bash wget git util-linux mtools python - py-pip gcc python2-dev libffi-dev libc-dev openssl-dev make + py-pip gcc python2-dev libffi-dev libc-dev openssl-dev make sudo - pip install "pytest>=3.0" "Fabric>=1.13.0, <2" pytest-html # Load image under test - export IMAGE_NAME=$DOCKER_REPOSITORY:pr