From f8f12b41b397c2b5060d580d7ff273215883b961 Mon Sep 17 00:00:00 2001 From: Ole Petter Date: Fri, 8 May 2020 10:53:35 +0200 Subject: [PATCH 1/2] feat(MEN-3052): Automatic decompression of input files Previously the user would have to manually decompress an input image prior to handing it over to mender-convert. With this change, files compressed in the formats: lzma, gzip, or zip archives will be automatically decompressed, converted, and then recompressed. Note that the zip archive can only contain one image file, otherwise the conversion will fail. Thus if the archive contains multiple files, human interaction is required. This simply involves unzipping the archive yourself, and then pass in the image, just like in the old workflow. Ticket: https://tracker.mender.io/browse/MEN-3052 Changelog: Added automatic decompression of input images, so that the convert tool now accepts compressed input images in the formats: lzma, gzip, and zip. The images will also be recompressed to the input format automatically. Signed-off-by: Ole Petter --- Dockerfile | 5 ++- mender-convert | 39 ++++++++++++++--- mender-convert-package | 4 ++ modules/cliparser.sh | 47 ++++++++++++++++++++ modules/decompressinput.sh | 88 ++++++++++++++++++++++++++++++++++++++ modules/zip.sh | 34 +++++++++++++++ 6 files changed, 209 insertions(+), 8 deletions(-) create mode 100644 modules/cliparser.sh create mode 100644 modules/decompressinput.sh create mode 100644 modules/zip.sh diff --git a/Dockerfile b/Dockerfile index a0cdaab..e3ef9e4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,7 +45,10 @@ RUN apt-get update && apt-get install -y \ # to regenerate the U-Boot boot.scr on platforms that need customization u-boot-tools \ # needed to run pxz - libgomp1 + libgomp1 \ +# zip and unzip archive + zip \ + unzip COPY --from=build /root/pxz/pxz /usr/bin/pxz diff --git a/mender-convert b/mender-convert index 2d438f8..b434cee 100755 --- a/mender-convert +++ b/mender-convert @@ -14,6 +14,23 @@ # See the License for the specific language governing permissions and # limitations under the License. +source modules/bootstrap.sh +source modules/cliparser.sh +source modules/decompressinput.sh + +############################################################################### +# Declaration of important variables for this file # +############################################################################### +declare override_extraargs="" # Override arguments passed to the sub-scripts +declare compression_type="" # Detected input file compression, also applied to the output +declare ocfile="./work/override_compression_config" +declare disk_image="" # Needed in parse_cli_options, and is passed to decompress_image() +declare -a overlays=() # [Dummy] Needed in parse_cli_options, not here +declare -a configs=() # [Dummy] Needed in parse_cli_options, not here +############################################################################### + +MENDER_CONVERT_VERSION=$(git describe --tags --dirty --exact-match 2>/dev/null || git rev-parse --short HEAD) + function show_help() { cat << EOF mender-convert @@ -45,9 +62,10 @@ function trap_exit() { EXIT_CODE=$? if [[ ${EXIT_CODE} -ne 0 && ${EXIT_CODE} -ne ${FATAL_EXIT_CODE} ]]; then log_error "mender-convert failed" - tac work/convert.log | sed '/DEBUG/q' | tac | sed 's/Running/When running/' + [ -e work/convert.log ] && tac work/convert.log | sed '/DEBUG/q' | tac | sed 's/Running/When running/' log_error "mender-convert exit code: ${EXIT_CODE}" fi + mv work/convert.log convert.log sudo rm -rf work } @@ -95,15 +113,22 @@ if [ -z "${MENDER_ARTIFACT_NAME}" ]; then echo -e "\tMENDER_ARTIFACT_NAME=\"release-1\" ./mender-convert" exit 1 fi - -source modules/bootstrap.sh - mkdir -p work touch work/convert.log -./mender-convert-extract "$@" -./mender-convert-modify "$@" -./mender-convert-package "$@" +parse_cli_options "$@" + +uncompressed_disk_image="${disk_image}" +compression_type=$(compression_type "${disk_image}") +if [[ ${compression_type} != "none" ]]; then + uncompressed_disk_image=$(decompress_image "${disk_image}" "./work") + echo "MENDER_COMPRESS_DISK_IMAGE=${compression_type}" > ${ocfile} + override_extraargs="--disk-image ${uncompressed_disk_image} --config ${ocfile}" +fi + +./mender-convert-extract "$@" ${override_extraargs} +./mender-convert-modify "$@" ${override_extraargs} +./mender-convert-package "$@" ${override_extraargs} echo "Output Artifacts and images can be found in the deploy directory:" ls -1 deploy/* diff --git a/mender-convert-package b/mender-convert-package index 18aa589..025394b 100755 --- a/mender-convert-package +++ b/mender-convert-package @@ -286,6 +286,10 @@ case "${MENDER_COMPRESS_DISK_IMAGE}" in log_info "Compressing ${img_path}.gz" run_and_log_cmd "pigz --best --force ${img_path}" ;; + zip) + log_info "Compressing ${img_path}.zip" + zip -9 "${img_path}.zip" "${img_path}" + ;; lzma) log_info "Compressing ${img_path}.xz" run_and_log_cmd "pxz --best --force ${img_path}" diff --git a/modules/cliparser.sh b/modules/cliparser.sh new file mode 100644 index 0000000..c31ed22 --- /dev/null +++ b/modules/cliparser.sh @@ -0,0 +1,47 @@ +# +# Copyright 2020 Northern.tech AS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +function parse_cli_options () { + while (( "$#" )); do + case "$1" in + -o | --overlay) + overlays+=("${2}") + shift 2 + ;; + -c | --config) + configs+=("${2}") + shift 2 + ;; + -d | --disk-image) + disk_image="${2}" + shift 2 + ;; + *) + log_fatal "Sorry but the provided option is not supported: $1" + ;; + esac + done + + if [ -z "${disk_image}" ]; then + log_warn "Sorry, but '--disk-image' is a mandatory option" + log_warn "See ./mender-convert --help for more information" + exit 1 + fi + + if [ ! -e ${disk_image} ]; then + log_fatal "File not found: ${disk_image}" + fi + +} diff --git a/modules/decompressinput.sh b/modules/decompressinput.sh new file mode 100644 index 0000000..9cb155e --- /dev/null +++ b/modules/decompressinput.sh @@ -0,0 +1,88 @@ +# +# Copyright 2020 Northern.tech AS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source modules/zip.sh +source modules/log.sh + +# compression_type +# +# $1 - Path to the compressed disk image +# +# @return - The MENDER_COMPRESS_IMAGE compression type +# +function compression_type () { + if [[ $# -ne 1 ]]; then + log_fatal "compression_type() requires one argument" + fi + local -r disk_image="${1}" + case "${disk_image}" in + *.img | *.sdimg) + echo "none" + ;; + *.gz) + echo "gzip" + ;; + *.zip) + echo "zip" + ;; + *.xz ) + echo "lzma" + ;; + * ) + log_fatal "Unsupported compression type: ${disk_image}. Please uncompress the image yourself." + ;; + esac +} + +# Decompresses the given input image +# +# $1 - Path to the compressed image +# $2 - Path to the output directory +# +# @return - Name of the uncompressed image +# +function decompress_image () { + if [[ $# -ne 2 ]]; then + log_fatal "decompress_image() requires an image argument and an output directory" + fi + local -r input_image="${1}" + local -r output_dir="${2}" + local disk_image="${output_dir}/$(basename ${input_image})" + case "$(compression_type ${disk_image})" in + none ) + : + ;; + gzip ) + log_info "Decompressing ${disk_image}..." + disk_image=${disk_image%.gz} + zcat "${input_image}" > "${disk_image}" + ;; + zip ) + log_info "Decompressing ${disk_image}..." + filename="$(zip_get_imgname ${input_image})" + unzip "${input_image}" -d "${output_dir}" &>/dev/null + disk_image="$(dirname ${disk_image})/${filename}" + ;; + lzma ) + log_info "Decompressing ${disk_image}..." + disk_image=${disk_image%.xz} + xzcat "${input_image}" > "${disk_image}" + ;; + * ) + log_fatal "Unsupported input image type: ${input_image}" + ;; + esac + echo "${disk_image}" +} diff --git a/modules/zip.sh b/modules/zip.sh new file mode 100644 index 0000000..736bdfe --- /dev/null +++ b/modules/zip.sh @@ -0,0 +1,34 @@ +# +# Copyright 2020 Northern.tech AS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Parse the filename from the zipped output +# +# $1 - Zip archive path +# +# @return - Name of the img contained in the archive +# +function zip_get_imgname () { + if [[ $# -ne 1 ]]; then + log_fatal "zip_get_imgname requires one argument" + fi + local -r disk_image="${1}" + # Assert that the archive holds only one file + nfiles="$(unzip -l ${disk_image} | awk '{nfiles=$2} END {print nfiles}')" + [[ "$nfiles" -ne 1 ]] && log_fatal "Zip archive has more than one file. Needs to be unzipped by a human. nfiles: $nfiles" + local -r filename="$(unzip -lq ${disk_image} | awk 'NR==3 {filename=$NF} END {print filename}')" + [[ ${filename} == *.img ]] || log_fatal "no img file found in the zip archive ${disk_image}." + echo "$(basename ${filename})" +} From 275615e58d59d3090aceafd17ddcc07bff2f35ee Mon Sep 17 00:00:00 2001 From: Ole Petter Date: Fri, 8 May 2020 11:06:43 +0200 Subject: [PATCH 2/2] test(automatic-decompression): Add tests for automatic decompression This change required some changes to the current test scripts. Most notably, the whole conversion is now run two times. One for the compressed input image, and one for the already decompressed image. Also, the test framework was cleaned up significantly, and the number of arguments passed to the 'convert_and_test' function has been decreased to remove the redundancies that were present (like passing the image name, and the image name with the extension, in two separate parameters). These changes are a result of the changes introduced with: * MEN-3052: https://tracker.mender.io/browse/MEN-3052 Changelog: None Signed-off-by: Ole Petter --- scripts/test/run-tests.sh | 78 ++++++++++++++++-------- scripts/test/test-utils.sh | 118 +++++++++++++++++++++---------------- 2 files changed, 122 insertions(+), 74 deletions(-) diff --git a/scripts/test/run-tests.sh b/scripts/test/run-tests.sh index 3eb3d11..d199e6b 100755 --- a/scripts/test/run-tests.sh +++ b/scripts/test/run-tests.sh @@ -15,20 +15,15 @@ fi WORKSPACE=./tests -BBB_DEBIAN_IMAGE="bone-debian-9.5-iot-armhf-2018-08-30-4gb.img" -BBB_DEBIAN_IMAGE_URL="http://debian.beagleboard.org/images/${BBB_DEBIAN_IMAGE}.xz" +BBB_DEBIAN_IMAGE_URL="http://debian.beagleboard.org/images/bone-debian-9.5-iot-armhf-2018-08-30-4gb.img.xz" -RASPBIAN_IMAGE="2019-09-26-raspbian-buster-lite" RASPBIAN_IMAGE_URL="http://downloads.raspberrypi.org/raspbian_lite/images/raspbian_lite-2019-09-30/2019-09-26-raspbian-buster-lite.zip" -TINKER_IMAGE="20170417-tinker-board-linaro-stretch-alip-v1.8" -TINKER_IMAGE_URL="http://dlcdnet.asus.com/pub/ASUS/mb/Linux/Tinker_Board_2GB/${TINKER_IMAGE}.zip" +TINKER_IMAGE_URL="http://dlcdnet.asus.com/pub/ASUS/mb/Linux/Tinker_Board_2GB/20170417-tinker-board-linaro-stretch-alip-v1.8.zip" -UBUNTU_IMAGE="Ubuntu-Bionic-x86-64.img" -UBUNTU_IMAGE_URL="https://d1b0l86ne08fsf.cloudfront.net/mender-convert/images/${UBUNTU_IMAGE}.gz" +UBUNTU_IMAGE_URL="https://d1b0l86ne08fsf.cloudfront.net/mender-convert/images/Ubuntu-Bionic-x86-64.img.gz" -UBUNTU_SERVER_RPI_IMAGE="ubuntu-18.04.4-preinstalled-server-armhf+raspi3.img" -UBUNTU_SERVER_RPI_IMAGE_URL="http://cdimage.ubuntu.com/ubuntu/releases/bionic/release/${UBUNTU_SERVER_RPI_IMAGE}.xz" +UBUNTU_SERVER_RPI_IMAGE_URL="http://cdimage.ubuntu.com/ubuntu/releases/bionic/release/ubuntu-18.04.4-preinstalled-server-armhf+raspi3.img.xz" # Keep common function declarations in separate utils script UTILS_PATH=${0/$(basename $0)/test-utils.sh} @@ -60,21 +55,39 @@ if [ "$1" == "--prebuilt-image" ]; then else if [ "$1" == "--all" -o "$1" == "--only" -a "$2" == "qemux86_64" ]; then + wget --progress=dot:giga -N ${UBUNTU_IMAGE_URL} -P input/ convert_and_test "qemux86_64" \ "release-1" \ - "${UBUNTU_IMAGE_URL}" \ - "${UBUNTU_IMAGE}" \ - "${UBUNTU_IMAGE}.gz" \ - "configs/qemux86-64_config" || test_result=$? + "input/Ubuntu-Bionic-x86-64.img.gz" \ + "--config configs/qemux86-64_config" || test_result=$? + + echo >&2 "----------------------------------------" + echo >&2 "Running the uncompressed test" + echo >&2 "----------------------------------------" + rm -rf deploy + gunzip --force "input/Ubuntu-Bionic-x86-64.img.gz" + convert_and_test "qemux86_64" \ + "release-1" \ + "input/Ubuntu-Bionic-x86-64.img" \ + "--config configs/qemux86-64_config" || test_result=$? fi if [ "$1" == "--all" -o "$1" == "--only" -a "$2" == "raspberrypi3" ]; then + wget --progress=dot:giga -N ${RASPBIAN_IMAGE_URL} -P input/ convert_and_test "raspberrypi3" \ "release-1" \ - "${RASPBIAN_IMAGE_URL}" \ - "${RASPBIAN_IMAGE}.img" \ - "${RASPBIAN_IMAGE}.zip" \ - "configs/raspberrypi3_config" || test_result=$? + "input/2019-09-26-raspbian-buster-lite.zip" \ + "--config configs/raspberrypi3_config" || test_result=$? + + echo >&2 "----------------------------------------" + echo >&2 "Running the uncompressed test" + echo >&2 "----------------------------------------" + rm -rf deploy + unzip -o "input/2019-09-26-raspbian-buster-lite.zip" -d "./input" + convert_and_test "raspberrypi3" \ + "release-1" \ + "input/2019-09-26-raspbian-buster-lite.img" \ + "--config configs/raspberrypi3_config" || test_result=$? fi if [ "$1" == "--all" -o "$1" == "--only" -a "$2" == "linaro-alip" ]; then @@ -88,20 +101,37 @@ else fi if [ "$1" == "--all" -o "$1" == "--only" -a "$2" == "beaglebone" ]; then + wget --progress=dot:giga -N ${BBB_DEBIAN_IMAGE_URL} -P input/ convert_and_test "beaglebone" \ "release-1" \ - "${BBB_DEBIAN_IMAGE_URL}" \ - "${BBB_DEBIAN_IMAGE}" \ - "${BBB_DEBIAN_IMAGE}.xz" || test_result=$? + "input/bone-debian-9.5-iot-armhf-2018-08-30-4gb.img.xz" || test_result=$? + + echo >&2 "----------------------------------------" + echo >&2 "Running the uncompressed test" + echo >&2 "----------------------------------------" + rm -rf deploy + unxz --force "input/bone-debian-9.5-iot-armhf-2018-08-30-4gb.img.xz" + convert_and_test "beaglebone" \ + "release-1" \ + "input/bone-debian-9.5-iot-armhf-2018-08-30-4gb.img" || test_result=$? fi if [ "$1" == "--all" -o "$1" == "--only" -a "$2" == "ubuntu" ]; then + wget --progress=dot:giga -N ${UBUNTU_SERVER_RPI_IMAGE_URL} -P input/ + convert_and_test "raspberrypi3" \ + "release-1" \ + "input/ubuntu-18.04.4-preinstalled-server-armhf+raspi3.img.xz" \ + "--config configs/raspberrypi3_config" || test_result=$? + + echo >&2 "----------------------------------------" + echo >&2 "Running the uncompressed test" + echo >&2 "----------------------------------------" + rm -rf deploy + unxz --force "input/ubuntu-18.04.4-preinstalled-server-armhf+raspi3.img.xz" convert_and_test "raspberrypi3" \ "release-1" \ - "${UBUNTU_SERVER_RPI_IMAGE_URL}" \ - "${UBUNTU_SERVER_RPI_IMAGE}" \ - "${UBUNTU_SERVER_RPI_IMAGE}.xz" \ - "configs/raspberrypi3_config" || test_result=$? + "input/ubuntu-18.04.4-preinstalled-server-armhf+raspi3.img" \ + "--config configs/raspberrypi3_config" || test_result=$? fi exit $test_result diff --git a/scripts/test/test-utils.sh b/scripts/test/test-utils.sh index 2e93a5b..4a182eb 100644 --- a/scripts/test/test-utils.sh +++ b/scripts/test/test-utils.sh @@ -4,76 +4,94 @@ WORKSPACE=${WORKSPACE:-./tests} MENDER_CONVERT_DIR=$PWD +# +# function image_name_after_conversion() +# +# Transforms the given input image name to the name given to the output image by +# mender-convert. +# +# That is, an input image 'bone-debian-9.5-iot-armhf-2018-08-30-4gb.img.xz' +# +# T(bone-debian-9.5-iot-armhf-2018-08-30-4gb.img.xz) +# -> bone-debian-9.5-iot-armhf-2018-08-30-4gb-beaglebone-mender +# +# $1 - image name, with optional .img and compression endings +# $2 - the compression used for the image +# $3 - the device type +# +function image_name_after_conversion () { + if (( $# < 1 )); then + echo "image_name_after_conversion requires one argument. $# given." + exit 1 + fi + local converted_image_name="$1" + local compression="${2}" + local device_type="${3}" + # Remove the compression if any + if [[ "${compression}" != "img" ]]; then + converted_image_name="${converted_image_name%.${compression}}" + fi + # remove the .img extension if any + converted_image_name="${converted_image_name%.img}" + # Add the extension which a successful mender-conversion will apply + converted_image_name="${converted_image_name}-${device_type}-mender.img" + # Add the compression back in + if [[ "${compression}" != "img" ]]; then + converted_image_name="${converted_image_name}.${compression}" + fi + echo "$(basename ${converted_image_name})" +} + +function assert () { + if (( $# < 2 )); then + echo >&2 "assert() requires at least an expected and an actual value" + fi + if [[ "$1" == "$2" ]]; then + return + fi + # Neither string nor value matched the assertion + echo >&2 "Assertion error: $1 not equal to $2" + [[ -n "$3" ]] && echo >&2 "$3" + exit 1 +} + +source modules/decompressinput.sh + convert_and_test() { device_type=$1 artifact_name=$2 - image_url=$3 - image_file=$4 - image_file_compressed=$5 - config=$6 # Optional - - wget --progress=dot:giga -N ${image_url} -P input/ - - echo "Extracting: ${image_file_compressed}" - case "${image_file_compressed}" in - *.gz) - gunzip -f input/${image_file_compressed} - ;; - *.zip) - cd input - unzip -o ${image_file_compressed} - cd - - ;; - *.xz) - xz -d -f input/${image_file_compressed} - ;; - *) - echo "Unknown image type: ${image_file_compressed}" - exit 1 - ;; - esac + image_file=$3 + extra_args=$4 # Optional rm -f ${WORKSPACE}/test_config - # Two motives for the following statement - # - # - speed up tests by avoiding decompression on all images (majority of images - # we test have a platform specific configuration) - # - # - test providing multiple '--config' options - # - # - (when no platform configuration is provided) test conversion without - # '--config' and with MENDER_COMPRESS_DISK_IMAGE=gzip. Compressed disk - # images is the default user facing option and we need to ensure that we - # cover this in the tests. - if [ -n "${config}" ]; then - echo "Will disable MENDER_COMPRESS_DISK_IMAGE for this image" - echo "MENDER_COMPRESS_DISK_IMAGE=none" > ${WORKSPACE}/test_config - local MENDER_CONVERT_EXTRA_ARGS="--config ${config} --config ${WORKSPACE}/test_config" - fi - MENDER_ARTIFACT_NAME=${artifact_name} ./docker-mender-convert \ - --disk-image input/${image_file} \ - ${MENDER_CONVERT_EXTRA_ARGS} + --disk-image ${image_file} \ + ${extra_args} + local compression="${image_file##*.}" local ret=0 - # The output image name after conversion - image_name="${image_file%.img}-${device_type}-mender" + image_name=$(image_name_after_conversion "${image_file}" "${compression}" "${device_type}") - run_tests "${device_type}" "$image_name" || ret=$? + converted_image_file="${MENDER_CONVERT_DIR}/deploy/$(basename ${image_name})" - rm -f deploy/${image_file}* + converted_image_uncompressed="$(decompress_image ${converted_image_file} ${MENDER_CONVERT_DIR}/deploy)" + + run_tests "${device_type}" "$(basename ${converted_image_uncompressed})" || ret=$? + + assert "${ret}" "0" "Failed to convert ${image_file}" - return $ret } run_tests() { device_type=$1 - converted_image_name=$2 + converted_image_file=$2 shift 2 pytest_args_extra=$@ + converted_image_name="${converted_image_file%.img}" + if pip3 list | grep -q -e pytest-html; then html_report_args="--html=${MENDER_CONVERT_DIR}/report_${device_type}.html --self-contained-html" fi