Browse Source

Merge pull request #198 from oleorhagen/MEN-3052

Testing
2.1.x
oleorhagen 5 years ago
committed by GitHub
parent
commit
87697a282b
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
  1. 5
      Dockerfile
  2. 39
      mender-convert
  3. 4
      mender-convert-package
  4. 47
      modules/cliparser.sh
  5. 88
      modules/decompressinput.sh
  6. 34
      modules/zip.sh
  7. 78
      scripts/test/run-tests.sh
  8. 118
      scripts/test/test-utils.sh

5
Dockerfile

@ -45,7 +45,10 @@ RUN apt-get update && apt-get install -y \
# to regenerate the U-Boot boot.scr on platforms that need customization
u-boot-tools \
# needed to run pxz
libgomp1
libgomp1 \
# zip and unzip archive
zip \
unzip
COPY --from=build /root/pxz/pxz /usr/bin/pxz

39
mender-convert

@ -14,6 +14,23 @@
# See the License for the specific language governing permissions and
# limitations under the License.
source modules/bootstrap.sh
source modules/cliparser.sh
source modules/decompressinput.sh
###############################################################################
# Declaration of important variables for this file #
###############################################################################
declare override_extraargs="" # Override arguments passed to the sub-scripts
declare compression_type="" # Detected input file compression, also applied to the output
declare ocfile="./work/override_compression_config"
declare disk_image="" # Needed in parse_cli_options, and is passed to decompress_image()
declare -a overlays=() # [Dummy] Needed in parse_cli_options, not here
declare -a configs=() # [Dummy] Needed in parse_cli_options, not here
###############################################################################
MENDER_CONVERT_VERSION=$(git describe --tags --dirty --exact-match 2>/dev/null || git rev-parse --short HEAD)
function show_help() {
cat << EOF
mender-convert
@ -45,9 +62,10 @@ function trap_exit() {
EXIT_CODE=$?
if [[ ${EXIT_CODE} -ne 0 && ${EXIT_CODE} -ne ${FATAL_EXIT_CODE} ]]; then
log_error "mender-convert failed"
tac work/convert.log | sed '/DEBUG/q' | tac | sed 's/Running/When running/'
[ -e work/convert.log ] && tac work/convert.log | sed '/DEBUG/q' | tac | sed 's/Running/When running/'
log_error "mender-convert exit code: ${EXIT_CODE}"
fi
mv work/convert.log convert.log
sudo rm -rf work
}
@ -95,15 +113,22 @@ if [ -z "${MENDER_ARTIFACT_NAME}" ]; then
echo -e "\tMENDER_ARTIFACT_NAME=\"release-1\" ./mender-convert"
exit 1
fi
source modules/bootstrap.sh
mkdir -p work
touch work/convert.log
./mender-convert-extract "$@"
./mender-convert-modify "$@"
./mender-convert-package "$@"
parse_cli_options "$@"
uncompressed_disk_image="${disk_image}"
compression_type=$(compression_type "${disk_image}")
if [[ ${compression_type} != "none" ]]; then
uncompressed_disk_image=$(decompress_image "${disk_image}" "./work")
echo "MENDER_COMPRESS_DISK_IMAGE=${compression_type}" > ${ocfile}
override_extraargs="--disk-image ${uncompressed_disk_image} --config ${ocfile}"
fi
./mender-convert-extract "$@" ${override_extraargs}
./mender-convert-modify "$@" ${override_extraargs}
./mender-convert-package "$@" ${override_extraargs}
echo "Output Artifacts and images can be found in the deploy directory:"
ls -1 deploy/*

4
mender-convert-package

@ -286,6 +286,10 @@ case "${MENDER_COMPRESS_DISK_IMAGE}" in
log_info "Compressing ${img_path}.gz"
run_and_log_cmd "pigz --best --force ${img_path}"
;;
zip)
log_info "Compressing ${img_path}.zip"
zip -9 "${img_path}.zip" "${img_path}"
;;
lzma)
log_info "Compressing ${img_path}.xz"
run_and_log_cmd "pxz --best --force ${img_path}"

47
modules/cliparser.sh

@ -0,0 +1,47 @@
#
# Copyright 2020 Northern.tech AS
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
function parse_cli_options () {
while (( "$#" )); do
case "$1" in
-o | --overlay)
overlays+=("${2}")
shift 2
;;
-c | --config)
configs+=("${2}")
shift 2
;;
-d | --disk-image)
disk_image="${2}"
shift 2
;;
*)
log_fatal "Sorry but the provided option is not supported: $1"
;;
esac
done
if [ -z "${disk_image}" ]; then
log_warn "Sorry, but '--disk-image' is a mandatory option"
log_warn "See ./mender-convert --help for more information"
exit 1
fi
if [ ! -e ${disk_image} ]; then
log_fatal "File not found: ${disk_image}"
fi
}

88
modules/decompressinput.sh

@ -0,0 +1,88 @@
#
# Copyright 2020 Northern.tech AS
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
source modules/zip.sh
source modules/log.sh
# compression_type
#
# $1 - Path to the compressed disk image
#
# @return - The MENDER_COMPRESS_IMAGE compression type
#
function compression_type () {
if [[ $# -ne 1 ]]; then
log_fatal "compression_type() requires one argument"
fi
local -r disk_image="${1}"
case "${disk_image}" in
*.img | *.sdimg)
echo "none"
;;
*.gz)
echo "gzip"
;;
*.zip)
echo "zip"
;;
*.xz )
echo "lzma"
;;
* )
log_fatal "Unsupported compression type: ${disk_image}. Please uncompress the image yourself."
;;
esac
}
# Decompresses the given input image
#
# $1 - Path to the compressed image
# $2 - Path to the output directory
#
# @return - Name of the uncompressed image
#
function decompress_image () {
if [[ $# -ne 2 ]]; then
log_fatal "decompress_image() requires an image argument and an output directory"
fi
local -r input_image="${1}"
local -r output_dir="${2}"
local disk_image="${output_dir}/$(basename ${input_image})"
case "$(compression_type ${disk_image})" in
none )
:
;;
gzip )
log_info "Decompressing ${disk_image}..."
disk_image=${disk_image%.gz}
zcat "${input_image}" > "${disk_image}"
;;
zip )
log_info "Decompressing ${disk_image}..."
filename="$(zip_get_imgname ${input_image})"
unzip "${input_image}" -d "${output_dir}" &>/dev/null
disk_image="$(dirname ${disk_image})/${filename}"
;;
lzma )
log_info "Decompressing ${disk_image}..."
disk_image=${disk_image%.xz}
xzcat "${input_image}" > "${disk_image}"
;;
* )
log_fatal "Unsupported input image type: ${input_image}"
;;
esac
echo "${disk_image}"
}

34
modules/zip.sh

@ -0,0 +1,34 @@
#
# Copyright 2020 Northern.tech AS
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Parse the filename from the zipped output
#
# $1 - Zip archive path
#
# @return - Name of the img contained in the archive
#
function zip_get_imgname () {
if [[ $# -ne 1 ]]; then
log_fatal "zip_get_imgname requires one argument"
fi
local -r disk_image="${1}"
# Assert that the archive holds only one file
nfiles="$(unzip -l ${disk_image} | awk '{nfiles=$2} END {print nfiles}')"
[[ "$nfiles" -ne 1 ]] && log_fatal "Zip archive has more than one file. Needs to be unzipped by a human. nfiles: $nfiles"
local -r filename="$(unzip -lq ${disk_image} | awk 'NR==3 {filename=$NF} END {print filename}')"
[[ ${filename} == *.img ]] || log_fatal "no img file found in the zip archive ${disk_image}."
echo "$(basename ${filename})"
}

78
scripts/test/run-tests.sh

@ -15,20 +15,15 @@ fi
WORKSPACE=./tests
BBB_DEBIAN_IMAGE="bone-debian-9.5-iot-armhf-2018-08-30-4gb.img"
BBB_DEBIAN_IMAGE_URL="http://debian.beagleboard.org/images/${BBB_DEBIAN_IMAGE}.xz"
BBB_DEBIAN_IMAGE_URL="http://debian.beagleboard.org/images/bone-debian-9.5-iot-armhf-2018-08-30-4gb.img.xz"
RASPBIAN_IMAGE="2019-09-26-raspbian-buster-lite"
RASPBIAN_IMAGE_URL="http://downloads.raspberrypi.org/raspbian_lite/images/raspbian_lite-2019-09-30/2019-09-26-raspbian-buster-lite.zip"
TINKER_IMAGE="20170417-tinker-board-linaro-stretch-alip-v1.8"
TINKER_IMAGE_URL="http://dlcdnet.asus.com/pub/ASUS/mb/Linux/Tinker_Board_2GB/${TINKER_IMAGE}.zip"
TINKER_IMAGE_URL="http://dlcdnet.asus.com/pub/ASUS/mb/Linux/Tinker_Board_2GB/20170417-tinker-board-linaro-stretch-alip-v1.8.zip"
UBUNTU_IMAGE="Ubuntu-Bionic-x86-64.img"
UBUNTU_IMAGE_URL="https://d1b0l86ne08fsf.cloudfront.net/mender-convert/images/${UBUNTU_IMAGE}.gz"
UBUNTU_IMAGE_URL="https://d1b0l86ne08fsf.cloudfront.net/mender-convert/images/Ubuntu-Bionic-x86-64.img.gz"
UBUNTU_SERVER_RPI_IMAGE="ubuntu-18.04.4-preinstalled-server-armhf+raspi3.img"
UBUNTU_SERVER_RPI_IMAGE_URL="http://cdimage.ubuntu.com/ubuntu/releases/bionic/release/${UBUNTU_SERVER_RPI_IMAGE}.xz"
UBUNTU_SERVER_RPI_IMAGE_URL="http://cdimage.ubuntu.com/ubuntu/releases/bionic/release/ubuntu-18.04.4-preinstalled-server-armhf+raspi3.img.xz"
# Keep common function declarations in separate utils script
UTILS_PATH=${0/$(basename $0)/test-utils.sh}
@ -60,21 +55,39 @@ if [ "$1" == "--prebuilt-image" ]; then
else
if [ "$1" == "--all" -o "$1" == "--only" -a "$2" == "qemux86_64" ]; then
wget --progress=dot:giga -N ${UBUNTU_IMAGE_URL} -P input/
convert_and_test "qemux86_64" \
"release-1" \
"${UBUNTU_IMAGE_URL}" \
"${UBUNTU_IMAGE}" \
"${UBUNTU_IMAGE}.gz" \
"configs/qemux86-64_config" || test_result=$?
"input/Ubuntu-Bionic-x86-64.img.gz" \
"--config configs/qemux86-64_config" || test_result=$?
echo >&2 "----------------------------------------"
echo >&2 "Running the uncompressed test"
echo >&2 "----------------------------------------"
rm -rf deploy
gunzip --force "input/Ubuntu-Bionic-x86-64.img.gz"
convert_and_test "qemux86_64" \
"release-1" \
"input/Ubuntu-Bionic-x86-64.img" \
"--config configs/qemux86-64_config" || test_result=$?
fi
if [ "$1" == "--all" -o "$1" == "--only" -a "$2" == "raspberrypi3" ]; then
wget --progress=dot:giga -N ${RASPBIAN_IMAGE_URL} -P input/
convert_and_test "raspberrypi3" \
"release-1" \
"${RASPBIAN_IMAGE_URL}" \
"${RASPBIAN_IMAGE}.img" \
"${RASPBIAN_IMAGE}.zip" \
"configs/raspberrypi3_config" || test_result=$?
"input/2019-09-26-raspbian-buster-lite.zip" \
"--config configs/raspberrypi3_config" || test_result=$?
echo >&2 "----------------------------------------"
echo >&2 "Running the uncompressed test"
echo >&2 "----------------------------------------"
rm -rf deploy
unzip -o "input/2019-09-26-raspbian-buster-lite.zip" -d "./input"
convert_and_test "raspberrypi3" \
"release-1" \
"input/2019-09-26-raspbian-buster-lite.img" \
"--config configs/raspberrypi3_config" || test_result=$?
fi
if [ "$1" == "--all" -o "$1" == "--only" -a "$2" == "linaro-alip" ]; then
@ -88,20 +101,37 @@ else
fi
if [ "$1" == "--all" -o "$1" == "--only" -a "$2" == "beaglebone" ]; then
wget --progress=dot:giga -N ${BBB_DEBIAN_IMAGE_URL} -P input/
convert_and_test "beaglebone" \
"release-1" \
"${BBB_DEBIAN_IMAGE_URL}" \
"${BBB_DEBIAN_IMAGE}" \
"${BBB_DEBIAN_IMAGE}.xz" || test_result=$?
"input/bone-debian-9.5-iot-armhf-2018-08-30-4gb.img.xz" || test_result=$?
echo >&2 "----------------------------------------"
echo >&2 "Running the uncompressed test"
echo >&2 "----------------------------------------"
rm -rf deploy
unxz --force "input/bone-debian-9.5-iot-armhf-2018-08-30-4gb.img.xz"
convert_and_test "beaglebone" \
"release-1" \
"input/bone-debian-9.5-iot-armhf-2018-08-30-4gb.img" || test_result=$?
fi
if [ "$1" == "--all" -o "$1" == "--only" -a "$2" == "ubuntu" ]; then
wget --progress=dot:giga -N ${UBUNTU_SERVER_RPI_IMAGE_URL} -P input/
convert_and_test "raspberrypi3" \
"release-1" \
"input/ubuntu-18.04.4-preinstalled-server-armhf+raspi3.img.xz" \
"--config configs/raspberrypi3_config" || test_result=$?
echo >&2 "----------------------------------------"
echo >&2 "Running the uncompressed test"
echo >&2 "----------------------------------------"
rm -rf deploy
unxz --force "input/ubuntu-18.04.4-preinstalled-server-armhf+raspi3.img.xz"
convert_and_test "raspberrypi3" \
"release-1" \
"${UBUNTU_SERVER_RPI_IMAGE_URL}" \
"${UBUNTU_SERVER_RPI_IMAGE}" \
"${UBUNTU_SERVER_RPI_IMAGE}.xz" \
"configs/raspberrypi3_config" || test_result=$?
"input/ubuntu-18.04.4-preinstalled-server-armhf+raspi3.img" \
"--config configs/raspberrypi3_config" || test_result=$?
fi
exit $test_result

118
scripts/test/test-utils.sh

@ -4,76 +4,94 @@ WORKSPACE=${WORKSPACE:-./tests}
MENDER_CONVERT_DIR=$PWD
#
# function image_name_after_conversion()
#
# Transforms the given input image name to the name given to the output image by
# mender-convert.
#
# That is, an input image 'bone-debian-9.5-iot-armhf-2018-08-30-4gb.img.xz'
#
# T(bone-debian-9.5-iot-armhf-2018-08-30-4gb.img.xz)
# -> bone-debian-9.5-iot-armhf-2018-08-30-4gb-beaglebone-mender
#
# $1 - image name, with optional .img and compression endings
# $2 - the compression used for the image
# $3 - the device type
#
function image_name_after_conversion () {
if (( $# < 1 )); then
echo "image_name_after_conversion requires one argument. $# given."
exit 1
fi
local converted_image_name="$1"
local compression="${2}"
local device_type="${3}"
# Remove the compression if any
if [[ "${compression}" != "img" ]]; then
converted_image_name="${converted_image_name%.${compression}}"
fi
# remove the .img extension if any
converted_image_name="${converted_image_name%.img}"
# Add the extension which a successful mender-conversion will apply
converted_image_name="${converted_image_name}-${device_type}-mender.img"
# Add the compression back in
if [[ "${compression}" != "img" ]]; then
converted_image_name="${converted_image_name}.${compression}"
fi
echo "$(basename ${converted_image_name})"
}
function assert () {
if (( $# < 2 )); then
echo >&2 "assert() requires at least an expected and an actual value"
fi
if [[ "$1" == "$2" ]]; then
return
fi
# Neither string nor value matched the assertion
echo >&2 "Assertion error: $1 not equal to $2"
[[ -n "$3" ]] && echo >&2 "$3"
exit 1
}
source modules/decompressinput.sh
convert_and_test() {
device_type=$1
artifact_name=$2
image_url=$3
image_file=$4
image_file_compressed=$5
config=$6 # Optional
wget --progress=dot:giga -N ${image_url} -P input/
echo "Extracting: ${image_file_compressed}"
case "${image_file_compressed}" in
*.gz)
gunzip -f input/${image_file_compressed}
;;
*.zip)
cd input
unzip -o ${image_file_compressed}
cd -
;;
*.xz)
xz -d -f input/${image_file_compressed}
;;
*)
echo "Unknown image type: ${image_file_compressed}"
exit 1
;;
esac
image_file=$3
extra_args=$4 # Optional
rm -f ${WORKSPACE}/test_config
# Two motives for the following statement
#
# - speed up tests by avoiding decompression on all images (majority of images
# we test have a platform specific configuration)
#
# - test providing multiple '--config' options
#
# - (when no platform configuration is provided) test conversion without
# '--config' and with MENDER_COMPRESS_DISK_IMAGE=gzip. Compressed disk
# images is the default user facing option and we need to ensure that we
# cover this in the tests.
if [ -n "${config}" ]; then
echo "Will disable MENDER_COMPRESS_DISK_IMAGE for this image"
echo "MENDER_COMPRESS_DISK_IMAGE=none" > ${WORKSPACE}/test_config
local MENDER_CONVERT_EXTRA_ARGS="--config ${config} --config ${WORKSPACE}/test_config"
fi
MENDER_ARTIFACT_NAME=${artifact_name} ./docker-mender-convert \
--disk-image input/${image_file} \
${MENDER_CONVERT_EXTRA_ARGS}
--disk-image ${image_file} \
${extra_args}
local compression="${image_file##*.}"
local ret=0
# The output image name after conversion
image_name="${image_file%.img}-${device_type}-mender"
image_name=$(image_name_after_conversion "${image_file}" "${compression}" "${device_type}")
run_tests "${device_type}" "$image_name" || ret=$?
converted_image_file="${MENDER_CONVERT_DIR}/deploy/$(basename ${image_name})"
rm -f deploy/${image_file}*
converted_image_uncompressed="$(decompress_image ${converted_image_file} ${MENDER_CONVERT_DIR}/deploy)"
run_tests "${device_type}" "$(basename ${converted_image_uncompressed})" || ret=$?
assert "${ret}" "0" "Failed to convert ${image_file}"
return $ret
}
run_tests() {
device_type=$1
converted_image_name=$2
converted_image_file=$2
shift 2
pytest_args_extra=$@
converted_image_name="${converted_image_file%.img}"
if pip3 list | grep -q -e pytest-html; then
html_report_args="--html=${MENDER_CONVERT_DIR}/report_${device_type}.html --self-contained-html"
fi

Loading…
Cancel
Save