Browse Source

feat(MEN-3052): Automatic decompression of input files

Previously the user would have to manually decompress an input image prior to
handing it over to mender-convert. With this change, files compressed in the
formats: lzma, gzip, or zip archives will be automatically decompressed,
converted, and then recompressed.

Note that the zip archive can only contain one image file, otherwise the
conversion will fail. Thus if the archive contains multiple files, human
interaction is required. This simply involves unzipping the archive yourself,
and then pass in the image, just like in the old workflow.

Ticket: https://tracker.mender.io/browse/MEN-3052

Changelog: Added automatic decompression of input images, so that the convert
tool now accepts compressed input images in the formats: lzma, gzip, and zip.
The images will also be recompressed to the input format automatically.

Signed-off-by: Ole Petter <ole.orhagen@northern.tech>
2.1.x
Ole Petter 5 years ago
parent
commit
f8f12b41b3
No known key found for this signature in database GPG Key ID: A7100375167A7B21
  1. 5
      Dockerfile
  2. 39
      mender-convert
  3. 4
      mender-convert-package
  4. 47
      modules/cliparser.sh
  5. 88
      modules/decompressinput.sh
  6. 34
      modules/zip.sh

5
Dockerfile

@ -45,7 +45,10 @@ RUN apt-get update && apt-get install -y \
# to regenerate the U-Boot boot.scr on platforms that need customization
u-boot-tools \
# needed to run pxz
libgomp1
libgomp1 \
# zip and unzip archive
zip \
unzip
COPY --from=build /root/pxz/pxz /usr/bin/pxz

39
mender-convert

@ -14,6 +14,23 @@
# See the License for the specific language governing permissions and
# limitations under the License.
source modules/bootstrap.sh
source modules/cliparser.sh
source modules/decompressinput.sh
###############################################################################
# Declaration of important variables for this file #
###############################################################################
declare override_extraargs="" # Override arguments passed to the sub-scripts
declare compression_type="" # Detected input file compression, also applied to the output
declare ocfile="./work/override_compression_config"
declare disk_image="" # Needed in parse_cli_options, and is passed to decompress_image()
declare -a overlays=() # [Dummy] Needed in parse_cli_options, not here
declare -a configs=() # [Dummy] Needed in parse_cli_options, not here
###############################################################################
MENDER_CONVERT_VERSION=$(git describe --tags --dirty --exact-match 2>/dev/null || git rev-parse --short HEAD)
function show_help() {
cat << EOF
mender-convert
@ -45,9 +62,10 @@ function trap_exit() {
EXIT_CODE=$?
if [[ ${EXIT_CODE} -ne 0 && ${EXIT_CODE} -ne ${FATAL_EXIT_CODE} ]]; then
log_error "mender-convert failed"
tac work/convert.log | sed '/DEBUG/q' | tac | sed 's/Running/When running/'
[ -e work/convert.log ] && tac work/convert.log | sed '/DEBUG/q' | tac | sed 's/Running/When running/'
log_error "mender-convert exit code: ${EXIT_CODE}"
fi
mv work/convert.log convert.log
sudo rm -rf work
}
@ -95,15 +113,22 @@ if [ -z "${MENDER_ARTIFACT_NAME}" ]; then
echo -e "\tMENDER_ARTIFACT_NAME=\"release-1\" ./mender-convert"
exit 1
fi
source modules/bootstrap.sh
mkdir -p work
touch work/convert.log
./mender-convert-extract "$@"
./mender-convert-modify "$@"
./mender-convert-package "$@"
parse_cli_options "$@"
uncompressed_disk_image="${disk_image}"
compression_type=$(compression_type "${disk_image}")
if [[ ${compression_type} != "none" ]]; then
uncompressed_disk_image=$(decompress_image "${disk_image}" "./work")
echo "MENDER_COMPRESS_DISK_IMAGE=${compression_type}" > ${ocfile}
override_extraargs="--disk-image ${uncompressed_disk_image} --config ${ocfile}"
fi
./mender-convert-extract "$@" ${override_extraargs}
./mender-convert-modify "$@" ${override_extraargs}
./mender-convert-package "$@" ${override_extraargs}
echo "Output Artifacts and images can be found in the deploy directory:"
ls -1 deploy/*

4
mender-convert-package

@ -286,6 +286,10 @@ case "${MENDER_COMPRESS_DISK_IMAGE}" in
log_info "Compressing ${img_path}.gz"
run_and_log_cmd "pigz --best --force ${img_path}"
;;
zip)
log_info "Compressing ${img_path}.zip"
zip -9 "${img_path}.zip" "${img_path}"
;;
lzma)
log_info "Compressing ${img_path}.xz"
run_and_log_cmd "pxz --best --force ${img_path}"

47
modules/cliparser.sh

@ -0,0 +1,47 @@
#
# Copyright 2020 Northern.tech AS
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
function parse_cli_options () {
while (( "$#" )); do
case "$1" in
-o | --overlay)
overlays+=("${2}")
shift 2
;;
-c | --config)
configs+=("${2}")
shift 2
;;
-d | --disk-image)
disk_image="${2}"
shift 2
;;
*)
log_fatal "Sorry but the provided option is not supported: $1"
;;
esac
done
if [ -z "${disk_image}" ]; then
log_warn "Sorry, but '--disk-image' is a mandatory option"
log_warn "See ./mender-convert --help for more information"
exit 1
fi
if [ ! -e ${disk_image} ]; then
log_fatal "File not found: ${disk_image}"
fi
}

88
modules/decompressinput.sh

@ -0,0 +1,88 @@
#
# Copyright 2020 Northern.tech AS
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
source modules/zip.sh
source modules/log.sh
# compression_type
#
# $1 - Path to the compressed disk image
#
# @return - The MENDER_COMPRESS_IMAGE compression type
#
function compression_type () {
if [[ $# -ne 1 ]]; then
log_fatal "compression_type() requires one argument"
fi
local -r disk_image="${1}"
case "${disk_image}" in
*.img | *.sdimg)
echo "none"
;;
*.gz)
echo "gzip"
;;
*.zip)
echo "zip"
;;
*.xz )
echo "lzma"
;;
* )
log_fatal "Unsupported compression type: ${disk_image}. Please uncompress the image yourself."
;;
esac
}
# Decompresses the given input image
#
# $1 - Path to the compressed image
# $2 - Path to the output directory
#
# @return - Name of the uncompressed image
#
function decompress_image () {
if [[ $# -ne 2 ]]; then
log_fatal "decompress_image() requires an image argument and an output directory"
fi
local -r input_image="${1}"
local -r output_dir="${2}"
local disk_image="${output_dir}/$(basename ${input_image})"
case "$(compression_type ${disk_image})" in
none )
:
;;
gzip )
log_info "Decompressing ${disk_image}..."
disk_image=${disk_image%.gz}
zcat "${input_image}" > "${disk_image}"
;;
zip )
log_info "Decompressing ${disk_image}..."
filename="$(zip_get_imgname ${input_image})"
unzip "${input_image}" -d "${output_dir}" &>/dev/null
disk_image="$(dirname ${disk_image})/${filename}"
;;
lzma )
log_info "Decompressing ${disk_image}..."
disk_image=${disk_image%.xz}
xzcat "${input_image}" > "${disk_image}"
;;
* )
log_fatal "Unsupported input image type: ${input_image}"
;;
esac
echo "${disk_image}"
}

34
modules/zip.sh

@ -0,0 +1,34 @@
#
# Copyright 2020 Northern.tech AS
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Parse the filename from the zipped output
#
# $1 - Zip archive path
#
# @return - Name of the img contained in the archive
#
function zip_get_imgname () {
if [[ $# -ne 1 ]]; then
log_fatal "zip_get_imgname requires one argument"
fi
local -r disk_image="${1}"
# Assert that the archive holds only one file
nfiles="$(unzip -l ${disk_image} | awk '{nfiles=$2} END {print nfiles}')"
[[ "$nfiles" -ne 1 ]] && log_fatal "Zip archive has more than one file. Needs to be unzipped by a human. nfiles: $nfiles"
local -r filename="$(unzip -lq ${disk_image} | awk 'NR==3 {filename=$NF} END {print filename}')"
[[ ${filename} == *.img ]] || log_fatal "no img file found in the zip archive ${disk_image}."
echo "$(basename ${filename})"
}
Loading…
Cancel
Save