From f8f12b41b397c2b5060d580d7ff273215883b961 Mon Sep 17 00:00:00 2001
From: Ole Petter <ole.orhagen@northern.tech>
Date: Fri, 8 May 2020 10:53:35 +0200
Subject: [PATCH] feat(MEN-3052): Automatic decompression of input files

Previously the user would have to manually decompress an input image prior to
handing it over to mender-convert. With this change, files compressed in the
formats: lzma, gzip, or zip archives will be automatically decompressed,
converted, and then recompressed.

Note that the zip archive can only contain one image file, otherwise the
conversion will fail. Thus if the archive contains multiple files, human
interaction is required. This simply involves unzipping the archive yourself,
and then pass in the image, just like in the old workflow.

Ticket: https://tracker.mender.io/browse/MEN-3052

Changelog: Added automatic decompression of input images, so that the convert
tool now accepts compressed input images in the formats: lzma, gzip, and zip.
The images will also be recompressed to the input format automatically.

Signed-off-by: Ole Petter <ole.orhagen@northern.tech>
---
 Dockerfile                 |  5 ++-
 mender-convert             | 39 ++++++++++++++---
 mender-convert-package     |  4 ++
 modules/cliparser.sh       | 47 ++++++++++++++++++++
 modules/decompressinput.sh | 88 ++++++++++++++++++++++++++++++++++++++
 modules/zip.sh             | 34 +++++++++++++++
 6 files changed, 209 insertions(+), 8 deletions(-)
 create mode 100644 modules/cliparser.sh
 create mode 100644 modules/decompressinput.sh
 create mode 100644 modules/zip.sh

diff --git a/Dockerfile b/Dockerfile
index a0cdaab..e3ef9e4 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -45,7 +45,10 @@ RUN apt-get update && apt-get install -y \
 # to regenerate the U-Boot boot.scr on platforms that need customization
     u-boot-tools \
 # needed to run pxz
-    libgomp1
+    libgomp1  \
+# zip and unzip archive
+    zip  \
+    unzip
 
 COPY --from=build /root/pxz/pxz /usr/bin/pxz
 
diff --git a/mender-convert b/mender-convert
index 2d438f8..b434cee 100755
--- a/mender-convert
+++ b/mender-convert
@@ -14,6 +14,23 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
+source modules/bootstrap.sh
+source modules/cliparser.sh
+source modules/decompressinput.sh
+
+###############################################################################
+#                Declaration of important variables for this file             #
+###############################################################################
+declare override_extraargs="" # Override arguments passed to the sub-scripts
+declare compression_type=""   # Detected input file compression, also applied to the output
+declare ocfile="./work/override_compression_config"
+declare disk_image=""         # Needed in parse_cli_options, and is passed to decompress_image()
+declare -a overlays=()        # [Dummy] Needed in parse_cli_options, not here
+declare -a configs=()        # [Dummy] Needed in parse_cli_options, not here
+###############################################################################
+
+MENDER_CONVERT_VERSION=$(git describe --tags --dirty --exact-match 2>/dev/null || git rev-parse --short HEAD)
+
 function show_help() {
   cat << EOF
 mender-convert
@@ -45,9 +62,10 @@ function trap_exit() {
   EXIT_CODE=$?
   if [[ ${EXIT_CODE} -ne 0 && ${EXIT_CODE} -ne ${FATAL_EXIT_CODE} ]]; then
     log_error "mender-convert failed"
-    tac work/convert.log | sed '/DEBUG/q' | tac | sed 's/Running/When running/'
+    [ -e work/convert.log ] && tac work/convert.log | sed '/DEBUG/q' | tac | sed 's/Running/When running/'
     log_error "mender-convert exit code: ${EXIT_CODE}"
   fi
+  mv work/convert.log convert.log
   sudo rm -rf work
 }
 
@@ -95,15 +113,22 @@ if [ -z "${MENDER_ARTIFACT_NAME}" ]; then
   echo -e "\tMENDER_ARTIFACT_NAME=\"release-1\" ./mender-convert"
   exit 1
 fi
-
-source modules/bootstrap.sh
-
 mkdir -p work
 touch work/convert.log
 
-./mender-convert-extract "$@"
-./mender-convert-modify "$@"
-./mender-convert-package "$@"
+parse_cli_options "$@"
+
+uncompressed_disk_image="${disk_image}"
+compression_type=$(compression_type "${disk_image}")
+if [[ ${compression_type} != "none" ]]; then
+  uncompressed_disk_image=$(decompress_image "${disk_image}" "./work")
+  echo "MENDER_COMPRESS_DISK_IMAGE=${compression_type}" > ${ocfile}
+  override_extraargs="--disk-image ${uncompressed_disk_image} --config ${ocfile}"
+fi
+
+./mender-convert-extract "$@" ${override_extraargs}
+./mender-convert-modify  "$@" ${override_extraargs}
+./mender-convert-package "$@" ${override_extraargs}
 
 echo "Output Artifacts and images can be found in the deploy directory:"
 ls -1 deploy/*
diff --git a/mender-convert-package b/mender-convert-package
index 18aa589..025394b 100755
--- a/mender-convert-package
+++ b/mender-convert-package
@@ -286,6 +286,10 @@ case "${MENDER_COMPRESS_DISK_IMAGE}" in
     log_info "Compressing ${img_path}.gz"
     run_and_log_cmd "pigz --best --force ${img_path}"
     ;;
+  zip)
+    log_info "Compressing ${img_path}.zip"
+    zip -9 "${img_path}.zip" "${img_path}"
+    ;;
   lzma)
     log_info "Compressing ${img_path}.xz"
     run_and_log_cmd "pxz --best --force ${img_path}"
diff --git a/modules/cliparser.sh b/modules/cliparser.sh
new file mode 100644
index 0000000..c31ed22
--- /dev/null
+++ b/modules/cliparser.sh
@@ -0,0 +1,47 @@
+#
+# Copyright 2020 Northern.tech AS
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+function parse_cli_options () {
+  while (( "$#" )); do
+    case "$1" in
+      -o | --overlay)
+        overlays+=("${2}")
+        shift 2
+        ;;
+      -c | --config)
+        configs+=("${2}")
+        shift 2
+        ;;
+      -d | --disk-image)
+        disk_image="${2}"
+        shift 2
+        ;;
+      *)
+        log_fatal "Sorry but the provided option is not supported: $1"
+        ;;
+    esac
+  done
+
+  if [ -z "${disk_image}" ]; then
+    log_warn "Sorry, but '--disk-image' is a mandatory option"
+    log_warn "See ./mender-convert --help for more information"
+    exit 1
+  fi
+
+  if [ ! -e ${disk_image} ]; then
+    log_fatal "File not found: ${disk_image}"
+  fi
+
+}
diff --git a/modules/decompressinput.sh b/modules/decompressinput.sh
new file mode 100644
index 0000000..9cb155e
--- /dev/null
+++ b/modules/decompressinput.sh
@@ -0,0 +1,88 @@
+#
+# Copyright 2020 Northern.tech AS
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+source modules/zip.sh
+source modules/log.sh
+
+# compression_type
+#
+# $1 - Path to the compressed disk image
+#
+# @return - The MENDER_COMPRESS_IMAGE compression type
+#
+function compression_type () {
+  if [[ $# -ne 1 ]]; then
+    log_fatal "compression_type() requires one argument"
+  fi
+  local -r disk_image="${1}"
+  case "${disk_image}" in
+    *.img | *.sdimg)
+      echo "none"
+      ;;
+    *.gz)
+      echo "gzip"
+      ;;
+    *.zip)
+      echo "zip"
+      ;;
+    *.xz )
+      echo "lzma"
+      ;;
+    * )
+      log_fatal "Unsupported compression type: ${disk_image}. Please uncompress the image yourself."
+      ;;
+  esac
+}
+
+# Decompresses the given input image
+#
+#  $1 - Path to the compressed image
+#  $2 - Path to the output directory
+#
+# @return - Name of the uncompressed image
+#
+function decompress_image () {
+  if [[ $# -ne 2 ]]; then
+    log_fatal "decompress_image() requires an image argument and an output directory"
+  fi
+  local -r input_image="${1}"
+  local -r output_dir="${2}"
+  local disk_image="${output_dir}/$(basename ${input_image})"
+  case "$(compression_type ${disk_image})" in
+    none )
+      :
+      ;;
+    gzip )
+      log_info "Decompressing ${disk_image}..."
+      disk_image=${disk_image%.gz}
+      zcat "${input_image}" > "${disk_image}"
+      ;;
+    zip )
+      log_info "Decompressing ${disk_image}..."
+      filename="$(zip_get_imgname ${input_image})"
+      unzip "${input_image}" -d "${output_dir}" &>/dev/null
+      disk_image="$(dirname ${disk_image})/${filename}"
+      ;;
+    lzma )
+      log_info "Decompressing ${disk_image}..."
+      disk_image=${disk_image%.xz}
+      xzcat "${input_image}" > "${disk_image}"
+      ;;
+    * )
+      log_fatal "Unsupported input image type: ${input_image}"
+      ;;
+  esac
+  echo "${disk_image}"
+}
diff --git a/modules/zip.sh b/modules/zip.sh
new file mode 100644
index 0000000..736bdfe
--- /dev/null
+++ b/modules/zip.sh
@@ -0,0 +1,34 @@
+#
+# Copyright 2020 Northern.tech AS
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+#
+# Parse the filename from the zipped output
+#
+# $1 - Zip archive path
+#
+# @return - Name of the img contained in the archive
+#
+function zip_get_imgname () {
+  if [[ $# -ne 1 ]]; then
+    log_fatal "zip_get_imgname requires one argument"
+  fi
+  local -r disk_image="${1}"
+  # Assert that the archive holds only one file
+  nfiles="$(unzip -l ${disk_image} | awk '{nfiles=$2} END {print nfiles}')"
+  [[ "$nfiles" -ne 1 ]] && log_fatal "Zip archive has more than one file. Needs to be unzipped by a human. nfiles: $nfiles"
+  local -r filename="$(unzip -lq ${disk_image} | awk 'NR==3 {filename=$NF} END {print filename}')"
+  [[ ${filename} == *.img ]] || log_fatal "no img file found in the zip archive ${disk_image}."
+  echo "$(basename ${filename})"
+}