MEN-2411: Refactor mender-convert to use make install target

When installing the client, call "make install" from source directory
and then modify the default setup with the user input (tenant token,
server url, demo...), eliminating this way the duplication on the
install recipe.

Also, setting the default build version to 2.0.0 beta

Other minor changes along the way:
* Include liblzma build dependency
* Change --mender-client parameter to be optional
* Abort on error in convert-stage-4.sh
* Bugfix in getting server_cert from command line option
* Error reporting when using unsupported device
* Clean-up obsolete --help example

Changelog: Title

Signed-off-by: Lluis Campos <lluis.campos@northern.tech>

Dockerfile

@@ -20,6 +20,7 @@ RUN apt-get update && apt-get install -y \
     libtool \
     pkg-config \
     python \
+    jq \
 # for mender-convert to run (mkfs.vfat is required for boot partition)
     sudo \
     dosfstools \
@@ -38,6 +39,16 @@ RUN echo "mtools_skip_check=1" >> $HOME/.mtoolsrc
 RUN wget -q -O /usr/bin/mender-artifact https://d1b0l86ne08fsf.cloudfront.net/mender-artifact/$MENDER_ARTIFACT_VERSION/mender-artifact \
     && chmod +x /usr/bin/mender-artifact
 
+# Build liblzma from source
+RUN wget -q https://tukaani.org/xz/xz-5.2.4.tar.gz \
+    && tar -C /root -xzf xz-5.2.4.tar.gz \
+    && cd /root/xz-5.2.4 \
+    && ./configure --host=arm-linux-gnueabihf --prefix=/root/xz-5.2.4/install \
+    && make \
+    && make install
+
+ENV LIBLZMA_INSTALL_PATH "/root/xz-5.2.4/install"
+
 # Golang environment, for cross-compiling the Mender client
 RUN wget https://dl.google.com/go/go$GOLANG_VERSION.linux-amd64.tar.gz \
     && tar -C /usr/local -xzf go$GOLANG_VERSION.linux-amd64.tar.gz \
@@ -52,17 +63,17 @@ ENV MENDER_CLIENT_VERSION=$mender_client_version
 ENV PATH "$PATH:/usr/local/go/bin"
 ENV GOPATH "/root/go"
 
-RUN go get github.com/mendersoftware/mender
+RUN go get -d github.com/mendersoftware/mender
 WORKDIR $GOPATH/src/github.com/mendersoftware/mender
 RUN git checkout $MENDER_CLIENT_VERSION
 
 RUN env CGO_ENABLED=1 \
+    CGO_CFLAGS="-I${LIBLZMA_INSTALL_PATH}/include" \
+    CGO_LDFLAGS="-L${LIBLZMA_INSTALL_PATH}/lib" \
     CC=arm-linux-gnueabihf-gcc \
     GOOS=linux \
     GOARCH=arm make build
 
-RUN cp $GOPATH/src/github.com/mendersoftware/mender/mender /
-
 WORKDIR /
 
 COPY docker-entrypoint.sh /usr/local/bin/

README.md

@@ -72,7 +72,6 @@ TENANT_TOKEN="<INSERT-TOKEN-FROM Hosted Mender>"
             --raw-disk-image $RAW_DISK_IMAGE                \
             --mender-disk-image $MENDER_DISK_IMAGE          \
             --device-type $DEVICE_TYPE                      \
-            --mender-client /mender                         \
             --artifact-name $ARTIFACT_NAME                  \
             --bootloader-toolchain arm-linux-gnueabihf      \
             --server-url "https://hosted.mender.io"         \

convert-stage-4.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -e
+
 show_help() {
   cat << EOF
 
@@ -23,26 +25,23 @@ Usage: $0 [options]
         --keep              - Keep intermediate files in output directory
         --help              - Show help and exit
 
-    Examples:
+    For examples, see: ./mender-convert --help
-
-        ./mender-convert install-mender-to-mender-disk-image
-                --mender-disk-image <mender_image_path>
-                --device-type <beaglebone | raspberrypi3>
-                --artifact-name release-1_1.5.0
-                --demo-host-ip 192.168.10.2
-                --mender-client <mender_binary_path>
 
 EOF
   exit 1
 }
 
+jq_inplace() {
+  jq_args="$1"
+  dest_file="$2"
+  sudo sh -c -e "jq \"${jq_args}\" ${dest_file} > ${dest_file}.tmp && mv ${dest_file}.tmp ${dest_file}"
+}
+
 tool_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 output_dir=${tool_dir}/output
 
-mender_client_repo="https://raw.githubusercontent.com/mendersoftware/mender"
-mender_client_revision="1.6.x"
 meta_mender_repo="https://raw.githubusercontent.com/mendersoftware/meta-mender"
-meta_mender_revision="sumo"
+meta_mender_revision="thud"
 
 mender_dir=$output_dir/mender
 device_type=
@@ -51,8 +50,6 @@ artifact_name=
 demo_host_ip=
 # Mender production server url passed as CLI option.
 server_url=
-# Actual server url.
-mender_server_url="https://docker.mender.io"
 # Mender production certificate.
 server_cert=
 # Mender tenant token passed as CLI option.
@@ -62,7 +59,9 @@ mender_tenant_token="dummy"
 
 declare -a mender_disk_mappings
 
-create_client_files() {
+append_rootfs_configuration() {
+  local conffile=$1
+
   local rootfsparta="/dev/mmcblk0p2"
   local rootfspartb="/dev/mmcblk0p3"
 
@@ -71,54 +70,10 @@ create_client_files() {
     rootfspartb="/dev/hda3"
   fi
 
-  # Default polling intervals for Production
+  jq_inplace '.RootfsPartA = \"'$rootfsparta'\" | .RootfsPartB = \"'$rootfspartb'\"' ${conffile}
-  local updatePollInterval="1800"
+}
-  local inventPollInterval="28800"
-  local retryPollInterval="300"
-  if [ -n "${demo}" ] && [ ${demo} -eq 1 ]; then
-    updatePollInterval="5"
-    inventPollInterval="5"
-    retryPollInterval="30"
-  fi
-
-  cat <<- EOF > $mender_dir/mender.service
-	[Unit]
-	Description=Mender OTA update service
-	After=systemd-resolved.service
-
-	[Service]
-	Type=idle
-	User=root
-	Group=root
-	ExecStartPre=/bin/mkdir -p -m 0700 /data/mender
-	ExecStartPre=/bin/ln -sf /etc/mender/tenant.conf /var/lib/mender/authtentoken
-	ExecStart=/usr/bin/mender -daemon
-	Restart=on-abort
-
-	[Install]
-	WantedBy=multi-user.target
-	EOF
-
-  cat <<- EOF > $mender_dir/mender.conf
-	{
-	    "InventoryPollIntervalSeconds": $inventPollInterval,
-	    "RetryPollIntervalSeconds": $retryPollInterval,
-	    "RootfsPartA": "$rootfsparta",
-	    "RootfsPartB": "$rootfspartb",
-	    "ServerCertificate": "/etc/mender/server.crt",
-	    "ServerURL": "$mender_server_url",
-	    "TenantToken": "$mender_tenant_token",
-	    "UpdatePollIntervalSeconds": $updatePollInterval
-	}
-	EOF
-
-  cat <<- EOF > $mender_dir/artifact_info
-	artifact_name=${artifact_name}
-	EOF
-
-  # Version file
-  echo -n "2" > $mender_dir/version
 
+create_client_files() {
   cat <<- EOF > $mender_dir/device_type
 	device_type=${device_type}
 	EOF
@@ -143,21 +98,9 @@ get_mender_files_from_upstream() {
 
   mkdir -p $mender_dir
 
-  log "\tDownloading inventory & identity scripts."
+  log "\tDownloading demo server certificate."
-
+
-  wget -nc -q -O $mender_dir/mender-device-identity \
+  wget -q -O $mender_dir/server.demo.crt \
-    $mender_client_repo/$mender_client_revision/support/mender-device-identity
-  wget -nc -q -O $mender_dir/mender-inventory-bootloader-integration \
-    $mender_client_repo/$mender_client_revision/support/mender-inventory-bootloader-integration
-  wget -nc -q -O $mender_dir/mender-inventory-hostinfo \
-    $mender_client_repo/$mender_client_revision/support/mender-inventory-hostinfo
-  wget -nc -q -O $mender_dir/mender-inventory-network \
-    $mender_client_repo/$mender_client_revision/support/mender-inventory-network
-  wget -nc -q -O $mender_dir/mender-inventory-os \
-    $mender_client_repo/$mender_client_revision/support/mender-inventory-os
-  wget -nc -q -O $mender_dir/mender-inventory-rootfs-type \
-    $mender_client_repo/$mender_client_revision/support/mender-inventory-rootfs-type
-  wget -nc -q -O $mender_dir/server.crt \
     $meta_mender_repo/$meta_mender_revision/meta-mender-demo/recipes-mender/mender/files/server.crt
 }
 
@@ -165,11 +108,8 @@ install_files() {
   local primary_dir=$1
   local data_dir=$2
 
-  identitydir="usr/share/mender/identity"
-  inventorydir="usr/share/mender/inventory"
   sysconfdir="etc/mender"
   bindir="usr/bin"
-  systemd_unitdir="lib/systemd/system"
   localstatedir="var/lib/mender"
   dataconfdir="mender"
   databootdir="u-boot"
@@ -202,41 +142,55 @@ install_files() {
       ;;
   esac
 
-  sudo install -d ${primary_dir}/${identitydir}
-  sudo install -d ${primary_dir}/${inventorydir}
-  sudo install -d ${primary_dir}/${sysconfdir}
-  sudo install -d ${primary_dir}/${sysconfdir}/scripts
-
   sudo ln -sf /data/${dataconfdir} ${primary_dir}/${localstatedir}
 
-  sudo install -m 0755 ${mender_client} ${primary_dir}/${bindir}/mender
+  # Call mender make install target
-
+  ( cd $GOPATH/src/github.com/mendersoftware/mender && \
-  sudo install -t ${primary_dir}/${identitydir} -m 0755 \
+  sudo make install prefix=$primary_dir )
-      ${mender_dir}/mender-device-identity
-
-  sudo install -t ${primary_dir}/${inventorydir} -m 0755 \
-      ${mender_dir}/mender-inventory-*
 
-  sudo install -m 0644 ${mender_dir}/mender.service ${primary_dir}/${systemd_unitdir}
+  # If specified, replace Mender client binary
+  if [ -n "${mender_client}" ]; then
+    sudo install -m 0755 ${mender_client} ${primary_dir}/${bindir}/mender
+  fi
 
   # Enable menderd service starting on boot.
   sudo ln -sf /lib/systemd/system/mender.service \
       ${primary_dir}/etc/systemd/system/multi-user.target.wants/mender.service
 
-  sudo install -m 0644 ${mender_dir}/mender.conf ${primary_dir}/${sysconfdir}
+  # By default production settings configuration is installed
+  if [ -n "${demo}" ] && [ ${demo} -eq 1 ]; then
+    sudo install -m 0644 ${primary_dir}/${sysconfdir}/mender.conf.demo ${primary_dir}/${sysconfdir}/mender.conf
+  fi
 
-  sudo install -m 0444 ${mender_dir}/server.crt ${primary_dir}/${sysconfdir}
+  # If specified, replace server URL
+  if [ -n "${server_url}" ]; then
+    jq_inplace '.ServerURL = \"'${server_url}'\"' ${primary_dir}/${sysconfdir}/mender.conf
+  fi
 
-  sudo install -m 0644 ${mender_dir}/artifact_info ${primary_dir}/${sysconfdir}
+  # Set tenant token
+  if [ -n "${tenant_token}" ]; then
+    jq_inplace '.TenantToken = \"'${tenant_token}'\"' ${primary_dir}/${sysconfdir}/mender.conf
+  fi
+
+  # Append RootfsPartA/B to mender.conf
+  append_rootfs_configuration ${primary_dir}/${sysconfdir}/mender.conf
 
-  sudo install -m 0644 ${mender_dir}/version ${primary_dir}/${sysconfdir}/scripts
+  # Set artifact name
+  if [ -n "${artifact_name}" ]; then
+    sudo sh -c -e "echo artifact_name=${artifact_name} > ${primary_dir}/${sysconfdir}/artifact_info";
+  fi
 
+  # Set demo server
   if [ -n "${demo_host_ip}" ]; then
     sudo sh -c -e "echo '$demo_host_ip docker.mender.io s3.docker.mender.io' >> $primary_dir/etc/hosts";
+    jq_inplace '.ServerURL = \"https://docker.mender.io\"' ${primary_dir}/${sysconfdir}/mender.conf
   fi
 
+  # Install provided or demo certificate
   if [ -n "${server_cert}" ]; then
-    sudo install -m 0444 ${server_cert} ${primary_dir}/${sysconfdir}
+    sudo install -m 0444 ${server_cert} ${primary_dir}/${sysconfdir}/server.crt
+  else
+    sudo install -m 0444 ${mender_dir}/server.demo.crt ${primary_dir}/${sysconfdir}/server.crt
   fi
 }
 
@@ -246,11 +200,6 @@ do_install_mender() {
     show_help
   fi
 
-  if [ -z "${mender_client}" ]; then
-    log "Mender client binary not set. Aborting."
-    show_help
-  fi
-
   if [ -z "${device_type}" ]; then
     log "Target device type name not set. Aborting."
     show_help
@@ -272,19 +221,18 @@ do_install_mender() {
     show_help
   fi
 
-  # TODO: more error checking of server types
-  if [ -n "${tenant_token}" ]; then
-    mender_tenant_token=$(echo ${tenant_token} | tr -d '\n')
-    mender_server_url="https://hosted.mender.io"
-  fi
-
-  if [ -n "${server_url}" ]; then
-    mender_server_url=${server_url}
-  fi
-
   [ ! -f $mender_disk_image ] && \
       { log "$mender_disk_image - file not found. Aborting."; exit 1; }
 
+  test -n "$(go version)" || \
+      { log "go binary not found in PATH. Aborting."; exit 1; }
+
+  test -n "$GOPATH" || \
+      { log "GOPATH not set. Aborting."; exit 1; }
+
+  test -d $GOPATH/src/github.com/mendersoftware/mender || \
+      { log "mender source not found in \$GOPATH/src/github.com/mendersoftware/mender. Aborting."; exit 1; }
+
   # Mount rootfs partition A.
   create_device_maps $mender_disk_image mender_disk_mappings
 

docker-build

@@ -4,6 +4,6 @@ set -e
 
 IMAGE_NAME=mender-convert
 
-MENDER_CLIENT_VERSION="1.7.0"
+MENDER_CLIENT_VERSION="2.0.0b1-build1"
 
 docker build . -t ${IMAGE_NAME} --build-arg mender_client_version=${MENDER_CLIENT_VERSION}

mender-convert

@@ -64,7 +64,7 @@ Examples:
                 --raw-disk-image <raw_disk_image_path>
                 [--mender-disk-image <mender_image_name>]
                 --device-type <beaglebone | raspberrypi3>
-                --mender-client <mender_binary_path>
+                [--mender-client <mender_binary_path>]
                 --artifact-name release-1_1.5.0
                 --bootloader-toolchain arm-linux-gnueabihf
                 --demo-host-ip 192.168.10.2
@@ -306,6 +306,10 @@ do_raw_disk_image_create_partitions() {
     "qemux86_64")
       do_make_sdimg_qemux86_64
       ;;
+    *)
+      log "Error: unsupported device type $device_type"
+      exit 1
+      ;;
   esac
 
   rc=$?
@@ -406,7 +410,7 @@ do_install_mender_to_mender_disk_image() {
   ((step++))
 
   if [ -z "$mender_disk_image" ] || [ -z "$device_type" ] || \
-     [ -z "$mender_client" ] || [ -z "$artifact_name" ]; then
+     [ -z "$artifact_name" ]; then
     show_help
     return 1
   fi
@@ -417,7 +421,11 @@ do_install_mender_to_mender_disk_image() {
       { log "Error: incorrect device type. Aborting."; return 1; }
 
   # mender-image-1.5.0
-  stage_4_args="-m $mender_disk_image -d $device_type -g ${mender_client} -a ${artifact_name}"
+  stage_4_args="-m $mender_disk_image -d $device_type -a ${artifact_name}"
+
+  if [ -n "$mender_client" ]; then
+    stage_4_args="${stage_4_args} --mender-client ${mender_client}"
+  fi
 
   if [ -n "$demo" ] && [ ${demo} -eq 1 ]; then
     stage_4_args="${stage_4_args} --demo"
@@ -490,6 +498,10 @@ do_install_bootloader_to_mender_disk_image() {
       update_test_config_file $device_type distro-feature "mender-uboot" \
                               mount-location "\/uboot"
       ;;
+    *)
+      log "Error: unsupported device type $device_type"
+      exit 1
+      ;;
   esac
 
   return 0
@@ -627,7 +639,7 @@ do_mender_disk_image_to_artifact() {
 
 do_from_raw_disk_image() {
   if [ -z "$raw_disk_image" ] || [ -z "$device_type" ] ||  \
-     [ -z "$artifact_name" ] || [ -z "$mender_client" ] || \
+     [ -z "$artifact_name" ] || \
      [ -z "$bootloader_toolchain" ]; then
     show_help
     return 1
@@ -702,7 +714,7 @@ while (( "$#" )); do
       shift 2
       ;;
     -c | --server-cert)
-      server_cert=$2
+      server_cert=$(get_path $2)
       shift 2
       ;;
     -u | --server-url)

mender-convert-functions.sh

@@ -211,6 +211,10 @@ set_mender_disk_alignment() {
       local lvar_uboot_env_size=$(( $lvar_partition_alignment * 2 ))
       local lvar_vfat_storage_offset=$(( $lvar_partition_alignment + $lvar_uboot_env_size ))
       ;;
+    *)
+      log "Error: unsupported device type $1"
+      exit 1
+      ;;
   esac
 
   eval $rvar_partition_alignment="'$lvar_partition_alignment'"
@@ -768,6 +772,10 @@ set_fstab() {
       blk_device=hda
       data_id=5
       ;;
+    *)
+      log "Error: unsupported device type $device_type"
+      exit 1
+      ;;
   esac
 
   # Add Mender specific entries to fstab.