diff --git a/README.md b/README.md new file mode 100644 index 0000000..b1e4831 --- /dev/null +++ b/README.md @@ -0,0 +1,63 @@ +[![Build Status](https://travis-ci.org/mendersoftware/mender-crossbuild.svg?branch=master)](https://travis-ci.org/mendersoftware/mender-crossbuild) +[![codecov](https://codecov.io/gh/mendersoftware/mender-crossbuild/branch/master/graph/badge.svg)](https://codecov.io/gh/mendersoftware/mender-crossbuild) + +Mender: over-the-air updater for embedded Linux devices +============================================== + +Mender is an open source over-the-air (OTA) software updater for embedded Linux +devices. Mender comprises a client running at the embedded device, as well as +a server that manages deployments across many devices. + +Embedded product teams often end up creating homegrown updaters at the last +minute due to the need to fix bugs in field-deployed devices. However, the most +important requirement for an embedded update process is *robustness*, for example +loss of power at any time should not brick a device. This creates a challenge +given the time constraints to develop and maintain a homegrown updater. + +Mender aims to address this challenge with a *robust* and *easy to use* updater +for embedded Linux devices, which is open source and available to anyone. + +Robustness is ensured with *atomic* image-based deployments using a dual A/B +rootfs partition layout. This makes it always possible to roll back to a working state, even +when losing power at any time during the update process. + +Ease of use is addressed with an intuitive UI, [comprehensive documentation](https://docs.mender.io/), a +[meta layer for the Yocto Project](https://github.com/mendersoftware/meta-mender) for *easy integration into existing environments*, +and high quality software (see the test coverage badge). + +This repository contains the Mender client updater, which can be run in standalone mode +(manually triggered through its command line interface) or managed mode (connected to the Mender server). + +Mender not only provides the client-side updater, but also the backend and UI +for managing deployments as open source. The Mender server is +designed as a microservices architecture and comprises several repositories. + + +## Generic conversion tool + +A tool for taking an existing embedded image (Debian, Ubuntu, Raspbian, etc) and converting it to a Mender image by restructuring partition table and adding necessary files. + +Since we are unlikely to be able to patch U-Boot this way, this depends on U-Boot/UEFI functionality. + +## Contributing + +We welcome and ask for your contribution. If you would like to contribute to Mender, please read our guide on how to best get started [contributing code or documentation](https://github.com/mendersoftware/mender/blob/master/CONTRIBUTING.md). + +## License + +Mender is licensed under the Apache License, Version 2.0. See [LICENSE](https://github.com/mendersoftware/mender-crossbuild/blob/master/LICENSE) for the full license text. + +## Security disclosure + +We take security very seriously. If you come across any issue regarding +security, please disclose the information by sending an email to +[security@mender.io](security@mender.io). Please do not create a new public +issue. We thank you in advance for your cooperation. + +## Connect with us + +* Join our [Google group](https://groups.google.com/a/lists.mender.io/forum/#!forum/mender) +* Follow us on [Twitter](https://twitter.com/mender_io?target=_blank). Please + feel free to tweet us questions. +* Fork us on [Github](https://github.com/mendersoftware) +* Email us at [contact@mender.io](mailto:contact@mender.io) diff --git a/bbb-convert-stage-2.sh b/bbb-convert-stage-2.sh new file mode 100755 index 0000000..c2be2b9 --- /dev/null +++ b/bbb-convert-stage-2.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +sdimg_boot_dir=$1 +embedded_rootfs_dir=$2 +uboot_backup_dir=${embedded_rootfs_dir}/opt/backup/uboot + +echo $sdimg_boot_dir +echo $embedded_rootfs_dir + +[ ! -d "${sdimg_boot_dir}" ] && \ + { echo "Error: boot location not mounted."; exit 1; } +[ ! -d "${embedded_rootfs_dir}" ] && \ + { echo "Error: embedded content not mounted."; exit 1; } +[[ ! -f $uboot_backup_dir/MLO || ! -f $uboot_backup_dir/u-boot.img ]] && \ + { echo "Error: cannot find U-Boot related files."; exit 1; } + +set_uenv() { + cat <<- 'EOF' | sudo tee --append $sdimg_boot_dir/uEnv.txt 2>&1 >/dev/null + loadaddr=0x82000000 + fdtaddr=0x88000000 + rdaddr=0x88080000 + + initrd_high=0xffffffff + fdt_high=0xffffffff + + loadximage=echo debug: [/boot/vmlinuz-${uname_r}] ... ; load mmc 0:2 ${loadaddr} /boot/vmlinuz-${uname_r} + loadxfdt=echo debug: [/boot/dtbs/${uname_r}/${fdtfile}] ... ;load mmc 0:2 ${fdtaddr} /boot/dtbs/${uname_r}/${fdtfile} + loadxrd=echo debug: [/boot/initrd.img-${uname_r}] ... ; load mmc 0:2 ${rdaddr} /boot/initrd.img-${uname_r}; setenv rdsize ${filesize} + loaduEnvtxt=load mmc 0:2 ${loadaddr} /boot/uEnv.txt ; env import -t ${loadaddr} ${filesize}; + check_dtb=if test -n ${dtb}; then setenv fdtfile ${dtb};fi; + loadall=run loaduEnvtxt; run check_dtb; run loadximage; run loadxrd; run loadxfdt; + + mmcargs=setenv bootargs console=tty0 console=${console} ${optargs} ${cape_disable} ${cape_enable} root=/dev/mmcblk0p2 rootfstype=${mmcrootfstype} ${cmdline} + + uenvcmd=run loadall; run mmcargs; echo debug: [${bootargs}] ... ; echo debug: [bootz ${loadaddr} ${rdaddr}:${rdsize} ${fdtaddr}] ... ; bootz ${loadaddr} ${rdaddr}:${rdsize} ${fdtaddr}; + EOF +} + +## dd if=MLO of=${sdimg_file} count=1 seek=1 bs=128k +## dd if=u-boot.img of=${sdimg_file} count=2 seek=1 bs=384k +# Copy U-Boot related files. +sudo cp ${uboot_backup_dir}/MLO ${sdimg_boot_dir} +sudo cp ${uboot_backup_dir}/u-boot.img ${sdimg_boot_dir} + +# Create U-Boot purposed uEnv.txt file. +set_uenv + +echo -e "\nStage done." + +exit 0 diff --git a/bbb-convert-stage-3.sh b/bbb-convert-stage-3.sh new file mode 100755 index 0000000..dda63ae --- /dev/null +++ b/bbb-convert-stage-3.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +sdimg_primary_dir=$1 +embedded_rootfs_dir=$2 + +[ ! -d "${sdimg_primary_dir}" ] && \ + { echo "Error: rootfs location not mounted."; exit 1; } +[ ! -d "${embedded_rootfs_dir}" ] && \ + { echo "Error: embedded content not mounted."; exit 1; } + +# Copy rootfs partition. +sudo cp -ar ${embedded_rootfs_dir}/* ${sdimg_primary_dir} + +# Add mountpoints. +sudo install -d -m 755 ${sdimg_primary_dir}/boot/efi +sudo install -d -m 755 ${sdimg_primary_dir}/data + +echo -e "\nStage done." + +exit 0 diff --git a/bbb-convert-stage-5.sh b/bbb-convert-stage-5.sh new file mode 100755 index 0000000..a5f87d8 --- /dev/null +++ b/bbb-convert-stage-5.sh @@ -0,0 +1,304 @@ +#!/bin/bash + +show_help() { +cat << EOF + +Tool adding GRUB specific files to Mender compliant .sdimg image file. + +Usage: $0 [options] + + Options: [-i|--image | -t|--toolchain -k|--keep | -d|--device-type] + + --image - .sdimg image generated with mender-conversion-tool + --toolchain - ARM specific toolchain path + --keep - prevent deleting GRUB workspace + --device-type - target device type identification + + Note: supported device types are: beaglebone, raspberrypi3 + + Examples: + + ./mender-conversion-tool.sh install_bootloader --image + --device-type beaglebone --toolchain arm-linux-gnueabihf + + Note: toolchain naming convention is arch-vendor-(os-)abi + + arch is for architecture: arm, mips, x86, i686... + vendor is tool chain supplier: apple, Codesourcery, Linux, + os is for operating system: linux, none (bare metal) + abi is for application binary interface convention: eabi, gnueabi, gnueabihf + +EOF +} + +tool_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +output_dir=${tool_dir}/output +grubenv_dir=$output_dir/grubenv +grubenv_build_dir=$output_dir/grubenv_build +image= +toolchain= +device_type= +keep=0 +efi_boot=EFI/BOOT +EFI_STUB_VER="4.12.0" + +declare -a sdimgmappings + +version() { + echo "$@" | awk -F. '{ printf("%d%03d%03d%03d\n", $1,$2,$3,$4); }' +} + +get_kernel_version() { + local search_path=$1/boot + local resultvar=$2 + + [ ! -d "$search_path" ] && { return 1; } + + kernel_image=$(ls -1 $search_path | grep -E "^vmlinuz") + + local myresult=${kernel_image#*-} + eval $resultvar="'$myresult'" + return 0 +} + +# Takes following arguments: +# +# $1 - linux kernel version +build_env_lock_boot_files() { + mkdir -p $grubenv_dir + mkdir -p $grubenv_build_dir + + git clone https://github.com/mendersoftware/grub-mender-grubenv.git $grubenv_dir + cd $grubenv_dir + + # Prepare configuration file. + cp mender_grubenv_defines.example mender_grubenv_defines + + local uname_r=$1 + local kernel_imagetype=vmlinuz-${uname_r} + local kernel_devicetree=dtbs/${uname_r}/am335x-boneblack.dtb + + sed -i '/^kernel_imagetype/s/=.*$/='${kernel_imagetype}'/' mender_grubenv_defines + sed -i '/^kernel_devicetree/s/=.*$/='${kernel_devicetree//\//\\/}'/' mender_grubenv_defines + + make + make DESTDIR=$grubenv_build_dir install + rm -rf $grubenv_dir +} + +# Takes following arguments: +# +# $1 - linux kernel version +build_grub_efi() { + local grub_arm=$output_dir/grub/arm + local grub_linux=$output_dir/grub/linux + local grub_repo_vc_dir=$output_dir/grub/.git + local repo_clean=0 + + local version=$(echo $1 | sed 's/[^0-9.]*\([0-9.]*\).*/\1/') + + # Build grub modules for arm platform and executables for the host. + if [ ! -d $grub_repo_vc_dir ]; then + git clone git://git.savannah.gnu.org/grub.git $output_dir/grub + local repo_clean=1 + fi + + cd $output_dir/grub/ + [[ repo_clean -eq 0 ]] && { make --quiet clean; make --quiet distclean; } + + if [ $(version $version) -lt $(version $EFI_STUB_VER) ]; then + # To avoid error message: "plain image kernel not supported - rebuild + # with CONFIG_(U)EFI_STUB enabled" - use a specific commit. + git checkout 9b37229f0 + fi + + mkdir -p $grub_arm + mkdir -p $grub_linux + + # First build linux tools. + ./autogen.sh + ./configure --quiet CC=gcc --target=x86_64 --with-platform=efi --prefix=$grub_linux + make --quiet + make --quiet install + + # Clean workspace. + make --quiet clean + make --quiet distclean + + # Now build ARM modules. + ./configure --host=$toolchain --with-platform=efi --prefix=$grub_arm \ + CFLAGS="-Os -march=armv7-a" CCASFLAGS="-march=armv7-a" --disable-werror + make --quiet + make --quiet install + + # Build grub.efi binary. + $grub_linux/bin/grub-mkimage -v -p /$efi_boot -o grub.efi --format=arm-efi \ + -d $grub_arm/lib/grub/arm-efi/ boot linux ext2 fat serial part_msdos \ + part_gpt normal efi_gop iso9660 configfile search loadenv test cat echo \ + gcry_sha256 halt hashsum loadenv reboot + + rc=$? + + [[ $rc -ne 0 ]] && { return 1; } || { return 0; } +} + +# Takes following arguments: +# +# $1 - boot partition mountpoint +set_uenv() { + local boot_dir=$1 + # Erase/create uEnv.txt file. + sudo install -b -m 644 /dev/null $boot_dir/uEnv.txt + + # Fill uEnv.txt file. + cat <<- 'EOF' | sudo tee $boot_dir/uEnv.txt 2>&1 >/dev/null + bootdir= + grubfile=EFI/BOOT/grub.efi + grubaddr=0x80007fc0 + loadgrub=fatload mmc 0:1 ${grubaddr} ${grubfile} + grubstart=bootefi ${grubaddr} + uenvcmd=mmc rescan; run loadgrub; run grubstart; + EOF +} + +# Takes following arguments: +# +# $1 - boot partition mountpoint +# $2 - primary partition mountpoint +install_files() { + local grubenv_dir=$grubenv_build_dir/boot/efi/EFI/BOOT/ + local boot_dir=$1 + local rootfs_dir=$2 + local efi_boot_dir=$boot_dir/$efi_boot + + # Make sure env, lock, lock.sha256sum files exists in working directory. + [[ ! -d $grubenv_dir/mender_grubenv1 || ! -d $grubenv_dir/mender_grubenv2 ]] && \ + { echo "Error: cannot find mender grub related files."; exit 1; } + + sudo install -d -m 755 $efi_boot_dir + + cd $grubenv_dir && find . -type f -exec sudo install -Dm 644 "{}" "$efi_boot_dir/{}" \; + cd ${output_dir} + + sudo install -m 0644 ${output_dir}/grub/grub.efi $efi_boot_dir + sudo install -m 0755 ${output_dir}/grub/arm/bin/grub-editenv $rootfs_dir/usr/bin + + sudo install -m 0755 $grubenv_build_dir/usr/bin/fw_printenv $rootfs_dir/sbin/fw_printenv + sudo install -m 0755 $grubenv_build_dir/usr/bin/fw_setenv $rootfs_dir/sbin/fw_setenv + + # Replace U-Boot default printenv/setenv commands. + sudo ln -fs /sbin/fw_printenv $rootfs_dir/usr/bin/fw_printenv + sudo ln -fs /sbin/fw_setenv $rootfs_dir/usr/bin/fw_setenv + + set_uenv $boot_dir +} + +do_install_bootloader() { + echo "Setting bootloader..." + + if [ -z "${image}" ]; then + echo ".sdimg image file not set. Aborting." + exit 1 + fi + + if [ -z "${toolchain}" ]; then + echo "ARM toolchain not set. Aborting." + exit 1 + fi + + if [ -z "${device_type}" ]; then + echo "Target device type name not set. Aborting." + exit 1 + fi + + if [[ $(which ${toolchain}-gcc) = 1 ]]; then + echo "Error: ARM GCC not found in PATH. Aborting." + exit 1 + fi + + [ ! -f $image ] && { echo "$image - file not found. Aborting."; exit 1; } + + # Map & mount Mender compliant image. + create_device_maps $image sdimgmappings + + mkdir -p $output_dir && cd $output_dir + + boot=${sdimgmappings[0]} + primary=${sdimgmappings[1]} + map_boot=/dev/mapper/"$boot" + map_primary=/dev/mapper/"$primary" + path_boot=$output_dir/sdimg/boot + path_primary=$output_dir/sdimg/primary + mkdir -p ${path_boot} ${path_primary} + + sudo mount ${map_boot} ${path_boot} + sudo mount ${map_primary} ${path_primary} + + get_kernel_version ${path_primary} kernel_version + echo -e "\nKernel version: $kernel_version" + + build_env_lock_boot_files $kernel_version + + build_grub_efi $kernel_version + rc=$? + + [[ $rc -ne 0 ]] && { echo "Error: grub.efi building failure. Aborting."; } \ + || { echo "Successful grub.efi building."; \ + install_files ${path_boot} ${path_primary}; } + + # Back to working directory. + cd $tool_dir && sync + + detach_device_maps ${sdimgmappings[@]} + # Clean files. + rm -rf $output_dir/sdimg + +# [[ $keep -eq 0 ]] && { rm -rf $output_dir; } + echo -e "\nAll done." +} + +PARAMS="" + +while (( "$#" )); do + case "$1" in + -i | --image) + image=$2 + shift 2 + ;; + -t | --toolchain) + toolchain=$2 + shift 2 + ;; + -d | --device-type) + device_type=$2 + shift 2 + ;; + -k | --keep) + keep=1 + shift 1 + ;; + -h | --help) + show_help + exit 0 + ;; + --) + shift + break + ;; + -*) + echo "Error: unsupported option $1" >&2 + exit 1 + ;; + *) + PARAMS="$PARAMS $1" + shift + ;; + esac +done + +eval set -- "$PARAMS" + +# Some commands expect elevated privileges. +sudo true + +do_install_bootloader diff --git a/convert-stage-4.sh b/convert-stage-4.sh new file mode 100755 index 0000000..80fb3e1 --- /dev/null +++ b/convert-stage-4.sh @@ -0,0 +1,351 @@ +#!/bin/bash + +show_help() { + cat << EOF + +Mender executables, service and configuration files installer. + +Usage: $0 [options] + + Options: [-i|--image | -m|--mender | -a|--artifact | -d|--device-type | + -p|--demo-ip | -u| --production-url | -o| --hosted-token] + + --image - .sdimg generated with mender-conversion-tool + --mender - mender client binary file + --artifact - artifact info + --device-type - target device type identification + --demo-ip - Mender demo server IP address + --production-url - Mender production server url + --certificate - Mender server certificate + + Examples: + + ./mender-conversion-tool.sh install_mender --image + --device-type beaglebone --artifact release-1_1.5.0 + --server 192.168.10.2 --mender + +EOF + exit 1 +} + +tool_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +output_dir=${tool_dir}/output + +mender_client_repo="https://raw.githubusercontent.com/mendersoftware/mender" +mender_client_revision="1.6.x" +meta_mender_repo="https://raw.githubusercontent.com/mendersoftware/meta-mender" +meta_mender_revision="sumo" + +mender_dir=$output_dir/mender +device_type= +artifact= +# Mender demo server IP address. +demo_ip= +# Mender production server url. +production_url= +# Used server url. +server_url="https://docker.mender.io" +# Mender production certificate. +certificate= +# Mender hosted token. +hosted_token= +# Mender tenant token. +tenant_token="dummy" + +declare -a sdimgmappings + +create_client_files() { + cat <<- EOF > $mender_dir/mender.service + [Unit] + Description=Mender OTA update service + After=systemd-resolved.service + + [Service] + Type=idle + User=root + Group=root + ExecStartPre=/bin/mkdir -p -m 0700 /data/mender + ExecStartPre=/bin/ln -sf /etc/mender/tenant.conf /var/lib/mender/authtentoken + ExecStart=/usr/bin/mender -daemon + Restart=on-abort + + [Install] + WantedBy=multi-user.target + EOF + + cat <<- EOF > $mender_dir/mender.conf + { + "InventoryPollIntervalSeconds": 5, + "RetryPollIntervalSeconds": 1, + "RootfsPartA": "/dev/mmcblk0p2", + "RootfsPartB": "/dev/mmcblk0p3", + "ServerCertificate": "/etc/mender/server.crt", + "ServerURL": "$server_url", + "TenantToken": "$tenant_token", + "UpdatePollIntervalSeconds": 5 + } + EOF + + cat <<- EOF > $mender_dir/artifact_info + artifact_name=${artifact} + EOF + + # Version file + echo -n "2" > $mender_dir/version + + cat <<- EOF > $mender_dir/device_type + device_type=${device_type} + EOF + + case "$device_type" in + "beaglebone") + cat <<- EOF > $mender_dir/fw_env.config + /dev/mmcblk0 0x800000 0x20000 + /dev/mmcblk0 0x1000000 0x20000 + EOF + ;; + "raspberrypi3") + cat <<- EOF > $mender_dir/fw_env.config + /dev/mmcblk0 0x400000 0x4000 + /dev/mmcblk0 0x800000 0x4000 + EOF + ;; + esac +} + +get_mender_files_from_upstream() { + mkdir -p $mender_dir + + echo -e "Downloading inventory & identity scripts..." + + wget -q -O $mender_dir/mender-device-identity \ + $mender_client_repo/$mender_client_revision/support/mender-device-identity + wget -q -O $mender_dir/mender-inventory-bootloader-integration \ + $mender_client_repo/$mender_client_revision/support/mender-inventory-bootloader-integration + wget -q -O $mender_dir/mender-inventory-hostinfo \ + $mender_client_repo/$mender_client_revision/support/mender-inventory-hostinfo + wget -q -O $mender_dir/mender-inventory-network \ + $mender_client_repo/$mender_client_revision/support/mender-inventory-network + wget -q -O $mender_dir/mender-inventory-os \ + $mender_client_repo/$mender_client_revision/support/mender-inventory-os + wget -q -O $mender_dir/mender-inventory-rootfs-type \ + $mender_client_repo/$mender_client_revision/support/mender-inventory-rootfs-type + wget -q -O $mender_dir/server.crt \ + $meta_mender_repo/$meta_mender_revision/meta-mender-demo/recipes-mender/mender/files/server.crt +} + +install_files() { + local primary_dir=$1 + local data_dir=$2 + + identitydir="usr/share/mender/identity" + inventorydir="usr/share/mender/inventory" + sysconfdir="etc/mender" + bindir="usr/bin" + systemd_unitdir="lib/systemd/system" + localstatedir="var/lib/mender" + dataconfdir="mender" + databootdir="u-boot" + + # Prepare 'data' partition + sudo install -d -m 755 ${data_dir}/${dataconfdir} + sudo install -d -m 755 ${data_dir}/${databootdir} + + sudo install -m 0644 ${mender_dir}/device_type ${data_dir}/${dataconfdir} + sudo install -m 0644 ${mender_dir}/fw_env.config ${data_dir}/${databootdir} + + sudo ln -sf /data/${databootdir}/fw_env.config ${primary_dir}/etc/fw_env.config + + # Prepare 'primary' partition + [ ! -d "$primary_dir/data" ] && \ + { echo "'data' mountpoint missing. Adding"; \ + sudo install -d -m 755 ${primary_dir}/data; } + + case "$device_type" in + "beaglebone") + [ ! -d "$primary_dir/boot/efi" ] && \ + { echo "'/boot/efi' mountpoint missing. Adding"; \ + sudo install -d -m 755 ${primary_dir}/boot/efi; } + ;; + "raspberrypi3") + [ ! -d "$primary_dir/uboot" ] && \ + { echo "'/boot/efi' mountpoint missing. Adding"; \ + sudo install -d -m 755 ${primary_dir}/uboot; } + ;; + esac + + sudo install -d ${primary_dir}/${identitydir} + sudo install -d ${primary_dir}/${inventorydir} + sudo install -d ${primary_dir}/${sysconfdir} + sudo install -d ${primary_dir}/${sysconfdir}/scripts + + sudo ln -s /data/${dataconfdir} ${primary_dir}/${localstatedir} + + sudo install -m 0755 ${mender} ${primary_dir}/${bindir}/mender + + sudo install -t ${primary_dir}/${identitydir} -m 0755 \ + ${mender_dir}/mender-device-identity + + sudo install -t ${primary_dir}/${inventorydir} -m 0755 \ + ${mender_dir}/mender-inventory-* + + sudo install -m 0644 ${mender_dir}/mender.service ${primary_dir}/${systemd_unitdir} + + # Enable menderd service starting on boot. + sudo ln -s /lib/systemd/system/mender.service \ + ${primary_dir}/etc/systemd/system/multi-user.target.wants/mender.service + + sudo install -m 0644 ${mender_dir}/mender.conf ${primary_dir}/${sysconfdir} + + sudo install -m 0444 ${mender_dir}/server.crt ${primary_dir}/${sysconfdir} + + sudo install -m 0644 ${mender_dir}/artifact_info ${primary_dir}/${sysconfdir} + + sudo install -m 0644 ${mender_dir}/version ${primary_dir}/${sysconfdir}/scripts + + if [ -n "${demo_ip}" ]; then + echo "$demo_ip docker.mender.io s3.docker.mender.io" | sudo tee -a $primary_dir/etc/hosts + fi + + if [ -n "${certificate}" ]; then + sudo install -m 0444 ${certificate} ${primary_dir}/${sysconfdir} + fi +} + +do_install_mender() { + if [ -z "${image}" ]; then + echo ".sdimg image file not set. Aborting." + show_help + fi + + if [ -z "${mender}" ]; then + echo "Mender client binary not set. Aborting." + show_help + fi + + if [ -z "${device_type}" ]; then + echo "Target device type name not set. Aborting." + show_help + fi + + if [ -z "${artifact}" ]; then + echo "Artifact info not set. Aborting." + show_help + fi + + if [ -z "${production_url}" ] && [ -z "${demo_ip}" ] && \ + [ -z "${hosted_token}" ]; then + echo "No server type specified. Aborting." + show_help + fi + + if [ -n "${production_url}" ] && [ -n "${demo_ip}" ]; then + echo "Incompatible server type choice. Aborting." + show_help + fi + + # TODO: more error checking of server types + if [ -n "${hosted_token}" ]; then + tenant_token=$(cat ${hosted_token} | tr -d '\n') + server_url="https://hosted.mender.io" + fi + + if [ -n "${production_url}" ]; then + server_url=${production_url} + fi + + [ ! -f $image ] && { echo "$image - file not found. Aborting."; exit 1; } + + # Mount rootfs partition A. + create_device_maps $image sdimgmappings + + mkdir -p $output_dir && cd $output_dir + + primary=${sdimgmappings[1]} + data=${sdimgmappings[3]} + + map_primary=/dev/mapper/"$primary" + map_data=/dev/mapper/"$data" + path_primary=$output_dir/sdimg/primary + path_data=$output_dir/sdimg/data + mkdir -p ${path_primary} ${path_data} + + sudo mount ${map_primary} ${path_primary} + sudo mount ${map_data} ${path_data} + + # Get Mender client related files. + get_mender_files_from_upstream + # Create all necessary client's files. + create_client_files + + # Create all required paths and install files. + install_files ${path_primary} ${path_data} + + # Back to working directory. + cd $tool_dir && sync + + # Clean stuff. + detach_device_maps ${sdimgmappings[@]} + rm -rf $output_dir/sdimg +} + +PARAMS="" + +while (( "$#" )); do + case "$1" in + -i | --image) + image=$2 + shift 2 + ;; + -m | --mender) + mender=$2 + shift 2 + ;; + -d | --device-type) + device_type=$2 + shift 2 + ;; + -a | --artifact) + artifact=$2 + shift 2 + ;; + -p | --demo-ip) + demo_ip=$2 + shift 2 + ;; + -c | --certificate) + certificate=$2 + shift 2 + ;; + -u | --production-url) + production_url=$2 + shift 2 + ;; + -o | --hosted-token) + hosted_token=$2 + shift 2 + ;; + -h | --help) + show_help + ;; + --) + shift + break + ;; + -*) + echo "Error: unsupported option $1" >&2 + exit 1 + ;; + *) + PARAMS="$PARAMS $1" + shift + ;; + esac +done + +eval set -- "$PARAMS" + +# Some commands expect elevated privileges. +sudo true + +do_install_mender diff --git a/mender-conversion-functions.sh b/mender-conversion-functions.sh new file mode 100755 index 0000000..ffadc6f --- /dev/null +++ b/mender-conversion-functions.sh @@ -0,0 +1,618 @@ +#!/bin/bash + +# Erase block size 8MiB +erase_block=8388608 +heads=255 +sectors=63 + +declare -a sdimg_partitions=("boot" "primary" "secondary" "data") +declare -a embedded_partitions=("boot" "rootfs") + +tool_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +output_dir=${tool_dir}/output + +embedded_base_dir=$output_dir/embedded +sdimg_base_dir=$output_dir/sdimg + +embedded_boot_dir=$embedded_base_dir/boot +embedded_rootfs_dir=$embedded_base_dir/rootfs +sdimg_boot_dir=$sdimg_base_dir/boot +sdimg_primary_dir=$sdimg_base_dir/primary +sdimg_secondary_dir=$sdimg_base_dir/secondary +sdimg_data_dir=$sdimg_base_dir/data + +# Takes following arguments: +# +# $1 -relative file path +get_path() { + echo "$(cd "$(dirname "$1")"; pwd)/$(basename "$1")" +} + +get_part_number_from_device() { + case "$1" in + /dev/*[0-9]p[1-9]) + echo ${1##*[0-9]p} + ;; + /dev/[sh]d[a-z][1-9]) + echo ${1##*d[a-z]} + ;; + ubi[0-9]_[0-9]) + echo ${1##*[0-9]_} + ;; + [a-z]*\.sdimg[1-9]) + echo ${1##*\.sdimg} + ;; + /dev/mapper/*[0-9]p[1-9]) + echo ${1##*[0-9]p} + ;; + *) + echo "Could not determine partition number from $1" + exit 1 + ;; + esac +} + +# Takes following arguments: +# +# $1 - raw disk image +# $2 - boot partition start offset (in sectors) +# $3 - boot partition size (in sectors) +create_single_disk_partition_table() { + local device=$1 + local bootstart=$2 + local stopsector=$(( $3 - 1 )) + + sed -e 's/\s*\([\+0-9a-zA-Z]*\).*/\1/' << EOF | sudo fdisk $device + d # delete partition + n # new partition + p # primary partition + 1 # partion number 1 + ${bootstart} + +${stopsector} + a # set boot flag + w # write the partition table + q # and we're done +EOF +} + +# Takes following arguments: +# +# $1 - raw disk image +# $2 - root filesystem partition start offset (in sectors) +# $3 - root filesystem partition size (in sectors) +create_double_disk_partition_table() { + local device=$1 + local rootfsstart=$2 + local rootfsstop=$(( $3 - 1 )) + + sed -e 's/\s*\([\+0-9a-zA-Z]*\).*/\1/' << EOF | sudo fdisk $device + d # delete partition + 2 + n + p + 2 + ${rootfsstart} + +${rootfsstop} + w # write the partition table + q # and we're done +EOF +} + +# Takes following arguments: +# +# $1 - embedded image path +# +# Calculates following values: +# +# $2 - number of partitions +# $3 - size of the sector (in bytes) +# $4 - boot partition start offset (in sectors) +# $5 - boot partition size (in sectors) +# $6 - root filesystem partition start offset (in sectors) +# $7 - root filesystem partition size (in sectors) +# $8 - boot flag +get_image_info() { + local limage=$1 + local rvar_count=$2 + local rvar_sectorsize=$3 + local rvar_bootstart=$4 + local rvar_bootsize=$5 + local rvar_rootfsstart=$6 + local rvar_rootfssize=$7 + local rvar_bootflag=$8 + + local lbootsize=0 + local lsubname=${limage:0:8} + local lfdisk="$(fdisk -u -l ${limage})" + + local lparts=($(echo "${lfdisk}" | grep "^${lsubname}" | cut -d' ' -f1)) + local lcount=${#lparts[@]} + + local lsectorsize=($(echo "${lfdisk}" | grep '^Sector' | cut -d' ' -f4)) + + local lfirstpartinfo="$(echo "${lfdisk}" | grep "^${lparts[0]}")" + + idx_start=2 + idx_size=4 + + if [[ $lcount -gt 1 ]]; then + local lsecondpartinfo="$(echo "${lfdisk}" | grep "^${lparts[1]}")" + local lsecondpartstart=($(echo "${lsecondpartinfo}" | tr -s ' ' | cut -d' ' -f${idx_start})) + local lsecondpartsize=($(echo "${lsecondpartinfo}" | tr -s ' ' | cut -d' ' -f${idx_size})) + fi + + eval $rvar_bootflag="0" + if [[ "$lfirstpartinfo" =~ .*\*.* ]]; then + eval $rvar_bootflag="1" + ((idx_start+=1)) + ((idx_size+=1)) + fi + + lfirstpartsize=($(echo "${lfirstpartinfo}" | tr -s ' ' | cut -d' ' -f${idx_size})) + lfirstpartstart=($(echo "${lfirstpartinfo}" | tr -s ' ' | cut -d' ' -f${idx_start})) + + eval $rvar_count="'$lcount'" + eval $rvar_sectorsize="'$lsectorsize'" + eval $rvar_bootstart="'$lfirstpartstart'" + eval $rvar_bootsize="'$lfirstpartsize'" + eval $rvar_rootfsstart="'$lsecondpartstart'" + eval $rvar_rootfssize="'$lsecondpartsize'" + + [[ $lcount -gt 2 ]] && \ + { echo "Unsupported type of source image. Aborting."; return 1; } || \ + { return 0; } +} + +# Takes following arguments: +# +# $1 - raw disk image path +# +# Calculates following values: +# +# $2 - number of partitions +# $3 - size of the sector (in bytes) +# $4 - rootfs A partition start offset (in sectors) +# $5 - rootfs A partition size (in sectors) +# $6 - rootfs B partition start offset (in sectors) +# $7 - rootfs B partition size (in sectors) +get_sdimg_info() { + local limage=$1 + local rvar_count=$2 + local rvar_sectorsize=$3 + local rvar_rootfs_a_start=$4 + local rvar_rootfs_a_size=$5 + local rvar_rootfs_b_start=$6 + local rvar_rootfs_b_size=$7 + + local lsubname=${limage:0:8} + local lfdisk="$(fdisk -u -l ${limage})" + + local lparts=($(echo "${lfdisk}" | grep "^${lsubname}" | cut -d' ' -f1)) + local lcount=${#lparts[@]} + + if [[ $lcount -ne 4 ]]; then + echo "Error: invalid source .sdimg file. Aborting." + return 1 + else + local lsectorsize=($(echo "${lfdisk}" | grep '^Sector' | cut -d' ' -f4)) + + local lrootfs_a_info="$(echo "${lfdisk}" | grep "^${lparts[1]}")" + local lrootfs_b_info="$(echo "${lfdisk}" | grep "^${lparts[2]}")" + + idx_start=2 + idx_size=4 + + local lrootfs_a_start=($(echo "${lrootfs_a_info}" | tr -s ' ' | cut -d' ' -f${idx_start})) + local lrootfs_a_size=($(echo "${lrootfs_a_info}" | tr -s ' ' | cut -d' ' -f${idx_size})) + local lrootfs_b_start=($(echo "${lrootfs_b_info}" | tr -s ' ' | cut -d' ' -f${idx_start})) + local lrootfs_b_size=($(echo "${lrootfs_b_info}" | tr -s ' ' | cut -d' ' -f${idx_size})) + + eval $rvar_count="'$lcount'" + eval $rvar_sectorsize="'$lsectorsize'" + eval $rvar_rootfs_a_start="'$lrootfs_a_start'" + eval $rvar_rootfs_a_size="'$lrootfs_a_size'" + eval $rvar_rootfs_b_start="'$lrootfs_b_start'" + eval $rvar_rootfs_b_size="'$lrootfs_b_size'" + + return 0 + fi +} + +# Takes following arguments: +# +# $1 - size variable to be aligned +# $2 - size of the sector +# +align_partition_size() { + # Final size is aligned to 8MiB. + local rvar_size=$1 + local -n ref=$1 + + local size_in_bytes=$(( $ref * $2 )) + local reminder=$(( ${size_in_bytes} % ${erase_block} )) + + if [ $reminder -ne 0 ]; then + size_in_bytes=$(( $size_in_bytes - $reminder + ${erase_block} )) + fi + + local lsize=$(( $size_in_bytes / $2 )) + + eval $rvar_size="'$lsize'" +} + +# Takes following arguments: +# +# $1 - embedded image +# +# Returns: +# +# $2 - boot partition start offset (in sectors) +# $3 - boot partition size (in sectors) +# $4 - root filesystem partition size (in sectors) +# $5 - sector size (in bytes) +# $6 - image type +analyse_embedded_image() { + local image=$1 + local count= + local sectorsize= + local bootstart= + local bootsize= + local rootfsstart= + local rootfssize= + local bootflag= + + local rvar_bootstart=$2 + local rvar_bootsize=$3 + local rvar_rootfssize=$4 + local rvar_sectorsize=$5 + local rvar_imagetype=$6 + + get_image_info $image count sectorsize bootstart bootsize \ + rootfsstart rootfssize bootflag + + [[ $? -ne 0 ]] && \ + { echo "Error: invalid/unsupported embedded image. Aborting."; exit 1; } + + if [[ $count -eq 1 ]]; then + echo -e "\nDetected single partition embedded image." + rootfssize=$bootsize + # Default size of the boot partition: 16MiB. + bootsize=$(( ($erase_block * 2) / $sectorsize )) + elif [[ $count -eq 2 ]]; then + echo -e "\nDetected multipartition ($count) embedded image." + fi + + align_partition_size bootsize $sectorsize + align_partition_size rootfssize $sectorsize + + eval $rvar_bootstart="'$bootstart'" + eval $rvar_bootsize="'$bootsize'" + eval $rvar_rootfssize="'$rootfssize'" + eval $rvar_sectorsize="'$sectorsize'" + eval $rvar_imagetype="'$count'" + + echo -e "\nEmbedded image processing summary:\ + \nboot part start sector: ${bootstart}\ + \nboot part size (sectors): ${bootsize}\ + \nrootfs part size (sectors): ${rootfssize}\ + \nsector size (bytes): ${sectorsize}\n" +} + +# Takes following arguments: +# +# $1 - boot partition start offset (in sectors) +# $2 - boot partition size (in sectors) +# $3 - root filesystem partition size (in sectors) +# $4 - data partition size (in MB) +# $5 - sector size (in bytes) +# +# Returns: +# +# $6 - aligned data partition size (in sectors) +# $7 - final .sdimg file size (in bytes) +calculate_sdimg_size() { + local rvar_datasize=$6 + local rvar_sdimgsize=$7 + + local datasize=$(( ($4 * 1024 * 1024) / $5 )) + + align_partition_size datasize $5 + + local sdimgsize=$(( ($1 + $2 + 2 * ${3} + $datasize) * $5 )) + + eval $rvar_datasize="'$datasize'" + eval $rvar_sdimgsize="'$sdimgsize'" +} + +# Takes following arguments: +# +# $1 - raw disk image +unmount_partitions() { + echo "1. Check if device is mounted..." + is_mounted=`grep ${1} /proc/self/mounts | wc -l` + if [ ${is_mounted} -ne 0 ]; then + sudo umount ${1}?* + fi +} + +# Takes following arguments: +# +# $1 - raw disk image +erase_filesystem() { + echo "2. Erase filesystem..." + sudo wipefs --all --force ${1}?* + sudo dd if=/dev/zero of=${1} bs=1M count=100 +} + +# Takes following arguments: +# +# $1 - raw disk image path +# $2 - raw disk image size +create_sdimg() { + local lfile=$1 + local lsize=$2 + local bs=$(( 1024*1024 )) + local count=$(( ${lsize} / ${bs} )) + + echo -e "\nWriting $lsize bytes to .sdimg file..." + dd if=/dev/zero of=${lfile} bs=${bs} count=${count} +} + +# Takes following arguments: +# +# $1 - raw disk image path +# $2 - raw disk image size +# $3 - boot partition start offset +# $4 - boot partition size +# $5 - root filesystem partiotion size +# $6 - data partition size +# $7 - sector size +format_sdimg() { + local lfile=$1 + local lsize=$2 + +# if [ -z "$3" ]; then +# echo "Error: no root filesystem size provided" +# exit 1 +# fi + +# if [ -z "$2" ]; then +# size=$(sudo blockdev --getsize64 ${sdimg_file}) +# else +# size=$2 +# fi + + cylinders=$(( ${lsize} / ${heads} / ${sectors} / ${7} )) + rootfs_size=$(( $5 - 1 )) + pboot_offset=$(( ${4} - 1 )) + primary_start=$(( ${3} + ${pboot_offset} + 1 )) + secondary_start=$(( ${primary_start} + ${rootfs_size} + 1 )) + data_start=$(( ${secondary_start} + ${rootfs_size} + 1 )) + data_offset=$(( ${6} - 1 )) + + echo $3 ${pboot_offset} $primary_start $secondary_start $data_start $data_offset + + echo -e "\nFormatting .sdimg file..." + + sed -e 's/\s*\([\+0-9a-zA-Z]*\).*/\1/' << EOF | sudo fdisk ${lfile} + o # clear the in memory partition table + x + h + ${heads} + s + ${sectors} + c + ${cylinders} + r + n # new partition + p # primary partition + 1 # partition number 1 + ${3} # default - start at beginning of disk + +${pboot_offset} # 16 MB boot parttion + t + c + a + n # new partition + p # primary partition + 2 # partion number 2 + ${primary_start} # start immediately after preceding partition + +${rootfs_size} + n # new partition + p # primary partition + 3 # partion number 3 + ${secondary_start} # start immediately after preceding partition + +${rootfs_size} + n # new partition + p # primary partition + ${data_start} # start immediately after preceding partition + +${data_offset} + p # print the in-memory partition table + w # write the partition table + q # and we're done +EOF +} + +# Takes following arguments: +# +# $1 - raw disk file +# +# Returns: +# +# $2 - number of detected partitions +verify_sdimg() { + local lfile=$1 + local rvar_no_of_parts=$2 + + local limage=$(basename $lfile) + local partitions=($(fdisk -l -u ${limage} | grep '^mender' | cut -d' ' -f1)) + + local no_of_parts=${#partitions[@]} + + [[ $no_of_parts -eq 4 ]] || \ + { echo "Error: incorrect number of partitions: $no_of_parts. Aborting."; return 1; } + + eval $rvar_no_of_parts=="'$no_of_parts='" + + return 0 +} + +# Takes following arguments: +# +# $1 - raw disk image +# $2 - partition mappings holder +create_device_maps() { + local -n mappings=$2 + + if [[ -n "$1" ]]; then + mapfile -t mappings < <( sudo kpartx -v -a $1 | grep 'loop' | cut -d' ' -f3 ) + [[ ${#mappings[@]} -eq 0 ]] \ + && { echo "Error: partition mappings failed. Aborting."; exit 1; } \ + || { echo "Mapped ${#mappings[@]} partition(s)."; } + else + echo "Error: no device passed. Aborting." + exit 1 + fi + + sudo partprobe /dev/${mappings[0]%p*} + + echo "Mapper device: ${mappings[0]%p*}" +} + +# Takes following arguments: +# +# $1 - partition mappings holder +detach_device_maps() { + local mappings=($@) + + [ ${#mappings[@]} -eq 0 ] && { echo "Nothing to detach."; return; } + + local mapper=${mappings[0]%p*} + + for mapping in ${mappings[@]} + do + map_dev=/dev/mapper/"$mapping" + is_mounted=`grep ${map_dev} /proc/self/mounts | wc -l` + if [ ${is_mounted} -ne 0 ]; then + sudo umount -l $map_dev + fi + done + + sudo kpartx -d /dev/$mapper & + sudo losetup -d /dev/$mapper & + wait && sync +} + +# Takes following arguments: +# +# $1 - partition mappings holder +make_sdimg_filesystem() { + local mappings=($@) + echo -e "\nCreating filesystem for ${#mappings[@]} partitions..." + + for mapping in ${mappings[@]} + do + map_dev=/dev/mapper/"$mapping" + part_no=$(get_part_number_from_device $map_dev) + + echo -e "\nFormatting partition: ${part_no}..." + + if [[ part_no -eq 1 ]]; then + sudo mkfs.vfat -n ${sdimg_partitions[${part_no} - 1]} $map_dev + else + sudo mkfs.ext4 -L ${sdimg_partitions[${part_no} - 1]} $map_dev + fi + done +} + +# Takes following arguments: +# +# $1 - partition mappings holder +mount_embedded() { + local mappings=($@) + + if [ ${#mappings[@]} -eq 1 ]; then + local path=$embedded_rootfs_dir + mkdir -p $path + sudo mount /dev/mapper/"${mappings[0]}" $path + return + fi + + for mapping in ${mappings[@]} + do + local part_no=${mapping#*p*p} + local path=$embedded_base_dir/${embedded_partitions[${part_no} - 1]} + mkdir -p $path + sudo mount /dev/mapper/"${mapping}" $path 2>&1 >/dev/null + done +} + +# Takes following arguments: +# +# $1 - partition mappings holder +mount_sdimg() { + local mappings=($@) + + for mapping in ${mappings[@]} + do + local part_no=${mapping#*p*p} + local path=$sdimg_base_dir/${sdimg_partitions[${part_no} - 1]} + mkdir -p $path + sudo mount /dev/mapper/"${mapping}" $path 2>&1 >/dev/null + done +} + +# Takes following arguments +# +# $1 - device type +set_fstab() { + echo -e "\nSetting fstab..." + local mountpoint= + local device_type=$1 + local sysconfdir="$sdimg_primary_dir/etc" + + [ ! -d "${sysconfdir}" ] && { echo "Error: cannot find rootfs config dir."; exit 1; } + + # Erase/create the fstab file. + sudo install -b -m 644 /dev/null ${sysconfdir}/fstab + + case "$device_type" in + "beaglebone") + mountpoint="/boot/efi" + ;; + "raspberrypi3") + mountpoint="/uboot" + ;; + esac + + # Add Mender specific entries to fstab. + sudo bash -c "cat <<- EOF > ${sysconfdir}/fstab + # stock fstab - you probably want to override this with a machine specific one + + /dev/root / auto defaults 1 1 + proc /proc proc defaults 0 0 + devpts /dev/pts devpts mode=0620,gid=5 0 0 + tmpfs /run tmpfs mode=0755,nodev,nosuid,strictatime 0 0 + tmpfs /var/volatile tmpfs defaults 0 0 + + # uncomment this if your device has a SD/MMC/Transflash slot + #/dev/mmcblk0p1 /media/card auto defaults,sync,noauto 0 0 + + # Where the U-Boot environment resides; for devices with SD card support ONLY! + /dev/mmcblk0p1 $mountpoint auto defaults,sync 0 0 + /dev/mmcblk0p4 /data auto defaults 0 0 + EOF" +} + +# Takes following arguments +# +# $1 - path to source raw disk image +# $2 - sector start (in 512 blocks) +# $3 - size (in 512 blocks) + +extract_file_from_image() { + local cmd="dd if=$1 of=${output_dir}/$4 skip=$2 bs=512 count=$3 status=progress" + + echo "Running command:" + echo " ${cmd}" + $(${cmd}) +} diff --git a/mender-conversion-tool.sh b/mender-conversion-tool.sh new file mode 100755 index 0000000..11a39e1 --- /dev/null +++ b/mender-conversion-tool.sh @@ -0,0 +1,618 @@ +#!/bin/bash + +show_help() { +cat << EOF + +Mender conversion tool + +A tool for taking an existing embedded image (Debian, Ubuntu, Raspbian, etc) and +convert it to a Mender image by restructuring partition table and adding +necessary files. + +Usage: $0 [prepare_image | make_sdimg | install_mender | install_bootloader | make_all] [options] + +Actions: + + prepare_image - shrinks existing embedded image + + make_sdimg - composes image file compliant with Mender partition + layout + + install_mender - installs Mender client related files + + install_bootloader - installs Grub related files + + make_artifact - creates Mender artifact file based on .sdimg file + + make_all - composes fully functional .sdimg file compliant + with Mender partition layout, having all necessary + files installed + +Options: [-e|--embedded | -i|--image | -s|--size-data | -d|--device-type | + -r|--rootfs_type | -p| --demo-ip | -c| --certificate | + -u| --production-url | -o| --hosted-token] + + embedded - raw disk embedded linux image path, e.g. Debian 9.3, Raspbian, etc. + image - raw disk .sdimg file name where the script writes to + size-data - size of data partition in MiB; default value 128MiB + device-type - target device identification used to build .sdimg name + rootfs_type - selects rootfs_a|rootfs_b as the source filesystem for an artifact + demo_ip - server demo ip used for testing purposes + certificate - server certificate file + production-url - production server url + hosted-token - mender hosted token + + Note: root filesystem size used in .sdimg creation can be found as an + output from 'prepare_image' command or, in case of using unmodified + embedded image, can be checked with any partition manipulation + program (e.g. parted, fdisk, etc.). + +Examples: + + To shrink the existing embedded image: + + ./mender-conversion-tool.sh prepare_image --embedded + + Output: Root filesystem size (sectors): 4521984 + + To prepare .sdimg file: + + ./mender-conversion-tool.sh make_sdimg --image + --embedded + --size-data 128 --device-type beaglebone + + Output: ready to use ./output/*.sdimg file which can be used to flash SD card + + To install Mender client related files: + + ./mender-conversion-tool.sh install_mender --image + --device-type beaglebone --artifact release-1_1.5.0 + --server 192.168.10.2 --mender + + Output: ./output/*.sdimg file with Mender client related files installed + + To install Grub/U-Boot related files: + + ./mender-conversion-tool.sh install_bootloader --image + --device-type beaglebone --toolchain arm-linux-gnueabihf + + Output: ./output/*.sdimg file with Grub/U-Boot related files installed + + To prepare .mender artifact file: + + ./mender-conversion-tool.sh make_artifact --image + --device-type beaglebone --artifact release-1_1.5.0 + --rootfs-type rootfs_a + + Note: artifact name format is: release-_ + + To compose .sdimg file in a single step: + + ./mender-conversion-tool.sh make_all --embedded + --image --device-type raspberrypi3 + --mender --artifact release-1_1.5.0 + --demo-ip 192.168.10.2 --toolchain arm-linux-gnueabihf --keep + +EOF +} + +if [ $# -eq 0 ]; then + show_help + exit 1 +fi + +tool_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +# Default sector size +sector_size= +# Boot partition start in sectors (512 bytes per sector). +pboot_start= +# Default 'boot' partition size in sectors: 16MiB +# (i.e 16777216 bytes, equals to 'erase_block' * 2) +pboot_size= +# Default 'data' partition size in MiB. +data_size=128 +# Data partition size in sectors. +pdata_size= +# Exemplary values for Beaglebone: 9.3: 4521984 9.4: 4423680 +prootfs_size= + +menderimage= +embeddedimage= +device_type= +partitions_number= +artifact= +rootfs_type= +image_type= +mender= +# Mender production certificate. +certificate= +# Mender production server url. +production_url= +# Mender demo server IP address. +demo_ip= +# Mender hosted token. +hosted_token= + +declare -a rootfs_types=("rootfs_a" "rootfs_b") +declare -a sdimgmappings +declare -a embedmappings + +do_prepare_image() { + if [ -z "${embeddedimage}" ]; then + echo "Embedded image not set. Aborting." + exit 1 + fi + + local count= + local bootstart= + local bootsize= + local rootfsstart= + local rootfssize= + local bootflag= + + # Gather information about embedded image. + get_image_info $embeddedimage count sector_size bootstart bootsize \ + rootfsstart rootfssize bootflag + + # Find first available loopback device. + loopdevice=($(losetup -f)) + + # Mount appropriate partition. + if [[ $count -eq 1 ]]; then + sudo losetup $loopdevice $embeddedimage -o $(($bootstart * $sector_size)) + elif [[ $count -eq 2 ]]; then + sudo losetup $loopdevice $embeddedimage -o $(($rootfsstart * $sector_size)) + else + echo "Error: invalid/unsupported embedded image. Aborting." + exit 1 + fi + + [ $? -ne 0 ] && { echo "Error: inaccesible loopback device"; exit 1; } + + block_size=($(sudo dumpe2fs -h $loopdevice | grep 'Block size' | tr -s ' ' | cut -d ' ' -f3)) + min_size_blocks=($(sudo resize2fs -P $loopdevice | awk '{print $NF}')) + + new_size_sectors=$(( $min_size_blocks * $block_size / $sector_size )) + align_partition_size new_size_sectors $sector_size + + echo -e "Root filesystem size:" + echo -e "\nminimal: $(( $min_size_blocks * $block_size ))" + echo -e "\naligned: $(( $new_size_sectors * $sector_size ))" + echo -e "\nsectors: $new_size_sectors" + + sudo e2fsck -y -f $loopdevice + sudo resize2fs -p $loopdevice ${new_size_sectors}s + sudo e2fsck -y -f $loopdevice + + sudo losetup -d $loopdevice + sudo losetup $loopdevice $embeddedimage + + if [[ $count -eq 1 ]]; then + create_single_disk_partition_table $loopdevice $bootstart $new_size_sectors + elif [[ $count -eq 2 ]]; then + echo + create_double_disk_partition_table $loopdevice $rootfsstart $new_size_sectors + fi + + sudo partprobe + endsector=($(sudo parted $loopdevice -ms unit s print | grep "^$count" | cut -f3 -d: | sed 's/[^0-9]*//g')) + + sudo losetup -d $loopdevice + echo "Image new endsector: $endsector" + truncate -s $((($endsector+1) * $sector_size)) $embeddedimage + echo "Root filesystem size (sectors): $new_size_sectors" +} + +do_make_sdimg() { + if [ -z "${embeddedimage}" ]; then + echo "Source embedded image not set. Aborting." + exit 1 + fi + + if [ -z "${device_type}" ]; then + echo "Target device type name not set. Aborting." + exit 1 + fi + + if [[ ! -f ${embeddedimage} ]]; then + echo "Source embedded image not found. Aborting." + exit 1 + fi + + mkdir -p $output_dir && cd $output_dir + + # In case of missing .sdimg name use the default format. + [ -z $menderimage ] && menderimage=$output_dir/mender_${device_type}.sdimg \ + || menderimage=$output_dir/$menderimage + + analyse_embedded_image ${embeddedimage} pboot_start pboot_size prootfs_size \ + sector_size image_type + + [ -z "${prootfs_size}" ] && \ + { echo "root filesystem size not set. Aborting."; exit 1; } + + local menderimage_size= + calculate_sdimg_size $pboot_start $pboot_size \ + $prootfs_size $data_size \ + $sector_size pdata_size menderimage_size + + echo -e "Creating Mender *.sdimg file:\ + \nimage size: ${menderimage_size} bytes\ + \nroot filesystem size: ${prootfs_size} sectors\ + \ndata partition size: $pdata_size sectors\n" + + create_sdimg $menderimage $menderimage_size + format_sdimg $menderimage $menderimage_size $pboot_start $pboot_size \ + $prootfs_size $pdata_size $sector_size + verify_sdimg $menderimage partitions_number + + create_device_maps $menderimage sdimgmappings + make_sdimg_filesystem ${sdimgmappings[@]} + + case "$device_type" in + "beaglebone") + do_make_sdimg_beaglebone + ;; + "raspberrypi3") + do_make_sdimg_raspberrypi3 + ;; + esac + + rc=$? + + echo -e "\nCleaning..." + # Clean and detach. + detach_device_maps ${sdimgmappings[@]} + detach_device_maps ${embedmappings[@]} + sync + rm -rf $embedded_base_dir + rm -rf $sdimg_base_dir + + [ $rc -eq 0 ] && { echo -e "\n$menderimage image created."; } \ + || { echo -e "\n$menderimage image composing failure."; } +} + +do_make_sdimg_beaglebone() { + local ret=0 + + create_device_maps $embeddedimage embedmappings + + mount_sdimg ${sdimgmappings[@]} + mount_embedded ${embedmappings[@]} + + echo -e "\nSetting boot partition..." + stage_2_args="$sdimg_boot_dir $embedded_rootfs_dir" + ${tool_dir}/bbb-convert-stage-2.sh ${stage_2_args} || ret=$? + [[ $ret -ne 0 ]] && { echo "Aborting."; return $ret; } + + echo -e "\nSetting root filesystem..." + stage_3_args="$sdimg_primary_dir $embedded_rootfs_dir" + ${tool_dir}/bbb-convert-stage-3.sh ${stage_3_args} || ret=$? + [[ $ret -ne 0 ]] && { echo "Aborting."; return $ret; } + + set_fstab $device_type + + return $ret +} + +do_make_sdimg_raspberrypi3() { + image_boot_part=$(fdisk -l ${embeddedimage} | grep FAT32) + + boot_part_start=$(echo ${image_boot_part} | awk '{print $2}') + boot_part_end=$(echo ${image_boot_part} | awk '{print $3}') + boot_part_size=$(echo ${image_boot_part} | awk '{print $4}') + + extract_file_from_image ${embeddedimage} ${boot_part_start} \ + ${boot_part_size} "boot.vfat" + + image_rootfs_part=$(fdisk -l ${embeddedimage} | grep Linux) + + rootfs_part_start=$(echo ${image_rootfs_part} | awk '{print $2}') + rootfs_part_end=$(echo ${image_rootfs_part} | awk '{print $3}') + rootfs_part_size=$(echo ${image_rootfs_part} | awk '{print $4}') + + extract_file_from_image ${embeddedimage} ${rootfs_part_start} \ + ${rootfs_part_size} "rootfs.img" + + echo -e "\nSetting boot partition..." + stage_2_args="$output_dir ${sdimgmappings[0]}" + ${tool_dir}/rpi3-convert-stage-2.sh ${stage_2_args} || ret=$? + [[ $ret -ne 0 ]] && { echo "Aborting."; return $ret; } + + echo -e "\nSetting root filesystem..." + stage_3_args="$output_dir ${sdimgmappings[1]}" + ${tool_dir}/rpi3-convert-stage-3.sh ${stage_3_args} || ret=$? + [[ $ret -ne 0 ]] && { echo "Aborting."; return $ret; } + + mount_sdimg ${sdimgmappings[@]} + + # Add mountpoints. + sudo install -d -m 755 ${sdimg_primary_dir}/uboot + sudo install -d -m 755 ${sdimg_primary_dir}/data + + set_fstab $device_type +} + +do_install_mender() { + # Mender executables, service and configuration files installer. + if [ -z "$menderimage" ] || [ -z "$device_type" ] || [ -z "$mender" ] || \ + [ -z "$artifact" ]; then + show_help + exit 1 + fi + # mender-image-1.5.0 + stage_4_args="-i $menderimage -d $device_type -m ${mender} -a ${artifact}" + + if [ -n "$demo_ip" ]; then + stage_4_args="${stage_4_args} -p ${demo_ip}" + fi + + if [ -n "$certificate" ]; then + stage_4_args="${stage_4_args} -c ${certificate}" + fi + + if [ -n "$production_url" ]; then + stage_4_args="${stage_4_args} -u ${production_url}" + fi + + if [ -n "${hosted_token}" ]; then + stage_4_args="${stage_4_args} -o ${hosted_token}" + fi + + eval set -- " ${stage_4_args}" + + export -f create_device_maps + export -f detach_device_maps + + ${tool_dir}/convert-stage-4.sh ${stage_4_args} +} + +do_install_bootloader() { + if [ -z "$menderimage" ] || [ -z "$device_type" ] || \ + [ -z "$toolchain" ]; then + show_help + exit 1 + fi + + case "$device_type" in + "beaglebone") + stage_5_args="-i $menderimage -d $device_type -t ${toolchain} $keep" + eval set -- " ${stage_5_args}" + export -f create_device_maps + export -f detach_device_maps + ${tool_dir}/bbb-convert-stage-5.sh ${stage_5_args} + ;; + "raspberrypi3") + stage_5_args="-i $menderimage -d $device_type -t ${toolchain} $keep" + eval set -- " ${stage_5_args}" + export -f create_device_maps + export -f detach_device_maps + export -f mount_sdimg + ${tool_dir}/rpi3-convert-stage-5.sh ${stage_5_args} + ;; + esac +} + +do_make_artifact() { + if [ -z "${menderimage}" ]; then + echo "Raw disk .sdimg image not set. Aborting." + exit 1 + fi + + if [ -z "${device_type}" ]; then + echo "Target device_type name not set. Aborting." + exit 1 + fi + + if [ -z "${artifact}" ]; then + echo "Artifact name not set. Aborting." + exit 1 + fi + + if [ -z "${rootfs_type}" ]; then + echo "Artifact name not set. rootfs_a will be used by default." + rootfs_type="rootfs_a" + fi + + inarray=$(echo ${rootfs_types[@]} | grep -o $rootfs_type | wc -w) + + [[ $inarray -eq 0 ]] && \ + { echo "Error: invalid rootfs type provided. Aborting."; exit 1; } + + local count= + local bootstart= + local rootfs_a_start= + local rootfs_a_size= + local rootfs_b_start= + local rootfs_b_size= + local rootfs_path= + local sdimg_device_type= + local abort=0 + + get_sdimg_info $menderimage count sector_size rootfs_a_start rootfs_a_size \ + rootfs_b_start rootfs_b_size + ret=$? + [[ $ret -ne 0 ]] && \ + { echo "Error: cannot validate input .sdimg image. Aborting."; exit 1; } + + create_device_maps $menderimage sdimgmappings + mount_sdimg ${sdimgmappings[@]} + + if [[ $rootfs_type == "rootfs_a" ]]; then + prootfs_size=$rootfs_a_size + rootfs_path=$sdimg_primary_dir + elif [[ $rootfs_type == "rootfs_b" ]]; then + prootfs_size=$rootfs_b_size + rootfs_path=$sdimg_secondary_dir + fi + + # Find .sdimg file's dedicated device type. + sdimg_device_type=$( cat $sdimg_data_dir/mender/device_type | sed 's/[^=].*=//' ) + + # Set 'artifact name' as passed in the command line. + sudo sed -i '/^artifact/s/=.*$/='${artifact}'/' "$rootfs_path/etc/mender/artifact_info" + + if [ "$sdimg_device_type" != "$device_type" ]; then + echo "Error: .mender and .sdimg device type not matching. Aborting." + abort=1 + fi + + if [[ $(which mender-artifact) = 1 ]]; then + echo "Error: mender-artifact not found in PATH. Aborting." + abort=1 + fi + + if [ $abort -eq 0 ]; then + local rootfs_file=${output_dir}/rootfs.ext4 + + echo "Creating a ext4 file-system image from modified root file-system" + dd if=/dev/zero of=$rootfs_file seek=${prootfs_size} count=0 bs=512 status=none + + sudo mkfs.ext4 -FF $rootfs_file -d $rootfs_path + + fsck.ext4 -fp $rootfs_file + + mender_artifact=${output_dir}/${device_type}_${artifact}.mender + echo "Writing Mender artifact to: ${mender_artifact}" + + #Create Mender artifact + mender-artifact write rootfs-image \ + --update ${rootfs_file} \ + --output-path ${mender_artifact} \ + --artifact-name ${artifact} \ + --device-type ${device_type} + + ret=$? + [[ $ret -eq 0 ]] && \ + { echo "Writing Mender artifact to ${mender_artifact} succeeded."; } || \ + { echo "Writing Mender artifact to ${mender_artifact} failed."; } + + rm $rootfs_file + fi + + # Clean and detach. + detach_device_maps ${sdimgmappings[@]} + + rm -rf $sdimg_base_dir +} + +do_make_all() { + do_make_sdimg + do_install_mender + do_install_bootloader +} + +#read -s -p "Enter password for sudo: " sudoPW +#echo "" + +PARAMS="" + +# Load necessary functions. +source ${tool_dir}/mender-conversion-functions.sh + +while (( "$#" )); do + case "$1" in + -r | --rootfs-type) + rootfs_type=$2 + shift 2 + ;; + -i | --image) + menderimage=$2 + shift 2 + ;; + -e | --embedded) + embeddedimage=$(get_path $2) + shift 2 + ;; + -s | --size-data) + data_size=$2 + shift 2 + ;; + -d | --device-type) + device_type=$2 + shift 2 + ;; + -a | --artifact) + artifact=$2 + shift 2 + ;; + -m | --mender) + mender=$(get_path $2) + shift 2 + ;; + -t | --toolchain) + toolchain=$2 + shift 2 + ;; + -p | --demo-ip) + demo_ip=$2 + shift 2 + ;; + -c | --certificate) + certificate=$2 + shift 2 + ;; + -u | --production-url) + production_url=$2 + shift 2 + ;; + -o | --hosted-token) + hosted_token=$2 + shift 2 + ;; + -k | --keep) + keep="-k" + shift 1 + ;; + -h | --help) + show_help + exit 0 + ;; + --) + shift + break + ;; + -*) + echo "Error: unsupported option $1" >&2 + exit 1 + ;; + *) + PARAMS="$PARAMS $1" + shift + ;; + esac +done + +[ -z "${data_size}" ] && \ + { echo "Default 'data' partition size set to 128MiB"; data_size=128; } + +eval set -- "$PARAMS" + +# Some commands expect elevated privileges. +sudo true + +case "$1" in + prepare_image) + do_prepare_image + ;; + make_sdimg) + do_make_sdimg + ;; + install_mender) + do_install_mender + ;; + install_bootloader) + do_install_bootloader + ;; + make_artifact) + do_make_artifact + ;; + make_all) + do_make_all + ;; + *) + show_help + ;; +esac + diff --git a/rpi3-convert-stage-2.sh b/rpi3-convert-stage-2.sh new file mode 100755 index 0000000..d6a2078 --- /dev/null +++ b/rpi3-convert-stage-2.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +output_dir=$1 +boot_mapping=$2 + +[ ! -f $output_dir/boot.vfat ] && \ + { echo "Error: extracted boot partition not found. Aborting."; exit 1; } + +# Make a copy of Linux kernel arguments and modify. +mcopy -o -i ${output_dir}/boot.vfat -s ::cmdline.txt ${output_dir}/cmdline.txt +sed -i 's/\b[ ]root=[^ ]*/ root=\/dev\/mmcblk0p2/' ${output_dir}/cmdline.txt +sed -i 's/\b[ ]console=tty1//' ${output_dir}/cmdline.txt +# Update Linux kernel command arguments with our custom configuration +mcopy -o -i ${output_dir}/boot.vfat -s ${output_dir}/cmdline.txt ::cmdline.txt + +mcopy -i ${output_dir}/boot.vfat -s ::config.txt ${output_dir}/config.txt +echo -e '\nenable_uart=1\n' >> ${output_dir}/config.txt +mcopy -o -i ${output_dir}/boot.vfat -s ${output_dir}/config.txt ::config.txt + +sudo dd if=${output_dir}/boot.vfat of=/dev/mapper/${boot_mapping} bs=1M + +echo -e "\nStage done." + +exit 0 diff --git a/rpi3-convert-stage-3.sh b/rpi3-convert-stage-3.sh new file mode 100755 index 0000000..b3044f0 --- /dev/null +++ b/rpi3-convert-stage-3.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +output_dir=$1 +rootfs_mapping=$2 + +[ ! -f ${output_dir}/rootfs.img ] && \ + { echo "Error: extracted rootfs partition not found. Aborting."; exit 1; } + +sudo dd if=${output_dir}/rootfs.img of=/dev/mapper/${rootfs_mapping} bs=8M + +# dd sets the original label, make sure label follows Mender naming convention. +sudo e2label /dev/mapper/${rootfs_mapping} "primary" + +echo -e "\nStage done." + +exit 0 diff --git a/rpi3-convert-stage-5.sh b/rpi3-convert-stage-5.sh new file mode 100755 index 0000000..ee4cef2 --- /dev/null +++ b/rpi3-convert-stage-5.sh @@ -0,0 +1,179 @@ +#!/bin/bash + +# Copyright 2018 Northern.tech AS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +application_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +output_dir=${application_dir}/output +bin_dir_pi=${output_dir}/bin/raspberrypi +sdimg_base_dir=$output_dir/sdimg + +echo "Running: $(basename $0)" +declare -a sdimgmappings +declare -a sdimg_partitions=("boot" "primary" "secondary" "data") + +build_uboot_files() { + mkdir -p $bin_dir_pi + + echo -e "Downloading U-Boot related files..." + + wget -q -O $bin_dir_pi/boot.scr \ + https://github.com/mirzak/mender-conversion-tools/raw/mirza/wip/bin/raspberrypi/boot.scr + wget -q -O $bin_dir_pi/fw_printenv \ + https://github.com/mirzak/mender-conversion-tools/raw/mirza/wip/bin/raspberrypi/fw_printenv + wget -q -O $bin_dir_pi/init_resize.sh \ + https://raw.githubusercontent.com/mirzak/mender-conversion-tools/mirza/wip/bin/raspberrypi/init_resize.sh + wget -q -O $bin_dir_pi/u-boot.bin \ + https://github.com/mirzak/mender-conversion-tools/raw/mirza/wip/bin/raspberrypi/u-boot.bin +} + +# Takes following arguments: +# +# $1 - boot partition mountpoint +# $2 - primary partition mountpoint +install_files() { + local boot_dir=$1 + local rootfs_dir=$2 + + # Make a copy of Linux kernel arguments and modify. + cp ${boot_dir}/cmdline.txt ${output_dir}/cmdline.txt + + sed -i 's/\b[ ]root=[^ ]*/ root=\${mender_kernel_root}/' ${output_dir}/cmdline.txt + + # If the the image that we are trying to convert has been booted once on a + # device, it will have removed the init_resize.sh init argument from cmdline. + # + # But we want it to run on our image as well to resize our data part so in + # case it is missing, add it back to cmdline.txt + if ! grep "init=/usr/lib/raspi-config/init_resize.sh" ${output_dir}/cmdline.txt; then + cmdline=$(cat ${output_dir}/cmdline.txt) + echo "${cmdline} init=/usr/lib/raspi-config/init_resize.sh" > ${output_dir}/cmdline.txt + fi + + # Update Linux kernel command arguments with our custom configuration + sudo cp ${output_dir}/cmdline.txt ${boot_dir} + + # Mask udisks2.service, otherwise it will mount the inactive part and we + # might write an update while it is mounted which often result in + # corruptions. + # + # TODO: Find a way to only blacklist mmcblk0pX devices instea of masking + # the service. + sudo ln -sf /dev/null ${rootfs_dir}/etc/systemd/system/udisks2.service + + # Extract Linux kernel and install to /boot directory on rootfs + sudo cp ${boot_dir}/kernel7.img ${rootfs_dir}/boot/zImage + + # Replace kernel with U-boot and add boot script + sudo mkdir -p ${rootfs_dir}/uboot + + sudo cp ${bin_dir_pi}/u-boot.bin ${boot_dir}/kernel7.img + sudo cp ${bin_dir_pi}/boot.scr ${boot_dir} + sudo cp ${bin_dir_pi}/init_resize.sh ${rootfs_dir}/usr/lib/raspi-config/init-resize.sh + + sudo cp ${boot_dir}/config.txt ${output_dir}/config.txt + + # dtoverlays seems to break U-boot for some reason, simply remove all of + # them as they do not actually work when U-boot is used. + sed -i /^dtoverlay=/d ${output_dir}/config.txt + + sudo cp ${output_dir}/config.txt ${boot_dir}/config.txt + + # Raspberry Pi configuration files, applications expect to find this on + # the device and in some cases parse the options to determinate + # functionality. + sudo ln -fs /uboot/config.txt ${rootfs_dir}/boot/config.txt + + sudo install -m 755 ${bin_dir_pi}/fw_printenv ${rootfs_dir}/sbin/fw_printenv + sudo ln -fs /sbin/fw_printenv ${rootfs_dir}/sbin/fw_setenv + + # Override init script to expand the data part instead of rootfs, which it + # normally expands in standard Raspberry Pi distributions. + sudo install -m 755 ${bin_dir_pi}/init_resize.sh \ + ${rootfs_dir}/usr/lib/raspi-config/init_resize.sh +} + +do_install_bootloader() { + echo "Setting bootloader..." + + if [ -z "${image}" ]; then + echo ".sdimg image file not set. Aborting." + exit 1 + fi + + [ ! -f $image ] && { echo "$image - file not found. Aborting."; exit 1; } + + # Map & mount Mender compliant image. + create_device_maps $image sdimgmappings + + mkdir -p $output_dir && cd $output_dir + + # Build patched U-Boot files. + build_uboot_files + + mount_sdimg ${sdimgmappings[@]} + + install_files ${output_dir}/sdimg/boot ${output_dir}/sdimg/primary + + detach_device_maps ${sdimgmappings[@]} + + echo -e "\nStage done." +} + +# Conditional once we support other boards +PARAMS="" + +while (( "$#" )); do + case "$1" in + -i | --image) + image=$2 + shift 2 + ;; + -t | --toolchain) + toolchain=$2 + shift 2 + ;; + -d | --device-type) + device_type=$2 + shift 2 + ;; + -k | --keep) + keep=1 + shift 1 + ;; + -h | --help) + show_help + exit 0 + ;; + --) + shift + break + ;; + -*) + echo "Error: unsupported option $1" >&2 + exit 1 + ;; + *) + PARAMS="$PARAMS $1" + shift + ;; + esac +done + +eval set -- "$PARAMS" + +# Some commands expect elevated privileges. +sudo true + +do_install_bootloader