diff --git a/.dockerignore b/.dockerignore index 24ae780..94e5f31 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,3 @@ -input/ -output/ -device-image-shell/output +deploy +input +work diff --git a/.gitignore b/.gitignore index 3ecb6c2..be018dc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ -input/ -output/ -device-image-shell/output -mendertesting/ -integration/ +deploy +input +work +rootfs_overlay_demo/* +tests diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..b19e133 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,45 @@ +FROM ubuntu:19.04 + +ARG MENDER_ARTIFACT_VERSION=3.1.0 + +RUN apt-get update && apt-get install -y \ +# For 'ar' command to unpack .deb + binutils \ + xz-utils \ +# to be able to detect file system types of extracted images + file \ +# to copy files between rootfs directories + rsync \ +# to generate partition table + parted \ +# mkfs.ext4 and family + e2fsprogs \ +# mkfs.xfs and family + xfsprogs \ +# Parallel gzip compression + pigz \ + sudo \ +# mkfs.vfat (required for boot partition) + dosfstools \ +# to download mender-artifact + wget \ +# to download mender-grub-env + git \ +# to compile mender-grub-env + make \ +# to get rid of 'sh: 1: udevadm: not found' errors triggered by parted + udev \ +# to create bmap index file (MENDER_USE_BMAP) + bmap-tools + +RUN wget -q -O /usr/bin/mender-artifact https://d1b0l86ne08fsf.cloudfront.net/mender-artifact/$MENDER_ARTIFACT_VERSION/linux/mender-artifact \ + && chmod +x /usr/bin/mender-artifact + +# allow us to keep original PATH variables when sudoing +RUN echo "Defaults secure_path=\"/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin:$PATH\"" > /etc/sudoers.d/secure_path_override +RUN chmod 0440 /etc/sudoers.d/secure_path_override + +WORKDIR / + +COPY docker-entrypoint.sh /usr/local/bin/ +ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..a819abc --- /dev/null +++ b/LICENSE @@ -0,0 +1,181 @@ +Copyright 2019 Northern.tech AS + +All content in this project is licensed under the Apache License v2, unless +indicated otherwise. + +------------------------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. diff --git a/LIC_FILES_CHKSUM.sha256 b/LIC_FILES_CHKSUM.sha256 new file mode 100644 index 0000000..2534c88 --- /dev/null +++ b/LIC_FILES_CHKSUM.sha256 @@ -0,0 +1,2 @@ +beb140be4cd64599bedc691a55b2729c9cc611a4b9d6ec44e01270105daf18a2 LICENSE +beb140be4cd64599bedc691a55b2729c9cc611a4b9d6ec44e01270105daf18a2 mendertesting/LICENSE diff --git a/README.md b/README.md new file mode 100644 index 0000000..0581708 --- /dev/null +++ b/README.md @@ -0,0 +1,153 @@ +# mender-convert + +Mender is an open source over-the-air (OTA) software updater for embedded Linux devices. Mender comprises a client running at the embedded device, as well as a server that manages deployments across many devices. + +This repository contains mender-convert, which is used to convert pre-built disk images (Debian, Ubuntu, Raspbian, etc) to a Mender compatible image by restructuring partition table and injecting the necessary files. + +For a full list of tested devices and images please visit [Mender Hub](https://hub.mender.io/c/board-integrations/debian-family). If your device and image combination is not listed as supported, this does not necessarily mean that it will not work, it probably just means that no one has tested and reported it back and usually only small tweaks are necessary to get this running on your device. + +![Mender logo](https://mender.io/user/pages/resources/06.digital-assets/mender.io.png) + + +## Getting started + +To start using Mender, we recommend that you begin with the Getting started +section in [the Mender documentation](https://docs.mender.io/). + +For more detailed information about `mender-convert` please visit the +[Debian family](https://docs.mender.io/artifacts/debian-family) section in +[the Mender documentation](https://docs.mender.io/). + + +### Prepare image and configuration + +Download the raw Raspberry Pi disk image into a subdirectory input: + +```bash +mkdir -p input +cd input +wget https://downloads.raspberrypi.org/raspbian_lite/images/raspbian_lite-2019-06-24/2019-06-20-raspbian-buster-lite.zip +``` + +Extract the raw Raspberry Pi disk image: + +```bash +unzip 2019-06-20-raspbian-buster-lite.zip && cd .. +``` + +Bootstrap the demo rootfs overlay that is configured to connect to +https://hosted.mender.io with polling intervals set appropriately for +demonstration purposes: + +``` +./scripts/bootstrap-rootfs-overlay-demo.sh \ + --output-dir ${PWD}/rootfs_overlay_demo \ + --tenant-token "Paste token from Hosted Mender" +``` + + +### Docker environment for mender-convert + +To make using mender-convert easier, a reference setup using a Docker +container is provided. + +You need to [install Docker Engine](https://docs.docker.com/install) to use +this environment. + + +#### Build the mender-convert container image + +Build a container with all required dependencies for `mender-convert`: + +```bash +./docker-build +``` + +This will create a container image which you can use to run `mender-convert` +without polluting your host environment with the necessary dependencies. + + +#### Use the mender-convert container image + +Run mender-convert from inside the container with your desired options, e.g. + +```bash +MENDER_ARTIFACT_NAME=release-1 ./docker-mender-convert \ + --disk-image input/2019-04-08-raspbian-stretch-lite.img \ + --config configs/raspberrypi3_config \ + --overlay rootfs_overlay_demo/ +``` + +Conversion will take 10-30 minutes, depending on image size and resources +available. You can watch `work/convert.log` for progress and diagnostics +information. + +After it finishes, you can find your images in the `deploy` directory on your +host machine! + +## Using mender-convert without Docker + +In order to be able to manipulate and create filesystem and disk images, +mender-convert has a few dependencies, and their version and name vary between +Linux distributions. Here is an example of how to install the dependencies on a +Debian based distribution: + +``` +sudo apt install $(cat requirements-deb.txt) +``` + +Start the conversion process with: + +```bash +MENDER_ARTIFACT_NAME=release-1 ./mender-convert \ + --disk-image input/2019-04-08-raspbian-stretch-lite.img \ + --config configs/raspberrypi3_config \ + --overlay rootfs_overlay_demo/ +``` + +**NOTE!** You will be prompted to enter `sudo` password during the conversion +process. This is required to be able to loopback mount images and for modifying +them. Our recommendation is to use the provided Docker container, to run the +tool in a isolated environment (at least to some degree). + + +## Contributing + +We welcome and ask for your contribution. If you would like to contribute to +Mender, please read our guide on how to best get started +[contributing code or documentation](https://github.com/mendersoftware/mender/blob/master/CONTRIBUTING.md). + + +## License + +Mender is licensed under the Apache License, Version 2.0. See +[LICENSE](https://github.com/mendersoftware/mender-convert/blob/master/LICENSE) +for the full license text. + + +## Security disclosure + +We take security very seriously. If you come across any issue regarding +security, please disclose the information by sending an email to +[security@mender.io](security@mender.io). Please do not create a new public +issue. We thank you in advance for your cooperation. + + +## Connect with us + +* Join the [Mender Hub discussion forum](https://hub.mender.io) +* Follow us on [Twitter](https://twitter.com/mender_io). Please + feel free to tweet us questions. +* Fork us on [Github](https://github.com/mendersoftware) +* Create an issue in the [bugtracker](https://tracker.mender.io/projects/MEN) +* Email us at [contact@mender.io](mailto:contact@mender.io) +* Connect to the [#mender IRC channel on Freenode](http://webchat.freenode.net/?channels=mender) + + +## Authors + +Mender was created by the team at [Northern.tech AS](https://northern.tech), +with many contributions from the community. Thanks +[everyone](https://github.com/mendersoftware/mender/graphs/contributors)! + +[Mender](https://mender.io) is sponsored by [Northern.tech AS](https://northern.tech). diff --git a/configs/beaglebone_black_base_config b/configs/beaglebone_black_base_config new file mode 100644 index 0000000..84fb04b --- /dev/null +++ b/configs/beaglebone_black_base_config @@ -0,0 +1,37 @@ +# Binaries generated with the following script: +# +# https://github.com/drewmoseley/mender-convert-integration-scripts/blob/master/build-uboot-bbb.sh +# + +# There are reported issues with GRUB bootloader integration, fallback to U-boot. +MENDER_GRUB_EFI_INTEGRATION=n + +# 4MB alignment +MENDER_PARTITION_ALIGNMENT="4194304" + +BEAGLEBONE_BLACK_BINARIES_URL="${MENDER_STORAGE_URL}/mender-convert/beaglebone/${BEAGLEBONE_BLACK_BINARIES}" + +function platform_modify() { + mkdir -p work/bbb/binaries + + run_and_log_cmd "wget -q ${BEAGLEBONE_BLACK_BINARIES_URL} -P work/bbb/binaries" + run_and_log_cmd "tar xvf work/bbb/binaries/${BEAGLEBONE_BLACK_BINARIES} -C work/bbb/binaries" + + # Mask udisks2.service, otherwise it will mount the inactive part and we + # might write an update while it is mounted which often result in + # corruptions. + # + # TODO: Find a way to only blacklist mmcblk0pX devices instead of masking + # the service. + run_and_log_cmd "sudo ln -sf /dev/null work/rootfs/etc/systemd/system/udisks2.service" + + # Place u-boot and MLO into rootfs/boot + run_and_log_cmd "sudo mkdir -p work/rootfs/boot" + run_and_log_cmd "sudo cp work/bbb/binaries/MLO work/boot/" + run_and_log_cmd "sudo cp work/bbb/binaries/u-boot.img work/boot/" + run_and_log_cmd "sudo cp work/bbb/binaries/fw_env.config work/rootfs/etc/" + run_and_log_cmd "sudo cp work/bbb/binaries/uboot-git-log.txt work/boot" + + run_and_log_cmd "sudo install -m 755 work/bbb/binaries/fw_printenv work/rootfs/sbin/fw_printenv" + run_and_log_cmd "sudo ln -fs /sbin/fw_printenv work/rootfs/sbin/fw_setenv" +} diff --git a/configs/beaglebone_black_debian_sdcard_config b/configs/beaglebone_black_debian_sdcard_config new file mode 100644 index 0000000..67a0b0d --- /dev/null +++ b/configs/beaglebone_black_debian_sdcard_config @@ -0,0 +1,6 @@ +# Binaries generated with the following script: +# +# https://github.com/drewmoseley/mender-convert-integration-scripts/blob/master/build-uboot-bbb.sh +# +BEAGLEBONE_BLACK_BINARIES="beaglebone-black-integration-debian-2018.07.002.tar" +source configs/beaglebone_black_base_config diff --git a/configs/mender_convert_config b/configs/mender_convert_config new file mode 100644 index 0000000..c865ded --- /dev/null +++ b/configs/mender_convert_config @@ -0,0 +1,147 @@ +# This is the default configuration used by mender-convert. You can override +# any option specified here by providing your own configuration file using the +# '--config' argument. +# +# NOTE! This file will always be sourced. + +# Compress generated disk image using gzip +# +# This is useful when you have large disk images, compressing them +# makes it easier to transfer them between e.g an build server and a local +# machine, and obviously saves space. +MENDER_COMPRESS_DISK_IMAGE=y + +# Compression algorithm for Mender Artifact +# +# Supported values are: lzma, gzip (default), none +# +# LZMA will produce a smaller Mender Artifact (2-3x) but will significantly +# increase time spent generating the Mender Artifact (10x) +MENDER_ARTIFACT_COMPRESSION="gzip" + +# Enable Mender systemd server +# +# You want this enabled if you want the Mender client to operate in managed mode +# and connect to a server. If you are not interested connecting to a server +# and will only be running standalone mode updates, then you can safely disable +# this. +MENDER_ENABLE_SYSTEMD=y + +# Set the fstab options for mounting the data partition +MENDER_DATA_PART_FSTAB_OPTS="defaults" + +# The file system will be grown to occupy the full block device. If the file +# system is already at maximum size, no action will be performed. +# +# This feature is utilizing x−systemd.growfs and will only work on systemd 242 +# and newer and nothing will be done if an older systemd version is used. +MENDER_DATA_PART_GROWFS=y + +# The name of the image or update that will be built. This is what the device +# will report that it is running, and different updates must have different +# names. +# +# This variable must be defined or the build will fail, but you should not +# set it in a configuration file and instead it should be an input to the +# mender-convert tool, e.g +# +# MENDER_ARTIFACT_NAME="release-1" ./mender-convert +# +#MENDER_ARTIFACT_NAME="" + +# A string that defines the type of device this image will be installed on and +# this value is used for compatibility checks between Mender Artifact and +# what the device is reporting. +# +# You can define multiple types by providing a space separated string of device +# types. +# +# It defaults to the content of '/etc/hostname', which is extracted from the +# input disk image +MENDER_DEVICE_TYPE="" + +# Total size of the physical storage medium that mender partitioned images +# will be written to, expressed in MiB. The size of rootfs partition will be +# calculated automatically by subtracting the sizes of boot +# (see MENDER_BOOT_PART_SIZE_MB) and data partitions +# (see MENDER_DATA_PART_SIZE_MB). +# +MENDER_STORAGE_TOTAL_SIZE_MB="8192" + +# The size of the boot partition in the generated .biosimg, .sdimg or .uefiimg +# file. +MENDER_BOOT_PART_SIZE_MB="40" + +# The size of the persistent data partition in the generated .biosimg, .sdimg +# or .uefiimg file. +# +# You rarely need to increase this number as this partition will be expanded +# on first boot to occupy the renaming free blocks on the physical storage. +MENDER_DATA_PART_SIZE_MB="128" + +# Alignment of partitions used when building partitioned images, expressed in +# bytes +# +# Default is 8MB +MENDER_PARTITION_ALIGNMENT="8388608" + +# Mender client version +# +# This is used to fetch the correct binaries +MENDER_CLIENT_VERSION="2.0.1" + +# File storage, containing binary files, do not modify this unless you know +# what you are doing. +MENDER_STORAGE_URL="https://d1b0l86ne08fsf.cloudfront.net" + +# Mender GitHub organization URL prefix +MENDER_GITHUB_ORG="https://github.com/mendersoftware" + +# Device file corresponding to the root filesystem partitions, without index. +MENDER_STORAGE_DEVICE=/dev/mmcblk0p + +# Partition index of boot partition +MENDER_BOOT_PART_INDEX="1" + +# Partition index of root filesystem A +MENDER_ROOTFS_PART_A_INDEX="2" + +# Partition index of root filesystem B +MENDER_ROOTFS_PART_B_INDEX="3" + +# Partition index of persistent data partition +MENDER_DATA_PART_INDEX="4" + +# Basename of DTB that should be loaded by the bootloader. +MENDER_KERNEL_DEVICETREE=kernel.dtb + +# Generate bmap index files +# +# https://github.com/intel/bmap-tools +MENDER_USE_BMAP="n" + +source configs/mender_grub_config + +# Function to create Mender Artifact +# +# This function is defined here, to allow it to be overridden which can be +# achieved by providing this function in a custom configuration file which will +# take precedence of this one. +# +# You might want to override this to e.g provide state-scripts or providing +# a private key to sign your artifact. +mender_create_artifact() { + local -r device_type="${1}" + local -r artifact_name="${2}" + + mender_artifact=deploy/${device_type}-${artifact_name}.mender + log_info "Writing Mender artifact to: ${mender_artifact}" + log_info "This can take up to 20 minutes depending on which compression method is used" + + run_and_log_cmd "mender-artifact --compression ${MENDER_ARTIFACT_COMPRESSION} \ + write rootfs-image \ + --file work/rootfs.img \ + --output-path ${mender_artifact} \ + --artifact-name ${artifact_name} \ + --device-type ${device_type}" +} diff --git a/configs/mender_grub_config b/configs/mender_grub_config new file mode 100644 index 0000000..30efd47 --- /dev/null +++ b/configs/mender_grub_config @@ -0,0 +1,35 @@ +# Use U-boot -> GRUB -> EFI boot-loader integration +# +# Only disable this if you know what you are doing +MENDER_GRUB_EFI_INTEGRATION=y + +# Specific Linux kernel boot arguments +# +# Typically you would read the content of /proc/cmdline on a "golden image" +# add the appropriate arguments here. +# +# This will override the defaults set by by grub-mender-grubenv, if not an +# empty string +MENDER_GRUB_KERNEL_BOOT_ARGS="" + +# grub-mender-grubenv is the Mender integration for the GRUB bootloader +MENDER_GRUBENV_VERSION="1.3.0" +MENDER_GRUBENV_URL="${MENDER_GITHUB_ORG}/grub-mender-grubenv/archive/${MENDER_GRUBENV_VERSION}.tar.gz" + +# Name of the storage device containing root filesystem partitions in GRUB +# format. +MENDER_GRUB_STORAGE_DEVICE=hd0 + +# Type of kernel (bzImage or zImage) +# +# mender-convert will try to determinate this value on its own, only set this +# if was not possible to auto-detect +MENDER_GRUB_KERNEL_IMAGETYPE="" + +# Type of initrd image +# +# mender-convert will try to determinate this value on its own, only set this +# if was not possible to auto-detect +MENDER_GRUB_INITRD_IMAGETYPE="" + +MENDER_GRUB_BINARY_STORAGE_URL="${MENDER_STORAGE_URL}/mender-convert/grub-efi" diff --git a/configs/qemux86-64_config b/configs/qemux86-64_config new file mode 100644 index 0000000..aa46612 --- /dev/null +++ b/configs/qemux86-64_config @@ -0,0 +1,16 @@ +# This configuration can be used to run on a qemux86-64 machine. +# +# This has been tested on images generated with the following command: +# +# mkosi -d ubuntu -r bionic -t gpt_ext4 -b --checksum --password password -o image.raw +# +# Converted with the following command: +# +# MENDER_ARTIFACT_NAME=release-1 ./mender-convert --disk-image input/image.raw --overlay rootfs_overlay_demo --config configs/qemux86-64_config +# +# and qemu is executed with the following command: +# +# qemu-system-x86_64 -enable-kvm -m 512 -smp 2 -bios /usr/share/ovmf/x64/OVMF_CODE.fd -drive format=raw,file=qemux86_64-release-1.sdimg + +MENDER_STORAGE_DEVICE=/dev/sda +MENDER_DEVICE_TYPE="qemux86_64" diff --git a/configs/raspberrypi3_config b/configs/raspberrypi3_config new file mode 100644 index 0000000..6315fd7 --- /dev/null +++ b/configs/raspberrypi3_config @@ -0,0 +1,8 @@ +# Binaries generated with the following script: +# +# https://d1b0l86ne08fsf.cloudfront.net/mender-convert/raspberrypi/raspberrypi-integration-scripts.tar.gz +RASPBERRYPI_BINARIES="raspberrypi3-integration-2018.07.tar.gz" +RASPBERRYPI_KERNEL_IMAGE="kernel7.img" +MENDER_KERNEL_IMAGETYPE=zImage + +source configs/raspberrypi_config diff --git a/configs/raspberrypi_config b/configs/raspberrypi_config new file mode 100644 index 0000000..f65ed2b --- /dev/null +++ b/configs/raspberrypi_config @@ -0,0 +1,82 @@ +# Raspberry Pi does not support GRUB bootloader integration, fallback to U-boot. +MENDER_GRUB_EFI_INTEGRATION=n + +# 4MB alignment +MENDER_PARTITION_ALIGNMENT="4194304" + +RASPBERRYPI_BINARIES_URL="${MENDER_STORAGE_URL}/mender-convert/raspberrypi/${RASPBERRYPI_BINARIES}" + +function platform_modify() { + mkdir -p work/rpi/binaries + + run_and_log_cmd "wget -q ${RASPBERRYPI_BINARIES_URL} -P work/rpi/binaries" + run_and_log_cmd "tar xvf work/rpi/binaries/${RASPBERRYPI_BINARIES} -C work/rpi/binaries" + + # Make a copy of Linux kernel arguments and modify. + run_and_log_cmd "cp work/boot/cmdline.txt work/rpi/cmdline.txt" + + # Set a dynamic rootfs part (required for Mender A/B update strategy) + run_and_log_cmd "sed -i 's/\b[ ]root=[^ ]*/ root=\${mender_kernel_root}/' work/rpi/cmdline.txt" + + # Root filesystem can not be resized when the disk is partition according + # to Mender layout, where the rootfs partition is the not last one which + # is a requirement to be able to do an "online" resize. + # + # This disables resize of rootfs on boot but applies the changes to + # cmdline.txt that are performed in the init_resize.sh script. + # + # Extracted from /usr/lib/raspi-config/init_resize.sh + run_and_log_cmd "sed -i 's| init=/usr/lib/raspi-config/init_resize\.sh||' work/rpi/cmdline.txt" + run_and_log_cmd "sed -i 's| sdhci\.debug_quirks2=4||' work/rpi/cmdline.txt" + if ! grep -q splash work/rpi/cmdline.txt; then + run_and_log_cmd "sed -i 's/ quiet//g' work/rpi/cmdline.txt" + fi + + # Update Linux kernel command arguments with our custom configuration + run_and_log_cmd "sudo cp work/rpi/cmdline.txt work/boot/" + + # Enable Full KMS (Kernel Mode Setting) graphics driver. This is the + # the open-source GPU driver that is part of mesa, also known as VC4. + # + # This is done because it seems that the firmware GPU driver does currently + # not play well when U-boot is enabled and HDMI output is "scrambled" + # + # Fixes: https://tracker.mender.io/browse/MEN-2685 + # + # Will need to revisit this later when there are new firware releases. + # + # Should be noted that RPi4 only works with this enabled and the + # configuration in the Yocto BSP (meta-raspberrypi) is also nowdays using + # this driver by default. + run_and_log_cmd "echo 'dtoverlay=vc4-kms-v3d' | sudo tee -a work/boot/config.txt" + + # Mask udisks2.service, otherwise it will mount the inactive part and we + # might write an update while it is mounted which often result in + # corruptions. + # + # TODO: Find a way to only blacklist mmcblk0pX devices instead of masking + # the service. + run_and_log_cmd "sudo ln -sf /dev/null work/rootfs/etc/systemd/system/udisks2.service" + + # Extract Linux kernel and install to /boot directory on rootfs + run_and_log_cmd "sudo cp work/boot/${RASPBERRYPI_KERNEL_IMAGE} work/rootfs/boot/${MENDER_KERNEL_IMAGETYPE}" + + # Replace kernel with U-boot and add boot script + run_and_log_cmd "sudo mkdir -p work/rootfs/uboot" + run_and_log_cmd "sudo cp work/rpi/binaries/u-boot.bin work/boot/${RASPBERRYPI_KERNEL_IMAGE}" + run_and_log_cmd "sudo cp work/rpi/binaries/boot.scr work/boot" + run_and_log_cmd "sudo cp work/rpi/binaries/fw_env.config work/rootfs/etc/" + + # Raspberry Pi applications expect to find this on the device and in some + # cases parse the options to determinate functionality. + run_and_log_cmd "sudo ln -fs /uboot/config.txt work/rootfs/boot/config.txt" + run_and_log_cmd "sudo ln -fs /uboot/overlays work/rootfs/boot/overlays" + run_and_log_cmd "sudo ln -fs /uboot/cmdline.txt work/rootfs/boot/cmdline.txt" + + run_and_log_cmd "sudo install -m 755 work/rpi/binaries/fw_printenv work/rootfs/sbin/fw_printenv" + run_and_log_cmd "sudo ln -fs /sbin/fw_printenv work/rootfs/sbin/fw_setenv" + + # Remove original 'resize2fs_once' script and its symbolic link. + run_and_log_cmd "sudo unlink work/rootfs/etc/rc3.d/S01resize2fs_once" + run_and_log_cmd "sudo rm work/rootfs/etc/init.d/resize2fs_once" +} diff --git a/configs/rockpro64_config b/configs/rockpro64_config new file mode 100644 index 0000000..9e9e631 --- /dev/null +++ b/configs/rockpro64_config @@ -0,0 +1,25 @@ +# ROCKPro64 do not support GRUB bootloader integration, fallback to U-boot. +MENDER_GRUB_EFI_INTEGRATION=n + +ROCKPRO64_BINARIES_URL="${MENDER_STORAGE_URL}/mender-convert/armbian/rockpro64/${ROCKPRO64_BINARIES}" + +function platform_modify() { + mkdir -p work/rockpro64 + + run_and_log_cmd "wget -Nq ${ROCKPRO64_BINARIES_URL} -P work/rockpro64" + run_and_log_cmd "tar xvf work/rockpro64/${ROCKPRO64_BINARIES} -C work/rockpro64" + + run_and_log_cmd "sudo cp work/rockpro64/boot.scr work/boot" + + run_and_log_cmd "sudo cp work/rockpro64/fw_env.config work/rootfs/etc/" + + # It is not possible to resize rootfs part when using Mender. so disable + # the service + run_and_log_cmd "sudo touch work/rootfs/root/.no_rootfs_resize" +} + +function platform_package() { + log_info "Embedding bootloader in disk image" + run_and_log_cmd "dd if=work/rockpro64/rksd_loader.img of=${sdimg_path} \ + seek=64 conv=notrunc status=none" +} diff --git a/configs/rockpro64_emmc_config b/configs/rockpro64_emmc_config new file mode 100644 index 0000000..3ae58f2 --- /dev/null +++ b/configs/rockpro64_emmc_config @@ -0,0 +1,8 @@ +MENDER_STORAGE_DEVICE=/dev/mmcblk1p + +# Binaries generated with the following script: +# +# https://d1b0l86ne08fsf.cloudfront.net/mender-convert/armbian/rockpro64/integration-scripts.tar.gz +ROCKPRO64_BINARIES="emmc-boot-integration-2017.09.tar.gz" + +source configs/rockpro64_config diff --git a/configs/rockpro64_sd_config b/configs/rockpro64_sd_config new file mode 100644 index 0000000..514f8c3 --- /dev/null +++ b/configs/rockpro64_sd_config @@ -0,0 +1,6 @@ +# Binaries generated with the following script: +# +# https://d1b0l86ne08fsf.cloudfront.net/mender-convert/armbian/rockpro64/integration-scripts.tar.gz +ROCKPRO64_BINARIES="sd-boot-integration-2017.09.tar.gz" + +source configs/rockpro64_config diff --git a/docker-build b/docker-build new file mode 100755 index 0000000..d758b0f --- /dev/null +++ b/docker-build @@ -0,0 +1,21 @@ +#!/bin/sh +# +# Copyright 2019 Northern.tech AS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +IMAGE_NAME=mender-convert + +eval docker build . -t ${IMAGE_NAME} diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100755 index 0000000..bfc703d --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,25 @@ +#!/bin/sh +# +# Copyright 2019 Northern.tech AS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +# run conversion, args provided to container (end of docker run ...) + +cd /mender-convert + +echo "Running mender-convert "$@"" + +./mender-convert "$@" diff --git a/docker-mender-convert b/docker-mender-convert new file mode 100755 index 0000000..ab9c36f --- /dev/null +++ b/docker-mender-convert @@ -0,0 +1,31 @@ +#!/bin/sh +# +# Copyright 2019 Northern.tech AS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +IMAGE_NAME=mender-convert + +MENDER_CONVERT_DIR="$(pwd)" +mkdir -p output + +docker run \ + --mount type=bind,source="$MENDER_CONVERT_DIR,target=/mender-convert" \ + --privileged=true \ + --cap-add=SYS_MODULE \ + -v /dev:/dev \ + -v /lib/modules:/lib/modules:ro \ + --env MENDER_ARTIFACT_NAME=${MENDER_ARTIFACT_NAME} \ + $IMAGE_NAME "$@" diff --git a/mender-convert b/mender-convert new file mode 100755 index 0000000..70f32a3 --- /dev/null +++ b/mender-convert @@ -0,0 +1,99 @@ +#!/usr/bin/env bash +# +# Copyright 2019 Northern.tech AS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +MENDER_CONVERT_VERSION=$(git describe --tags --dirty --exact-match 2>/dev/null || git rev-parse --short HEAD) + +function show_help() { + cat << EOF +mender-convert + +A tool that takes an existing embedded image (Debian, Ubuntu, Raspbian, etc) +and converts it to a Mender image by restructuring partition table and adding +necessary files. + +Usage: $0 COMMAND [options] + +Options: + -d, --disk-image - Path to disk image to convert (Debian, Raspbian, + Ubuntu, etc) + -o, --overlay - Path to root filesystem overlay directory, you + are able to provide this option multiple times + -c, --config - Path to configuration file, you are able to + provide this option multiple times + -v, --version - output version information and exit + -h, --help - display this help and exit + +EOF +} + +function show_version() { + echo "Version: ${MENDER_CONVERT_VERSION}" +} + +function trap_exit() { + sudo rm -rf work +} + +function trap_term() { + echo "Program interrupted by user" +} + +trap trap_term INT TERM +trap trap_exit EXIT + +mender_convert_dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +if [ "${mender_convert_dir}" != "${PWD}" ]; then + echo "You must execute mender-convert from the root directory: ${mender_convert_dir}" + exit 1 +fi + +if [ -z "${MENDER_ARTIFACT_NAME}" ]; then + echo "Sorry, it seems that you have not defined MENDER_ARTIFACT_NAME" + echo "You can do this with the following command:" + echo "" + echo -e "\tMENDER_ARTIFACT_NAME="release-1" ./mender-convert" + exit 1 +fi + +source modules/bootstrap.sh + +# We only handle a selection of the arguments here, the rest are passed on +# to the sub-scripts. +while (( "$#" )); do + case "$1" in + -h | --help) + show_help + exit 0 + ;; + -v | --version) + show_version + exit 0 + ;; + *) + break; + ;; + esac +done + +mkdir -p work +touch work/convert.log + +./mender-convert-extract "$@" +./mender-convert-modify "$@" +./mender-convert-package "$@" + +echo "Output Artifacts and images can be found in deploy directory:" +ls -1 deploy/* diff --git a/mender-convert-extract b/mender-convert-extract new file mode 100755 index 0000000..708d745 --- /dev/null +++ b/mender-convert-extract @@ -0,0 +1,99 @@ +#!/bin/bash +# +# Copyright 2019 Northern.tech AS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "Running $(basename $0): $@" + +source modules/bootstrap.sh +source modules/disk.sh + +disk_image="" +while (( "$#" )); do + case "$1" in + -o | --overlay) + overlays+="${2}" + shift 2 + ;; + -c | --config) + configs+="${2}" + shift 2 + ;; + -d | --disk-image) + disk_image="${2}" + shift 2 + ;; + *) + log_fatal "Sorry but the provided option is not supported: $1" + ;; + esac +done + +if [ -z "${disk_image}" ]; then + log_warn "Sorry, but '--disk-image' is a mandatory option" + log_warn "See ./mender-convert --help for more information" + exit 1 +fi + +if [ ! -e ${disk_image} ]; then + log_fatal "File not found: ${disk_image}" +fi + +source modules/config.sh "${configs[@]}" + +MKFS_VFAT="/usr/bin/mkfs.vfat" +if [ ! -f ${MKFS_VFAT} ]; then + MKFS_VFAT="/sbin/mkfs.vfat" +fi + +declare -i nr_of_parts=$(disk_get_nr_of_parts ${disk_image}) + +log_info "Validating disk image" + +if [ ${nr_of_parts} -eq 0 ]; then + log_fatal "Sorry, but could not find any valid partitions for: ${disk_image}" +fi + +log_info "Disk parsed succesfully" +log_info "NUMBER OF PARTS: ${nr_of_parts} TYPE: $(disk_get_part_value ${disk_image} 1 SCHEME)" + +for ((n=1;n<=${nr_of_parts};n++)); do + part_dst_file="work/part-${n}.fs" + + if [ "$(disk_get_part_value ${disk_image} ${n} TYPE)" == "0x8e" ]; then + log_fatal "Detected an LVM volume group on disk. Unfortunatly this is not yet supported" + fi + + log_info "PART ${n}: SIZE: $(disk_get_part_value ${disk_image} ${n} SIZE) TYPE: $(disk_get_part_value ${disk_image} ${n} TYPE)" + log_info "PART ${n}: extracting to ${part_dst_file}" + + disk_extract_part "${disk_image}" $(disk_get_part_value ${disk_image} ${n} START) \ + $(disk_get_part_value ${disk_image} ${n} SECTORS) ${part_dst_file} +done + +# Some disk images will not have an boot partition for us to extract, +# but we do require it be present and hence we might need to generate +# a filesystem image here. +if [ ${nr_of_parts} -eq 1 ]; then + log_info "Generating boot partition (required, does not exist in original image)" + run_and_log_cmd "dd if=/dev/zero of=work/boot-generated.vfat count=${MENDER_BOOT_PART_SIZE_MB} bs=1M status=none" + run_and_log_cmd "${MKFS_VFAT} work/boot-generated.vfat" +fi + +# Extract boot gap, that is the area from sector 1 until first part, and +# this is done because some images might contain the bootloader embedded here +# and we might need to keep this area intact when we generate our custom +# image. +disk_extract_part "${disk_image}" \ + 1 $(( $(disk_get_part_value ${disk_image} 1 START) - 1)) work/boot-gap.bin diff --git a/mender-convert-modify b/mender-convert-modify new file mode 100755 index 0000000..0ca16e7 --- /dev/null +++ b/mender-convert-modify @@ -0,0 +1,215 @@ +#!/bin/bash +# +# Copyright 2019 Northern.tech AS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default that can be overridden by providing this method in a +# configuration file passed with '--config' +function platform_modify() { + true +} + +function trap_exit() { + echo "mender-convert-modify has finished. Cleaning..." + sudo umount -f work/boot + sudo umount -f work/rootfs +} + +function trap_term() { + true +} + +trap trap_term INT TERM +trap trap_exit EXIT + +echo "Running $(basename $0): $@" + +source modules/bootstrap.sh +source modules/disk.sh +source modules/probe.sh + +while (( "$#" )); do + case "$1" in + -o | --overlay) + overlays+="${2}" + shift 2 + ;; + -c | --config) + configs+="${2}" + shift 2 + ;; + -d | --disk-image) + disk_image="${2}" + shift 2 + ;; + *) + log_fatal "Sorry, but the provided option is not supported: $1" + ;; + esac +done + +source modules/config.sh "${configs[@]}" + +boot_part=$(disk_boot_part) +root_part=$(disk_root_part) + +# Create mount points +mkdir -p work/boot +mkdir -p work/rootfs + +sudo mount ${boot_part} work/boot +sudo mount ${root_part} work/rootfs + +mkdir -p work/mender-deb/files + +log_info "Installing Mender client and related files" + +deb_arch=$(probe_debian_arch_name) +deb_name="mender-client_${MENDER_CLIENT_VERSION}-1_${deb_arch}.deb" +run_and_log_cmd "wget -Nq ${MENDER_STORAGE_URL}/mender-convert/deb/${deb_name} -P work/mender-deb" + +cd work/mender-deb +run_and_log_cmd "ar -xv ${deb_name}" +run_and_log_cmd "tar -xf data.tar.xz -C files" +cd - > /dev/null 2>&1 + +run_and_log_cmd "sudo rsync --archive --keep-dirlinks --verbose work/mender-deb/files/ work/rootfs/" + +if [ "${MENDER_ENABLE_SYSTEMD}" == "y" ]; then + run_and_log_cmd "sudo ln -sf /lib/systemd/system/mender.service \ + work/rootfs/etc/systemd/system/multi-user.target.wants/mender.service" +fi + +if [ "${MENDER_GRUB_EFI_INTEGRATION}" == "y" ]; then + run_and_log_cmd "wget -Nq '${MENDER_GRUBENV_URL}' -P work/" + run_and_log_cmd "tar xvf work/${MENDER_GRUBENV_VERSION}.tar.gz -C work/" + + if [ -z "${MENDER_GRUB_KERNEL_IMAGETYPE}" ]; then + kernel_imagetype=$(probe_kernel_in_boot_and_root) + else + kernel_imagetype="${MENDER_GRUB_KERNEL_IMAGETYPE}" + fi + + if [ -z "${MENDER_GRUB_INITRD_IMAGETYPE}" ]; then + initrd_imagetype=$(probe_initrd_in_boot_and_root) + else + kernel_imagetype="${MENDER_GRUB_INITRD_IMAGETYPE}" + fi + + cat <<- EOF > work/grub-mender-grubenv-${MENDER_GRUBENV_VERSION}/mender_grubenv_defines +mender_rootfsa_part=${MENDER_ROOTFS_PART_A_INDEX} +mender_rootfsb_part=${MENDER_ROOTFS_PART_B_INDEX} +mender_kernel_root_base=${MENDER_STORAGE_DEVICE} +mender_grub_storage_device=${MENDER_GRUB_STORAGE_DEVICE} +kernel_imagetype=${kernel_imagetype} +initrd_imagetype=${initrd_imagetype} +kernel_devicetree=${MENDER_KERNEL_DEVICETREE} +EOF + + if [ -n "${MENDER_GRUB_KERNEL_BOOT_ARGS}" ]; then + cat <<- EOF > work/grub-mender-grubenv-${MENDER_GRUBENV_VERSION}/11_bootargs_grub.cfg +set bootargs="${MENDER_GRUB_KERNEL_BOOT_ARGS}" +EOF + fi + + cd work/grub-mender-grubenv-${MENDER_GRUBENV_VERSION} + run_and_log_cmd "make 2>&1" + run_and_log_cmd "sudo make DESTDIR=../ BOOT_DIR=boot install-boot-files" + run_and_log_cmd "sudo make DESTDIR=../rootfs install-tools" + cd - > /dev/null 2>&1 + + # Remove conflicting boot files. These files do not necessarily effect the + # functionality, but lets get rid of them to avoid confusion. + # + # There is no Mender integration for EFI boot or systemd-boot. + sudo rm -rf work/boot/loader + sudo rm -rf work/boot/EFI/Linux + sudo rm -rf work/boot/EFI/systemd + sudo rm -rf work/boot/NvVars + + log_info "Installing GRUB" + + arch=$(probe_arch) + efi_name=$(probe_grub_efi_name) + efi_target_name=$(probe_grub_efi_target_name) + + log_info "GRUB EFI: ${efi_target_name}" + + run_and_log_cmd "wget -Nq ${MENDER_GRUB_BINARY_STORAGE_URL}/${arch}/${efi_name} -P work/" + run_and_log_cmd "wget -Nq ${MENDER_GRUB_BINARY_STORAGE_URL}/${arch}/grub-editenv -P work/" + + run_and_log_cmd "sudo install -m 751 work/grub-editenv work/rootfs/usr/bin/" + + run_and_log_cmd "sudo mkdir -p work/boot/EFI/BOOT" + run_and_log_cmd "sudo cp work/${efi_name} -P work/boot/EFI/BOOT/${efi_target_name}" +fi + +run_and_log_cmd "sudo mkdir -p work/rootfs/data/mender" +run_and_log_cmd "sudo ln -sf /data/mender work/rootfs/var/lib/mender" + +cat <<- EOF > work/mender.conf.data +{ + "RootfsPartA": "${MENDER_STORAGE_DEVICE}${MENDER_ROOTFS_PART_A_INDEX}", + "RootfsPartB": "${MENDER_STORAGE_DEVICE}${MENDER_ROOTFS_PART_B_INDEX}" +} +EOF + +run_and_log_cmd "sudo cp work/mender.conf.data work/rootfs/data/mender/mender.conf" + +if [ -z "${MENDER_DEVICE_TYPE}" ]; then + # Observed systems who do not have this file, e.g images generated with mkosi + if [ -f work/rootfs/etc/hostname ]; then + device_type=$(cat work/rootfs/etc/hostname) + else + device_type="default" + fi +else + device_type="${MENDER_DEVICE_TYPE}" +fi + +run_and_log_cmd "echo 'device_type=${device_type}' > work/device_type" +run_and_log_cmd "sudo install -m 0444 work/device_type work/rootfs/data/mender/" +run_and_log_cmd "sudo echo 'artifact_name=${MENDER_ARTIFACT_NAME}' \ + > work/rootfs/etc/mender/artifact_info" + +log_info "Installing a custom /etc/fstab (see work/convert.log for more info)" + +if [ "${MENDER_GRUB_EFI_INTEGRATION}" == "y" ]; then + boot_part_mountpoint="/boot/efi" +else + boot_part_mountpoint="/uboot" +fi + +run_and_log_cmd "sudo mkdir -p work/rootfs/${boot_part_mountpoint}" + +if [ "${MENDER_DATA_PART_GROWFS}" == "y" ]; then + MENDER_DATA_PART_FSTAB_OPTS="${MENDER_DATA_PART_FSTAB_OPTS},x-systemd.growfs" +fi + +sudo bash -c "cat <<- EOF > work/rootfs/etc/fstab +# stock fstab - you probably want to override this with a machine specific one +/dev/root / auto defaults 1 1 +proc /proc proc defaults 0 0 + +${MENDER_STORAGE_DEVICE}${MENDER_BOOT_PART_INDEX} ${boot_part_mountpoint} auto defaults,sync 0 0 +${MENDER_STORAGE_DEVICE}${MENDER_DATA_PART_INDEX} /data auto ${MENDER_DATA_PART_FSTAB_OPTS} 0 0 +EOF" + +log_info "Performing platform specific modifications (if any)" +platform_modify + +for overlay in "${overlays[@]}"; do + log_info "Applying rootfs overlay: ${overlay}" + run_and_log_cmd "sudo rsync --archive --keep-dirlinks --verbose ${overlay}/ work/rootfs/" +done diff --git a/mender-convert-package b/mender-convert-package new file mode 100755 index 0000000..f8e968e --- /dev/null +++ b/mender-convert-package @@ -0,0 +1,259 @@ +#!/bin/bash +# +# Copyright 2019 Northern.tech AS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default that can be overridden by providing this method in a +# configuration file passed with '--config' +function platform_package() { + true +} + +function trap_exit() { + echo "mender-convert-package has finished. Cleaning..." + sudo umount -f work/boot > /dev/null + sudo umount -f work/rootfs > /dev/null +} + +function trap_term() { + true +} + +trap trap_term INT TERM +trap trap_exit EXIT + +echo "Running $(basename $0): $@" + +source modules/bootstrap.sh +source modules/disk.sh + +while (( "$#" )); do + case "$1" in + -o | --overlay) + overlays+="${2}" + shift 2 + ;; + -c | --config) + configs+="${2}" + shift 2 + ;; + -d | --disk-image) + disk_image="${2}" + shift 2 + ;; + *) + log_fatal "Sorry but the provided option is not supported: $1" + ;; + esac +done + +source modules/config.sh "${configs[@]}" + +PARTED="/usr/bin/parted" +if [ ! -f ${PARTED} ]; then + PARTED="/sbin/parted" +fi + +output_dir=work + +boot_part=$(disk_boot_part) +root_part=$(disk_root_part) + +# Final output +mkdir -p deploy + +# Create mount points +mkdir -p work/boot +mkdir -p work/rootfs + +sudo mount ${boot_part} work/boot +sudo mount ${root_part} work/rootfs + +# Convert to 512 blocks +disk_image_total_sectors=$(disk_mb_to_sectors ${MENDER_STORAGE_TOTAL_SIZE_MB}) +boot_part_sectors=$(disk_mb_to_sectors ${MENDER_BOOT_PART_SIZE_MB}) +data_part_sectors=$(disk_mb_to_sectors ${MENDER_DATA_PART_SIZE_MB}) +alignment_sectors=$((${MENDER_PARTITION_ALIGNMENT} / 512)) + +if [ "${MENDER_GRUB_EFI_INTEGRATION}" == "y" ]; then + boot_part_start=${alignment_sectors} + overhead_sectors=$(( ${alignment_sectors} * 4 )) +else + # The two first blocks are typically reserved for U-boot environment. + boot_part_start=$(( ${alignment_sectors} * 3)) + overhead_sectors=$(( ${alignment_sectors} * 6 )) +fi + +# Validate boot part size +actual_boot_size=$(du --apparent-size -s --block-size=512 ${boot_part} | cut -f 1) +if [ ${actual_boot_size} -gt ${boot_part_sectors} ]; then + log_warn "The allocated boot part size $(disk_sectors_to_mb ${boot_part_sectors}) MiB is too small." + log_warn "The actual boot part size is $(disk_sectors_to_mb ${actual_boot_size}) MiB" + log_warn "Will adjust MENDER_BOOT_PART_SIZE_MB automatically" + log_warn "Considered adjusting the configuration file to avoid this message" + boot_part_sectors=${actual_boot_size} +fi + +# Make sure that the boot part sector size is aligned to +# MENDER_PARTITION_ALIGNMENT, it is possible that the boot part is extracted +# as-is from the input image, and the input image might not have used the same +# alignment as our configuration +boot_part_sectors=$(disk_align_sectors ${boot_part_sectors} ${MENDER_PARTITION_ALIGNMENT} ) + +# Calculate rootfs size +rootfs_part_sectors=$(( (${disk_image_total_sectors} - ${data_part_sectors} \ + - ${boot_part_sectors} - ${overhead_sectors}) / 2 )) + +# Make sure rootfs size is aligned to MENDER_PARTITION_ALIGNMENT +rootfs_part_sectors=$(disk_align_sectors ${rootfs_part_sectors} ${MENDER_PARTITION_ALIGNMENT} ) + +device_type=$(cat work/rootfs/data/mender/device_type | sed 's/[^=]*=//') +artifact_name=$(cat work/rootfs/etc/mender/artifact_info | sed 's/[^=]*=//') + +image_name="${device_type}-${artifact_name}" + +actual_rootfs_size=$(sudo du --apparent-size -s --block-size=512 work/rootfs | cut -f 1) + +# 50 % free space, not to be confused with rootfs_part_sectors +rootfs_image_sectors=$(awk -v r1="$actual_rootfs_size" 'BEGIN{printf "%.0f", r1 * 1.50}') + +if [ ${rootfs_image_sectors} -gt ${rootfs_part_sectors} ]; then + log_warn "The calculated rootfs partition size $(disk_sectors_to_mb ${rootfs_part_sectors}) MiB is too small." + log_warn "The actual rootfs image size is $(disk_sectors_to_mb ${rootfs_image_sectors}) MiB" + log_fatal "You can try adjusting the MENDER_STORAGE_TOTAL_SIZE_MB variable to increase available space" +fi + +# Extract file-system type from rootfs +if file ${root_part} | grep -q ext4; then + image_fs_type="ext4" +elif file ${root_part} | grep -q XFS; then + image_fs_type="xfs" +else + log_warn "$(file work/${root_part})" + log_fatal "Could not determinate root file-system type. Aborting..." +fi + +disk_create_file_system_from_folder "work/rootfs/data/" "work/data.img" \ + "${data_part_sectors}" "${image_fs_type}" + +# Clear this area as it will be contained in the data.img +sudo rm -rf work/rootfs/data/* + +disk_create_file_system_from_folder "work/rootfs/" "work/rootfs.img" \ + "${rootfs_image_sectors}" "${image_fs_type}" + +log_info "Copying root filesystem image to deploy directory" +run_and_log_cmd "cp --sparse=always work/rootfs.img deploy/${image_name}.${image_fs_type}" + +mender_create_artifact "${device_type}" "${artifact_name}" + +log_info "Creating Mender compatible disk-image" + +sdimg_path=deploy/${image_name}.sdimg + +log_info "Total disk size: $(disk_sectors_to_mb ${disk_image_total_sectors}) MiB" +log_info " Boot partition $(disk_sectors_to_mb ${boot_part_sectors}) MiB" +log_info " RootFS $(disk_sectors_to_mb ${rootfs_part_sectors}) x 2 MiB" +log_info " Data $(disk_sectors_to_mb ${data_part_sectors}) MiB" + +# Initialize sdcard image file +run_and_log_cmd \ + "dd if=/dev/zero of=${sdimg_path} bs=512 count=0 seek=${disk_image_total_sectors} status=none" + +# boot_part_start, is defined at the beginning of this file +boot_part_end=$(( ${boot_part_start} + ${boot_part_sectors} - 1 )) + +rootfsa_start=$(disk_align_sectors ${boot_part_end} ${MENDER_PARTITION_ALIGNMENT} ) +rootfsa_end=$(( ${rootfsa_start} + ${rootfs_part_sectors} - 1 )) + +rootfsb_start=$(disk_align_sectors ${rootfsa_end} ${MENDER_PARTITION_ALIGNMENT} ) +rootfsb_end=$(( ${rootfsb_start} + ${rootfs_part_sectors} - 1 )) + +data_start=$(disk_align_sectors ${rootfsb_end} ${MENDER_PARTITION_ALIGNMENT} ) +data_end=$(( ${data_start} + ${data_part_sectors} - 1 )) + +# Create partition table. TODO: GPT support +run_and_log_cmd "${PARTED} -s ${sdimg_path} mklabel msdos" +run_and_log_cmd "${PARTED} -s ${sdimg_path} unit s mkpart primary fat32 ${boot_part_start} ${boot_part_end}" +run_and_log_cmd "${PARTED} -s ${sdimg_path} set 1 boot on" +run_and_log_cmd "${PARTED} -s ${sdimg_path} -- unit s mkpart primary ext2 ${rootfsa_start} ${rootfsa_end}" +run_and_log_cmd "${PARTED} -s ${sdimg_path} -- unit s mkpart primary ext2 ${rootfsb_start} ${rootfsb_end}" +run_and_log_cmd "${PARTED} -s ${sdimg_path} -- unit s mkpart primary ext2 ${data_start} ${data_end}" +run_and_log_cmd "${PARTED} -s ${sdimg_path} print" + +# Burn Partitions +disk_write_at_offset "${boot_part}" "${sdimg_path}" "${boot_part_start}" +disk_write_at_offset "${output_dir}/rootfs.img" "${sdimg_path}" "${rootfsa_start}" +disk_write_at_offset "${output_dir}/rootfs.img" "${sdimg_path}" "${rootfsb_start}" +disk_write_at_offset "${output_dir}/data.img" "${sdimg_path}" "${data_start}" + +log_info "Performing platform specific package operations (if any)" +platform_package + +# Create bmap index +if [ "${MENDER_USE_BMAP}" == "y" ]; then + BMAP_TOOL="/usr/bin/bmaptool" + if [ ! -e "${BMAP_TOOL}" ]; then + log_error "You have enabled the MENDER_USE_BMAP option, but we could not find the required 'bmaptool'" + log_fatal "You can install 'bmaptool' with: apt-get install bmap-tools (on Debian based distributions)" + fi + run_and_log_cmd "${BMAP_TOOL} create ${sdimg_path} > ${sdimg_path}.bmap" +fi + +if [ "${MENDER_COMPRESS_DISK_IMAGE}" == "y" ]; then + log_info "Compressing ${sdimg_path}.gz" + run_and_log_cmd "pigz --best --force ${sdimg_path}" +fi + +log_info "Conversion has completed! \o/" + +##################### Create configuration file for tests ###################### + +if [ "${MENDER_GRUB_EFI_INTEGRATION}" == "y" ]; then + boot_part_mountpoint="/boot/efi" + + # This is the name of the DISTRO_FEATURES in Yocto + distro_feature="mender-grub" +else + boot_part_mountpoint="/uboot" + + # This is the name of the DISTRO_FEATURES in Yocto + distro_feature="mender-uboot" +fi + +cat <<- EOF > deploy/${image_name}.cfg +MENDER_BOOT_PART="${MENDER_STORAGE_DEVICE}${MENDER_BOOT_PART_INDEX}" +MENDER_ROOTFS_PART_A="${MENDER_STORAGE_DEVICE}${MENDER_ROOTFS_PART_A_INDEX}" +MENDER_ROOTFS_PART_B="${MENDER_STORAGE_DEVICE}${MENDER_ROOTFS_PART_B_INDEX}" +MENDER_BOOT_PART_MOUNT_LOCATION="${boot_part_mountpoint}" +MENDER_BOOT_PART_SIZE_MB="$(disk_sectors_to_mb ${boot_part_sectors})" +MENDER_DATA_PART_SIZE_MB="${MENDER_DATA_PART_SIZE_MB}" +MENDER_DEVICE_TYPE="${device_type}" +MENDER_PARTITIONING_OVERHEAD_KB="$(( (${overhead_sectors} * 512) / 1024 ))" +MENDER_PARTITION_ALIGNMENT="${MENDER_PARTITION_ALIGNMENT}" +MENDER_STORAGE_DEVICE="${MENDER_STORAGE_DEVICE}" +MENDER_STORAGE_TOTAL_SIZE_MB="${MENDER_STORAGE_TOTAL_SIZE_MB}" +MENDER_UBOOT_ENV_STORAGE_DEVICE_OFFSET="12582912" +MENDER_ARTIFACT_NAME="${artifact_name}" +DISTRO_FEATURES="${distro_feature}" +DEPLOY_DIR_IMAGE="${PWD}/deploy" +MENDER_MACHINE="${device_type}" +EOF + +# Something that the tests expect to be defined (originally from Yocto) +cat <<- EOF >> deploy/${image_name}.cfg +IMAGE_FSTYPES="${image_fs_type} mender sdimg" +ARTIFACTIMG_FSTYPE="${image_fs_type}" +LAYER_CONF_VERSION="2" +EOF diff --git a/modules/bootstrap.sh b/modules/bootstrap.sh new file mode 100644 index 0000000..33644e6 --- /dev/null +++ b/modules/bootstrap.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# +# Copyright 2019 Northern.tech AS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Credit to: +# https://github.com/gruntwork-io/bash-commons/blob/master/modules/bash-commons/src/bootstrap.sh + +# Sets some Bash options to encourage well formed code. +# For example, some of the options here will cause the script to terminate as +# soon as a command fails. Another option will cause an error if an undefined +# variable is used. +# See: https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html + +# Any trap on ERR is inherited by shell functions, command substitutions, and +# commands executed in a subshell environment. The ERR trap is normally not +# inherited in such cases. +set -o errtrace + +# Any trap on DEBUG and RETURN are inherited by shell functions, command +# substitutions, and commands executed in a subshell environment. The DEBUG and +# RETURN traps are normally not inherited in such cases. +set -o functrace + +# Exit if any command exits with a non-zero exit status. +set -o errexit + +# Exit if script uses undefined variables. +set -o nounset + +# Prevent masking an error in a pipeline. +# Look at the end of the 'Use set -e' section for an excellent explanation. +# see: https://www.davidpashley.com/articles/writing-robust-shell-scripts/ +set -o pipefail + +# Make debugging easier when you use `set -x` +# See: http://wiki.bash-hackers.org/scripting/debuggingtips#making_xtrace_more_useful +export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }' + +source modules/log.sh +source modules/run.sh diff --git a/modules/config.sh b/modules/config.sh new file mode 100644 index 0000000..82e6c5e --- /dev/null +++ b/modules/config.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# +# Copyright 2019 Northern.tech AS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +for config in configs/mender_convert_config "$@"; do + log_info "Using configuration file: ${config}" + source ${config} +done diff --git a/modules/disk.sh b/modules/disk.sh new file mode 100644 index 0000000..b29b97d --- /dev/null +++ b/modules/disk.sh @@ -0,0 +1,176 @@ +#!/usr/bin/env bash +# +# Copyright 2019 Northern.tech AS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Print desired information about a partition +# +# Example usage: +# +# part_one_start=$(disk_get_part_value ${disk_image_path} 1 START) +# +# $1 - path to disk image +# $2 - part number +# $3 - information to be printed +# +# We use 'partx' to parse input disk image and it supports the following fields: +# +# NR partition number +# START start of the partition in sectors +# END end of the partition in sectors +# SECTORS number of sectors +# SIZE human readable size +# NAME partition name +# UUID partition UUID +# TYPE partition type (a string, a UUID, or hex) +# FLAGS partition flags +# SCHEME partition table type (dos, gpt, ...) +disk_get_part_value() { + echo "$(partx -o ${3} -g -r --nr ${2} ${1})" +} + +# Prints number of partitions found in disk image +# +# Example usage: +# +# nr_of_parts=$(disk_get_nr_of_parts ${disk_image_path}) +# +# $1 - path to disk image +disk_get_nr_of_parts() { + echo "$(partx -l ${1} | wc -l)" +} + +# Extract a file system image from a disk image +# +# $1 - path to disk image +# $2 - sector start (in 512 blocks) +# $3 - size (in 512 blocks) +# $4 - path to output file +disk_extract_part() { + run_and_log_cmd "dd if=$1 of=$4 skip=$2 bs=512 count=$3 conv=sparse status=none" +} + +# Convert MiB to number of 512 sectors +# +# $1 - MiB value +disk_mb_to_sectors() { + echo "$(( (${1} * 1024 * 1024) / 512 ))" +} + +# Convert 512 sectors to MiB +# +# $1 - number of 512 sectors +disk_sectors_to_mb() { + echo "$(( (${1} * 512) / 1024 / 1024 ))" +} + + +# Align value (result is number of 512 sectors) +# +# $1 - value to align (number of 512 sectors) +# $2 - alignment in bytes +disk_align_sectors() { + local size_in_bytes=$(( ${1} * 512)) + local reminder=$(( ${size_in_bytes} % ${2} )) + + if [ $reminder -ne 0 ]; then + size_in_bytes=$(( $size_in_bytes - $reminder + ${2} )) + fi + + echo "$(( $size_in_bytes / 512 ))" +} + +# Write file at offset of another file +# +# $1 - file to write +# $2 - destination file +# $3 - offset in number of 512 sectors +disk_write_at_offset() { + run_and_log_cmd "dd if=${1} of=${2} seek=${3} conv=sparse status=none" +} + +# Create file system image from directory content +# +# $1 - base folder +# $2 - destination file +# $3 - image size in sectors +# $4 - file system type, e.g ext4, xfs, ... +disk_create_file_system_from_folder() { + log_info "Creating a file-system image from: ${1}" + + run_and_log_cmd "dd if=/dev/zero of=${2} seek=${3} count=0 bs=512 status=none" + + case "$4" in + "ext4") + MKFS_EXT4="/usr/bin/mkfs.ext4" + if [ ! -f ${MKFS_EXT4} ]; then + MKFS_EXT4="/sbin/mkfs.ext4" + fi + run_and_log_cmd "${MKFS_EXT4} -q -F ${2}" + ;; + + "xfs") + MKFS_XFS="/usr/bin/mkfs.xfs" + if [ ! -f ${MKFS_XFS} ]; then + MKFS_XFS="/sbin/mkfs.xfs" + fi + run_and_log_cmd "${MKFS_XFS} -q -f ${2}" + ;; + *) + log_fatal "Unknown file system type specified: ${4}" + esac + + run_and_log_cmd "mkdir -p work/output" + run_and_log_cmd "sudo mount ${2} work/output" + run_and_log_cmd "sudo rsync --archive --delete ${1} work/output/" + run_and_log_cmd "sudo umount work/output" +} + +# Print path to the boot partition filesystem image +# +disk_boot_part() { + # Why this little dance you might wonder. + # + # Some disk images do not have a boot partition, but our integration does + # require one and this is why there is a difference here. + # + # If input image did not have a boot part it will be generated and be called: + # + # work/boot-generated.vfat + # + # This is mostly done to make it obvious that we have created an empty vfat file. + # + # But if the disk image already had a boot part, we assume that it is the first + # partition that was extracted. + # + # We also need to adjust the part number of the rootfs part depending on if + # boot part was extracted or generated. + boot_part="work/boot-generated.vfat" + if [ ! -f ${boot_part} ]; then + boot_part="work/part-1.fs" + fi + echo "${boot_part}" +} + +# Print path to the root partition filesystem image +# +disk_root_part() { + boot_part="work/boot-generated.vfat" + if [ ! -f ${boot_part} ]; then + root_part="work/part-2.fs" + else + root_part="work/part-1.fs" + fi + echo "${root_part}" +} diff --git a/modules/log.sh b/modules/log.sh new file mode 100644 index 0000000..b13261b --- /dev/null +++ b/modules/log.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# +# Copyright 2019 Northern.tech AS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This must be an absolute path, as users might call the log functions +# from sub-directories +log_file="${PWD}/work/convert.log" + +# Log the given message at the given level. +function log { + local -r level="$1" + local -r message="$2" + local -r timestamp=$(date +"%Y-%m-%d %H:%M:%S") + local -r script_name="$(basename "$0")" + >&2 echo -e "${timestamp} [${level}] [$script_name] ${message}" | tee -a ${log_file} +} + +function local_log_debug { + local -r level="DEBUG" + local -r message="$1" + local -r timestamp=$(date +"%Y-%m-%d %H:%M:%S") + local -r script_name="$(basename "$0")" + echo -e "${timestamp} [${level}] [$script_name] ${message}" >> ${log_file} +} + +# Log the given message at DEBUG level. +function log_debug { + local -r message="$1" + local_log_debug "$message" +} + +# Log the given message at INFO level. +function log_info { + local -r message="$1" + log "INFO" "$message" +} + +# Log the given message at WARN level. +function log_warn { + local -r message="$1" + log "WARN" "$message" +} + +# Log the given message at ERROR level. +function log_error { + local -r message="$1" + log "ERROR" "$message" +} + +# Log the given message at FATAL level. +function log_fatal { + local -r message="$1" + log "FATAL" "$message" + exit 1 +} diff --git a/modules/probe.sh b/modules/probe.sh new file mode 100644 index 0000000..736c4fc --- /dev/null +++ b/modules/probe.sh @@ -0,0 +1,237 @@ +#!/usr/bin/env bash +# +# Copyright 2019 Northern.tech AS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Prints target architecture +# +# No input parameters and these work on the assumption that boot and root parts +# are mounted at work/boot and work/rootfs +probe_arch() { + # --dereference, means to follow symlinks because 'ls' could be a symlink + # to busybox + file_info="" + for location in bin/ls usr/bin/ls; do + if [ -e work/rootfs/${location} ]; then + file_info=$(file -b --dereference work/rootfs/${location}) + break + fi + done + + if [ -z "${file_info}" ]; then + log_fatal "Sorry, where not able to determinate target architecture" + fi + + target_arch="unknown" + if grep -q x86-64 <<< "${file_info}"; then + target_arch="x86-64" + elif grep -Eq "ELF 32-bit.*ARM" <<< "${file_info}"; then + target_arch="arm" + elif grep -Eq "ELF 64-bit.*aarch64" <<< "${file_info}"; then + target_arch="aarch64" + else + log_fatal "Unsupported architecture: ${file_info}" + fi + echo "${target_arch}" +} + +# Prints GRUB EFI name depending on target architecture +# +# No input parameters and these work on the assumption that boot and root parts +# are mounted at work/boot and work/rootfs +probe_grub_efi_name() { + efi_name="" + arch=$(probe_arch) + case "${arch}" in + "x86-64") + efi_name="grub-efi-bootx64.efi" + ;; + "arm") + efi_name="grub-efi-bootarm.efi" + ;; + "aarch64") + efi_name="grub-efi-bootaa64.efi" + ;; + *) + log_fatal "Unknown arch: ${arch}" + esac + echo "$efi_name" +} + +# Prints Debian arch name depending on target architecture +# +# No input parameters and these work on the assumption that boot and root parts +# are mounted at work/boot and work/rootfs +probe_debian_arch_name() { + deb_arch="" + arch=$(probe_arch) + case "${arch}" in + "x86-64") + deb_arch="amd64" + ;; + "arm") + deb_arch="armhf" + ;; + "aarch64") + deb_arch="arm64" + ;; + *) + log_fatal "Unknown arch: ${arch}" + esac + echo "${deb_arch}" +} + +# Prints GRUB EFI target name depending on target architecture +# +# This is what the file name should be when put on target boot part. +# +# No input parameters and these work on the assumption that boot and root parts +# are mounted at work/boot and work/rootfs +probe_grub_efi_target_name() { + efi_target_name="" + arch=$(probe_arch) + case "$arch" in + "x86-64") + efi_target_name="bootx64.efi" + ;; + "arm") + efi_target_name="bootarm.efi" + ;; + "aarch64") + efi_target_name="bootaa64.efi" + ;; + *) + log_fatal "Unknown arch: ${arch}" + esac + echo "$efi_target_name" +} + +# Prints path to the Linux kernel image +# +# $1 - directory in which the search is performed +# +probe_kernel_image() { + kernel_image_path="" + for image in vmlinuz zImage bzImage; do + # Linux kernel image type and naming varies between different platforms. + # + # The wildcard at the end is important, because it is common to suffix the + # Linux kernel version to the image type/name, e.g: + # + # vmlinuz-4.14-x86_64 + # vmlinuz-3.10.0-862.el7.x86_64 + # vmlinuz-4.15.0-20-generic + # + kernel_image_path=$(sudo find ${1} -name ${image}* ! -name '*-rescue-*') + if [ -n "${kernel_image_path}" ]; then + break + fi + done + echo "${kernel_image_path}" +} + +# Prints path to the initrd/initramfs image +# +# $1 - directory in which the search is performed +# +probe_initrd_image() { + initrd_image_path="" + for image in initramfs initrd; do + # initrd/initramfs naming varies between different platforms. + # + # The wildcard at the end is important, because it is common to suffix the + # Linux kernel version to the image name, e.g: + # + # initrd.img-4.15.0-20-generic + # + initrd_image_path=$(sudo find ${1} -name ${image}* ! -name '*-rescue-*') + if [ -n "${initrd_image_path}" ]; then + break + fi + done + echo "${initrd_image_path}" +} + + +# Prints Linux kernel image name +# +# It will look for it in both boot and rootfs parts. If image is only present +# in boot part, it will move it to rootfs/boot +# +# No input parameters and these work on the assumption that boot and root parts +# are mounted at work/boot and work/rootfs +probe_kernel_in_boot_and_root() { + kernel_imagetype_path="" + + # Important to check rootfs/boot first, because it might be possible that + # they are stored in both partitions, and in this case we want to find the + # image in the rootfs first and use that, to avoid copying it over from + # boot part when it is already there. + for boot in work/rootfs/boot work/boot; do + kernel_imagetype_path=$(probe_kernel_image ${boot}) + if [ -n "${kernel_imagetype_path}" ] && [ "${boot}" == "work/boot" ]; then + log_info "Found Linux kernel image in boot part, moving to rootfs/boot" + sudo cp ${kernel_imagetype_path} work/rootfs/boot + break; + elif [ -n "${kernel_imagetype_path}" ]; then + break; + fi + done + + if [ -n "${kernel_imagetype_path}" ]; then + log_info "Found Linux kernel image: \n\n\t${kernel_imagetype_path}\n" + kernel_imagetype=$(basename ${kernel_imagetype_path}) + else + log_warn "Unfortunatly we where not able to find the Linux kernel image." + log_fatal "Please specifc the image name using MENDER_GRUB_KERNEL_IMAGETYPE" + fi + echo "${kernel_imagetype}" +} + +# Prints initrd/initramfs image name +# +# It will look for it in both boot and rootfs parts. If image is only present +# in boot part, it will move it to rootfs/boot +# +# No input parameters and these work on the assumption that boot and root parts +# are mounted at work/boot and work/rootfs +probe_initrd_in_boot_and_root() { + initrd_image_path="" + + # Important to check rootfs/boot first, because it might be possible that + # they are stored in both partitions, and in this case we want to find the + # image in the rootfs first and use that, to avoid copying it over from + # boot part when it is already there. + for boot in work/rootfs/boot work/boot; do + initrd_image_path=$(probe_initrd_image ${boot}) + if [ -n "${initrd_image_path}" ] && [ "${boot}" == "work/boot" ]; then + sudo cp ${initrd_image_path} ${target_rootfs_dir}/boot + break; + elif [ -n "${initrd_image_path}" ]; then + break; + fi + done + + if [ -n "${initrd_image_path}" ]; then + log_info "Found initramfs image: \n\n\t${initrd_image_path}\n" + initrd_imagetype=$(basename ${initrd_image_path}) + else + log_info "Unfortunatly we where not able to find the initrd image." + log_info "Please specifc the image name using MENDER_GRUB_INITRD_IMAGETYPE \ +(only required if your board is using this)" + initrd_imagetype="" + fi + + echo "${initrd_imagetype}" +} diff --git a/modules/run.sh b/modules/run.sh new file mode 100644 index 0000000..6a1a618 --- /dev/null +++ b/modules/run.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# +# Copyright 2019 Northern.tech AS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Run a command, capture output and log it +# +# $1 - command to run +function run_and_log_cmd() { + local -r cmd="${1}" + + log_debug "Running: \n\r\n\r\t ${cmd}" + log_debug "Run result: \n\r" + + result=$(eval ${cmd}) + log_debug "${result}" +} diff --git a/requirements-deb.txt b/requirements-deb.txt new file mode 100644 index 0000000..4d91f68 --- /dev/null +++ b/requirements-deb.txt @@ -0,0 +1 @@ +binutils xz-utils file rsync parted e2fsprogs xfsprogs pigz dosfstools wget git make bmap-tools diff --git a/rootfs_overlay_demo/.gitkeep b/rootfs_overlay_demo/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/scripts/README-run-tests.md b/scripts/README-run-tests.md new file mode 100644 index 0000000..f6b5219 --- /dev/null +++ b/scripts/README-run-tests.md @@ -0,0 +1,13 @@ +# Run tests + +Run the following commands to install test dependencies (assumes that all mender-convert dependencies are already installed): + + sudo apt-get update e2fsprogs=1.44.1-1 + sudo apt-get -qy --force-yes install python-pip + sudo pip2 install pytest --upgrade + sudo pip2 install pytest-xdist --upgrade + sudo pip2 install pytest-html --upgrade + +Run tests: + + ./scripts/run-tests.sh diff --git a/scripts/bootstrap-rootfs-overlay-demo.sh b/scripts/bootstrap-rootfs-overlay-demo.sh new file mode 100755 index 0000000..57bac6f --- /dev/null +++ b/scripts/bootstrap-rootfs-overlay-demo.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# +# Copyright 2019 Northern.tech AS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Exit if any command exits with a non-zero exit status. +set -o errexit + +root_dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )/../" && pwd ) +if [ "${root_dir}" != "${PWD}" ]; then + echo "You must execute $(basename $0) from the root directory: ${root_dir}" + exit 1 +fi + +# Do not actually paste it here, this is just the default value that will +# end up in mender.conf if no token is specified using '--tenant-token' +tenant_token="Paste your Hosted Mender token here" +output_dir="" +while (( "$#" )); do + case "$1" in + -t | --tenant-token) + tenant_token="${2}" + shift 2 + ;; + -o | --output-dir) + output_dir="${2}" + shift 2 + ;; + *) + echo "Sorry but the provided option is not supported: $1" + echo "Usage: $(basename $0) --tenant-token" + exit 1 + ;; + esac +done + +if [ -z "${output_dir}" ]; then + echo "Sorry, but you need to provide an output directory using the '-o/--output-dir' option" + exit 1 +fi + +mkdir -p ${output_dir}/etc/mender +cat <<- EOF > ${output_dir}/etc/mender/mender.conf +{ + "InventoryPollIntervalSeconds": 5, + "RetryPollIntervalSeconds": 30, + "ServerURL": "https://hosted.mender.io/", + "TenantToken": "${tenant_token}", + "UpdatePollIntervalSeconds": 5 +} +EOF + +echo "Configuration file written to: ${output_dir}/etc/mender" diff --git a/scripts/run-tests.sh b/scripts/run-tests.sh new file mode 100755 index 0000000..d431344 --- /dev/null +++ b/scripts/run-tests.sh @@ -0,0 +1,143 @@ +#!/bin/bash + +set -e + +root_dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )/../" && pwd ) +if [ "${root_dir}" != "${PWD}" ]; then + echo "You must execute $(basename $0) from the root directory: ${root_dir}" + exit 1 +fi + +WORKSPACE=./tests + +# Relative to where scripts are executed (${WORKSPACE}/mender-image-tests) +MENDER_CONVERT_DIR=../../ + +BBB_DEBIAN_IMAGE="bone-debian-9.5-iot-armhf-2018-10-07-4gb.img" +BBB_DEBIAN_IMAGE_URL="http://debian.beagleboard.org/images/${BBB_DEBIAN_IMAGE}.xz" + +RASPBIAN_IMAGE="2019-04-08-raspbian-stretch-lite" +RASPBIAN_IMAGE_URL="https://downloads.raspberrypi.org/raspbian_lite/images/raspbian_lite-2019-04-09/${RASPBIAN_IMAGE}.zip" + +TINKER_IMAGE="20170417-tinker-board-linaro-stretch-alip-v1.8" +TINKER_IMAGE_URL="http://dlcdnet.asus.com/pub/ASUS/mb/Linux/Tinker_Board_2GB/${TINKER_IMAGE}.zip" + +UBUNTU_IMAGE="Ubuntu-Bionic-x86-64.img" +UBUNTU_IMAGE_URL="https://d1b0l86ne08fsf.cloudfront.net/mender-convert/images/${UBUNTU_IMAGE}.gz" + +MENDER_ACCEPTANCE_URL="https://raw.githubusercontent.com/mendersoftware/meta-mender/master/tests/acceptance" + +# Some distros do not have /sbin in path for "normal users" +export PATH="${PATH}:/sbin" + +convert_and_test() { + device_type=$1 + artifact_name=$2 + image_url=$3 + image_name=$4 + image_name_compressed=$5 + config=$6 # Optional + + wget -N ${image_url} -P input/ + + echo "Extracting: ${image_name_compressed}" + case "${image_name_compressed}" in + *.gz) + gunzip -f input/${image_name_compressed} + ;; + *.zip) + cd input + unzip -o ${image_name_compressed} + cd - + ;; + *.xz) + xz -d -f input/${image_name_compressed} + ;; + *) + echo "Unknown image type: ${image_name_compressed}" + exit 1 + esac + + rm -f ${WORKSPACE}/test_config + + if [ -n "${config}" ]; then + cp ${config} ${WORKSPACE}/test_config + fi + + # Disable compression of disk image when testing, otherwise we need to + # unpack each image we test which is time consuming + echo "MENDER_COMPRESS_DISK_IMAGE=n" >> ${WORKSPACE}/test_config + + echo "Configuration used:" + cat ${WORKSPACE}/test_config + + MENDER_ARTIFACT_NAME=${artifact_name} ./docker-mender-convert \ + --disk-image input/${image_name} \ + --config ${WORKSPACE}/test_config + + cd ${WORKSPACE}/mender-image-tests + + py.test --verbose \ + --junit-xml="${WORKSPACE}/results.xml" \ + --test-conversion \ + --test-variables="${MENDER_CONVERT_DIR}/deploy/${device_type}-${artifact_name}.cfg" \ + --board-type="${device_type}" \ + --mender-image=${device_type}-${artifact_name}.sdimg \ + --sdimg-location="${MENDER_CONVERT_DIR}/deploy" + + cd - +} + +get_pytest_files() { + wget -N ${MENDER_ACCEPTANCE_URL}/pytest.ini -P $WORKSPACE/mender-image-tests + wget -N ${MENDER_ACCEPTANCE_URL}/common.py -P $WORKSPACE/mender-image-tests + wget -N ${MENDER_ACCEPTANCE_URL}/conftest.py -P $WORKSPACE/mender-image-tests + wget -N ${MENDER_ACCEPTANCE_URL}/fixtures.py -P $WORKSPACE/mender-image-tests +} + +if [ ! -d ${WORKSPACE}/mender-image-tests ]; then + git clone https://github.com/mendersoftware/mender-image-tests ${WORKSPACE}/mender-image-tests +else + cd ${WORKSPACE}/mender-image-tests + git pull + cd - +fi + +if ! [ -x "$(command -v mender-artifact)" ]; then + echo "mender-artifact: not found in PATH." + github_PR_status "failure" "mender-artifact: not found in PATH." + exit 1 +fi + +mkdir -p ${WORKSPACE} + +get_pytest_files + +./docker-build + +convert_and_test "qemux86_64" \ + "release-1" \ + "${UBUNTU_IMAGE_URL}" \ + "${UBUNTU_IMAGE}" \ + "${UBUNTU_IMAGE}.gz" \ + "configs/qemux86-64_config" + + +convert_and_test "raspberrypi" \ + "release-1" \ + "${RASPBIAN_IMAGE_URL}" \ + "${RASPBIAN_IMAGE}.img" \ + "${RASPBIAN_IMAGE}.zip" \ + "configs/raspberrypi3_config" + +convert_and_test "linaro-alip" \ + "release-1" \ + "${TINKER_IMAGE_URL}" \ + "${TINKER_IMAGE}.img" \ + "${TINKER_IMAGE}.zip" + +convert_and_test "beaglebone" \ + "release-1" \ + "${BBB_DEBIAN_IMAGE_URL}" \ + "${BBB_DEBIAN_IMAGE}" \ + "${BBB_DEBIAN_IMAGE}.xz"