Browse Source

MEN-2608: add mender-convert (version 2)

The goals of the re-write was to achieve the following:

1. It should be easy to extend the tool to support other boards or distributions
2. We should not compile code in the tool (rely on binaries built elsewhere)
   compiling code increases complexity, due to requirement of toolchains etc.
3. The tool shall not be designed around specific hardware/platform types
    - This is the case today with the usage of --device-type flag
4 The tool should be to convert images without knowing anything about the hardware/platform
  relates to above 3.
5. Configuration interface should be simplified
    - command line flags -> configuration files
6. Platform specific code shall be provided trough “hooks”, and are not part of the “core” mender-convert code
7. It shall be easy to extend functionality
    - support for rootfs overlay to inject user applications/configurations
    - ability to override how the Mender Artifact is generated (to be able to sign and include state-scripts)
8. Code structure should be modular
    - Eases Maintenance and possibility of making isolated changes

Changelog: Title

Signed-off-by: Mirza Krak <mirza.krak@northern.tech>
2.0.x
Mirza Krak 6 years ago
parent
commit
fac622517c
  1. 6
      .dockerignore
  2. 10
      .gitignore
  3. 45
      Dockerfile
  4. 181
      LICENSE
  5. 2
      LIC_FILES_CHKSUM.sha256
  6. 153
      README.md
  7. 37
      configs/beaglebone_black_base_config
  8. 6
      configs/beaglebone_black_debian_sdcard_config
  9. 147
      configs/mender_convert_config
  10. 35
      configs/mender_grub_config
  11. 16
      configs/qemux86-64_config
  12. 8
      configs/raspberrypi3_config
  13. 82
      configs/raspberrypi_config
  14. 25
      configs/rockpro64_config
  15. 8
      configs/rockpro64_emmc_config
  16. 6
      configs/rockpro64_sd_config
  17. 21
      docker-build
  18. 25
      docker-entrypoint.sh
  19. 31
      docker-mender-convert
  20. 99
      mender-convert
  21. 99
      mender-convert-extract
  22. 215
      mender-convert-modify
  23. 259
      mender-convert-package
  24. 52
      modules/bootstrap.sh
  25. 20
      modules/config.sh
  26. 176
      modules/disk.sh
  27. 67
      modules/log.sh
  28. 237
      modules/probe.sh
  29. 28
      modules/run.sh
  30. 1
      requirements-deb.txt
  31. 0
      rootfs_overlay_demo/.gitkeep
  32. 13
      scripts/README-run-tests.md
  33. 64
      scripts/bootstrap-rootfs-overlay-demo.sh
  34. 143
      scripts/run-tests.sh

6
.dockerignore

@ -1,3 +1,3 @@
input/
output/
device-image-shell/output
deploy
input
work

10
.gitignore

@ -1,5 +1,5 @@
input/
output/
device-image-shell/output
mendertesting/
integration/
deploy
input
work
rootfs_overlay_demo/*
tests

45
Dockerfile

@ -0,0 +1,45 @@
FROM ubuntu:19.04
ARG MENDER_ARTIFACT_VERSION=3.1.0
RUN apt-get update && apt-get install -y \
# For 'ar' command to unpack .deb
binutils \
xz-utils \
# to be able to detect file system types of extracted images
file \
# to copy files between rootfs directories
rsync \
# to generate partition table
parted \
# mkfs.ext4 and family
e2fsprogs \
# mkfs.xfs and family
xfsprogs \
# Parallel gzip compression
pigz \
sudo \
# mkfs.vfat (required for boot partition)
dosfstools \
# to download mender-artifact
wget \
# to download mender-grub-env
git \
# to compile mender-grub-env
make \
# to get rid of 'sh: 1: udevadm: not found' errors triggered by parted
udev \
# to create bmap index file (MENDER_USE_BMAP)
bmap-tools
RUN wget -q -O /usr/bin/mender-artifact https://d1b0l86ne08fsf.cloudfront.net/mender-artifact/$MENDER_ARTIFACT_VERSION/linux/mender-artifact \
&& chmod +x /usr/bin/mender-artifact
# allow us to keep original PATH variables when sudoing
RUN echo "Defaults secure_path=\"/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin:$PATH\"" > /etc/sudoers.d/secure_path_override
RUN chmod 0440 /etc/sudoers.d/secure_path_override
WORKDIR /
COPY docker-entrypoint.sh /usr/local/bin/
ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]

181
LICENSE

@ -0,0 +1,181 @@
Copyright 2019 Northern.tech AS
All content in this project is licensed under the Apache License v2, unless
indicated otherwise.
-------------------------------------------------------------------------------
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.

2
LIC_FILES_CHKSUM.sha256

@ -0,0 +1,2 @@
beb140be4cd64599bedc691a55b2729c9cc611a4b9d6ec44e01270105daf18a2 LICENSE
beb140be4cd64599bedc691a55b2729c9cc611a4b9d6ec44e01270105daf18a2 mendertesting/LICENSE

153
README.md

@ -0,0 +1,153 @@
# mender-convert
Mender is an open source over-the-air (OTA) software updater for embedded Linux devices. Mender comprises a client running at the embedded device, as well as a server that manages deployments across many devices.
This repository contains mender-convert, which is used to convert pre-built disk images (Debian, Ubuntu, Raspbian, etc) to a Mender compatible image by restructuring partition table and injecting the necessary files.
For a full list of tested devices and images please visit [Mender Hub](https://hub.mender.io/c/board-integrations/debian-family). If your device and image combination is not listed as supported, this does not necessarily mean that it will not work, it probably just means that no one has tested and reported it back and usually only small tweaks are necessary to get this running on your device.
![Mender logo](https://mender.io/user/pages/resources/06.digital-assets/mender.io.png)
## Getting started
To start using Mender, we recommend that you begin with the Getting started
section in [the Mender documentation](https://docs.mender.io/).
For more detailed information about `mender-convert` please visit the
[Debian family](https://docs.mender.io/artifacts/debian-family) section in
[the Mender documentation](https://docs.mender.io/).
### Prepare image and configuration
Download the raw Raspberry Pi disk image into a subdirectory input:
```bash
mkdir -p input
cd input
wget https://downloads.raspberrypi.org/raspbian_lite/images/raspbian_lite-2019-06-24/2019-06-20-raspbian-buster-lite.zip
```
Extract the raw Raspberry Pi disk image:
```bash
unzip 2019-06-20-raspbian-buster-lite.zip && cd ..
```
Bootstrap the demo rootfs overlay that is configured to connect to
https://hosted.mender.io with polling intervals set appropriately for
demonstration purposes:
```
./scripts/bootstrap-rootfs-overlay-demo.sh \
--output-dir ${PWD}/rootfs_overlay_demo \
--tenant-token "Paste token from Hosted Mender"
```
### Docker environment for mender-convert
To make using mender-convert easier, a reference setup using a Docker
container is provided.
You need to [install Docker Engine](https://docs.docker.com/install) to use
this environment.
#### Build the mender-convert container image
Build a container with all required dependencies for `mender-convert`:
```bash
./docker-build
```
This will create a container image which you can use to run `mender-convert`
without polluting your host environment with the necessary dependencies.
#### Use the mender-convert container image
Run mender-convert from inside the container with your desired options, e.g.
```bash
MENDER_ARTIFACT_NAME=release-1 ./docker-mender-convert \
--disk-image input/2019-04-08-raspbian-stretch-lite.img \
--config configs/raspberrypi3_config \
--overlay rootfs_overlay_demo/
```
Conversion will take 10-30 minutes, depending on image size and resources
available. You can watch `work/convert.log` for progress and diagnostics
information.
After it finishes, you can find your images in the `deploy` directory on your
host machine!
## Using mender-convert without Docker
In order to be able to manipulate and create filesystem and disk images,
mender-convert has a few dependencies, and their version and name vary between
Linux distributions. Here is an example of how to install the dependencies on a
Debian based distribution:
```
sudo apt install $(cat requirements-deb.txt)
```
Start the conversion process with:
```bash
MENDER_ARTIFACT_NAME=release-1 ./mender-convert \
--disk-image input/2019-04-08-raspbian-stretch-lite.img \
--config configs/raspberrypi3_config \
--overlay rootfs_overlay_demo/
```
**NOTE!** You will be prompted to enter `sudo` password during the conversion
process. This is required to be able to loopback mount images and for modifying
them. Our recommendation is to use the provided Docker container, to run the
tool in a isolated environment (at least to some degree).
## Contributing
We welcome and ask for your contribution. If you would like to contribute to
Mender, please read our guide on how to best get started
[contributing code or documentation](https://github.com/mendersoftware/mender/blob/master/CONTRIBUTING.md).
## License
Mender is licensed under the Apache License, Version 2.0. See
[LICENSE](https://github.com/mendersoftware/mender-convert/blob/master/LICENSE)
for the full license text.
## Security disclosure
We take security very seriously. If you come across any issue regarding
security, please disclose the information by sending an email to
[security@mender.io](security@mender.io). Please do not create a new public
issue. We thank you in advance for your cooperation.
## Connect with us
* Join the [Mender Hub discussion forum](https://hub.mender.io)
* Follow us on [Twitter](https://twitter.com/mender_io). Please
feel free to tweet us questions.
* Fork us on [Github](https://github.com/mendersoftware)
* Create an issue in the [bugtracker](https://tracker.mender.io/projects/MEN)
* Email us at [contact@mender.io](mailto:contact@mender.io)
* Connect to the [#mender IRC channel on Freenode](http://webchat.freenode.net/?channels=mender)
## Authors
Mender was created by the team at [Northern.tech AS](https://northern.tech),
with many contributions from the community. Thanks
[everyone](https://github.com/mendersoftware/mender/graphs/contributors)!
[Mender](https://mender.io) is sponsored by [Northern.tech AS](https://northern.tech).

37
configs/beaglebone_black_base_config

@ -0,0 +1,37 @@
# Binaries generated with the following script:
#
# https://github.com/drewmoseley/mender-convert-integration-scripts/blob/master/build-uboot-bbb.sh
#
# There are reported issues with GRUB bootloader integration, fallback to U-boot.
MENDER_GRUB_EFI_INTEGRATION=n
# 4MB alignment
MENDER_PARTITION_ALIGNMENT="4194304"
BEAGLEBONE_BLACK_BINARIES_URL="${MENDER_STORAGE_URL}/mender-convert/beaglebone/${BEAGLEBONE_BLACK_BINARIES}"
function platform_modify() {
mkdir -p work/bbb/binaries
run_and_log_cmd "wget -q ${BEAGLEBONE_BLACK_BINARIES_URL} -P work/bbb/binaries"
run_and_log_cmd "tar xvf work/bbb/binaries/${BEAGLEBONE_BLACK_BINARIES} -C work/bbb/binaries"
# Mask udisks2.service, otherwise it will mount the inactive part and we
# might write an update while it is mounted which often result in
# corruptions.
#
# TODO: Find a way to only blacklist mmcblk0pX devices instead of masking
# the service.
run_and_log_cmd "sudo ln -sf /dev/null work/rootfs/etc/systemd/system/udisks2.service"
# Place u-boot and MLO into rootfs/boot
run_and_log_cmd "sudo mkdir -p work/rootfs/boot"
run_and_log_cmd "sudo cp work/bbb/binaries/MLO work/boot/"
run_and_log_cmd "sudo cp work/bbb/binaries/u-boot.img work/boot/"
run_and_log_cmd "sudo cp work/bbb/binaries/fw_env.config work/rootfs/etc/"
run_and_log_cmd "sudo cp work/bbb/binaries/uboot-git-log.txt work/boot"
run_and_log_cmd "sudo install -m 755 work/bbb/binaries/fw_printenv work/rootfs/sbin/fw_printenv"
run_and_log_cmd "sudo ln -fs /sbin/fw_printenv work/rootfs/sbin/fw_setenv"
}

6
configs/beaglebone_black_debian_sdcard_config

@ -0,0 +1,6 @@
# Binaries generated with the following script:
#
# https://github.com/drewmoseley/mender-convert-integration-scripts/blob/master/build-uboot-bbb.sh
#
BEAGLEBONE_BLACK_BINARIES="beaglebone-black-integration-debian-2018.07.002.tar"
source configs/beaglebone_black_base_config

147
configs/mender_convert_config

@ -0,0 +1,147 @@
# This is the default configuration used by mender-convert. You can override
# any option specified here by providing your own configuration file using the
# '--config' argument.
#
# NOTE! This file will always be sourced.
# Compress generated disk image using gzip
#
# This is useful when you have large disk images, compressing them
# makes it easier to transfer them between e.g an build server and a local
# machine, and obviously saves space.
MENDER_COMPRESS_DISK_IMAGE=y
# Compression algorithm for Mender Artifact
#
# Supported values are: lzma, gzip (default), none
#
# LZMA will produce a smaller Mender Artifact (2-3x) but will significantly
# increase time spent generating the Mender Artifact (10x)
MENDER_ARTIFACT_COMPRESSION="gzip"
# Enable Mender systemd server
#
# You want this enabled if you want the Mender client to operate in managed mode
# and connect to a server. If you are not interested connecting to a server
# and will only be running standalone mode updates, then you can safely disable
# this.
MENDER_ENABLE_SYSTEMD=y
# Set the fstab options for mounting the data partition
MENDER_DATA_PART_FSTAB_OPTS="defaults"
# The file system will be grown to occupy the full block device. If the file
# system is already at maximum size, no action will be performed.
#
# This feature is utilizing x−systemd.growfs and will only work on systemd 242
# and newer and nothing will be done if an older systemd version is used.
MENDER_DATA_PART_GROWFS=y
# The name of the image or update that will be built. This is what the device
# will report that it is running, and different updates must have different
# names.
#
# This variable must be defined or the build will fail, but you should not
# set it in a configuration file and instead it should be an input to the
# mender-convert tool, e.g
#
# MENDER_ARTIFACT_NAME="release-1" ./mender-convert
#
#MENDER_ARTIFACT_NAME=""
# A string that defines the type of device this image will be installed on and
# this value is used for compatibility checks between Mender Artifact and
# what the device is reporting.
#
# You can define multiple types by providing a space separated string of device
# types.
#
# It defaults to the content of '/etc/hostname', which is extracted from the
# input disk image
MENDER_DEVICE_TYPE=""
# Total size of the physical storage medium that mender partitioned images
# will be written to, expressed in MiB. The size of rootfs partition will be
# calculated automatically by subtracting the sizes of boot
# (see MENDER_BOOT_PART_SIZE_MB) and data partitions
# (see MENDER_DATA_PART_SIZE_MB).
#
MENDER_STORAGE_TOTAL_SIZE_MB="8192"
# The size of the boot partition in the generated .biosimg, .sdimg or .uefiimg
# file.
MENDER_BOOT_PART_SIZE_MB="40"
# The size of the persistent data partition in the generated .biosimg, .sdimg
# or .uefiimg file.
#
# You rarely need to increase this number as this partition will be expanded
# on first boot to occupy the renaming free blocks on the physical storage.
MENDER_DATA_PART_SIZE_MB="128"
# Alignment of partitions used when building partitioned images, expressed in
# bytes
#
# Default is 8MB
MENDER_PARTITION_ALIGNMENT="8388608"
# Mender client version
#
# This is used to fetch the correct binaries
MENDER_CLIENT_VERSION="2.0.1"
# File storage, containing binary files, do not modify this unless you know
# what you are doing.
MENDER_STORAGE_URL="https://d1b0l86ne08fsf.cloudfront.net"
# Mender GitHub organization URL prefix
MENDER_GITHUB_ORG="https://github.com/mendersoftware"
# Device file corresponding to the root filesystem partitions, without index.
MENDER_STORAGE_DEVICE=/dev/mmcblk0p
# Partition index of boot partition
MENDER_BOOT_PART_INDEX="1"
# Partition index of root filesystem A
MENDER_ROOTFS_PART_A_INDEX="2"
# Partition index of root filesystem B
MENDER_ROOTFS_PART_B_INDEX="3"
# Partition index of persistent data partition
MENDER_DATA_PART_INDEX="4"
# Basename of DTB that should be loaded by the bootloader.
MENDER_KERNEL_DEVICETREE=kernel.dtb
# Generate bmap index files
#
# https://github.com/intel/bmap-tools
MENDER_USE_BMAP="n"
source configs/mender_grub_config
# Function to create Mender Artifact
#
# This function is defined here, to allow it to be overridden which can be
# achieved by providing this function in a custom configuration file which will
# take precedence of this one.
#
# You might want to override this to e.g provide state-scripts or providing
# a private key to sign your artifact.
mender_create_artifact() {
local -r device_type="${1}"
local -r artifact_name="${2}"
mender_artifact=deploy/${device_type}-${artifact_name}.mender
log_info "Writing Mender artifact to: ${mender_artifact}"
log_info "This can take up to 20 minutes depending on which compression method is used"
run_and_log_cmd "mender-artifact --compression ${MENDER_ARTIFACT_COMPRESSION} \
write rootfs-image \
--file work/rootfs.img \
--output-path ${mender_artifact} \
--artifact-name ${artifact_name} \
--device-type ${device_type}"
}

35
configs/mender_grub_config

@ -0,0 +1,35 @@
# Use U-boot -> GRUB -> EFI boot-loader integration
#
# Only disable this if you know what you are doing
MENDER_GRUB_EFI_INTEGRATION=y
# Specific Linux kernel boot arguments
#
# Typically you would read the content of /proc/cmdline on a "golden image"
# add the appropriate arguments here.
#
# This will override the defaults set by by grub-mender-grubenv, if not an
# empty string
MENDER_GRUB_KERNEL_BOOT_ARGS=""
# grub-mender-grubenv is the Mender integration for the GRUB bootloader
MENDER_GRUBENV_VERSION="1.3.0"
MENDER_GRUBENV_URL="${MENDER_GITHUB_ORG}/grub-mender-grubenv/archive/${MENDER_GRUBENV_VERSION}.tar.gz"
# Name of the storage device containing root filesystem partitions in GRUB
# format.
MENDER_GRUB_STORAGE_DEVICE=hd0
# Type of kernel (bzImage or zImage)
#
# mender-convert will try to determinate this value on its own, only set this
# if was not possible to auto-detect
MENDER_GRUB_KERNEL_IMAGETYPE=""
# Type of initrd image
#
# mender-convert will try to determinate this value on its own, only set this
# if was not possible to auto-detect
MENDER_GRUB_INITRD_IMAGETYPE=""
MENDER_GRUB_BINARY_STORAGE_URL="${MENDER_STORAGE_URL}/mender-convert/grub-efi"

16
configs/qemux86-64_config

@ -0,0 +1,16 @@
# This configuration can be used to run on a qemux86-64 machine.
#
# This has been tested on images generated with the following command:
#
# mkosi -d ubuntu -r bionic -t gpt_ext4 -b --checksum --password password -o image.raw
#
# Converted with the following command:
#
# MENDER_ARTIFACT_NAME=release-1 ./mender-convert --disk-image input/image.raw --overlay rootfs_overlay_demo --config configs/qemux86-64_config
#
# and qemu is executed with the following command:
#
# qemu-system-x86_64 -enable-kvm -m 512 -smp 2 -bios /usr/share/ovmf/x64/OVMF_CODE.fd -drive format=raw,file=qemux86_64-release-1.sdimg
MENDER_STORAGE_DEVICE=/dev/sda
MENDER_DEVICE_TYPE="qemux86_64"

8
configs/raspberrypi3_config

@ -0,0 +1,8 @@
# Binaries generated with the following script:
#
# https://d1b0l86ne08fsf.cloudfront.net/mender-convert/raspberrypi/raspberrypi-integration-scripts.tar.gz
RASPBERRYPI_BINARIES="raspberrypi3-integration-2018.07.tar.gz"
RASPBERRYPI_KERNEL_IMAGE="kernel7.img"
MENDER_KERNEL_IMAGETYPE=zImage
source configs/raspberrypi_config

82
configs/raspberrypi_config

@ -0,0 +1,82 @@
# Raspberry Pi does not support GRUB bootloader integration, fallback to U-boot.
MENDER_GRUB_EFI_INTEGRATION=n
# 4MB alignment
MENDER_PARTITION_ALIGNMENT="4194304"
RASPBERRYPI_BINARIES_URL="${MENDER_STORAGE_URL}/mender-convert/raspberrypi/${RASPBERRYPI_BINARIES}"
function platform_modify() {
mkdir -p work/rpi/binaries
run_and_log_cmd "wget -q ${RASPBERRYPI_BINARIES_URL} -P work/rpi/binaries"
run_and_log_cmd "tar xvf work/rpi/binaries/${RASPBERRYPI_BINARIES} -C work/rpi/binaries"
# Make a copy of Linux kernel arguments and modify.
run_and_log_cmd "cp work/boot/cmdline.txt work/rpi/cmdline.txt"
# Set a dynamic rootfs part (required for Mender A/B update strategy)
run_and_log_cmd "sed -i 's/\b[ ]root=[^ ]*/ root=\${mender_kernel_root}/' work/rpi/cmdline.txt"
# Root filesystem can not be resized when the disk is partition according
# to Mender layout, where the rootfs partition is the not last one which
# is a requirement to be able to do an "online" resize.
#
# This disables resize of rootfs on boot but applies the changes to
# cmdline.txt that are performed in the init_resize.sh script.
#
# Extracted from /usr/lib/raspi-config/init_resize.sh
run_and_log_cmd "sed -i 's| init=/usr/lib/raspi-config/init_resize\.sh||' work/rpi/cmdline.txt"
run_and_log_cmd "sed -i 's| sdhci\.debug_quirks2=4||' work/rpi/cmdline.txt"
if ! grep -q splash work/rpi/cmdline.txt; then
run_and_log_cmd "sed -i 's/ quiet//g' work/rpi/cmdline.txt"
fi
# Update Linux kernel command arguments with our custom configuration
run_and_log_cmd "sudo cp work/rpi/cmdline.txt work/boot/"
# Enable Full KMS (Kernel Mode Setting) graphics driver. This is the
# the open-source GPU driver that is part of mesa, also known as VC4.
#
# This is done because it seems that the firmware GPU driver does currently
# not play well when U-boot is enabled and HDMI output is "scrambled"
#
# Fixes: https://tracker.mender.io/browse/MEN-2685
#
# Will need to revisit this later when there are new firware releases.
#
# Should be noted that RPi4 only works with this enabled and the
# configuration in the Yocto BSP (meta-raspberrypi) is also nowdays using
# this driver by default.
run_and_log_cmd "echo 'dtoverlay=vc4-kms-v3d' | sudo tee -a work/boot/config.txt"
# Mask udisks2.service, otherwise it will mount the inactive part and we
# might write an update while it is mounted which often result in
# corruptions.
#
# TODO: Find a way to only blacklist mmcblk0pX devices instead of masking
# the service.
run_and_log_cmd "sudo ln -sf /dev/null work/rootfs/etc/systemd/system/udisks2.service"
# Extract Linux kernel and install to /boot directory on rootfs
run_and_log_cmd "sudo cp work/boot/${RASPBERRYPI_KERNEL_IMAGE} work/rootfs/boot/${MENDER_KERNEL_IMAGETYPE}"
# Replace kernel with U-boot and add boot script
run_and_log_cmd "sudo mkdir -p work/rootfs/uboot"
run_and_log_cmd "sudo cp work/rpi/binaries/u-boot.bin work/boot/${RASPBERRYPI_KERNEL_IMAGE}"
run_and_log_cmd "sudo cp work/rpi/binaries/boot.scr work/boot"
run_and_log_cmd "sudo cp work/rpi/binaries/fw_env.config work/rootfs/etc/"
# Raspberry Pi applications expect to find this on the device and in some
# cases parse the options to determinate functionality.
run_and_log_cmd "sudo ln -fs /uboot/config.txt work/rootfs/boot/config.txt"
run_and_log_cmd "sudo ln -fs /uboot/overlays work/rootfs/boot/overlays"
run_and_log_cmd "sudo ln -fs /uboot/cmdline.txt work/rootfs/boot/cmdline.txt"
run_and_log_cmd "sudo install -m 755 work/rpi/binaries/fw_printenv work/rootfs/sbin/fw_printenv"
run_and_log_cmd "sudo ln -fs /sbin/fw_printenv work/rootfs/sbin/fw_setenv"
# Remove original 'resize2fs_once' script and its symbolic link.
run_and_log_cmd "sudo unlink work/rootfs/etc/rc3.d/S01resize2fs_once"
run_and_log_cmd "sudo rm work/rootfs/etc/init.d/resize2fs_once"
}

25
configs/rockpro64_config

@ -0,0 +1,25 @@
# ROCKPro64 do not support GRUB bootloader integration, fallback to U-boot.
MENDER_GRUB_EFI_INTEGRATION=n
ROCKPRO64_BINARIES_URL="${MENDER_STORAGE_URL}/mender-convert/armbian/rockpro64/${ROCKPRO64_BINARIES}"
function platform_modify() {
mkdir -p work/rockpro64
run_and_log_cmd "wget -Nq ${ROCKPRO64_BINARIES_URL} -P work/rockpro64"
run_and_log_cmd "tar xvf work/rockpro64/${ROCKPRO64_BINARIES} -C work/rockpro64"
run_and_log_cmd "sudo cp work/rockpro64/boot.scr work/boot"
run_and_log_cmd "sudo cp work/rockpro64/fw_env.config work/rootfs/etc/"
# It is not possible to resize rootfs part when using Mender. so disable
# the service
run_and_log_cmd "sudo touch work/rootfs/root/.no_rootfs_resize"
}
function platform_package() {
log_info "Embedding bootloader in disk image"
run_and_log_cmd "dd if=work/rockpro64/rksd_loader.img of=${sdimg_path} \
seek=64 conv=notrunc status=none"
}

8
configs/rockpro64_emmc_config

@ -0,0 +1,8 @@
MENDER_STORAGE_DEVICE=/dev/mmcblk1p
# Binaries generated with the following script:
#
# https://d1b0l86ne08fsf.cloudfront.net/mender-convert/armbian/rockpro64/integration-scripts.tar.gz
ROCKPRO64_BINARIES="emmc-boot-integration-2017.09.tar.gz"
source configs/rockpro64_config

6
configs/rockpro64_sd_config

@ -0,0 +1,6 @@
# Binaries generated with the following script:
#
# https://d1b0l86ne08fsf.cloudfront.net/mender-convert/armbian/rockpro64/integration-scripts.tar.gz
ROCKPRO64_BINARIES="sd-boot-integration-2017.09.tar.gz"
source configs/rockpro64_config

21
docker-build

@ -0,0 +1,21 @@
#!/bin/sh
#
# Copyright 2019 Northern.tech AS
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -e
IMAGE_NAME=mender-convert
eval docker build . -t ${IMAGE_NAME}

25
docker-entrypoint.sh

@ -0,0 +1,25 @@
#!/bin/sh
#
# Copyright 2019 Northern.tech AS
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -e
# run conversion, args provided to container (end of docker run ...)
cd /mender-convert
echo "Running mender-convert "$@""
./mender-convert "$@"

31
docker-mender-convert

@ -0,0 +1,31 @@
#!/bin/sh
#
# Copyright 2019 Northern.tech AS
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -e
IMAGE_NAME=mender-convert
MENDER_CONVERT_DIR="$(pwd)"
mkdir -p output
docker run \
--mount type=bind,source="$MENDER_CONVERT_DIR,target=/mender-convert" \
--privileged=true \
--cap-add=SYS_MODULE \
-v /dev:/dev \
-v /lib/modules:/lib/modules:ro \
--env MENDER_ARTIFACT_NAME=${MENDER_ARTIFACT_NAME} \
$IMAGE_NAME "$@"

99
mender-convert

@ -0,0 +1,99 @@
#!/usr/bin/env bash
#
# Copyright 2019 Northern.tech AS
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
MENDER_CONVERT_VERSION=$(git describe --tags --dirty --exact-match 2>/dev/null || git rev-parse --short HEAD)
function show_help() {
cat << EOF
mender-convert
A tool that takes an existing embedded image (Debian, Ubuntu, Raspbian, etc)
and converts it to a Mender image by restructuring partition table and adding
necessary files.
Usage: $0 COMMAND [options]
Options:
-d, --disk-image - Path to disk image to convert (Debian, Raspbian,
Ubuntu, etc)
-o, --overlay - Path to root filesystem overlay directory, you
are able to provide this option multiple times
-c, --config - Path to configuration file, you are able to
provide this option multiple times
-v, --version - output version information and exit
-h, --help - display this help and exit
EOF
}
function show_version() {
echo "Version: ${MENDER_CONVERT_VERSION}"
}
function trap_exit() {
sudo rm -rf work
}
function trap_term() {
echo "Program interrupted by user"
}
trap trap_term INT TERM
trap trap_exit EXIT
mender_convert_dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
if [ "${mender_convert_dir}" != "${PWD}" ]; then
echo "You must execute mender-convert from the root directory: ${mender_convert_dir}"
exit 1
fi
if [ -z "${MENDER_ARTIFACT_NAME}" ]; then
echo "Sorry, it seems that you have not defined MENDER_ARTIFACT_NAME"
echo "You can do this with the following command:"
echo ""
echo -e "\tMENDER_ARTIFACT_NAME="release-1" ./mender-convert"
exit 1
fi
source modules/bootstrap.sh
# We only handle a selection of the arguments here, the rest are passed on
# to the sub-scripts.
while (( "$#" )); do
case "$1" in
-h | --help)
show_help
exit 0
;;
-v | --version)
show_version
exit 0
;;
*)
break;
;;
esac
done
mkdir -p work
touch work/convert.log
./mender-convert-extract "$@"
./mender-convert-modify "$@"
./mender-convert-package "$@"
echo "Output Artifacts and images can be found in deploy directory:"
ls -1 deploy/*

99
mender-convert-extract

@ -0,0 +1,99 @@
#!/bin/bash
#
# Copyright 2019 Northern.tech AS
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
echo "Running $(basename $0): $@"
source modules/bootstrap.sh
source modules/disk.sh
disk_image=""
while (( "$#" )); do
case "$1" in
-o | --overlay)
overlays+="${2}"
shift 2
;;
-c | --config)
configs+="${2}"
shift 2
;;
-d | --disk-image)
disk_image="${2}"
shift 2
;;
*)
log_fatal "Sorry but the provided option is not supported: $1"
;;
esac
done
if [ -z "${disk_image}" ]; then
log_warn "Sorry, but '--disk-image' is a mandatory option"
log_warn "See ./mender-convert --help for more information"
exit 1
fi
if [ ! -e ${disk_image} ]; then
log_fatal "File not found: ${disk_image}"
fi
source modules/config.sh "${configs[@]}"
MKFS_VFAT="/usr/bin/mkfs.vfat"
if [ ! -f ${MKFS_VFAT} ]; then
MKFS_VFAT="/sbin/mkfs.vfat"
fi
declare -i nr_of_parts=$(disk_get_nr_of_parts ${disk_image})
log_info "Validating disk image"
if [ ${nr_of_parts} -eq 0 ]; then
log_fatal "Sorry, but could not find any valid partitions for: ${disk_image}"
fi
log_info "Disk parsed succesfully"
log_info "NUMBER OF PARTS: ${nr_of_parts} TYPE: $(disk_get_part_value ${disk_image} 1 SCHEME)"
for ((n=1;n<=${nr_of_parts};n++)); do
part_dst_file="work/part-${n}.fs"
if [ "$(disk_get_part_value ${disk_image} ${n} TYPE)" == "0x8e" ]; then
log_fatal "Detected an LVM volume group on disk. Unfortunatly this is not yet supported"
fi
log_info "PART ${n}: SIZE: $(disk_get_part_value ${disk_image} ${n} SIZE) TYPE: $(disk_get_part_value ${disk_image} ${n} TYPE)"
log_info "PART ${n}: extracting to ${part_dst_file}"
disk_extract_part "${disk_image}" $(disk_get_part_value ${disk_image} ${n} START) \
$(disk_get_part_value ${disk_image} ${n} SECTORS) ${part_dst_file}
done
# Some disk images will not have an boot partition for us to extract,
# but we do require it be present and hence we might need to generate
# a filesystem image here.
if [ ${nr_of_parts} -eq 1 ]; then
log_info "Generating boot partition (required, does not exist in original image)"
run_and_log_cmd "dd if=/dev/zero of=work/boot-generated.vfat count=${MENDER_BOOT_PART_SIZE_MB} bs=1M status=none"
run_and_log_cmd "${MKFS_VFAT} work/boot-generated.vfat"
fi
# Extract boot gap, that is the area from sector 1 until first part, and
# this is done because some images might contain the bootloader embedded here
# and we might need to keep this area intact when we generate our custom
# image.
disk_extract_part "${disk_image}" \
1 $(( $(disk_get_part_value ${disk_image} 1 START) - 1)) work/boot-gap.bin

215
mender-convert-modify

@ -0,0 +1,215 @@
#!/bin/bash
#
# Copyright 2019 Northern.tech AS
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Default that can be overridden by providing this method in a
# configuration file passed with '--config'
function platform_modify() {
true
}
function trap_exit() {
echo "mender-convert-modify has finished. Cleaning..."
sudo umount -f work/boot
sudo umount -f work/rootfs
}
function trap_term() {
true
}
trap trap_term INT TERM
trap trap_exit EXIT
echo "Running $(basename $0): $@"
source modules/bootstrap.sh
source modules/disk.sh
source modules/probe.sh
while (( "$#" )); do
case "$1" in
-o | --overlay)
overlays+="${2}"
shift 2
;;
-c | --config)
configs+="${2}"
shift 2
;;
-d | --disk-image)
disk_image="${2}"
shift 2
;;
*)
log_fatal "Sorry, but the provided option is not supported: $1"
;;
esac
done
source modules/config.sh "${configs[@]}"
boot_part=$(disk_boot_part)
root_part=$(disk_root_part)
# Create mount points
mkdir -p work/boot
mkdir -p work/rootfs
sudo mount ${boot_part} work/boot
sudo mount ${root_part} work/rootfs
mkdir -p work/mender-deb/files
log_info "Installing Mender client and related files"
deb_arch=$(probe_debian_arch_name)
deb_name="mender-client_${MENDER_CLIENT_VERSION}-1_${deb_arch}.deb"
run_and_log_cmd "wget -Nq ${MENDER_STORAGE_URL}/mender-convert/deb/${deb_name} -P work/mender-deb"
cd work/mender-deb
run_and_log_cmd "ar -xv ${deb_name}"
run_and_log_cmd "tar -xf data.tar.xz -C files"
cd - > /dev/null 2>&1
run_and_log_cmd "sudo rsync --archive --keep-dirlinks --verbose work/mender-deb/files/ work/rootfs/"
if [ "${MENDER_ENABLE_SYSTEMD}" == "y" ]; then
run_and_log_cmd "sudo ln -sf /lib/systemd/system/mender.service \
work/rootfs/etc/systemd/system/multi-user.target.wants/mender.service"
fi
if [ "${MENDER_GRUB_EFI_INTEGRATION}" == "y" ]; then
run_and_log_cmd "wget -Nq '${MENDER_GRUBENV_URL}' -P work/"
run_and_log_cmd "tar xvf work/${MENDER_GRUBENV_VERSION}.tar.gz -C work/"
if [ -z "${MENDER_GRUB_KERNEL_IMAGETYPE}" ]; then
kernel_imagetype=$(probe_kernel_in_boot_and_root)
else
kernel_imagetype="${MENDER_GRUB_KERNEL_IMAGETYPE}"
fi
if [ -z "${MENDER_GRUB_INITRD_IMAGETYPE}" ]; then
initrd_imagetype=$(probe_initrd_in_boot_and_root)
else
kernel_imagetype="${MENDER_GRUB_INITRD_IMAGETYPE}"
fi
cat <<- EOF > work/grub-mender-grubenv-${MENDER_GRUBENV_VERSION}/mender_grubenv_defines
mender_rootfsa_part=${MENDER_ROOTFS_PART_A_INDEX}
mender_rootfsb_part=${MENDER_ROOTFS_PART_B_INDEX}
mender_kernel_root_base=${MENDER_STORAGE_DEVICE}
mender_grub_storage_device=${MENDER_GRUB_STORAGE_DEVICE}
kernel_imagetype=${kernel_imagetype}
initrd_imagetype=${initrd_imagetype}
kernel_devicetree=${MENDER_KERNEL_DEVICETREE}
EOF
if [ -n "${MENDER_GRUB_KERNEL_BOOT_ARGS}" ]; then
cat <<- EOF > work/grub-mender-grubenv-${MENDER_GRUBENV_VERSION}/11_bootargs_grub.cfg
set bootargs="${MENDER_GRUB_KERNEL_BOOT_ARGS}"
EOF
fi
cd work/grub-mender-grubenv-${MENDER_GRUBENV_VERSION}
run_and_log_cmd "make 2>&1"
run_and_log_cmd "sudo make DESTDIR=../ BOOT_DIR=boot install-boot-files"
run_and_log_cmd "sudo make DESTDIR=../rootfs install-tools"
cd - > /dev/null 2>&1
# Remove conflicting boot files. These files do not necessarily effect the
# functionality, but lets get rid of them to avoid confusion.
#
# There is no Mender integration for EFI boot or systemd-boot.
sudo rm -rf work/boot/loader
sudo rm -rf work/boot/EFI/Linux
sudo rm -rf work/boot/EFI/systemd
sudo rm -rf work/boot/NvVars
log_info "Installing GRUB"
arch=$(probe_arch)
efi_name=$(probe_grub_efi_name)
efi_target_name=$(probe_grub_efi_target_name)
log_info "GRUB EFI: ${efi_target_name}"
run_and_log_cmd "wget -Nq ${MENDER_GRUB_BINARY_STORAGE_URL}/${arch}/${efi_name} -P work/"
run_and_log_cmd "wget -Nq ${MENDER_GRUB_BINARY_STORAGE_URL}/${arch}/grub-editenv -P work/"
run_and_log_cmd "sudo install -m 751 work/grub-editenv work/rootfs/usr/bin/"
run_and_log_cmd "sudo mkdir -p work/boot/EFI/BOOT"
run_and_log_cmd "sudo cp work/${efi_name} -P work/boot/EFI/BOOT/${efi_target_name}"
fi
run_and_log_cmd "sudo mkdir -p work/rootfs/data/mender"
run_and_log_cmd "sudo ln -sf /data/mender work/rootfs/var/lib/mender"
cat <<- EOF > work/mender.conf.data
{
"RootfsPartA": "${MENDER_STORAGE_DEVICE}${MENDER_ROOTFS_PART_A_INDEX}",
"RootfsPartB": "${MENDER_STORAGE_DEVICE}${MENDER_ROOTFS_PART_B_INDEX}"
}
EOF
run_and_log_cmd "sudo cp work/mender.conf.data work/rootfs/data/mender/mender.conf"
if [ -z "${MENDER_DEVICE_TYPE}" ]; then
# Observed systems who do not have this file, e.g images generated with mkosi
if [ -f work/rootfs/etc/hostname ]; then
device_type=$(cat work/rootfs/etc/hostname)
else
device_type="default"
fi
else
device_type="${MENDER_DEVICE_TYPE}"
fi
run_and_log_cmd "echo 'device_type=${device_type}' > work/device_type"
run_and_log_cmd "sudo install -m 0444 work/device_type work/rootfs/data/mender/"
run_and_log_cmd "sudo echo 'artifact_name=${MENDER_ARTIFACT_NAME}' \
> work/rootfs/etc/mender/artifact_info"
log_info "Installing a custom /etc/fstab (see work/convert.log for more info)"
if [ "${MENDER_GRUB_EFI_INTEGRATION}" == "y" ]; then
boot_part_mountpoint="/boot/efi"
else
boot_part_mountpoint="/uboot"
fi
run_and_log_cmd "sudo mkdir -p work/rootfs/${boot_part_mountpoint}"
if [ "${MENDER_DATA_PART_GROWFS}" == "y" ]; then
MENDER_DATA_PART_FSTAB_OPTS="${MENDER_DATA_PART_FSTAB_OPTS},x-systemd.growfs"
fi
sudo bash -c "cat <<- EOF > work/rootfs/etc/fstab
# stock fstab - you probably want to override this with a machine specific one
/dev/root / auto defaults 1 1
proc /proc proc defaults 0 0
${MENDER_STORAGE_DEVICE}${MENDER_BOOT_PART_INDEX} ${boot_part_mountpoint} auto defaults,sync 0 0
${MENDER_STORAGE_DEVICE}${MENDER_DATA_PART_INDEX} /data auto ${MENDER_DATA_PART_FSTAB_OPTS} 0 0
EOF"
log_info "Performing platform specific modifications (if any)"
platform_modify
for overlay in "${overlays[@]}"; do
log_info "Applying rootfs overlay: ${overlay}"
run_and_log_cmd "sudo rsync --archive --keep-dirlinks --verbose ${overlay}/ work/rootfs/"
done

259
mender-convert-package

@ -0,0 +1,259 @@
#!/bin/bash
#
# Copyright 2019 Northern.tech AS
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Default that can be overridden by providing this method in a
# configuration file passed with '--config'
function platform_package() {
true
}
function trap_exit() {
echo "mender-convert-package has finished. Cleaning..."
sudo umount -f work/boot > /dev/null
sudo umount -f work/rootfs > /dev/null
}
function trap_term() {
true
}
trap trap_term INT TERM
trap trap_exit EXIT
echo "Running $(basename $0): $@"
source modules/bootstrap.sh
source modules/disk.sh
while (( "$#" )); do
case "$1" in
-o | --overlay)
overlays+="${2}"
shift 2
;;
-c | --config)
configs+="${2}"
shift 2
;;
-d | --disk-image)
disk_image="${2}"
shift 2
;;
*)
log_fatal "Sorry but the provided option is not supported: $1"
;;
esac
done
source modules/config.sh "${configs[@]}"
PARTED="/usr/bin/parted"
if [ ! -f ${PARTED} ]; then
PARTED="/sbin/parted"
fi
output_dir=work
boot_part=$(disk_boot_part)
root_part=$(disk_root_part)
# Final output
mkdir -p deploy
# Create mount points
mkdir -p work/boot
mkdir -p work/rootfs
sudo mount ${boot_part} work/boot
sudo mount ${root_part} work/rootfs
# Convert to 512 blocks
disk_image_total_sectors=$(disk_mb_to_sectors ${MENDER_STORAGE_TOTAL_SIZE_MB})
boot_part_sectors=$(disk_mb_to_sectors ${MENDER_BOOT_PART_SIZE_MB})
data_part_sectors=$(disk_mb_to_sectors ${MENDER_DATA_PART_SIZE_MB})
alignment_sectors=$((${MENDER_PARTITION_ALIGNMENT} / 512))
if [ "${MENDER_GRUB_EFI_INTEGRATION}" == "y" ]; then
boot_part_start=${alignment_sectors}
overhead_sectors=$(( ${alignment_sectors} * 4 ))
else
# The two first blocks are typically reserved for U-boot environment.
boot_part_start=$(( ${alignment_sectors} * 3))
overhead_sectors=$(( ${alignment_sectors} * 6 ))
fi
# Validate boot part size
actual_boot_size=$(du --apparent-size -s --block-size=512 ${boot_part} | cut -f 1)
if [ ${actual_boot_size} -gt ${boot_part_sectors} ]; then
log_warn "The allocated boot part size $(disk_sectors_to_mb ${boot_part_sectors}) MiB is too small."
log_warn "The actual boot part size is $(disk_sectors_to_mb ${actual_boot_size}) MiB"
log_warn "Will adjust MENDER_BOOT_PART_SIZE_MB automatically"
log_warn "Considered adjusting the configuration file to avoid this message"
boot_part_sectors=${actual_boot_size}
fi
# Make sure that the boot part sector size is aligned to
# MENDER_PARTITION_ALIGNMENT, it is possible that the boot part is extracted
# as-is from the input image, and the input image might not have used the same
# alignment as our configuration
boot_part_sectors=$(disk_align_sectors ${boot_part_sectors} ${MENDER_PARTITION_ALIGNMENT} )
# Calculate rootfs size
rootfs_part_sectors=$(( (${disk_image_total_sectors} - ${data_part_sectors} \
- ${boot_part_sectors} - ${overhead_sectors}) / 2 ))
# Make sure rootfs size is aligned to MENDER_PARTITION_ALIGNMENT
rootfs_part_sectors=$(disk_align_sectors ${rootfs_part_sectors} ${MENDER_PARTITION_ALIGNMENT} )
device_type=$(cat work/rootfs/data/mender/device_type | sed 's/[^=]*=//')
artifact_name=$(cat work/rootfs/etc/mender/artifact_info | sed 's/[^=]*=//')
image_name="${device_type}-${artifact_name}"
actual_rootfs_size=$(sudo du --apparent-size -s --block-size=512 work/rootfs | cut -f 1)
# 50 % free space, not to be confused with rootfs_part_sectors
rootfs_image_sectors=$(awk -v r1="$actual_rootfs_size" 'BEGIN{printf "%.0f", r1 * 1.50}')
if [ ${rootfs_image_sectors} -gt ${rootfs_part_sectors} ]; then
log_warn "The calculated rootfs partition size $(disk_sectors_to_mb ${rootfs_part_sectors}) MiB is too small."
log_warn "The actual rootfs image size is $(disk_sectors_to_mb ${rootfs_image_sectors}) MiB"
log_fatal "You can try adjusting the MENDER_STORAGE_TOTAL_SIZE_MB variable to increase available space"
fi
# Extract file-system type from rootfs
if file ${root_part} | grep -q ext4; then
image_fs_type="ext4"
elif file ${root_part} | grep -q XFS; then
image_fs_type="xfs"
else
log_warn "$(file work/${root_part})"
log_fatal "Could not determinate root file-system type. Aborting..."
fi
disk_create_file_system_from_folder "work/rootfs/data/" "work/data.img" \
"${data_part_sectors}" "${image_fs_type}"
# Clear this area as it will be contained in the data.img
sudo rm -rf work/rootfs/data/*
disk_create_file_system_from_folder "work/rootfs/" "work/rootfs.img" \
"${rootfs_image_sectors}" "${image_fs_type}"
log_info "Copying root filesystem image to deploy directory"
run_and_log_cmd "cp --sparse=always work/rootfs.img deploy/${image_name}.${image_fs_type}"
mender_create_artifact "${device_type}" "${artifact_name}"
log_info "Creating Mender compatible disk-image"
sdimg_path=deploy/${image_name}.sdimg
log_info "Total disk size: $(disk_sectors_to_mb ${disk_image_total_sectors}) MiB"
log_info " Boot partition $(disk_sectors_to_mb ${boot_part_sectors}) MiB"
log_info " RootFS $(disk_sectors_to_mb ${rootfs_part_sectors}) x 2 MiB"
log_info " Data $(disk_sectors_to_mb ${data_part_sectors}) MiB"
# Initialize sdcard image file
run_and_log_cmd \
"dd if=/dev/zero of=${sdimg_path} bs=512 count=0 seek=${disk_image_total_sectors} status=none"
# boot_part_start, is defined at the beginning of this file
boot_part_end=$(( ${boot_part_start} + ${boot_part_sectors} - 1 ))
rootfsa_start=$(disk_align_sectors ${boot_part_end} ${MENDER_PARTITION_ALIGNMENT} )
rootfsa_end=$(( ${rootfsa_start} + ${rootfs_part_sectors} - 1 ))
rootfsb_start=$(disk_align_sectors ${rootfsa_end} ${MENDER_PARTITION_ALIGNMENT} )
rootfsb_end=$(( ${rootfsb_start} + ${rootfs_part_sectors} - 1 ))
data_start=$(disk_align_sectors ${rootfsb_end} ${MENDER_PARTITION_ALIGNMENT} )
data_end=$(( ${data_start} + ${data_part_sectors} - 1 ))
# Create partition table. TODO: GPT support
run_and_log_cmd "${PARTED} -s ${sdimg_path} mklabel msdos"
run_and_log_cmd "${PARTED} -s ${sdimg_path} unit s mkpart primary fat32 ${boot_part_start} ${boot_part_end}"
run_and_log_cmd "${PARTED} -s ${sdimg_path} set 1 boot on"
run_and_log_cmd "${PARTED} -s ${sdimg_path} -- unit s mkpart primary ext2 ${rootfsa_start} ${rootfsa_end}"
run_and_log_cmd "${PARTED} -s ${sdimg_path} -- unit s mkpart primary ext2 ${rootfsb_start} ${rootfsb_end}"
run_and_log_cmd "${PARTED} -s ${sdimg_path} -- unit s mkpart primary ext2 ${data_start} ${data_end}"
run_and_log_cmd "${PARTED} -s ${sdimg_path} print"
# Burn Partitions
disk_write_at_offset "${boot_part}" "${sdimg_path}" "${boot_part_start}"
disk_write_at_offset "${output_dir}/rootfs.img" "${sdimg_path}" "${rootfsa_start}"
disk_write_at_offset "${output_dir}/rootfs.img" "${sdimg_path}" "${rootfsb_start}"
disk_write_at_offset "${output_dir}/data.img" "${sdimg_path}" "${data_start}"
log_info "Performing platform specific package operations (if any)"
platform_package
# Create bmap index
if [ "${MENDER_USE_BMAP}" == "y" ]; then
BMAP_TOOL="/usr/bin/bmaptool"
if [ ! -e "${BMAP_TOOL}" ]; then
log_error "You have enabled the MENDER_USE_BMAP option, but we could not find the required 'bmaptool'"
log_fatal "You can install 'bmaptool' with: apt-get install bmap-tools (on Debian based distributions)"
fi
run_and_log_cmd "${BMAP_TOOL} create ${sdimg_path} > ${sdimg_path}.bmap"
fi
if [ "${MENDER_COMPRESS_DISK_IMAGE}" == "y" ]; then
log_info "Compressing ${sdimg_path}.gz"
run_and_log_cmd "pigz --best --force ${sdimg_path}"
fi
log_info "Conversion has completed! \o/"
##################### Create configuration file for tests ######################
if [ "${MENDER_GRUB_EFI_INTEGRATION}" == "y" ]; then
boot_part_mountpoint="/boot/efi"
# This is the name of the DISTRO_FEATURES in Yocto
distro_feature="mender-grub"
else
boot_part_mountpoint="/uboot"
# This is the name of the DISTRO_FEATURES in Yocto
distro_feature="mender-uboot"
fi
cat <<- EOF > deploy/${image_name}.cfg
MENDER_BOOT_PART="${MENDER_STORAGE_DEVICE}${MENDER_BOOT_PART_INDEX}"
MENDER_ROOTFS_PART_A="${MENDER_STORAGE_DEVICE}${MENDER_ROOTFS_PART_A_INDEX}"
MENDER_ROOTFS_PART_B="${MENDER_STORAGE_DEVICE}${MENDER_ROOTFS_PART_B_INDEX}"
MENDER_BOOT_PART_MOUNT_LOCATION="${boot_part_mountpoint}"
MENDER_BOOT_PART_SIZE_MB="$(disk_sectors_to_mb ${boot_part_sectors})"
MENDER_DATA_PART_SIZE_MB="${MENDER_DATA_PART_SIZE_MB}"
MENDER_DEVICE_TYPE="${device_type}"
MENDER_PARTITIONING_OVERHEAD_KB="$(( (${overhead_sectors} * 512) / 1024 ))"
MENDER_PARTITION_ALIGNMENT="${MENDER_PARTITION_ALIGNMENT}"
MENDER_STORAGE_DEVICE="${MENDER_STORAGE_DEVICE}"
MENDER_STORAGE_TOTAL_SIZE_MB="${MENDER_STORAGE_TOTAL_SIZE_MB}"
MENDER_UBOOT_ENV_STORAGE_DEVICE_OFFSET="12582912"
MENDER_ARTIFACT_NAME="${artifact_name}"
DISTRO_FEATURES="${distro_feature}"
DEPLOY_DIR_IMAGE="${PWD}/deploy"
MENDER_MACHINE="${device_type}"
EOF
# Something that the tests expect to be defined (originally from Yocto)
cat <<- EOF >> deploy/${image_name}.cfg
IMAGE_FSTYPES="${image_fs_type} mender sdimg"
ARTIFACTIMG_FSTYPE="${image_fs_type}"
LAYER_CONF_VERSION="2"
EOF

52
modules/bootstrap.sh

@ -0,0 +1,52 @@
#!/usr/bin/env bash
#
# Copyright 2019 Northern.tech AS
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Credit to:
# https://github.com/gruntwork-io/bash-commons/blob/master/modules/bash-commons/src/bootstrap.sh
# Sets some Bash options to encourage well formed code.
# For example, some of the options here will cause the script to terminate as
# soon as a command fails. Another option will cause an error if an undefined
# variable is used.
# See: https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html
# Any trap on ERR is inherited by shell functions, command substitutions, and
# commands executed in a subshell environment. The ERR trap is normally not
# inherited in such cases.
set -o errtrace
# Any trap on DEBUG and RETURN are inherited by shell functions, command
# substitutions, and commands executed in a subshell environment. The DEBUG and
# RETURN traps are normally not inherited in such cases.
set -o functrace
# Exit if any command exits with a non-zero exit status.
set -o errexit
# Exit if script uses undefined variables.
set -o nounset
# Prevent masking an error in a pipeline.
# Look at the end of the 'Use set -e' section for an excellent explanation.
# see: https://www.davidpashley.com/articles/writing-robust-shell-scripts/
set -o pipefail
# Make debugging easier when you use `set -x`
# See: http://wiki.bash-hackers.org/scripting/debuggingtips#making_xtrace_more_useful
export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }'
source modules/log.sh
source modules/run.sh

20
modules/config.sh

@ -0,0 +1,20 @@
#!/usr/bin/env bash
#
# Copyright 2019 Northern.tech AS
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
for config in configs/mender_convert_config "$@"; do
log_info "Using configuration file: ${config}"
source ${config}
done

176
modules/disk.sh

@ -0,0 +1,176 @@
#!/usr/bin/env bash
#
# Copyright 2019 Northern.tech AS
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Print desired information about a partition
#
# Example usage:
#
# part_one_start=$(disk_get_part_value ${disk_image_path} 1 START)
#
# $1 - path to disk image
# $2 - part number
# $3 - information to be printed
#
# We use 'partx' to parse input disk image and it supports the following fields:
#
# NR partition number
# START start of the partition in sectors
# END end of the partition in sectors
# SECTORS number of sectors
# SIZE human readable size
# NAME partition name
# UUID partition UUID
# TYPE partition type (a string, a UUID, or hex)
# FLAGS partition flags
# SCHEME partition table type (dos, gpt, ...)
disk_get_part_value() {
echo "$(partx -o ${3} -g -r --nr ${2} ${1})"
}
# Prints number of partitions found in disk image
#
# Example usage:
#
# nr_of_parts=$(disk_get_nr_of_parts ${disk_image_path})
#
# $1 - path to disk image
disk_get_nr_of_parts() {
echo "$(partx -l ${1} | wc -l)"
}
# Extract a file system image from a disk image
#
# $1 - path to disk image
# $2 - sector start (in 512 blocks)
# $3 - size (in 512 blocks)
# $4 - path to output file
disk_extract_part() {
run_and_log_cmd "dd if=$1 of=$4 skip=$2 bs=512 count=$3 conv=sparse status=none"
}
# Convert MiB to number of 512 sectors
#
# $1 - MiB value
disk_mb_to_sectors() {
echo "$(( (${1} * 1024 * 1024) / 512 ))"
}
# Convert 512 sectors to MiB
#
# $1 - number of 512 sectors
disk_sectors_to_mb() {
echo "$(( (${1} * 512) / 1024 / 1024 ))"
}
# Align value (result is number of 512 sectors)
#
# $1 - value to align (number of 512 sectors)
# $2 - alignment in bytes
disk_align_sectors() {
local size_in_bytes=$(( ${1} * 512))
local reminder=$(( ${size_in_bytes} % ${2} ))
if [ $reminder -ne 0 ]; then
size_in_bytes=$(( $size_in_bytes - $reminder + ${2} ))
fi
echo "$(( $size_in_bytes / 512 ))"
}
# Write file at offset of another file
#
# $1 - file to write
# $2 - destination file
# $3 - offset in number of 512 sectors
disk_write_at_offset() {
run_and_log_cmd "dd if=${1} of=${2} seek=${3} conv=sparse status=none"
}
# Create file system image from directory content
#
# $1 - base folder
# $2 - destination file
# $3 - image size in sectors
# $4 - file system type, e.g ext4, xfs, ...
disk_create_file_system_from_folder() {
log_info "Creating a file-system image from: ${1}"
run_and_log_cmd "dd if=/dev/zero of=${2} seek=${3} count=0 bs=512 status=none"
case "$4" in
"ext4")
MKFS_EXT4="/usr/bin/mkfs.ext4"
if [ ! -f ${MKFS_EXT4} ]; then
MKFS_EXT4="/sbin/mkfs.ext4"
fi
run_and_log_cmd "${MKFS_EXT4} -q -F ${2}"
;;
"xfs")
MKFS_XFS="/usr/bin/mkfs.xfs"
if [ ! -f ${MKFS_XFS} ]; then
MKFS_XFS="/sbin/mkfs.xfs"
fi
run_and_log_cmd "${MKFS_XFS} -q -f ${2}"
;;
*)
log_fatal "Unknown file system type specified: ${4}"
esac
run_and_log_cmd "mkdir -p work/output"
run_and_log_cmd "sudo mount ${2} work/output"
run_and_log_cmd "sudo rsync --archive --delete ${1} work/output/"
run_and_log_cmd "sudo umount work/output"
}
# Print path to the boot partition filesystem image
#
disk_boot_part() {
# Why this little dance you might wonder.
#
# Some disk images do not have a boot partition, but our integration does
# require one and this is why there is a difference here.
#
# If input image did not have a boot part it will be generated and be called:
#
# work/boot-generated.vfat
#
# This is mostly done to make it obvious that we have created an empty vfat file.
#
# But if the disk image already had a boot part, we assume that it is the first
# partition that was extracted.
#
# We also need to adjust the part number of the rootfs part depending on if
# boot part was extracted or generated.
boot_part="work/boot-generated.vfat"
if [ ! -f ${boot_part} ]; then
boot_part="work/part-1.fs"
fi
echo "${boot_part}"
}
# Print path to the root partition filesystem image
#
disk_root_part() {
boot_part="work/boot-generated.vfat"
if [ ! -f ${boot_part} ]; then
root_part="work/part-2.fs"
else
root_part="work/part-1.fs"
fi
echo "${root_part}"
}

67
modules/log.sh

@ -0,0 +1,67 @@
#!/usr/bin/env bash
#
# Copyright 2019 Northern.tech AS
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This must be an absolute path, as users might call the log functions
# from sub-directories
log_file="${PWD}/work/convert.log"
# Log the given message at the given level.
function log {
local -r level="$1"
local -r message="$2"
local -r timestamp=$(date +"%Y-%m-%d %H:%M:%S")
local -r script_name="$(basename "$0")"
>&2 echo -e "${timestamp} [${level}] [$script_name] ${message}" | tee -a ${log_file}
}
function local_log_debug {
local -r level="DEBUG"
local -r message="$1"
local -r timestamp=$(date +"%Y-%m-%d %H:%M:%S")
local -r script_name="$(basename "$0")"
echo -e "${timestamp} [${level}] [$script_name] ${message}" >> ${log_file}
}
# Log the given message at DEBUG level.
function log_debug {
local -r message="$1"
local_log_debug "$message"
}
# Log the given message at INFO level.
function log_info {
local -r message="$1"
log "INFO" "$message"
}
# Log the given message at WARN level.
function log_warn {
local -r message="$1"
log "WARN" "$message"
}
# Log the given message at ERROR level.
function log_error {
local -r message="$1"
log "ERROR" "$message"
}
# Log the given message at FATAL level.
function log_fatal {
local -r message="$1"
log "FATAL" "$message"
exit 1
}

237
modules/probe.sh

@ -0,0 +1,237 @@
#!/usr/bin/env bash
#
# Copyright 2019 Northern.tech AS
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Prints target architecture
#
# No input parameters and these work on the assumption that boot and root parts
# are mounted at work/boot and work/rootfs
probe_arch() {
# --dereference, means to follow symlinks because 'ls' could be a symlink
# to busybox
file_info=""
for location in bin/ls usr/bin/ls; do
if [ -e work/rootfs/${location} ]; then
file_info=$(file -b --dereference work/rootfs/${location})
break
fi
done
if [ -z "${file_info}" ]; then
log_fatal "Sorry, where not able to determinate target architecture"
fi
target_arch="unknown"
if grep -q x86-64 <<< "${file_info}"; then
target_arch="x86-64"
elif grep -Eq "ELF 32-bit.*ARM" <<< "${file_info}"; then
target_arch="arm"
elif grep -Eq "ELF 64-bit.*aarch64" <<< "${file_info}"; then
target_arch="aarch64"
else
log_fatal "Unsupported architecture: ${file_info}"
fi
echo "${target_arch}"
}
# Prints GRUB EFI name depending on target architecture
#
# No input parameters and these work on the assumption that boot and root parts
# are mounted at work/boot and work/rootfs
probe_grub_efi_name() {
efi_name=""
arch=$(probe_arch)
case "${arch}" in
"x86-64")
efi_name="grub-efi-bootx64.efi"
;;
"arm")
efi_name="grub-efi-bootarm.efi"
;;
"aarch64")
efi_name="grub-efi-bootaa64.efi"
;;
*)
log_fatal "Unknown arch: ${arch}"
esac
echo "$efi_name"
}
# Prints Debian arch name depending on target architecture
#
# No input parameters and these work on the assumption that boot and root parts
# are mounted at work/boot and work/rootfs
probe_debian_arch_name() {
deb_arch=""
arch=$(probe_arch)
case "${arch}" in
"x86-64")
deb_arch="amd64"
;;
"arm")
deb_arch="armhf"
;;
"aarch64")
deb_arch="arm64"
;;
*)
log_fatal "Unknown arch: ${arch}"
esac
echo "${deb_arch}"
}
# Prints GRUB EFI target name depending on target architecture
#
# This is what the file name should be when put on target boot part.
#
# No input parameters and these work on the assumption that boot and root parts
# are mounted at work/boot and work/rootfs
probe_grub_efi_target_name() {
efi_target_name=""
arch=$(probe_arch)
case "$arch" in
"x86-64")
efi_target_name="bootx64.efi"
;;
"arm")
efi_target_name="bootarm.efi"
;;
"aarch64")
efi_target_name="bootaa64.efi"
;;
*)
log_fatal "Unknown arch: ${arch}"
esac
echo "$efi_target_name"
}
# Prints path to the Linux kernel image
#
# $1 - directory in which the search is performed
#
probe_kernel_image() {
kernel_image_path=""
for image in vmlinuz zImage bzImage; do
# Linux kernel image type and naming varies between different platforms.
#
# The wildcard at the end is important, because it is common to suffix the
# Linux kernel version to the image type/name, e.g:
#
# vmlinuz-4.14-x86_64
# vmlinuz-3.10.0-862.el7.x86_64
# vmlinuz-4.15.0-20-generic
#
kernel_image_path=$(sudo find ${1} -name ${image}* ! -name '*-rescue-*')
if [ -n "${kernel_image_path}" ]; then
break
fi
done
echo "${kernel_image_path}"
}
# Prints path to the initrd/initramfs image
#
# $1 - directory in which the search is performed
#
probe_initrd_image() {
initrd_image_path=""
for image in initramfs initrd; do
# initrd/initramfs naming varies between different platforms.
#
# The wildcard at the end is important, because it is common to suffix the
# Linux kernel version to the image name, e.g:
#
# initrd.img-4.15.0-20-generic
#
initrd_image_path=$(sudo find ${1} -name ${image}* ! -name '*-rescue-*')
if [ -n "${initrd_image_path}" ]; then
break
fi
done
echo "${initrd_image_path}"
}
# Prints Linux kernel image name
#
# It will look for it in both boot and rootfs parts. If image is only present
# in boot part, it will move it to rootfs/boot
#
# No input parameters and these work on the assumption that boot and root parts
# are mounted at work/boot and work/rootfs
probe_kernel_in_boot_and_root() {
kernel_imagetype_path=""
# Important to check rootfs/boot first, because it might be possible that
# they are stored in both partitions, and in this case we want to find the
# image in the rootfs first and use that, to avoid copying it over from
# boot part when it is already there.
for boot in work/rootfs/boot work/boot; do
kernel_imagetype_path=$(probe_kernel_image ${boot})
if [ -n "${kernel_imagetype_path}" ] && [ "${boot}" == "work/boot" ]; then
log_info "Found Linux kernel image in boot part, moving to rootfs/boot"
sudo cp ${kernel_imagetype_path} work/rootfs/boot
break;
elif [ -n "${kernel_imagetype_path}" ]; then
break;
fi
done
if [ -n "${kernel_imagetype_path}" ]; then
log_info "Found Linux kernel image: \n\n\t${kernel_imagetype_path}\n"
kernel_imagetype=$(basename ${kernel_imagetype_path})
else
log_warn "Unfortunatly we where not able to find the Linux kernel image."
log_fatal "Please specifc the image name using MENDER_GRUB_KERNEL_IMAGETYPE"
fi
echo "${kernel_imagetype}"
}
# Prints initrd/initramfs image name
#
# It will look for it in both boot and rootfs parts. If image is only present
# in boot part, it will move it to rootfs/boot
#
# No input parameters and these work on the assumption that boot and root parts
# are mounted at work/boot and work/rootfs
probe_initrd_in_boot_and_root() {
initrd_image_path=""
# Important to check rootfs/boot first, because it might be possible that
# they are stored in both partitions, and in this case we want to find the
# image in the rootfs first and use that, to avoid copying it over from
# boot part when it is already there.
for boot in work/rootfs/boot work/boot; do
initrd_image_path=$(probe_initrd_image ${boot})
if [ -n "${initrd_image_path}" ] && [ "${boot}" == "work/boot" ]; then
sudo cp ${initrd_image_path} ${target_rootfs_dir}/boot
break;
elif [ -n "${initrd_image_path}" ]; then
break;
fi
done
if [ -n "${initrd_image_path}" ]; then
log_info "Found initramfs image: \n\n\t${initrd_image_path}\n"
initrd_imagetype=$(basename ${initrd_image_path})
else
log_info "Unfortunatly we where not able to find the initrd image."
log_info "Please specifc the image name using MENDER_GRUB_INITRD_IMAGETYPE \
(only required if your board is using this)"
initrd_imagetype=""
fi
echo "${initrd_imagetype}"
}

28
modules/run.sh

@ -0,0 +1,28 @@
#!/usr/bin/env bash
#
# Copyright 2019 Northern.tech AS
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Run a command, capture output and log it
#
# $1 - command to run
function run_and_log_cmd() {
local -r cmd="${1}"
log_debug "Running: \n\r\n\r\t ${cmd}"
log_debug "Run result: \n\r"
result=$(eval ${cmd})
log_debug "${result}"
}

1
requirements-deb.txt

@ -0,0 +1 @@
binutils xz-utils file rsync parted e2fsprogs xfsprogs pigz dosfstools wget git make bmap-tools

0
rootfs_overlay_demo/.gitkeep

13
scripts/README-run-tests.md

@ -0,0 +1,13 @@
# Run tests
Run the following commands to install test dependencies (assumes that all mender-convert dependencies are already installed):
sudo apt-get update e2fsprogs=1.44.1-1
sudo apt-get -qy --force-yes install python-pip
sudo pip2 install pytest --upgrade
sudo pip2 install pytest-xdist --upgrade
sudo pip2 install pytest-html --upgrade
Run tests:
./scripts/run-tests.sh

64
scripts/bootstrap-rootfs-overlay-demo.sh

@ -0,0 +1,64 @@
#!/bin/bash
#
# Copyright 2019 Northern.tech AS
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Exit if any command exits with a non-zero exit status.
set -o errexit
root_dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )/../" && pwd )
if [ "${root_dir}" != "${PWD}" ]; then
echo "You must execute $(basename $0) from the root directory: ${root_dir}"
exit 1
fi
# Do not actually paste it here, this is just the default value that will
# end up in mender.conf if no token is specified using '--tenant-token'
tenant_token="Paste your Hosted Mender token here"
output_dir=""
while (( "$#" )); do
case "$1" in
-t | --tenant-token)
tenant_token="${2}"
shift 2
;;
-o | --output-dir)
output_dir="${2}"
shift 2
;;
*)
echo "Sorry but the provided option is not supported: $1"
echo "Usage: $(basename $0) --tenant-token"
exit 1
;;
esac
done
if [ -z "${output_dir}" ]; then
echo "Sorry, but you need to provide an output directory using the '-o/--output-dir' option"
exit 1
fi
mkdir -p ${output_dir}/etc/mender
cat <<- EOF > ${output_dir}/etc/mender/mender.conf
{
"InventoryPollIntervalSeconds": 5,
"RetryPollIntervalSeconds": 30,
"ServerURL": "https://hosted.mender.io/",
"TenantToken": "${tenant_token}",
"UpdatePollIntervalSeconds": 5
}
EOF
echo "Configuration file written to: ${output_dir}/etc/mender"

143
scripts/run-tests.sh

@ -0,0 +1,143 @@
#!/bin/bash
set -e
root_dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )/../" && pwd )
if [ "${root_dir}" != "${PWD}" ]; then
echo "You must execute $(basename $0) from the root directory: ${root_dir}"
exit 1
fi
WORKSPACE=./tests
# Relative to where scripts are executed (${WORKSPACE}/mender-image-tests)
MENDER_CONVERT_DIR=../../
BBB_DEBIAN_IMAGE="bone-debian-9.5-iot-armhf-2018-10-07-4gb.img"
BBB_DEBIAN_IMAGE_URL="http://debian.beagleboard.org/images/${BBB_DEBIAN_IMAGE}.xz"
RASPBIAN_IMAGE="2019-04-08-raspbian-stretch-lite"
RASPBIAN_IMAGE_URL="https://downloads.raspberrypi.org/raspbian_lite/images/raspbian_lite-2019-04-09/${RASPBIAN_IMAGE}.zip"
TINKER_IMAGE="20170417-tinker-board-linaro-stretch-alip-v1.8"
TINKER_IMAGE_URL="http://dlcdnet.asus.com/pub/ASUS/mb/Linux/Tinker_Board_2GB/${TINKER_IMAGE}.zip"
UBUNTU_IMAGE="Ubuntu-Bionic-x86-64.img"
UBUNTU_IMAGE_URL="https://d1b0l86ne08fsf.cloudfront.net/mender-convert/images/${UBUNTU_IMAGE}.gz"
MENDER_ACCEPTANCE_URL="https://raw.githubusercontent.com/mendersoftware/meta-mender/master/tests/acceptance"
# Some distros do not have /sbin in path for "normal users"
export PATH="${PATH}:/sbin"
convert_and_test() {
device_type=$1
artifact_name=$2
image_url=$3
image_name=$4
image_name_compressed=$5
config=$6 # Optional
wget -N ${image_url} -P input/
echo "Extracting: ${image_name_compressed}"
case "${image_name_compressed}" in
*.gz)
gunzip -f input/${image_name_compressed}
;;
*.zip)
cd input
unzip -o ${image_name_compressed}
cd -
;;
*.xz)
xz -d -f input/${image_name_compressed}
;;
*)
echo "Unknown image type: ${image_name_compressed}"
exit 1
esac
rm -f ${WORKSPACE}/test_config
if [ -n "${config}" ]; then
cp ${config} ${WORKSPACE}/test_config
fi
# Disable compression of disk image when testing, otherwise we need to
# unpack each image we test which is time consuming
echo "MENDER_COMPRESS_DISK_IMAGE=n" >> ${WORKSPACE}/test_config
echo "Configuration used:"
cat ${WORKSPACE}/test_config
MENDER_ARTIFACT_NAME=${artifact_name} ./docker-mender-convert \
--disk-image input/${image_name} \
--config ${WORKSPACE}/test_config
cd ${WORKSPACE}/mender-image-tests
py.test --verbose \
--junit-xml="${WORKSPACE}/results.xml" \
--test-conversion \
--test-variables="${MENDER_CONVERT_DIR}/deploy/${device_type}-${artifact_name}.cfg" \
--board-type="${device_type}" \
--mender-image=${device_type}-${artifact_name}.sdimg \
--sdimg-location="${MENDER_CONVERT_DIR}/deploy"
cd -
}
get_pytest_files() {
wget -N ${MENDER_ACCEPTANCE_URL}/pytest.ini -P $WORKSPACE/mender-image-tests
wget -N ${MENDER_ACCEPTANCE_URL}/common.py -P $WORKSPACE/mender-image-tests
wget -N ${MENDER_ACCEPTANCE_URL}/conftest.py -P $WORKSPACE/mender-image-tests
wget -N ${MENDER_ACCEPTANCE_URL}/fixtures.py -P $WORKSPACE/mender-image-tests
}
if [ ! -d ${WORKSPACE}/mender-image-tests ]; then
git clone https://github.com/mendersoftware/mender-image-tests ${WORKSPACE}/mender-image-tests
else
cd ${WORKSPACE}/mender-image-tests
git pull
cd -
fi
if ! [ -x "$(command -v mender-artifact)" ]; then
echo "mender-artifact: not found in PATH."
github_PR_status "failure" "mender-artifact: not found in PATH."
exit 1
fi
mkdir -p ${WORKSPACE}
get_pytest_files
./docker-build
convert_and_test "qemux86_64" \
"release-1" \
"${UBUNTU_IMAGE_URL}" \
"${UBUNTU_IMAGE}" \
"${UBUNTU_IMAGE}.gz" \
"configs/qemux86-64_config"
convert_and_test "raspberrypi" \
"release-1" \
"${RASPBIAN_IMAGE_URL}" \
"${RASPBIAN_IMAGE}.img" \
"${RASPBIAN_IMAGE}.zip" \
"configs/raspberrypi3_config"
convert_and_test "linaro-alip" \
"release-1" \
"${TINKER_IMAGE_URL}" \
"${TINKER_IMAGE}.img" \
"${TINKER_IMAGE}.zip"
convert_and_test "beaglebone" \
"release-1" \
"${BBB_DEBIAN_IMAGE_URL}" \
"${BBB_DEBIAN_IMAGE}" \
"${BBB_DEBIAN_IMAGE}.xz"
Loading…
Cancel
Save