Browse Source

Merge pull request #69 from apodogrocki/MEN-2242

Add flag for total storage size
1.1.x
Adam Podogrocki 6 years ago
committed by GitHub
parent
commit
52394e2e37
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
  1. 1
      convert-stage-3.sh
  2. 101
      mender-convert
  3. 183
      mender-convert-functions.sh

1
convert-stage-3.sh

@ -9,6 +9,7 @@ build_log=$output_dir/build.log
sudo dd if=${output_dir}/rootfs.img of=/dev/mapper/${rootfs_mapping} bs=8M conv=sparse >> "$build_log" 2>&1
sync
sudo resize2fs /dev/mapper/${rootfs_mapping} >> "$build_log" 2>&1
# Check Linux ext4 file system just in case.
sudo fsck.ext4 -fp /dev/mapper/${rootfs_mapping} >> "$build_log" 2>&1
# Make sure the rootfs partition's label follows Mender naming convention.

101
mender-convert

@ -35,27 +35,29 @@ Expert commands:
Options: [-r|--raw-disk-image | -m|--mender-disk-image | -s|--data-part-size-mb |
-d|--device-type | -p|--rootfs-partition-id | -i|--demo-host-ip |
-c|--server-cert | -u|--server-url | -t|--tenant-token |
-g|--mender-client -b|--bootloader-toolchain | -a|--artifact-name |
-k|--keep | -v|--version]
raw-disk-image - raw disk embedded Linux (Debian, Raspbian,
Ubuntu, etc.) image path
mender-disk-image - Mender disk image name where the script writes to
should have "sdimg" suffix
data-part-size-mb - data partition size in MB; default value 128MB
device-type - target device identification used to build
Mender image
rootfs-partition-id - selects root filesystem (primary|secondary)
as the source filesystem for an artifact
demo-host-ip - server demo ip used for testing purposes
server-cert - server certificate file
server-url - production server url
tenant-token - Mender tenant token
mender-client - Mender client binary
bootloader-toolchain - GNU Arm Embedded Toolchain
artifact-name - Mender artifact name
keep - keep intermediate files in output directory
version - print the version
-g|--mender-client | -b|--bootloader-toolchain | -a|--artifact-name |
-e|--storage-total-size-mb | -k|--keep | -v|--version]
raw-disk-image - raw disk embedded Linux (Debian, Raspbian,
Ubuntu, etc.) image path
mender-disk-image - Mender disk image name where the script writes to
should have "sdimg" suffix
data-part-size-mb - data partition size in MB; default value 128MB
device-type - target device identification used to build
Mender image
rootfs-partition-id - selects root filesystem (primary|secondary)
as the source filesystem for an artifact
demo-host-ip - server demo ip used for testing purposes
server-cert - server certificate file
server-url - production server url
tenant-token - Mender tenant token
mender-client - Mender client binary
bootloader-toolchain - GNU Arm Embedded Toolchain
artifact-name - Mender artifact name
storage-total-size-mb - total storage size in MB; default value 8000MB;
it is used to calculate the rootfs final size
keep - keep intermediate files in output directory
version - print the version
Note: root filesystem size used in Mender image creation can be found as
an output from 'raw-disk-image-shrink-rootfs' command or, in case
@ -146,11 +148,6 @@ fi
tool_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
# Default sector size
sector_size=
# Default 'data' partition size in MiB.
data_part_size_mb=128
mender_disk_image=
raw_disk_image=
device_type=
@ -190,7 +187,8 @@ do_raw_disk_image_shrink_rootfs() {
partition_alignment=$PART_ALIGN_8MB
# Gather information about raw disk image.
get_raw_disk_sizes ${raw_disk_image} raw_disk_counts sector_size raw_disk_sizes
get_raw_disk_sizes ${raw_disk_image} raw_disk_counts raw_disk_sizes
local sector_size = ${raw_disk_sizes[sector_size]}
# Find first available loopback device.
loopdevice=($(losetup -f))
@ -268,16 +266,27 @@ do_raw_disk_image_create_partitions() {
set_mender_disk_alignment $device_type partition_alignment vfat_storage_offset
get_raw_disk_sizes ${raw_disk_image} raw_disk_counts sector_size raw_disk_sizes
get_raw_disk_sizes ${raw_disk_image} raw_disk_counts raw_disk_sizes
rc=$?
[ $rc -eq 0 ] || { return 1; }
set_mender_disk_sizes ${raw_disk_counts} ${sector_size} \
${partition_alignment} ${vfat_storage_offset} \
${data_part_size_mb} raw_disk_sizes mender_disk_sizes
log "\tDetected raw disk image with $raw_disk_counts partition(s)."
log "\tCalculating partitions' sizes of the Mender image."
set_mender_disk_sizes ${raw_disk_counts} ${partition_alignment} \
${vfat_storage_offset} ${data_part_size_mb} \
raw_disk_sizes mender_disk_sizes
local mender_disk_image_size=
calculate_mender_disk_size ${partition_alignment} mender_disk_sizes \
mender_disk_counts mender_disk_image_size
mender_image_size_to_total_storage_size ${mender_disk_counts} \
mender_disk_image_size ${storage_total_size_mb} \
mender_disk_sizes
rc=$?
[ $rc -eq 0 ] || { return 1; }
if [[ ${raw_disk_counts} -gt 1 ]]; then
log "\tExtracting boot partition from raw disk image."
@ -298,10 +307,7 @@ do_raw_disk_image_create_partitions() {
${raw_disk_sizes[pboot_size]} "rootfs.img"
fi
local mender_disk_image_size=
calculate_mender_disk_size $sector_size ${partition_alignment} \
mender_disk_sizes mender_disk_image_size
local sector_size=${mender_disk_sizes[sector_size]}
log "\tCreating blank Mender disk image:\
\n\t\timage size: ${mender_disk_image_size} bytes\
\n\t\tboot partition size: $((${mender_disk_sizes[pboot_size]} * $sector_size)) bytes\
@ -313,11 +319,11 @@ do_raw_disk_image_create_partitions() {
fi
create_test_config_file $device_type $partition_alignment $mender_disk_image_size \
$sector_size mender_disk_sizes
mender_disk_sizes
create_mender_disk $mender_disk_image $mender_disk_image_size
format_mender_disk $mender_disk_image $mender_disk_image_size $sector_size \
$partition_alignment mender_disk_sizes mender_disk_counts
format_mender_disk $mender_disk_image $mender_disk_image_size \
$partition_alignment mender_disk_sizes
rc=$?
[ $rc -eq 0 ] || { return 1; }
@ -335,7 +341,7 @@ do_raw_disk_image_create_partitions() {
"beaglebone")
do_make_sdimg_beaglebone
;;
"raspberrypi3"|"raspberrypi0w")
"raspberrypi3" | "raspberrypi0w")
do_make_sdimg_raspberrypi3
;;
"qemux86_64")
@ -561,6 +567,7 @@ do_mender_disk_image_to_artifact() {
{ log "Error: invalid rootfs partition id provided. Aborting."; return 1; }
local count=0
local sector_size=0
local prootfs_start=0
local prootfs_size=0
local rootfs_a_start=0
@ -742,6 +749,10 @@ while (( "$#" )); do
tenant_token=$2
shift 2
;;
-e | --storage-total-size-mb)
storage_total_size_mb=$2
shift 2
;;
-k | --keep)
keep="-k"
shift 1
@ -770,7 +781,11 @@ while (( "$#" )); do
done
[ -z "${data_part_size_mb}" ] && \
{ log "Default 'data' partition size set to 128MB"; data_part_size_mb=128; }
{ log "*** Data partition size set to default value: 128MB ***"; \
data_part_size_mb=$DATA_PART_SIZE_MB; }
[ -z "${storage_total_size_mb}" ] && \
{ log "*** Total storage size set to default value: 8GB ***"; \
storage_total_size_mb=$STORAGE_TOTAL_SIZE_MB; }
eval set -- "$PARAMS"
@ -856,7 +871,7 @@ case "$1" in
esac
[[ $keep != "-k" ]] && { rm -f $output_dir/boot.vfat \
$output_dir/cmdline.txt $output_dir/config.txt \
$output_dir/syslinux.cfg $output_dir/rootfs.img; }
$output_dir/cmdline.txt $output_dir/config.txt \
$output_dir/syslinux.cfg $output_dir/rootfs.img; }
exit 0

183
mender-convert-functions.sh

@ -8,6 +8,14 @@ declare -i vfat_storage_offset
PART_ALIGN_4MB=4194304
PART_ALIGN_8MB=8388608
# Default 'data' partition size in MiB.
declare -i data_part_size_mb
# Default total storage size in MiB.
declare -i storage_total_size_mb
DATA_PART_SIZE_MB=128
STORAGE_TOTAL_SIZE_MB=8000
# Number of required heads in a final image.
declare -i -r heads=255
# Number of required sectors in a final image.
@ -130,14 +138,12 @@ EOF
# Calculates following values:
#
# $2 - number of partitions
# $3 - sector size
# $4 - array of partitions' sizes for raw disk
# $3 - array of partitions' sizes for raw disk
#
get_raw_disk_sizes() {
local limage=$1
local rvar_count=$2
local rvar_sectorsize=$3
shift 3
shift 2
local rvar_array=($@)
local lsubname=${limage:0:10}
@ -177,9 +183,9 @@ get_raw_disk_sizes() {
eval $rvar_array[pboot_start]="'$(echo "${lfirstpartinfo}" | tr -s ' ' | cut -d' ' -f${idx_start})'"
eval $rvar_array[pboot_size]="'$(echo "${lfirstpartinfo}" | tr -s ' ' | cut -d' ' -f${idx_size})'"
eval $rvar_array[sector_size]="'$lsectorsize'"
eval $rvar_count="'$lcount'"
eval $rvar_sectorsize="'$lsectorsize'"
return 0
}
@ -200,7 +206,7 @@ set_mender_disk_alignment() {
local lvar_partition_alignment=${PART_ALIGN_8MB}
local lvar_vfat_storage_offset=$lvar_partition_alignment
;;
"raspberrypi3"|"raspberrypi0w")
"raspberrypi3" | "raspberrypi0w")
local lvar_partition_alignment=${PART_ALIGN_4MB}
local lvar_uboot_env_size=$(( $lvar_partition_alignment * 2 ))
local lvar_vfat_storage_offset=$(( $lvar_partition_alignment + $lvar_uboot_env_size ))
@ -291,27 +297,28 @@ align_partition_size() {
# Takes following arguments:
#
# $1 - number of partition of the raw disk image
# $2 - sector size of the raw disk image
# $3 - mender image partition alignment
# $4 - mender image's boot partition offset
# $5 - data partition size (in MB)
# $6 - array of partitions' sizes for raw image
# $2 - mender image partition alignment
# $3 - mender image's boot partition offset
# $4 - data partition size (in MB)
# $5 - array of partitions' sizes for raw image
#
# Returns:
#
# $7 - array of partitions' sizes for Mender image
# $6 - array of partitions' sizes for Mender image
#
set_mender_disk_sizes() {
local count=$1
local sectorsize=$2
local alignment=$3
local offset=$4
local datasize=$(( ($5 * 1024 * 1024) / $2 ))
local _raw_sizes=$(declare -p $6)
local alignment=$2
local offset=$3
local datasize_mb=$4
local _raw_sizes=$(declare -p $5)
eval "declare -A raw_sizes="${_raw_sizes#*=}
shift 6
shift 5
local rvar_array=($@)
local sector_size=${raw_sizes[sector_size]}
local datasize=$(( ($datasize_mb * 1024 * 1024) / $sector_size ))
local bootstart=
local bootsize=
local rootfsstart=
@ -320,7 +327,7 @@ set_mender_disk_sizes() {
if [[ $count -eq 1 ]]; then
# Default size of the boot partition: 16MiB.
bootsize=$(( (${alignment} * 2) / ${sectorsize} ))
bootsize=$(( (${alignment} * 2) / ${sector_size} ))
# Root filesystem size is determined by the size of the single partition.
rootfssize=${raw_sizes[pboot_size]}
else
@ -329,11 +336,11 @@ set_mender_disk_sizes() {
fi
# Boot partition storage offset is defined from the top down.
bootstart=$(( ${offset} / ${sectorsize} ))
bootstart=$(( ${offset} / ${sector_size} ))
align_partition_size bootsize $sectorsize
align_partition_size rootfssize $sectorsize
align_partition_size datasize $sectorsize
align_partition_size bootsize $sector_size
align_partition_size rootfssize $sector_size
align_partition_size datasize $sector_size
eval $rvar_array[pboot_start]="'$bootstart'"
eval $rvar_array[pboot_size]="'$bootsize'"
@ -343,41 +350,119 @@ set_mender_disk_sizes() {
if [[ $count -eq 3 ]]; then
# Add space for Swap partition.
local swapsize=${raw_sizes[pswap_size]}
align_partition_size swapsize $sectorsize
align_partition_size swapsize $sector_size
eval $rvar_array[pswap_size]="'$swapsize'"
fi
eval $rvar_array[sector_size]="'$sector_size'"
}
# Takes following arguments:
#
# $1 - sector size (in bytes)
# $2 - partition alignment
# $3 - array of partitions' sizes for Mender image
# $1 - partition alignment
# $2 - array of partitions' sizes for Mender image
#
# Returns:
#
# #3 - number of partitions of the Mender disk image
# $4 - final Mender disk image size (in bytes)
#
calculate_mender_disk_size() {
local _mender_sizes=$(declare -p $3)
local _mender_sizes=$(declare -p $2)
eval "declare -A mender_sizes="${_mender_sizes#*=}
local sector_size=${mender_sizes[sector_size]}
local rvar_counts=$3
local rvar_sdimgsize=$4
local sdimgsize=$(( (${mender_sizes[pboot_start]} + ${mender_sizes[pboot_size]} + \
2 * ${mender_sizes[prootfs_size]} + \
${mender_sizes[pdata_size]}) * $1 ))
${mender_sizes[pdata_size]}) * $sector_size ))
eval $rvar_counts="'4'"
if [ -v mender_sizes[pswap_size] ]; then
log "\tSwap partition found."
# Add size of the swap partition to the total size.
sdimgsize=$(( $sdimgsize + (${mender_sizes[pswap_size]} * $1) ))
sdimgsize=$(( $sdimgsize + (${mender_sizes[pswap_size]} * $sector_size) ))
# Add alignment used as swap partition offset.
sdimgsize=$(( $sdimgsize + 2 * ${2} ))
sdimgsize=$(( $sdimgsize + 2 * ${1} ))
eval $rvar_counts="'6'"
fi
eval $rvar_sdimgsize="'$sdimgsize'"
}
# Takes following arguments:
#
# $1 - number of partition of the Mender image
# $2 - calculated total size of the Mender image in bytes
# $3 - expected total size of the Mender image in mb
# $4 - array of partitions' sizes for Mender image
#
# Returns:
#
# Size of the rootfs partition updated and adjusted to total storage size
#
mender_image_size_to_total_storage_size() {
local count=$1
local rvar_image_size=$2
local -n image_size_bytes=$2
local storage_size_mb=$3
local _mender_sizes=$(declare -p $4)
eval "declare -A mender_sizes="${_mender_sizes#*=}
shift 3
local rvar_array=($@)
local image_size_mb=$(( (($image_size_bytes / 1024) / 1024) ))
log "\tAdjust Mender disk image size to the total storage size (${storage_size_mb}MB)."
if [ $image_size_mb -gt $storage_size_mb ]; then
log "\tDefined total storage size of ${3}MB is too small."
log "\tMinimal required storage is ${image_size_mb}MB. Aborting."
return 1
elif [ $image_size_mb -eq $storage_size_mb ]; then
# Simply continue.
log "\tCalculated Mender image size exactly fits the defined total storage."
return 0
fi
# Get spare space for rootfs a/b partitions (in sectors).
local sector_size=${mender_sizes[sector_size]}
local image_size_s=$(( $image_size_bytes / $sector_size ))
local storage_size_bytes=$(( ($storage_size_mb * 1024 * 1024) ))
local storage_size_s=$(( $storage_size_bytes / $sector_size ))
local rootfs_overplus_bytes=0
local spare_storage_bytes=$(( $storage_size_bytes - $image_size_bytes ))
if [ $(($spare_storage_bytes % 2)) -ne 0 ]; then
log "\tAdditional space for rootfs partitions not divisible by 2.\
\n\tFinal image will be smaller than ${storage_size_mb}MB"
fi
rootfs_overplus_bytes=$(( $spare_storage_bytes / 2 ))
local reminder=$(( ${rootfs_overplus_bytes} % ${partition_alignment} ))
if [ $reminder -ne 0 ]; then
log "\tAdditional space for rootfs partitions not aligned.\
\n\tFinal image will be smaller than ${storage_size_mb}MB"
fi
rootfs_overplus_bytes=$(($rootfs_overplus_bytes - $reminder))
rootfs_overplus_s=$(( $rootfs_overplus_bytes / $sector_size ))
local prootfs_size=${mender_sizes[prootfs_size]}
prootfs_size=$(( $prootfs_size + $rootfs_overplus_s ))
image_size_bytes=$(( $image_size_bytes + 2 * $rootfs_overplus_bytes ))
eval $rvar_array[prootfs_size]="'$prootfs_size'"
eval $rvar_image_size="'$image_size_bytes'"
return 0
}
# Takes following arguments:
#
# $1 - raw disk image
@ -405,26 +490,21 @@ create_mender_disk() {
#
# $1 - Mender disk image path
# $2 - Mender disk image size
# $3 - sector size in bytes
# $4 - partition alignment
# $5 - array of partitions' sizes for Mender image
#
# Returns:
#
# $6 - number of partitions of the Mender disk image
# $3 - partition alignment
# $4 - array of partitions' sizes for Mender image
#
format_mender_disk() {
local lfile=$1
local lsize=$2
local sectorsize=$3
local alignment=$(($4 / ${sectorsize}))
local rc=0
local _mender_sizes=$(declare -p $5)
local _mender_sizes=$(declare -p $4)
eval "declare -A mender_sizes="${_mender_sizes#*=}
local rvar_counts=$6
cylinders=$(( ${lsize} / ${heads} / ${sectors} / ${sectorsize} ))
local sector_size=${mender_sizes[sector_size]}
local alignment=$(($3 / ${sector_size}))
cylinders=$(( ${lsize} / ${heads} / ${sectors} / ${sector_size} ))
pboot_start=${mender_sizes[pboot_start]}
pboot_size=${mender_sizes[pboot_size]}
@ -479,10 +559,8 @@ EOF
sudo parted -s ${lfile} -- unit s mkpart extended ${pextended_start} 100% || rc=$?
sudo parted -s ${lfile} -- unit s mkpart logical ext4 ${pdata_start} ${pdata_end} || rc=$?
sudo parted -s ${lfile} -- unit s mkpart logical linux-swap ${pswap_start} ${pswap_end} || rc=$?
eval $rvar_counts="'6'"
else
sudo parted -s ${lfile} -- unit s mkpart primary ext4 ${pdata_start} ${pdata_end} || rc=$?
eval $rvar_counts="'4'"
fi
[[ $rc -eq 0 ]] && { log "\tChanges in partition table applied."; }
@ -746,19 +824,20 @@ extract_file_from_image() {
# $1 - device type
# $2 - partition alignment in bytes
# $3 - total size in bytes
# $4 - sector size in bytes
# $5 - array of partitions' sizes for Mender image
# $4 - array of partitions' sizes for Mender image
#
create_test_config_file() {
local device_type=$1
local alignment=$2
local mender_image_size_mb=$(( (($3 / 1024) / 1024) ))
local _mender_sizes=$(declare -p $5)
local _mender_sizes=$(declare -p $4)
eval "declare -A mender_sizes="${_mender_sizes#*=}
local boot_offset=$(( (${mender_sizes[pboot_start]} * $4) ))
local boot_size_mb=$(( (((${mender_sizes[pboot_size]} * $4) / 1024) / 1024) ))
local rootfs_size_mb=$(( (((${mender_sizes[prootfs_size]} * $4) / 1024) / 1024) ))
local data_size_mb=$(( (((${mender_sizes[pdata_size]} * $4) / 1024) / 1024) ))
local sector_size=${mender_sizes[sector_size]}
local boot_offset=$(( (${mender_sizes[pboot_start]} * $sector_size) ))
local boot_size_mb=$(( (((${mender_sizes[pboot_size]} * $sector_size) / 1024) / 1024) ))
local rootfs_size_mb=$(( (((${mender_sizes[prootfs_size]} * $sector_size) / 1024) / 1024) ))
local data_size_mb=$(( (((${mender_sizes[pdata_size]} * $sector_size) / 1024) / 1024) ))
cp ${files_dir}/variables.template ${output_dir}/${device_type}_variables.cfg

Loading…
Cancel
Save