diff --git a/convert-stage-3.sh b/convert-stage-3.sh index fa6328f..6587356 100755 --- a/convert-stage-3.sh +++ b/convert-stage-3.sh @@ -9,6 +9,7 @@ build_log=$output_dir/build.log sudo dd if=${output_dir}/rootfs.img of=/dev/mapper/${rootfs_mapping} bs=8M conv=sparse >> "$build_log" 2>&1 sync +sudo resize2fs /dev/mapper/${rootfs_mapping} >> "$build_log" 2>&1 # Check Linux ext4 file system just in case. sudo fsck.ext4 -fp /dev/mapper/${rootfs_mapping} >> "$build_log" 2>&1 # Make sure the rootfs partition's label follows Mender naming convention. diff --git a/mender-convert b/mender-convert index 94e48fa..8d8f19f 100755 --- a/mender-convert +++ b/mender-convert @@ -35,27 +35,29 @@ Expert commands: Options: [-r|--raw-disk-image | -m|--mender-disk-image | -s|--data-part-size-mb | -d|--device-type | -p|--rootfs-partition-id | -i|--demo-host-ip | -c|--server-cert | -u|--server-url | -t|--tenant-token | - -g|--mender-client -b|--bootloader-toolchain | -a|--artifact-name | - -k|--keep | -v|--version] - - raw-disk-image - raw disk embedded Linux (Debian, Raspbian, - Ubuntu, etc.) image path - mender-disk-image - Mender disk image name where the script writes to - should have "sdimg" suffix - data-part-size-mb - data partition size in MB; default value 128MB - device-type - target device identification used to build - Mender image - rootfs-partition-id - selects root filesystem (primary|secondary) - as the source filesystem for an artifact - demo-host-ip - server demo ip used for testing purposes - server-cert - server certificate file - server-url - production server url - tenant-token - Mender tenant token - mender-client - Mender client binary - bootloader-toolchain - GNU Arm Embedded Toolchain - artifact-name - Mender artifact name - keep - keep intermediate files in output directory - version - print the version + -g|--mender-client | -b|--bootloader-toolchain | -a|--artifact-name | + -e|--storage-total-size-mb | -k|--keep | -v|--version] + + raw-disk-image - raw disk embedded Linux (Debian, Raspbian, + Ubuntu, etc.) image path + mender-disk-image - Mender disk image name where the script writes to + should have "sdimg" suffix + data-part-size-mb - data partition size in MB; default value 128MB + device-type - target device identification used to build + Mender image + rootfs-partition-id - selects root filesystem (primary|secondary) + as the source filesystem for an artifact + demo-host-ip - server demo ip used for testing purposes + server-cert - server certificate file + server-url - production server url + tenant-token - Mender tenant token + mender-client - Mender client binary + bootloader-toolchain - GNU Arm Embedded Toolchain + artifact-name - Mender artifact name + storage-total-size-mb - total storage size in MB; default value 8000MB; + it is used to calculate the rootfs final size + keep - keep intermediate files in output directory + version - print the version Note: root filesystem size used in Mender image creation can be found as an output from 'raw-disk-image-shrink-rootfs' command or, in case @@ -146,11 +148,6 @@ fi tool_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -# Default sector size -sector_size= -# Default 'data' partition size in MiB. -data_part_size_mb=128 - mender_disk_image= raw_disk_image= device_type= @@ -190,7 +187,8 @@ do_raw_disk_image_shrink_rootfs() { partition_alignment=$PART_ALIGN_8MB # Gather information about raw disk image. - get_raw_disk_sizes ${raw_disk_image} raw_disk_counts sector_size raw_disk_sizes + get_raw_disk_sizes ${raw_disk_image} raw_disk_counts raw_disk_sizes + local sector_size = ${raw_disk_sizes[sector_size]} # Find first available loopback device. loopdevice=($(losetup -f)) @@ -268,16 +266,27 @@ do_raw_disk_image_create_partitions() { set_mender_disk_alignment $device_type partition_alignment vfat_storage_offset - get_raw_disk_sizes ${raw_disk_image} raw_disk_counts sector_size raw_disk_sizes + get_raw_disk_sizes ${raw_disk_image} raw_disk_counts raw_disk_sizes rc=$? [ $rc -eq 0 ] || { return 1; } - set_mender_disk_sizes ${raw_disk_counts} ${sector_size} \ - ${partition_alignment} ${vfat_storage_offset} \ - ${data_part_size_mb} raw_disk_sizes mender_disk_sizes - log "\tDetected raw disk image with $raw_disk_counts partition(s)." + log "\tCalculating partitions' sizes of the Mender image." + set_mender_disk_sizes ${raw_disk_counts} ${partition_alignment} \ + ${vfat_storage_offset} ${data_part_size_mb} \ + raw_disk_sizes mender_disk_sizes + + local mender_disk_image_size= + calculate_mender_disk_size ${partition_alignment} mender_disk_sizes \ + mender_disk_counts mender_disk_image_size + + mender_image_size_to_total_storage_size ${mender_disk_counts} \ + mender_disk_image_size ${storage_total_size_mb} \ + mender_disk_sizes + rc=$? + [ $rc -eq 0 ] || { return 1; } + if [[ ${raw_disk_counts} -gt 1 ]]; then log "\tExtracting boot partition from raw disk image." @@ -298,10 +307,7 @@ do_raw_disk_image_create_partitions() { ${raw_disk_sizes[pboot_size]} "rootfs.img" fi - local mender_disk_image_size= - calculate_mender_disk_size $sector_size ${partition_alignment} \ - mender_disk_sizes mender_disk_image_size - + local sector_size=${mender_disk_sizes[sector_size]} log "\tCreating blank Mender disk image:\ \n\t\timage size: ${mender_disk_image_size} bytes\ \n\t\tboot partition size: $((${mender_disk_sizes[pboot_size]} * $sector_size)) bytes\ @@ -313,11 +319,11 @@ do_raw_disk_image_create_partitions() { fi create_test_config_file $device_type $partition_alignment $mender_disk_image_size \ - $sector_size mender_disk_sizes + mender_disk_sizes create_mender_disk $mender_disk_image $mender_disk_image_size - format_mender_disk $mender_disk_image $mender_disk_image_size $sector_size \ - $partition_alignment mender_disk_sizes mender_disk_counts + format_mender_disk $mender_disk_image $mender_disk_image_size \ + $partition_alignment mender_disk_sizes rc=$? [ $rc -eq 0 ] || { return 1; } @@ -335,7 +341,7 @@ do_raw_disk_image_create_partitions() { "beaglebone") do_make_sdimg_beaglebone ;; - "raspberrypi3"|"raspberrypi0w") + "raspberrypi3" | "raspberrypi0w") do_make_sdimg_raspberrypi3 ;; "qemux86_64") @@ -561,6 +567,7 @@ do_mender_disk_image_to_artifact() { { log "Error: invalid rootfs partition id provided. Aborting."; return 1; } local count=0 + local sector_size=0 local prootfs_start=0 local prootfs_size=0 local rootfs_a_start=0 @@ -742,6 +749,10 @@ while (( "$#" )); do tenant_token=$2 shift 2 ;; + -e | --storage-total-size-mb) + storage_total_size_mb=$2 + shift 2 + ;; -k | --keep) keep="-k" shift 1 @@ -770,7 +781,11 @@ while (( "$#" )); do done [ -z "${data_part_size_mb}" ] && \ - { log "Default 'data' partition size set to 128MB"; data_part_size_mb=128; } + { log "*** Data partition size set to default value: 128MB ***"; \ + data_part_size_mb=$DATA_PART_SIZE_MB; } +[ -z "${storage_total_size_mb}" ] && \ + { log "*** Total storage size set to default value: 8GB ***"; \ + storage_total_size_mb=$STORAGE_TOTAL_SIZE_MB; } eval set -- "$PARAMS" @@ -856,7 +871,7 @@ case "$1" in esac [[ $keep != "-k" ]] && { rm -f $output_dir/boot.vfat \ - $output_dir/cmdline.txt $output_dir/config.txt \ - $output_dir/syslinux.cfg $output_dir/rootfs.img; } + $output_dir/cmdline.txt $output_dir/config.txt \ + $output_dir/syslinux.cfg $output_dir/rootfs.img; } exit 0 diff --git a/mender-convert-functions.sh b/mender-convert-functions.sh index 9d225ae..25479ce 100755 --- a/mender-convert-functions.sh +++ b/mender-convert-functions.sh @@ -8,6 +8,14 @@ declare -i vfat_storage_offset PART_ALIGN_4MB=4194304 PART_ALIGN_8MB=8388608 +# Default 'data' partition size in MiB. +declare -i data_part_size_mb +# Default total storage size in MiB. +declare -i storage_total_size_mb + +DATA_PART_SIZE_MB=128 +STORAGE_TOTAL_SIZE_MB=8000 + # Number of required heads in a final image. declare -i -r heads=255 # Number of required sectors in a final image. @@ -130,14 +138,12 @@ EOF # Calculates following values: # # $2 - number of partitions -# $3 - sector size -# $4 - array of partitions' sizes for raw disk +# $3 - array of partitions' sizes for raw disk # get_raw_disk_sizes() { local limage=$1 local rvar_count=$2 - local rvar_sectorsize=$3 - shift 3 + shift 2 local rvar_array=($@) local lsubname=${limage:0:10} @@ -177,9 +183,9 @@ get_raw_disk_sizes() { eval $rvar_array[pboot_start]="'$(echo "${lfirstpartinfo}" | tr -s ' ' | cut -d' ' -f${idx_start})'" eval $rvar_array[pboot_size]="'$(echo "${lfirstpartinfo}" | tr -s ' ' | cut -d' ' -f${idx_size})'" + eval $rvar_array[sector_size]="'$lsectorsize'" eval $rvar_count="'$lcount'" - eval $rvar_sectorsize="'$lsectorsize'" return 0 } @@ -200,7 +206,7 @@ set_mender_disk_alignment() { local lvar_partition_alignment=${PART_ALIGN_8MB} local lvar_vfat_storage_offset=$lvar_partition_alignment ;; - "raspberrypi3"|"raspberrypi0w") + "raspberrypi3" | "raspberrypi0w") local lvar_partition_alignment=${PART_ALIGN_4MB} local lvar_uboot_env_size=$(( $lvar_partition_alignment * 2 )) local lvar_vfat_storage_offset=$(( $lvar_partition_alignment + $lvar_uboot_env_size )) @@ -291,27 +297,28 @@ align_partition_size() { # Takes following arguments: # # $1 - number of partition of the raw disk image -# $2 - sector size of the raw disk image -# $3 - mender image partition alignment -# $4 - mender image's boot partition offset -# $5 - data partition size (in MB) -# $6 - array of partitions' sizes for raw image +# $2 - mender image partition alignment +# $3 - mender image's boot partition offset +# $4 - data partition size (in MB) +# $5 - array of partitions' sizes for raw image # # Returns: # -# $7 - array of partitions' sizes for Mender image +# $6 - array of partitions' sizes for Mender image # set_mender_disk_sizes() { local count=$1 - local sectorsize=$2 - local alignment=$3 - local offset=$4 - local datasize=$(( ($5 * 1024 * 1024) / $2 )) - local _raw_sizes=$(declare -p $6) + local alignment=$2 + local offset=$3 + local datasize_mb=$4 + local _raw_sizes=$(declare -p $5) eval "declare -A raw_sizes="${_raw_sizes#*=} - shift 6 + shift 5 local rvar_array=($@) + local sector_size=${raw_sizes[sector_size]} + local datasize=$(( ($datasize_mb * 1024 * 1024) / $sector_size )) + local bootstart= local bootsize= local rootfsstart= @@ -320,7 +327,7 @@ set_mender_disk_sizes() { if [[ $count -eq 1 ]]; then # Default size of the boot partition: 16MiB. - bootsize=$(( (${alignment} * 2) / ${sectorsize} )) + bootsize=$(( (${alignment} * 2) / ${sector_size} )) # Root filesystem size is determined by the size of the single partition. rootfssize=${raw_sizes[pboot_size]} else @@ -329,11 +336,11 @@ set_mender_disk_sizes() { fi # Boot partition storage offset is defined from the top down. - bootstart=$(( ${offset} / ${sectorsize} )) + bootstart=$(( ${offset} / ${sector_size} )) - align_partition_size bootsize $sectorsize - align_partition_size rootfssize $sectorsize - align_partition_size datasize $sectorsize + align_partition_size bootsize $sector_size + align_partition_size rootfssize $sector_size + align_partition_size datasize $sector_size eval $rvar_array[pboot_start]="'$bootstart'" eval $rvar_array[pboot_size]="'$bootsize'" @@ -343,41 +350,119 @@ set_mender_disk_sizes() { if [[ $count -eq 3 ]]; then # Add space for Swap partition. local swapsize=${raw_sizes[pswap_size]} - align_partition_size swapsize $sectorsize + align_partition_size swapsize $sector_size eval $rvar_array[pswap_size]="'$swapsize'" fi + + eval $rvar_array[sector_size]="'$sector_size'" } # Takes following arguments: # -# $1 - sector size (in bytes) -# $2 - partition alignment -# $3 - array of partitions' sizes for Mender image +# $1 - partition alignment +# $2 - array of partitions' sizes for Mender image # # Returns: # +# #3 - number of partitions of the Mender disk image # $4 - final Mender disk image size (in bytes) # calculate_mender_disk_size() { - local _mender_sizes=$(declare -p $3) + local _mender_sizes=$(declare -p $2) eval "declare -A mender_sizes="${_mender_sizes#*=} + local sector_size=${mender_sizes[sector_size]} + local rvar_counts=$3 local rvar_sdimgsize=$4 local sdimgsize=$(( (${mender_sizes[pboot_start]} + ${mender_sizes[pboot_size]} + \ 2 * ${mender_sizes[prootfs_size]} + \ - ${mender_sizes[pdata_size]}) * $1 )) + ${mender_sizes[pdata_size]}) * $sector_size )) + + eval $rvar_counts="'4'" if [ -v mender_sizes[pswap_size] ]; then log "\tSwap partition found." # Add size of the swap partition to the total size. - sdimgsize=$(( $sdimgsize + (${mender_sizes[pswap_size]} * $1) )) + sdimgsize=$(( $sdimgsize + (${mender_sizes[pswap_size]} * $sector_size) )) # Add alignment used as swap partition offset. - sdimgsize=$(( $sdimgsize + 2 * ${2} )) + sdimgsize=$(( $sdimgsize + 2 * ${1} )) + eval $rvar_counts="'6'" fi eval $rvar_sdimgsize="'$sdimgsize'" } +# Takes following arguments: +# +# $1 - number of partition of the Mender image +# $2 - calculated total size of the Mender image in bytes +# $3 - expected total size of the Mender image in mb +# $4 - array of partitions' sizes for Mender image +# +# Returns: +# +# Size of the rootfs partition updated and adjusted to total storage size +# +mender_image_size_to_total_storage_size() { + local count=$1 + local rvar_image_size=$2 + local -n image_size_bytes=$2 + local storage_size_mb=$3 + + local _mender_sizes=$(declare -p $4) + eval "declare -A mender_sizes="${_mender_sizes#*=} + + shift 3 + local rvar_array=($@) + + local image_size_mb=$(( (($image_size_bytes / 1024) / 1024) )) + + log "\tAdjust Mender disk image size to the total storage size (${storage_size_mb}MB)." + + if [ $image_size_mb -gt $storage_size_mb ]; then + log "\tDefined total storage size of ${3}MB is too small." + log "\tMinimal required storage is ${image_size_mb}MB. Aborting." + return 1 + elif [ $image_size_mb -eq $storage_size_mb ]; then + # Simply continue. + log "\tCalculated Mender image size exactly fits the defined total storage." + return 0 + fi + + # Get spare space for rootfs a/b partitions (in sectors). + local sector_size=${mender_sizes[sector_size]} + local image_size_s=$(( $image_size_bytes / $sector_size )) + local storage_size_bytes=$(( ($storage_size_mb * 1024 * 1024) )) + local storage_size_s=$(( $storage_size_bytes / $sector_size )) + local rootfs_overplus_bytes=0 + + local spare_storage_bytes=$(( $storage_size_bytes - $image_size_bytes )) + + if [ $(($spare_storage_bytes % 2)) -ne 0 ]; then + log "\tAdditional space for rootfs partitions not divisible by 2.\ + \n\tFinal image will be smaller than ${storage_size_mb}MB" + fi + rootfs_overplus_bytes=$(( $spare_storage_bytes / 2 )) + + local reminder=$(( ${rootfs_overplus_bytes} % ${partition_alignment} )) + if [ $reminder -ne 0 ]; then + log "\tAdditional space for rootfs partitions not aligned.\ + \n\tFinal image will be smaller than ${storage_size_mb}MB" + fi + rootfs_overplus_bytes=$(($rootfs_overplus_bytes - $reminder)) + rootfs_overplus_s=$(( $rootfs_overplus_bytes / $sector_size )) + + local prootfs_size=${mender_sizes[prootfs_size]} + prootfs_size=$(( $prootfs_size + $rootfs_overplus_s )) + + image_size_bytes=$(( $image_size_bytes + 2 * $rootfs_overplus_bytes )) + + eval $rvar_array[prootfs_size]="'$prootfs_size'" + eval $rvar_image_size="'$image_size_bytes'" + + return 0 +} + # Takes following arguments: # # $1 - raw disk image @@ -405,26 +490,21 @@ create_mender_disk() { # # $1 - Mender disk image path # $2 - Mender disk image size -# $3 - sector size in bytes -# $4 - partition alignment -# $5 - array of partitions' sizes for Mender image -# -# Returns: -# -# $6 - number of partitions of the Mender disk image +# $3 - partition alignment +# $4 - array of partitions' sizes for Mender image # format_mender_disk() { local lfile=$1 local lsize=$2 - local sectorsize=$3 - local alignment=$(($4 / ${sectorsize})) local rc=0 - local _mender_sizes=$(declare -p $5) + local _mender_sizes=$(declare -p $4) eval "declare -A mender_sizes="${_mender_sizes#*=} - local rvar_counts=$6 - cylinders=$(( ${lsize} / ${heads} / ${sectors} / ${sectorsize} )) + local sector_size=${mender_sizes[sector_size]} + local alignment=$(($3 / ${sector_size})) + + cylinders=$(( ${lsize} / ${heads} / ${sectors} / ${sector_size} )) pboot_start=${mender_sizes[pboot_start]} pboot_size=${mender_sizes[pboot_size]} @@ -479,10 +559,8 @@ EOF sudo parted -s ${lfile} -- unit s mkpart extended ${pextended_start} 100% || rc=$? sudo parted -s ${lfile} -- unit s mkpart logical ext4 ${pdata_start} ${pdata_end} || rc=$? sudo parted -s ${lfile} -- unit s mkpart logical linux-swap ${pswap_start} ${pswap_end} || rc=$? - eval $rvar_counts="'6'" else sudo parted -s ${lfile} -- unit s mkpart primary ext4 ${pdata_start} ${pdata_end} || rc=$? - eval $rvar_counts="'4'" fi [[ $rc -eq 0 ]] && { log "\tChanges in partition table applied."; } @@ -746,19 +824,20 @@ extract_file_from_image() { # $1 - device type # $2 - partition alignment in bytes # $3 - total size in bytes -# $4 - sector size in bytes -# $5 - array of partitions' sizes for Mender image +# $4 - array of partitions' sizes for Mender image # create_test_config_file() { local device_type=$1 local alignment=$2 local mender_image_size_mb=$(( (($3 / 1024) / 1024) )) - local _mender_sizes=$(declare -p $5) + local _mender_sizes=$(declare -p $4) eval "declare -A mender_sizes="${_mender_sizes#*=} - local boot_offset=$(( (${mender_sizes[pboot_start]} * $4) )) - local boot_size_mb=$(( (((${mender_sizes[pboot_size]} * $4) / 1024) / 1024) )) - local rootfs_size_mb=$(( (((${mender_sizes[prootfs_size]} * $4) / 1024) / 1024) )) - local data_size_mb=$(( (((${mender_sizes[pdata_size]} * $4) / 1024) / 1024) )) + local sector_size=${mender_sizes[sector_size]} + + local boot_offset=$(( (${mender_sizes[pboot_start]} * $sector_size) )) + local boot_size_mb=$(( (((${mender_sizes[pboot_size]} * $sector_size) / 1024) / 1024) )) + local rootfs_size_mb=$(( (((${mender_sizes[prootfs_size]} * $sector_size) / 1024) / 1024) )) + local data_size_mb=$(( (((${mender_sizes[pdata_size]} * $sector_size) / 1024) / 1024) )) cp ${files_dir}/variables.template ${output_dir}/${device_type}_variables.cfg