diff --git a/.dockerignore b/.dockerignore index ba64707..24ae780 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,3 @@ input/ output/ +device-image-shell/output diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..24ae780 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +input/ +output/ +device-image-shell/output diff --git a/bbb-convert-stage-2.sh b/bbb-convert-stage-2.sh index c2106b2..87f74c5 100755 --- a/bbb-convert-stage-2.sh +++ b/bbb-convert-stage-2.sh @@ -1,46 +1,43 @@ #!/bin/bash -sdimg_boot_dir=$1 -embedded_rootfs_dir=$2 +output_dir=$1 +boot_mapping=$2 +embedded_rootfs_dir=$3 uboot_backup_dir=${embedded_rootfs_dir}/opt/backup/uboot +build_log=$output_dir/build.log -[ ! -d "${sdimg_boot_dir}" ] && \ - { log "Error: boot location not mounted."; exit 1; } +[ ! -f $output_dir/boot.vfat ] && \ + { log "Error: extracted boot partition not found. Aborting."; exit 1; } [ ! -d "${embedded_rootfs_dir}" ] && \ { log "Error: embedded content not mounted."; exit 1; } [[ ! -f $uboot_backup_dir/MLO || ! -f $uboot_backup_dir/u-boot.img ]] && \ { log "Error: cannot find U-Boot related files."; exit 1; } -set_uenv() { - cat <<- 'EOF' | sudo tee --append $sdimg_boot_dir/uEnv.txt 2>&1 >/dev/null - loadaddr=0x82000000 - fdtaddr=0x88000000 - rdaddr=0x88080000 +cat <<- 'EOF' | sudo tee --append ${output_dir}/uEnv.txt 2>&1 >/dev/null +loadaddr=0x82000000 +fdtaddr=0x88000000 +rdaddr=0x88080000 - initrd_high=0xffffffff - fdt_high=0xffffffff +initrd_high=0xffffffff +fdt_high=0xffffffff - loadximage=echo debug: [/boot/vmlinuz-${uname_r}] ... ; load mmc 0:2 ${loadaddr} /boot/vmlinuz-${uname_r} - loadxfdt=echo debug: [/boot/dtbs/${uname_r}/${fdtfile}] ... ;load mmc 0:2 ${fdtaddr} /boot/dtbs/${uname_r}/${fdtfile} - loadxrd=echo debug: [/boot/initrd.img-${uname_r}] ... ; load mmc 0:2 ${rdaddr} /boot/initrd.img-${uname_r}; setenv rdsize ${filesize} - loaduEnvtxt=load mmc 0:2 ${loadaddr} /boot/uEnv.txt ; env import -t ${loadaddr} ${filesize}; - check_dtb=if test -n ${dtb}; then setenv fdtfile ${dtb};fi; - loadall=run loaduEnvtxt; run check_dtb; run loadximage; run loadxrd; run loadxfdt; +loadximage=echo debug: [/boot/vmlinuz-${uname_r}] ... ; load mmc 0:2 ${loadaddr} /boot/vmlinuz-${uname_r} +loadxfdt=echo debug: [/boot/dtbs/${uname_r}/${fdtfile}] ... ;load mmc 0:2 ${fdtaddr} /boot/dtbs/${uname_r}/${fdtfile} +loadxrd=echo debug: [/boot/initrd.img-${uname_r}] ... ; load mmc 0:2 ${rdaddr} /boot/initrd.img-${uname_r}; setenv rdsize ${filesize} +loaduEnvtxt=load mmc 0:2 ${loadaddr} /boot/uEnv.txt ; env import -t ${loadaddr} ${filesize}; +check_dtb=if test -n ${dtb}; then setenv fdtfile ${dtb};fi; +loadall=run loaduEnvtxt; run check_dtb; run loadximage; run loadxrd; run loadxfdt; - mmcargs=setenv bootargs console=tty0 console=${console} ${optargs} ${cape_disable} ${cape_enable} root=/dev/mmcblk0p2 rootfstype=${mmcrootfstype} ${cmdline} +mmcargs=setenv bootargs console=tty0 console=${console} ${optargs} ${cape_disable} ${cape_enable} root=/dev/mmcblk0p2 rootfstype=${mmcrootfstype} ${cmdline} - uenvcmd=run loadall; run mmcargs; echo debug: [${bootargs}] ... ; echo debug: [bootz ${loadaddr} ${rdaddr}:${rdsize} ${fdtaddr}] ... ; bootz ${loadaddr} ${rdaddr}:${rdsize} ${fdtaddr}; - EOF -} +uenvcmd=run loadall; run mmcargs; echo debug: [${bootargs}] ... ; echo debug: [bootz ${loadaddr} ${rdaddr}:${rdsize} ${fdtaddr}] ... ; bootz ${loadaddr} ${rdaddr}:${rdsize} ${fdtaddr}; +EOF -## dd if=MLO of=${sdimg_file} count=1 seek=1 bs=128k -## dd if=u-boot.img of=${sdimg_file} count=2 seek=1 bs=384k -# Copy U-Boot related files. -sudo cp ${uboot_backup_dir}/MLO ${sdimg_boot_dir} -sudo cp ${uboot_backup_dir}/u-boot.img ${sdimg_boot_dir} +mcopy -o -i ${output_dir}/boot.vfat -s ${output_dir}/uEnv.txt ::uEnv.txt +mcopy -o -i ${output_dir}/boot.vfat -s ${uboot_backup_dir}/MLO ::MLO +mcopy -o -i ${output_dir}/boot.vfat -s ${uboot_backup_dir}/u-boot.img ::u-boot.img -# Create U-Boot purposed uEnv.txt file. -set_uenv +sudo dd if=${output_dir}/boot.vfat of=/dev/mapper/${boot_mapping} bs=1M conv=sparse >> "$build_log" 2>&1 log "\tDone." diff --git a/bbb-convert-stage-3.sh b/bbb-convert-stage-3.sh deleted file mode 100755 index 20ae0a2..0000000 --- a/bbb-convert-stage-3.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -sdimg_primary_dir=$1 -embedded_rootfs_dir=$2 - -[ ! -d "${sdimg_primary_dir}" ] && \ - { log "Error: rootfs location not mounted."; exit 1; } -[ ! -d "${embedded_rootfs_dir}" ] && \ - { log "Error: embedded content not mounted."; exit 1; } - -# Copy rootfs partition. -sudo cp -ar ${embedded_rootfs_dir}/* ${sdimg_primary_dir} - -# Add mountpoints. -sudo install -d -m 755 ${sdimg_primary_dir}/boot/efi -sudo install -d -m 755 ${sdimg_primary_dir}/data - -log "\tDone." - -exit 0 diff --git a/convert-stage-3.sh b/convert-stage-3.sh index e657395..fa6328f 100755 --- a/convert-stage-3.sh +++ b/convert-stage-3.sh @@ -7,7 +7,7 @@ build_log=$output_dir/build.log [ ! -f ${output_dir}/rootfs.img ] && \ { log "Error: extracted rootfs partition not found. Aborting."; exit 1; } -sudo dd if=${output_dir}/rootfs.img of=/dev/mapper/${rootfs_mapping} bs=8M >> "$build_log" 2>&1 +sudo dd if=${output_dir}/rootfs.img of=/dev/mapper/${rootfs_mapping} bs=8M conv=sparse >> "$build_log" 2>&1 sync # Check Linux ext4 file system just in case. sudo fsck.ext4 -fp /dev/mapper/${rootfs_mapping} >> "$build_log" 2>&1 diff --git a/convert-stage-4.sh b/convert-stage-4.sh index 3139459..6db352f 100755 --- a/convert-stage-4.sh +++ b/convert-stage-4.sh @@ -171,26 +171,18 @@ install_files() { sudo ln -sf /data/${databootdir}/fw_env.config ${primary_dir}/etc/fw_env.config - # Prepare 'primary' partition - [ ! -d "$primary_dir/data" ] && \ - { log "\t'data' mountpoint missing. Adding"; \ - sudo install -d -m 755 ${primary_dir}/data; } + # Create mount-points + # + # Note that only one of /boot/efi or /uboot will be used depending on what + # type of Mender integration is used (GRUB or U-boot). I do not see any + # problems with keeping an empty directory to reduce complexity of creating + # this directory structure. + sudo install -d -m 755 ${primary_dir}/data + sudo install -d -m 755 ${primary_dir}/boot/efi + sudo install -d -m 755 ${primary_dir}/uboot case "$device_type" in - "beaglebone") - [ ! -d "$primary_dir/boot/efi" ] && \ - { log "\t'/boot/efi' mountpoint missing. Adding"; \ - sudo install -d -m 755 ${primary_dir}/boot/efi; } - ;; - "raspberrypi3"|"raspberrypi0w") - [ ! -d "$primary_dir/uboot" ] && \ - { log "\t'/uboot' mountpoint missing. Adding"; \ - sudo install -d -m 755 ${primary_dir}/uboot; } - ;; "qemux86_64") - [ ! -d "$primary_dir/boot/efi" ] && \ - { log "\t'/boot/efi' mountpoint missing. Adding"; \ - sudo install -d -m 755 ${primary_dir}/boot/efi; } sudo install -d ${primary_dir}/lib64 sudo ln -sf /lib/ld-linux-x86-64.so.2 ${primary_dir}/lib64/ld-linux-x86-64.so.2 ;; diff --git a/mender-convert b/mender-convert index 79595f9..94e48fa 100755 --- a/mender-convert +++ b/mender-convert @@ -266,21 +266,6 @@ do_raw_disk_image_create_partitions() { # Change current directory to 'output' directory. cd $output_dir - # Make sure the user's given Mender image name has a correct extension. - # If Mender image name is not provided, then use following syntax: - # mender--.sdimg - if [ -n "${mender_disk_image}" ]; then - local mender_disk_basename=$(basename -- "$mender_disk_image") - if [[ $mender_disk_basename =~ \.sdimg$ ]]; then - mender_disk_image=$output_dir/$mender_disk_basename - else - local mender_disk_filename="${mender_disk_basename%.*}" - mender_disk_image=$output_dir/${mender_disk_filename}.sdimg - fi - else - mender_disk_image=$output_dir/mender-${device_type}-${artifact_name}.sdimg - fi - set_mender_disk_alignment $device_type partition_alignment vfat_storage_offset get_raw_disk_sizes ${raw_disk_image} raw_disk_counts sector_size raw_disk_sizes @@ -293,11 +278,31 @@ do_raw_disk_image_create_partitions() { log "\tDetected raw disk image with $raw_disk_counts partition(s)." + if [[ ${raw_disk_counts} -gt 1 ]]; then + log "\tExtracting boot partition from raw disk image." + + extract_file_from_image ${raw_disk_image} ${raw_disk_sizes[pboot_start]} \ + ${raw_disk_sizes[pboot_size]} "boot.vfat" + + log "\tExtracting root filesystem partition from raw disk image." + extract_file_from_image ${raw_disk_image} ${raw_disk_sizes[prootfs_start]} \ + ${raw_disk_sizes[prootfs_size]} "rootfs.img" + else + log "\tGenerating boot partition (required, does not exist in original image)" + + dd if=/dev/zero of=${output_dir}/boot.vfat count=${mender_disk_sizes[pboot_size]} bs=512 >> "$build_log" 2>&1 + mkfs.vfat ${output_dir}/boot.vfat >> "$build_log" 2>&1 + + log "\tExtracting root filesystem partition from raw disk image." + extract_file_from_image ${raw_disk_image} ${raw_disk_sizes[pboot_start]} \ + ${raw_disk_sizes[pboot_size]} "rootfs.img" + fi + local mender_disk_image_size= calculate_mender_disk_size $sector_size ${partition_alignment} \ mender_disk_sizes mender_disk_image_size - log "\tCreating Mender disk image:\ + log "\tCreating blank Mender disk image:\ \n\t\timage size: ${mender_disk_image_size} bytes\ \n\t\tboot partition size: $((${mender_disk_sizes[pboot_size]} * $sector_size)) bytes\ \n\t\troot filesystem size: $((${mender_disk_sizes[prootfs_size]} * $sector_size)) bytes\ @@ -358,21 +363,23 @@ do_make_sdimg_beaglebone() { local ret=0 create_device_maps $raw_disk_image raw_disk_mappings - - mount_mender_disk ${mender_disk_mappings[@]} mount_raw_disk ${raw_disk_mappings[@]} log "$step/$total Setting boot partition..." ((step++)) - stage_2_args="$sdimg_boot_dir $embedded_rootfs_dir" + + stage_2_args="$output_dir ${mender_disk_mappings[0]} ${embedded_rootfs_dir}" ${tool_dir}/bbb-convert-stage-2.sh ${stage_2_args} || ret=$? [[ $ret -ne 0 ]] && { log "Aborting."; return $ret; } log "$step/$total Setting root filesystem partition..." ((step++)) - stage_3_args="$sdimg_primary_dir $embedded_rootfs_dir" - ${tool_dir}/bbb-convert-stage-3.sh ${stage_3_args} || ret=$? - [[ $ret -ne 0 ]] && { log "Aborting."; return $ret; } + + stage_3_args="$output_dir ${mender_disk_mappings[1]}" + ${tool_dir}/convert-stage-3.sh ${stage_3_args} || ret=$? + [[ $ret -ne 0 ]] && { return $ret; } + + mount_mender_disk ${mender_disk_mappings[@]} log "$step/$total Setting file system table..." ((step++)) @@ -384,15 +391,6 @@ do_make_sdimg_beaglebone() { do_make_sdimg_raspberrypi3() { log "$step/$total Setting boot partition..." ((step++)) - local raw_boot_part=$(fdisk -l ${raw_disk_image} | grep FAT32) - - local boot_part_start=$(echo ${raw_boot_part} | awk '{print $2}') - local boot_part_end=$(echo ${raw_boot_part} | awk '{print $3}') - local boot_part_size=$(echo ${raw_boot_part} | awk '{print $4}') - - log "\tExtracting boot partition from raw disk image." - extract_file_from_image ${raw_disk_image} ${boot_part_start} \ - ${boot_part_size} "boot.vfat" stage_2_args="$output_dir ${mender_disk_mappings[0]}" ${tool_dir}/rpi-convert-stage-2.sh ${stage_2_args} || ret=$? @@ -400,15 +398,6 @@ do_make_sdimg_raspberrypi3() { log "$step/$total Setting root filesystem partition..." ((step++)) - local raw_rootfs_part=$(fdisk -l ${raw_disk_image} | grep Linux) - - local rootfs_part_start=$(echo ${raw_rootfs_part} | awk '{print $2}') - local rootfs_part_end=$(echo ${raw_rootfs_part} | awk '{print $3}') - local rootfs_part_size=$(echo ${raw_rootfs_part} | awk '{print $4}') - - log "\tExtracting root filesystem partition from raw disk image." - extract_file_from_image ${raw_disk_image} ${rootfs_part_start} \ - ${rootfs_part_size} "rootfs.img" stage_3_args="$output_dir ${mender_disk_mappings[1]}" ${tool_dir}/convert-stage-3.sh ${stage_3_args} || ret=$? @@ -416,18 +405,10 @@ do_make_sdimg_raspberrypi3() { mount_mender_disk ${mender_disk_mappings[@]} - # Add mountpoints. - sudo install -d -m 755 ${sdimg_primary_dir}/uboot - sudo install -d -m 755 ${sdimg_primary_dir}/data - log "$step/$total Setting file system table..." ((step++)) set_fstab $device_type - [[ $keep != "-k" ]] && { rm -f $output_dir/boot.vfat\ - $output_dir/cmdline.txt $output_dir/config.txt\ - $output_dir/rootfs.img; } - return 0 } @@ -435,31 +416,12 @@ do_make_sdimg_qemux86_64() { log "$step/$total Setting boot partition..." ((step++)) - local raw_boot_part=$(fdisk -l ${raw_disk_image} | grep 'Microsoft' | sed s/*//) - - local boot_part_start=$(echo ${raw_boot_part} | awk '{print $2}') - local boot_part_end=$(echo ${raw_boot_part} | awk '{print $3}') - local boot_part_size=$(echo ${raw_boot_part} | awk '{print $4}') - - log "\tExtracting boot partition from raw disk image." - extract_file_from_image ${raw_disk_image} ${boot_part_start} \ - ${boot_part_size} "boot.vfat" - stage_2_args="$output_dir ${mender_disk_mappings[0]}" ${tool_dir}/qemux86_64-convert-stage-2.sh ${stage_2_args} || ret=$? [[ $ret -ne 0 ]] && { return $ret; } log "$step/$total Setting root filesystem partition..." ((step++)) - local raw_rootfs_part=$(fdisk -l ${raw_disk_image} | grep 'Linux filesystem') - - local rootfs_part_start=$(echo ${raw_rootfs_part} | awk '{print $2}') - local rootfs_part_end=$(echo ${raw_rootfs_part} | awk '{print $3}') - local rootfs_part_size=$(echo ${raw_rootfs_part} | awk '{print $4}') - - log "\tExtracting root filesystem partition from raw disk image." - extract_file_from_image ${raw_disk_image} ${rootfs_part_start} \ - ${rootfs_part_size} "rootfs.img" stage_3_args="$output_dir ${mender_disk_mappings[1]}" ${tool_dir}/convert-stage-3.sh ${stage_3_args} || ret=$? @@ -467,17 +429,10 @@ do_make_sdimg_qemux86_64() { mount_mender_disk ${mender_disk_mappings[@]} - # Add mountpoints. - sudo install -d -m 755 ${sdimg_primary_dir}/boot/efi - sudo install -d -m 755 ${sdimg_primary_dir}/data - log "$step/$total Setting file system table..." ((step++)) set_fstab $device_type - [[ $keep != "-k" ]] && { rm -f $output_dir/boot.vfat\ - $output_dir/syslinux.cfg $output_dir/rootfs.img; } - return 0 } @@ -649,11 +604,6 @@ do_mender_disk_image_to_artifact() { rootfs_path=$sdimg_secondary_dir fi - local mender_disk_basename=$(basename -- "$mender_disk_image") - local mender_disk_filename="${mender_disk_basename%.*}" - local mender_rootfs_basename=${mender_disk_filename}.ext4 - mender_rootfs_image=${output_dir}/$mender_rootfs_basename - # Extract root filesystem ext4 image to use it to generate Mender artifact. # Ext4 disk image will be also verified in acceptance tests. extract_file_from_image $mender_disk_image $prootfs_start \ @@ -682,9 +632,6 @@ do_mender_disk_image_to_artifact() { sudo losetup -d $loopdevice rm -rf ${output_dir}/mnt - # Note: expected Mender Artifact name follows the scheme: - # mender--.mender. - mender_artifact=${output_dir}/${mender_disk_filename}.mender log "\tWriting Mender artifact to: ${mender_artifact}" #Create Mender artifact @@ -835,6 +782,26 @@ logsetup # Some commands expect elevated privileges. sudo true +# Make sure the user's given Mender image name has a correct extension. +# If Mender image name is not provided, then use following syntax: +# mender--.sdimg +if [ -n "${mender_disk_image}" ]; then + mender_disk_filename=$(basename -- "$mender_disk_image") + if [[ $mender_disk_filename =~ \.sdimg$ ]]; then + mender_disk_image=$output_dir/${mender_disk_filename} + else + mender_disk_image=$output_dir/${mender_disk_filename}.sdimg + fi +else + mender_disk_filename="mender-${device_type}-${artifact_name}" + mender_disk_image=$output_dir/${mender_disk_filename}.sdimg +fi + +mender_disk_basename="${mender_disk_filename%.*}" +mender_rootfs_basename=${mender_disk_basename}.ext4 +mender_rootfs_image=${output_dir}/${mender_rootfs_basename} +mender_artifact=${output_dir}/${mender_disk_basename}.mender + case "$1" in raw-disk-image-shrink-rootfs) total=1 @@ -888,4 +855,8 @@ case "$1" in ;; esac +[[ $keep != "-k" ]] && { rm -f $output_dir/boot.vfat \ + $output_dir/cmdline.txt $output_dir/config.txt \ + $output_dir/syslinux.cfg $output_dir/rootfs.img; } + exit 0 diff --git a/mender-convert-functions.sh b/mender-convert-functions.sh index 142257f..9d225ae 100755 --- a/mender-convert-functions.sh +++ b/mender-convert-functions.sh @@ -396,10 +396,9 @@ unmount_partitions() { create_mender_disk() { local lfile=$1 local lsize=$2 - local bs=$(( 1024*1024 )) - local count=$(( ${lsize} / ${bs} )) - dd if=/dev/zero of=${lfile} bs=${bs} count=${count}>> "$build_log" 2>&1 + # Generates a sparse image + dd if=/dev/zero of=${lfile} seek=${lsize} bs=1 count=0 >> "$build_log" 2>&1 } # Takes following arguments: @@ -737,10 +736,8 @@ set_fstab() { # $3 - size (in 512 blocks) extract_file_from_image() { - local cmd="dd if=$1 of=${output_dir}/$4 skip=$2 bs=512 count=$3" - - [ "${4##*.}" == "ext4" ] && { log "\tStoring data in $4 root file system image."; } \ - || { log "\tStoring data in $4 disk image."; } + log "\tStoring data in $4." + local cmd="dd if=$1 of=${output_dir}/$4 skip=$2 bs=512 count=$3 conv=sparse" $(${cmd}>> "$build_log" 2>&1) } diff --git a/qemux86_64-convert-stage-2.sh b/qemux86_64-convert-stage-2.sh index ffad07f..8f94d0e 100755 --- a/qemux86_64-convert-stage-2.sh +++ b/qemux86_64-convert-stage-2.sh @@ -14,7 +14,7 @@ sed -i 's/\b[ ]root=[^ ]*/ root=\/dev\/hda2/' ${output_dir}/grub.cfg # Update Linux kernel command arguments with our custom configuration mcopy -o -i ${output_dir}/boot.vfat -s ${output_dir}/grub.cfg ::EFI/BOOT/grub.cfg -sudo dd if=${output_dir}/boot.vfat of=/dev/mapper/${boot_mapping} bs=1M >> "$build_log" 2>&1 +sudo dd if=${output_dir}/boot.vfat of=/dev/mapper/${boot_mapping} bs=1M conv=sparse >> "$build_log" 2>&1 log "\tDone." diff --git a/rpi-convert-stage-2.sh b/rpi-convert-stage-2.sh index 57f5f4f..4dcb160 100755 --- a/rpi-convert-stage-2.sh +++ b/rpi-convert-stage-2.sh @@ -18,7 +18,7 @@ mcopy -on -i ${output_dir}/boot.vfat -s ::config.txt ${output_dir}/config.txt echo -e '\nenable_uart=1\n' >> ${output_dir}/config.txt mcopy -o -i ${output_dir}/boot.vfat -s ${output_dir}/config.txt ::config.txt -sudo dd if=${output_dir}/boot.vfat of=/dev/mapper/${boot_mapping} bs=1M >> "$build_log" 2>&1 +sudo dd if=${output_dir}/boot.vfat of=/dev/mapper/${boot_mapping} bs=1M conv=sparse >> "$build_log" 2>&1 log "\tDone."