Make docker container independent from mender-convert repository

Changelog: The mender-convert docker image does now contain the mender-convert application and does no longer need companion checkouts. Changelog: All input files passed to docker-mender-convert must be located within the input directory as opposed to anywhere in the mender-convert checkout directory Signed-off-by: Simon Ensslen <simon.ensslen@griesser.ch>
3 years ago · 5fc630e935
7 changed files with 112 additions and 55 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -102,14 +102,17 @@ build:
    - export IMAGE_NAME=$DOCKER_REPOSITORY:pr
    - docker load -i image.tar

+    - mkdir -p input
+    - cd input
    - wget -q ${RASPBIAN_URL}
    - unzip ${RASPBIAN_NAME}.zip
+    - cd ..

    - eval "$(curl https://raw.githubusercontent.com/mendersoftware/mendertesting/master/mender-ci-common.sh)"

  script:
-    - env MENDER_ARTIFACT_NAME=${RASPBIAN_NAME}-mender
-      ./docker-mender-convert --disk-image ${RASPBIAN_NAME}.img
+    - env MENDER_ARTIFACT_NAME=${RASPBIAN_NAME}-mender 
+      ./docker-mender-convert --disk-image input/${RASPBIAN_NAME}.img
      --config configs/${RASPBERRYPI_PLATFORM}_config
      --config configs/images/raspberrypi_raspbian_config

@ -223,27 +226,37 @@ test_acceptance_prebuilt_raspberrypi4:
 test_acceptance_qemux86_64:
  <<: *test_acceptance
  script:
-    - ./scripts/test/run-tests.sh --config versions_override_config --only qemux86_64
+    - mkdir -p input/config
+    - cp versions_override_config input/config/versions_override_config
+    - ./scripts/test/run-tests.sh --config input/config/versions_override_config --only qemux86_64

 test_acceptance_debian_qemux86_64:
  <<: *test_acceptance
  script:
-    - ./scripts/test/run-tests.sh --config versions_override_config --only debian-qemux86_64
+    - mkdir -p input/config
+    - cp versions_override_config input/config/versions_override_config
+    - ./scripts/test/run-tests.sh --config input/config/versions_override_config --only debian-qemux86_64

 test_acceptance_raspberrypi:
  <<: *test_acceptance
  script:
-    - ./scripts/test/run-tests.sh --config versions_override_config --only raspberrypi3
+    - mkdir -p input/config
+    - cp versions_override_config input/config/versions_override_config
+    - ./scripts/test/run-tests.sh --config input/config/versions_override_config --only raspberrypi3

 test_acceptance_beaglebone:
  <<: *test_acceptance
  script:
-    - ./scripts/test/run-tests.sh --config versions_override_config --only beaglebone
+    - mkdir -p input/config
+    - cp versions_override_config input/config/versions_override_config
+    - ./scripts/test/run-tests.sh --config input/config/versions_override_config --only beaglebone

 test_acceptance_ubuntu:
  <<: *test_acceptance
  script:
-    - ./scripts/test/run-tests.sh --config versions_override_config --only ubuntu
+    - mkdir -p input/config
+    - cp versions_override_config input/config/versions_override_config
+    - ./scripts/test/run-tests.sh --config input/config/versions_override_config --only ubuntu

 .template:publish:s3:
  stage: publish
@ -275,6 +288,23 @@ test_acceptance_ubuntu:
          --key ${RASPBIAN_NAME}/arm/${PUBLISH_NAME}
    - done

+.template:publish:docker-image:
+  stage: publish
+  tags:
+    - docker
+  image: docker
+  services:
+    - name: docker:20.10.8-dind
+      alias: docker
+      command: ["--mtu=1440"] # https://gitlab.com/gitlab-com/gl-infra/production/-/issues/5590
+  script:
+    - docker tag $DOCKER_REPOSITORY:pr $DOCKER_REPOSITORY:$MENDER_CONVERT_PUBLISH_VERSION
+    - docker push $DOCKER_REPOSITORY:$MENDER_CONVERT_PUBLISH_VERSION
+    - echo "PUBLISH_IMAGE_DIGEST=$(docker inspect --format='{{index .RepoDigests 0}}' $DOCKER_REPOSITORY:$MENDER_CONVERT_PUBLISH_VERSION)" >> publish.env
+  artifacts:
+    reports:
+      dotenv: publish.env
+
 publish:s3:manual:
  when: manual
  extends: .template:publish:s3
@ -283,3 +313,12 @@ publish:s3:automatic:
  rules:
    - if: '$PUBLISH_MENDER_CONVERT_AUTOMATIC == "true"'
  extends: .template:publish:s3
+
+publish:docker-image:manual:
+  when: manual
+  extends: .template:publish:docker-image
+
+publish:docker-image:automatic:
+  rules:
+    - if: '$PUBLISH_MENDER_CONVERT_AUTOMATIC == "true"'
+  extends: .template:publish:docker-image
--- a/14
+++ b/14
@ -72,5 +72,15 @@ RUN wget -q -O /usr/bin/mender-artifact https://downloads.mender.io/mender-artif

 WORKDIR /

-COPY docker-entrypoint.sh /usr/local/bin/
-ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]
+COPY . /mender-convert
+
+RUN mkdir -p /mender-convert/work
+RUN mkdir -p /mender-convert/input
+RUN mkdir -p /mender-convert/deploy
+RUN mkdir -p /mender-convert/logs
+
+VOLUME ["/mender-convert/input"]
+VOLUME ["/mender-convert/deploy"]
+VOLUME ["/mender-convert/logs"]
+
+ENTRYPOINT ["/mender-convert/docker-entrypoint.sh"]
--- a/README.md
+++ b/README.md
@ -152,15 +152,22 @@ necessary dependencies.
 Run mender-convert from inside the container with your desired options, e.g.

 ```bash
+mkdir -p input/image
+cp $PATH_TO_DISK_IMAGE/$INPUT_DISK_IMAGE input/image
+
+mkdir -p input/config
+cp $PATH_TO_MY_OWN_CONFIG/$CUSTOM_CONFIG input/config
+
 MENDER_ARTIFACT_NAME=release-1 ./docker-mender-convert \
-   --disk-image input/$INPUT_DISK_IMAGE \
+   --disk-image input/image/$INPUT_DISK_IMAGE \
   --config configs/raspberrypi3_config \
+   --config input/config/$CUSTOM_CONFIG \
   --overlay rootfs_overlay_demo/
 ```

 Conversion will take 10-30 minutes, depending on image size and resources
-available. You can watch `WORKDIR/convert.log` for progress and diagnostics
-information.
+available. You can watch `log/convert.log.XXXXX` for progress and diagnostics
+information. The exact log file path is printed before the conversion starts.

 After it finishes, you can find your images in the `deploy` directory on your
 host machine!
@ -188,7 +195,7 @@ MENDER_ARTIFACT_NAME=release-1 ./mender-convert \
 **NOTE!** You will be prompted to enter `sudo` password during the conversion
 process. This is required to be able to loopback mount images and for modifying
 them. Our recommendation is to use the provided Docker container, to run the
-tool in a isolated environment.
+tool in an isolated environment.

 -------------------------------------------------------------------------------

--- a/4
+++ b/4
@ -16,6 +16,6 @@

 set -e

-IMAGE_NAME=${IMAGE_NAME:-mender-convert}
+IMAGE_NAME=${IMAGE_NAME:-"mendersoftware/mender-convert"}

-eval docker build . -t ${IMAGE_NAME} "$@"
+eval docker build . -t "${IMAGE_NAME}" "$*"
--- a/docker-entrypoint.sh
+++ b/docker-entrypoint.sh
@ -20,9 +20,9 @@ set -e

 cd /mender-convert

-echo "Running mender-convert "$@""
+echo "Running mender-convert $*"

 ./mender-convert "$@"

-# Set owner and group to same as launch directory.
-[ -d deploy ] && chown -R --reference=. deploy
+# Set owner and group to same as input directory.
+[ -d deploy ] && chown -R --reference=input deploy
--- a/35
+++ b/35
@ -16,38 +16,35 @@

 set -e

-. modules/git.sh
+IMAGE_NAME=${IMAGE_NAME:-"mendersoftware/mender-convert"}
+DEPLOY_DIRECTORY=${DEPLOY_DIRECTORY:-"$(pwd)/deploy"}
+INPUT_DIRECTORY=${INPUT_DIRECTORY:-"$(pwd)/input"}
+LOGS_DIRECTORY=${LOGS_DIRECTORY:-"$(pwd)/logs"}

-IMAGE_NAME=${IMAGE_NAME:-mender-convert}
+mkdir -p "$DEPLOY_DIRECTORY"
+mkdir -p "$LOGS_DIRECTORY"

-MENDER_CONVERT_DIR="$(pwd)"
+# Create a unique log file so we can run multiple containers in parallel
+LOG_FILE="convert.log.$(date +%s)-$$"

-#Pass in version in-case we are added as a git submodule
-MENDER_CONVERT_VERSION=$(git_mender_convert_version)
-
-# Create a unique work directory so we can run multiple containers in parallel
-WORK_DIR=$(mktemp -d $(pwd)/work.XXXX)
-WORK_SUFFIX=$(echo $WORK_DIR | awk -F. '{print $NF}')
-LOG_FILE="convert.log.${WORK_SUFFIX}"
+echo "using log file at: ${LOGS_DIRECTORY}/${LOG_FILE}"

 set +e
 docker run \
  --rm \
-  -v $MENDER_CONVERT_DIR:/mender-convert \
-  -v $WORK_DIR:/mender-convert/work \
+  -v "$INPUT_DIRECTORY":/mender-convert/input \
+  -v "$LOGS_DIRECTORY":/mender-convert/logs \
+  -v "$DEPLOY_DIRECTORY":/mender-convert/deploy \
  --privileged=true \
  --cap-add=SYS_MODULE \
  -v /dev:/dev \
  -v /lib/modules:/lib/modules:ro \
-  --env MENDER_ARTIFACT_NAME=${MENDER_ARTIFACT_NAME} \
-  --env MENDER_CONVERT_VERSION=${MENDER_CONVERT_VERSION} \
-  --env MENDER_CONVERT_LOG_FILE=logs/${LOG_FILE} \
-  $IMAGE_NAME "$@"
+  --env MENDER_ARTIFACT_NAME="${MENDER_ARTIFACT_NAME}" \
+  --env MENDER_CONVERT_LOG_FILE=logs/"${LOG_FILE}" \
+  "$IMAGE_NAME" "$@"

 exit_code=$?

-[ ${exit_code} -eq 0 ] && rmdir ${WORK_DIR}
-
-echo "Log file available at: logs/${LOG_FILE}"
+echo "Log file available at: ${LOGS_DIRECTORY}/${LOG_FILE}"

 exit ${exit_code}
--- a/scripts/test/run-tests.sh
+++ b/scripts/test/run-tests.sh
@ -89,11 +89,13 @@ if [ -n "$PREBUILT_IMAGE" ]; then

 else
  if [ "$TEST_ALL" == "1" -o "$TEST_PLATFORM" == "qemux86_64" ]; then
-    wget --progress=dot:giga -N ${UBUNTU_IMAGE_URL} -P input/
+    wget --progress=dot:giga -N ${UBUNTU_IMAGE_URL} -P input/image/
+    mkdir -p input/tests
+    cp -r "tests/ssh-public-key-overlay" "input/tests/ssh-public-key-overlay"
    convert_and_test "qemux86_64" \
                     "release-1" \
-                     "input/Ubuntu-Focal-x86-64.img.gz" \
-                     "--overlay tests/ssh-public-key-overlay" \
+                     "input/image/Ubuntu-Focal-x86-64.img.gz" \
+                     "--overlay input/tests/ssh-public-key-overlay" \
                     "--config configs/qemux86-64_config $EXTRA_CONFIG" \
                     || test_result=$?

@ -101,9 +103,9 @@ else
    echo >&2 "Running the uncompressed test"
    echo >&2 "----------------------------------------"
    rm -rf deploy
-    gunzip --force "input/Ubuntu-Focal-x86-64.img.gz"
+    gunzip --force "input/image/Ubuntu-Focal-x86-64.img.gz"
    run_convert "release-2" \
-                "input/Ubuntu-Focal-x86-64.img" \
+                "input/image/Ubuntu-Focal-x86-64.img" \
                "--config configs/qemux86-64_config $EXTRA_CONFIG" || test_result=$?
    ret=0
    test -f deploy/Ubuntu-Focal-x86-64-qemux86_64-mender.img || ret=$?
@ -111,11 +113,11 @@ else
  fi

  if [ "$TEST_ALL" == "1" -o "$TEST_PLATFORM" == "raspberrypi3" ]; then
-    wget --progress=dot:giga -N ${RASPBIAN_IMAGE_URL} -P input/
+    wget --progress=dot:giga -N ${RASPBIAN_IMAGE_URL} -P input/image/
    RASPBIAN_IMAGE="${RASPBIAN_IMAGE_URL##*/}"
    convert_and_test "raspberrypi3" \
                     "release-1" \
-                     "input/${RASPBIAN_IMAGE}" \
+                     "input/image/${RASPBIAN_IMAGE}" \
                     "--config configs/raspberrypi3_config $EXTRA_CONFIG" \
                     -- \
                     "-k" "'not test_update'" \
@ -123,27 +125,27 @@ else
  fi

  if [ "$TEST_ALL" == "1" -o "$TEST_PLATFORM" == "beaglebone" ]; then
-    wget --progress=dot:giga -N ${BBB_DEBIAN_SDCARD_IMAGE_URL} -P input/
+    wget --progress=dot:giga -N ${BBB_DEBIAN_SDCARD_IMAGE_URL} -P input/image/
    BBB_DEBIAN_SDCARD_IMAGE_COMPRESSED="${BBB_DEBIAN_SDCARD_IMAGE_URL##*/}"
    BBB_DEBIAN_SDCARD_IMAGE_UNCOMPRESSED="${BBB_DEBIAN_SDCARD_IMAGE_COMPRESSED%.xz}"
    # Convert uncompressed images to speed up this job
-    unxz --force "input/${BBB_DEBIAN_SDCARD_IMAGE_COMPRESSED}"
+    unxz --force "input/image/${BBB_DEBIAN_SDCARD_IMAGE_COMPRESSED}"
    convert_and_test "beaglebone-sdcard" \
                     "release-1" \
-                     "input/${BBB_DEBIAN_SDCARD_IMAGE_UNCOMPRESSED}" \
+                     "input/image/${BBB_DEBIAN_SDCARD_IMAGE_UNCOMPRESSED}" \
                     "--config configs/beaglebone_black_debian_sdcard_config $EXTRA_CONFIG" \
                     -- \
                     "-k" "'not test_update'" \
                     || test_result=$?

    rm -rf deploy
-    wget --progress=dot:giga -N ${BBB_DEBIAN_EMMC_IMAGE_URL} -P input/
+    wget --progress=dot:giga -N ${BBB_DEBIAN_EMMC_IMAGE_URL} -P input/image/
    BBB_DEBIAN_EMMC_IMAGE_COMPRESSED="${BBB_DEBIAN_EMMC_IMAGE_URL##*/}"
    BBB_DEBIAN_EMMC_IMAGE_UNCOMPRESSED="${BBB_DEBIAN_EMMC_IMAGE_COMPRESSED%.xz}"
-    unxz --force "input/${BBB_DEBIAN_EMMC_IMAGE_COMPRESSED}"
+    unxz --force "input/image/${BBB_DEBIAN_EMMC_IMAGE_COMPRESSED}"
    convert_and_test "beaglebone-emmc" \
                     "release-1" \
-                     "input/${BBB_DEBIAN_EMMC_IMAGE_UNCOMPRESSED}" \
+                     "input/image/${BBB_DEBIAN_EMMC_IMAGE_UNCOMPRESSED}" \
                     "--config configs/beaglebone_black_debian_emmc_config $EXTRA_CONFIG" \
                     -- \
                     "-k" "'not test_update'" \
@ -151,11 +153,11 @@ else
  fi

  if [ "$TEST_ALL" == "1" -o "$TEST_PLATFORM" == "ubuntu" ]; then
-    wget --progress=dot:giga -N ${UBUNTU_SERVER_RPI_IMAGE_URL} -P input/
+    wget --progress=dot:giga -N ${UBUNTU_SERVER_RPI_IMAGE_URL} -P input/image/
    UBUNTU_SERVER_RPI_IMAGE_COMPRESSED="${UBUNTU_SERVER_RPI_IMAGE_URL##*/}"
    convert_and_test "raspberrypi3" \
                     "release-1" \
-                     "input/${UBUNTU_SERVER_RPI_IMAGE_COMPRESSED}" \
+                     "input/image/${UBUNTU_SERVER_RPI_IMAGE_COMPRESSED}" \
                     "--config configs/raspberrypi3_config $EXTRA_CONFIG" \
                     -- \
                     "-k" "'not test_update'" \
@ -163,11 +165,13 @@ else
  fi

  if [ "$TEST_ALL" == "1" -o "$TEST_PLATFORM" == "debian-qemux86_64" ]; then
-    wget --progress=dot:giga -N ${DEBIAN_IMAGE_URL} -P input/
+    wget --progress=dot:giga -N ${DEBIAN_IMAGE_URL} -P input/image/
+    mkdir -p input/tests
+    cp -r "tests/ssh-public-key-overlay" "input/tests/ssh-public-key-overlay"
    convert_and_test "qemux86_64" \
                     "release-1" \
-                     "input/Debian-11-x86-64.img.gz" \
-                     "--overlay tests/ssh-public-key-overlay" \
+                     "input/image/Debian-11-x86-64.img.gz" \
+                     "--overlay input/tests/ssh-public-key-overlay" \
                     "--config configs/debian-qemux86-64_config $EXTRA_CONFIG" \
                     || test_result=$?
  fi