Merge pull request #440 from mendersoftware/docker_container

Testing #433 with solved merge conflicts
3 years ago · b6709f3e4c
7 changed files with 130 additions and 54 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -102,14 +102,17 @@ build:
    - export IMAGE_NAME=$DOCKER_REPOSITORY:pr
    - docker load -i image.tar

+    - mkdir -p input
+    - cd input
    - wget -q ${RASPBIAN_URL}
    - unzip ${RASPBIAN_NAME}.zip
+    - cd ..

    - eval "$(curl https://raw.githubusercontent.com/mendersoftware/mendertesting/master/mender-ci-common.sh)"

  script:
-    - env MENDER_ARTIFACT_NAME=${RASPBIAN_NAME}-mender
-      ./docker-mender-convert --disk-image ${RASPBIAN_NAME}.img
+    - env MENDER_ARTIFACT_NAME=${RASPBIAN_NAME}-mender 
+      ./docker-mender-convert --disk-image input/${RASPBIAN_NAME}.img
      --config configs/${RASPBERRYPI_PLATFORM}_config
      --config configs/images/raspberrypi_raspbian_config

@ -223,27 +226,37 @@ test_acceptance_prebuilt_raspberrypi4:
 test_acceptance_qemux86_64:
  <<: *test_acceptance
  script:
-    - ./scripts/test/run-tests.sh --config versions_override_config --only ubuntu-qemux86-64
+    - mkdir -p input/config
+    - cp versions_override_config input/config/versions_override_config
+    - ./scripts/test/run-tests.sh --config input/config/versions_override_config --only ubuntu-qemux86-64

 test_acceptance_debian_qemux86_64:
  <<: *test_acceptance
  script:
-    - ./scripts/test/run-tests.sh --config versions_override_config --only debian-qemux86-64
+    - mkdir -p input/config
+    - cp versions_override_config input/config/versions_override_config
+    - ./scripts/test/run-tests.sh --config input/config/versions_override_config --only debian-qemux86-64

 test_acceptance_raspberrypi:
  <<: *test_acceptance
  script:
-    - ./scripts/test/run-tests.sh --config versions_override_config --only raspberrypi3
+    - mkdir -p input/config
+    - cp versions_override_config input/config/versions_override_config
+    - ./scripts/test/run-tests.sh --config input/config/versions_override_config --only raspberrypi3

 test_acceptance_beaglebone:
  <<: *test_acceptance
  script:
-    - ./scripts/test/run-tests.sh --config versions_override_config --only beaglebone
+    - mkdir -p input/config
+    - cp versions_override_config input/config/versions_override_config
+    - ./scripts/test/run-tests.sh --config input/config/versions_override_config --only beaglebone

 test_acceptance_ubuntu:
  <<: *test_acceptance
  script:
-    - ./scripts/test/run-tests.sh --config versions_override_config --only ubuntu
+    - mkdir -p input/config
+    - cp versions_override_config input/config/versions_override_config
+    - ./scripts/test/run-tests.sh --config input/config/versions_override_config --only ubuntu

 .template:publish:s3:
  stage: publish
@ -275,6 +288,28 @@ test_acceptance_ubuntu:
          --key ${RASPBIAN_NAME}/arm/${PUBLISH_NAME}
    - done

+.template:publish:docker-image:
+  stage: publish
+  tags:
+    - docker
+  image: docker
+  services:
+    - name: docker:20.10.8-dind
+      alias: docker
+      command: ["--mtu=1440"] # https://gitlab.com/gitlab-com/gl-infra/production/-/issues/5590
+  before_script:
+    - docker login -u $DOCKER_HUB_USERNAME -p $DOCKER_HUB_PASSWORD
+    - docker load -i image.tar
+  script:
+    - IMAGE_VERSION=${MENDER_CONVERT_PUBLISH_VERSION:-${CI_COMMIT_REF_NAME}}
+
+    - docker tag $DOCKER_REPOSITORY:pr $DOCKER_REPOSITORY:$IMAGE_VERSION
+    - docker push $DOCKER_REPOSITORY:$IMAGE_VERSION
+    - echo "PUBLISH_IMAGE_DIGEST=$(docker inspect --format='{{index .RepoDigests 0}}' $DOCKER_REPOSITORY:$IMAGE_VERSION)" >> publish.env
+  artifacts:
+    reports:
+      dotenv: publish.env
+
 publish:s3:manual:
  when: manual
  extends: .template:publish:s3
@ -283,3 +318,12 @@ publish:s3:automatic:
  rules:
    - if: '$PUBLISH_MENDER_CONVERT_AUTOMATIC == "true"'
  extends: .template:publish:s3
+
+publish:docker-image:manual:
+  when: manual
+  extends: .template:publish:docker-image
+
+publish:docker-image:automatic:
+  rules:
+    - if: '$CI_COMMIT_BRANCH =~ /^(master|staging|production|feature-.+|[0-9]+\.[0-9]+\.([0-9]+|x))$/'
+  extends: .template:publish:docker-image
--- a/14
+++ b/14
@ -74,5 +74,15 @@ RUN wget -q -O /usr/bin/mender-artifact https://downloads.mender.io/mender-artif

 WORKDIR /

-COPY docker-entrypoint.sh /usr/local/bin/
-ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]
+COPY . /mender-convert
+
+RUN mkdir -p /mender-convert/work
+RUN mkdir -p /mender-convert/input
+RUN mkdir -p /mender-convert/deploy
+RUN mkdir -p /mender-convert/logs
+
+VOLUME ["/mender-convert/input"]
+VOLUME ["/mender-convert/deploy"]
+VOLUME ["/mender-convert/logs"]
+
+ENTRYPOINT ["/mender-convert/docker-entrypoint.sh"]
--- a/README.md
+++ b/README.md
@ -152,15 +152,22 @@ necessary dependencies.
 Run mender-convert from inside the container with your desired options, e.g.

 ```bash
+mkdir -p input/image
+cp $PATH_TO_DISK_IMAGE/$INPUT_DISK_IMAGE input/image
+
+mkdir -p input/config
+cp $PATH_TO_MY_OWN_CONFIG/$CUSTOM_CONFIG input/config
+
 MENDER_ARTIFACT_NAME=release-1 ./docker-mender-convert \
-   --disk-image input/$INPUT_DISK_IMAGE \
+   --disk-image input/image/$INPUT_DISK_IMAGE \
   --config configs/raspberrypi3_config \
+   --config input/config/$CUSTOM_CONFIG \
   --overlay rootfs_overlay_demo/
 ```

 Conversion will take 10-30 minutes, depending on image size and resources
-available. You can watch `WORKDIR/convert.log` for progress and diagnostics
-information.
+available. You can watch `log/convert.log.XXXXX` for progress and diagnostics
+information. The exact log file path is printed before the conversion starts.

 After it finishes, you can find your images in the `deploy` directory on your
 host machine!
@ -188,7 +195,7 @@ MENDER_ARTIFACT_NAME=release-1 ./mender-convert \
 **NOTE!** You will be prompted to enter `sudo` password during the conversion
 process. This is required to be able to loopback mount images and for modifying
 them. Our recommendation is to use the provided Docker container, to run the
-tool in a isolated environment.
+tool in an isolated environment.

 -------------------------------------------------------------------------------

--- a/11
+++ b/11
@ -16,6 +16,13 @@

 set -e

-IMAGE_NAME=${IMAGE_NAME:-mender-convert}
+GIT_PROVIDED_TAG_NAME="$(git describe --exact HEAD 2>/dev/null || \
+    git for-each-ref "refs/heads/*" --format '%(refname:short)' --points-at HEAD 2>/dev/null)"

-eval docker build . -t ${IMAGE_NAME} "$@"
+if [ -z "$IMAGE_NAME" -a -z "$GIT_PROVIDED_TAG_NAME" ]; then
+  echo "Could not deduce mendersoftware/mender-convert container version from currently checked out commit. Using latest." 1>&2
+fi
+
+IMAGE_NAME=${IMAGE_NAME:-"mendersoftware/mender-convert${GIT_PROVIDED_TAG_NAME:+:$GIT_PROVIDED_TAG_NAME}"}
+
+eval docker build . -t "${IMAGE_NAME}" "$*"
--- a/docker-entrypoint.sh
+++ b/docker-entrypoint.sh
@ -20,9 +20,9 @@ set -e

 cd /mender-convert

-echo "Running mender-convert "$@""
+echo "Running mender-convert $*"

 ./mender-convert "$@"

-# Set owner and group to same as launch directory.
-[ -d deploy ] && chown -R --reference=. deploy
+# Set owner and group to same as input directory.
+[ -d deploy ] && chown -R --reference=input deploy
--- a/40
+++ b/40
@ -16,38 +16,42 @@

 set -e

-. modules/git.sh
+GIT_PROVIDED_TAG_NAME="$(git describe --exact HEAD 2>/dev/null || \
+    git for-each-ref "refs/heads/*" --format '%(refname:short)' --points-at HEAD 2>/dev/null)"

-IMAGE_NAME=${IMAGE_NAME:-mender-convert}
+if [ -z "$IMAGE_NAME" -a -z "$GIT_PROVIDED_TAG_NAME" ]; then
+  echo "Could not deduce mendersoftware/mender-convert container version from currently checked out commit. Using latest." 1>&2
+fi

-MENDER_CONVERT_DIR="$(pwd)"
+IMAGE_NAME=${IMAGE_NAME:-"mendersoftware/mender-convert${GIT_PROVIDED_TAG_NAME:+:$GIT_PROVIDED_TAG_NAME}"}
+DEPLOY_DIRECTORY=${DEPLOY_DIRECTORY:-"$(pwd)/deploy"}
+INPUT_DIRECTORY=${INPUT_DIRECTORY:-"$(pwd)/input"}
+LOGS_DIRECTORY=${LOGS_DIRECTORY:-"$(pwd)/logs"}

-#Pass in version in-case we are added as a git submodule
-MENDER_CONVERT_VERSION=$(git_mender_convert_version)
+mkdir -p "$DEPLOY_DIRECTORY"
+mkdir -p "$LOGS_DIRECTORY"

-# Create a unique work directory so we can run multiple containers in parallel
-WORK_DIR=$(mktemp -d $(pwd)/work.XXXX)
-WORK_SUFFIX=$(echo $WORK_DIR | awk -F. '{print $NF}')
-LOG_FILE="convert.log.${WORK_SUFFIX}"
+# Create a unique log file so we can run multiple containers in parallel
+LOG_FILE="convert.log.$(date +%s)-$$"
+
+echo "using log file at: ${LOGS_DIRECTORY}/${LOG_FILE}"

 set +e
 docker run \
  --rm \
-  -v $MENDER_CONVERT_DIR:/mender-convert \
-  -v $WORK_DIR:/mender-convert/work \
+  -v "$INPUT_DIRECTORY":/mender-convert/input \
+  -v "$LOGS_DIRECTORY":/mender-convert/logs \
+  -v "$DEPLOY_DIRECTORY":/mender-convert/deploy \
  --privileged=true \
  --cap-add=SYS_MODULE \
  -v /dev:/dev \
  -v /lib/modules:/lib/modules:ro \
-  --env MENDER_ARTIFACT_NAME=${MENDER_ARTIFACT_NAME} \
-  --env MENDER_CONVERT_VERSION=${MENDER_CONVERT_VERSION} \
-  --env MENDER_CONVERT_LOG_FILE=logs/${LOG_FILE} \
-  $IMAGE_NAME "$@"
+  --env MENDER_ARTIFACT_NAME="${MENDER_ARTIFACT_NAME}" \
+  --env MENDER_CONVERT_LOG_FILE=logs/"${LOG_FILE}" \
+  "$IMAGE_NAME" "$@"

 exit_code=$?

-[ ${exit_code} -eq 0 ] && rmdir ${WORK_DIR}
-
-echo "Log file available at: logs/${LOG_FILE}"
+echo "Log file available at: ${LOGS_DIRECTORY}/${LOG_FILE}"

 exit ${exit_code}
--- a/scripts/test/run-tests.sh
+++ b/scripts/test/run-tests.sh
@ -88,11 +88,13 @@ if [ -n "$PREBUILT_IMAGE" ]; then

 else
  if [ "$TEST_ALL" == "1" -o "$TEST_PLATFORM" == "ubuntu-qemux86-64" ]; then
-    wget --progress=dot:giga -N ${UBUNTU_IMAGE_URL} -P input/
+    wget --progress=dot:giga -N ${UBUNTU_IMAGE_URL} -P input/image/
+    mkdir -p input/tests
+    cp -r "tests/ssh-public-key-overlay" "input/tests/ssh-public-key-overlay"
    convert_and_test "qemux86-64" \
                     "release-1" \
-                     "input/Ubuntu-Focal-x86-64.img.gz" \
-                     "--overlay tests/ssh-public-key-overlay" \
+                     "input/image/Ubuntu-Focal-x86-64.img.gz" \
+                     "--overlay input/tests/ssh-public-key-overlay" \
                     "--config configs/ubuntu-qemux86-64_config $EXTRA_CONFIG" \
                     || test_result=$?

@ -100,9 +102,9 @@ else
    echo >&2 "Running the uncompressed test"
    echo >&2 "----------------------------------------"
    rm -rf deploy
-    gunzip --force "input/Ubuntu-Focal-x86-64.img.gz"
+    gunzip --force "input/image/Ubuntu-Focal-x86-64.img.gz"
    run_convert "release-2" \
-                "input/Ubuntu-Focal-x86-64.img" \
+                "input/image/Ubuntu-Focal-x86-64.img" \
                "--config configs/ubuntu-qemux86-64_config $EXTRA_CONFIG" || test_result=$?
    ret=0
    test -f deploy/Ubuntu-Focal-x86-64-qemux86-64-mender.img || ret=$?
@ -110,55 +112,57 @@ else
  fi

  if [ "$TEST_ALL" == "1" -o "$TEST_PLATFORM" == "raspberrypi3" ]; then
-    wget --progress=dot:giga -N ${RASPBIAN_IMAGE_URL} -P input/
+    wget --progress=dot:giga -N ${RASPBIAN_IMAGE_URL} -P input/image/
    RASPBIAN_IMAGE="${RASPBIAN_IMAGE_URL##*/}"
    convert_and_test "raspberrypi3" \
                     "release-1" \
-                     "input/${RASPBIAN_IMAGE}" \
+                     "input/image/${RASPBIAN_IMAGE}" \
                     "--config configs/raspberrypi3_config $EXTRA_CONFIG" \
                     || test_result=$?
  fi

  if [ "$TEST_ALL" == "1" -o "$TEST_PLATFORM" == "beaglebone" ]; then
-    wget --progress=dot:giga -N ${BBB_DEBIAN_SDCARD_IMAGE_URL} -P input/
+    wget --progress=dot:giga -N ${BBB_DEBIAN_SDCARD_IMAGE_URL} -P input/image/
    BBB_DEBIAN_SDCARD_IMAGE_COMPRESSED="${BBB_DEBIAN_SDCARD_IMAGE_URL##*/}"
    BBB_DEBIAN_SDCARD_IMAGE_UNCOMPRESSED="${BBB_DEBIAN_SDCARD_IMAGE_COMPRESSED%.xz}"
    # Convert uncompressed images to speed up this job
-    unxz --force "input/${BBB_DEBIAN_SDCARD_IMAGE_COMPRESSED}"
+    unxz --force "input/image/${BBB_DEBIAN_SDCARD_IMAGE_COMPRESSED}"
    convert_and_test "beaglebone-sdcard" \
                     "release-1" \
-                     "input/${BBB_DEBIAN_SDCARD_IMAGE_UNCOMPRESSED}" \
+                     "input/image/${BBB_DEBIAN_SDCARD_IMAGE_UNCOMPRESSED}" \
                     "--config configs/beaglebone_black_debian_sdcard_config $EXTRA_CONFIG" \
                     || test_result=$?

    rm -rf deploy
-    wget --progress=dot:giga -N ${BBB_DEBIAN_EMMC_IMAGE_URL} -P input/
+    wget --progress=dot:giga -N ${BBB_DEBIAN_EMMC_IMAGE_URL} -P input/image/
    BBB_DEBIAN_EMMC_IMAGE_COMPRESSED="${BBB_DEBIAN_EMMC_IMAGE_URL##*/}"
    BBB_DEBIAN_EMMC_IMAGE_UNCOMPRESSED="${BBB_DEBIAN_EMMC_IMAGE_COMPRESSED%.xz}"
-    unxz --force "input/${BBB_DEBIAN_EMMC_IMAGE_COMPRESSED}"
+    unxz --force "input/image/${BBB_DEBIAN_EMMC_IMAGE_COMPRESSED}"
    convert_and_test "beaglebone-emmc" \
                     "release-1" \
-                     "input/${BBB_DEBIAN_EMMC_IMAGE_UNCOMPRESSED}" \
+                     "input/image/${BBB_DEBIAN_EMMC_IMAGE_UNCOMPRESSED}" \
                     "--config configs/beaglebone_black_debian_emmc_config $EXTRA_CONFIG" \
                     || test_result=$?
  fi

  if [ "$TEST_ALL" == "1" -o "$TEST_PLATFORM" == "ubuntu" ]; then
-    wget --progress=dot:giga -N ${UBUNTU_SERVER_RPI_IMAGE_URL} -P input/
+    wget --progress=dot:giga -N ${UBUNTU_SERVER_RPI_IMAGE_URL} -P input/image/
    UBUNTU_SERVER_RPI_IMAGE_COMPRESSED="${UBUNTU_SERVER_RPI_IMAGE_URL##*/}"
    convert_and_test "raspberrypi3" \
                     "release-1" \
-                     "input/${UBUNTU_SERVER_RPI_IMAGE_COMPRESSED}" \
+                     "input/image/${UBUNTU_SERVER_RPI_IMAGE_COMPRESSED}" \
                     "--config configs/raspberrypi3_config $EXTRA_CONFIG" \
                     || test_result=$?
  fi

  if [ "$TEST_ALL" == "1" -o "$TEST_PLATFORM" == "debian-qemux86-64" ]; then
-    wget --progress=dot:giga -N ${DEBIAN_IMAGE_URL} -P input/
+    wget --progress=dot:giga -N ${DEBIAN_IMAGE_URL} -P input/image/
+    mkdir -p input/tests
+    cp -r "tests/ssh-public-key-overlay" "input/tests/ssh-public-key-overlay"
    convert_and_test "qemux86-64" \
                     "release-1" \
-                     "input/Debian-11-x86-64.img.gz" \
-                     "--overlay tests/ssh-public-key-overlay" \
+                     "input/image/Debian-11-x86-64.img.gz" \
+                     "--overlay input/tests/ssh-public-key-overlay" \
                     "--config configs/debian-qemux86-64_config $EXTRA_CONFIG" \
                     || test_result=$?
  fi