From bbceda77667f7dba9158ca66c1d1124a9df139f8 Mon Sep 17 00:00:00 2001 From: Mayank Date: Wed, 1 Jul 2020 23:13:32 +0530 Subject: [PATCH 001/104] Test run image pull --- stage0/00-configure-apt/00-run.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/stage0/00-configure-apt/00-run.sh b/stage0/00-configure-apt/00-run.sh index 9d21ffb..8d143ed 100755 --- a/stage0/00-configure-apt/00-run.sh +++ b/stage0/00-configure-apt/00-run.sh @@ -15,3 +15,10 @@ on_chroot << EOF apt-get update apt-get dist-upgrade -y EOF + +on_chroot << EOF +wget -q "https://raw.githubusercontent.com/moby/moby/master/contrib/download-frozen-image-v2.sh" +chmod +x download-frozen-image-v2.sh +./download-frozen-image-v2.sh docker-images getumbrel/dashboard:latest +tar -cC 'target_dir' . | docker load +EOF \ No newline at end of file From f1259cceacf290bf39de9f7d7cdac809c9faf411 Mon Sep 17 00:00:00 2001 From: Mayank Date: Wed, 1 Jul 2020 23:50:15 +0530 Subject: [PATCH 002/104] Move bundling to stage 2 --- stage0/00-configure-apt/00-run.sh | 7 ------- stage2/05-docker-compose/01-run.sh | 8 ++++++++ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/stage0/00-configure-apt/00-run.sh b/stage0/00-configure-apt/00-run.sh index 8d143ed..f978814 100755 --- a/stage0/00-configure-apt/00-run.sh +++ b/stage0/00-configure-apt/00-run.sh @@ -14,11 +14,4 @@ on_chroot apt-key add - < files/raspberrypi.gpg.key on_chroot << EOF apt-get update apt-get dist-upgrade -y -EOF - -on_chroot << EOF -wget -q "https://raw.githubusercontent.com/moby/moby/master/contrib/download-frozen-image-v2.sh" -chmod +x download-frozen-image-v2.sh -./download-frozen-image-v2.sh docker-images getumbrel/dashboard:latest -tar -cC 'target_dir' . | docker load EOF \ No newline at end of file diff --git a/stage2/05-docker-compose/01-run.sh b/stage2/05-docker-compose/01-run.sh index 0e6b389..5d99a2c 100755 --- a/stage2/05-docker-compose/01-run.sh +++ b/stage2/05-docker-compose/01-run.sh @@ -30,3 +30,11 @@ cp files/compose-service ${ROOTFS_DIR}/etc/init.d/umbrelbox echo "Docker stuff installed!" + +echo "Downloading images from Docker Hub" +on_chroot << EOF +wget -q "https://raw.githubusercontent.com/moby/moby/master/contrib/download-frozen-image-v2.sh" +chmod +x download-frozen-image-v2.sh +./download-frozen-image-v2.sh docker-images getumbrel/middleware:latest +tar -cC 'target_dir' . | docker load +EOF \ No newline at end of file From 8ddc02e02643ad4a1825318e5ed4712d737e5b6e Mon Sep 17 00:00:00 2001 From: Mayank Date: Thu, 2 Jul 2020 00:01:25 +0530 Subject: [PATCH 003/104] Install dependencies for download script --- stage2/05-docker-compose/00-packages | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/stage2/05-docker-compose/00-packages b/stage2/05-docker-compose/00-packages index 0899c29..ebfe8dc 100644 --- a/stage2/05-docker-compose/00-packages +++ b/stage2/05-docker-compose/00-packages @@ -1 +1,4 @@ -git \ No newline at end of file +git +curl +jq +go \ No newline at end of file From 26a15e01c81be9064ab5e9e43cca8e9263b6d226 Mon Sep 17 00:00:00 2001 From: Mayank Date: Thu, 2 Jul 2020 02:18:59 +0530 Subject: [PATCH 004/104] Install golang --- stage2/05-docker-compose/00-packages | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stage2/05-docker-compose/00-packages b/stage2/05-docker-compose/00-packages index ebfe8dc..a6a85b7 100644 --- a/stage2/05-docker-compose/00-packages +++ b/stage2/05-docker-compose/00-packages @@ -1,4 +1,4 @@ git curl jq -go \ No newline at end of file +golang \ No newline at end of file From 91a5b663056c085b47f9eb2354374e0eb9c660ce Mon Sep 17 00:00:00 2001 From: Mayank Date: Thu, 2 Jul 2020 20:08:18 +0530 Subject: [PATCH 005/104] Download images on runner --- depends | 1 + stage0/00-configure-apt/00-run.sh | 23 +++++++++++++++++++++++ stage2/05-docker-compose/00-packages | 5 +---- stage2/05-docker-compose/01-run.sh | 10 +--------- 4 files changed, 26 insertions(+), 13 deletions(-) diff --git a/depends b/depends index a86bc82..87cf16f 100644 --- a/depends +++ b/depends @@ -17,3 +17,4 @@ file git lsmod:kmod bc +jq \ No newline at end of file diff --git a/stage0/00-configure-apt/00-run.sh b/stage0/00-configure-apt/00-run.sh index f978814..8af1031 100755 --- a/stage0/00-configure-apt/00-run.sh +++ b/stage0/00-configure-apt/00-run.sh @@ -14,4 +14,27 @@ on_chroot apt-key add - < files/raspberrypi.gpg.key on_chroot << EOF apt-get update apt-get dist-upgrade -y +EOF + +echo "Installing Go" +cd ~ +curl -O "https://golang.org/dl/go1.14.4.linux-amd64.tar.gz" +tar xvf go1.14.4.linux-amd64.tar.gz +sudo chown -R root:root ./go +sudo mv go /usr/local +export GOPATH=$HOME/work +export PATH=$PATH:/usr/local/go/bin:$GOPATH/bin + +go version +echo "Go installed, now downloading images" + +wget -q "https://raw.githubusercontent.com/moby/moby/master/contrib/download-frozen-image-v2.sh" +chmod +x download-frozen-image-v2.sh +./download-frozen-image-v2.sh docker-images getumbrel/middleware:latest +ls docker-images/ +cp -avr docker-images/ ${ROOTFS_DIR}/home/${FIRST_USER_NAME}/docker-images + +on_chroot << EOF +ls /home/${FIRST_USER_NAME}/ +ls /home/${FIRST_USER_NAME}/docker-images EOF \ No newline at end of file diff --git a/stage2/05-docker-compose/00-packages b/stage2/05-docker-compose/00-packages index a6a85b7..0899c29 100644 --- a/stage2/05-docker-compose/00-packages +++ b/stage2/05-docker-compose/00-packages @@ -1,4 +1 @@ -git -curl -jq -golang \ No newline at end of file +git \ No newline at end of file diff --git a/stage2/05-docker-compose/01-run.sh b/stage2/05-docker-compose/01-run.sh index 5d99a2c..bf23baa 100755 --- a/stage2/05-docker-compose/01-run.sh +++ b/stage2/05-docker-compose/01-run.sh @@ -29,12 +29,4 @@ echo "Copying the compose service to rootfs (etc/init.d)" cp files/compose-service ${ROOTFS_DIR}/etc/init.d/umbrelbox -echo "Docker stuff installed!" - -echo "Downloading images from Docker Hub" -on_chroot << EOF -wget -q "https://raw.githubusercontent.com/moby/moby/master/contrib/download-frozen-image-v2.sh" -chmod +x download-frozen-image-v2.sh -./download-frozen-image-v2.sh docker-images getumbrel/middleware:latest -tar -cC 'target_dir' . | docker load -EOF \ No newline at end of file +echo "Docker stuff installed!" \ No newline at end of file From c9bc3621f162c102f4762e33b4c19f033f0b35b8 Mon Sep 17 00:00:00 2001 From: Mayank Date: Thu, 2 Jul 2020 20:26:39 +0530 Subject: [PATCH 006/104] Install go with apt --- depends | 3 ++- stage0/00-configure-apt/00-run.sh | 8 -------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/depends b/depends index 87cf16f..420b1ec 100644 --- a/depends +++ b/depends @@ -17,4 +17,5 @@ file git lsmod:kmod bc -jq \ No newline at end of file +jq +golang-go \ No newline at end of file diff --git a/stage0/00-configure-apt/00-run.sh b/stage0/00-configure-apt/00-run.sh index 8af1031..d79a647 100755 --- a/stage0/00-configure-apt/00-run.sh +++ b/stage0/00-configure-apt/00-run.sh @@ -17,14 +17,6 @@ apt-get dist-upgrade -y EOF echo "Installing Go" -cd ~ -curl -O "https://golang.org/dl/go1.14.4.linux-amd64.tar.gz" -tar xvf go1.14.4.linux-amd64.tar.gz -sudo chown -R root:root ./go -sudo mv go /usr/local -export GOPATH=$HOME/work -export PATH=$PATH:/usr/local/go/bin:$GOPATH/bin - go version echo "Go installed, now downloading images" From 0f6be91e7ddc2046927c99fec5c0905289adf6bc Mon Sep 17 00:00:00 2001 From: Mayank Date: Thu, 2 Jul 2020 20:30:05 +0530 Subject: [PATCH 007/104] Install go with apt --- depends | 4 +--- stage0/00-configure-apt/00-run.sh | 2 ++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/depends b/depends index 420b1ec..fd4cde7 100644 --- a/depends +++ b/depends @@ -16,6 +16,4 @@ xxd file git lsmod:kmod -bc -jq -golang-go \ No newline at end of file +bc \ No newline at end of file diff --git a/stage0/00-configure-apt/00-run.sh b/stage0/00-configure-apt/00-run.sh index d79a647..e915d05 100755 --- a/stage0/00-configure-apt/00-run.sh +++ b/stage0/00-configure-apt/00-run.sh @@ -17,6 +17,8 @@ apt-get dist-upgrade -y EOF echo "Installing Go" +apt-get update +apt-get install golang-go go version echo "Go installed, now downloading images" From 25822cc800d3c579d39a7fd42f132d0f451656b2 Mon Sep 17 00:00:00 2001 From: Mayank Date: Thu, 2 Jul 2020 20:52:58 +0530 Subject: [PATCH 008/104] Change dir --- stage0/00-configure-apt/00-run.sh | 7 ++++--- stage2/05-docker-compose/01-run.sh | 6 +++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/stage0/00-configure-apt/00-run.sh b/stage0/00-configure-apt/00-run.sh index e915d05..39df22c 100755 --- a/stage0/00-configure-apt/00-run.sh +++ b/stage0/00-configure-apt/00-run.sh @@ -26,9 +26,10 @@ wget -q "https://raw.githubusercontent.com/moby/moby/master/contrib/download-fro chmod +x download-frozen-image-v2.sh ./download-frozen-image-v2.sh docker-images getumbrel/middleware:latest ls docker-images/ -cp -avr docker-images/ ${ROOTFS_DIR}/home/${FIRST_USER_NAME}/docker-images +mkdir -p ${ROOTFS_DIR}/etc/tmp/docker-images +cp -avr docker-images/ ${ROOTFS_DIR}/etc/tmp/docker-images on_chroot << EOF -ls /home/${FIRST_USER_NAME}/ -ls /home/${FIRST_USER_NAME}/docker-images +ls /tmp +ls /tmp/docker-images EOF \ No newline at end of file diff --git a/stage2/05-docker-compose/01-run.sh b/stage2/05-docker-compose/01-run.sh index bf23baa..76a3b62 100755 --- a/stage2/05-docker-compose/01-run.sh +++ b/stage2/05-docker-compose/01-run.sh @@ -29,4 +29,8 @@ echo "Copying the compose service to rootfs (etc/init.d)" cp files/compose-service ${ROOTFS_DIR}/etc/init.d/umbrelbox -echo "Docker stuff installed!" \ No newline at end of file +echo "Docker stuff installed!" + +on_chroot << EOF +tar -cC '/tmp/docker-images' . | docker load +EOF \ No newline at end of file From 9c5b6edea5779acc4841943e79f8a752a07dc5bb Mon Sep 17 00:00:00 2001 From: Mayank Date: Thu, 2 Jul 2020 21:09:41 +0530 Subject: [PATCH 009/104] Dir typo --- stage0/00-configure-apt/00-run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stage0/00-configure-apt/00-run.sh b/stage0/00-configure-apt/00-run.sh index 39df22c..b4901a7 100755 --- a/stage0/00-configure-apt/00-run.sh +++ b/stage0/00-configure-apt/00-run.sh @@ -26,8 +26,8 @@ wget -q "https://raw.githubusercontent.com/moby/moby/master/contrib/download-fro chmod +x download-frozen-image-v2.sh ./download-frozen-image-v2.sh docker-images getumbrel/middleware:latest ls docker-images/ -mkdir -p ${ROOTFS_DIR}/etc/tmp/docker-images -cp -avr docker-images/ ${ROOTFS_DIR}/etc/tmp/docker-images +mkdir -p ${ROOTFS_DIR}/tmp/docker-images +cp -avr docker-images/ ${ROOTFS_DIR}/tmp/docker-images on_chroot << EOF ls /tmp From 041ca6456922e0c6e39d88e9bb45cd9b9d3f0be6 Mon Sep 17 00:00:00 2001 From: Mayank Date: Thu, 2 Jul 2020 22:04:25 +0530 Subject: [PATCH 010/104] Load images --- stage0/00-configure-apt/00-run.sh | 4 ++-- stage2/01-sys-tweaks/files/rc.local | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/stage0/00-configure-apt/00-run.sh b/stage0/00-configure-apt/00-run.sh index b4901a7..46661f4 100755 --- a/stage0/00-configure-apt/00-run.sh +++ b/stage0/00-configure-apt/00-run.sh @@ -27,9 +27,9 @@ chmod +x download-frozen-image-v2.sh ./download-frozen-image-v2.sh docker-images getumbrel/middleware:latest ls docker-images/ mkdir -p ${ROOTFS_DIR}/tmp/docker-images -cp -avr docker-images/ ${ROOTFS_DIR}/tmp/docker-images +cp -avr docker-images/ ${ROOTFS_DIR}/tmp on_chroot << EOF -ls /tmp +echo "Verifying images on device" ls /tmp/docker-images EOF \ No newline at end of file diff --git a/stage2/01-sys-tweaks/files/rc.local b/stage2/01-sys-tweaks/files/rc.local index 4991680..261e184 100755 --- a/stage2/01-sys-tweaks/files/rc.local +++ b/stage2/01-sys-tweaks/files/rc.local @@ -54,6 +54,10 @@ if [ ! -f /home/umbrel/statuses/service-configured ]; then # Making this as done so we don't go through updates again touch /home/umbrel/statuses/service-configured chown -R umbrel.umbrel /home/umbrel/statuses/service-configured + + #Load docker images + echo "Loading docker images" + tar -cC '/tmp/docker-images' . | docker load fi # Next stage, check if service-configured From d876851d88cb6662f4b0f3b578d71dd618f4fccf Mon Sep 17 00:00:00 2001 From: Mayank Date: Thu, 2 Jul 2020 22:50:23 +0530 Subject: [PATCH 011/104] Download multiple images --- stage0/00-configure-apt/00-run.sh | 2 +- stage2/05-docker-compose/01-run.sh | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/stage0/00-configure-apt/00-run.sh b/stage0/00-configure-apt/00-run.sh index 46661f4..94eccc5 100755 --- a/stage0/00-configure-apt/00-run.sh +++ b/stage0/00-configure-apt/00-run.sh @@ -24,7 +24,7 @@ echo "Go installed, now downloading images" wget -q "https://raw.githubusercontent.com/moby/moby/master/contrib/download-frozen-image-v2.sh" chmod +x download-frozen-image-v2.sh -./download-frozen-image-v2.sh docker-images getumbrel/middleware:latest +./download-frozen-image-v2.sh docker-images getumbrel/middleware:latest getumbrel/manager:latest getumbrel/dashboard:latest ls docker-images/ mkdir -p ${ROOTFS_DIR}/tmp/docker-images cp -avr docker-images/ ${ROOTFS_DIR}/tmp diff --git a/stage2/05-docker-compose/01-run.sh b/stage2/05-docker-compose/01-run.sh index 76a3b62..bf23baa 100755 --- a/stage2/05-docker-compose/01-run.sh +++ b/stage2/05-docker-compose/01-run.sh @@ -29,8 +29,4 @@ echo "Copying the compose service to rootfs (etc/init.d)" cp files/compose-service ${ROOTFS_DIR}/etc/init.d/umbrelbox -echo "Docker stuff installed!" - -on_chroot << EOF -tar -cC '/tmp/docker-images' . | docker load -EOF \ No newline at end of file +echo "Docker stuff installed!" \ No newline at end of file From 3d52644be744e01a6d48e48ae2dde5b1d83a1058 Mon Sep 17 00:00:00 2001 From: Mayank Date: Thu, 2 Jul 2020 23:38:10 +0530 Subject: [PATCH 012/104] Download images from compose file --- stage0/00-configure-apt/00-run.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/stage0/00-configure-apt/00-run.sh b/stage0/00-configure-apt/00-run.sh index 94eccc5..d743f27 100755 --- a/stage0/00-configure-apt/00-run.sh +++ b/stage0/00-configure-apt/00-run.sh @@ -23,8 +23,11 @@ go version echo "Go installed, now downloading images" wget -q "https://raw.githubusercontent.com/moby/moby/master/contrib/download-frozen-image-v2.sh" -chmod +x download-frozen-image-v2.sh -./download-frozen-image-v2.sh docker-images getumbrel/middleware:latest getumbrel/manager:latest getumbrel/dashboard:latest +wget -q "https://raw.githubusercontent.com/getumbrel/umbrel-compose/master/docker-compose.yml" +IMAGES=`grep '^\s*image' docker-compose.yml | sed 's/image://' +echo "Images to download $IMAGES" +chmod +x download-froze +./download-frozen-image-v2.sh docker-images $IMAGES ls docker-images/ mkdir -p ${ROOTFS_DIR}/tmp/docker-images cp -avr docker-images/ ${ROOTFS_DIR}/tmp From 0cc1c8069f1678d106ac16a581c4a10d8ef3484b Mon Sep 17 00:00:00 2001 From: Mayank Date: Thu, 2 Jul 2020 23:55:00 +0530 Subject: [PATCH 013/104] Typo --- stage0/00-configure-apt/00-run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stage0/00-configure-apt/00-run.sh b/stage0/00-configure-apt/00-run.sh index d743f27..60011cd 100755 --- a/stage0/00-configure-apt/00-run.sh +++ b/stage0/00-configure-apt/00-run.sh @@ -24,7 +24,7 @@ echo "Go installed, now downloading images" wget -q "https://raw.githubusercontent.com/moby/moby/master/contrib/download-frozen-image-v2.sh" wget -q "https://raw.githubusercontent.com/getumbrel/umbrel-compose/master/docker-compose.yml" -IMAGES=`grep '^\s*image' docker-compose.yml | sed 's/image://' +IMAGES=`grep '^\s*image' docker-compose.yml | sed 's/image://' | sort | uniq` echo "Images to download $IMAGES" chmod +x download-froze ./download-frozen-image-v2.sh docker-images $IMAGES From 7adfa5bb3f0a20561d17fce2d40442c52110beb3 Mon Sep 17 00:00:00 2001 From: Mayank Date: Fri, 3 Jul 2020 00:11:20 +0530 Subject: [PATCH 014/104] Script name --- stage0/00-configure-apt/00-run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stage0/00-configure-apt/00-run.sh b/stage0/00-configure-apt/00-run.sh index 60011cd..ea6b1fc 100755 --- a/stage0/00-configure-apt/00-run.sh +++ b/stage0/00-configure-apt/00-run.sh @@ -26,7 +26,7 @@ wget -q "https://raw.githubusercontent.com/moby/moby/master/contrib/download-fro wget -q "https://raw.githubusercontent.com/getumbrel/umbrel-compose/master/docker-compose.yml" IMAGES=`grep '^\s*image' docker-compose.yml | sed 's/image://' | sort | uniq` echo "Images to download $IMAGES" -chmod +x download-froze +chmod +x ./download-frozen-image-v2.sh ./download-frozen-image-v2.sh docker-images $IMAGES ls docker-images/ mkdir -p ${ROOTFS_DIR}/tmp/docker-images From 95520a4c18873e86bcebdd897e7468b79a095b05 Mon Sep 17 00:00:00 2001 From: Mayank Date: Fri, 3 Jul 2020 00:33:12 +0530 Subject: [PATCH 015/104] Download from a pre-defined list --- stage0/00-configure-apt/00-run.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/stage0/00-configure-apt/00-run.sh b/stage0/00-configure-apt/00-run.sh index ea6b1fc..76511cc 100755 --- a/stage0/00-configure-apt/00-run.sh +++ b/stage0/00-configure-apt/00-run.sh @@ -24,7 +24,8 @@ echo "Go installed, now downloading images" wget -q "https://raw.githubusercontent.com/moby/moby/master/contrib/download-frozen-image-v2.sh" wget -q "https://raw.githubusercontent.com/getumbrel/umbrel-compose/master/docker-compose.yml" -IMAGES=`grep '^\s*image' docker-compose.yml | sed 's/image://' | sort | uniq` +#IMAGES=`grep '^\s*image' docker-compose.yml | sed 's/image://' | sort | uniq` +IMAGES="getumbrel/dashboard:v0.2.1 getumbrel/manager:v0.1.1 getumbrel/middleware:v0.1.1 lncm/bitcoind:v0.20.0 lncm/lnd:v0.9.2-root-experimental nginx:1.17.8 alpine:3.11" echo "Images to download $IMAGES" chmod +x ./download-frozen-image-v2.sh ./download-frozen-image-v2.sh docker-images $IMAGES From 0dce822610e79c75df82b2318a84c19c3a3fb376 Mon Sep 17 00:00:00 2001 From: Mayank Date: Fri, 3 Jul 2020 07:36:27 +0530 Subject: [PATCH 016/104] Move bundling --- stage0/00-configure-apt/00-run.sh | 22 ---------------------- stage2/05-docker-compose/01-run.sh | 17 ++++++++++++++++- 2 files changed, 16 insertions(+), 23 deletions(-) diff --git a/stage0/00-configure-apt/00-run.sh b/stage0/00-configure-apt/00-run.sh index 76511cc..f978814 100755 --- a/stage0/00-configure-apt/00-run.sh +++ b/stage0/00-configure-apt/00-run.sh @@ -14,26 +14,4 @@ on_chroot apt-key add - < files/raspberrypi.gpg.key on_chroot << EOF apt-get update apt-get dist-upgrade -y -EOF - -echo "Installing Go" -apt-get update -apt-get install golang-go -go version -echo "Go installed, now downloading images" - -wget -q "https://raw.githubusercontent.com/moby/moby/master/contrib/download-frozen-image-v2.sh" -wget -q "https://raw.githubusercontent.com/getumbrel/umbrel-compose/master/docker-compose.yml" -#IMAGES=`grep '^\s*image' docker-compose.yml | sed 's/image://' | sort | uniq` -IMAGES="getumbrel/dashboard:v0.2.1 getumbrel/manager:v0.1.1 getumbrel/middleware:v0.1.1 lncm/bitcoind:v0.20.0 lncm/lnd:v0.9.2-root-experimental nginx:1.17.8 alpine:3.11" -echo "Images to download $IMAGES" -chmod +x ./download-frozen-image-v2.sh -./download-frozen-image-v2.sh docker-images $IMAGES -ls docker-images/ -mkdir -p ${ROOTFS_DIR}/tmp/docker-images -cp -avr docker-images/ ${ROOTFS_DIR}/tmp - -on_chroot << EOF -echo "Verifying images on device" -ls /tmp/docker-images EOF \ No newline at end of file diff --git a/stage2/05-docker-compose/01-run.sh b/stage2/05-docker-compose/01-run.sh index bf23baa..57364be 100755 --- a/stage2/05-docker-compose/01-run.sh +++ b/stage2/05-docker-compose/01-run.sh @@ -29,4 +29,19 @@ echo "Copying the compose service to rootfs (etc/init.d)" cp files/compose-service ${ROOTFS_DIR}/etc/init.d/umbrelbox -echo "Docker stuff installed!" \ No newline at end of file +echo "Docker stuff installed!" + +echo "Bundling Docker images required to run Umbrel services" +apt-get update +apt-get install golang-go +go version +wget -q "https://raw.githubusercontent.com/moby/moby/master/contrib/download-frozen-image-v2.sh" +wget -q "https://raw.githubusercontent.com/getumbrel/umbrel-compose/master/docker-compose.yml" +#IMAGES=`grep '^\s*image' docker-compose.yml | sed 's/image://' | sort | uniq` +IMAGES="getumbrel/dashboard:v0.2.1 getumbrel/manager:v0.1.1 getumbrel/middleware:v0.1.1 lncm/bitcoind:v0.20.0 lncm/lnd:v0.9.2-root-experimental nginx:1.17.8 alpine:3.11" +echo "Images to download: $IMAGES" +chmod +x ./download-frozen-image-v2.sh +./download-frozen-image-v2.sh docker-images $IMAGES + +mkdir -p ${ROOTFS_DIR}/tmp/docker-images +cp -avr docker-images/ ${ROOTFS_DIR}/tmp \ No newline at end of file From 3fadeeeeb49cfc0d4cfcb702f424eb8bddde455b Mon Sep 17 00:00:00 2001 From: Mayank Date: Fri, 3 Jul 2020 09:46:22 +0530 Subject: [PATCH 017/104] Change location --- stage2/01-sys-tweaks/files/rc.local | 2 +- stage2/05-docker-compose/01-run.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/stage2/01-sys-tweaks/files/rc.local b/stage2/01-sys-tweaks/files/rc.local index 261e184..1c7e16b 100755 --- a/stage2/01-sys-tweaks/files/rc.local +++ b/stage2/01-sys-tweaks/files/rc.local @@ -57,7 +57,7 @@ if [ ! -f /home/umbrel/statuses/service-configured ]; then #Load docker images echo "Loading docker images" - tar -cC '/tmp/docker-images' . | docker load + tar -cC '/home/umbrel/images' . | docker load fi # Next stage, check if service-configured diff --git a/stage2/05-docker-compose/01-run.sh b/stage2/05-docker-compose/01-run.sh index 57364be..01a36df 100755 --- a/stage2/05-docker-compose/01-run.sh +++ b/stage2/05-docker-compose/01-run.sh @@ -43,5 +43,5 @@ echo "Images to download: $IMAGES" chmod +x ./download-frozen-image-v2.sh ./download-frozen-image-v2.sh docker-images $IMAGES -mkdir -p ${ROOTFS_DIR}/tmp/docker-images -cp -avr docker-images/ ${ROOTFS_DIR}/tmp \ No newline at end of file +mkdir ${ROOTFS_DIR}/home/${FIRST_USER_NAME}/images +cp -avr docker-images/ ${ROOTFS_DIR}/home/${FIRST_USER_NAME}/images \ No newline at end of file From d77b8e2136c13308b2143496c3251cf2a9e6a566 Mon Sep 17 00:00:00 2001 From: Mayank Date: Fri, 3 Jul 2020 10:19:16 +0530 Subject: [PATCH 018/104] Dynamically download images --- stage2/05-docker-compose/01-run.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/stage2/05-docker-compose/01-run.sh b/stage2/05-docker-compose/01-run.sh index 01a36df..162c64f 100755 --- a/stage2/05-docker-compose/01-run.sh +++ b/stage2/05-docker-compose/01-run.sh @@ -37,11 +37,10 @@ apt-get install golang-go go version wget -q "https://raw.githubusercontent.com/moby/moby/master/contrib/download-frozen-image-v2.sh" wget -q "https://raw.githubusercontent.com/getumbrel/umbrel-compose/master/docker-compose.yml" -#IMAGES=`grep '^\s*image' docker-compose.yml | sed 's/image://' | sort | uniq` -IMAGES="getumbrel/dashboard:v0.2.1 getumbrel/manager:v0.1.1 getumbrel/middleware:v0.1.1 lncm/bitcoind:v0.20.0 lncm/lnd:v0.9.2-root-experimental nginx:1.17.8 alpine:3.11" -echo "Images to download: $IMAGES" +IMAGES=`grep '^\s*image' docker-compose.yml | sed 's/image://' | sort | uniq` +echo "List of images to download: $IMAGES" chmod +x ./download-frozen-image-v2.sh -./download-frozen-image-v2.sh docker-images $IMAGES +./download-frozen-image-v2.sh docker-images "${IMAGES//\"}" mkdir ${ROOTFS_DIR}/home/${FIRST_USER_NAME}/images cp -avr docker-images/ ${ROOTFS_DIR}/home/${FIRST_USER_NAME}/images \ No newline at end of file From f7d99eec0c2cdf528ce2f17abc7422f7ff09c028 Mon Sep 17 00:00:00 2001 From: Mayank Date: Fri, 3 Jul 2020 12:06:28 +0530 Subject: [PATCH 019/104] Format list of images --- stage2/05-docker-compose/01-run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stage2/05-docker-compose/01-run.sh b/stage2/05-docker-compose/01-run.sh index 162c64f..ade2ec5 100755 --- a/stage2/05-docker-compose/01-run.sh +++ b/stage2/05-docker-compose/01-run.sh @@ -37,7 +37,7 @@ apt-get install golang-go go version wget -q "https://raw.githubusercontent.com/moby/moby/master/contrib/download-frozen-image-v2.sh" wget -q "https://raw.githubusercontent.com/getumbrel/umbrel-compose/master/docker-compose.yml" -IMAGES=`grep '^\s*image' docker-compose.yml | sed 's/image://' | sort | uniq` +IMAGES=$(grep '^\s*image' docker-compose.yml | sed 's/image://' | sed 's/\"//g' | sed '/^$/d;s/[[:blank:]]//g' | sort | uniq | tr \\n " ") echo "List of images to download: $IMAGES" chmod +x ./download-frozen-image-v2.sh ./download-frozen-image-v2.sh docker-images "${IMAGES//\"}" From 4ebaddb9447a539a167f7315ed4be97bfb76793b Mon Sep 17 00:00:00 2001 From: Mayank Date: Fri, 3 Jul 2020 12:41:07 +0530 Subject: [PATCH 020/104] Test save/load logic --- .github/workflows/on-push.yml | 13 +++++++++++++ build-test.sh | 8 ++++++++ 2 files changed, 21 insertions(+) create mode 100644 .github/workflows/on-push.yml create mode 100755 build-test.sh diff --git a/.github/workflows/on-push.yml b/.github/workflows/on-push.yml new file mode 100644 index 0000000..a73a264 --- /dev/null +++ b/.github/workflows/on-push.yml @@ -0,0 +1,13 @@ +name: Testing only +on: + push: + branches: + - * + +jobs: + build: + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v1 + - name: Run Build script + run: sudo ./build-test.sh diff --git a/build-test.sh b/build-test.sh new file mode 100755 index 0000000..03b2c64 --- /dev/null +++ b/build-test.sh @@ -0,0 +1,8 @@ +docker --version +wget -q "https://raw.githubusercontent.com/getumbrel/umbrel-compose/master/docker-compose.yml" +IMAGES=$(grep '^\s*image' docker-compose.yml | sed 's/image://' | sed 's/\"//g' | sed '/^$/d;s/[[:blank:]]//g' | sort | uniq) +echo "List of images to download: $IMAGES" +docker pull --platform=linux/arm/v7 $IMAGES +docker images +docker save $IMAGES -o umbrel-docker-images.tar +ls -al \ No newline at end of file From 6ea5da37f475d4384842e1668e3ec3330a83af56 Mon Sep 17 00:00:00 2001 From: Mayank Date: Fri, 3 Jul 2020 12:42:12 +0530 Subject: [PATCH 021/104] Syntax --- .github/workflows/on-push.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/on-push.yml b/.github/workflows/on-push.yml index a73a264..ad5ebbc 100644 --- a/.github/workflows/on-push.yml +++ b/.github/workflows/on-push.yml @@ -1,8 +1,6 @@ name: Testing only on: - push: - branches: - - * + push jobs: build: From 09b2714fb8ca2f8241a7735d9179c02e6e806249 Mon Sep 17 00:00:00 2001 From: Mayank Date: Fri, 3 Jul 2020 12:48:34 +0530 Subject: [PATCH 022/104] Pull one by one --- build-test.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build-test.sh b/build-test.sh index 03b2c64..9f48bcb 100755 --- a/build-test.sh +++ b/build-test.sh @@ -2,7 +2,9 @@ docker --version wget -q "https://raw.githubusercontent.com/getumbrel/umbrel-compose/master/docker-compose.yml" IMAGES=$(grep '^\s*image' docker-compose.yml | sed 's/image://' | sed 's/\"//g' | sed '/^$/d;s/[[:blank:]]//g' | sort | uniq) echo "List of images to download: $IMAGES" -docker pull --platform=linux/arm/v7 $IMAGES +while IFS= read -r image; do + docker pull --platform=linux/arm/v7 $image +done <<< "$IMAGES" docker images docker save $IMAGES -o umbrel-docker-images.tar ls -al \ No newline at end of file From 2dae0c8277b87ee5ff47b1e076db894d72031aeb Mon Sep 17 00:00:00 2001 From: Mayank Date: Fri, 3 Jul 2020 12:51:08 +0530 Subject: [PATCH 023/104] Use bash shell --- build-test.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build-test.sh b/build-test.sh index 9f48bcb..645a5fa 100755 --- a/build-test.sh +++ b/build-test.sh @@ -1,3 +1,5 @@ +#!/bin/bash -e + docker --version wget -q "https://raw.githubusercontent.com/getumbrel/umbrel-compose/master/docker-compose.yml" IMAGES=$(grep '^\s*image' docker-compose.yml | sed 's/image://' | sed 's/\"//g' | sed '/^$/d;s/[[:blank:]]//g' | sort | uniq) From 492dafec3bc1e95c7f224dbff38af491263f535e Mon Sep 17 00:00:00 2001 From: Mayank Date: Fri, 3 Jul 2020 12:52:40 +0530 Subject: [PATCH 024/104] Enable experimental features --- build-test.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/build-test.sh b/build-test.sh index 645a5fa..9640b87 100755 --- a/build-test.sh +++ b/build-test.sh @@ -1,6 +1,7 @@ #!/bin/bash -e docker --version +export DOCKER_CLI_EXPERIMENTAL=enabled wget -q "https://raw.githubusercontent.com/getumbrel/umbrel-compose/master/docker-compose.yml" IMAGES=$(grep '^\s*image' docker-compose.yml | sed 's/image://' | sed 's/\"//g' | sed '/^$/d;s/[[:blank:]]//g' | sort | uniq) echo "List of images to download: $IMAGES" From db78f87f04759ad86e529d8d476fc5c777963032 Mon Sep 17 00:00:00 2001 From: Mayank Date: Fri, 3 Jul 2020 12:55:17 +0530 Subject: [PATCH 025/104] Alternate way --- build-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-test.sh b/build-test.sh index 9640b87..ec26456 100755 --- a/build-test.sh +++ b/build-test.sh @@ -1,7 +1,7 @@ #!/bin/bash -e docker --version -export DOCKER_CLI_EXPERIMENTAL=enabled +echo '{"experimental": true}' | sudo tee -a /etc/docker/daemon.json wget -q "https://raw.githubusercontent.com/getumbrel/umbrel-compose/master/docker-compose.yml" IMAGES=$(grep '^\s*image' docker-compose.yml | sed 's/image://' | sed 's/\"//g' | sed '/^$/d;s/[[:blank:]]//g' | sort | uniq) echo "List of images to download: $IMAGES" From a374b491dd3acb0bca6fe18a9fcac60a73e2a33e Mon Sep 17 00:00:00 2001 From: Mayank Date: Fri, 3 Jul 2020 12:57:18 +0530 Subject: [PATCH 026/104] Restart Docker --- build-test.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build-test.sh b/build-test.sh index ec26456..0856d09 100755 --- a/build-test.sh +++ b/build-test.sh @@ -1,7 +1,9 @@ #!/bin/bash -e docker --version +sudo rm -rf /etc/docker/daemon.json echo '{"experimental": true}' | sudo tee -a /etc/docker/daemon.json +sudo systemctl restart docker wget -q "https://raw.githubusercontent.com/getumbrel/umbrel-compose/master/docker-compose.yml" IMAGES=$(grep '^\s*image' docker-compose.yml | sed 's/image://' | sed 's/\"//g' | sed '/^$/d;s/[[:blank:]]//g' | sort | uniq) echo "List of images to download: $IMAGES" From 7210d49aa5600b85e5617ff81f2ddf400d556166 Mon Sep 17 00:00:00 2001 From: Mayank Date: Fri, 3 Jul 2020 13:05:47 +0530 Subject: [PATCH 027/104] Segregated logic --- .github/workflows/on-push.yml | 5 +++++ build-test.sh | 5 ----- stage2/05-docker-compose/01-run.sh | 17 ++++++++--------- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/.github/workflows/on-push.yml b/.github/workflows/on-push.yml index ad5ebbc..8588220 100644 --- a/.github/workflows/on-push.yml +++ b/.github/workflows/on-push.yml @@ -7,5 +7,10 @@ jobs: runs-on: ubuntu-18.04 steps: - uses: actions/checkout@v1 + - name: Enable experimental features in Docker + - run: | + sudo rm -rf /etc/docker/daemon.json \ + echo '{"experimental": true}' | sudo tee -a /etc/docker/daemon.json \ + sudo systemctl restart docker - name: Run Build script run: sudo ./build-test.sh diff --git a/build-test.sh b/build-test.sh index 0856d09..36a83d3 100755 --- a/build-test.sh +++ b/build-test.sh @@ -1,15 +1,10 @@ #!/bin/bash -e -docker --version -sudo rm -rf /etc/docker/daemon.json -echo '{"experimental": true}' | sudo tee -a /etc/docker/daemon.json -sudo systemctl restart docker wget -q "https://raw.githubusercontent.com/getumbrel/umbrel-compose/master/docker-compose.yml" IMAGES=$(grep '^\s*image' docker-compose.yml | sed 's/image://' | sed 's/\"//g' | sed '/^$/d;s/[[:blank:]]//g' | sort | uniq) echo "List of images to download: $IMAGES" while IFS= read -r image; do docker pull --platform=linux/arm/v7 $image done <<< "$IMAGES" -docker images docker save $IMAGES -o umbrel-docker-images.tar ls -al \ No newline at end of file diff --git a/stage2/05-docker-compose/01-run.sh b/stage2/05-docker-compose/01-run.sh index ade2ec5..6861c16 100755 --- a/stage2/05-docker-compose/01-run.sh +++ b/stage2/05-docker-compose/01-run.sh @@ -32,15 +32,14 @@ cp files/compose-service ${ROOTFS_DIR}/etc/init.d/umbrelbox echo "Docker stuff installed!" echo "Bundling Docker images required to run Umbrel services" -apt-get update -apt-get install golang-go -go version -wget -q "https://raw.githubusercontent.com/moby/moby/master/contrib/download-frozen-image-v2.sh" + wget -q "https://raw.githubusercontent.com/getumbrel/umbrel-compose/master/docker-compose.yml" -IMAGES=$(grep '^\s*image' docker-compose.yml | sed 's/image://' | sed 's/\"//g' | sed '/^$/d;s/[[:blank:]]//g' | sort | uniq | tr \\n " ") +IMAGES=$(grep '^\s*image' docker-compose.yml | sed 's/image://' | sed 's/\"//g' | sed '/^$/d;s/[[:blank:]]//g' | sort | uniq) echo "List of images to download: $IMAGES" -chmod +x ./download-frozen-image-v2.sh -./download-frozen-image-v2.sh docker-images "${IMAGES//\"}" -mkdir ${ROOTFS_DIR}/home/${FIRST_USER_NAME}/images -cp -avr docker-images/ ${ROOTFS_DIR}/home/${FIRST_USER_NAME}/images \ No newline at end of file +while IFS= read -r image; do + docker pull --platform=linux/arm/v7 $image +done <<< "$IMAGES" + +docker save $IMAGES -o umbrel-docker-images.tar +cp umbrel-docker-images.tar ${ROOTFS_DIR}/home/${FIRST_USER_NAME}/umbrel-docker-images.tar \ No newline at end of file From b1017e43a4fcc9765007cead74d5fb1cdb1ba9f2 Mon Sep 17 00:00:00 2001 From: Mayank Date: Fri, 3 Jul 2020 13:06:19 +0530 Subject: [PATCH 028/104] Syntax --- .github/workflows/on-push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/on-push.yml b/.github/workflows/on-push.yml index 8588220..244e0a7 100644 --- a/.github/workflows/on-push.yml +++ b/.github/workflows/on-push.yml @@ -8,7 +8,7 @@ jobs: steps: - uses: actions/checkout@v1 - name: Enable experimental features in Docker - - run: | + run: | sudo rm -rf /etc/docker/daemon.json \ echo '{"experimental": true}' | sudo tee -a /etc/docker/daemon.json \ sudo systemctl restart docker From eb740cd09e4d8d66a9427270e325aa9b7817f355 Mon Sep 17 00:00:00 2001 From: Mayank Date: Fri, 3 Jul 2020 13:07:39 +0530 Subject: [PATCH 029/104] Removed slahes --- .github/workflows/on-push.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/on-push.yml b/.github/workflows/on-push.yml index 244e0a7..e07550d 100644 --- a/.github/workflows/on-push.yml +++ b/.github/workflows/on-push.yml @@ -9,8 +9,8 @@ jobs: - uses: actions/checkout@v1 - name: Enable experimental features in Docker run: | - sudo rm -rf /etc/docker/daemon.json \ - echo '{"experimental": true}' | sudo tee -a /etc/docker/daemon.json \ + sudo rm -rf /etc/docker/daemon.json + echo '{"experimental": true}' | sudo tee -a /etc/docker/daemon.json sudo systemctl restart docker - name: Run Build script run: sudo ./build-test.sh From 7e96d3d1cb34665bc867dd9231b4e8988da2fdad Mon Sep 17 00:00:00 2001 From: Mayank Date: Fri, 3 Jul 2020 13:12:44 +0530 Subject: [PATCH 030/104] Get file size --- .github/workflows/on-tag-only.yml | 14 ++++++++++++++ build-test.sh | 2 +- stage2/01-sys-tweaks/files/rc.local | 2 +- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/workflows/on-tag-only.yml b/.github/workflows/on-tag-only.yml index e80a1f0..3228ece 100644 --- a/.github/workflows/on-tag-only.yml +++ b/.github/workflows/on-tag-only.yml @@ -5,19 +5,33 @@ on: jobs: build: + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v1 - name: Run Apt-get update run: sudo apt-get update + - name: Install Dependencies run: sudo apt-get -y install quilt qemu-user-static debootstrap bsdtar + + - name: Enable experimental features in Docker + run: | + sudo rm -rf /etc/docker/daemon.json + echo '{"experimental": true}' | sudo tee -a /etc/docker/daemon.json + sudo systemctl restart docker + - name: Run Build script run: sudo ./build.sh + - name: Copy ZIP to current working dir run: cp deploy/*.zip ./ + - name: Debug current working dir run: ls -la + - name: Release Image to placeholder or release uses: meeDamian/github-release@v1.0.0 if: startsWith(github.ref, 'refs/tags/') diff --git a/build-test.sh b/build-test.sh index 36a83d3..4e3d097 100755 --- a/build-test.sh +++ b/build-test.sh @@ -7,4 +7,4 @@ while IFS= read -r image; do docker pull --platform=linux/arm/v7 $image done <<< "$IMAGES" docker save $IMAGES -o umbrel-docker-images.tar -ls -al \ No newline at end of file +du -h umbrel-docker-images.tar \ No newline at end of file diff --git a/stage2/01-sys-tweaks/files/rc.local b/stage2/01-sys-tweaks/files/rc.local index 1c7e16b..2368717 100755 --- a/stage2/01-sys-tweaks/files/rc.local +++ b/stage2/01-sys-tweaks/files/rc.local @@ -57,7 +57,7 @@ if [ ! -f /home/umbrel/statuses/service-configured ]; then #Load docker images echo "Loading docker images" - tar -cC '/home/umbrel/images' . | docker load + docker load -i /home/umbrel/umbrel-docker-images.tar fi # Next stage, check if service-configured From 746d66aa00ae7a68eaa5feafbd101ee8eca858d6 Mon Sep 17 00:00:00 2001 From: Mayank Date: Fri, 3 Jul 2020 13:15:56 +0530 Subject: [PATCH 031/104] Try load/save on build --- stage2/05-docker-compose/01-run.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/stage2/05-docker-compose/01-run.sh b/stage2/05-docker-compose/01-run.sh index 6861c16..7cb8587 100755 --- a/stage2/05-docker-compose/01-run.sh +++ b/stage2/05-docker-compose/01-run.sh @@ -42,4 +42,5 @@ while IFS= read -r image; do done <<< "$IMAGES" docker save $IMAGES -o umbrel-docker-images.tar +du -h umbrel-docker-images.tar cp umbrel-docker-images.tar ${ROOTFS_DIR}/home/${FIRST_USER_NAME}/umbrel-docker-images.tar \ No newline at end of file From 3b170ec6088101957713a4aa7eaec272080629f2 Mon Sep 17 00:00:00 2001 From: Mayank Date: Fri, 3 Jul 2020 16:15:27 +0530 Subject: [PATCH 032/104] Log rc.local --- stage2/01-sys-tweaks/files/rc.local | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/stage2/01-sys-tweaks/files/rc.local b/stage2/01-sys-tweaks/files/rc.local index 2368717..8771ea1 100755 --- a/stage2/01-sys-tweaks/files/rc.local +++ b/stage2/01-sys-tweaks/files/rc.local @@ -11,6 +11,10 @@ # # By default this script does nothing. +exec 2> /tmp/rc.local.log # send stderr from rc.local to a log file +exec 1>&2 # send stdout to the same log file +set -x # tell sh to display commands before execution + # Print the IP address _IP=$(hostname -I) || true if [ "$_IP" ]; then From e3e225187f2a26456c39726ea29aac9ebe297f8d Mon Sep 17 00:00:00 2001 From: Mayank Date: Sat, 4 Jul 2020 10:52:35 +0530 Subject: [PATCH 033/104] Status file for loading Docker images --- stage2/01-sys-tweaks/files/rc.local | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/stage2/01-sys-tweaks/files/rc.local b/stage2/01-sys-tweaks/files/rc.local index 8771ea1..a823a54 100755 --- a/stage2/01-sys-tweaks/files/rc.local +++ b/stage2/01-sys-tweaks/files/rc.local @@ -42,6 +42,19 @@ if [ ! -f /home/umbrel/statuses/disk-partitioned ]; then fi fi +# Check if docker images have been loaded +if [ ! -f /home/umbrel/statuses/images-loaded ]; then + if [ -f /home/umbrel/umbrel-docker-images.tar ]; then + echo "Loading docker images" + docker load -i /home/umbrel/umbrel-docker-images.tar || exit 1 + rm -f /home/umbrel/umbrel-docker-images.tar + touch /home/umbrel/statuses/images-loaded + chown -R umbrel.umbrel /home/umbrel/statuses/images-loaded + else + echo "Could not find pre-bundled Docker images" + fi +fi + if [ ! -f /home/umbrel/statuses/service-configured ]; then if [ -f /home/umbrel/bin/rpcauth.py ]; then echo "Running rpcauth.py, and generating secrets for bitcoind" @@ -58,10 +71,6 @@ if [ ! -f /home/umbrel/statuses/service-configured ]; then # Making this as done so we don't go through updates again touch /home/umbrel/statuses/service-configured chown -R umbrel.umbrel /home/umbrel/statuses/service-configured - - #Load docker images - echo "Loading docker images" - docker load -i /home/umbrel/umbrel-docker-images.tar fi # Next stage, check if service-configured From 7e9d65eb0f3ecb8240d6316aa6fd119d9d9e4bba Mon Sep 17 00:00:00 2001 From: Mayank Date: Sat, 4 Jul 2020 15:45:25 +0530 Subject: [PATCH 034/104] Compare gzip size --- build-test.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build-test.sh b/build-test.sh index 4e3d097..f02e365 100755 --- a/build-test.sh +++ b/build-test.sh @@ -7,4 +7,6 @@ while IFS= read -r image; do docker pull --platform=linux/arm/v7 $image done <<< "$IMAGES" docker save $IMAGES -o umbrel-docker-images.tar -du -h umbrel-docker-images.tar \ No newline at end of file +du -h umbrel-docker-images.tar +docker save $IMAGES | gzip > umbrel-docker-images.tar.gz +du -h umbrel-docker-images.tar.gz \ No newline at end of file From 8bf6d5275032f0fd2a764b6e88b37e47fcef9b5a Mon Sep 17 00:00:00 2001 From: Mayank Date: Sat, 4 Jul 2020 16:00:03 +0530 Subject: [PATCH 035/104] Use gzip compression --- stage2/01-sys-tweaks/files/rc.local | 11 +++++++---- stage2/05-docker-compose/01-run.sh | 6 +++--- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/stage2/01-sys-tweaks/files/rc.local b/stage2/01-sys-tweaks/files/rc.local index a823a54..38153ae 100755 --- a/stage2/01-sys-tweaks/files/rc.local +++ b/stage2/01-sys-tweaks/files/rc.local @@ -44,10 +44,13 @@ fi # Check if docker images have been loaded if [ ! -f /home/umbrel/statuses/images-loaded ]; then - if [ -f /home/umbrel/umbrel-docker-images.tar ]; then - echo "Loading docker images" - docker load -i /home/umbrel/umbrel-docker-images.tar || exit 1 - rm -f /home/umbrel/umbrel-docker-images.tar + if [ -f /home/umbrel/umbrel-docker-images.tar.gz ]; then + echo "Loading pre-bundled Docker images" + start=`date +%s` + docker load -i /home/umbrel/umbrel-docker-images.tar.gz || exit 1 + end=`date +%s` + echo "It took $((($end - $start)/60)) minutes ($((end - start)) seconds) to load all the images" + rm -f /home/umbrel/umbrel-docker-images.tar.gz touch /home/umbrel/statuses/images-loaded chown -R umbrel.umbrel /home/umbrel/statuses/images-loaded else diff --git a/stage2/05-docker-compose/01-run.sh b/stage2/05-docker-compose/01-run.sh index 7cb8587..dcae09e 100755 --- a/stage2/05-docker-compose/01-run.sh +++ b/stage2/05-docker-compose/01-run.sh @@ -41,6 +41,6 @@ while IFS= read -r image; do docker pull --platform=linux/arm/v7 $image done <<< "$IMAGES" -docker save $IMAGES -o umbrel-docker-images.tar -du -h umbrel-docker-images.tar -cp umbrel-docker-images.tar ${ROOTFS_DIR}/home/${FIRST_USER_NAME}/umbrel-docker-images.tar \ No newline at end of file +docker save $IMAGES | gzip > umbrel-docker-images.tar.gz +du -h umbrel-docker-images.tar.gz +cp umbrel-docker-images.tar.gz ${ROOTFS_DIR}/home/${FIRST_USER_NAME}/umbrel-docker-images.tar.gz \ No newline at end of file From 0b4acd4cad49d27e3714ba146dca5627850ad26c Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 09:00:11 +0530 Subject: [PATCH 036/104] Pull images in docker --- build-test.sh | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/build-test.sh b/build-test.sh index f02e365..a293d4f 100755 --- a/build-test.sh +++ b/build-test.sh @@ -1,12 +1,20 @@ #!/bin/bash -e +set -x wget -q "https://raw.githubusercontent.com/getumbrel/umbrel-compose/master/docker-compose.yml" IMAGES=$(grep '^\s*image' docker-compose.yml | sed 's/image://' | sed 's/\"//g' | sed '/^$/d;s/[[:blank:]]//g' | sort | uniq) echo "List of images to download: $IMAGES" + +mkdir dockerpi +docker run -it -v dockerpi:/sdcard lukechilds/dockerpi --name "dockerpi" +docker exec -it dockerpi curl -fsSL https://get.docker.com -o docker-install.sh +docker exec -it dockerpi ./docker-install.sh while IFS= read -r image; do - docker pull --platform=linux/arm/v7 $image + docker exec -it docker pull --platform=linux/arm/v7 $image done <<< "$IMAGES" -docker save $IMAGES -o umbrel-docker-images.tar -du -h umbrel-docker-images.tar -docker save $IMAGES | gzip > umbrel-docker-images.tar.gz -du -h umbrel-docker-images.tar.gz \ No newline at end of file +ls dockerpi +ls dockerpi/var/lib/tor +# docker save $IMAGES -o umbrel-docker-images.tar +# du -h umbrel-docker-images.tar +# docker save $IMAGES | gzip > umbrel-docker-images.tar.gz +# du -h umbrel-docker-images.tar.gz \ No newline at end of file From 4b4a9f7c390b19dfcb1a95f4c3756fae56b0b9df Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 09:04:26 +0530 Subject: [PATCH 037/104] Log --- build-test.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/build-test.sh b/build-test.sh index a293d4f..a88183b 100755 --- a/build-test.sh +++ b/build-test.sh @@ -6,9 +6,14 @@ IMAGES=$(grep '^\s*image' docker-compose.yml | sed 's/image://' | sed 's/\"//g' echo "List of images to download: $IMAGES" mkdir dockerpi +echo "Running Docker Pi" docker run -it -v dockerpi:/sdcard lukechilds/dockerpi --name "dockerpi" +docker ps +echo "Downloading install script" docker exec -it dockerpi curl -fsSL https://get.docker.com -o docker-install.sh +echo "Running install script" docker exec -it dockerpi ./docker-install.sh +echo "Pulling images in docker" while IFS= read -r image; do docker exec -it docker pull --platform=linux/arm/v7 $image done <<< "$IMAGES" From 5db41993f07dab0d84f6c73d67f71794cc1196ae Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 09:06:01 +0530 Subject: [PATCH 038/104] Make it non-interactive --- build-test.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build-test.sh b/build-test.sh index a88183b..4749097 100755 --- a/build-test.sh +++ b/build-test.sh @@ -7,15 +7,15 @@ echo "List of images to download: $IMAGES" mkdir dockerpi echo "Running Docker Pi" -docker run -it -v dockerpi:/sdcard lukechilds/dockerpi --name "dockerpi" +docker run -v dockerpi:/sdcard lukechilds/dockerpi --name "dockerpi" docker ps echo "Downloading install script" -docker exec -it dockerpi curl -fsSL https://get.docker.com -o docker-install.sh +docker exec -i dockerpi curl -fsSL https://get.docker.com -o docker-install.sh echo "Running install script" -docker exec -it dockerpi ./docker-install.sh +docker exec -i dockerpi ./docker-install.sh echo "Pulling images in docker" while IFS= read -r image; do - docker exec -it docker pull --platform=linux/arm/v7 $image + docker exec -i docker pull --platform=linux/arm/v7 $image done <<< "$IMAGES" ls dockerpi ls dockerpi/var/lib/tor From f694d1280fa4fa9c73e7896a719631269bdd39db Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 09:10:21 +0530 Subject: [PATCH 039/104] -d mode --- build-test.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build-test.sh b/build-test.sh index 4749097..66e2770 100755 --- a/build-test.sh +++ b/build-test.sh @@ -7,15 +7,15 @@ echo "List of images to download: $IMAGES" mkdir dockerpi echo "Running Docker Pi" -docker run -v dockerpi:/sdcard lukechilds/dockerpi --name "dockerpi" +docker run --name dockerpi -v dockerpi:/sdcard lukechilds/dockerpi docker ps echo "Downloading install script" -docker exec -i dockerpi curl -fsSL https://get.docker.com -o docker-install.sh +docker exec -d dockerpi curl -fsSL https://get.docker.com -o docker-install.sh echo "Running install script" -docker exec -i dockerpi ./docker-install.sh +docker exec -d dockerpi ./docker-install.sh echo "Pulling images in docker" while IFS= read -r image; do - docker exec -i docker pull --platform=linux/arm/v7 $image + docker exec -d docker pull --platform=linux/arm/v7 $image done <<< "$IMAGES" ls dockerpi ls dockerpi/var/lib/tor From 9f1cd156304c8104c21a0cdc99914241a7da3422 Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 09:26:44 +0530 Subject: [PATCH 040/104] See what's installed --- build-test.sh | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/build-test.sh b/build-test.sh index 66e2770..ad87849 100755 --- a/build-test.sh +++ b/build-test.sh @@ -1,24 +1,26 @@ #!/bin/bash -e set -x -wget -q "https://raw.githubusercontent.com/getumbrel/umbrel-compose/master/docker-compose.yml" -IMAGES=$(grep '^\s*image' docker-compose.yml | sed 's/image://' | sed 's/\"//g' | sed '/^$/d;s/[[:blank:]]//g' | sort | uniq) -echo "List of images to download: $IMAGES" - -mkdir dockerpi -echo "Running Docker Pi" -docker run --name dockerpi -v dockerpi:/sdcard lukechilds/dockerpi docker ps -echo "Downloading install script" -docker exec -d dockerpi curl -fsSL https://get.docker.com -o docker-install.sh -echo "Running install script" -docker exec -d dockerpi ./docker-install.sh -echo "Pulling images in docker" -while IFS= read -r image; do - docker exec -d docker pull --platform=linux/arm/v7 $image -done <<< "$IMAGES" -ls dockerpi -ls dockerpi/var/lib/tor + +# wget -q "https://raw.githubusercontent.com/getumbrel/umbrel-compose/master/docker-compose.yml" +# IMAGES=$(grep '^\s*image' docker-compose.yml | sed 's/image://' | sed 's/\"//g' | sed '/^$/d;s/[[:blank:]]//g' | sort | uniq) +# echo "List of images to download: $IMAGES" + +# mkdir dockerpi +# echo "Running Docker Pi" +# docker run --name dockerpi -v dockerpi:/sdcard lukechilds/dockerpi +# docker ps +# echo "Downloading install script" +# docker exec -d dockerpi curl -fsSL https://get.docker.com -o docker-install.sh +# echo "Running install script" +# docker exec -d dockerpi ./docker-install.sh +# echo "Pulling images in docker" +# while IFS= read -r image; do +# docker exec -d docker pull --platform=linux/arm/v7 $image +# done <<< "$IMAGES" +# ls dockerpi +# ls dockerpi/var/lib/tor # docker save $IMAGES -o umbrel-docker-images.tar # du -h umbrel-docker-images.tar # docker save $IMAGES | gzip > umbrel-docker-images.tar.gz From bc28cc1de336f281ffbd11efccd9285652a24485 Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 09:27:24 +0530 Subject: [PATCH 041/104] See images --- build-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-test.sh b/build-test.sh index ad87849..f6e96f9 100755 --- a/build-test.sh +++ b/build-test.sh @@ -1,7 +1,7 @@ #!/bin/bash -e set -x -docker ps +docker images -a # wget -q "https://raw.githubusercontent.com/getumbrel/umbrel-compose/master/docker-compose.yml" # IMAGES=$(grep '^\s*image' docker-compose.yml | sed 's/image://' | sed 's/\"//g' | sed '/^$/d;s/[[:blank:]]//g' | sort | uniq) From d525db65f64bd8478b574a04d5d6ec1929106dca Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 09:34:31 +0530 Subject: [PATCH 042/104] Use Qeemu --- build-test.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build-test.sh b/build-test.sh index f6e96f9..631bc0e 100755 --- a/build-test.sh +++ b/build-test.sh @@ -1,7 +1,10 @@ #!/bin/bash -e set -x -docker images -a +uname -m +sudo apt-get install qemu binfmt-support qemu-user-static # Install the qemu packages +docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # This step will execute the registering scripts +docker run --rm -t multiarch/debian-debootstrap:armhf-buster-slim uname -m # wget -q "https://raw.githubusercontent.com/getumbrel/umbrel-compose/master/docker-compose.yml" # IMAGES=$(grep '^\s*image' docker-compose.yml | sed 's/image://' | sed 's/\"//g' | sed '/^$/d;s/[[:blank:]]//g' | sort | uniq) From 21b38693b5ae772214bf3b09b6b11a405bd2bb87 Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 09:36:31 +0530 Subject: [PATCH 043/104] Remove packages --- build-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-test.sh b/build-test.sh index 631bc0e..e435750 100755 --- a/build-test.sh +++ b/build-test.sh @@ -2,7 +2,7 @@ set -x uname -m -sudo apt-get install qemu binfmt-support qemu-user-static # Install the qemu packages +# sudo apt-get install qemu binfmt-support qemu-user-static # Install the qemu packages docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # This step will execute the registering scripts docker run --rm -t multiarch/debian-debootstrap:armhf-buster-slim uname -m From 63df95e96b4dea4f9cd8d13d3520b7fd9f3ba2b4 Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 09:41:07 +0530 Subject: [PATCH 044/104] Test docker --- build-test.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build-test.sh b/build-test.sh index e435750..41f4946 100755 --- a/build-test.sh +++ b/build-test.sh @@ -4,7 +4,10 @@ set -x uname -m # sudo apt-get install qemu binfmt-support qemu-user-static # Install the qemu packages docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # This step will execute the registering scripts -docker run --rm -t multiarch/debian-debootstrap:armhf-buster-slim uname -m + +mkdir docker-dir +docker run --rm -t --name dockerpi -v docker-dir:/var/lib/docker multiarch/debian-debootstrap:armhf-buster-slim uname -m +docker ps # wget -q "https://raw.githubusercontent.com/getumbrel/umbrel-compose/master/docker-compose.yml" # IMAGES=$(grep '^\s*image' docker-compose.yml | sed 's/image://' | sed 's/\"//g' | sed '/^$/d;s/[[:blank:]]//g' | sort | uniq) From 19f8e04e69151568bf75a53cb115f30b916aade5 Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 09:43:01 +0530 Subject: [PATCH 045/104] Persistent container --- build-test.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build-test.sh b/build-test.sh index 41f4946..f1f5b39 100755 --- a/build-test.sh +++ b/build-test.sh @@ -6,8 +6,9 @@ uname -m docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # This step will execute the registering scripts mkdir docker-dir -docker run --rm -t --name dockerpi -v docker-dir:/var/lib/docker multiarch/debian-debootstrap:armhf-buster-slim uname -m +docker run --rm -t --name dockerdebian -v docker-dir:/var/lib/docker multiarch/debian-debootstrap:armhf-buster-slim bash docker ps +docker exec -t dockerdebian uname -m # wget -q "https://raw.githubusercontent.com/getumbrel/umbrel-compose/master/docker-compose.yml" # IMAGES=$(grep '^\s*image' docker-compose.yml | sed 's/image://' | sed 's/\"//g' | sed '/^$/d;s/[[:blank:]]//g' | sort | uniq) From c003bb9418612ea8f9be0819be6d059cf5c53962 Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 09:47:41 +0530 Subject: [PATCH 046/104] Detached --- build-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-test.sh b/build-test.sh index f1f5b39..23e9205 100755 --- a/build-test.sh +++ b/build-test.sh @@ -6,7 +6,7 @@ uname -m docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # This step will execute the registering scripts mkdir docker-dir -docker run --rm -t --name dockerdebian -v docker-dir:/var/lib/docker multiarch/debian-debootstrap:armhf-buster-slim bash +docker run --rm -d --name dockerdebian -v docker-dir:/var/lib/docker multiarch/debian-debootstrap:armhf-buster-slim bash docker ps docker exec -t dockerdebian uname -m From f67a1793debc1a36527128c126df91bfef9a5f52 Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 09:48:59 +0530 Subject: [PATCH 047/104] Sleep --- build-test.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/build-test.sh b/build-test.sh index 23e9205..3171144 100755 --- a/build-test.sh +++ b/build-test.sh @@ -7,6 +7,7 @@ docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # This st mkdir docker-dir docker run --rm -d --name dockerdebian -v docker-dir:/var/lib/docker multiarch/debian-debootstrap:armhf-buster-slim bash +sleep 10s docker ps docker exec -t dockerdebian uname -m From a6cd00005b73681ae6d1ae449f149ead0ac1e727 Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 09:51:14 +0530 Subject: [PATCH 048/104] Check persistence --- build-test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build-test.sh b/build-test.sh index 3171144..125c612 100755 --- a/build-test.sh +++ b/build-test.sh @@ -6,8 +6,8 @@ uname -m docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # This step will execute the registering scripts mkdir docker-dir -docker run --rm -d --name dockerdebian -v docker-dir:/var/lib/docker multiarch/debian-debootstrap:armhf-buster-slim bash -sleep 10s +docker run --rm -dt --name dockerdebian -v docker-dir:/var/lib/docker multiarch/debian-debootstrap:armhf-buster-slim bash +sleep 5s docker ps docker exec -t dockerdebian uname -m From 57772d567254ef90992300eaf21661007ec6f8a3 Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 09:53:24 +0530 Subject: [PATCH 049/104] Check Docker --- build-test.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build-test.sh b/build-test.sh index 125c612..c2a4d01 100755 --- a/build-test.sh +++ b/build-test.sh @@ -7,9 +7,8 @@ docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # This st mkdir docker-dir docker run --rm -dt --name dockerdebian -v docker-dir:/var/lib/docker multiarch/debian-debootstrap:armhf-buster-slim bash -sleep 5s -docker ps docker exec -t dockerdebian uname -m +docker exec -t dockerdebian docker --version # wget -q "https://raw.githubusercontent.com/getumbrel/umbrel-compose/master/docker-compose.yml" # IMAGES=$(grep '^\s*image' docker-compose.yml | sed 's/image://' | sed 's/\"//g' | sed '/^$/d;s/[[:blank:]]//g' | sort | uniq) From ddd3f1d5a81330e15874d70688e50a44eed67681 Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 09:57:38 +0530 Subject: [PATCH 050/104] Docker installation --- build-test.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/build-test.sh b/build-test.sh index c2a4d01..039c88a 100755 --- a/build-test.sh +++ b/build-test.sh @@ -7,8 +7,12 @@ docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # This st mkdir docker-dir docker run --rm -dt --name dockerdebian -v docker-dir:/var/lib/docker multiarch/debian-debootstrap:armhf-buster-slim bash -docker exec -t dockerdebian uname -m -docker exec -t dockerdebian docker --version +docker exec -t dockerdebian bash < Date: Sun, 5 Jul 2020 10:01:34 +0530 Subject: [PATCH 051/104] Remove TTY --- build-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-test.sh b/build-test.sh index 039c88a..5aa698a 100755 --- a/build-test.sh +++ b/build-test.sh @@ -7,7 +7,7 @@ docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # This st mkdir docker-dir docker run --rm -dt --name dockerdebian -v docker-dir:/var/lib/docker multiarch/debian-debootstrap:armhf-buster-slim bash -docker exec -t dockerdebian bash < Date: Sun, 5 Jul 2020 10:03:47 +0530 Subject: [PATCH 052/104] Check Docker installation --- build-test.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/build-test.sh b/build-test.sh index 5aa698a..bca938a 100755 --- a/build-test.sh +++ b/build-test.sh @@ -13,6 +13,7 @@ curl -fsSL https://get.docker.com -o get-docker.sh ./get-docker.sh docker --version EOF +docker exec -t dockerdebian docker --version # wget -q "https://raw.githubusercontent.com/getumbrel/umbrel-compose/master/docker-compose.yml" # IMAGES=$(grep '^\s*image' docker-compose.yml | sed 's/image://' | sed 's/\"//g' | sed '/^$/d;s/[[:blank:]]//g' | sort | uniq) From 51ea81685efbee364b70dc4a7a7d2169602aa914 Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 10:07:18 +0530 Subject: [PATCH 053/104] Alternate install script --- build-test.sh | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/build-test.sh b/build-test.sh index bca938a..47a2898 100755 --- a/build-test.sh +++ b/build-test.sh @@ -7,12 +7,7 @@ docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # This st mkdir docker-dir docker run --rm -dt --name dockerdebian -v docker-dir:/var/lib/docker multiarch/debian-debootstrap:armhf-buster-slim bash -docker exec dockerdebian bash < Date: Sun, 5 Jul 2020 10:08:23 +0530 Subject: [PATCH 054/104] Container name --- build-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-test.sh b/build-test.sh index 47a2898..99dbc48 100755 --- a/build-test.sh +++ b/build-test.sh @@ -7,7 +7,7 @@ docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # This st mkdir docker-dir docker run --rm -dt --name dockerdebian -v docker-dir:/var/lib/docker multiarch/debian-debootstrap:armhf-buster-slim bash -docker exec bash -c 'ls /var/lib/docker; curl -fsSL https://get.docker.com -o get-docker.sh; ./get-docker.sh; docker --version; ls /var/lib/docker;' +docker exec -t dockerdebian bash -c 'ls /var/lib/docker; curl -fsSL https://get.docker.com -o get-docker.sh; ./get-docker.sh; docker --version; ls /var/lib/docker;' docker exec -t dockerdebian docker --version # wget -q "https://raw.githubusercontent.com/getumbrel/umbrel-compose/master/docker-compose.yml" From 6b04458421919029d8bb93536cc4e489b95beef8 Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 10:12:12 +0530 Subject: [PATCH 055/104] Install curl --- build-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-test.sh b/build-test.sh index 99dbc48..efa5ed2 100755 --- a/build-test.sh +++ b/build-test.sh @@ -7,7 +7,7 @@ docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # This st mkdir docker-dir docker run --rm -dt --name dockerdebian -v docker-dir:/var/lib/docker multiarch/debian-debootstrap:armhf-buster-slim bash -docker exec -t dockerdebian bash -c 'ls /var/lib/docker; curl -fsSL https://get.docker.com -o get-docker.sh; ./get-docker.sh; docker --version; ls /var/lib/docker;' +docker exec -t dockerdebian bash -c 'apt-get update; apt-get install curl -y; ls /var/lib/docker; curl -fsSL https://get.docker.com -o get-docker.sh; ./get-docker.sh; docker --version; ls /var/lib/docker;' docker exec -t dockerdebian docker --version # wget -q "https://raw.githubusercontent.com/getumbrel/umbrel-compose/master/docker-compose.yml" From cb974e3296e9d302972c772b1f942647947c401a Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 10:14:31 +0530 Subject: [PATCH 056/104] Skip SSL --- build-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-test.sh b/build-test.sh index efa5ed2..ecfb2ab 100755 --- a/build-test.sh +++ b/build-test.sh @@ -7,7 +7,7 @@ docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # This st mkdir docker-dir docker run --rm -dt --name dockerdebian -v docker-dir:/var/lib/docker multiarch/debian-debootstrap:armhf-buster-slim bash -docker exec -t dockerdebian bash -c 'apt-get update; apt-get install curl -y; ls /var/lib/docker; curl -fsSL https://get.docker.com -o get-docker.sh; ./get-docker.sh; docker --version; ls /var/lib/docker;' +docker exec -t dockerdebian bash -c 'apt-get update; apt-get install curl -y; ls /var/lib/docker; curl -kfsSL https://get.docker.com -o get-docker.sh; ./get-docker.sh; docker --version; ls /var/lib/docker;' docker exec -t dockerdebian docker --version # wget -q "https://raw.githubusercontent.com/getumbrel/umbrel-compose/master/docker-compose.yml" From 7665e3d604c1072ef3ba698e63d7384937069a22 Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 10:16:55 +0530 Subject: [PATCH 057/104] Sudo run install script --- build-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-test.sh b/build-test.sh index ecfb2ab..3a86a10 100755 --- a/build-test.sh +++ b/build-test.sh @@ -7,7 +7,7 @@ docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # This st mkdir docker-dir docker run --rm -dt --name dockerdebian -v docker-dir:/var/lib/docker multiarch/debian-debootstrap:armhf-buster-slim bash -docker exec -t dockerdebian bash -c 'apt-get update; apt-get install curl -y; ls /var/lib/docker; curl -kfsSL https://get.docker.com -o get-docker.sh; ./get-docker.sh; docker --version; ls /var/lib/docker;' +docker exec -t dockerdebian bash -c 'apt-get update; apt-get install curl -y; ls /var/lib/docker; curl -kfsSL https://get.docker.com -o get-docker.sh; sudo sh get-docker.sh; docker --version; ls /var/lib/docker;' docker exec -t dockerdebian docker --version # wget -q "https://raw.githubusercontent.com/getumbrel/umbrel-compose/master/docker-compose.yml" From 4cbeb19bd71212c3cf341860b9994ed7bc581b81 Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 10:19:14 +0530 Subject: [PATCH 058/104] Make script executable --- build-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-test.sh b/build-test.sh index 3a86a10..588d6bf 100755 --- a/build-test.sh +++ b/build-test.sh @@ -7,7 +7,7 @@ docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # This st mkdir docker-dir docker run --rm -dt --name dockerdebian -v docker-dir:/var/lib/docker multiarch/debian-debootstrap:armhf-buster-slim bash -docker exec -t dockerdebian bash -c 'apt-get update; apt-get install curl -y; ls /var/lib/docker; curl -kfsSL https://get.docker.com -o get-docker.sh; sudo sh get-docker.sh; docker --version; ls /var/lib/docker;' +docker exec -t dockerdebian bash -c 'apt-get update; apt-get install curl -y; ls /var/lib/docker; curl -kfsSL https://get.docker.com -o get-docker.sh; chmod +x get-docker.sh; sh get-docker.sh; docker --version; ls /var/lib/docker;' docker exec -t dockerdebian docker --version # wget -q "https://raw.githubusercontent.com/getumbrel/umbrel-compose/master/docker-compose.yml" From 7d4bff1e1c06d803f6c85c5f1f571f1a4206ae04 Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 10:22:02 +0530 Subject: [PATCH 059/104] Export CA bundle path --- build-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-test.sh b/build-test.sh index 588d6bf..45f86ce 100755 --- a/build-test.sh +++ b/build-test.sh @@ -7,7 +7,7 @@ docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # This st mkdir docker-dir docker run --rm -dt --name dockerdebian -v docker-dir:/var/lib/docker multiarch/debian-debootstrap:armhf-buster-slim bash -docker exec -t dockerdebian bash -c 'apt-get update; apt-get install curl -y; ls /var/lib/docker; curl -kfsSL https://get.docker.com -o get-docker.sh; chmod +x get-docker.sh; sh get-docker.sh; docker --version; ls /var/lib/docker;' +docker exec -t dockerdebian bash -c 'apt-get update; apt-get install curl -y; export CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt; ls /var/lib/docker; curl -fsSL https://get.docker.com -o get-docker.sh; chmod +x get-docker.sh; sh get-docker.sh; docker --version; ls /var/lib/docker;' docker exec -t dockerdebian docker --version # wget -q "https://raw.githubusercontent.com/getumbrel/umbrel-compose/master/docker-compose.yml" From 6062624908303d9156159ac43729b877c7d364f6 Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 10:27:48 +0530 Subject: [PATCH 060/104] Pull images --- build-test.sh | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/build-test.sh b/build-test.sh index 45f86ce..0f56a44 100755 --- a/build-test.sh +++ b/build-test.sh @@ -7,12 +7,14 @@ docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # This st mkdir docker-dir docker run --rm -dt --name dockerdebian -v docker-dir:/var/lib/docker multiarch/debian-debootstrap:armhf-buster-slim bash -docker exec -t dockerdebian bash -c 'apt-get update; apt-get install curl -y; export CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt; ls /var/lib/docker; curl -fsSL https://get.docker.com -o get-docker.sh; chmod +x get-docker.sh; sh get-docker.sh; docker --version; ls /var/lib/docker;' +docker exec -t dockerdebian bash -c 'apt-get update; apt-get install curl -y; export CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt; curl -fsSL https://get.docker.com -o get-docker.sh; chmod +x get-docker.sh; sh get-docker.sh;' docker exec -t dockerdebian docker --version -# wget -q "https://raw.githubusercontent.com/getumbrel/umbrel-compose/master/docker-compose.yml" -# IMAGES=$(grep '^\s*image' docker-compose.yml | sed 's/image://' | sed 's/\"//g' | sed '/^$/d;s/[[:blank:]]//g' | sort | uniq) -# echo "List of images to download: $IMAGES" +ls docker-dir + +wget -q "https://raw.githubusercontent.com/getumbrel/umbrel-compose/master/docker-compose.yml" +IMAGES=$(grep '^\s*image' docker-compose.yml | sed 's/image://' | sed 's/\"//g' | sed '/^$/d;s/[[:blank:]]//g' | sort | uniq) +echo "List of images to download: $IMAGES" # mkdir dockerpi # echo "Running Docker Pi" @@ -22,10 +24,13 @@ docker exec -t dockerdebian docker --version # docker exec -d dockerpi curl -fsSL https://get.docker.com -o docker-install.sh # echo "Running install script" # docker exec -d dockerpi ./docker-install.sh -# echo "Pulling images in docker" -# while IFS= read -r image; do -# docker exec -d docker pull --platform=linux/arm/v7 $image -# done <<< "$IMAGES" +echo "Pulling images in docker" +while IFS= read -r image; do + docker exec -t dockerdebian docker pull --platform=linux/arm/v7 $image +done <<< "$IMAGES" + +ls docker-dir + # ls dockerpi # ls dockerpi/var/lib/tor # docker save $IMAGES -o umbrel-docker-images.tar From 8c3438224742d9a1de6fe28eccafc6b8ff572ab1 Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 10:43:16 +0530 Subject: [PATCH 061/104] Skip Qemu --- build-test.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build-test.sh b/build-test.sh index 0f56a44..b79f3b6 100755 --- a/build-test.sh +++ b/build-test.sh @@ -3,10 +3,11 @@ set -x uname -m # sudo apt-get install qemu binfmt-support qemu-user-static # Install the qemu packages -docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # This step will execute the registering scripts +# docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # This step will execute the registering scripts mkdir docker-dir -docker run --rm -dt --name dockerdebian -v docker-dir:/var/lib/docker multiarch/debian-debootstrap:armhf-buster-slim bash +# docker run --rm -dt --name dockerdebian -v docker-dir:/var/lib/docker multiarch/debian-debootstrap:armhf-buster-slim bash +docker run --rm -dt --name dockerdebian -v docker-dir:/var/lib/docker debian:buster-slim docker exec -t dockerdebian bash -c 'apt-get update; apt-get install curl -y; export CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt; curl -fsSL https://get.docker.com -o get-docker.sh; chmod +x get-docker.sh; sh get-docker.sh;' docker exec -t dockerdebian docker --version From 9a460021c5a06a1e45be49c7a66291294534d51b Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 10:43:34 +0530 Subject: [PATCH 062/104] Start bash --- build-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-test.sh b/build-test.sh index b79f3b6..348d301 100755 --- a/build-test.sh +++ b/build-test.sh @@ -7,7 +7,7 @@ uname -m mkdir docker-dir # docker run --rm -dt --name dockerdebian -v docker-dir:/var/lib/docker multiarch/debian-debootstrap:armhf-buster-slim bash -docker run --rm -dt --name dockerdebian -v docker-dir:/var/lib/docker debian:buster-slim +docker run --rm -dt --name dockerdebian -v docker-dir:/var/lib/docker debian:buster-slim bash docker exec -t dockerdebian bash -c 'apt-get update; apt-get install curl -y; export CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt; curl -fsSL https://get.docker.com -o get-docker.sh; chmod +x get-docker.sh; sh get-docker.sh;' docker exec -t dockerdebian docker --version From 38288959c7f03951cddc326c166cf98b23cb26df Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 11:02:36 +0530 Subject: [PATCH 063/104] Test Docker pull on host --- build-test.sh | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/build-test.sh b/build-test.sh index 348d301..1945d43 100755 --- a/build-test.sh +++ b/build-test.sh @@ -1,17 +1,21 @@ #!/bin/bash -e set -x -uname -m +# uname -m # sudo apt-get install qemu binfmt-support qemu-user-static # Install the qemu packages # docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # This step will execute the registering scripts -mkdir docker-dir +# mkdir docker-dir # docker run --rm -dt --name dockerdebian -v docker-dir:/var/lib/docker multiarch/debian-debootstrap:armhf-buster-slim bash -docker run --rm -dt --name dockerdebian -v docker-dir:/var/lib/docker debian:buster-slim bash -docker exec -t dockerdebian bash -c 'apt-get update; apt-get install curl -y; export CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt; curl -fsSL https://get.docker.com -o get-docker.sh; chmod +x get-docker.sh; sh get-docker.sh;' -docker exec -t dockerdebian docker --version +# docker run --rm -dt --name dockerdebian -v docker-dir:/var/lib/docker debian:buster-slim bash +# docker exec -t dockerdebian bash -c 'apt-get update; apt-get install curl -y; export CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt; curl -fsSL https://get.docker.com -o get-docker.sh; chmod +x get-docker.sh; sh get-docker.sh;' +# docker exec -t dockerdebian docker --version -ls docker-dir +# ls docker-dir + +ls /var/lib/docker/overlay2/ +docker rmi -f $(docker images -a -q) +ls /var/lib/docker/overlay2/ wget -q "https://raw.githubusercontent.com/getumbrel/umbrel-compose/master/docker-compose.yml" IMAGES=$(grep '^\s*image' docker-compose.yml | sed 's/image://' | sed 's/\"//g' | sed '/^$/d;s/[[:blank:]]//g' | sort | uniq) @@ -27,10 +31,10 @@ echo "List of images to download: $IMAGES" # docker exec -d dockerpi ./docker-install.sh echo "Pulling images in docker" while IFS= read -r image; do - docker exec -t dockerdebian docker pull --platform=linux/arm/v7 $image + docker pull --platform=linux/arm/v7 $image done <<< "$IMAGES" -ls docker-dir +ls /var/lib/docker/overlay2/ # ls dockerpi # ls dockerpi/var/lib/tor From 9759a89bbf458b9ddd948b0b3008b996eaf96c0a Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 11:19:01 +0530 Subject: [PATCH 064/104] Test with priviledge --- build-test.sh | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/build-test.sh b/build-test.sh index 1945d43..f873a2b 100755 --- a/build-test.sh +++ b/build-test.sh @@ -1,21 +1,17 @@ #!/bin/bash -e set -x -# uname -m +uname -m # sudo apt-get install qemu binfmt-support qemu-user-static # Install the qemu packages # docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # This step will execute the registering scripts -# mkdir docker-dir +mkdir docker-dir # docker run --rm -dt --name dockerdebian -v docker-dir:/var/lib/docker multiarch/debian-debootstrap:armhf-buster-slim bash -# docker run --rm -dt --name dockerdebian -v docker-dir:/var/lib/docker debian:buster-slim bash -# docker exec -t dockerdebian bash -c 'apt-get update; apt-get install curl -y; export CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt; curl -fsSL https://get.docker.com -o get-docker.sh; chmod +x get-docker.sh; sh get-docker.sh;' -# docker exec -t dockerdebian docker --version +docker run --rm -dt --name dockerdebian --privileged=true -v docker-dir:/var/lib/docker debian:buster-slim bash +docker exec -t dockerdebian bash -c 'apt-get update; apt-get install curl -y; export CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt; curl -fsSL https://get.docker.com -o get-docker.sh; chmod +x get-docker.sh; sh get-docker.sh;' +docker exec -t dockerdebian docker --version -# ls docker-dir - -ls /var/lib/docker/overlay2/ -docker rmi -f $(docker images -a -q) -ls /var/lib/docker/overlay2/ +ls docker-dir wget -q "https://raw.githubusercontent.com/getumbrel/umbrel-compose/master/docker-compose.yml" IMAGES=$(grep '^\s*image' docker-compose.yml | sed 's/image://' | sed 's/\"//g' | sed '/^$/d;s/[[:blank:]]//g' | sort | uniq) @@ -31,10 +27,10 @@ echo "List of images to download: $IMAGES" # docker exec -d dockerpi ./docker-install.sh echo "Pulling images in docker" while IFS= read -r image; do - docker pull --platform=linux/arm/v7 $image + docker exec -t dockerdebian docker pull --platform=linux/arm/v7 $image done <<< "$IMAGES" -ls /var/lib/docker/overlay2/ +ls docker-dir # ls dockerpi # ls dockerpi/var/lib/tor From 732ba52673a22118556d17b5ec89746f790fd597 Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 11:23:16 +0530 Subject: [PATCH 065/104] Try again with priviledged --- build-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-test.sh b/build-test.sh index f873a2b..6adcaba 100755 --- a/build-test.sh +++ b/build-test.sh @@ -7,7 +7,7 @@ uname -m mkdir docker-dir # docker run --rm -dt --name dockerdebian -v docker-dir:/var/lib/docker multiarch/debian-debootstrap:armhf-buster-slim bash -docker run --rm -dt --name dockerdebian --privileged=true -v docker-dir:/var/lib/docker debian:buster-slim bash +docker run --rm -dt --name dockerdebian --privileged -v docker-dir:/var/lib/docker debian:buster-slim bash docker exec -t dockerdebian bash -c 'apt-get update; apt-get install curl -y; export CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt; curl -fsSL https://get.docker.com -o get-docker.sh; chmod +x get-docker.sh; sh get-docker.sh;' docker exec -t dockerdebian docker --version From a79c7ced917ddf48c0b5616bf6bb2978d2abd3dc Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 11:25:22 +0530 Subject: [PATCH 066/104] Check service status --- build-test.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build-test.sh b/build-test.sh index 6adcaba..5118977 100755 --- a/build-test.sh +++ b/build-test.sh @@ -10,6 +10,8 @@ mkdir docker-dir docker run --rm -dt --name dockerdebian --privileged -v docker-dir:/var/lib/docker debian:buster-slim bash docker exec -t dockerdebian bash -c 'apt-get update; apt-get install curl -y; export CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt; curl -fsSL https://get.docker.com -o get-docker.sh; chmod +x get-docker.sh; sh get-docker.sh;' docker exec -t dockerdebian docker --version +docker exec -t dockerdebian service docker status +docker exec -t dockerdebian service docker start ls docker-dir From b481868944525d6ae0939bfc6caa1ff1dbd586df Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 11:28:30 +0530 Subject: [PATCH 067/104] Try dind --- build-test.sh | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/build-test.sh b/build-test.sh index 5118977..d570966 100755 --- a/build-test.sh +++ b/build-test.sh @@ -7,11 +7,9 @@ uname -m mkdir docker-dir # docker run --rm -dt --name dockerdebian -v docker-dir:/var/lib/docker multiarch/debian-debootstrap:armhf-buster-slim bash -docker run --rm -dt --name dockerdebian --privileged -v docker-dir:/var/lib/docker debian:buster-slim bash -docker exec -t dockerdebian bash -c 'apt-get update; apt-get install curl -y; export CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt; curl -fsSL https://get.docker.com -o get-docker.sh; chmod +x get-docker.sh; sh get-docker.sh;' -docker exec -t dockerdebian docker --version -docker exec -t dockerdebian service docker status -docker exec -t dockerdebian service docker start +docker run --rm -dt --name dind -v docker-dir:/var/lib/docker docker:stable-dind bash +# docker exec -t dockerdebian bash -c 'apt-get update; apt-get install curl -y; export CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt; curl -fsSL https://get.docker.com -o get-docker.sh; chmod +x get-docker.sh; sh get-docker.sh;' +docker exec -t dind docker --version ls docker-dir @@ -29,7 +27,7 @@ echo "List of images to download: $IMAGES" # docker exec -d dockerpi ./docker-install.sh echo "Pulling images in docker" while IFS= read -r image; do - docker exec -t dockerdebian docker pull --platform=linux/arm/v7 $image + docker exec -t dind docker pull --platform=linux/arm/v7 $image done <<< "$IMAGES" ls docker-dir From 25391f1447eb8693cade59d2ac2ef99e0d7610a6 Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 11:29:53 +0530 Subject: [PATCH 068/104] Only run to pull --- build-test.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build-test.sh b/build-test.sh index d570966..0af0843 100755 --- a/build-test.sh +++ b/build-test.sh @@ -7,9 +7,9 @@ uname -m mkdir docker-dir # docker run --rm -dt --name dockerdebian -v docker-dir:/var/lib/docker multiarch/debian-debootstrap:armhf-buster-slim bash -docker run --rm -dt --name dind -v docker-dir:/var/lib/docker docker:stable-dind bash +# docker run --rm -dt --name dind -v docker-dir:/var/lib/docker docker:stable-dind bash # docker exec -t dockerdebian bash -c 'apt-get update; apt-get install curl -y; export CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt; curl -fsSL https://get.docker.com -o get-docker.sh; chmod +x get-docker.sh; sh get-docker.sh;' -docker exec -t dind docker --version +# docker exec -t dind docker --version ls docker-dir @@ -27,7 +27,7 @@ echo "List of images to download: $IMAGES" # docker exec -d dockerpi ./docker-install.sh echo "Pulling images in docker" while IFS= read -r image; do - docker exec -t dind docker pull --platform=linux/arm/v7 $image + docker run --rm -dt --name dind -v docker-dir:/var/lib/docker docker:stable-dind docker pull --platform=linux/arm/v7 $image done <<< "$IMAGES" ls docker-dir From e0fe70da91c1f6116951a6559f33f00071a5eaa0 Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 11:31:26 +0530 Subject: [PATCH 069/104] Remove container name --- build-test.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build-test.sh b/build-test.sh index 0af0843..7ababcc 100755 --- a/build-test.sh +++ b/build-test.sh @@ -25,11 +25,12 @@ echo "List of images to download: $IMAGES" # docker exec -d dockerpi curl -fsSL https://get.docker.com -o docker-install.sh # echo "Running install script" # docker exec -d dockerpi ./docker-install.sh +docker ps echo "Pulling images in docker" while IFS= read -r image; do - docker run --rm -dt --name dind -v docker-dir:/var/lib/docker docker:stable-dind docker pull --platform=linux/arm/v7 $image + docker run --rm -dt -v docker-dir:/var/lib/docker docker:stable-dind docker pull --platform=linux/arm/v7 $image done <<< "$IMAGES" - +docker ps ls docker-dir # ls dockerpi From 6f0547f82438ddd4b3d41443245be8f545eeace7 Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 11:32:36 +0530 Subject: [PATCH 070/104] Don't run in detached --- build-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-test.sh b/build-test.sh index 7ababcc..2451a1a 100755 --- a/build-test.sh +++ b/build-test.sh @@ -28,7 +28,7 @@ echo "List of images to download: $IMAGES" docker ps echo "Pulling images in docker" while IFS= read -r image; do - docker run --rm -dt -v docker-dir:/var/lib/docker docker:stable-dind docker pull --platform=linux/arm/v7 $image + docker run --rm -t -v docker-dir:/var/lib/docker docker:stable-dind docker pull --platform=linux/arm/v7 $image done <<< "$IMAGES" docker ps ls docker-dir From a862b17b9f9194dfadab5c27fe245f28607c42b3 Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 11:52:07 +0530 Subject: [PATCH 071/104] Re-install Docker --- build-test.sh | 48 +++++++++++++++++++++++------------------------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/build-test.sh b/build-test.sh index 2451a1a..bb66a80 100755 --- a/build-test.sh +++ b/build-test.sh @@ -2,36 +2,34 @@ set -x uname -m -# sudo apt-get install qemu binfmt-support qemu-user-static # Install the qemu packages -# docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # This step will execute the registering scripts -mkdir docker-dir -# docker run --rm -dt --name dockerdebian -v docker-dir:/var/lib/docker multiarch/debian-debootstrap:armhf-buster-slim bash -# docker run --rm -dt --name dind -v docker-dir:/var/lib/docker docker:stable-dind bash -# docker exec -t dockerdebian bash -c 'apt-get update; apt-get install curl -y; export CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt; curl -fsSL https://get.docker.com -o get-docker.sh; chmod +x get-docker.sh; sh get-docker.sh;' -# docker exec -t dind docker --version +echo "Removing Docker" +docker ps +docker --version +sudo apt-get purge docker-ce docker-ce-cli containerd.io moby-engine +sudo rm -rf /var/lib/docker + +ls /var/lib + +curl -fsSL https://get.docker.com -o get-docker.sh +sudo sh get-docker.sh +docker --version +docker ps -ls docker-dir +ls /var/lib -wget -q "https://raw.githubusercontent.com/getumbrel/umbrel-compose/master/docker-compose.yml" -IMAGES=$(grep '^\s*image' docker-compose.yml | sed 's/image://' | sed 's/\"//g' | sed '/^$/d;s/[[:blank:]]//g' | sort | uniq) -echo "List of images to download: $IMAGES" -# mkdir dockerpi -# echo "Running Docker Pi" -# docker run --name dockerpi -v dockerpi:/sdcard lukechilds/dockerpi +# wget -q "https://raw.githubusercontent.com/getumbrel/umbrel-compose/master/docker-compose.yml" +# IMAGES=$(grep '^\s*image' docker-compose.yml | sed 's/image://' | sed 's/\"//g' | sed '/^$/d;s/[[:blank:]]//g' | sort | uniq) +# echo "List of images to download: $IMAGES" + + +# echo "Pulling images in docker" +# while IFS= read -r image; do +# docker run --rm -t -v docker-dir:/var/lib/docker docker:stable-dind docker pull --platform=linux/arm/v7 $image +# done <<< "$IMAGES" # docker ps -# echo "Downloading install script" -# docker exec -d dockerpi curl -fsSL https://get.docker.com -o docker-install.sh -# echo "Running install script" -# docker exec -d dockerpi ./docker-install.sh -docker ps -echo "Pulling images in docker" -while IFS= read -r image; do - docker run --rm -t -v docker-dir:/var/lib/docker docker:stable-dind docker pull --platform=linux/arm/v7 $image -done <<< "$IMAGES" -docker ps -ls docker-dir +# ls docker-dir # ls dockerpi # ls dockerpi/var/lib/tor From f231a81a3574d555f786c43c5adffcba4a9b13b6 Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 11:59:40 +0530 Subject: [PATCH 072/104] Download images --- build-test.sh | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/build-test.sh b/build-test.sh index bb66a80..60ed02c 100755 --- a/build-test.sh +++ b/build-test.sh @@ -5,32 +5,38 @@ uname -m echo "Removing Docker" docker ps -docker --version -sudo apt-get purge docker-ce docker-ce-cli containerd.io moby-engine +docker images -a +du -h /var/lib/docker/overlay2 +sudo systemctl restart stop +sudo apt-get purge docker-ce docker-ce-cli containerd.io moby-engine moby-cli sudo rm -rf /var/lib/docker -ls /var/lib - curl -fsSL https://get.docker.com -o get-docker.sh sudo sh get-docker.sh docker --version docker ps +docker images -a +sudo rm -rf /etc/docker/daemon.json +echo '{"experimental": true}' | sudo tee -a /etc/docker/daemon.json + +ls /var/lib/docker +ls /var/lib/docker/overlay2 -ls /var/lib +wget -q "https://raw.githubusercontent.com/getumbrel/umbrel-compose/master/docker-compose.yml" +IMAGES=$(grep '^\s*image' docker-compose.yml | sed 's/image://' | sed 's/\"//g' | sed '/^$/d;s/[[:blank:]]//g' | sort | uniq) -# wget -q "https://raw.githubusercontent.com/getumbrel/umbrel-compose/master/docker-compose.yml" -# IMAGES=$(grep '^\s*image' docker-compose.yml | sed 's/image://' | sed 's/\"//g' | sed '/^$/d;s/[[:blank:]]//g' | sort | uniq) -# echo "List of images to download: $IMAGES" +echo "Pulling images in docker" +while IFS= read -r image; do + docker run --rm -t -v docker-dir:/var/lib/docker docker:stable-dind docker pull --platform=linux/arm/v7 $image +done <<< "$IMAGES" +docker images -a -# echo "Pulling images in docker" -# while IFS= read -r image; do -# docker run --rm -t -v docker-dir:/var/lib/docker docker:stable-dind docker pull --platform=linux/arm/v7 $image -# done <<< "$IMAGES" -# docker ps -# ls docker-dir +ls /var/lib/docker +ls /var/lib/docker/overlay2 +du -h /var/lib/docker/overlay2 # ls dockerpi # ls dockerpi/var/lib/tor # docker save $IMAGES -o umbrel-docker-images.tar From b305d03fa8a3a0601e1e2761280920f47c3ccfd1 Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 12:01:19 +0530 Subject: [PATCH 073/104] Stop docker --- build-test.sh | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/build-test.sh b/build-test.sh index 60ed02c..43b0ded 100755 --- a/build-test.sh +++ b/build-test.sh @@ -7,7 +7,7 @@ echo "Removing Docker" docker ps docker images -a du -h /var/lib/docker/overlay2 -sudo systemctl restart stop +sudo systemctl stop docker sudo apt-get purge docker-ce docker-ce-cli containerd.io moby-engine moby-cli sudo rm -rf /var/lib/docker @@ -19,9 +19,6 @@ docker images -a sudo rm -rf /etc/docker/daemon.json echo '{"experimental": true}' | sudo tee -a /etc/docker/daemon.json -ls /var/lib/docker -ls /var/lib/docker/overlay2 - wget -q "https://raw.githubusercontent.com/getumbrel/umbrel-compose/master/docker-compose.yml" IMAGES=$(grep '^\s*image' docker-compose.yml | sed 's/image://' | sed 's/\"//g' | sed '/^$/d;s/[[:blank:]]//g' | sort | uniq) @@ -34,8 +31,6 @@ done <<< "$IMAGES" docker images -a -ls /var/lib/docker -ls /var/lib/docker/overlay2 du -h /var/lib/docker/overlay2 # ls dockerpi # ls dockerpi/var/lib/tor From 6eba4f6f5d3e56b6fb2e7aef1e362717a594638a Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 12:03:48 +0530 Subject: [PATCH 074/104] Remove dind --- build-test.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build-test.sh b/build-test.sh index 43b0ded..41fbdd2 100755 --- a/build-test.sh +++ b/build-test.sh @@ -6,7 +6,7 @@ uname -m echo "Removing Docker" docker ps docker images -a -du -h /var/lib/docker/overlay2 +du -sh /var/lib/docker/overlay2 sudo systemctl stop docker sudo apt-get purge docker-ce docker-ce-cli containerd.io moby-engine moby-cli sudo rm -rf /var/lib/docker @@ -26,12 +26,12 @@ IMAGES=$(grep '^\s*image' docker-compose.yml | sed 's/image://' | sed 's/\"//g' echo "Pulling images in docker" while IFS= read -r image; do - docker run --rm -t -v docker-dir:/var/lib/docker docker:stable-dind docker pull --platform=linux/arm/v7 $image + docker pull --platform=linux/arm/v7 $image done <<< "$IMAGES" docker images -a -du -h /var/lib/docker/overlay2 +du -sh /var/lib/docker/overlay2 # ls dockerpi # ls dockerpi/var/lib/tor # docker save $IMAGES -o umbrel-docker-images.tar From 8cb0a4bb9155fc64edb1f6d810130dd2269a77a5 Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 12:10:32 +0530 Subject: [PATCH 075/104] Copy Docker data to Rootfs --- stage2/05-docker-compose/01-run.sh | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/stage2/05-docker-compose/01-run.sh b/stage2/05-docker-compose/01-run.sh index dcae09e..7ae68d1 100755 --- a/stage2/05-docker-compose/01-run.sh +++ b/stage2/05-docker-compose/01-run.sh @@ -33,6 +33,23 @@ echo "Docker stuff installed!" echo "Bundling Docker images required to run Umbrel services" +echo "Removing Docker" +docker ps +docker images -a +du -sh /var/lib/docker/overlay2 +sudo systemctl stop docker +sudo apt-get purge docker-ce docker-ce-cli containerd.io moby-engine moby-cli +sudo rm -rf /var/lib/docker + +curl -fsSL https://get.docker.com -o get-docker.sh +sudo sh get-docker.sh +docker --version +docker ps +docker images -a +sudo rm -rf /etc/docker/daemon.json +echo '{"experimental": true}' | sudo tee -a /etc/docker/daemon.json + + wget -q "https://raw.githubusercontent.com/getumbrel/umbrel-compose/master/docker-compose.yml" IMAGES=$(grep '^\s*image' docker-compose.yml | sed 's/image://' | sed 's/\"//g' | sed '/^$/d;s/[[:blank:]]//g' | sort | uniq) echo "List of images to download: $IMAGES" @@ -41,6 +58,7 @@ while IFS= read -r image; do docker pull --platform=linux/arm/v7 $image done <<< "$IMAGES" -docker save $IMAGES | gzip > umbrel-docker-images.tar.gz -du -h umbrel-docker-images.tar.gz -cp umbrel-docker-images.tar.gz ${ROOTFS_DIR}/home/${FIRST_USER_NAME}/umbrel-docker-images.tar.gz \ No newline at end of file +# docker save $IMAGES | gzip > umbrel-docker-images.tar.gz +# du -h umbrel-docker-images.tar.gz +rsync -avPHSX /var/lib/docker ${ROOTFS_DIR}/var/lib/docker/ +# cp umbrel-docker-images.tar.gz ${ROOTFS_DIR}/home/${FIRST_USER_NAME}/umbrel-docker-images.tar.gz \ No newline at end of file From eee512fde9fb77adf32216644e1f710c1a415abc Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 12:11:58 +0530 Subject: [PATCH 076/104] Copy only overlay2 --- stage2/05-docker-compose/01-run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stage2/05-docker-compose/01-run.sh b/stage2/05-docker-compose/01-run.sh index 7ae68d1..eccd4ea 100755 --- a/stage2/05-docker-compose/01-run.sh +++ b/stage2/05-docker-compose/01-run.sh @@ -60,5 +60,5 @@ done <<< "$IMAGES" # docker save $IMAGES | gzip > umbrel-docker-images.tar.gz # du -h umbrel-docker-images.tar.gz -rsync -avPHSX /var/lib/docker ${ROOTFS_DIR}/var/lib/docker/ +rsync -avPHSX /var/lib/docker/overlay2 ${ROOTFS_DIR}/var/lib/docker/overlay2/ # cp umbrel-docker-images.tar.gz ${ROOTFS_DIR}/home/${FIRST_USER_NAME}/umbrel-docker-images.tar.gz \ No newline at end of file From bc9946e8096a3314cb14a5a844013cf766337132 Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 15:59:59 +0530 Subject: [PATCH 077/104] overlay2 only run --- stage2/05-docker-compose/01-run.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/stage2/05-docker-compose/01-run.sh b/stage2/05-docker-compose/01-run.sh index eccd4ea..ecb9f5e 100755 --- a/stage2/05-docker-compose/01-run.sh +++ b/stage2/05-docker-compose/01-run.sh @@ -60,5 +60,6 @@ done <<< "$IMAGES" # docker save $IMAGES | gzip > umbrel-docker-images.tar.gz # du -h umbrel-docker-images.tar.gz +mkdir -p ${ROOTFS_DIR}/var/lib/docker/overlay2 rsync -avPHSX /var/lib/docker/overlay2 ${ROOTFS_DIR}/var/lib/docker/overlay2/ # cp umbrel-docker-images.tar.gz ${ROOTFS_DIR}/home/${FIRST_USER_NAME}/umbrel-docker-images.tar.gz \ No newline at end of file From 3e0fa3d3092d77baa9e5b5594801d943f995af7f Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 16:03:24 +0530 Subject: [PATCH 078/104] Copy entire docker dir --- stage2/05-docker-compose/01-run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stage2/05-docker-compose/01-run.sh b/stage2/05-docker-compose/01-run.sh index ecb9f5e..6efbf1a 100755 --- a/stage2/05-docker-compose/01-run.sh +++ b/stage2/05-docker-compose/01-run.sh @@ -60,6 +60,6 @@ done <<< "$IMAGES" # docker save $IMAGES | gzip > umbrel-docker-images.tar.gz # du -h umbrel-docker-images.tar.gz -mkdir -p ${ROOTFS_DIR}/var/lib/docker/overlay2 -rsync -avPHSX /var/lib/docker/overlay2 ${ROOTFS_DIR}/var/lib/docker/overlay2/ +# mkdir -p ${ROOTFS_DIR}/var/lib/docker/overlay2 +rsync -avPHSX /var/lib/docker ${ROOTFS_DIR}/var/lib/docker/ # cp umbrel-docker-images.tar.gz ${ROOTFS_DIR}/home/${FIRST_USER_NAME}/umbrel-docker-images.tar.gz \ No newline at end of file From ba49f4e617660e78b4280964af76900b2dfc0d96 Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 16:15:04 +0530 Subject: [PATCH 079/104] v2.0.0 github-release action --- .github/workflows/on-tag-only.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/on-tag-only.yml b/.github/workflows/on-tag-only.yml index 3228ece..f746833 100644 --- a/.github/workflows/on-tag-only.yml +++ b/.github/workflows/on-tag-only.yml @@ -33,7 +33,7 @@ jobs: run: ls -la - name: Release Image to placeholder or release - uses: meeDamian/github-release@v1.0.0 + uses: meeDamian/github-release@v2.0.0 if: startsWith(github.ref, 'refs/tags/') with: gzip: false From 7b6ab5a3454543c0bf07953142364c76be900bd3 Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 16:19:33 +0530 Subject: [PATCH 080/104] Version v2.0 release action --- .github/workflows/on-tag-only.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/on-tag-only.yml b/.github/workflows/on-tag-only.yml index f746833..48c662e 100644 --- a/.github/workflows/on-tag-only.yml +++ b/.github/workflows/on-tag-only.yml @@ -33,7 +33,7 @@ jobs: run: ls -la - name: Release Image to placeholder or release - uses: meeDamian/github-release@v2.0.0 + uses: meeDamian/github-release@v2.0 if: startsWith(github.ref, 'refs/tags/') with: gzip: false From 1978fc8de1130608ca4581b3bc450735a01193d0 Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 16:20:09 +0530 Subject: [PATCH 081/104] Copy only overlay2 --- stage2/05-docker-compose/01-run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stage2/05-docker-compose/01-run.sh b/stage2/05-docker-compose/01-run.sh index 6efbf1a..ecb9f5e 100755 --- a/stage2/05-docker-compose/01-run.sh +++ b/stage2/05-docker-compose/01-run.sh @@ -60,6 +60,6 @@ done <<< "$IMAGES" # docker save $IMAGES | gzip > umbrel-docker-images.tar.gz # du -h umbrel-docker-images.tar.gz -# mkdir -p ${ROOTFS_DIR}/var/lib/docker/overlay2 -rsync -avPHSX /var/lib/docker ${ROOTFS_DIR}/var/lib/docker/ +mkdir -p ${ROOTFS_DIR}/var/lib/docker/overlay2 +rsync -avPHSX /var/lib/docker/overlay2 ${ROOTFS_DIR}/var/lib/docker/overlay2/ # cp umbrel-docker-images.tar.gz ${ROOTFS_DIR}/home/${FIRST_USER_NAME}/umbrel-docker-images.tar.gz \ No newline at end of file From 6fe6d20889e8706c8bc15186f2498972305b7d8e Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 16:21:04 +0530 Subject: [PATCH 082/104] Action syntax --- .github/workflows/on-tag-only.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/on-tag-only.yml b/.github/workflows/on-tag-only.yml index 48c662e..67b72da 100644 --- a/.github/workflows/on-tag-only.yml +++ b/.github/workflows/on-tag-only.yml @@ -33,7 +33,7 @@ jobs: run: ls -la - name: Release Image to placeholder or release - uses: meeDamian/github-release@v2.0 + uses: meeDamian/github-release@2.0 if: startsWith(github.ref, 'refs/tags/') with: gzip: false From 0b857895e99cb0121e60e8e97211cd0db578f2d8 Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 16:21:45 +0530 Subject: [PATCH 083/104] Copy entire docker lib --- stage2/05-docker-compose/01-run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stage2/05-docker-compose/01-run.sh b/stage2/05-docker-compose/01-run.sh index ecb9f5e..6efbf1a 100755 --- a/stage2/05-docker-compose/01-run.sh +++ b/stage2/05-docker-compose/01-run.sh @@ -60,6 +60,6 @@ done <<< "$IMAGES" # docker save $IMAGES | gzip > umbrel-docker-images.tar.gz # du -h umbrel-docker-images.tar.gz -mkdir -p ${ROOTFS_DIR}/var/lib/docker/overlay2 -rsync -avPHSX /var/lib/docker/overlay2 ${ROOTFS_DIR}/var/lib/docker/overlay2/ +# mkdir -p ${ROOTFS_DIR}/var/lib/docker/overlay2 +rsync -avPHSX /var/lib/docker ${ROOTFS_DIR}/var/lib/docker/ # cp umbrel-docker-images.tar.gz ${ROOTFS_DIR}/home/${FIRST_USER_NAME}/umbrel-docker-images.tar.gz \ No newline at end of file From 527b37e90b59a4043879218c89f8af140b94adf5 Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 16:25:41 +0530 Subject: [PATCH 084/104] Copy only overlay2 --- stage2/05-docker-compose/01-run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stage2/05-docker-compose/01-run.sh b/stage2/05-docker-compose/01-run.sh index 6efbf1a..ecb9f5e 100755 --- a/stage2/05-docker-compose/01-run.sh +++ b/stage2/05-docker-compose/01-run.sh @@ -60,6 +60,6 @@ done <<< "$IMAGES" # docker save $IMAGES | gzip > umbrel-docker-images.tar.gz # du -h umbrel-docker-images.tar.gz -# mkdir -p ${ROOTFS_DIR}/var/lib/docker/overlay2 -rsync -avPHSX /var/lib/docker ${ROOTFS_DIR}/var/lib/docker/ +mkdir -p ${ROOTFS_DIR}/var/lib/docker/overlay2 +rsync -avPHSX /var/lib/docker/overlay2 ${ROOTFS_DIR}/var/lib/docker/overlay2/ # cp umbrel-docker-images.tar.gz ${ROOTFS_DIR}/home/${FIRST_USER_NAME}/umbrel-docker-images.tar.gz \ No newline at end of file From a070173e0ddfe551eb4526cc2fc2e3584b6e86fe Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 23:40:41 +0530 Subject: [PATCH 085/104] Upload artifact - overlay2 --- .github/workflows/on-tag-only.yml | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/.github/workflows/on-tag-only.yml b/.github/workflows/on-tag-only.yml index 67b72da..d8d6f80 100644 --- a/.github/workflows/on-tag-only.yml +++ b/.github/workflows/on-tag-only.yml @@ -32,11 +32,16 @@ jobs: - name: Debug current working dir run: ls -la - - name: Release Image to placeholder or release - uses: meeDamian/github-release@2.0 - if: startsWith(github.ref, 'refs/tags/') + # - name: Release Image to placeholder or release + # uses: meeDamian/github-release@2.0 + # if: startsWith(github.ref, 'refs/tags/') + # with: + # gzip: false + # files: "*.zip" + # token: ${{ secrets.GITHUB_TOKEN }} + # allow_override: true + + - uses: actions/upload-artifact@v2 + - name: Upload artifact with: - gzip: false - files: "*.zip" - token: ${{ secrets.GITHUB_TOKEN }} - allow_override: true + path: *.zip From 6180aee9cf3805bf22efb5fb57464bf447261ca5 Mon Sep 17 00:00:00 2001 From: Mayank Date: Sun, 5 Jul 2020 23:48:53 +0530 Subject: [PATCH 086/104] Trigger build for entire docker dir --- stage2/05-docker-compose/01-run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stage2/05-docker-compose/01-run.sh b/stage2/05-docker-compose/01-run.sh index ecb9f5e..0bab652 100755 --- a/stage2/05-docker-compose/01-run.sh +++ b/stage2/05-docker-compose/01-run.sh @@ -60,6 +60,6 @@ done <<< "$IMAGES" # docker save $IMAGES | gzip > umbrel-docker-images.tar.gz # du -h umbrel-docker-images.tar.gz -mkdir -p ${ROOTFS_DIR}/var/lib/docker/overlay2 -rsync -avPHSX /var/lib/docker/overlay2 ${ROOTFS_DIR}/var/lib/docker/overlay2/ +mkdir -p ${ROOTFS_DIR}/var/lib/docker +rsync -avPHSX /var/lib/docker ${ROOTFS_DIR}/var/lib/docker/ # cp umbrel-docker-images.tar.gz ${ROOTFS_DIR}/home/${FIRST_USER_NAME}/umbrel-docker-images.tar.gz \ No newline at end of file From b607ff5ee99e68b8cc653a5458b4915ca2279ead Mon Sep 17 00:00:00 2001 From: Mayank Date: Mon, 6 Jul 2020 00:01:19 +0530 Subject: [PATCH 087/104] Wrap path --- .github/workflows/on-tag-only.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/on-tag-only.yml b/.github/workflows/on-tag-only.yml index d8d6f80..d928806 100644 --- a/.github/workflows/on-tag-only.yml +++ b/.github/workflows/on-tag-only.yml @@ -44,4 +44,4 @@ jobs: - uses: actions/upload-artifact@v2 - name: Upload artifact with: - path: *.zip + path: "*.zip" From cf56e14b7b6f80855a178d3ed11f9ad789cec88d Mon Sep 17 00:00:00 2001 From: Mayank Date: Mon, 6 Jul 2020 00:04:44 +0530 Subject: [PATCH 088/104] Copy only overlay2 --- stage2/05-docker-compose/01-run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stage2/05-docker-compose/01-run.sh b/stage2/05-docker-compose/01-run.sh index 0bab652..ecb9f5e 100755 --- a/stage2/05-docker-compose/01-run.sh +++ b/stage2/05-docker-compose/01-run.sh @@ -60,6 +60,6 @@ done <<< "$IMAGES" # docker save $IMAGES | gzip > umbrel-docker-images.tar.gz # du -h umbrel-docker-images.tar.gz -mkdir -p ${ROOTFS_DIR}/var/lib/docker -rsync -avPHSX /var/lib/docker ${ROOTFS_DIR}/var/lib/docker/ +mkdir -p ${ROOTFS_DIR}/var/lib/docker/overlay2 +rsync -avPHSX /var/lib/docker/overlay2 ${ROOTFS_DIR}/var/lib/docker/overlay2/ # cp umbrel-docker-images.tar.gz ${ROOTFS_DIR}/home/${FIRST_USER_NAME}/umbrel-docker-images.tar.gz \ No newline at end of file From 6c14500291714bf8c63a47e1fb9daff27908b59c Mon Sep 17 00:00:00 2001 From: Mayank Date: Mon, 6 Jul 2020 00:08:45 +0530 Subject: [PATCH 089/104] Test run --- .github/workflows/on-tag-only.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/on-tag-only.yml b/.github/workflows/on-tag-only.yml index d928806..aaaf7ca 100644 --- a/.github/workflows/on-tag-only.yml +++ b/.github/workflows/on-tag-only.yml @@ -41,7 +41,7 @@ jobs: # token: ${{ secrets.GITHUB_TOKEN }} # allow_override: true - - uses: actions/upload-artifact@v2 - name: Upload artifact + uses: actions/upload-artifact@v2 with: path: "*.zip" From 3a820b14b2ebc8f47b3ab6d6719108e77a417567 Mon Sep 17 00:00:00 2001 From: Mayank Date: Mon, 6 Jul 2020 00:10:58 +0530 Subject: [PATCH 090/104] Disable push workflow on tag --- .github/workflows/on-push.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/on-push.yml b/.github/workflows/on-push.yml index e07550d..78b6b7b 100644 --- a/.github/workflows/on-push.yml +++ b/.github/workflows/on-push.yml @@ -1,6 +1,8 @@ name: Testing only on: - push + push: + branch: + - * jobs: build: From ebbf6fd1a641570e0a44d6b37f7254e820e1870d Mon Sep 17 00:00:00 2001 From: Mayank Date: Mon, 6 Jul 2020 00:12:30 +0530 Subject: [PATCH 091/104] Coopy entire Docker lib --- stage2/05-docker-compose/01-run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stage2/05-docker-compose/01-run.sh b/stage2/05-docker-compose/01-run.sh index ecb9f5e..0bab652 100755 --- a/stage2/05-docker-compose/01-run.sh +++ b/stage2/05-docker-compose/01-run.sh @@ -60,6 +60,6 @@ done <<< "$IMAGES" # docker save $IMAGES | gzip > umbrel-docker-images.tar.gz # du -h umbrel-docker-images.tar.gz -mkdir -p ${ROOTFS_DIR}/var/lib/docker/overlay2 -rsync -avPHSX /var/lib/docker/overlay2 ${ROOTFS_DIR}/var/lib/docker/overlay2/ +mkdir -p ${ROOTFS_DIR}/var/lib/docker +rsync -avPHSX /var/lib/docker ${ROOTFS_DIR}/var/lib/docker/ # cp umbrel-docker-images.tar.gz ${ROOTFS_DIR}/home/${FIRST_USER_NAME}/umbrel-docker-images.tar.gz \ No newline at end of file From 374404e2a77c7cc5e1a7ac6b4208d03b949c5116 Mon Sep 17 00:00:00 2001 From: Mayank Date: Mon, 6 Jul 2020 09:14:06 +0530 Subject: [PATCH 092/104] Download overlay2 Run 1 --- .github/workflows/on-push.yml | 10 +++- .github/workflows/on-tag-only.yml | 88 +++++++++++++++---------------- build-test.sh | 14 +++-- 3 files changed, 61 insertions(+), 51 deletions(-) diff --git a/.github/workflows/on-push.yml b/.github/workflows/on-push.yml index 78b6b7b..6917491 100644 --- a/.github/workflows/on-push.yml +++ b/.github/workflows/on-push.yml @@ -1,8 +1,7 @@ name: Testing only on: push: - branch: - - * + tags: [ '*' ] jobs: build: @@ -16,3 +15,10 @@ jobs: sudo systemctl restart docker - name: Run Build script run: sudo ./build-test.sh + - name: Release zip + uses: meeDamian/github-release@2.0 + with: + gzip: false + files: "overlay2.tar.gz" + token: ${{ secrets.GITHUB_TOKEN }} + allow_override: true diff --git a/.github/workflows/on-tag-only.yml b/.github/workflows/on-tag-only.yml index aaaf7ca..d3dcdb5 100644 --- a/.github/workflows/on-tag-only.yml +++ b/.github/workflows/on-tag-only.yml @@ -1,47 +1,47 @@ -name: Automatically Build image on tag ONLY -on: - push: - tags: [ '*' ] +# name: Automatically Build image on tag ONLY +# on: +# push: +# tags: [ '*' ] -jobs: - build: +# jobs: +# build: - runs-on: ubuntu-18.04 +# runs-on: ubuntu-18.04 - steps: - - - uses: actions/checkout@v1 - - name: Run Apt-get update - run: sudo apt-get update - - - name: Install Dependencies - run: sudo apt-get -y install quilt qemu-user-static debootstrap bsdtar - - - name: Enable experimental features in Docker - run: | - sudo rm -rf /etc/docker/daemon.json - echo '{"experimental": true}' | sudo tee -a /etc/docker/daemon.json - sudo systemctl restart docker - - - name: Run Build script - run: sudo ./build.sh - - - name: Copy ZIP to current working dir - run: cp deploy/*.zip ./ - - - name: Debug current working dir - run: ls -la - - # - name: Release Image to placeholder or release - # uses: meeDamian/github-release@2.0 - # if: startsWith(github.ref, 'refs/tags/') - # with: - # gzip: false - # files: "*.zip" - # token: ${{ secrets.GITHUB_TOKEN }} - # allow_override: true - - - name: Upload artifact - uses: actions/upload-artifact@v2 - with: - path: "*.zip" +# steps: + +# - uses: actions/checkout@v1 +# - name: Run Apt-get update +# run: sudo apt-get update + +# - name: Install Dependencies +# run: sudo apt-get -y install quilt qemu-user-static debootstrap bsdtar + +# - name: Enable experimental features in Docker +# run: | +# sudo rm -rf /etc/docker/daemon.json +# echo '{"experimental": true}' | sudo tee -a /etc/docker/daemon.json +# sudo systemctl restart docker + +# - name: Run Build script +# run: sudo ./build.sh + +# - name: Copy ZIP to current working dir +# run: cp deploy/*.zip ./ + +# - name: Debug current working dir +# run: ls -la + +# # - name: Release Image to placeholder or release +# # uses: meeDamian/github-release@2.0 +# # if: startsWith(github.ref, 'refs/tags/') +# # with: +# # gzip: false +# # files: "*.zip" +# # token: ${{ secrets.GITHUB_TOKEN }} +# # allow_override: true + +# - name: Upload artifact +# uses: actions/upload-artifact@v2 +# with: +# path: "*.zip" diff --git a/build-test.sh b/build-test.sh index 41fbdd2..e55a378 100755 --- a/build-test.sh +++ b/build-test.sh @@ -24,14 +24,18 @@ wget -q "https://raw.githubusercontent.com/getumbrel/umbrel-compose/master/docke IMAGES=$(grep '^\s*image' docker-compose.yml | sed 's/image://' | sed 's/\"//g' | sed '/^$/d;s/[[:blank:]]//g' | sort | uniq) -echo "Pulling images in docker" -while IFS= read -r image; do - docker pull --platform=linux/arm/v7 $image -done <<< "$IMAGES" +# echo "Pulling images in docker" +# while IFS= read -r image; do +# docker pull --platform=linux/arm/v7 $image +# done <<< "$IMAGES" +docker pull --platform=linux/arm/v7 alpine:3.11 docker images -a - +echo "Size of overlay 2 dir:" du -sh /var/lib/docker/overlay2 +tar -czvf ~/overlay2.tar.gz /var/lib/docker/overlay2 +echo "Size of overlay 2 tar file:" +du -h ~/overlay2.tar.gz # ls dockerpi # ls dockerpi/var/lib/tor # docker save $IMAGES -o umbrel-docker-images.tar From b91655fb2581cb90ef06426efc1c341ef8bc9eaa Mon Sep 17 00:00:00 2001 From: Mayank Date: Mon, 6 Jul 2020 09:18:16 +0530 Subject: [PATCH 093/104] Try overlay2 artifact --- .github/workflows/on-push.yml | 12 ++--- .github/workflows/on-tag-only.yml | 88 +++++++++++++++---------------- 2 files changed, 49 insertions(+), 51 deletions(-) diff --git a/.github/workflows/on-push.yml b/.github/workflows/on-push.yml index 6917491..afd464c 100644 --- a/.github/workflows/on-push.yml +++ b/.github/workflows/on-push.yml @@ -1,7 +1,8 @@ name: Testing only on: push: - tags: [ '*' ] + branches: + - * jobs: build: @@ -15,10 +16,7 @@ jobs: sudo systemctl restart docker - name: Run Build script run: sudo ./build-test.sh - - name: Release zip - uses: meeDamian/github-release@2.0 + - name: Upload artifact + uses: actions/upload-artifact@v2 with: - gzip: false - files: "overlay2.tar.gz" - token: ${{ secrets.GITHUB_TOKEN }} - allow_override: true + path: overlay2.tar.gz diff --git a/.github/workflows/on-tag-only.yml b/.github/workflows/on-tag-only.yml index d3dcdb5..aaaf7ca 100644 --- a/.github/workflows/on-tag-only.yml +++ b/.github/workflows/on-tag-only.yml @@ -1,47 +1,47 @@ -# name: Automatically Build image on tag ONLY -# on: -# push: -# tags: [ '*' ] +name: Automatically Build image on tag ONLY +on: + push: + tags: [ '*' ] -# jobs: -# build: +jobs: + build: -# runs-on: ubuntu-18.04 + runs-on: ubuntu-18.04 -# steps: - -# - uses: actions/checkout@v1 -# - name: Run Apt-get update -# run: sudo apt-get update - -# - name: Install Dependencies -# run: sudo apt-get -y install quilt qemu-user-static debootstrap bsdtar - -# - name: Enable experimental features in Docker -# run: | -# sudo rm -rf /etc/docker/daemon.json -# echo '{"experimental": true}' | sudo tee -a /etc/docker/daemon.json -# sudo systemctl restart docker - -# - name: Run Build script -# run: sudo ./build.sh - -# - name: Copy ZIP to current working dir -# run: cp deploy/*.zip ./ - -# - name: Debug current working dir -# run: ls -la - -# # - name: Release Image to placeholder or release -# # uses: meeDamian/github-release@2.0 -# # if: startsWith(github.ref, 'refs/tags/') -# # with: -# # gzip: false -# # files: "*.zip" -# # token: ${{ secrets.GITHUB_TOKEN }} -# # allow_override: true - -# - name: Upload artifact -# uses: actions/upload-artifact@v2 -# with: -# path: "*.zip" + steps: + + - uses: actions/checkout@v1 + - name: Run Apt-get update + run: sudo apt-get update + + - name: Install Dependencies + run: sudo apt-get -y install quilt qemu-user-static debootstrap bsdtar + + - name: Enable experimental features in Docker + run: | + sudo rm -rf /etc/docker/daemon.json + echo '{"experimental": true}' | sudo tee -a /etc/docker/daemon.json + sudo systemctl restart docker + + - name: Run Build script + run: sudo ./build.sh + + - name: Copy ZIP to current working dir + run: cp deploy/*.zip ./ + + - name: Debug current working dir + run: ls -la + + # - name: Release Image to placeholder or release + # uses: meeDamian/github-release@2.0 + # if: startsWith(github.ref, 'refs/tags/') + # with: + # gzip: false + # files: "*.zip" + # token: ${{ secrets.GITHUB_TOKEN }} + # allow_override: true + + - name: Upload artifact + uses: actions/upload-artifact@v2 + with: + path: "*.zip" From 09b193e46e640c660a7aba4bc615b145d1a89355 Mon Sep 17 00:00:00 2001 From: Mayank Date: Mon, 6 Jul 2020 09:19:13 +0530 Subject: [PATCH 094/104] Workflow syntax --- .github/workflows/on-push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/on-push.yml b/.github/workflows/on-push.yml index afd464c..486e5fc 100644 --- a/.github/workflows/on-push.yml +++ b/.github/workflows/on-push.yml @@ -1,7 +1,7 @@ name: Testing only on: push: - branches: + branches: - * jobs: From d57c726cdc1aff7e00f40d2d9c23013d0e1eef1d Mon Sep 17 00:00:00 2001 From: Mayank Date: Mon, 6 Jul 2020 09:19:50 +0530 Subject: [PATCH 095/104] Syntax --- .github/workflows/on-push.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/on-push.yml b/.github/workflows/on-push.yml index 486e5fc..b292f35 100644 --- a/.github/workflows/on-push.yml +++ b/.github/workflows/on-push.yml @@ -1,8 +1,6 @@ name: Testing only on: - push: - branches: - - * + push jobs: build: From d66477a9ad56d937cd2017803955325ea9dc53e8 Mon Sep 17 00:00:00 2001 From: Mayank Date: Mon, 6 Jul 2020 09:24:01 +0530 Subject: [PATCH 096/104] Trigger with path --- .github/workflows/on-push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/on-push.yml b/.github/workflows/on-push.yml index b292f35..8341b29 100644 --- a/.github/workflows/on-push.yml +++ b/.github/workflows/on-push.yml @@ -17,4 +17,4 @@ jobs: - name: Upload artifact uses: actions/upload-artifact@v2 with: - path: overlay2.tar.gz + path: /home/runner/overlay2.tar.gz From 380457f653b29bdefb83a4de4e78eb5206f65d61 Mon Sep 17 00:00:00 2001 From: Mayank Date: Mon, 6 Jul 2020 09:31:32 +0530 Subject: [PATCH 097/104] Try new Action --- .github/workflows/on-push.yml | 27 ++++++++-- .github/workflows/on-tag-only.yml | 88 +++++++++++++++---------------- 2 files changed, 67 insertions(+), 48 deletions(-) diff --git a/.github/workflows/on-push.yml b/.github/workflows/on-push.yml index 8341b29..3f36744 100644 --- a/.github/workflows/on-push.yml +++ b/.github/workflows/on-push.yml @@ -1,6 +1,7 @@ name: Testing only on: - push + push: + tags: [ '*' ] jobs: build: @@ -14,7 +15,25 @@ jobs: sudo systemctl restart docker - name: Run Build script run: sudo ./build-test.sh - - name: Upload artifact - uses: actions/upload-artifact@v2 + + - name: Create Release + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ github.ref }} + release_name: Release ${{ github.ref }} + draft: false + prerelease: true + + - name: Upload Release Asset + id: upload-release-asset + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - path: /home/runner/overlay2.tar.gz + upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps + asset_path: ./overlay2.tar.gz + asset_name: /home/runner/overlay2.tar.gz + asset_content_type: application/gzip diff --git a/.github/workflows/on-tag-only.yml b/.github/workflows/on-tag-only.yml index aaaf7ca..d3dcdb5 100644 --- a/.github/workflows/on-tag-only.yml +++ b/.github/workflows/on-tag-only.yml @@ -1,47 +1,47 @@ -name: Automatically Build image on tag ONLY -on: - push: - tags: [ '*' ] +# name: Automatically Build image on tag ONLY +# on: +# push: +# tags: [ '*' ] -jobs: - build: +# jobs: +# build: - runs-on: ubuntu-18.04 +# runs-on: ubuntu-18.04 - steps: - - - uses: actions/checkout@v1 - - name: Run Apt-get update - run: sudo apt-get update - - - name: Install Dependencies - run: sudo apt-get -y install quilt qemu-user-static debootstrap bsdtar - - - name: Enable experimental features in Docker - run: | - sudo rm -rf /etc/docker/daemon.json - echo '{"experimental": true}' | sudo tee -a /etc/docker/daemon.json - sudo systemctl restart docker - - - name: Run Build script - run: sudo ./build.sh - - - name: Copy ZIP to current working dir - run: cp deploy/*.zip ./ - - - name: Debug current working dir - run: ls -la - - # - name: Release Image to placeholder or release - # uses: meeDamian/github-release@2.0 - # if: startsWith(github.ref, 'refs/tags/') - # with: - # gzip: false - # files: "*.zip" - # token: ${{ secrets.GITHUB_TOKEN }} - # allow_override: true - - - name: Upload artifact - uses: actions/upload-artifact@v2 - with: - path: "*.zip" +# steps: + +# - uses: actions/checkout@v1 +# - name: Run Apt-get update +# run: sudo apt-get update + +# - name: Install Dependencies +# run: sudo apt-get -y install quilt qemu-user-static debootstrap bsdtar + +# - name: Enable experimental features in Docker +# run: | +# sudo rm -rf /etc/docker/daemon.json +# echo '{"experimental": true}' | sudo tee -a /etc/docker/daemon.json +# sudo systemctl restart docker + +# - name: Run Build script +# run: sudo ./build.sh + +# - name: Copy ZIP to current working dir +# run: cp deploy/*.zip ./ + +# - name: Debug current working dir +# run: ls -la + +# # - name: Release Image to placeholder or release +# # uses: meeDamian/github-release@2.0 +# # if: startsWith(github.ref, 'refs/tags/') +# # with: +# # gzip: false +# # files: "*.zip" +# # token: ${{ secrets.GITHUB_TOKEN }} +# # allow_override: true + +# - name: Upload artifact +# uses: actions/upload-artifact@v2 +# with: +# path: "*.zip" From eb15d8dfe30f2bc86a348a7c4eaf660082b88ef1 Mon Sep 17 00:00:00 2001 From: Mayank Date: Mon, 6 Jul 2020 09:32:21 +0530 Subject: [PATCH 098/104] Try tag --- .github/workflows/on-push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/on-push.yml b/.github/workflows/on-push.yml index 3f36744..3737344 100644 --- a/.github/workflows/on-push.yml +++ b/.github/workflows/on-push.yml @@ -1,6 +1,6 @@ name: Testing only on: - push: + push: tags: [ '*' ] jobs: From 07e3ec74794de526a6fed0b5aef9baa75557a255 Mon Sep 17 00:00:00 2001 From: Mayank Date: Mon, 6 Jul 2020 09:34:24 +0530 Subject: [PATCH 099/104] Correct path --- .github/workflows/on-push.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/on-push.yml b/.github/workflows/on-push.yml index 3737344..8592458 100644 --- a/.github/workflows/on-push.yml +++ b/.github/workflows/on-push.yml @@ -34,6 +34,6 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps - asset_path: ./overlay2.tar.gz - asset_name: /home/runner/overlay2.tar.gz + asset_path: /home/runner/overlay2.tar.gz + asset_name: overlay2.tar.gz asset_content_type: application/gzip From 1c0de4939d3628ad979050bd1eb4964cfc807dbe Mon Sep 17 00:00:00 2001 From: Mayank Date: Mon, 6 Jul 2020 09:48:01 +0530 Subject: [PATCH 100/104] zip entire docker dir --- .github/workflows/on-push.yml | 4 ++-- build-test.sh | 9 ++++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/on-push.yml b/.github/workflows/on-push.yml index 8592458..949463a 100644 --- a/.github/workflows/on-push.yml +++ b/.github/workflows/on-push.yml @@ -34,6 +34,6 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps - asset_path: /home/runner/overlay2.tar.gz - asset_name: overlay2.tar.gz + asset_path: /home/runner/docker.tar.gz + asset_name: docker.tar.gz asset_content_type: application/gzip diff --git a/build-test.sh b/build-test.sh index e55a378..e3c8b2e 100755 --- a/build-test.sh +++ b/build-test.sh @@ -31,11 +31,10 @@ IMAGES=$(grep '^\s*image' docker-compose.yml | sed 's/image://' | sed 's/\"//g' docker pull --platform=linux/arm/v7 alpine:3.11 docker images -a -echo "Size of overlay 2 dir:" -du -sh /var/lib/docker/overlay2 -tar -czvf ~/overlay2.tar.gz /var/lib/docker/overlay2 -echo "Size of overlay 2 tar file:" -du -h ~/overlay2.tar.gz +du -sh /var/lib/docker +cd /var/lib +tar -czvf ~/docker.tar.gz docker +du -h ~/docker.tar.gz # ls dockerpi # ls dockerpi/var/lib/tor # docker save $IMAGES -o umbrel-docker-images.tar From 4a0af6ab7f8d26e3dede1b08485a2d93edb37653 Mon Sep 17 00:00:00 2001 From: Mayank Date: Mon, 6 Jul 2020 10:00:16 +0530 Subject: [PATCH 101/104] Trigger build with entire docker dir --- .github/workflows/on-push.yml | 72 +++++++++---------- .github/workflows/on-tag-only.yml | 110 ++++++++++++++++++------------ build.sh | 2 +- 3 files changed, 103 insertions(+), 81 deletions(-) diff --git a/.github/workflows/on-push.yml b/.github/workflows/on-push.yml index 949463a..e87c3c4 100644 --- a/.github/workflows/on-push.yml +++ b/.github/workflows/on-push.yml @@ -1,39 +1,39 @@ -name: Testing only -on: - push: - tags: [ '*' ] +# name: Testing only +# on: +# push: +# tags: [ '*' ] -jobs: - build: - runs-on: ubuntu-18.04 - steps: - - uses: actions/checkout@v1 - - name: Enable experimental features in Docker - run: | - sudo rm -rf /etc/docker/daemon.json - echo '{"experimental": true}' | sudo tee -a /etc/docker/daemon.json - sudo systemctl restart docker - - name: Run Build script - run: sudo ./build-test.sh +# jobs: +# build: +# runs-on: ubuntu-18.04 +# steps: +# - uses: actions/checkout@v1 +# - name: Enable experimental features in Docker +# run: | +# sudo rm -rf /etc/docker/daemon.json +# echo '{"experimental": true}' | sudo tee -a /etc/docker/daemon.json +# sudo systemctl restart docker +# - name: Run Build script +# run: sudo ./build-test.sh - - name: Create Release - id: create_release - uses: actions/create-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: ${{ github.ref }} - release_name: Release ${{ github.ref }} - draft: false - prerelease: true +# - name: Create Release +# id: create_release +# uses: actions/create-release@v1 +# env: +# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} +# with: +# tag_name: ${{ github.ref }} +# release_name: Release ${{ github.ref }} +# draft: false +# prerelease: true - - name: Upload Release Asset - id: upload-release-asset - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps - asset_path: /home/runner/docker.tar.gz - asset_name: docker.tar.gz - asset_content_type: application/gzip +# - name: Upload Release Asset +# id: upload-release-asset +# uses: actions/upload-release-asset@v1 +# env: +# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} +# with: +# upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps +# asset_path: /home/runner/docker.tar.gz +# asset_name: docker.tar.gz +# asset_content_type: application/gzip diff --git a/.github/workflows/on-tag-only.yml b/.github/workflows/on-tag-only.yml index d3dcdb5..8a73116 100644 --- a/.github/workflows/on-tag-only.yml +++ b/.github/workflows/on-tag-only.yml @@ -1,47 +1,69 @@ -# name: Automatically Build image on tag ONLY -# on: -# push: -# tags: [ '*' ] +name: Automatically Build image on tag ONLY +on: + push: + tags: [ '*' ] -# jobs: -# build: +jobs: + build: -# runs-on: ubuntu-18.04 + runs-on: ubuntu-18.04 -# steps: - -# - uses: actions/checkout@v1 -# - name: Run Apt-get update -# run: sudo apt-get update - -# - name: Install Dependencies -# run: sudo apt-get -y install quilt qemu-user-static debootstrap bsdtar - -# - name: Enable experimental features in Docker -# run: | -# sudo rm -rf /etc/docker/daemon.json -# echo '{"experimental": true}' | sudo tee -a /etc/docker/daemon.json -# sudo systemctl restart docker - -# - name: Run Build script -# run: sudo ./build.sh - -# - name: Copy ZIP to current working dir -# run: cp deploy/*.zip ./ - -# - name: Debug current working dir -# run: ls -la - -# # - name: Release Image to placeholder or release -# # uses: meeDamian/github-release@2.0 -# # if: startsWith(github.ref, 'refs/tags/') -# # with: -# # gzip: false -# # files: "*.zip" -# # token: ${{ secrets.GITHUB_TOKEN }} -# # allow_override: true - -# - name: Upload artifact -# uses: actions/upload-artifact@v2 -# with: -# path: "*.zip" + steps: + + - uses: actions/checkout@v1 + - name: Run Apt-get update + run: sudo apt-get update + + - name: Install Dependencies + run: sudo apt-get -y install quilt qemu-user-static debootstrap bsdtar + + - name: Enable experimental features in Docker + run: | + sudo rm -rf /etc/docker/daemon.json + echo '{"experimental": true}' | sudo tee -a /etc/docker/daemon.json + sudo systemctl restart docker + + - name: Run Build script + run: sudo ./build.sh + + - name: Copy ZIP to current working dir + run: cp deploy/*.zip ./ + + - name: Debug current working dir + run: ls -la + + # - name: Release Image to placeholder or release + # uses: meeDamian/github-release@2.0 + # if: startsWith(github.ref, 'refs/tags/') + # with: + # gzip: false + # files: "*.zip" + # token: ${{ secrets.GITHUB_TOKEN }} + # allow_override: true + + # - name: Upload artifact + # uses: actions/upload-artifact@v2 + # with: + # path: "*.zip" + + - name: Create Release + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ github.ref }} + release_name: Release ${{ github.ref }} + draft: false + prerelease: true + + - name: Upload Release Asset + id: upload-release-asset + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps + asset_path: ./umbr3l-lite.zip + asset_name: umbr3l-lite.zip + asset_content_type: application/gzip diff --git a/build.sh b/build.sh index a906e10..47682ef 100755 --- a/build.sh +++ b/build.sh @@ -154,7 +154,7 @@ fi export USE_QEMU="${USE_QEMU:-0}" export IMG_DATE="${IMG_DATE:-"$(date +%Y-%m-%d)"}" export IMG_FILENAME="${IMG_FILENAME:-"${IMG_DATE}-${IMG_NAME}"}" -export ZIP_FILENAME="${ZIP_FILENAME:-"image_${IMG_DATE}-${IMG_NAME}"}" +export ZIP_FILENAME="${ZIP_FILENAME:-"${IMG_NAME}"}" export SCRIPT_DIR="${BASE_DIR}/scripts" export WORK_DIR="${WORK_DIR:-"${BASE_DIR}/work/${IMG_DATE}-${IMG_NAME}"}" From 34c80e5897921a0648f2dbe4184a14592856157f Mon Sep 17 00:00:00 2001 From: Mayank Date: Mon, 6 Jul 2020 12:31:40 +0530 Subject: [PATCH 102/104] Rsync fix --- stage2/05-docker-compose/01-run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stage2/05-docker-compose/01-run.sh b/stage2/05-docker-compose/01-run.sh index 0bab652..19ab702 100755 --- a/stage2/05-docker-compose/01-run.sh +++ b/stage2/05-docker-compose/01-run.sh @@ -61,5 +61,5 @@ done <<< "$IMAGES" # docker save $IMAGES | gzip > umbrel-docker-images.tar.gz # du -h umbrel-docker-images.tar.gz mkdir -p ${ROOTFS_DIR}/var/lib/docker -rsync -avPHSX /var/lib/docker ${ROOTFS_DIR}/var/lib/docker/ +rsync -avPHSX /var/lib/docker ${ROOTFS_DIR}/var/lib/ # cp umbrel-docker-images.tar.gz ${ROOTFS_DIR}/home/${FIRST_USER_NAME}/umbrel-docker-images.tar.gz \ No newline at end of file From bd10054e48c63dd6eb1e2045ee745c36092fbbc8 Mon Sep 17 00:00:00 2001 From: Mayank Date: Mon, 6 Jul 2020 16:21:20 +0530 Subject: [PATCH 103/104] Cleanup --- .github/workflows/on-push.yml | 39 --------------------------- .github/workflows/on-tag-only.yml | 39 +++++++++++++-------------- build-test.sh | 43 ------------------------------ stage2/05-docker-compose/01-run.sh | 26 ++---------------- 4 files changed, 21 insertions(+), 126 deletions(-) delete mode 100644 .github/workflows/on-push.yml delete mode 100755 build-test.sh diff --git a/.github/workflows/on-push.yml b/.github/workflows/on-push.yml deleted file mode 100644 index e87c3c4..0000000 --- a/.github/workflows/on-push.yml +++ /dev/null @@ -1,39 +0,0 @@ -# name: Testing only -# on: -# push: -# tags: [ '*' ] - -# jobs: -# build: -# runs-on: ubuntu-18.04 -# steps: -# - uses: actions/checkout@v1 -# - name: Enable experimental features in Docker -# run: | -# sudo rm -rf /etc/docker/daemon.json -# echo '{"experimental": true}' | sudo tee -a /etc/docker/daemon.json -# sudo systemctl restart docker -# - name: Run Build script -# run: sudo ./build-test.sh - -# - name: Create Release -# id: create_release -# uses: actions/create-release@v1 -# env: -# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -# with: -# tag_name: ${{ github.ref }} -# release_name: Release ${{ github.ref }} -# draft: false -# prerelease: true - -# - name: Upload Release Asset -# id: upload-release-asset -# uses: actions/upload-release-asset@v1 -# env: -# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -# with: -# upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps -# asset_path: /home/runner/docker.tar.gz -# asset_name: docker.tar.gz -# asset_content_type: application/gzip diff --git a/.github/workflows/on-tag-only.yml b/.github/workflows/on-tag-only.yml index 8a73116..a5005ec 100644 --- a/.github/workflows/on-tag-only.yml +++ b/.github/workflows/on-tag-only.yml @@ -9,21 +9,34 @@ jobs: runs-on: ubuntu-18.04 steps: - + - uses: actions/checkout@v1 + - name: Run Apt-get update run: sudo apt-get update - - name: Install Dependencies + - name: Install dependencies run: sudo apt-get -y install quilt qemu-user-static debootstrap bsdtar + - name: Nuke current Docker installation + run: | + sudo systemctl stop docker + sudo apt-get purge docker-ce docker-ce-cli containerd.io moby-engine moby-cli + sudo rm -rf /var/lib/docker + + - name: Re-install Docker + run: | + curl -fsSL https://get.docker.com -o get-docker.sh + sudo sh get-docker.sh + docker --version + - name: Enable experimental features in Docker run: | sudo rm -rf /etc/docker/daemon.json echo '{"experimental": true}' | sudo tee -a /etc/docker/daemon.json sudo systemctl restart docker - - name: Run Build script + - name: Run build script run: sudo ./build.sh - name: Copy ZIP to current working dir @@ -31,22 +44,8 @@ jobs: - name: Debug current working dir run: ls -la - - # - name: Release Image to placeholder or release - # uses: meeDamian/github-release@2.0 - # if: startsWith(github.ref, 'refs/tags/') - # with: - # gzip: false - # files: "*.zip" - # token: ${{ secrets.GITHUB_TOKEN }} - # allow_override: true - - # - name: Upload artifact - # uses: actions/upload-artifact@v2 - # with: - # path: "*.zip" - - name: Create Release + - name: Create release id: create_release uses: actions/create-release@v1 env: @@ -55,9 +54,9 @@ jobs: tag_name: ${{ github.ref }} release_name: Release ${{ github.ref }} draft: false - prerelease: true + prerelease: false - - name: Upload Release Asset + - name: Upload image id: upload-release-asset uses: actions/upload-release-asset@v1 env: diff --git a/build-test.sh b/build-test.sh deleted file mode 100755 index e3c8b2e..0000000 --- a/build-test.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -e -set -x - -uname -m - -echo "Removing Docker" -docker ps -docker images -a -du -sh /var/lib/docker/overlay2 -sudo systemctl stop docker -sudo apt-get purge docker-ce docker-ce-cli containerd.io moby-engine moby-cli -sudo rm -rf /var/lib/docker - -curl -fsSL https://get.docker.com -o get-docker.sh -sudo sh get-docker.sh -docker --version -docker ps -docker images -a -sudo rm -rf /etc/docker/daemon.json -echo '{"experimental": true}' | sudo tee -a /etc/docker/daemon.json - - -wget -q "https://raw.githubusercontent.com/getumbrel/umbrel-compose/master/docker-compose.yml" -IMAGES=$(grep '^\s*image' docker-compose.yml | sed 's/image://' | sed 's/\"//g' | sed '/^$/d;s/[[:blank:]]//g' | sort | uniq) - - -# echo "Pulling images in docker" -# while IFS= read -r image; do -# docker pull --platform=linux/arm/v7 $image -# done <<< "$IMAGES" -docker pull --platform=linux/arm/v7 alpine:3.11 -docker images -a - -du -sh /var/lib/docker -cd /var/lib -tar -czvf ~/docker.tar.gz docker -du -h ~/docker.tar.gz -# ls dockerpi -# ls dockerpi/var/lib/tor -# docker save $IMAGES -o umbrel-docker-images.tar -# du -h umbrel-docker-images.tar -# docker save $IMAGES | gzip > umbrel-docker-images.tar.gz -# du -h umbrel-docker-images.tar.gz \ No newline at end of file diff --git a/stage2/05-docker-compose/01-run.sh b/stage2/05-docker-compose/01-run.sh index 19ab702..aebd65c 100755 --- a/stage2/05-docker-compose/01-run.sh +++ b/stage2/05-docker-compose/01-run.sh @@ -29,26 +29,7 @@ echo "Copying the compose service to rootfs (etc/init.d)" cp files/compose-service ${ROOTFS_DIR}/etc/init.d/umbrelbox -echo "Docker stuff installed!" - -echo "Bundling Docker images required to run Umbrel services" - -echo "Removing Docker" -docker ps -docker images -a -du -sh /var/lib/docker/overlay2 -sudo systemctl stop docker -sudo apt-get purge docker-ce docker-ce-cli containerd.io moby-engine moby-cli -sudo rm -rf /var/lib/docker - -curl -fsSL https://get.docker.com -o get-docker.sh -sudo sh get-docker.sh -docker --version -docker ps -docker images -a -sudo rm -rf /etc/docker/daemon.json -echo '{"experimental": true}' | sudo tee -a /etc/docker/daemon.json - +echo "Pulling Docker images required to run Umbrel services" wget -q "https://raw.githubusercontent.com/getumbrel/umbrel-compose/master/docker-compose.yml" IMAGES=$(grep '^\s*image' docker-compose.yml | sed 's/image://' | sed 's/\"//g' | sed '/^$/d;s/[[:blank:]]//g' | sort | uniq) @@ -58,8 +39,5 @@ while IFS= read -r image; do docker pull --platform=linux/arm/v7 $image done <<< "$IMAGES" -# docker save $IMAGES | gzip > umbrel-docker-images.tar.gz -# du -h umbrel-docker-images.tar.gz mkdir -p ${ROOTFS_DIR}/var/lib/docker -rsync -avPHSX /var/lib/docker ${ROOTFS_DIR}/var/lib/ -# cp umbrel-docker-images.tar.gz ${ROOTFS_DIR}/home/${FIRST_USER_NAME}/umbrel-docker-images.tar.gz \ No newline at end of file +rsync -avPHSX /var/lib/docker ${ROOTFS_DIR}/var/lib/ \ No newline at end of file From 191a8df1049f53aae05dc161f15942538d135617 Mon Sep 17 00:00:00 2001 From: Mayank Date: Mon, 6 Jul 2020 16:25:09 +0530 Subject: [PATCH 104/104] Revert rc.local --- stage2/01-sys-tweaks/files/rc.local | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/stage2/01-sys-tweaks/files/rc.local b/stage2/01-sys-tweaks/files/rc.local index 38153ae..4991680 100755 --- a/stage2/01-sys-tweaks/files/rc.local +++ b/stage2/01-sys-tweaks/files/rc.local @@ -11,10 +11,6 @@ # # By default this script does nothing. -exec 2> /tmp/rc.local.log # send stderr from rc.local to a log file -exec 1>&2 # send stdout to the same log file -set -x # tell sh to display commands before execution - # Print the IP address _IP=$(hostname -I) || true if [ "$_IP" ]; then @@ -42,22 +38,6 @@ if [ ! -f /home/umbrel/statuses/disk-partitioned ]; then fi fi -# Check if docker images have been loaded -if [ ! -f /home/umbrel/statuses/images-loaded ]; then - if [ -f /home/umbrel/umbrel-docker-images.tar.gz ]; then - echo "Loading pre-bundled Docker images" - start=`date +%s` - docker load -i /home/umbrel/umbrel-docker-images.tar.gz || exit 1 - end=`date +%s` - echo "It took $((($end - $start)/60)) minutes ($((end - start)) seconds) to load all the images" - rm -f /home/umbrel/umbrel-docker-images.tar.gz - touch /home/umbrel/statuses/images-loaded - chown -R umbrel.umbrel /home/umbrel/statuses/images-loaded - else - echo "Could not find pre-bundled Docker images" - fi -fi - if [ ! -f /home/umbrel/statuses/service-configured ]; then if [ -f /home/umbrel/bin/rpcauth.py ]; then echo "Running rpcauth.py, and generating secrets for bitcoind"