Merge pull request #1 from Samourai-Wallet/feat_mydojo_upgrade

my-dojo upgrade mechanism

db-scripts/1_db.sql → db-scripts/1_db.sql.tpl

@@ -25,10 +25,9 @@
 -- Table structure for table `addresses`
 --
 
-DROP TABLE IF EXISTS `addresses`;
 /*!40101 SET @saved_cs_client     = @@character_set_client */;
 /*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `addresses` (
+CREATE TABLE IF NOT EXISTS `addresses` (
   `addrID` int(10) unsigned NOT NULL AUTO_INCREMENT,
   `addrAddress` varchar(74) DEFAULT NULL,
   PRIMARY KEY (`addrID`),
@@ -40,10 +39,9 @@ CREATE TABLE `addresses` (
 -- Table structure for table `banned_addresses`
 --
 
-DROP TABLE IF EXISTS `banned_addresses`;
 /*!40101 SET @saved_cs_client     = @@character_set_client */;
 /*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `banned_addresses` (
+CREATE TABLE IF NOT EXISTS `banned_addresses` (
   `bannedAddressId` int(11) NOT NULL AUTO_INCREMENT,
   `addrAddress` varchar(35) NOT NULL,
   PRIMARY KEY (`bannedAddressId`),
@@ -55,10 +53,9 @@ CREATE TABLE `banned_addresses` (
 -- Table structure for table `blocks`
 --
 
-DROP TABLE IF EXISTS `blocks`;
 /*!40101 SET @saved_cs_client     = @@character_set_client */;
 /*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `blocks` (
+CREATE TABLE IF NOT EXISTS `blocks` (
   `blockID` int(10) unsigned NOT NULL AUTO_INCREMENT,
   `blockHash` char(64) NOT NULL DEFAULT '',
   `blockParent` int(10) unsigned DEFAULT NULL,
@@ -76,10 +73,9 @@ CREATE TABLE `blocks` (
 -- Table structure for table `hd`
 --
 
-DROP TABLE IF EXISTS `hd`;
 /*!40101 SET @saved_cs_client     = @@character_set_client */;
 /*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `hd` (
+CREATE TABLE IF NOT EXISTS `hd` (
   `hdID` int(10) unsigned NOT NULL AUTO_INCREMENT,
   `hdXpub` char(112) DEFAULT NULL,
   `hdCreated` int(10) unsigned NOT NULL DEFAULT '0',
@@ -94,10 +90,9 @@ CREATE TABLE `hd` (
 -- Table structure for table `hd_addresses`
 --
 
-DROP TABLE IF EXISTS `hd_addresses`;
 /*!40101 SET @saved_cs_client     = @@character_set_client */;
 /*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `hd_addresses` (
+CREATE TABLE IF NOT EXISTS `hd_addresses` (
   `hdAddrID` int(10) unsigned NOT NULL AUTO_INCREMENT,
   `hdID` int(10) unsigned NOT NULL DEFAULT '0',
   `addrID` int(10) unsigned NOT NULL DEFAULT '0',
@@ -116,10 +111,9 @@ CREATE TABLE `hd_addresses` (
 -- Table structure for table `inputs`
 --
 
-DROP TABLE IF EXISTS `inputs`;
 /*!40101 SET @saved_cs_client     = @@character_set_client */;
 /*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `inputs` (
+CREATE TABLE IF NOT EXISTS `inputs` (
   `inID` int(10) unsigned NOT NULL AUTO_INCREMENT,
   `outID` int(10) unsigned NOT NULL DEFAULT '0',
   `txnID` int(10) unsigned NOT NULL DEFAULT '0',
@@ -138,10 +132,9 @@ CREATE TABLE `inputs` (
 -- Table structure for table `outputs`
 --
 
-DROP TABLE IF EXISTS `outputs`;
 /*!40101 SET @saved_cs_client     = @@character_set_client */;
 /*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `outputs` (
+CREATE TABLE IF NOT EXISTS `outputs` (
   `outID` int(10) unsigned NOT NULL AUTO_INCREMENT,
   `txnID` int(10) unsigned NOT NULL DEFAULT '0',
   `addrID` int(10) unsigned NOT NULL DEFAULT '0',
@@ -161,10 +154,9 @@ CREATE TABLE `outputs` (
 -- Table structure for table `transactions`
 --
 
-DROP TABLE IF EXISTS `transactions`;
 /*!40101 SET @saved_cs_client     = @@character_set_client */;
 /*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `transactions` (
+CREATE TABLE IF NOT EXISTS `transactions` (
   `txnID` int(10) unsigned NOT NULL AUTO_INCREMENT,
   `txnTxid` char(64) DEFAULT NULL,
   `txnCreated` int(10) unsigned NOT NULL DEFAULT '0',
@@ -183,10 +175,9 @@ CREATE TABLE `transactions` (
 -- Table structure for table `scheduled_transactions`
 --
 
-DROP TABLE IF EXISTS `scheduled_transactions`;
 /*!40101 SET @saved_cs_client     = @@character_set_client */;
 /*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `scheduled_transactions` (
+CREATE TABLE IF NOT EXISTS `scheduled_transactions` (
   `schID` int(10) unsigned NOT NULL AUTO_INCREMENT,
   `schTxid` char(64) NOT NULL DEFAULT '',
   `schCreated` int(10) unsigned NOT NULL DEFAULT '0',

db-scripts/2_update.sql.tpl

@@ -0,0 +1,7 @@
+# SQL scripts for incremental updates
+
+# Copyright © 2019 – Katana Cryptographic Ltd. All Rights Reserved.
+ 
+--
+-- UPDATES vX.Y.Z
+--

doc/DOCKER_setup.md

@@ -54,41 +54,43 @@ MyDojo is a set of Docker containers providing a full Samourai backend composed
 * Tor Browser installed on the host machine (or on another machine if your host is a headless server)
 
 
-## Setup ##
+## First-time Setup ##
+
+This procedure allows to install a new Dojo from scratch.
 
 * Install [Docker and Docker Compose](https://docs.docker.com/compose/install/) on the host machine and check that your installation is working.
 
 * Install [Tor Browser](https://www.torproject.org/projects/torbrowser.html.en) on the host machine.
 
-* Download the most recent version of Dojo from [Github](https://github.com/Samourai-Wallet/samourai-dojo/archive/master.zip)
+* Download the most recent release of Dojo from [Github](https://github.com/Samourai-Wallet/samourai-dojo/archive/master.zip)
 
-* Uncompress the archive on the host machine in a temporary directory of your choice (named /tmp_dir in this doc)
+* Uncompress the archive on the host machine in a temporary directory of your choice (named <tmp_dir> in this doc)
 
-* Create a directory for Dojo (named /dojo_dir in this doc)
+* Create a directory for Dojo (named <dojo_dir> in this doc)
 
-* Copy the content of the "/tmp_dir/samourai-dojo-master" directory into the "/dojo_dir" directory
+* Copy the content of the "<tmp_dir>/samourai-dojo-master" directory into the <dojo_dir> directory
 
 * Customize the configuration of your Dojo
 
-  * Go to the "/dojo_dir/docker/my_dojo/conf" directory
+  * Go to the "<dojo_dir>/docker/my_dojo/conf" directory
 
-  * Edit docker-bitcoin.conf and provide a new value for the following parameters:
+  * Edit docker-bitcoin.conf.tpl and provide a new value for the following parameters:
       * BITCOIND_RPC_USER = login protecting the access to the RPC API of your full node,
       * BITCOIND_RPC_PASSWORD = password protecting the access to the RPC API of your full node.
       * If your machine has a lot of RAM, it's recommended that you increase the value of BITCOIND_DB_CACHE for a faster Initial Block Download.
 
-  * Edit docker-mysql.conf and provide a new value for the following parameters:
+  * Edit docker-mysql.conf.tpl and provide a new value for the following parameters:
       * MYSQL_ROOT_PASSWORD = password protecting the root account of MySQL,
       * MYSQL_USER = login of the account used to access the database of your Dojo,
       * MYSQL_PASSWORD = password of the account used to access the database of your Dojo.
 
-  * Edit docker-node.conf and provide a new value for the following parameters:
+  * Edit docker-node.conf.tpl and provide a new value for the following parameters:
       * NODE_API_KEY = API key which will be required from your Samourai Wallet / Sentinel for its interactions with the API of your Dojo,
       * NODE_ADMIN_KEY = API key which will be required from the maintenance tool for accessing a set of advanced features provided by the API of your Dojo,
       * NODE_JWT_SECRET = secret used by your Dojo for the initialization of a cryptographic key signing Json Web Tokens.
     These parameters will protect the access to your Dojo. Be sure to provide alphanumeric values with enough entropy.
 
-* Open the docker quickstart terminal or a terminal console and go to the "/dojo_dir/docker/my_dojo" directory. This directory contains a script named dojo.sh which will be your entrypoint for all operations related to the management of your Dojo.
+* Open the docker quickstart terminal or a terminal console and go to the "<dojo_dir>/docker/my_dojo" directory. This directory contains a script named dojo.sh which will be your entrypoint for all operations related to the management of your Dojo.
 
 
 * Launch the installation of your Dojo with
@@ -118,6 +120,31 @@ Exit the logs with CTRL+C when the syncing of the database has completed.
 * Restrict the access to your host machine as much as possible by configuring its firewall.
 
 
+## Upgrade ##
+
+This procedure allows to upgrade your Dojo with a new version.
+
+* Stop your Dojo with
+
+```
+./dojo.sh stop
+```
+
+* Download the most recent release of Dojo from [Github](https://github.com/Samourai-Wallet/samourai-dojo/releases)
+
+* Uncompress the archive on the host machine in a temporary directory of your choice (named <tmp_dir> in this doc)
+
+* Copy the content of the "<tmp_dir>/samourai-dojo-1.x.y" directory into the <dojo_dir> directory (overwrite).
+
+* Launch the upgrade of your Dojo with
+
+```
+./dojo.sh upgrade
+```
+
+Docker and Docker Compose are going to build new images and containers for your Dojo. After completion, the updated version of your Dojo will be launched automatically.
+
+
 ## Dojo shell script ##
 
 dojo.sh is a multifeature tool allowing to interact with your Dojo.
@@ -160,6 +187,9 @@ Available commands:
 
   uninstall                     Delete your Dojo. Be careful! This command will also remove all data.
 
+  upgrade                       Upgrade your Dojo.
+
+  version                       Display the version of dojo.
 ```
 
 

docker/my-dojo/.env

@@ -9,7 +9,13 @@
 #########################################
 
 COMPOSE_CONVERT_WINDOWS_PATHS=1
+
 DOJO_VERSION_TAG=1.0.0
+DOJO_DB_VERSION_TAG=1.0.0
+DOJO_BITCOIND_VERSION_TAG=1.0.0
+DOJO_NODEJS_VERSION_TAG=1.0.0
+DOJO_NGINX_VERSION_TAG=1.0.0
+DOJO_TOR_VERSION_TAG=1.0.0
 
 
 #########################################

docker/my-dojo/docker-compose.yaml

@@ -2,7 +2,7 @@ version: "3.2"
 
 services:
   db:
-    image: "samouraiwallet/dojo-db:1.0.0"
+    image: "samouraiwallet/dojo-db:${DOJO_DB_VERSION_TAG}"
     container_name: db
     build:
       context: ./../..
@@ -20,7 +20,7 @@ services:
         ipv4_address: 172.28.1.1
 
   bitcoind:
-    image: "samouraiwallet/dojo-bitcoind:1.0.0"
+    image: "samouraiwallet/dojo-bitcoind:${DOJO_BITCOIND_VERSION_TAG}"
     container_name: bitcoind
     build:
       context: ./bitcoin
@@ -43,7 +43,7 @@ services:
         ipv4_address: 172.28.1.5
 
   node:
-    image: "samouraiwallet/dojo-nodejs:1.0.0"
+    image: "samouraiwallet/dojo-nodejs:${DOJO_NODEJS_VERSION_TAG}"
     container_name: nodejs
     build:
       context: ./../..
@@ -68,7 +68,7 @@ services:
         ipv4_address: 172.28.1.2
 
   nginx:
-    image: "samouraiwallet/dojo-nginx:1.0.0"
+    image: "samouraiwallet/dojo-nginx:${DOJO_NGINX_VERSION_TAG}"
     container_name: nginx
     build:
       context: ./nginx
@@ -89,7 +89,7 @@ services:
         ipv4_address: 172.28.1.3
 
   tor:
-    image: "samouraiwallet/dojo-tor:1.0.0"
+    image: "samouraiwallet/dojo-tor:${DOJO_TOR_VERSION_TAG}"
     container_name: tor
     build:
       context: ./tor

docker/my-dojo/dojo.sh

@@ -1,7 +1,14 @@
 #!/bin/bash
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-source "$DIR/conf/docker-bitcoind.conf"
+
+if [ -f "$DIR/conf/docker-bitcoind.conf" ]; then
+  source "$DIR/conf/docker-bitcoind.conf"
+fi
+
+if [ -f "$DIR/.env" ]; then
+  source "$DIR/.env"
+fi
 
   
 # Start
@@ -42,8 +49,22 @@ restart() {
 
 # Install
 install() {
-  docker-compose up -d --remove-orphans
-  docker-compose logs --tail=0 --follow
+  source "$DIR/install/install-scripts.sh"
+
+  launchInstall=1
+
+  if [ -z "$1" ]; then
+    get_confirmation
+    launchInstall=$?
+  else
+    launchInstall=0
+  fi
+
+  if [ $launchInstall -eq 0 ]; then
+    init_config_files
+    docker-compose up -d --remove-orphans
+    docker-compose logs --tail=0 --follow
+  fi
 }
 
 # Delete everything
@@ -51,15 +72,37 @@ uninstall() {
   docker-compose rm
   docker-compose down
 
-  docker image rm samouraiwallet/dojo-db:1.0.0
-  docker image rm samouraiwallet/dojo-bitcoind:1.0.0
-  docker image rm samouraiwallet/dojo-nodejs:1.0.0
-  docker image rm samouraiwallet/dojo-nginx:1.0.0
-  docker image rm samouraiwallet/dojo-tor:1.0.0
+  docker image rm samouraiwallet/dojo-db:"$DOJO_DB_VERSION_TAG"
+  docker image rm samouraiwallet/dojo-bitcoind:"$DOJO_BITCOIND_VERSION_TAG"
+  docker image rm samouraiwallet/dojo-nodejs:"$DOJO_NODEJS_VERSION_TAG"
+  docker image rm samouraiwallet/dojo-nginx:"$DOJO_NGINX_VERSION_TAG"
+  docker image rm samouraiwallet/dojo-tor:"$DOJO_TOR_VERSION_TAG"
 
   docker volume prune
 }
 
+# Upgrade
+upgrade() {
+  source "$DIR/install/upgrade-scripts.sh"
+
+  launchUpgrade=1
+
+  if [ -z "$1" ]; then
+    get_confirmation
+    launchUpgrade=$?
+  else
+    launchUpgrade=0
+  fi
+
+  if [ $launchUpgrade -eq 0 ]; then
+    update_config_files
+    docker-compose build
+    docker-compose up -d --remove-orphans
+    update_dojo_db
+    docker-compose logs --tail=0 --follow
+  fi
+}
+
 # Display the onion address
 onion() {
   V2_ADDR=$( docker exec -it tor cat /var/lib/tor/hsv2dojo/hostname )
@@ -69,6 +112,11 @@ onion() {
   echo "API Hidden Service address (v2) = $V2_ADDR"
 }
 
+# Display the version of this dojo
+version() {
+  echo "Dojo v$DOJO_VERSION_TAG"
+}
+
 # Display logs
 logs_node() {
   if [ $3 -eq 0 ]; then
@@ -137,6 +185,10 @@ help() {
   echo "  stop                          Stop your dojo."
   echo " "
   echo "  uninstall                     Delete your dojo. Be careful! This command will also remove all data."
+  echo " "
+  echo "  upgrade                       Upgrade your dojo."
+  echo " "
+  echo "  version                       Display the version of dojo"
 }
 
 
@@ -174,7 +226,7 @@ case "$subcommand" in
     help
     ;;
   install )
-    install
+    install $1
     ;;
   logs )
     module=$1; shift
@@ -219,4 +271,10 @@ case "$subcommand" in
   uninstall )
     uninstall
     ;;
+  upgrade )
+    upgrade $1
+    ;;
+  version )
+    version
+    ;;
 esac

docker/my-dojo/install/install-scripts.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# Confirm installation
+get_confirmation() {
+  while true; do
+    echo "This operation is going to install Dojo v$DOJO_VERSION_TAG on your computer."
+    read -p "Do you wish to continue? [y/n]" yn
+    case $yn in
+      [Yy]* ) return 0;;
+      [Nn]* ) echo "Installation was cancelled."; return 1;;
+      * ) echo "Please answer yes or no.";;
+    esac
+  done
+}
+
+# Initialize configuration files from templates
+init_config_files() {
+  cp ../../db-scripts/1_db.sql.tpl ../../db-scripts/1_db.sql
+  echo "Initialized 1_db.sql"
+
+  if [ -f ../../db-scripts/2_update.sql ]; then
+    rm ../../db-scripts/2_update.sql
+    echo "Deleted 2_update.sql"
+  fi
+
+  cp ./conf/docker-bitcoind.conf.tpl ./conf/docker-bitcoind.conf
+  echo "Initialized docker-bitcoind.conf"
+  
+  cp ./conf/docker-mysql.conf.tpl ./conf/docker-mysql.conf
+  echo "Initialized docker-mysql.conf"
+  
+  cp ./conf/docker-node.conf.tpl ./conf/docker-node.conf
+  echo "Initialized docker-node.conf"
+}

docker/my-dojo/install/upgrade-scripts.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+# Confirm upgrade operation
+get_confirmation() {
+  while true; do
+    echo "This operation is going to upgrade your Dojo to v$DOJO_VERSION_TAG."
+    read -p "Do you wish to continue? [y/n]" yn
+    case $yn in
+      [Yy]* ) return 0;;
+      [Nn]* ) echo "Upgrade was cancelled."; return 1;;
+      * ) echo "Please answer yes or no.";;
+    esac
+  done
+}
+
+# Update configuration files from templates
+update_config_files() {
+  if [ -f ../../db-scripts/1_db.sql ]; then
+    rm ../../db-scripts/1_db.sql
+    echo "Deleted 1_db.sql"
+  fi
+
+  cp ../../db-scripts/2_update.sql.tpl ../../db-scripts/2_update.sql
+  echo "Initialized 2_update.sql"
+
+  update_config_file ./conf/docker-bitcoind.conf ./conf/docker-bitcoind.conf.tpl
+  echo "Initialized docker-bitcoind.conf"
+
+  update_config_file ./conf/docker-mysql.conf ./conf/docker-mysql.conf.tpl
+  echo "Initialized docker-mysql.conf"
+
+  update_config_file ./conf/docker-node.conf ./conf/docker-node.conf.tpl
+  echo "Initialized docker-node.conf"
+}
+
+# Update a configuration file from template
+update_config_file() {
+  sed "s/^#.*//g;s/=.*//g;/^$/d" $1 > ./original.keys.raw
+  grep -f ./original.keys.raw $1 > ./original.lines.raw
+
+  cp -p $1 "$1.save"
+  cp -p $2 $1
+
+  while IFS='=' read -r key val ; do 
+    sed -i "s/$key=.*/$key=$val/g" "$1"
+  done < ./original.lines.raw
+
+  rm ./original.keys.raw
+  rm ./original.lines.raw
+}
+
+# Update dojo database
+update_dojo_db() {
+  docker exec -d db /update-db.sh
+}

docker/my-dojo/mysql/Dockerfile

@@ -1,7 +1,13 @@
-FROM      mysql:5.7.25
+FROM    mysql:5.7.25
 
 # Copy mysql config
-COPY      ./docker/my-dojo/mysql/mysql-dojo.cnf /etc/mysql/conf.d/mysql-dojo.cnf
+COPY    ./docker/my-dojo/mysql/mysql-dojo.cnf /etc/mysql/conf.d/mysql-dojo.cnf
+
+# Copy update-db script
+COPY    ./docker/my-dojo/mysql/update-db.sh /update-db.sh
+
+RUN     chmod u+x /update-db.sh && \
+        chmod g+x /update-db.sh
 
 # Copy content of mysql scripts into /docker-entrypoint-initdb.d
-COPY      ./db-scripts/ /docker-entrypoint-initdb.d
+COPY    ./db-scripts/ /docker-entrypoint-initdb.d

docker/my-dojo/mysql/update-db.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+for i in {30..0}; do
+  if echo "SELECT 1" | mysql -h"db" -u"root" -p"$MYSQL_ROOT_PASSWORD" &> /dev/null; then
+    break
+  fi
+  echo "MySQL init process in progress..."
+  sleep 1
+done
+
+if [ -f /docker-entrypoint-initdb.d/2_update.sql ]; then
+  mysql -h"db" -u"root" -p"$MYSQL_ROOT_PASSWORD" "$MYSQL_DATABASE" < /docker-entrypoint-initdb.d/2_update.sql
+  echo "Updated database with 2_update.sql"
+fi