Merge pull request #46 from Samourai-Wallet/feat_mydojo_testnet

add testnet support to my-dojo

doc/DOCKER_advanced_setups.md

@@ -1,9 +1,9 @@
 # MyDojo - Advanced Setups
 
 
-The 3 configuration files of Dojo provide a few advanced options allowing to tune your setup.
+The configuration files of Dojo provide a few advanced options allowing to tune your setup.
 
-A word of caution, though, the default values of these options try to maximize your privacy at a network level. All the advanced setups described in this document may damage your privacy. Use at your own risk!
+A word of caution, though, the default values of these options try to maximize your privacy at a network level. Most of the advanced setups described in this document may damage your privacy. Use at your own risk!
 
 
 <a name="external_bitcoind"/>
@@ -159,3 +159,27 @@ nano ./conf/docker-bitcoind.conf
 ```
 
 Note: this option has no effect if your setup relies on a external full node (i.e. if BITCOIND_INSTALL is set to "off").
+
+
+<a name="testnet"/>
+
+## Support of testnet ##
+
+By default, Dojo is installed for running on Bitcoin mainnet.
+
+The following steps allow to install an instance of Dojo running on Bitcoin testnet.
+
+```
+# Edit the common config template file
+nano ./conf/docker-common.conf.tpl
+
+#
+# Set the value of COMMON_BTC_NETWORK to "testnet"
+#
+# Save and exit nano
+#
+```
+
+Note: This option must be set before the first installation of Dojo and mustn't be changed after this first installation.
+
+Known limitation: A single instance of Dojo can be run per machine (a same machine can't host both a mainnet and a testnet instance of Dojo).

doc/DOCKER_setup.md

@@ -177,16 +177,17 @@ Note: The upgrade process will override all manual modifications of the files st
 
 ## Configuration files ##
 
-Each new release of Dojo is packaged with 3 template files stored in the `<dojo_dir>/docker/my-dojo/conf` directory:
+Each new release of Dojo is packaged with 4 template files stored in the `<dojo_dir>/docker/my-dojo/conf` directory:
+- docker-common.conf.tpl
 - docker-bitcoin.conf.tpl
 - docker-mysql.conf.tpl
 - docker-node.conf.tpl
 
-These templates files define default values for configuration options of your Dojo.
+These template files define default values for configuration options of your Dojo.
 
 During the first-time installation (dojo.sh install) these templates are used to initialize the configuration files (files with .conf extension) that will be used by your Dojo.
 
-During an upgrade (dojo.sh upgrade), the content of the templates files is merged with the content of the configuration files, preserving the values that you may have modified in the configuration files. A backup of the configuration files is saved in the same directory (files with .save extension).
+During an upgrade (dojo.sh upgrade), the content of the template files is merged with the content of the configuration files, preserving the values that you may have modified in the configuration files. A backup of the configuration files is saved in the same directory (files with .save extension).
 
 Most options provided in the configuration files can be later modified. New values will become active after a call to
 

docker/my-dojo/.env

@@ -12,9 +12,9 @@ COMPOSE_CONVERT_WINDOWS_PATHS=1
 
 DOJO_VERSION_TAG=1.2.0
 DOJO_DB_VERSION_TAG=1.1.0
-DOJO_BITCOIND_VERSION_TAG=1.1.0
+DOJO_BITCOIND_VERSION_TAG=1.2.0
-DOJO_NODEJS_VERSION_TAG=1.1.0
+DOJO_NODEJS_VERSION_TAG=1.2.0
-DOJO_NGINX_VERSION_TAG=1.1.0
+DOJO_NGINX_VERSION_TAG=1.2.0
 DOJO_TOR_VERSION_TAG=1.1.0
 
 
@@ -41,6 +41,7 @@ NODE_GAP_EXTERNAL=100
 NODE_GAP_INTERNAL=100
 NODE_ADDR_FILTER_THRESHOLD=1000
 NODE_URL_OXT_API=https://api.oxt.me
+NODE_URL_BTCCOM_API=https://tchain.api.btc.com/v3
 NODE_ADDR_DERIVATION_MIN_CHILD=2
 NODE_ADDR_DERIVATION_MAX_CHILD=2
 NODE_ADDR_DERIVATION_THRESHOLD=10

docker/my-dojo/bitcoin/restart.sh

@@ -35,4 +35,8 @@ if [ "$BITCOIND_RPC_EXTERNAL" == "on" ]; then
   bitcoind_options+=(-zmqpubrawblock=tcp://0.0.0.0:9503)
 fi
 
+if [ "$COMMON_BTC_NETWORK" == "testnet" ]; then
+  bitcoind_options+=(-testnet)
+fi
+
 bitcoind "${bitcoind_options[@]}"

docker/my-dojo/conf/docker-common.conf.tpl

@@ -0,0 +1,13 @@
+#
+# EXPERT AND DEV SETTINGS
+#
+
+
+#
+# NETWORK ENVIRONMENT
+#
+
+# Select a Bitcoin network
+# Do not modify this value after the first install
+# Value: mainnet | testnet
+COMMON_BTC_NETWORK=mainnet

docker/my-dojo/docker-compose.yaml

@@ -27,6 +27,7 @@ services:
       dockerfile: ./docker/my-dojo/node/Dockerfile
     env_file:
       - ./.env
+      - ./conf/docker-common.conf
       - ./conf/docker-mysql.conf
       - ./conf/docker-bitcoind.conf
       - ./conf/docker-node.conf
@@ -50,6 +51,7 @@ services:
       context: ./nginx
     env_file:
       - ./.env
+      - ./conf/docker-common.conf
     restart: on-failure
     command: "/wait-for node:8080 --timeout=360 -- nginx"
     expose:

docker/my-dojo/dojo.sh

@@ -10,6 +10,7 @@ source_file() {
 }
 
 source_file "$DIR/conf/docker-bitcoind.conf"
+source_file "$DIR/conf/docker-common.conf"
 source_file "$DIR/.env"
 
 
@@ -187,6 +188,7 @@ logs_node() {
 
 logs() {
   source_file "$DIR/conf/docker-bitcoind.conf"
+  source_file "$DIR/conf/docker-common.conf"
 
   case $1 in
     db )
@@ -194,7 +196,12 @@ logs() {
       ;;
     bitcoind )
       if [ "$BITCOIND_INSTALL" == "on" ]; then
-        docker exec -ti bitcoind tail -f /home/bitcoin/.bitcoin/debug.log
+        if [ "$COMMON_BTC_NETWORK" == "testnet" ]; then
+          bitcoindDataDir="/home/bitcoin/.bitcoin/testnet3"
+        else
+          bitcoindDataDir="/home/bitcoin/.bitcoin"
+        fi
+        docker exec -ti bitcoind tail -f "$bitcoindDataDir/debug.log"
       else
         echo -e "Command not supported for your setup.\nCause: Your Dojo is using an external bitcoind"
       fi

docker/my-dojo/install/install-scripts.sh

@@ -1,9 +1,21 @@
 #!/bin/bash
 
+if [ -f ./conf/docker-bitcoind.conf ]; then
+  source ./conf/docker-bitcoind.conf
+else
+  source ./conf/docker-bitcoind.conf.tpl
+fi
+
+if [ -f ./conf/docker-common.conf ]; then
+  source ./conf/docker-common.conf
+else
+  source ./conf/docker-common.conf.tpl
+fi
+
 # Confirm installation
 get_confirmation() {
   while true; do
-    echo "This operation is going to install Dojo v$DOJO_VERSION_TAG on your computer."
+    echo "This operation is going to install Dojo v$DOJO_VERSION_TAG for $COMMON_BTC_NETWORK on your computer."
     read -p "Do you wish to continue? [y/n]" yn
     case $yn in
       [Yy]* ) return 0;;
@@ -15,6 +27,7 @@ get_confirmation() {
 
 # Initialize configuration files from templates
 init_config_files() {
+  # Initialize db scripts
   cp ../../db-scripts/1_db.sql.tpl ../../db-scripts/1_db.sql
   echo "Initialized 1_db.sql"
 
@@ -23,6 +36,10 @@ init_config_files() {
     echo "Deleted 2_update.sql"
   fi
 
+  # Initialize config files for MyDojo
+  cp ./conf/docker-common.conf.tpl ./conf/docker-common.conf
+  echo "Initialized docker-common.conf"
+
   cp ./conf/docker-bitcoind.conf.tpl ./conf/docker-bitcoind.conf
   echo "Initialized docker-bitcoind.conf"
   
@@ -31,4 +48,17 @@ init_config_files() {
   
   cp ./conf/docker-node.conf.tpl ./conf/docker-node.conf
   echo "Initialized docker-node.conf"
+
+  # Initialize config files for nginx and the maintenance tool 
+  if [ "$COMMON_BTC_NETWORK" == "testnet" ]; then
+    cp ./nginx/testnet.conf ./nginx/dojo.conf
+    echo "Initialized dojo.conf (nginx)"
+    cp ../../static/admin/conf/index-testnet.js ../../static/admin/conf/index.js
+    echo "Initialized index.js (admin module)"
+  else
+    cp ./nginx/mainnet.conf ./nginx/dojo.conf
+    echo "Initialized dojo.conf (nginx)"
+    cp ../../static/admin/conf/index-mainnet.js ../../static/admin/conf/index.js
+    echo "Initialized index.js (admin module)"
+  fi
 }

docker/my-dojo/install/upgrade-scripts.sh

@@ -1,9 +1,17 @@
 #!/bin/bash
 
+if [ -f ./conf/docker-common.conf ]; then
+  source ./conf/docker-common.conf
+else
+  source ./conf/docker-common.conf.tpl
+fi
+
+source ./conf/docker-bitcoind.conf
+
 # Confirm upgrade operation
 get_confirmation() {
   while true; do
-    echo "This operation is going to upgrade your Dojo to v$DOJO_VERSION_TAG."
+    echo "This operation is going to upgrade your Dojo to v$DOJO_VERSION_TAG for $COMMON_BTC_NETWORK."
     read -p "Do you wish to continue? [y/n]" yn
     case $yn in
       [Yy]* ) return 0;;
@@ -15,6 +23,7 @@ get_confirmation() {
 
 # Update configuration files from templates
 update_config_files() {
+  # Initialize db scripts
   if [ -f ../../db-scripts/1_db.sql ]; then
     rm ../../db-scripts/1_db.sql
     echo "Deleted 1_db.sql"
@@ -23,6 +32,14 @@ update_config_files() {
   cp ../../db-scripts/2_update.sql.tpl ../../db-scripts/2_update.sql
   echo "Initialized 2_update.sql"
 
+  # Initialize config files for MyDojo
+  if [ -f ./conf/docker-common.conf ]; then
+    update_config_file ./conf/docker-common.conf ./conf/docker-common.conf.tpl
+  else
+    cp ./conf/docker-common.conf.tpl ./conf/docker-common.conf
+  fi
+  echo "Initialized docker-common.conf"
+
   update_config_file ./conf/docker-bitcoind.conf ./conf/docker-bitcoind.conf.tpl
   echo "Initialized docker-bitcoind.conf"
 
@@ -31,6 +48,19 @@ update_config_files() {
 
   update_config_file ./conf/docker-node.conf ./conf/docker-node.conf.tpl
   echo "Initialized docker-node.conf"
+
+  # Initialize config files for nginx and the maintenance tool 
+  if [ "$COMMON_BTC_NETWORK" == "testnet" ]; then
+    cp ./nginx/testnet.conf ./nginx/dojo.conf
+    echo "Initialized dojo.conf (nginx)"
+    cp ../../static/admin/conf/index-testnet.js ../../static/admin/conf/index.js
+    echo "Initialized index.js (admin module)"
+  else
+    cp ./nginx/mainnet.conf ./nginx/dojo.conf
+    echo "Initialized dojo.conf (nginx)"
+    cp ../../static/admin/conf/index-mainnet.js ../../static/admin/conf/index.js
+    echo "Initialized index.js (admin module)"
+  fi
 }
 
 # Update a configuration file from template

docker/my-dojo/nginx/testnet.conf

@@ -0,0 +1,63 @@
+# Proxy WebSockets
+# https://www.nginx.com/blog/websocket-nginx/
+map $http_upgrade $connection_upgrade {
+    default upgrade;
+    '' close;
+}
+
+# WebSocket server listening here
+upstream websocket {
+    server node:8080;
+}
+
+# Site Configuration
+server {
+    listen 80;
+    server_name _;
+
+    # Set proxy timeouts for the application
+    proxy_connect_timeout 600;
+    proxy_read_timeout 600;
+    proxy_send_timeout 600;
+    send_timeout 600;
+
+    # Proxy WebSocket connections first
+    location /test/v2/inv {
+        proxy_pass http://websocket;
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection $connection_upgrade;
+    }
+
+    # PushTX server is separate, so proxy first
+    location /test/v2/pushtx/ {
+        proxy_pass http://node:8081/;
+    }
+
+    # Proxy requests to maintenance tool
+    location /admin/ {
+        proxy_pass http://node:8080/static/admin/;
+    }
+
+    # Proxy all other v2 requests to the accounts server
+    location /test/v2/ {
+        proxy_pass http://node:8080/;
+    }
+
+    # Redirect onion address to maintenance tool
+    location = / {
+        return 301 /admin;
+    }
+
+    # Serve remaining requests
+    location / {
+        return 200 '{"status":"ok"}';
+        add_header Content-Type application/json;
+    }
+
+    location /test/ {
+        return 200 '{"status":"ok"}';
+        add_header Content-Type application/json;
+    }
+}
+

docker/my-dojo/node/keys.index.js

@@ -3,6 +3,9 @@
  * Copyright (c) 2016-2018, Samourai Wallet (CC BY-NC-ND 4.0 License).
  */
 
+const bitcoinNetwork = (process.env.COMMON_BTC_NETWORK == 'testnet')
+  ? 'testnet'
+  : 'bitcoin'
 
 /**
  * Desired structure of /keys/index.js, which is ignored in the repository.
@@ -11,7 +14,7 @@ module.exports = {
   /*
    * Mainnet parameters
    */
-  bitcoin: {
+  [bitcoinNetwork]: {
     /*
      * Dojo version
      */
@@ -196,8 +199,10 @@ module.exports = {
       // Use a SOCKS5 proxy for all communications with external services
       // Values: null if no socks5 proxy used, otherwise the url of the socks5 proxy
       socks5Proxy: 'socks5h://172.28.1.4:9050',
-      // OXT
+      // OXT (mainnet)
-      oxt: process.env.NODE_URL_OXT_API
+      oxt: process.env.NODE_URL_OXT_API,
+      // BTC.COM (testnet)
+      btccom: process.env.NODE_URL_BTCCOM_API
     },
     /*
      * Max number of transactions per address

docker/my-dojo/node/restart.sh

@@ -1,13 +1,13 @@
 #!/bin/bash
 
 cd /home/node/app/accounts
-forever start -a -l /dev/null -o /data/logs/api-output.log -e /data/logs/api-error.log index.js
+forever start -a -l /dev/null -o /data/logs/api-output.log -e /data/logs/api-error.log index.js "$COMMON_BTC_NETWORK"
 
 cd /home/node/app/pushtx
-forever start -a -l /dev/null -o /data/logs/pushtx-output.log -e /data/logs/pushtx-error.log index.js
+forever start -a -l /dev/null -o /data/logs/pushtx-output.log -e /data/logs/pushtx-error.log index.js "$COMMON_BTC_NETWORK"
-forever start -a -l /dev/null -o /data/logs/pushtx-orchest-output.log -e /data/logs/pushtx-orchest-error.log index-orchestrator.js
+forever start -a -l /dev/null -o /data/logs/pushtx-orchest-output.log -e /data/logs/pushtx-orchest-error.log index-orchestrator.js "$COMMON_BTC_NETWORK"
 
 cd /home/node/app/tracker
-forever start -a -l /dev/null -o /data/logs/tracker-output.log -e /data/logs/tracker-error.log index.js
+forever start -a -l /dev/null -o /data/logs/tracker-output.log -e /data/logs/tracker-error.log index.js "$COMMON_BTC_NETWORK"
 
 forever --fifo logs 0

docker/my-dojo/overrides/bitcoind.install.yaml

@@ -8,6 +8,7 @@ services:
       context: ./bitcoin
     env_file:
       - ./.env
+      - ./conf/docker-common.conf
       - ./conf/docker-bitcoind.conf
     restart: on-failure
     command: "/wait-for-it.sh tor:9050 --timeout=360 --strict -- /restart.sh"

static/admin/conf/index-testnet.js

@@ -0,0 +1,25 @@
+var conf = {
+
+  // Admin tool
+  adminTool: {
+    baseUri: '/admin'
+    //baseUri: '/static/admin'
+  },
+
+  // API
+  api: {
+    baseUri: '/test/v2'
+    //baseUri: ''
+  },
+
+  // Url prefixes
+  prefixes: {
+    // Prefix for /support endpoint
+    support: 'support',
+    // Prefix for /status endpoint
+    status: 'status',
+    // Prefix for pushtx /status endpoint
+    statusPushtx: 'status'
+  }
+
+};