manage all logs with docker logs system

doc/DOCKER_mac_setup.MD

@@ -111,9 +111,9 @@ __"Download the most recent release of Dojo from Github"__ until you reach __"La
 Once you Reach Step __"Launch the Installation of Your Dojo with"__ from above you will need to read and follow the instructions from [here](https://github.com/Samourai-Wallet/samourai-dojo/blob/develop/doc/DOCKER_advanced_setups.md)
 Once adjustments are made to your external bitcoind bitcoin.conf __(location dependent on what device you have bitcoind)__ and docker-bitcoind.conf.tpl __(dojo_dir > docker > my-dojo > conf)__ you can proceed with Install and revert back to original instructions [here](https://github.com/Samourai-Wallet/samourai-dojo/blob/develop/doc/DOCKER_setup.md) at section __"Launch the Installation of Your Dojo with"__
 
-_Note: For tracking progress, open terminal, change directory to my-dojo and run /dojo.sh logs tracker
+_Note: For tracking progress, open terminal, change directory to my-dojo and run /dojo.sh logs nodejs
 __Some possible optimization tips:__
-If you notice that progress has stopped. Click the whale icon and select Restart. Restart Logs Tracker from step above to verify progress has resumed.
+If you notice that progress has stopped. Click the whale icon and select Restart. Restart logs nodejs from step above to verify progress has resumed.
 
 This may optimize speed: open __Activity Monitor__, check the PID (Process ID) of your docker process. Open Terminal and type:
 

doc/DOCKER_setup.md

@@ -182,7 +182,7 @@ Docker and Docker Compose are going to build the images and containers of your D
 * Monitor the progress made for the initialization of the database with this command displaying the logs of the tracker
 
 ```
-./dojo.sh logs tracker
+./dojo.sh logs nodejs
 ```
 
 Exit the logs with CTRL+C when the syncing of the database has completed.
@@ -245,24 +245,22 @@ Available commands:
 
   install                       Install your Dojo.
 
-  logs [module] [options]       Display the logs of your Dojo. Use CTRL+C to stop the logs.
+  logs [module] [options]       Display the logs of your dojo.
+                                  By default, the command displays the live logs. Use CTRL+C to stop the logs.
+                                  Use the -n option to display past logs.
 
                                 Available modules:
-                                  dojo.sh logs                : display the logs of all containers
+                                  dojo.sh logs                : display the logs of all the Docker containers
                                   dojo.sh logs bitcoind       : display the logs of bitcoind
                                   dojo.sh logs db             : display the logs of the MySQL database
                                   dojo.sh logs tor            : display the logs of tor
+                                  dojo.sh logs nginx          : display the logs of nginx
                                   dojo.sh logs indexer        : display the logs of the internal indexer
-                                  dojo.sh logs api            : display the logs of the REST API (nodejs)
-                                  dojo.sh logs tracker        : display the logs of the Tracker (nodejs)
-                                  dojo.sh logs pushtx         : display the logs of the pushTx API (nodejs)
-                                  dojo.sh logs pushtx-orchest : display the logs of the Orchestrator (nodejs)
+                                  dojo.sh logs nodejs         : display the logs of NodeJS modules (API, Tracker, PushTx API, Orchestrator)
                                   dojo.sh logs explorer       : display the logs of the Explorer
                                   dojo.sh logs whirlpool      : display the logs of the Whirlpool client
 
-                                Available options (for api, tracker, pushtx, pushtx-orchest and explorer modules):
-                                  -d [VALUE]                  : select the type of log to be displayed.
-                                                                VALUE can be output (default) or error.
+                                Available options:
                                   -n [VALUE]                  : display the last VALUE lines
 
   onion                         Display the Tor onion addresses allowing to access the API, maintenance tool and block explorer of your Dojo.

docker/my-dojo/.env

@@ -11,13 +11,13 @@
 COMPOSE_CONVERT_WINDOWS_PATHS=1
 
 DOJO_VERSION_TAG=1.6.0
-DOJO_DB_VERSION_TAG=1.1.1
-DOJO_BITCOIND_VERSION_TAG=1.5.0
+DOJO_DB_VERSION_TAG=1.2.0
+DOJO_BITCOIND_VERSION_TAG=1.6.0
 DOJO_NODEJS_VERSION_TAG=1.6.0
 DOJO_NGINX_VERSION_TAG=1.5.0
 DOJO_TOR_VERSION_TAG=1.4.0
 DOJO_EXPLORER_VERSION_TAG=1.3.0
-DOJO_INDEXER_VERSION_TAG=1.0.0
+DOJO_INDEXER_VERSION_TAG=1.1.0
 DOJO_WHIRLPOOL_VERSION_TAG=1.0.0
 
 

docker/my-dojo/bitcoin/restart.sh

@@ -6,6 +6,7 @@ echo "## Start bitcoind #############################"
 bitcoind_options=(
   -bind=172.28.1.5
   -datadir=/home/bitcoin/.bitcoin
+  -printtoconsole=1
   -dbcache=$BITCOIND_DB_CACHE
   -disablewallet=1
   -dns=$BITCOIND_DNS

docker/my-dojo/docker-compose.yaml

@@ -15,6 +15,11 @@ services:
       - "3306"
     volumes:
       - data-mysql:/var/lib/mysql
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "20m"
+        max-file: "10"
     networks:
       dojonet:
         ipv4_address: 172.28.1.1
@@ -40,8 +45,12 @@ services:
       - "8081"
       - "8082"
     volumes:
-      - data-nodejs:/data
       - data-tor:/var/lib/tor
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "20m"
+        max-file: "10"
     depends_on:
       - db
     networks:
@@ -61,8 +70,11 @@ services:
     expose:
       - "80"
       - "9080"
-    volumes:
-      - data-nginx:/data
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "20m"
+        max-file: "10"
     depends_on:
       - node
     networks:
@@ -87,6 +99,11 @@ services:
     command: /restart.sh
     volumes:
       - data-tor:/var/lib/tor
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "20m"
+        max-file: "10"
     networks:
       dmznet:
         ipv4_address: 172.29.1.4
@@ -115,6 +132,4 @@ networks:
 
 volumes:
   data-mysql:
-  data-nodejs:
-  data-nginx:
   data-tor:

docker/my-dojo/dojo.sh

@@ -198,7 +198,7 @@ install() {
     docker_up --remove-orphans
     # Display the logs
     if [ $noLog -eq 1 ]; then
-      logs
+      logs "" 0
     fi
   fi
 }
@@ -326,7 +326,7 @@ upgrade() {
     update_dojo_db
     # Display the logs
     if [ $noLog -eq 1 ]; then
-      logs
+      logs "" 0
     fi
   fi
 }
@@ -381,78 +381,55 @@ whirlpool() {
 }
 
 # Display logs
-logs_node() {
-  if [ $3 -eq 0 ]; then
-    docker exec -ti nodejs tail -f /data/logs/$1-$2.log
-  else
-    docker exec -ti nodejs tail -n $3 /data/logs/$1-$2.log
-  fi 
-}
-
-logs_explorer() {
-  if [ $3 -eq 0 ]; then
-    docker exec -ti explorer tail -f /data/logs/$1-$2.log
-  else
-    docker exec -ti explorer tail -n $3 /data/logs/$1-$2.log
-  fi 
-}
-
-logs_whirlpool() {
-  if [ $3 -eq 0 ]; then
-    docker exec -ti whirlpool tail -f /home/whirlpool/.whirlpool-cli/whirlpool-output.log
+display_logs() {
+  yamlFiles=$(select_yaml_files)
+  if [ $2 -eq 0 ]; then
+    docker-compose $yamlFiles logs --tail=50 --follow $1
   else
-    docker exec -ti whirlpool tail -n $3 /home/whirlpool/.whirlpool-cli/whirlpool-output.log
+    docker-compose $yamlFiles logs --tail=$2 $1
   fi 
 }
 
 logs() {
   source_file "$DIR/conf/docker-bitcoind.conf"
   source_file "$DIR/conf/docker-indexer.conf"
+  source_file "$DIR/conf/docker-explorer.conf"
   source_file "$DIR/conf/docker-whirlpool.conf"
   source_file "$DIR/conf/docker-common.conf"
 
   case $1 in
-    db )
-      docker-compose logs --tail=50 --follow db
+    db | tor | nginx | node )
+      display_logs $1 $2
       ;;
     bitcoind )
       if [ "$BITCOIND_INSTALL" == "on" ]; then
-        if [ "$COMMON_BTC_NETWORK" == "testnet" ]; then
-          bitcoindDataDir="/home/bitcoin/.bitcoin/testnet3"
-        else
-          bitcoindDataDir="/home/bitcoin/.bitcoin"
-        fi
-        docker exec -ti bitcoind tail -f "$bitcoindDataDir/debug.log"
+        display_logs $1 $2
       else
         echo -e "Command not supported for your setup.\nCause: Your Dojo is using an external bitcoind"
       fi
       ;;
     indexer )
       if [ "$INDEXER_INSTALL" == "on" ]; then
-        yamlFiles=$(select_yaml_files)
-        eval "docker-compose $yamlFiles logs --tail=50 --follow indexer"
+        display_logs $1 $2
       else
-        echo -e "Command not supported for your setup.\nCause: Your Dojo is not using an internal indexer"
+        echo -e "Command not supported for your setup.\nCause: Your Dojo is not running the internal indexer"
       fi
       ;;
-    tor )
-      docker-compose logs --tail=50 --follow tor
-      ;;
-    api | pushtx | pushtx-orchest | tracker )
-      logs_node $1 $2 $3
-      ;;
     explorer )
-      logs_explorer $1 $2 $3
+      if [ "$EXPLORER_INSTALL" == "on" ]; then
+        display_logs $1 $2
+      else
+        echo -e "Command not supported for your setup.\nCause: Your Dojo is not running the internal block explorer"
+      fi
       ;;
     whirlpool )
       if [ "$WHIRLPOOL_INSTALL" == "on" ]; then
-        logs_whirlpool $1 $2 $3
+        display_logs $1 $2
       else
         echo -e "Command not supported for your setup.\nCause: Your Dojo is not running a whirlpool client"
       fi
       ;;
     * )
-      yamlFiles=$(select_yaml_files)
       services="nginx node tor db" 
       if [ "$BITCOIND_INSTALL" == "on" ]; then
         services="$services bitcoind"
@@ -466,7 +443,7 @@ logs() {
       if [ "$WHIRLPOOL_INSTALL" == "on" ]; then
         services="$services whirlpool"
       fi
-      eval "docker-compose $yamlFiles logs --tail=0 --follow $services"
+      display_logs "$services" $2
       ;;
   esac
 }
@@ -489,24 +466,22 @@ help() {
   echo "                                Available options:"
   echo "                                  --nolog     : do not display the logs after Dojo has been laucnhed."
   echo " "
-  echo "  logs [module] [options]       Display the logs of your dojo. Use CTRL+C to stop the logs."
+  echo "  logs [module] [options]       Display the logs of your dojo."
+  echo "                                  By default, the command displays the live logs. Use CTRL+C to stop the logs."
+  echo "                                  Use the -n option to display past logs."
   echo " "
   echo "                                Available modules:"
   echo "                                  dojo.sh logs                : display the logs of all the Docker containers"
   echo "                                  dojo.sh logs bitcoind       : display the logs of bitcoind"
   echo "                                  dojo.sh logs db             : display the logs of the MySQL database"
   echo "                                  dojo.sh logs tor            : display the logs of tor"
+  echo "                                  dojo.sh logs nginx          : display the logs of nginx"
   echo "                                  dojo.sh logs indexer        : display the logs of the internal indexer"
-  echo "                                  dojo.sh logs api            : display the logs of the REST API (nodejs)"
-  echo "                                  dojo.sh logs tracker        : display the logs of the Tracker (nodejs)"
-  echo "                                  dojo.sh logs pushtx         : display the logs of the pushTx API (nodejs)"
-  echo "                                  dojo.sh logs pushtx-orchest : display the logs of the pushTx Orchestrator (nodejs)"
+  echo "                                  dojo.sh logs node           : display the logs of NodeJS modules (API, Tracker, PushTx API, Orchestrator)"
   echo "                                  dojo.sh logs explorer       : display the logs of the Explorer"
   echo "                                  dojo.sh logs whirlpool      : display the logs of the Whirlpool client"
   echo " "
-  echo "                                Available options (only available for api, tracker, pushtx, pushtx-orchest, explorer and whirlpool modules):"
-  echo "                                  -d [VALUE]                  : select the type of log to be displayed."
-  echo "                                                                VALUE can be output (default) or error."
+  echo "                                Available options:"
   echo "                                  -n [VALUE]                  : display the last VALUE lines"
   echo " "
   echo "  onion                         Display the Tor onion address allowing your wallet to access your dojo."
@@ -580,15 +555,11 @@ case "$subcommand" in
     ;;
   logs )
     module=$1; shift
-    display="output"
     numlines=0
 
     # Process package options
-    while getopts ":d:n:" opt; do
+    while getopts ":n:" opt; do
       case ${opt} in
-        d )
-          display=$OPTARG
-          ;;
         n )
           numlines=$OPTARG
           ;;
@@ -604,7 +575,7 @@ case "$subcommand" in
     done
     shift $((OPTIND -1))
 
-    logs $module $display $numlines
+    logs "$module" $numlines
     ;;
   onion )
     onion

docker/my-dojo/explorer/Dockerfile

@@ -1,24 +1,18 @@
 FROM    node:12-buster
 
-ENV     LOGS_DIR             /data/logs
 ENV     APP_DIR              /home/node/app
 
 ENV     EXPLORER_URL         https://github.com/janoside/btc-rpc-explorer/archive
 ENV     EXPLORER_VERSION     2.0.0
 
-
 # Install netcat
 RUN     set -ex && \
         apt-get update && \
         apt-get install -y netcat
 
-# Create logs  and apps directory
-RUN     mkdir -p "$LOGS_DIR" && \
-        chown -R node:node "$LOGS_DIR" && \
-        mkdir "$APP_DIR"
-
 # Download the source code and install it
 RUN     set -ex && \
+        mkdir "$APP_DIR" && \
         wget -qO explorer.tar.gz "$EXPLORER_URL/v$EXPLORER_VERSION.tar.gz" && \
         tar -xzvf explorer.tar.gz -C "$APP_DIR/" --strip-components 1 && \
         rm explorer.tar.gz && \

docker/my-dojo/explorer/restart.sh

@@ -42,4 +42,4 @@ if [ "$NODE_ACTIVE_INDEXER" == "local_indexer" ]; then
   fi
 fi
 
-node ./bin/cli.js "${explorer_options[@]}" > /data/logs/explorer-error.log  2> /data/logs/explorer-output.log
+node ./bin/cli.js "${explorer_options[@]}"

docker/my-dojo/nginx/Dockerfile

@@ -1,11 +1,5 @@
 FROM      nginx:1.15.10-alpine
 
-# Create data directory
-ENV       LOGS_DIR /data/logs
-
-RUN       mkdir -p "$LOGS_DIR" && \
-          chown -R nginx:nginx "$LOGS_DIR"
-
 # Copy configuration files
 COPY      ./nginx.conf /etc/nginx/nginx.conf
 COPY      ./dojo.conf /etc/nginx/sites-enabled/dojo.conf

docker/my-dojo/nginx/nginx.conf

@@ -2,8 +2,9 @@ user  nginx;
 worker_processes  auto;
 daemon off;
 
-# Log critical errors and higher
-error_log  /data/logs/error.log crit;
+# Log critical errors and higher to stderr
+# (see https://github.com/nginxinc/docker-nginx/blob/594ce7a8bc26c85af88495ac94d5cd0096b306f7/mainline/alpine/Dockerfile#L104)
+error_log  /var/log/nginx/error.log crit;
 pid        /var/run/nginx.pid;
 
 

docker/my-dojo/node/Dockerfile

@@ -1,9 +1,7 @@
 FROM    node:12-buster
 
-ENV     LOGS_DIR    /data/logs
 ENV     APP_DIR     /home/node/app
 
-
 # Add node user to tor group
 RUN     addgroup --system -gid 1107 tor && \
         usermod -a -G tor node
@@ -11,13 +9,9 @@ RUN     addgroup --system -gid 1107 tor && \
 # Install forever
 RUN     npm install -g forever
 
-# Create data directory
-RUN     mkdir -p "$LOGS_DIR" && \
-        chown -R node:node "$LOGS_DIR"
-
 # Create app directory
 RUN     mkdir "$APP_DIR" && \
-        chown -R node:node "$APP_DIR"
+        chown -R node:node "$APP_DIR" 
 
 # Copy app source files into APP_DIR
 COPY    . "$APP_DIR"

docker/my-dojo/node/restart.sh

@@ -1,13 +1,17 @@
 #!/bin/bash
 
 cd /home/node/app/accounts
-forever start -a -l /dev/null -o /data/logs/api-output.log -e /data/logs/api-error.log index.js "$COMMON_BTC_NETWORK"
+forever start -a -l /dev/stdout -o /dev/null -e /dev/null index.js "$COMMON_BTC_NETWORK"
 
 cd /home/node/app/pushtx
-forever start -a -l /dev/null -o /data/logs/pushtx-output.log -e /data/logs/pushtx-error.log index.js "$COMMON_BTC_NETWORK"
-forever start -a -l /dev/null -o /data/logs/pushtx-orchest-output.log -e /data/logs/pushtx-orchest-error.log index-orchestrator.js "$COMMON_BTC_NETWORK"
+forever start -a -l /dev/stdout -o /dev/null -e /dev/null index.js "$COMMON_BTC_NETWORK"
+forever start -a -l /dev/stdout -o /dev/null -e /dev/null index-orchestrator.js "$COMMON_BTC_NETWORK"
 
 cd /home/node/app/tracker
-forever start -a -l /dev/null -o /data/logs/tracker-output.log -e /data/logs/tracker-error.log index.js "$COMMON_BTC_NETWORK"
+forever start -a -l /dev/stdout -o /dev/null -e /dev/null index.js "$COMMON_BTC_NETWORK"
 
-forever --fifo logs 0
+# Keep the container up
+while true
+do
+  sleep 1
+done

docker/my-dojo/overrides/bitcoind.install.yaml

@@ -20,6 +20,11 @@ services:
     volumes:
       - data-bitcoind:/home/bitcoin/.bitcoin
       - data-tor:/var/lib/tor
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "20m"
+        max-file: "10"
     depends_on:
       - db
       - tor

docker/my-dojo/overrides/explorer.install.yaml

@@ -16,8 +16,11 @@ services:
     command: "/home/node/app/restart.sh"
     expose:
       - "3002"
-    volumes:
-      - data-explorer:/data/logs
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "20m"
+        max-file: "10"
     networks:
       dojonet:
         ipv4_address: 172.28.1.7
@@ -25,6 +28,3 @@ services:
   node:
     depends_on:
       - explorer
-
-volumes:
-  data-explorer:

docker/my-dojo/overrides/indexer.install.yaml

@@ -17,6 +17,11 @@ services:
       - "50001"
     volumes:
       - data-indexer:/home/indexer
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "20m"
+        max-file: "10"
     depends_on:
       - tor
     networks:

docker/my-dojo/overrides/whirlpool.install.yaml

@@ -16,6 +16,11 @@ services:
       - "8898"
     volumes:
       - data-whirlpool:/home/whirlpool
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "20m"
+        max-file: "10"
     networks:
       whirlnet:
         ipv4_address: 172.30.1.8

docker/my-dojo/whirlpool/restart.sh

@@ -12,7 +12,6 @@ whirlpool_options=(
   --cli.torConfig.coordinator.onion=true
   --cli.torConfig.backend.enabled=false
   --cli.torConfig.backend.onion=false
-  --logging.file="/home/whirlpool/.whirlpool-cli/whirlpool-output.log"
 )
 
 if [ "$COMMON_BTC_NETWORK" == "testnet" ]; then

lib/logger.js

@@ -59,7 +59,7 @@ class Logger {
     const s = util.pad10(D.getUTCSeconds())
     const ms = util.pad100(D.getUTCMilliseconds())
     
-    const parts = ['[', y, m, d, ' ', h, ':', mn, ':', s, '.', ms, ' ', mib, ' MiB', '] ', data]
+    const parts = [y, '-', m, '-', d, 'T', h, ':', mn, ':', s, '.', ms,'Z ', data]
     return parts.join('')
   }
 }