From c16604d60258ce0e7e4d0e20e939e72b204db9ea Mon Sep 17 00:00:00 2001 From: kenshin-samourai Date: Thu, 7 Nov 2019 16:05:19 +0100 Subject: [PATCH 01/29] add new local indexer docker module to dojo (experimental) the local indexer is an instance of addrindexrs (see https://github.com/Samourai-Wallet/addrindexrs) running inside dojo --- docker/my-dojo/.env | 8 + docker/my-dojo/conf/docker-indexer.conf.tpl | 42 +++++ docker/my-dojo/conf/docker-node.conf.tpl | 2 +- docker/my-dojo/dojo.sh | 20 ++ docker/my-dojo/indexer/Dockerfile | 46 +++++ docker/my-dojo/indexer/restart.sh | 22 +++ docker/my-dojo/indexer/wait-for-it.sh | 178 ++++++++++++++++++ docker/my-dojo/install/install-scripts.sh | 3 + docker/my-dojo/install/upgrade-scripts.sh | 3 + docker/my-dojo/node/keys.index.js | 12 +- docker/my-dojo/overrides/indexer.install.yaml | 27 +++ 11 files changed, 361 insertions(+), 2 deletions(-) create mode 100644 docker/my-dojo/conf/docker-indexer.conf.tpl create mode 100644 docker/my-dojo/indexer/Dockerfile create mode 100644 docker/my-dojo/indexer/restart.sh create mode 100644 docker/my-dojo/indexer/wait-for-it.sh create mode 100644 docker/my-dojo/overrides/indexer.install.yaml diff --git a/docker/my-dojo/.env b/docker/my-dojo/.env index ea9f6ef..8841053 100644 --- a/docker/my-dojo/.env +++ b/docker/my-dojo/.env @@ -16,6 +16,7 @@ DOJO_BITCOIND_VERSION_TAG=1.2.0 DOJO_NODEJS_VERSION_TAG=1.3.0 DOJO_NGINX_VERSION_TAG=1.3.0 DOJO_TOR_VERSION_TAG=1.2.0 +DOJO_INDEXER_VERSION_TAG=1.0.0 ######################################### @@ -57,3 +58,10 @@ NODE_PREFIX_STATUS_PUSHTX=status NODE_TRACKER_MEMPOOL_PERIOD=10000 NODE_TRACKER_UNCONF_TXS_PERIOD=300000 + + +######################################### +# INDEXER +######################################### + +INDEXER_BATCH_SIZE=10 diff --git a/docker/my-dojo/conf/docker-indexer.conf.tpl b/docker/my-dojo/conf/docker-indexer.conf.tpl new file mode 100644 index 0000000..d7ad83c --- /dev/null +++ b/docker/my-dojo/conf/docker-indexer.conf.tpl @@ -0,0 +1,42 @@ +######################################### +# CONFIGURATION OF A LOCAL INDEXER +######################################### + +# Install and run a local indexer inside Docker +# Set this option to 'off' for using an indexer hosted outside of Docker +# or when using a different data source (local bitcoind, OXT) +# Value: on | off +INDEXER_INSTALL=off + +# IP address of the local indexer used by Dojo +# Set value to 172.28.1.6 if INDEXER_INSTALL is set to 'on' +# Type: string +INDEXER_IP=172.28.1.6 + +# Port of the RPC API +# Set value to 50001 if INDEXER_INSTALL is set to 'on' +# Type: integer +INDEXER_RPC_PORT=50001 + +# Support of batch requests by the local indexer +# Set value to inactive if INDEXER_INSTALL is set to 'on' +# Value: active | inactive +INDEXER_BATCH_SUPPORT=inactive + + +# +# EXPERT SETTINGS +# (ACTIVE IF INDEXER_INSTALL IS SET TO ON) +# + +# Number of blocks to get in one JSONRPC request from bitcoind +# Type: integer +INDEXER_BATCH_SIZE=10 + +# Total size of block txids to cache (in MB) +# Type: integer +INDEXER_BLK_TXIDS_CACHE_SIZE_MB=10 + +# Number of transactions to lookup before returning an error +# Type: integer +INDEXER_TXID_LIMIT=501 \ No newline at end of file diff --git a/docker/my-dojo/conf/docker-node.conf.tpl b/docker/my-dojo/conf/docker-node.conf.tpl index 403ed36..e7adcfa 100644 --- a/docker/my-dojo/conf/docker-node.conf.tpl +++ b/docker/my-dojo/conf/docker-node.conf.tpl @@ -21,7 +21,7 @@ NODE_ADMIN_KEY=myAdminKey NODE_JWT_SECRET=myJwtSecret # Indexer or third-party service used for imports and rescans of addresses -# Values: local_bitcoind | third_party_explorer +# Values: local_bitcoind | local_indexer | third_party_explorer NODE_ACTIVE_INDEXER=local_bitcoind # FEE TYPE USED FOR FEES ESTIMATIONS BY BITCOIND diff --git a/docker/my-dojo/dojo.sh b/docker/my-dojo/dojo.sh index d5f5620..b0e280c 100755 --- a/docker/my-dojo/dojo.sh +++ b/docker/my-dojo/dojo.sh @@ -14,6 +14,7 @@ source_file() { } # Source config files +source_file "$DIR/conf/docker-indexer.conf" source_file "$DIR/conf/docker-bitcoind.conf" source_file "$DIR/conf/docker-common.conf" source_file "$DIR/.env" @@ -33,6 +34,10 @@ select_yaml_files() { fi fi + if [ "$INDEXER_INSTALL" == "on" ]; then + yamlFiles="$yamlFiles -f $DIR/overrides/indexer.install.yaml" + fi + # Return yamlFiles echo "$yamlFiles" } @@ -114,6 +119,7 @@ uninstall() { docker image rm samouraiwallet/dojo-nodejs:"$DOJO_NODEJS_VERSION_TAG" docker image rm samouraiwallet/dojo-nginx:"$DOJO_NGINX_VERSION_TAG" docker image rm samouraiwallet/dojo-tor:"$DOJO_TOR_VERSION_TAG" + docker image rm samouraiwallet/dojo-indexer:"$DOJO_INDEXER_VERSION_TAG" docker volume prune } @@ -136,6 +142,7 @@ clean() { del_images_for samouraiwallet/dojo-nodejs "$DOJO_NODEJS_VERSION_TAG" del_images_for samouraiwallet/dojo-nginx "$DOJO_NGINX_VERSION_TAG" del_images_for samouraiwallet/dojo-tor "$DOJO_TOR_VERSION_TAG" + del_images_for samouraiwallet/dojo-indexer "$DOJO_INDEXER_VERSION_TAG" } # Upgrade @@ -194,6 +201,7 @@ logs_node() { logs() { source_file "$DIR/conf/docker-bitcoind.conf" + source_file "$DIR/conf/docker-indexer.conf" source_file "$DIR/conf/docker-common.conf" case $1 in @@ -212,6 +220,14 @@ logs() { echo -e "Command not supported for your setup.\nCause: Your Dojo is using an external bitcoind" fi ;; + indexer ) + if [ "$INDEXER_INSTALL" == "on" ]; then + yamlFiles=$(select_yaml_files) + eval "docker-compose $yamlFiles logs --tail=50 --follow indexer" + else + echo -e "Command not supported for your setup.\nCause: Your Dojo is not using an internal indexer" + fi + ;; tor ) docker-compose logs --tail=50 --follow tor ;; @@ -224,6 +240,9 @@ logs() { if [ "$BITCOIND_INSTALL" == "on" ]; then services="$services bitcoind" fi + if [ "$INDEXER_INSTALL" == "on" ]; then + services="$services indexer" + fi eval "docker-compose $yamlFiles logs --tail=0 --follow $services" ;; esac @@ -251,6 +270,7 @@ help() { echo " dojo.sh logs bitcoind : display the logs of bitcoind" echo " dojo.sh logs db : display the logs of the MySQL database" echo " dojo.sh logs tor : display the logs of tor" + echo " dojo.sh logs indexer : display the logs of the internal indexer" echo " dojo.sh logs api : display the logs of the REST API (nodejs)" echo " dojo.sh logs tracker : display the logs of the Tracker (nodejs)" echo " dojo.sh logs pushtx : display the logs of the pushTx API (nodejs)" diff --git a/docker/my-dojo/indexer/Dockerfile b/docker/my-dojo/indexer/Dockerfile new file mode 100644 index 0000000..259f827 --- /dev/null +++ b/docker/my-dojo/indexer/Dockerfile @@ -0,0 +1,46 @@ +FROM rust:1.37.0-slim + +ENV INDEXER_HOME /home/indexer +ENV INDEXER_VERSION 0..1.0 +ENV INDEXER_URL https://github.com/Samourai-Wallet/addrindexrs.git + +RUN apt-get update && \ + apt-get install -y clang cmake git && \ + apt-get install -y libsnappy-dev + +# Create group and user indexer +RUN addgroup --system -gid 1109 indexer && \ + adduser --system --ingroup indexer -uid 1106 indexer + +# Create data directory +RUN mkdir "$INDEXER_HOME/addrindexrs" && \ + chown -h indexer:indexer "$INDEXER_HOME/addrindexrs" + +# Copy restart script +COPY ./restart.sh /restart.sh +RUN chown indexer:indexer /restart.sh && \ + chmod 777 /restart.sh + +# Copy wait-for-it script +COPY ./wait-for-it.sh /wait-for-it.sh +RUN chown indexer:indexer /wait-for-it.sh && \ + chmod u+x /wait-for-it.sh && \ + chmod g+x /wait-for-it.sh + +USER indexer + +# Install addrindexrs +RUN cd "$INDEXER_HOME" && \ + git clone "$INDEXER_URL" "$INDEXER_HOME/addrindexrs" && \ + cd addrindexrs && \ + git checkout "master" +# TODO uncomment +# git checkout "tags/v$INDEXER_VERSION" + +RUN cd "$INDEXER_HOME/addrindexrs" && \ + cargo build --release && \ + cargo install --path . + +EXPOSE 50001 + +STOPSIGNAL SIGINT diff --git a/docker/my-dojo/indexer/restart.sh b/docker/my-dojo/indexer/restart.sh new file mode 100644 index 0000000..0db33b1 --- /dev/null +++ b/docker/my-dojo/indexer/restart.sh @@ -0,0 +1,22 @@ +#!/bin/bash +set -e + +indexer_options=( + -vvvv + --index-batch-size="$INDEXER_BATCH_SIZE" + --jsonrpc-import + --db-dir="/home/indexer/db" + --indexer-rpc-addr="172.28.1.6:50001" + --daemon-rpc-addr="$BITCOIND_IP:$BITCOIND_RPC_PORT" + --cookie="$BITCOIND_RPC_USER:$BITCOIND_RPC_PASSWORD" + --txid-limit="$INDEXER_TXID_LIMIT" + --blocktxids-cache-size-mb="$INDEXER_BLK_TXIDS_CACHE_SIZE_MB" +) + +if [ "$COMMON_BTC_NETWORK" == "testnet" ]; then + bitcoind_options+=(--network="testnet") +else + bitcoind_options+=(--network="mainnet") +fi + +addrindexrs "${indexer_options[@]}" diff --git a/docker/my-dojo/indexer/wait-for-it.sh b/docker/my-dojo/indexer/wait-for-it.sh new file mode 100644 index 0000000..071c2be --- /dev/null +++ b/docker/my-dojo/indexer/wait-for-it.sh @@ -0,0 +1,178 @@ +#!/usr/bin/env bash +# Use this script to test if a given TCP host/port are available + +WAITFORIT_cmdname=${0##*/} + +echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } + +usage() +{ + cat << USAGE >&2 +Usage: + $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args] + -h HOST | --host=HOST Host or IP under test + -p PORT | --port=PORT TCP port under test + Alternatively, you specify the host and port as host:port + -s | --strict Only execute subcommand if the test succeeds + -q | --quiet Don't output any status messages + -t TIMEOUT | --timeout=TIMEOUT + Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit 1 +} + +wait_for() +{ + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + else + echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout" + fi + WAITFORIT_start_ts=$(date +%s) + while : + do + if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then + nc -z $WAITFORIT_HOST $WAITFORIT_PORT + WAITFORIT_result=$? + else + (echo > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1 + WAITFORIT_result=$? + fi + if [[ $WAITFORIT_result -eq 0 ]]; then + WAITFORIT_end_ts=$(date +%s) + echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds" + break + fi + sleep 1 + done + return $WAITFORIT_result +} + +wait_for_wrapper() +{ + # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 + if [[ $WAITFORIT_QUIET -eq 1 ]]; then + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + else + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + fi + WAITFORIT_PID=$! + trap "kill -INT -$WAITFORIT_PID" INT + wait $WAITFORIT_PID + WAITFORIT_RESULT=$? + if [[ $WAITFORIT_RESULT -ne 0 ]]; then + echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + fi + return $WAITFORIT_RESULT +} + +# process arguments +while [[ $# -gt 0 ]] +do + case "$1" in + *:* ) + WAITFORIT_hostport=(${1//:/ }) + WAITFORIT_HOST=${WAITFORIT_hostport[0]} + WAITFORIT_PORT=${WAITFORIT_hostport[1]} + shift 1 + ;; + --child) + WAITFORIT_CHILD=1 + shift 1 + ;; + -q | --quiet) + WAITFORIT_QUIET=1 + shift 1 + ;; + -s | --strict) + WAITFORIT_STRICT=1 + shift 1 + ;; + -h) + WAITFORIT_HOST="$2" + if [[ $WAITFORIT_HOST == "" ]]; then break; fi + shift 2 + ;; + --host=*) + WAITFORIT_HOST="${1#*=}" + shift 1 + ;; + -p) + WAITFORIT_PORT="$2" + if [[ $WAITFORIT_PORT == "" ]]; then break; fi + shift 2 + ;; + --port=*) + WAITFORIT_PORT="${1#*=}" + shift 1 + ;; + -t) + WAITFORIT_TIMEOUT="$2" + if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi + shift 2 + ;; + --timeout=*) + WAITFORIT_TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + WAITFORIT_CLI=("$@") + break + ;; + --help) + usage + ;; + *) + echoerr "Unknown argument: $1" + usage + ;; + esac +done + +if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then + echoerr "Error: you need to provide a host and port to test." + usage +fi + +WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15} +WAITFORIT_STRICT=${WAITFORIT_STRICT:-0} +WAITFORIT_CHILD=${WAITFORIT_CHILD:-0} +WAITFORIT_QUIET=${WAITFORIT_QUIET:-0} + +# check to see if timeout is from busybox? +WAITFORIT_TIMEOUT_PATH=$(type -p timeout) +WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH) +if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then + WAITFORIT_ISBUSY=1 + WAITFORIT_BUSYTIMEFLAG="-t" + +else + WAITFORIT_ISBUSY=0 + WAITFORIT_BUSYTIMEFLAG="" +fi + +if [[ $WAITFORIT_CHILD -gt 0 ]]; then + wait_for + WAITFORIT_RESULT=$? + exit $WAITFORIT_RESULT +else + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + wait_for_wrapper + WAITFORIT_RESULT=$? + else + wait_for + WAITFORIT_RESULT=$? + fi +fi + +if [[ $WAITFORIT_CLI != "" ]]; then + if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then + echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess" + exit $WAITFORIT_RESULT + fi + exec "${WAITFORIT_CLI[@]}" +else + exit $WAITFORIT_RESULT +fi diff --git a/docker/my-dojo/install/install-scripts.sh b/docker/my-dojo/install/install-scripts.sh index 88b6bcd..d6ee261 100755 --- a/docker/my-dojo/install/install-scripts.sh +++ b/docker/my-dojo/install/install-scripts.sh @@ -52,6 +52,9 @@ init_config_files() { cp ./conf/docker-tor.conf.tpl ./conf/docker-tor.conf echo "Initialized docker-tor.conf" + cp ./conf/docker-indexer.conf.tpl ./conf/docker-indexer.conf + echo "Initialized docker-indexer.conf" + # Initialize config files for nginx and the maintenance tool if [ "$COMMON_BTC_NETWORK" == "testnet" ]; then cp ./nginx/testnet.conf ./nginx/dojo.conf diff --git a/docker/my-dojo/install/upgrade-scripts.sh b/docker/my-dojo/install/upgrade-scripts.sh index 40e735a..bd84026 100755 --- a/docker/my-dojo/install/upgrade-scripts.sh +++ b/docker/my-dojo/install/upgrade-scripts.sh @@ -48,6 +48,9 @@ update_config_files() { update_config_file ./conf/docker-tor.conf ./conf/docker-tor.conf.tpl echo "Initialized docker-tor.conf" + update_config_file ./conf/docker-indexer.conf ./conf/docker-indexer.conf.tpl + echo "Initialized docker-indexer.conf" + # Initialize config files for nginx and the maintenance tool if [ "$COMMON_BTC_NETWORK" == "testnet" ]; then cp ./nginx/testnet.conf ./nginx/dojo.conf diff --git a/docker/my-dojo/node/keys.index.js b/docker/my-dojo/node/keys.index.js index 300a1a5..a6aae97 100644 --- a/docker/my-dojo/node/keys.index.js +++ b/docker/my-dojo/node/keys.index.js @@ -156,8 +156,18 @@ module.exports = { */ indexer: { // Active indexer - // Values: local_bitcoind | third_party_explorer + // Values: local_bitcoind | local_indexer | third_party_explorer active: process.env.NODE_ACTIVE_INDEXER, + // Local indexer + localIndexer: { + // IP address or hostname of the local indexer + host: process.env.INDEXER_IP, + // Port + port: parseInt(process.env.INDEXER_RPC_PORT), + // Support of batch requests by the local indexer + // Values: active | inactive + batchRequests: process.env.INDEXER_BATCH_SUPPORT + }, // Use a SOCKS5 proxy for all communications with external services // Values: null if no socks5 proxy used, otherwise the url of the socks5 proxy socks5Proxy: 'socks5h://172.28.1.4:9050', diff --git a/docker/my-dojo/overrides/indexer.install.yaml b/docker/my-dojo/overrides/indexer.install.yaml new file mode 100644 index 0000000..e130624 --- /dev/null +++ b/docker/my-dojo/overrides/indexer.install.yaml @@ -0,0 +1,27 @@ +version: "3.2" + +services: + indexer: + image: "samouraiwallet/dojo-indexer:${DOJO_INDEXER_VERSION_TAG}" + container_name: indexer + build: + context: ./indexer + env_file: + - ./.env + - ./conf/docker-common.conf + - ./conf/docker-bitcoind.conf + - ./conf/docker-indexer.conf + restart: on-failure + command: "/wait-for-it.sh tor:9050 --timeout=360 --strict -- /restart.sh" + expose: + - "50001" + volumes: + - data-indexer:/home/indexer/addrindexrs + depends_on: + - tor + networks: + dojonet: + ipv4_address: 172.28.1.6 + +volumes: + data-indexer: From def48aabda07c07a4470629cdd3f1f49307cf781 Mon Sep 17 00:00:00 2001 From: kenshin-samourai Date: Thu, 7 Nov 2019 19:47:15 +0100 Subject: [PATCH 02/29] fix path of docker volume for local indexer --- docker/my-dojo/overrides/indexer.install.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/my-dojo/overrides/indexer.install.yaml b/docker/my-dojo/overrides/indexer.install.yaml index e130624..92635d4 100644 --- a/docker/my-dojo/overrides/indexer.install.yaml +++ b/docker/my-dojo/overrides/indexer.install.yaml @@ -16,7 +16,7 @@ services: expose: - "50001" volumes: - - data-indexer:/home/indexer/addrindexrs + - data-indexer:/home/indexer depends_on: - tor networks: From 8c7b8b08513c24493b878fb0fa266b41f4b2939c Mon Sep 17 00:00:00 2001 From: kenshin-samourai Date: Thu, 7 Nov 2019 22:29:56 +0100 Subject: [PATCH 03/29] add missing network parameter --- lib/remote-importer/local-indexer-wrapper.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/remote-importer/local-indexer-wrapper.js b/lib/remote-importer/local-indexer-wrapper.js index 8891608..9fcc37b 100644 --- a/lib/remote-importer/local-indexer-wrapper.js +++ b/lib/remote-importer/local-indexer-wrapper.js @@ -38,7 +38,7 @@ class LocalIndexerWrapper extends Wrapper { * @returns {string} returns the script hash associated to the address */ _getScriptHash(address) { - const bScriptPubKey = bitcoin.address.toOutputScript(address) + const bScriptPubKey = bitcoin.address.toOutputScript(address, activeNet) const bScriptHash = bitcoin.crypto.sha256(bScriptPubKey) return util.reverseBuffer(bScriptHash).toString('hex') } From d2615e30863b1a2ab0eea1656675e41e10abdf67 Mon Sep 17 00:00:00 2001 From: kenshin-samourai Date: Thu, 7 Nov 2019 22:30:56 +0100 Subject: [PATCH 04/29] add loading of docker-indexer.conf in nodejs container --- docker/my-dojo/docker-compose.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/my-dojo/docker-compose.yaml b/docker/my-dojo/docker-compose.yaml index 4a6633f..c2c16cf 100644 --- a/docker/my-dojo/docker-compose.yaml +++ b/docker/my-dojo/docker-compose.yaml @@ -30,6 +30,7 @@ services: - ./conf/docker-common.conf - ./conf/docker-mysql.conf - ./conf/docker-bitcoind.conf + - ./conf/docker-indexer.conf - ./conf/docker-node.conf restart: on-failure command: "/home/node/app/wait-for-it.sh db:3306 --timeout=360 --strict -- /home/node/app/restart.sh" From d93239370d4823207ff4d9bf19425303d9ab8eba Mon Sep 17 00:00:00 2001 From: kenshin-samourai Date: Thu, 7 Nov 2019 16:05:19 +0100 Subject: [PATCH 05/29] add new local indexer docker module to dojo (experimental) the local indexer is an instance of addrindexrs (see https://github.com/Samourai-Wallet/addrindexrs) running inside dojo --- docker/my-dojo/.env | 8 + docker/my-dojo/conf/docker-indexer.conf.tpl | 42 +++++ docker/my-dojo/conf/docker-node.conf.tpl | 2 +- docker/my-dojo/dojo.sh | 20 ++ docker/my-dojo/indexer/Dockerfile | 46 +++++ docker/my-dojo/indexer/restart.sh | 22 +++ docker/my-dojo/indexer/wait-for-it.sh | 178 ++++++++++++++++++ docker/my-dojo/install/install-scripts.sh | 3 + docker/my-dojo/install/upgrade-scripts.sh | 3 + docker/my-dojo/node/keys.index.js | 12 +- docker/my-dojo/overrides/indexer.install.yaml | 27 +++ 11 files changed, 361 insertions(+), 2 deletions(-) create mode 100644 docker/my-dojo/conf/docker-indexer.conf.tpl create mode 100644 docker/my-dojo/indexer/Dockerfile create mode 100644 docker/my-dojo/indexer/restart.sh create mode 100644 docker/my-dojo/indexer/wait-for-it.sh create mode 100644 docker/my-dojo/overrides/indexer.install.yaml diff --git a/docker/my-dojo/.env b/docker/my-dojo/.env index 57b9ff1..ba3003e 100644 --- a/docker/my-dojo/.env +++ b/docker/my-dojo/.env @@ -16,6 +16,7 @@ DOJO_BITCOIND_VERSION_TAG=1.3.0 DOJO_NODEJS_VERSION_TAG=1.3.0 DOJO_NGINX_VERSION_TAG=1.3.0 DOJO_TOR_VERSION_TAG=1.2.0 +DOJO_INDEXER_VERSION_TAG=1.0.0 ######################################### @@ -57,3 +58,10 @@ NODE_PREFIX_STATUS_PUSHTX=status NODE_TRACKER_MEMPOOL_PERIOD=10000 NODE_TRACKER_UNCONF_TXS_PERIOD=300000 + + +######################################### +# INDEXER +######################################### + +INDEXER_BATCH_SIZE=10 diff --git a/docker/my-dojo/conf/docker-indexer.conf.tpl b/docker/my-dojo/conf/docker-indexer.conf.tpl new file mode 100644 index 0000000..d7ad83c --- /dev/null +++ b/docker/my-dojo/conf/docker-indexer.conf.tpl @@ -0,0 +1,42 @@ +######################################### +# CONFIGURATION OF A LOCAL INDEXER +######################################### + +# Install and run a local indexer inside Docker +# Set this option to 'off' for using an indexer hosted outside of Docker +# or when using a different data source (local bitcoind, OXT) +# Value: on | off +INDEXER_INSTALL=off + +# IP address of the local indexer used by Dojo +# Set value to 172.28.1.6 if INDEXER_INSTALL is set to 'on' +# Type: string +INDEXER_IP=172.28.1.6 + +# Port of the RPC API +# Set value to 50001 if INDEXER_INSTALL is set to 'on' +# Type: integer +INDEXER_RPC_PORT=50001 + +# Support of batch requests by the local indexer +# Set value to inactive if INDEXER_INSTALL is set to 'on' +# Value: active | inactive +INDEXER_BATCH_SUPPORT=inactive + + +# +# EXPERT SETTINGS +# (ACTIVE IF INDEXER_INSTALL IS SET TO ON) +# + +# Number of blocks to get in one JSONRPC request from bitcoind +# Type: integer +INDEXER_BATCH_SIZE=10 + +# Total size of block txids to cache (in MB) +# Type: integer +INDEXER_BLK_TXIDS_CACHE_SIZE_MB=10 + +# Number of transactions to lookup before returning an error +# Type: integer +INDEXER_TXID_LIMIT=501 \ No newline at end of file diff --git a/docker/my-dojo/conf/docker-node.conf.tpl b/docker/my-dojo/conf/docker-node.conf.tpl index 403ed36..e7adcfa 100644 --- a/docker/my-dojo/conf/docker-node.conf.tpl +++ b/docker/my-dojo/conf/docker-node.conf.tpl @@ -21,7 +21,7 @@ NODE_ADMIN_KEY=myAdminKey NODE_JWT_SECRET=myJwtSecret # Indexer or third-party service used for imports and rescans of addresses -# Values: local_bitcoind | third_party_explorer +# Values: local_bitcoind | local_indexer | third_party_explorer NODE_ACTIVE_INDEXER=local_bitcoind # FEE TYPE USED FOR FEES ESTIMATIONS BY BITCOIND diff --git a/docker/my-dojo/dojo.sh b/docker/my-dojo/dojo.sh index d5f5620..b0e280c 100755 --- a/docker/my-dojo/dojo.sh +++ b/docker/my-dojo/dojo.sh @@ -14,6 +14,7 @@ source_file() { } # Source config files +source_file "$DIR/conf/docker-indexer.conf" source_file "$DIR/conf/docker-bitcoind.conf" source_file "$DIR/conf/docker-common.conf" source_file "$DIR/.env" @@ -33,6 +34,10 @@ select_yaml_files() { fi fi + if [ "$INDEXER_INSTALL" == "on" ]; then + yamlFiles="$yamlFiles -f $DIR/overrides/indexer.install.yaml" + fi + # Return yamlFiles echo "$yamlFiles" } @@ -114,6 +119,7 @@ uninstall() { docker image rm samouraiwallet/dojo-nodejs:"$DOJO_NODEJS_VERSION_TAG" docker image rm samouraiwallet/dojo-nginx:"$DOJO_NGINX_VERSION_TAG" docker image rm samouraiwallet/dojo-tor:"$DOJO_TOR_VERSION_TAG" + docker image rm samouraiwallet/dojo-indexer:"$DOJO_INDEXER_VERSION_TAG" docker volume prune } @@ -136,6 +142,7 @@ clean() { del_images_for samouraiwallet/dojo-nodejs "$DOJO_NODEJS_VERSION_TAG" del_images_for samouraiwallet/dojo-nginx "$DOJO_NGINX_VERSION_TAG" del_images_for samouraiwallet/dojo-tor "$DOJO_TOR_VERSION_TAG" + del_images_for samouraiwallet/dojo-indexer "$DOJO_INDEXER_VERSION_TAG" } # Upgrade @@ -194,6 +201,7 @@ logs_node() { logs() { source_file "$DIR/conf/docker-bitcoind.conf" + source_file "$DIR/conf/docker-indexer.conf" source_file "$DIR/conf/docker-common.conf" case $1 in @@ -212,6 +220,14 @@ logs() { echo -e "Command not supported for your setup.\nCause: Your Dojo is using an external bitcoind" fi ;; + indexer ) + if [ "$INDEXER_INSTALL" == "on" ]; then + yamlFiles=$(select_yaml_files) + eval "docker-compose $yamlFiles logs --tail=50 --follow indexer" + else + echo -e "Command not supported for your setup.\nCause: Your Dojo is not using an internal indexer" + fi + ;; tor ) docker-compose logs --tail=50 --follow tor ;; @@ -224,6 +240,9 @@ logs() { if [ "$BITCOIND_INSTALL" == "on" ]; then services="$services bitcoind" fi + if [ "$INDEXER_INSTALL" == "on" ]; then + services="$services indexer" + fi eval "docker-compose $yamlFiles logs --tail=0 --follow $services" ;; esac @@ -251,6 +270,7 @@ help() { echo " dojo.sh logs bitcoind : display the logs of bitcoind" echo " dojo.sh logs db : display the logs of the MySQL database" echo " dojo.sh logs tor : display the logs of tor" + echo " dojo.sh logs indexer : display the logs of the internal indexer" echo " dojo.sh logs api : display the logs of the REST API (nodejs)" echo " dojo.sh logs tracker : display the logs of the Tracker (nodejs)" echo " dojo.sh logs pushtx : display the logs of the pushTx API (nodejs)" diff --git a/docker/my-dojo/indexer/Dockerfile b/docker/my-dojo/indexer/Dockerfile new file mode 100644 index 0000000..259f827 --- /dev/null +++ b/docker/my-dojo/indexer/Dockerfile @@ -0,0 +1,46 @@ +FROM rust:1.37.0-slim + +ENV INDEXER_HOME /home/indexer +ENV INDEXER_VERSION 0..1.0 +ENV INDEXER_URL https://github.com/Samourai-Wallet/addrindexrs.git + +RUN apt-get update && \ + apt-get install -y clang cmake git && \ + apt-get install -y libsnappy-dev + +# Create group and user indexer +RUN addgroup --system -gid 1109 indexer && \ + adduser --system --ingroup indexer -uid 1106 indexer + +# Create data directory +RUN mkdir "$INDEXER_HOME/addrindexrs" && \ + chown -h indexer:indexer "$INDEXER_HOME/addrindexrs" + +# Copy restart script +COPY ./restart.sh /restart.sh +RUN chown indexer:indexer /restart.sh && \ + chmod 777 /restart.sh + +# Copy wait-for-it script +COPY ./wait-for-it.sh /wait-for-it.sh +RUN chown indexer:indexer /wait-for-it.sh && \ + chmod u+x /wait-for-it.sh && \ + chmod g+x /wait-for-it.sh + +USER indexer + +# Install addrindexrs +RUN cd "$INDEXER_HOME" && \ + git clone "$INDEXER_URL" "$INDEXER_HOME/addrindexrs" && \ + cd addrindexrs && \ + git checkout "master" +# TODO uncomment +# git checkout "tags/v$INDEXER_VERSION" + +RUN cd "$INDEXER_HOME/addrindexrs" && \ + cargo build --release && \ + cargo install --path . + +EXPOSE 50001 + +STOPSIGNAL SIGINT diff --git a/docker/my-dojo/indexer/restart.sh b/docker/my-dojo/indexer/restart.sh new file mode 100644 index 0000000..0db33b1 --- /dev/null +++ b/docker/my-dojo/indexer/restart.sh @@ -0,0 +1,22 @@ +#!/bin/bash +set -e + +indexer_options=( + -vvvv + --index-batch-size="$INDEXER_BATCH_SIZE" + --jsonrpc-import + --db-dir="/home/indexer/db" + --indexer-rpc-addr="172.28.1.6:50001" + --daemon-rpc-addr="$BITCOIND_IP:$BITCOIND_RPC_PORT" + --cookie="$BITCOIND_RPC_USER:$BITCOIND_RPC_PASSWORD" + --txid-limit="$INDEXER_TXID_LIMIT" + --blocktxids-cache-size-mb="$INDEXER_BLK_TXIDS_CACHE_SIZE_MB" +) + +if [ "$COMMON_BTC_NETWORK" == "testnet" ]; then + bitcoind_options+=(--network="testnet") +else + bitcoind_options+=(--network="mainnet") +fi + +addrindexrs "${indexer_options[@]}" diff --git a/docker/my-dojo/indexer/wait-for-it.sh b/docker/my-dojo/indexer/wait-for-it.sh new file mode 100644 index 0000000..071c2be --- /dev/null +++ b/docker/my-dojo/indexer/wait-for-it.sh @@ -0,0 +1,178 @@ +#!/usr/bin/env bash +# Use this script to test if a given TCP host/port are available + +WAITFORIT_cmdname=${0##*/} + +echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } + +usage() +{ + cat << USAGE >&2 +Usage: + $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args] + -h HOST | --host=HOST Host or IP under test + -p PORT | --port=PORT TCP port under test + Alternatively, you specify the host and port as host:port + -s | --strict Only execute subcommand if the test succeeds + -q | --quiet Don't output any status messages + -t TIMEOUT | --timeout=TIMEOUT + Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit 1 +} + +wait_for() +{ + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + else + echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout" + fi + WAITFORIT_start_ts=$(date +%s) + while : + do + if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then + nc -z $WAITFORIT_HOST $WAITFORIT_PORT + WAITFORIT_result=$? + else + (echo > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1 + WAITFORIT_result=$? + fi + if [[ $WAITFORIT_result -eq 0 ]]; then + WAITFORIT_end_ts=$(date +%s) + echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds" + break + fi + sleep 1 + done + return $WAITFORIT_result +} + +wait_for_wrapper() +{ + # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 + if [[ $WAITFORIT_QUIET -eq 1 ]]; then + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + else + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + fi + WAITFORIT_PID=$! + trap "kill -INT -$WAITFORIT_PID" INT + wait $WAITFORIT_PID + WAITFORIT_RESULT=$? + if [[ $WAITFORIT_RESULT -ne 0 ]]; then + echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + fi + return $WAITFORIT_RESULT +} + +# process arguments +while [[ $# -gt 0 ]] +do + case "$1" in + *:* ) + WAITFORIT_hostport=(${1//:/ }) + WAITFORIT_HOST=${WAITFORIT_hostport[0]} + WAITFORIT_PORT=${WAITFORIT_hostport[1]} + shift 1 + ;; + --child) + WAITFORIT_CHILD=1 + shift 1 + ;; + -q | --quiet) + WAITFORIT_QUIET=1 + shift 1 + ;; + -s | --strict) + WAITFORIT_STRICT=1 + shift 1 + ;; + -h) + WAITFORIT_HOST="$2" + if [[ $WAITFORIT_HOST == "" ]]; then break; fi + shift 2 + ;; + --host=*) + WAITFORIT_HOST="${1#*=}" + shift 1 + ;; + -p) + WAITFORIT_PORT="$2" + if [[ $WAITFORIT_PORT == "" ]]; then break; fi + shift 2 + ;; + --port=*) + WAITFORIT_PORT="${1#*=}" + shift 1 + ;; + -t) + WAITFORIT_TIMEOUT="$2" + if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi + shift 2 + ;; + --timeout=*) + WAITFORIT_TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + WAITFORIT_CLI=("$@") + break + ;; + --help) + usage + ;; + *) + echoerr "Unknown argument: $1" + usage + ;; + esac +done + +if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then + echoerr "Error: you need to provide a host and port to test." + usage +fi + +WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15} +WAITFORIT_STRICT=${WAITFORIT_STRICT:-0} +WAITFORIT_CHILD=${WAITFORIT_CHILD:-0} +WAITFORIT_QUIET=${WAITFORIT_QUIET:-0} + +# check to see if timeout is from busybox? +WAITFORIT_TIMEOUT_PATH=$(type -p timeout) +WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH) +if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then + WAITFORIT_ISBUSY=1 + WAITFORIT_BUSYTIMEFLAG="-t" + +else + WAITFORIT_ISBUSY=0 + WAITFORIT_BUSYTIMEFLAG="" +fi + +if [[ $WAITFORIT_CHILD -gt 0 ]]; then + wait_for + WAITFORIT_RESULT=$? + exit $WAITFORIT_RESULT +else + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + wait_for_wrapper + WAITFORIT_RESULT=$? + else + wait_for + WAITFORIT_RESULT=$? + fi +fi + +if [[ $WAITFORIT_CLI != "" ]]; then + if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then + echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess" + exit $WAITFORIT_RESULT + fi + exec "${WAITFORIT_CLI[@]}" +else + exit $WAITFORIT_RESULT +fi diff --git a/docker/my-dojo/install/install-scripts.sh b/docker/my-dojo/install/install-scripts.sh index 88b6bcd..d6ee261 100755 --- a/docker/my-dojo/install/install-scripts.sh +++ b/docker/my-dojo/install/install-scripts.sh @@ -52,6 +52,9 @@ init_config_files() { cp ./conf/docker-tor.conf.tpl ./conf/docker-tor.conf echo "Initialized docker-tor.conf" + cp ./conf/docker-indexer.conf.tpl ./conf/docker-indexer.conf + echo "Initialized docker-indexer.conf" + # Initialize config files for nginx and the maintenance tool if [ "$COMMON_BTC_NETWORK" == "testnet" ]; then cp ./nginx/testnet.conf ./nginx/dojo.conf diff --git a/docker/my-dojo/install/upgrade-scripts.sh b/docker/my-dojo/install/upgrade-scripts.sh index 40e735a..bd84026 100755 --- a/docker/my-dojo/install/upgrade-scripts.sh +++ b/docker/my-dojo/install/upgrade-scripts.sh @@ -48,6 +48,9 @@ update_config_files() { update_config_file ./conf/docker-tor.conf ./conf/docker-tor.conf.tpl echo "Initialized docker-tor.conf" + update_config_file ./conf/docker-indexer.conf ./conf/docker-indexer.conf.tpl + echo "Initialized docker-indexer.conf" + # Initialize config files for nginx and the maintenance tool if [ "$COMMON_BTC_NETWORK" == "testnet" ]; then cp ./nginx/testnet.conf ./nginx/dojo.conf diff --git a/docker/my-dojo/node/keys.index.js b/docker/my-dojo/node/keys.index.js index 300a1a5..a6aae97 100644 --- a/docker/my-dojo/node/keys.index.js +++ b/docker/my-dojo/node/keys.index.js @@ -156,8 +156,18 @@ module.exports = { */ indexer: { // Active indexer - // Values: local_bitcoind | third_party_explorer + // Values: local_bitcoind | local_indexer | third_party_explorer active: process.env.NODE_ACTIVE_INDEXER, + // Local indexer + localIndexer: { + // IP address or hostname of the local indexer + host: process.env.INDEXER_IP, + // Port + port: parseInt(process.env.INDEXER_RPC_PORT), + // Support of batch requests by the local indexer + // Values: active | inactive + batchRequests: process.env.INDEXER_BATCH_SUPPORT + }, // Use a SOCKS5 proxy for all communications with external services // Values: null if no socks5 proxy used, otherwise the url of the socks5 proxy socks5Proxy: 'socks5h://172.28.1.4:9050', diff --git a/docker/my-dojo/overrides/indexer.install.yaml b/docker/my-dojo/overrides/indexer.install.yaml new file mode 100644 index 0000000..e130624 --- /dev/null +++ b/docker/my-dojo/overrides/indexer.install.yaml @@ -0,0 +1,27 @@ +version: "3.2" + +services: + indexer: + image: "samouraiwallet/dojo-indexer:${DOJO_INDEXER_VERSION_TAG}" + container_name: indexer + build: + context: ./indexer + env_file: + - ./.env + - ./conf/docker-common.conf + - ./conf/docker-bitcoind.conf + - ./conf/docker-indexer.conf + restart: on-failure + command: "/wait-for-it.sh tor:9050 --timeout=360 --strict -- /restart.sh" + expose: + - "50001" + volumes: + - data-indexer:/home/indexer/addrindexrs + depends_on: + - tor + networks: + dojonet: + ipv4_address: 172.28.1.6 + +volumes: + data-indexer: From 9830f1e9c0197e007327db6c4a4779156953bca3 Mon Sep 17 00:00:00 2001 From: kenshin-samourai Date: Thu, 7 Nov 2019 19:47:15 +0100 Subject: [PATCH 06/29] fix path of docker volume for local indexer --- docker/my-dojo/overrides/indexer.install.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/my-dojo/overrides/indexer.install.yaml b/docker/my-dojo/overrides/indexer.install.yaml index e130624..92635d4 100644 --- a/docker/my-dojo/overrides/indexer.install.yaml +++ b/docker/my-dojo/overrides/indexer.install.yaml @@ -16,7 +16,7 @@ services: expose: - "50001" volumes: - - data-indexer:/home/indexer/addrindexrs + - data-indexer:/home/indexer depends_on: - tor networks: From cec37a47fffcfbc858ea0a5db7e42ae3f1889834 Mon Sep 17 00:00:00 2001 From: kenshin-samourai Date: Thu, 7 Nov 2019 22:30:56 +0100 Subject: [PATCH 07/29] add loading of docker-indexer.conf in nodejs container --- docker/my-dojo/docker-compose.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/my-dojo/docker-compose.yaml b/docker/my-dojo/docker-compose.yaml index eb2ef0e..02d1423 100644 --- a/docker/my-dojo/docker-compose.yaml +++ b/docker/my-dojo/docker-compose.yaml @@ -30,6 +30,7 @@ services: - ./conf/docker-common.conf - ./conf/docker-mysql.conf - ./conf/docker-bitcoind.conf + - ./conf/docker-indexer.conf - ./conf/docker-node.conf restart: on-failure command: "/home/node/app/wait-for-it.sh db:3306 --timeout=720 --strict -- /home/node/app/restart.sh" From 34b96424aafb2d18cd58b7a4d51432f8212c5f96 Mon Sep 17 00:00:00 2001 From: kenshin-samourai Date: Thu, 7 Nov 2019 16:05:19 +0100 Subject: [PATCH 08/29] add new local indexer docker module to dojo (experimental) the local indexer is an instance of addrindexrs (see https://github.com/Samourai-Wallet/addrindexrs) running inside dojo --- docker/my-dojo/.env | 8 + docker/my-dojo/conf/docker-indexer.conf.tpl | 42 +++++ docker/my-dojo/conf/docker-node.conf.tpl | 2 +- docker/my-dojo/dojo.sh | 20 ++ docker/my-dojo/indexer/Dockerfile | 46 +++++ docker/my-dojo/indexer/restart.sh | 22 +++ docker/my-dojo/indexer/wait-for-it.sh | 178 ++++++++++++++++++ docker/my-dojo/install/install-scripts.sh | 3 + docker/my-dojo/install/upgrade-scripts.sh | 3 + docker/my-dojo/node/keys.index.js | 12 +- docker/my-dojo/overrides/indexer.install.yaml | 27 +++ 11 files changed, 361 insertions(+), 2 deletions(-) create mode 100644 docker/my-dojo/conf/docker-indexer.conf.tpl create mode 100644 docker/my-dojo/indexer/Dockerfile create mode 100644 docker/my-dojo/indexer/restart.sh create mode 100644 docker/my-dojo/indexer/wait-for-it.sh create mode 100644 docker/my-dojo/overrides/indexer.install.yaml diff --git a/docker/my-dojo/.env b/docker/my-dojo/.env index 57b9ff1..ba3003e 100644 --- a/docker/my-dojo/.env +++ b/docker/my-dojo/.env @@ -16,6 +16,7 @@ DOJO_BITCOIND_VERSION_TAG=1.3.0 DOJO_NODEJS_VERSION_TAG=1.3.0 DOJO_NGINX_VERSION_TAG=1.3.0 DOJO_TOR_VERSION_TAG=1.2.0 +DOJO_INDEXER_VERSION_TAG=1.0.0 ######################################### @@ -57,3 +58,10 @@ NODE_PREFIX_STATUS_PUSHTX=status NODE_TRACKER_MEMPOOL_PERIOD=10000 NODE_TRACKER_UNCONF_TXS_PERIOD=300000 + + +######################################### +# INDEXER +######################################### + +INDEXER_BATCH_SIZE=10 diff --git a/docker/my-dojo/conf/docker-indexer.conf.tpl b/docker/my-dojo/conf/docker-indexer.conf.tpl new file mode 100644 index 0000000..d7ad83c --- /dev/null +++ b/docker/my-dojo/conf/docker-indexer.conf.tpl @@ -0,0 +1,42 @@ +######################################### +# CONFIGURATION OF A LOCAL INDEXER +######################################### + +# Install and run a local indexer inside Docker +# Set this option to 'off' for using an indexer hosted outside of Docker +# or when using a different data source (local bitcoind, OXT) +# Value: on | off +INDEXER_INSTALL=off + +# IP address of the local indexer used by Dojo +# Set value to 172.28.1.6 if INDEXER_INSTALL is set to 'on' +# Type: string +INDEXER_IP=172.28.1.6 + +# Port of the RPC API +# Set value to 50001 if INDEXER_INSTALL is set to 'on' +# Type: integer +INDEXER_RPC_PORT=50001 + +# Support of batch requests by the local indexer +# Set value to inactive if INDEXER_INSTALL is set to 'on' +# Value: active | inactive +INDEXER_BATCH_SUPPORT=inactive + + +# +# EXPERT SETTINGS +# (ACTIVE IF INDEXER_INSTALL IS SET TO ON) +# + +# Number of blocks to get in one JSONRPC request from bitcoind +# Type: integer +INDEXER_BATCH_SIZE=10 + +# Total size of block txids to cache (in MB) +# Type: integer +INDEXER_BLK_TXIDS_CACHE_SIZE_MB=10 + +# Number of transactions to lookup before returning an error +# Type: integer +INDEXER_TXID_LIMIT=501 \ No newline at end of file diff --git a/docker/my-dojo/conf/docker-node.conf.tpl b/docker/my-dojo/conf/docker-node.conf.tpl index 403ed36..e7adcfa 100644 --- a/docker/my-dojo/conf/docker-node.conf.tpl +++ b/docker/my-dojo/conf/docker-node.conf.tpl @@ -21,7 +21,7 @@ NODE_ADMIN_KEY=myAdminKey NODE_JWT_SECRET=myJwtSecret # Indexer or third-party service used for imports and rescans of addresses -# Values: local_bitcoind | third_party_explorer +# Values: local_bitcoind | local_indexer | third_party_explorer NODE_ACTIVE_INDEXER=local_bitcoind # FEE TYPE USED FOR FEES ESTIMATIONS BY BITCOIND diff --git a/docker/my-dojo/dojo.sh b/docker/my-dojo/dojo.sh index d5f5620..b0e280c 100755 --- a/docker/my-dojo/dojo.sh +++ b/docker/my-dojo/dojo.sh @@ -14,6 +14,7 @@ source_file() { } # Source config files +source_file "$DIR/conf/docker-indexer.conf" source_file "$DIR/conf/docker-bitcoind.conf" source_file "$DIR/conf/docker-common.conf" source_file "$DIR/.env" @@ -33,6 +34,10 @@ select_yaml_files() { fi fi + if [ "$INDEXER_INSTALL" == "on" ]; then + yamlFiles="$yamlFiles -f $DIR/overrides/indexer.install.yaml" + fi + # Return yamlFiles echo "$yamlFiles" } @@ -114,6 +119,7 @@ uninstall() { docker image rm samouraiwallet/dojo-nodejs:"$DOJO_NODEJS_VERSION_TAG" docker image rm samouraiwallet/dojo-nginx:"$DOJO_NGINX_VERSION_TAG" docker image rm samouraiwallet/dojo-tor:"$DOJO_TOR_VERSION_TAG" + docker image rm samouraiwallet/dojo-indexer:"$DOJO_INDEXER_VERSION_TAG" docker volume prune } @@ -136,6 +142,7 @@ clean() { del_images_for samouraiwallet/dojo-nodejs "$DOJO_NODEJS_VERSION_TAG" del_images_for samouraiwallet/dojo-nginx "$DOJO_NGINX_VERSION_TAG" del_images_for samouraiwallet/dojo-tor "$DOJO_TOR_VERSION_TAG" + del_images_for samouraiwallet/dojo-indexer "$DOJO_INDEXER_VERSION_TAG" } # Upgrade @@ -194,6 +201,7 @@ logs_node() { logs() { source_file "$DIR/conf/docker-bitcoind.conf" + source_file "$DIR/conf/docker-indexer.conf" source_file "$DIR/conf/docker-common.conf" case $1 in @@ -212,6 +220,14 @@ logs() { echo -e "Command not supported for your setup.\nCause: Your Dojo is using an external bitcoind" fi ;; + indexer ) + if [ "$INDEXER_INSTALL" == "on" ]; then + yamlFiles=$(select_yaml_files) + eval "docker-compose $yamlFiles logs --tail=50 --follow indexer" + else + echo -e "Command not supported for your setup.\nCause: Your Dojo is not using an internal indexer" + fi + ;; tor ) docker-compose logs --tail=50 --follow tor ;; @@ -224,6 +240,9 @@ logs() { if [ "$BITCOIND_INSTALL" == "on" ]; then services="$services bitcoind" fi + if [ "$INDEXER_INSTALL" == "on" ]; then + services="$services indexer" + fi eval "docker-compose $yamlFiles logs --tail=0 --follow $services" ;; esac @@ -251,6 +270,7 @@ help() { echo " dojo.sh logs bitcoind : display the logs of bitcoind" echo " dojo.sh logs db : display the logs of the MySQL database" echo " dojo.sh logs tor : display the logs of tor" + echo " dojo.sh logs indexer : display the logs of the internal indexer" echo " dojo.sh logs api : display the logs of the REST API (nodejs)" echo " dojo.sh logs tracker : display the logs of the Tracker (nodejs)" echo " dojo.sh logs pushtx : display the logs of the pushTx API (nodejs)" diff --git a/docker/my-dojo/indexer/Dockerfile b/docker/my-dojo/indexer/Dockerfile new file mode 100644 index 0000000..259f827 --- /dev/null +++ b/docker/my-dojo/indexer/Dockerfile @@ -0,0 +1,46 @@ +FROM rust:1.37.0-slim + +ENV INDEXER_HOME /home/indexer +ENV INDEXER_VERSION 0..1.0 +ENV INDEXER_URL https://github.com/Samourai-Wallet/addrindexrs.git + +RUN apt-get update && \ + apt-get install -y clang cmake git && \ + apt-get install -y libsnappy-dev + +# Create group and user indexer +RUN addgroup --system -gid 1109 indexer && \ + adduser --system --ingroup indexer -uid 1106 indexer + +# Create data directory +RUN mkdir "$INDEXER_HOME/addrindexrs" && \ + chown -h indexer:indexer "$INDEXER_HOME/addrindexrs" + +# Copy restart script +COPY ./restart.sh /restart.sh +RUN chown indexer:indexer /restart.sh && \ + chmod 777 /restart.sh + +# Copy wait-for-it script +COPY ./wait-for-it.sh /wait-for-it.sh +RUN chown indexer:indexer /wait-for-it.sh && \ + chmod u+x /wait-for-it.sh && \ + chmod g+x /wait-for-it.sh + +USER indexer + +# Install addrindexrs +RUN cd "$INDEXER_HOME" && \ + git clone "$INDEXER_URL" "$INDEXER_HOME/addrindexrs" && \ + cd addrindexrs && \ + git checkout "master" +# TODO uncomment +# git checkout "tags/v$INDEXER_VERSION" + +RUN cd "$INDEXER_HOME/addrindexrs" && \ + cargo build --release && \ + cargo install --path . + +EXPOSE 50001 + +STOPSIGNAL SIGINT diff --git a/docker/my-dojo/indexer/restart.sh b/docker/my-dojo/indexer/restart.sh new file mode 100644 index 0000000..0db33b1 --- /dev/null +++ b/docker/my-dojo/indexer/restart.sh @@ -0,0 +1,22 @@ +#!/bin/bash +set -e + +indexer_options=( + -vvvv + --index-batch-size="$INDEXER_BATCH_SIZE" + --jsonrpc-import + --db-dir="/home/indexer/db" + --indexer-rpc-addr="172.28.1.6:50001" + --daemon-rpc-addr="$BITCOIND_IP:$BITCOIND_RPC_PORT" + --cookie="$BITCOIND_RPC_USER:$BITCOIND_RPC_PASSWORD" + --txid-limit="$INDEXER_TXID_LIMIT" + --blocktxids-cache-size-mb="$INDEXER_BLK_TXIDS_CACHE_SIZE_MB" +) + +if [ "$COMMON_BTC_NETWORK" == "testnet" ]; then + bitcoind_options+=(--network="testnet") +else + bitcoind_options+=(--network="mainnet") +fi + +addrindexrs "${indexer_options[@]}" diff --git a/docker/my-dojo/indexer/wait-for-it.sh b/docker/my-dojo/indexer/wait-for-it.sh new file mode 100644 index 0000000..071c2be --- /dev/null +++ b/docker/my-dojo/indexer/wait-for-it.sh @@ -0,0 +1,178 @@ +#!/usr/bin/env bash +# Use this script to test if a given TCP host/port are available + +WAITFORIT_cmdname=${0##*/} + +echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } + +usage() +{ + cat << USAGE >&2 +Usage: + $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args] + -h HOST | --host=HOST Host or IP under test + -p PORT | --port=PORT TCP port under test + Alternatively, you specify the host and port as host:port + -s | --strict Only execute subcommand if the test succeeds + -q | --quiet Don't output any status messages + -t TIMEOUT | --timeout=TIMEOUT + Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit 1 +} + +wait_for() +{ + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + else + echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout" + fi + WAITFORIT_start_ts=$(date +%s) + while : + do + if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then + nc -z $WAITFORIT_HOST $WAITFORIT_PORT + WAITFORIT_result=$? + else + (echo > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1 + WAITFORIT_result=$? + fi + if [[ $WAITFORIT_result -eq 0 ]]; then + WAITFORIT_end_ts=$(date +%s) + echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds" + break + fi + sleep 1 + done + return $WAITFORIT_result +} + +wait_for_wrapper() +{ + # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 + if [[ $WAITFORIT_QUIET -eq 1 ]]; then + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + else + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + fi + WAITFORIT_PID=$! + trap "kill -INT -$WAITFORIT_PID" INT + wait $WAITFORIT_PID + WAITFORIT_RESULT=$? + if [[ $WAITFORIT_RESULT -ne 0 ]]; then + echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + fi + return $WAITFORIT_RESULT +} + +# process arguments +while [[ $# -gt 0 ]] +do + case "$1" in + *:* ) + WAITFORIT_hostport=(${1//:/ }) + WAITFORIT_HOST=${WAITFORIT_hostport[0]} + WAITFORIT_PORT=${WAITFORIT_hostport[1]} + shift 1 + ;; + --child) + WAITFORIT_CHILD=1 + shift 1 + ;; + -q | --quiet) + WAITFORIT_QUIET=1 + shift 1 + ;; + -s | --strict) + WAITFORIT_STRICT=1 + shift 1 + ;; + -h) + WAITFORIT_HOST="$2" + if [[ $WAITFORIT_HOST == "" ]]; then break; fi + shift 2 + ;; + --host=*) + WAITFORIT_HOST="${1#*=}" + shift 1 + ;; + -p) + WAITFORIT_PORT="$2" + if [[ $WAITFORIT_PORT == "" ]]; then break; fi + shift 2 + ;; + --port=*) + WAITFORIT_PORT="${1#*=}" + shift 1 + ;; + -t) + WAITFORIT_TIMEOUT="$2" + if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi + shift 2 + ;; + --timeout=*) + WAITFORIT_TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + WAITFORIT_CLI=("$@") + break + ;; + --help) + usage + ;; + *) + echoerr "Unknown argument: $1" + usage + ;; + esac +done + +if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then + echoerr "Error: you need to provide a host and port to test." + usage +fi + +WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15} +WAITFORIT_STRICT=${WAITFORIT_STRICT:-0} +WAITFORIT_CHILD=${WAITFORIT_CHILD:-0} +WAITFORIT_QUIET=${WAITFORIT_QUIET:-0} + +# check to see if timeout is from busybox? +WAITFORIT_TIMEOUT_PATH=$(type -p timeout) +WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH) +if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then + WAITFORIT_ISBUSY=1 + WAITFORIT_BUSYTIMEFLAG="-t" + +else + WAITFORIT_ISBUSY=0 + WAITFORIT_BUSYTIMEFLAG="" +fi + +if [[ $WAITFORIT_CHILD -gt 0 ]]; then + wait_for + WAITFORIT_RESULT=$? + exit $WAITFORIT_RESULT +else + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + wait_for_wrapper + WAITFORIT_RESULT=$? + else + wait_for + WAITFORIT_RESULT=$? + fi +fi + +if [[ $WAITFORIT_CLI != "" ]]; then + if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then + echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess" + exit $WAITFORIT_RESULT + fi + exec "${WAITFORIT_CLI[@]}" +else + exit $WAITFORIT_RESULT +fi diff --git a/docker/my-dojo/install/install-scripts.sh b/docker/my-dojo/install/install-scripts.sh index 88b6bcd..d6ee261 100755 --- a/docker/my-dojo/install/install-scripts.sh +++ b/docker/my-dojo/install/install-scripts.sh @@ -52,6 +52,9 @@ init_config_files() { cp ./conf/docker-tor.conf.tpl ./conf/docker-tor.conf echo "Initialized docker-tor.conf" + cp ./conf/docker-indexer.conf.tpl ./conf/docker-indexer.conf + echo "Initialized docker-indexer.conf" + # Initialize config files for nginx and the maintenance tool if [ "$COMMON_BTC_NETWORK" == "testnet" ]; then cp ./nginx/testnet.conf ./nginx/dojo.conf diff --git a/docker/my-dojo/install/upgrade-scripts.sh b/docker/my-dojo/install/upgrade-scripts.sh index 40e735a..bd84026 100755 --- a/docker/my-dojo/install/upgrade-scripts.sh +++ b/docker/my-dojo/install/upgrade-scripts.sh @@ -48,6 +48,9 @@ update_config_files() { update_config_file ./conf/docker-tor.conf ./conf/docker-tor.conf.tpl echo "Initialized docker-tor.conf" + update_config_file ./conf/docker-indexer.conf ./conf/docker-indexer.conf.tpl + echo "Initialized docker-indexer.conf" + # Initialize config files for nginx and the maintenance tool if [ "$COMMON_BTC_NETWORK" == "testnet" ]; then cp ./nginx/testnet.conf ./nginx/dojo.conf diff --git a/docker/my-dojo/node/keys.index.js b/docker/my-dojo/node/keys.index.js index 300a1a5..a6aae97 100644 --- a/docker/my-dojo/node/keys.index.js +++ b/docker/my-dojo/node/keys.index.js @@ -156,8 +156,18 @@ module.exports = { */ indexer: { // Active indexer - // Values: local_bitcoind | third_party_explorer + // Values: local_bitcoind | local_indexer | third_party_explorer active: process.env.NODE_ACTIVE_INDEXER, + // Local indexer + localIndexer: { + // IP address or hostname of the local indexer + host: process.env.INDEXER_IP, + // Port + port: parseInt(process.env.INDEXER_RPC_PORT), + // Support of batch requests by the local indexer + // Values: active | inactive + batchRequests: process.env.INDEXER_BATCH_SUPPORT + }, // Use a SOCKS5 proxy for all communications with external services // Values: null if no socks5 proxy used, otherwise the url of the socks5 proxy socks5Proxy: 'socks5h://172.28.1.4:9050', diff --git a/docker/my-dojo/overrides/indexer.install.yaml b/docker/my-dojo/overrides/indexer.install.yaml new file mode 100644 index 0000000..e130624 --- /dev/null +++ b/docker/my-dojo/overrides/indexer.install.yaml @@ -0,0 +1,27 @@ +version: "3.2" + +services: + indexer: + image: "samouraiwallet/dojo-indexer:${DOJO_INDEXER_VERSION_TAG}" + container_name: indexer + build: + context: ./indexer + env_file: + - ./.env + - ./conf/docker-common.conf + - ./conf/docker-bitcoind.conf + - ./conf/docker-indexer.conf + restart: on-failure + command: "/wait-for-it.sh tor:9050 --timeout=360 --strict -- /restart.sh" + expose: + - "50001" + volumes: + - data-indexer:/home/indexer/addrindexrs + depends_on: + - tor + networks: + dojonet: + ipv4_address: 172.28.1.6 + +volumes: + data-indexer: From 503b40af4be738c4a216dd89152938278c67299c Mon Sep 17 00:00:00 2001 From: kenshin-samourai Date: Thu, 7 Nov 2019 19:47:15 +0100 Subject: [PATCH 09/29] fix path of docker volume for local indexer --- docker/my-dojo/overrides/indexer.install.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/my-dojo/overrides/indexer.install.yaml b/docker/my-dojo/overrides/indexer.install.yaml index e130624..92635d4 100644 --- a/docker/my-dojo/overrides/indexer.install.yaml +++ b/docker/my-dojo/overrides/indexer.install.yaml @@ -16,7 +16,7 @@ services: expose: - "50001" volumes: - - data-indexer:/home/indexer/addrindexrs + - data-indexer:/home/indexer depends_on: - tor networks: From 253e1ae765139b12bbbce93cf3501aed8011d92c Mon Sep 17 00:00:00 2001 From: kenshin-samourai Date: Thu, 7 Nov 2019 22:30:56 +0100 Subject: [PATCH 10/29] add loading of docker-indexer.conf in nodejs container --- docker/my-dojo/docker-compose.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/my-dojo/docker-compose.yaml b/docker/my-dojo/docker-compose.yaml index eb2ef0e..02d1423 100644 --- a/docker/my-dojo/docker-compose.yaml +++ b/docker/my-dojo/docker-compose.yaml @@ -30,6 +30,7 @@ services: - ./conf/docker-common.conf - ./conf/docker-mysql.conf - ./conf/docker-bitcoind.conf + - ./conf/docker-indexer.conf - ./conf/docker-node.conf restart: on-failure command: "/home/node/app/wait-for-it.sh db:3306 --timeout=720 --strict -- /home/node/app/restart.sh" From 69ec5a7f2ee34de6f955cb53a6fb3c13a97d46a5 Mon Sep 17 00:00:00 2001 From: kenshin-samourai Date: Thu, 7 Nov 2019 16:05:19 +0100 Subject: [PATCH 11/29] add new local indexer docker module to dojo (experimental) the local indexer is an instance of addrindexrs (see https://github.com/Samourai-Wallet/addrindexrs) running inside dojo --- docker/my-dojo/.env | 8 + docker/my-dojo/conf/docker-indexer.conf.tpl | 42 +++++ docker/my-dojo/conf/docker-node.conf.tpl | 2 +- docker/my-dojo/dojo.sh | 20 ++ docker/my-dojo/indexer/Dockerfile | 46 +++++ docker/my-dojo/indexer/restart.sh | 22 +++ docker/my-dojo/indexer/wait-for-it.sh | 178 ++++++++++++++++++ docker/my-dojo/install/install-scripts.sh | 3 + docker/my-dojo/install/upgrade-scripts.sh | 3 + docker/my-dojo/node/keys.index.js | 12 +- docker/my-dojo/overrides/indexer.install.yaml | 27 +++ 11 files changed, 361 insertions(+), 2 deletions(-) create mode 100644 docker/my-dojo/conf/docker-indexer.conf.tpl create mode 100644 docker/my-dojo/indexer/Dockerfile create mode 100644 docker/my-dojo/indexer/restart.sh create mode 100644 docker/my-dojo/indexer/wait-for-it.sh create mode 100644 docker/my-dojo/overrides/indexer.install.yaml diff --git a/docker/my-dojo/.env b/docker/my-dojo/.env index 388973b..1f76f55 100644 --- a/docker/my-dojo/.env +++ b/docker/my-dojo/.env @@ -16,6 +16,7 @@ DOJO_BITCOIND_VERSION_TAG=1.3.0 DOJO_NODEJS_VERSION_TAG=1.3.0 DOJO_NGINX_VERSION_TAG=1.3.0 DOJO_TOR_VERSION_TAG=1.2.0 +DOJO_INDEXER_VERSION_TAG=1.0.0 ######################################### @@ -57,3 +58,10 @@ NODE_PREFIX_STATUS_PUSHTX=status NODE_TRACKER_MEMPOOL_PERIOD=10000 NODE_TRACKER_UNCONF_TXS_PERIOD=300000 + + +######################################### +# INDEXER +######################################### + +INDEXER_BATCH_SIZE=10 diff --git a/docker/my-dojo/conf/docker-indexer.conf.tpl b/docker/my-dojo/conf/docker-indexer.conf.tpl new file mode 100644 index 0000000..d7ad83c --- /dev/null +++ b/docker/my-dojo/conf/docker-indexer.conf.tpl @@ -0,0 +1,42 @@ +######################################### +# CONFIGURATION OF A LOCAL INDEXER +######################################### + +# Install and run a local indexer inside Docker +# Set this option to 'off' for using an indexer hosted outside of Docker +# or when using a different data source (local bitcoind, OXT) +# Value: on | off +INDEXER_INSTALL=off + +# IP address of the local indexer used by Dojo +# Set value to 172.28.1.6 if INDEXER_INSTALL is set to 'on' +# Type: string +INDEXER_IP=172.28.1.6 + +# Port of the RPC API +# Set value to 50001 if INDEXER_INSTALL is set to 'on' +# Type: integer +INDEXER_RPC_PORT=50001 + +# Support of batch requests by the local indexer +# Set value to inactive if INDEXER_INSTALL is set to 'on' +# Value: active | inactive +INDEXER_BATCH_SUPPORT=inactive + + +# +# EXPERT SETTINGS +# (ACTIVE IF INDEXER_INSTALL IS SET TO ON) +# + +# Number of blocks to get in one JSONRPC request from bitcoind +# Type: integer +INDEXER_BATCH_SIZE=10 + +# Total size of block txids to cache (in MB) +# Type: integer +INDEXER_BLK_TXIDS_CACHE_SIZE_MB=10 + +# Number of transactions to lookup before returning an error +# Type: integer +INDEXER_TXID_LIMIT=501 \ No newline at end of file diff --git a/docker/my-dojo/conf/docker-node.conf.tpl b/docker/my-dojo/conf/docker-node.conf.tpl index 403ed36..e7adcfa 100644 --- a/docker/my-dojo/conf/docker-node.conf.tpl +++ b/docker/my-dojo/conf/docker-node.conf.tpl @@ -21,7 +21,7 @@ NODE_ADMIN_KEY=myAdminKey NODE_JWT_SECRET=myJwtSecret # Indexer or third-party service used for imports and rescans of addresses -# Values: local_bitcoind | third_party_explorer +# Values: local_bitcoind | local_indexer | third_party_explorer NODE_ACTIVE_INDEXER=local_bitcoind # FEE TYPE USED FOR FEES ESTIMATIONS BY BITCOIND diff --git a/docker/my-dojo/dojo.sh b/docker/my-dojo/dojo.sh index d5f5620..b0e280c 100755 --- a/docker/my-dojo/dojo.sh +++ b/docker/my-dojo/dojo.sh @@ -14,6 +14,7 @@ source_file() { } # Source config files +source_file "$DIR/conf/docker-indexer.conf" source_file "$DIR/conf/docker-bitcoind.conf" source_file "$DIR/conf/docker-common.conf" source_file "$DIR/.env" @@ -33,6 +34,10 @@ select_yaml_files() { fi fi + if [ "$INDEXER_INSTALL" == "on" ]; then + yamlFiles="$yamlFiles -f $DIR/overrides/indexer.install.yaml" + fi + # Return yamlFiles echo "$yamlFiles" } @@ -114,6 +119,7 @@ uninstall() { docker image rm samouraiwallet/dojo-nodejs:"$DOJO_NODEJS_VERSION_TAG" docker image rm samouraiwallet/dojo-nginx:"$DOJO_NGINX_VERSION_TAG" docker image rm samouraiwallet/dojo-tor:"$DOJO_TOR_VERSION_TAG" + docker image rm samouraiwallet/dojo-indexer:"$DOJO_INDEXER_VERSION_TAG" docker volume prune } @@ -136,6 +142,7 @@ clean() { del_images_for samouraiwallet/dojo-nodejs "$DOJO_NODEJS_VERSION_TAG" del_images_for samouraiwallet/dojo-nginx "$DOJO_NGINX_VERSION_TAG" del_images_for samouraiwallet/dojo-tor "$DOJO_TOR_VERSION_TAG" + del_images_for samouraiwallet/dojo-indexer "$DOJO_INDEXER_VERSION_TAG" } # Upgrade @@ -194,6 +201,7 @@ logs_node() { logs() { source_file "$DIR/conf/docker-bitcoind.conf" + source_file "$DIR/conf/docker-indexer.conf" source_file "$DIR/conf/docker-common.conf" case $1 in @@ -212,6 +220,14 @@ logs() { echo -e "Command not supported for your setup.\nCause: Your Dojo is using an external bitcoind" fi ;; + indexer ) + if [ "$INDEXER_INSTALL" == "on" ]; then + yamlFiles=$(select_yaml_files) + eval "docker-compose $yamlFiles logs --tail=50 --follow indexer" + else + echo -e "Command not supported for your setup.\nCause: Your Dojo is not using an internal indexer" + fi + ;; tor ) docker-compose logs --tail=50 --follow tor ;; @@ -224,6 +240,9 @@ logs() { if [ "$BITCOIND_INSTALL" == "on" ]; then services="$services bitcoind" fi + if [ "$INDEXER_INSTALL" == "on" ]; then + services="$services indexer" + fi eval "docker-compose $yamlFiles logs --tail=0 --follow $services" ;; esac @@ -251,6 +270,7 @@ help() { echo " dojo.sh logs bitcoind : display the logs of bitcoind" echo " dojo.sh logs db : display the logs of the MySQL database" echo " dojo.sh logs tor : display the logs of tor" + echo " dojo.sh logs indexer : display the logs of the internal indexer" echo " dojo.sh logs api : display the logs of the REST API (nodejs)" echo " dojo.sh logs tracker : display the logs of the Tracker (nodejs)" echo " dojo.sh logs pushtx : display the logs of the pushTx API (nodejs)" diff --git a/docker/my-dojo/indexer/Dockerfile b/docker/my-dojo/indexer/Dockerfile new file mode 100644 index 0000000..259f827 --- /dev/null +++ b/docker/my-dojo/indexer/Dockerfile @@ -0,0 +1,46 @@ +FROM rust:1.37.0-slim + +ENV INDEXER_HOME /home/indexer +ENV INDEXER_VERSION 0..1.0 +ENV INDEXER_URL https://github.com/Samourai-Wallet/addrindexrs.git + +RUN apt-get update && \ + apt-get install -y clang cmake git && \ + apt-get install -y libsnappy-dev + +# Create group and user indexer +RUN addgroup --system -gid 1109 indexer && \ + adduser --system --ingroup indexer -uid 1106 indexer + +# Create data directory +RUN mkdir "$INDEXER_HOME/addrindexrs" && \ + chown -h indexer:indexer "$INDEXER_HOME/addrindexrs" + +# Copy restart script +COPY ./restart.sh /restart.sh +RUN chown indexer:indexer /restart.sh && \ + chmod 777 /restart.sh + +# Copy wait-for-it script +COPY ./wait-for-it.sh /wait-for-it.sh +RUN chown indexer:indexer /wait-for-it.sh && \ + chmod u+x /wait-for-it.sh && \ + chmod g+x /wait-for-it.sh + +USER indexer + +# Install addrindexrs +RUN cd "$INDEXER_HOME" && \ + git clone "$INDEXER_URL" "$INDEXER_HOME/addrindexrs" && \ + cd addrindexrs && \ + git checkout "master" +# TODO uncomment +# git checkout "tags/v$INDEXER_VERSION" + +RUN cd "$INDEXER_HOME/addrindexrs" && \ + cargo build --release && \ + cargo install --path . + +EXPOSE 50001 + +STOPSIGNAL SIGINT diff --git a/docker/my-dojo/indexer/restart.sh b/docker/my-dojo/indexer/restart.sh new file mode 100644 index 0000000..0db33b1 --- /dev/null +++ b/docker/my-dojo/indexer/restart.sh @@ -0,0 +1,22 @@ +#!/bin/bash +set -e + +indexer_options=( + -vvvv + --index-batch-size="$INDEXER_BATCH_SIZE" + --jsonrpc-import + --db-dir="/home/indexer/db" + --indexer-rpc-addr="172.28.1.6:50001" + --daemon-rpc-addr="$BITCOIND_IP:$BITCOIND_RPC_PORT" + --cookie="$BITCOIND_RPC_USER:$BITCOIND_RPC_PASSWORD" + --txid-limit="$INDEXER_TXID_LIMIT" + --blocktxids-cache-size-mb="$INDEXER_BLK_TXIDS_CACHE_SIZE_MB" +) + +if [ "$COMMON_BTC_NETWORK" == "testnet" ]; then + bitcoind_options+=(--network="testnet") +else + bitcoind_options+=(--network="mainnet") +fi + +addrindexrs "${indexer_options[@]}" diff --git a/docker/my-dojo/indexer/wait-for-it.sh b/docker/my-dojo/indexer/wait-for-it.sh new file mode 100644 index 0000000..071c2be --- /dev/null +++ b/docker/my-dojo/indexer/wait-for-it.sh @@ -0,0 +1,178 @@ +#!/usr/bin/env bash +# Use this script to test if a given TCP host/port are available + +WAITFORIT_cmdname=${0##*/} + +echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } + +usage() +{ + cat << USAGE >&2 +Usage: + $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args] + -h HOST | --host=HOST Host or IP under test + -p PORT | --port=PORT TCP port under test + Alternatively, you specify the host and port as host:port + -s | --strict Only execute subcommand if the test succeeds + -q | --quiet Don't output any status messages + -t TIMEOUT | --timeout=TIMEOUT + Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit 1 +} + +wait_for() +{ + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + else + echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout" + fi + WAITFORIT_start_ts=$(date +%s) + while : + do + if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then + nc -z $WAITFORIT_HOST $WAITFORIT_PORT + WAITFORIT_result=$? + else + (echo > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1 + WAITFORIT_result=$? + fi + if [[ $WAITFORIT_result -eq 0 ]]; then + WAITFORIT_end_ts=$(date +%s) + echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds" + break + fi + sleep 1 + done + return $WAITFORIT_result +} + +wait_for_wrapper() +{ + # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 + if [[ $WAITFORIT_QUIET -eq 1 ]]; then + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + else + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + fi + WAITFORIT_PID=$! + trap "kill -INT -$WAITFORIT_PID" INT + wait $WAITFORIT_PID + WAITFORIT_RESULT=$? + if [[ $WAITFORIT_RESULT -ne 0 ]]; then + echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + fi + return $WAITFORIT_RESULT +} + +# process arguments +while [[ $# -gt 0 ]] +do + case "$1" in + *:* ) + WAITFORIT_hostport=(${1//:/ }) + WAITFORIT_HOST=${WAITFORIT_hostport[0]} + WAITFORIT_PORT=${WAITFORIT_hostport[1]} + shift 1 + ;; + --child) + WAITFORIT_CHILD=1 + shift 1 + ;; + -q | --quiet) + WAITFORIT_QUIET=1 + shift 1 + ;; + -s | --strict) + WAITFORIT_STRICT=1 + shift 1 + ;; + -h) + WAITFORIT_HOST="$2" + if [[ $WAITFORIT_HOST == "" ]]; then break; fi + shift 2 + ;; + --host=*) + WAITFORIT_HOST="${1#*=}" + shift 1 + ;; + -p) + WAITFORIT_PORT="$2" + if [[ $WAITFORIT_PORT == "" ]]; then break; fi + shift 2 + ;; + --port=*) + WAITFORIT_PORT="${1#*=}" + shift 1 + ;; + -t) + WAITFORIT_TIMEOUT="$2" + if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi + shift 2 + ;; + --timeout=*) + WAITFORIT_TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + WAITFORIT_CLI=("$@") + break + ;; + --help) + usage + ;; + *) + echoerr "Unknown argument: $1" + usage + ;; + esac +done + +if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then + echoerr "Error: you need to provide a host and port to test." + usage +fi + +WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15} +WAITFORIT_STRICT=${WAITFORIT_STRICT:-0} +WAITFORIT_CHILD=${WAITFORIT_CHILD:-0} +WAITFORIT_QUIET=${WAITFORIT_QUIET:-0} + +# check to see if timeout is from busybox? +WAITFORIT_TIMEOUT_PATH=$(type -p timeout) +WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH) +if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then + WAITFORIT_ISBUSY=1 + WAITFORIT_BUSYTIMEFLAG="-t" + +else + WAITFORIT_ISBUSY=0 + WAITFORIT_BUSYTIMEFLAG="" +fi + +if [[ $WAITFORIT_CHILD -gt 0 ]]; then + wait_for + WAITFORIT_RESULT=$? + exit $WAITFORIT_RESULT +else + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + wait_for_wrapper + WAITFORIT_RESULT=$? + else + wait_for + WAITFORIT_RESULT=$? + fi +fi + +if [[ $WAITFORIT_CLI != "" ]]; then + if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then + echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess" + exit $WAITFORIT_RESULT + fi + exec "${WAITFORIT_CLI[@]}" +else + exit $WAITFORIT_RESULT +fi diff --git a/docker/my-dojo/install/install-scripts.sh b/docker/my-dojo/install/install-scripts.sh index 88b6bcd..d6ee261 100755 --- a/docker/my-dojo/install/install-scripts.sh +++ b/docker/my-dojo/install/install-scripts.sh @@ -52,6 +52,9 @@ init_config_files() { cp ./conf/docker-tor.conf.tpl ./conf/docker-tor.conf echo "Initialized docker-tor.conf" + cp ./conf/docker-indexer.conf.tpl ./conf/docker-indexer.conf + echo "Initialized docker-indexer.conf" + # Initialize config files for nginx and the maintenance tool if [ "$COMMON_BTC_NETWORK" == "testnet" ]; then cp ./nginx/testnet.conf ./nginx/dojo.conf diff --git a/docker/my-dojo/install/upgrade-scripts.sh b/docker/my-dojo/install/upgrade-scripts.sh index 40e735a..bd84026 100755 --- a/docker/my-dojo/install/upgrade-scripts.sh +++ b/docker/my-dojo/install/upgrade-scripts.sh @@ -48,6 +48,9 @@ update_config_files() { update_config_file ./conf/docker-tor.conf ./conf/docker-tor.conf.tpl echo "Initialized docker-tor.conf" + update_config_file ./conf/docker-indexer.conf ./conf/docker-indexer.conf.tpl + echo "Initialized docker-indexer.conf" + # Initialize config files for nginx and the maintenance tool if [ "$COMMON_BTC_NETWORK" == "testnet" ]; then cp ./nginx/testnet.conf ./nginx/dojo.conf diff --git a/docker/my-dojo/node/keys.index.js b/docker/my-dojo/node/keys.index.js index 300a1a5..a6aae97 100644 --- a/docker/my-dojo/node/keys.index.js +++ b/docker/my-dojo/node/keys.index.js @@ -156,8 +156,18 @@ module.exports = { */ indexer: { // Active indexer - // Values: local_bitcoind | third_party_explorer + // Values: local_bitcoind | local_indexer | third_party_explorer active: process.env.NODE_ACTIVE_INDEXER, + // Local indexer + localIndexer: { + // IP address or hostname of the local indexer + host: process.env.INDEXER_IP, + // Port + port: parseInt(process.env.INDEXER_RPC_PORT), + // Support of batch requests by the local indexer + // Values: active | inactive + batchRequests: process.env.INDEXER_BATCH_SUPPORT + }, // Use a SOCKS5 proxy for all communications with external services // Values: null if no socks5 proxy used, otherwise the url of the socks5 proxy socks5Proxy: 'socks5h://172.28.1.4:9050', diff --git a/docker/my-dojo/overrides/indexer.install.yaml b/docker/my-dojo/overrides/indexer.install.yaml new file mode 100644 index 0000000..e130624 --- /dev/null +++ b/docker/my-dojo/overrides/indexer.install.yaml @@ -0,0 +1,27 @@ +version: "3.2" + +services: + indexer: + image: "samouraiwallet/dojo-indexer:${DOJO_INDEXER_VERSION_TAG}" + container_name: indexer + build: + context: ./indexer + env_file: + - ./.env + - ./conf/docker-common.conf + - ./conf/docker-bitcoind.conf + - ./conf/docker-indexer.conf + restart: on-failure + command: "/wait-for-it.sh tor:9050 --timeout=360 --strict -- /restart.sh" + expose: + - "50001" + volumes: + - data-indexer:/home/indexer/addrindexrs + depends_on: + - tor + networks: + dojonet: + ipv4_address: 172.28.1.6 + +volumes: + data-indexer: From f1f86429eb241d5a32c918cc7638d2e5856f3166 Mon Sep 17 00:00:00 2001 From: kenshin-samourai Date: Thu, 7 Nov 2019 19:47:15 +0100 Subject: [PATCH 12/29] fix path of docker volume for local indexer --- docker/my-dojo/overrides/indexer.install.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/my-dojo/overrides/indexer.install.yaml b/docker/my-dojo/overrides/indexer.install.yaml index e130624..92635d4 100644 --- a/docker/my-dojo/overrides/indexer.install.yaml +++ b/docker/my-dojo/overrides/indexer.install.yaml @@ -16,7 +16,7 @@ services: expose: - "50001" volumes: - - data-indexer:/home/indexer/addrindexrs + - data-indexer:/home/indexer depends_on: - tor networks: From 686f7320eacaa44de867cb129245be34209f9602 Mon Sep 17 00:00:00 2001 From: kenshin-samourai Date: Thu, 7 Nov 2019 22:30:56 +0100 Subject: [PATCH 13/29] add loading of docker-indexer.conf in nodejs container --- docker/my-dojo/docker-compose.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/my-dojo/docker-compose.yaml b/docker/my-dojo/docker-compose.yaml index eb2ef0e..02d1423 100644 --- a/docker/my-dojo/docker-compose.yaml +++ b/docker/my-dojo/docker-compose.yaml @@ -30,6 +30,7 @@ services: - ./conf/docker-common.conf - ./conf/docker-mysql.conf - ./conf/docker-bitcoind.conf + - ./conf/docker-indexer.conf - ./conf/docker-node.conf restart: on-failure command: "/home/node/app/wait-for-it.sh db:3306 --timeout=720 --strict -- /home/node/app/restart.sh" From 3f9c2195a99c8f8f4851e624ff508772b0ac6964 Mon Sep 17 00:00:00 2001 From: kenshin-samourai Date: Thu, 7 Nov 2019 16:05:19 +0100 Subject: [PATCH 14/29] add new local indexer docker module to dojo (experimental) the local indexer is an instance of addrindexrs (see https://github.com/Samourai-Wallet/addrindexrs) running inside dojo --- docker/my-dojo/.env | 8 + docker/my-dojo/conf/docker-indexer.conf.tpl | 42 +++++ docker/my-dojo/conf/docker-node.conf.tpl | 2 +- docker/my-dojo/dojo.sh | 20 ++ docker/my-dojo/indexer/Dockerfile | 46 +++++ docker/my-dojo/indexer/restart.sh | 22 +++ docker/my-dojo/indexer/wait-for-it.sh | 178 ++++++++++++++++++ docker/my-dojo/install/install-scripts.sh | 5 +- docker/my-dojo/install/upgrade-scripts.sh | 3 + docker/my-dojo/node/keys.index.js | 12 +- docker/my-dojo/overrides/indexer.install.yaml | 27 +++ 11 files changed, 362 insertions(+), 3 deletions(-) create mode 100644 docker/my-dojo/conf/docker-indexer.conf.tpl create mode 100644 docker/my-dojo/indexer/Dockerfile create mode 100644 docker/my-dojo/indexer/restart.sh create mode 100644 docker/my-dojo/indexer/wait-for-it.sh create mode 100644 docker/my-dojo/overrides/indexer.install.yaml diff --git a/docker/my-dojo/.env b/docker/my-dojo/.env index a301808..e144426 100644 --- a/docker/my-dojo/.env +++ b/docker/my-dojo/.env @@ -17,6 +17,7 @@ DOJO_NODEJS_VERSION_TAG=1.4.0 DOJO_NGINX_VERSION_TAG=1.4.0 DOJO_TOR_VERSION_TAG=1.3.0 DOJO_EXPLORER_VERSION_TAG=1.0.0 +DOJO_INDEXER_VERSION_TAG=1.0.0 ######################################### @@ -58,3 +59,10 @@ NODE_PREFIX_STATUS_PUSHTX=status NODE_TRACKER_MEMPOOL_PERIOD=10000 NODE_TRACKER_UNCONF_TXS_PERIOD=300000 + + +######################################### +# INDEXER +######################################### + +INDEXER_BATCH_SIZE=10 diff --git a/docker/my-dojo/conf/docker-indexer.conf.tpl b/docker/my-dojo/conf/docker-indexer.conf.tpl new file mode 100644 index 0000000..d7ad83c --- /dev/null +++ b/docker/my-dojo/conf/docker-indexer.conf.tpl @@ -0,0 +1,42 @@ +######################################### +# CONFIGURATION OF A LOCAL INDEXER +######################################### + +# Install and run a local indexer inside Docker +# Set this option to 'off' for using an indexer hosted outside of Docker +# or when using a different data source (local bitcoind, OXT) +# Value: on | off +INDEXER_INSTALL=off + +# IP address of the local indexer used by Dojo +# Set value to 172.28.1.6 if INDEXER_INSTALL is set to 'on' +# Type: string +INDEXER_IP=172.28.1.6 + +# Port of the RPC API +# Set value to 50001 if INDEXER_INSTALL is set to 'on' +# Type: integer +INDEXER_RPC_PORT=50001 + +# Support of batch requests by the local indexer +# Set value to inactive if INDEXER_INSTALL is set to 'on' +# Value: active | inactive +INDEXER_BATCH_SUPPORT=inactive + + +# +# EXPERT SETTINGS +# (ACTIVE IF INDEXER_INSTALL IS SET TO ON) +# + +# Number of blocks to get in one JSONRPC request from bitcoind +# Type: integer +INDEXER_BATCH_SIZE=10 + +# Total size of block txids to cache (in MB) +# Type: integer +INDEXER_BLK_TXIDS_CACHE_SIZE_MB=10 + +# Number of transactions to lookup before returning an error +# Type: integer +INDEXER_TXID_LIMIT=501 \ No newline at end of file diff --git a/docker/my-dojo/conf/docker-node.conf.tpl b/docker/my-dojo/conf/docker-node.conf.tpl index 403ed36..e7adcfa 100644 --- a/docker/my-dojo/conf/docker-node.conf.tpl +++ b/docker/my-dojo/conf/docker-node.conf.tpl @@ -21,7 +21,7 @@ NODE_ADMIN_KEY=myAdminKey NODE_JWT_SECRET=myJwtSecret # Indexer or third-party service used for imports and rescans of addresses -# Values: local_bitcoind | third_party_explorer +# Values: local_bitcoind | local_indexer | third_party_explorer NODE_ACTIVE_INDEXER=local_bitcoind # FEE TYPE USED FOR FEES ESTIMATIONS BY BITCOIND diff --git a/docker/my-dojo/dojo.sh b/docker/my-dojo/dojo.sh index 089cd93..3352e56 100755 --- a/docker/my-dojo/dojo.sh +++ b/docker/my-dojo/dojo.sh @@ -14,6 +14,7 @@ source_file() { } # Source config files +source_file "$DIR/conf/docker-indexer.conf" source_file "$DIR/conf/docker-bitcoind.conf" source_file "$DIR/conf/docker-explorer.conf" source_file "$DIR/conf/docker-common.conf" @@ -38,6 +39,10 @@ select_yaml_files() { yamlFiles="$yamlFiles -f $DIR/overrides/explorer.install.yaml" fi + if [ "$INDEXER_INSTALL" == "on" ]; then + yamlFiles="$yamlFiles -f $DIR/overrides/indexer.install.yaml" + fi + # Return yamlFiles echo "$yamlFiles" } @@ -155,6 +160,7 @@ uninstall() { docker image rm samouraiwallet/dojo-nodejs:"$DOJO_NODEJS_VERSION_TAG" docker image rm samouraiwallet/dojo-nginx:"$DOJO_NGINX_VERSION_TAG" docker image rm samouraiwallet/dojo-tor:"$DOJO_TOR_VERSION_TAG" + docker image rm samouraiwallet/dojo-indexer:"$DOJO_INDEXER_VERSION_TAG" docker volume prune } @@ -178,6 +184,7 @@ clean() { del_images_for samouraiwallet/dojo-nodejs "$DOJO_NODEJS_VERSION_TAG" del_images_for samouraiwallet/dojo-nginx "$DOJO_NGINX_VERSION_TAG" del_images_for samouraiwallet/dojo-tor "$DOJO_TOR_VERSION_TAG" + del_images_for samouraiwallet/dojo-indexer "$DOJO_INDEXER_VERSION_TAG" } # Upgrade @@ -250,6 +257,7 @@ logs_explorer() { logs() { source_file "$DIR/conf/docker-bitcoind.conf" + source_file "$DIR/conf/docker-indexer.conf" source_file "$DIR/conf/docker-common.conf" case $1 in @@ -268,6 +276,14 @@ logs() { echo -e "Command not supported for your setup.\nCause: Your Dojo is using an external bitcoind" fi ;; + indexer ) + if [ "$INDEXER_INSTALL" == "on" ]; then + yamlFiles=$(select_yaml_files) + eval "docker-compose $yamlFiles logs --tail=50 --follow indexer" + else + echo -e "Command not supported for your setup.\nCause: Your Dojo is not using an internal indexer" + fi + ;; tor ) docker-compose logs --tail=50 --follow tor ;; @@ -286,6 +302,9 @@ logs() { if [ "$EXPLORER_INSTALL" == "on" ]; then services="$services explorer" fi + if [ "$INDEXER_INSTALL" == "on" ]; then + services="$services indexer" + fi eval "docker-compose $yamlFiles logs --tail=0 --follow $services" ;; esac @@ -313,6 +332,7 @@ help() { echo " dojo.sh logs bitcoind : display the logs of bitcoind" echo " dojo.sh logs db : display the logs of the MySQL database" echo " dojo.sh logs tor : display the logs of tor" + echo " dojo.sh logs indexer : display the logs of the internal indexer" echo " dojo.sh logs api : display the logs of the REST API (nodejs)" echo " dojo.sh logs tracker : display the logs of the Tracker (nodejs)" echo " dojo.sh logs pushtx : display the logs of the pushTx API (nodejs)" diff --git a/docker/my-dojo/indexer/Dockerfile b/docker/my-dojo/indexer/Dockerfile new file mode 100644 index 0000000..259f827 --- /dev/null +++ b/docker/my-dojo/indexer/Dockerfile @@ -0,0 +1,46 @@ +FROM rust:1.37.0-slim + +ENV INDEXER_HOME /home/indexer +ENV INDEXER_VERSION 0..1.0 +ENV INDEXER_URL https://github.com/Samourai-Wallet/addrindexrs.git + +RUN apt-get update && \ + apt-get install -y clang cmake git && \ + apt-get install -y libsnappy-dev + +# Create group and user indexer +RUN addgroup --system -gid 1109 indexer && \ + adduser --system --ingroup indexer -uid 1106 indexer + +# Create data directory +RUN mkdir "$INDEXER_HOME/addrindexrs" && \ + chown -h indexer:indexer "$INDEXER_HOME/addrindexrs" + +# Copy restart script +COPY ./restart.sh /restart.sh +RUN chown indexer:indexer /restart.sh && \ + chmod 777 /restart.sh + +# Copy wait-for-it script +COPY ./wait-for-it.sh /wait-for-it.sh +RUN chown indexer:indexer /wait-for-it.sh && \ + chmod u+x /wait-for-it.sh && \ + chmod g+x /wait-for-it.sh + +USER indexer + +# Install addrindexrs +RUN cd "$INDEXER_HOME" && \ + git clone "$INDEXER_URL" "$INDEXER_HOME/addrindexrs" && \ + cd addrindexrs && \ + git checkout "master" +# TODO uncomment +# git checkout "tags/v$INDEXER_VERSION" + +RUN cd "$INDEXER_HOME/addrindexrs" && \ + cargo build --release && \ + cargo install --path . + +EXPOSE 50001 + +STOPSIGNAL SIGINT diff --git a/docker/my-dojo/indexer/restart.sh b/docker/my-dojo/indexer/restart.sh new file mode 100644 index 0000000..0db33b1 --- /dev/null +++ b/docker/my-dojo/indexer/restart.sh @@ -0,0 +1,22 @@ +#!/bin/bash +set -e + +indexer_options=( + -vvvv + --index-batch-size="$INDEXER_BATCH_SIZE" + --jsonrpc-import + --db-dir="/home/indexer/db" + --indexer-rpc-addr="172.28.1.6:50001" + --daemon-rpc-addr="$BITCOIND_IP:$BITCOIND_RPC_PORT" + --cookie="$BITCOIND_RPC_USER:$BITCOIND_RPC_PASSWORD" + --txid-limit="$INDEXER_TXID_LIMIT" + --blocktxids-cache-size-mb="$INDEXER_BLK_TXIDS_CACHE_SIZE_MB" +) + +if [ "$COMMON_BTC_NETWORK" == "testnet" ]; then + bitcoind_options+=(--network="testnet") +else + bitcoind_options+=(--network="mainnet") +fi + +addrindexrs "${indexer_options[@]}" diff --git a/docker/my-dojo/indexer/wait-for-it.sh b/docker/my-dojo/indexer/wait-for-it.sh new file mode 100644 index 0000000..071c2be --- /dev/null +++ b/docker/my-dojo/indexer/wait-for-it.sh @@ -0,0 +1,178 @@ +#!/usr/bin/env bash +# Use this script to test if a given TCP host/port are available + +WAITFORIT_cmdname=${0##*/} + +echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } + +usage() +{ + cat << USAGE >&2 +Usage: + $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args] + -h HOST | --host=HOST Host or IP under test + -p PORT | --port=PORT TCP port under test + Alternatively, you specify the host and port as host:port + -s | --strict Only execute subcommand if the test succeeds + -q | --quiet Don't output any status messages + -t TIMEOUT | --timeout=TIMEOUT + Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit 1 +} + +wait_for() +{ + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + else + echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout" + fi + WAITFORIT_start_ts=$(date +%s) + while : + do + if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then + nc -z $WAITFORIT_HOST $WAITFORIT_PORT + WAITFORIT_result=$? + else + (echo > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1 + WAITFORIT_result=$? + fi + if [[ $WAITFORIT_result -eq 0 ]]; then + WAITFORIT_end_ts=$(date +%s) + echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds" + break + fi + sleep 1 + done + return $WAITFORIT_result +} + +wait_for_wrapper() +{ + # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 + if [[ $WAITFORIT_QUIET -eq 1 ]]; then + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + else + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + fi + WAITFORIT_PID=$! + trap "kill -INT -$WAITFORIT_PID" INT + wait $WAITFORIT_PID + WAITFORIT_RESULT=$? + if [[ $WAITFORIT_RESULT -ne 0 ]]; then + echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + fi + return $WAITFORIT_RESULT +} + +# process arguments +while [[ $# -gt 0 ]] +do + case "$1" in + *:* ) + WAITFORIT_hostport=(${1//:/ }) + WAITFORIT_HOST=${WAITFORIT_hostport[0]} + WAITFORIT_PORT=${WAITFORIT_hostport[1]} + shift 1 + ;; + --child) + WAITFORIT_CHILD=1 + shift 1 + ;; + -q | --quiet) + WAITFORIT_QUIET=1 + shift 1 + ;; + -s | --strict) + WAITFORIT_STRICT=1 + shift 1 + ;; + -h) + WAITFORIT_HOST="$2" + if [[ $WAITFORIT_HOST == "" ]]; then break; fi + shift 2 + ;; + --host=*) + WAITFORIT_HOST="${1#*=}" + shift 1 + ;; + -p) + WAITFORIT_PORT="$2" + if [[ $WAITFORIT_PORT == "" ]]; then break; fi + shift 2 + ;; + --port=*) + WAITFORIT_PORT="${1#*=}" + shift 1 + ;; + -t) + WAITFORIT_TIMEOUT="$2" + if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi + shift 2 + ;; + --timeout=*) + WAITFORIT_TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + WAITFORIT_CLI=("$@") + break + ;; + --help) + usage + ;; + *) + echoerr "Unknown argument: $1" + usage + ;; + esac +done + +if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then + echoerr "Error: you need to provide a host and port to test." + usage +fi + +WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15} +WAITFORIT_STRICT=${WAITFORIT_STRICT:-0} +WAITFORIT_CHILD=${WAITFORIT_CHILD:-0} +WAITFORIT_QUIET=${WAITFORIT_QUIET:-0} + +# check to see if timeout is from busybox? +WAITFORIT_TIMEOUT_PATH=$(type -p timeout) +WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH) +if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then + WAITFORIT_ISBUSY=1 + WAITFORIT_BUSYTIMEFLAG="-t" + +else + WAITFORIT_ISBUSY=0 + WAITFORIT_BUSYTIMEFLAG="" +fi + +if [[ $WAITFORIT_CHILD -gt 0 ]]; then + wait_for + WAITFORIT_RESULT=$? + exit $WAITFORIT_RESULT +else + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + wait_for_wrapper + WAITFORIT_RESULT=$? + else + wait_for + WAITFORIT_RESULT=$? + fi +fi + +if [[ $WAITFORIT_CLI != "" ]]; then + if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then + echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess" + exit $WAITFORIT_RESULT + fi + exec "${WAITFORIT_CLI[@]}" +else + exit $WAITFORIT_RESULT +fi diff --git a/docker/my-dojo/install/install-scripts.sh b/docker/my-dojo/install/install-scripts.sh index 79b2142..3db46a3 100755 --- a/docker/my-dojo/install/install-scripts.sh +++ b/docker/my-dojo/install/install-scripts.sh @@ -61,7 +61,9 @@ init_config_files() { cp ./conf/docker-tor.conf.tpl ./conf/docker-tor.conf echo "Initialized docker-tor.conf" - # Initialize config files for nginx and the maintenance tool + cp ./conf/docker-indexer.conf.tpl ./conf/docker-indexer.conf + echo "Initialized docker-indexer.conf" + if [ "$EXPLORER_INSTALL" == "on" ]; then cp ./nginx/explorer.conf ./nginx/dojo-explorer.conf else @@ -69,6 +71,7 @@ init_config_files() { fi echo "Initialized dojo-explorer.conf (nginx)" + # Initialize config files for nginx and the maintenance tool if [ "$COMMON_BTC_NETWORK" == "testnet" ]; then cp ./nginx/testnet.conf ./nginx/dojo.conf echo "Initialized dojo.conf (nginx)" diff --git a/docker/my-dojo/install/upgrade-scripts.sh b/docker/my-dojo/install/upgrade-scripts.sh index 68554aa..f59dbcb 100755 --- a/docker/my-dojo/install/upgrade-scripts.sh +++ b/docker/my-dojo/install/upgrade-scripts.sh @@ -57,6 +57,9 @@ update_config_files() { update_config_file ./conf/docker-tor.conf ./conf/docker-tor.conf.tpl echo "Initialized docker-tor.conf" + update_config_file ./conf/docker-indexer.conf ./conf/docker-indexer.conf.tpl + echo "Initialized docker-indexer.conf" + # Initialize config files for nginx and the maintenance tool if [ "$EXPLORER_INSTALL" == "on" ]; then cp ./nginx/explorer.conf ./nginx/dojo-explorer.conf diff --git a/docker/my-dojo/node/keys.index.js b/docker/my-dojo/node/keys.index.js index 300a1a5..a6aae97 100644 --- a/docker/my-dojo/node/keys.index.js +++ b/docker/my-dojo/node/keys.index.js @@ -156,8 +156,18 @@ module.exports = { */ indexer: { // Active indexer - // Values: local_bitcoind | third_party_explorer + // Values: local_bitcoind | local_indexer | third_party_explorer active: process.env.NODE_ACTIVE_INDEXER, + // Local indexer + localIndexer: { + // IP address or hostname of the local indexer + host: process.env.INDEXER_IP, + // Port + port: parseInt(process.env.INDEXER_RPC_PORT), + // Support of batch requests by the local indexer + // Values: active | inactive + batchRequests: process.env.INDEXER_BATCH_SUPPORT + }, // Use a SOCKS5 proxy for all communications with external services // Values: null if no socks5 proxy used, otherwise the url of the socks5 proxy socks5Proxy: 'socks5h://172.28.1.4:9050', diff --git a/docker/my-dojo/overrides/indexer.install.yaml b/docker/my-dojo/overrides/indexer.install.yaml new file mode 100644 index 0000000..e130624 --- /dev/null +++ b/docker/my-dojo/overrides/indexer.install.yaml @@ -0,0 +1,27 @@ +version: "3.2" + +services: + indexer: + image: "samouraiwallet/dojo-indexer:${DOJO_INDEXER_VERSION_TAG}" + container_name: indexer + build: + context: ./indexer + env_file: + - ./.env + - ./conf/docker-common.conf + - ./conf/docker-bitcoind.conf + - ./conf/docker-indexer.conf + restart: on-failure + command: "/wait-for-it.sh tor:9050 --timeout=360 --strict -- /restart.sh" + expose: + - "50001" + volumes: + - data-indexer:/home/indexer/addrindexrs + depends_on: + - tor + networks: + dojonet: + ipv4_address: 172.28.1.6 + +volumes: + data-indexer: From 604674e2a6c33c25ad2b097dfb376c3d31e80c59 Mon Sep 17 00:00:00 2001 From: kenshin-samourai Date: Thu, 7 Nov 2019 19:47:15 +0100 Subject: [PATCH 15/29] fix path of docker volume for local indexer --- docker/my-dojo/overrides/indexer.install.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/my-dojo/overrides/indexer.install.yaml b/docker/my-dojo/overrides/indexer.install.yaml index e130624..92635d4 100644 --- a/docker/my-dojo/overrides/indexer.install.yaml +++ b/docker/my-dojo/overrides/indexer.install.yaml @@ -16,7 +16,7 @@ services: expose: - "50001" volumes: - - data-indexer:/home/indexer/addrindexrs + - data-indexer:/home/indexer depends_on: - tor networks: From 9c42f0857f517054738f46b4de618a02ed4769e6 Mon Sep 17 00:00:00 2001 From: kenshin-samourai Date: Thu, 7 Nov 2019 22:30:56 +0100 Subject: [PATCH 16/29] add loading of docker-indexer.conf in nodejs container --- docker/my-dojo/docker-compose.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/my-dojo/docker-compose.yaml b/docker/my-dojo/docker-compose.yaml index b4e82f9..1c09374 100644 --- a/docker/my-dojo/docker-compose.yaml +++ b/docker/my-dojo/docker-compose.yaml @@ -31,6 +31,7 @@ services: - ./conf/docker-mysql.conf - ./conf/docker-bitcoind.conf - ./conf/docker-explorer.conf + - ./conf/docker-indexer.conf - ./conf/docker-node.conf restart: always command: "/home/node/app/wait-for-it.sh db:3306 --timeout=720 --strict -- /home/node/app/restart.sh" From 3bfc93c9d8f50301b88b75edc2fa82a3532518a9 Mon Sep 17 00:00:00 2001 From: kenshin-samourai Date: Tue, 31 Dec 2019 16:03:00 +0100 Subject: [PATCH 17/29] connect the explorer and the indexer --- docker/my-dojo/explorer/restart.sh | 6 ++++++ docker/my-dojo/overrides/explorer.install.yaml | 2 ++ 2 files changed, 8 insertions(+) diff --git a/docker/my-dojo/explorer/restart.sh b/docker/my-dojo/explorer/restart.sh index 4da82fa..985debc 100644 --- a/docker/my-dojo/explorer/restart.sh +++ b/docker/my-dojo/explorer/restart.sh @@ -18,4 +18,10 @@ explorer_options=( # Blacklist all functions provided by the RPC API explorer_options+=(--rpc-blacklist "addnode,analyzepsbt,clearbanned,combinepsbt,combinerawtransaction,converttopsbt,createmultisig,createpsbt,createrawtransaction,decodepsbt,decoderawtransaction,decodescript,deriveaddresses,disconnectnode,echo,echojson,estimaterawfee,estimatesmartfee,finalizepsbt,generatetoaddress,generatetodescriptor,getaddednodeinfo,getbestblockhash,getblock,getblockchaininfo,getblockcount,getblockfilter,getblockhash,getblockheader,getblockstats,getblocktemplate,getchaintips,getchaintxstats,getconnectioncount,getdescriptorinfo,getdifficulty,getmemoryinfo,getmempoolancestors,getmempooldescendants,getmempoolentry,getmempoolinfo,getmininginfo,getnettotals,getnetworkhashps,getnetworkinfo,getnodeaddresses,getpeerinfo,getrawmempool,getrawtransaction,getrpcinfo,gettxout,gettxoutproof,gettxoutsetinfo,help,invalidateblock,joinpsbts,listbanned,logging,ping,preciousblock,prioritisetransaction,pruneblockchain,reconsiderblock,savemempool,scantxoutset,sendrawtransaction,setban,setmocktime,setnetworkactive,signmessagewithprivkey,signrawtransactionwithkey,stop,submitblock,submitheader,syncwithvalidationinterfacequeue,testmempoolaccept,uptime,utxoupdatepsbt,validateaddress,verifychain,verifymessage,verifytxoutproof,waitforblock,waitforblockheight,waitfornewblock") +# Use the local indexer if one is defined for Dojo +if [ "$NODE_ACTIVE_INDEXER" == "local_indexer" ]; then + explorer_options+=(--address-api electrumx) + explorer_options+=(--electrumx-servers "tcp://$INDEXER_IP:$INDEXER_RPC_PORT") +fi + node ./bin/cli.js "${explorer_options[@]}" > /data/logs/explorer-error.log 2> /data/logs/explorer-output.log diff --git a/docker/my-dojo/overrides/explorer.install.yaml b/docker/my-dojo/overrides/explorer.install.yaml index 06b97a1..44ed011 100644 --- a/docker/my-dojo/overrides/explorer.install.yaml +++ b/docker/my-dojo/overrides/explorer.install.yaml @@ -9,6 +9,8 @@ services: env_file: - ./.env - ./conf/docker-bitcoind.conf + - ./conf/docker-node.conf + - ./conf/docker-indexer.conf - ./conf/docker-explorer.conf restart: always command: "/home/node/app/restart.sh" From b132cf58e24b0fe6dffaf1d6731e5fa21036a5f8 Mon Sep 17 00:00:00 2001 From: kenshin-samourai Date: Wed, 1 Jan 2020 17:48:15 +0100 Subject: [PATCH 18/29] optimize build --- docker/my-dojo/indexer/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/my-dojo/indexer/Dockerfile b/docker/my-dojo/indexer/Dockerfile index 259f827..8c9519e 100644 --- a/docker/my-dojo/indexer/Dockerfile +++ b/docker/my-dojo/indexer/Dockerfile @@ -38,7 +38,6 @@ RUN cd "$INDEXER_HOME" && \ # git checkout "tags/v$INDEXER_VERSION" RUN cd "$INDEXER_HOME/addrindexrs" && \ - cargo build --release && \ cargo install --path . EXPOSE 50001 From 89af2718682ce1b96a7c006aefa8e159debdde42 Mon Sep 17 00:00:00 2001 From: kenshin-samourai Date: Wed, 1 Jan 2020 17:49:14 +0100 Subject: [PATCH 19/29] force the explorer to wait for the local indexer --- docker/my-dojo/explorer/Dockerfile | 5 +++++ docker/my-dojo/explorer/restart.sh | 17 +++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/docker/my-dojo/explorer/Dockerfile b/docker/my-dojo/explorer/Dockerfile index 081460a..b4c60b4 100644 --- a/docker/my-dojo/explorer/Dockerfile +++ b/docker/my-dojo/explorer/Dockerfile @@ -7,6 +7,11 @@ ENV EXPLORER_URL https://github.com/janoside/btc-rpc-explorer/archiv ENV EXPLORER_VERSION 1.1.5 +# Install netcat +RUN set -ex && \ + apt-get update && \ + apt-get install -y netcat + # Create logs and apps directory RUN mkdir -p "$LOGS_DIR" && \ chown -R node:node "$LOGS_DIR" && \ diff --git a/docker/my-dojo/explorer/restart.sh b/docker/my-dojo/explorer/restart.sh index 985debc..46a07d3 100644 --- a/docker/my-dojo/explorer/restart.sh +++ b/docker/my-dojo/explorer/restart.sh @@ -22,6 +22,23 @@ explorer_options+=(--rpc-blacklist "addnode,analyzepsbt,clearbanned,combinepsbt, if [ "$NODE_ACTIVE_INDEXER" == "local_indexer" ]; then explorer_options+=(--address-api electrumx) explorer_options+=(--electrumx-servers "tcp://$INDEXER_IP:$INDEXER_RPC_PORT") + + # Wait for the local indexer + timeout="720" + i="0" + while [ $i -lt $timeout ] + do + nc -z "$INDEXER_IP" "$INDEXER_RPC_PORT" > /dev/null + if [ $? -eq 0 ] ; then + break + fi + sleep 1 + i=$[$i+1] + done + if [ $i -eq $timeout ]; then + echo "Operation timed out" + exit 1 + fi fi node ./bin/cli.js "${explorer_options[@]}" > /data/logs/explorer-error.log 2> /data/logs/explorer-output.log From 6969007557aea8237a93e40af17678135f46c83e Mon Sep 17 00:00:00 2001 From: kenshin-samourai Date: Thu, 7 Nov 2019 16:05:19 +0100 Subject: [PATCH 20/29] add new local indexer docker module to dojo (experimental) the local indexer is an instance of addrindexrs (see https://github.com/Samourai-Wallet/addrindexrs) running inside dojo --- docker/my-dojo/.env | 8 + docker/my-dojo/conf/docker-indexer.conf.tpl | 42 +++++ docker/my-dojo/conf/docker-node.conf.tpl | 2 +- docker/my-dojo/dojo.sh | 20 ++ docker/my-dojo/indexer/Dockerfile | 46 +++++ docker/my-dojo/indexer/restart.sh | 22 +++ docker/my-dojo/indexer/wait-for-it.sh | 178 ++++++++++++++++++ docker/my-dojo/install/install-scripts.sh | 4 + docker/my-dojo/install/upgrade-scripts.sh | 3 + docker/my-dojo/node/keys.index.js | 12 +- docker/my-dojo/overrides/indexer.install.yaml | 27 +++ 11 files changed, 362 insertions(+), 2 deletions(-) create mode 100644 docker/my-dojo/conf/docker-indexer.conf.tpl create mode 100644 docker/my-dojo/indexer/Dockerfile create mode 100644 docker/my-dojo/indexer/restart.sh create mode 100644 docker/my-dojo/indexer/wait-for-it.sh create mode 100644 docker/my-dojo/overrides/indexer.install.yaml diff --git a/docker/my-dojo/.env b/docker/my-dojo/.env index 02530e8..27b0332 100644 --- a/docker/my-dojo/.env +++ b/docker/my-dojo/.env @@ -17,6 +17,7 @@ DOJO_NODEJS_VERSION_TAG=1.4.0 DOJO_NGINX_VERSION_TAG=1.4.0 DOJO_TOR_VERSION_TAG=1.3.0 DOJO_EXPLORER_VERSION_TAG=1.0.0 +DOJO_INDEXER_VERSION_TAG=1.0.0 ######################################### @@ -58,3 +59,10 @@ NODE_PREFIX_STATUS_PUSHTX=status NODE_TRACKER_MEMPOOL_PERIOD=10000 NODE_TRACKER_UNCONF_TXS_PERIOD=300000 + + +######################################### +# INDEXER +######################################### + +INDEXER_BATCH_SIZE=10 diff --git a/docker/my-dojo/conf/docker-indexer.conf.tpl b/docker/my-dojo/conf/docker-indexer.conf.tpl new file mode 100644 index 0000000..d7ad83c --- /dev/null +++ b/docker/my-dojo/conf/docker-indexer.conf.tpl @@ -0,0 +1,42 @@ +######################################### +# CONFIGURATION OF A LOCAL INDEXER +######################################### + +# Install and run a local indexer inside Docker +# Set this option to 'off' for using an indexer hosted outside of Docker +# or when using a different data source (local bitcoind, OXT) +# Value: on | off +INDEXER_INSTALL=off + +# IP address of the local indexer used by Dojo +# Set value to 172.28.1.6 if INDEXER_INSTALL is set to 'on' +# Type: string +INDEXER_IP=172.28.1.6 + +# Port of the RPC API +# Set value to 50001 if INDEXER_INSTALL is set to 'on' +# Type: integer +INDEXER_RPC_PORT=50001 + +# Support of batch requests by the local indexer +# Set value to inactive if INDEXER_INSTALL is set to 'on' +# Value: active | inactive +INDEXER_BATCH_SUPPORT=inactive + + +# +# EXPERT SETTINGS +# (ACTIVE IF INDEXER_INSTALL IS SET TO ON) +# + +# Number of blocks to get in one JSONRPC request from bitcoind +# Type: integer +INDEXER_BATCH_SIZE=10 + +# Total size of block txids to cache (in MB) +# Type: integer +INDEXER_BLK_TXIDS_CACHE_SIZE_MB=10 + +# Number of transactions to lookup before returning an error +# Type: integer +INDEXER_TXID_LIMIT=501 \ No newline at end of file diff --git a/docker/my-dojo/conf/docker-node.conf.tpl b/docker/my-dojo/conf/docker-node.conf.tpl index 403ed36..e7adcfa 100644 --- a/docker/my-dojo/conf/docker-node.conf.tpl +++ b/docker/my-dojo/conf/docker-node.conf.tpl @@ -21,7 +21,7 @@ NODE_ADMIN_KEY=myAdminKey NODE_JWT_SECRET=myJwtSecret # Indexer or third-party service used for imports and rescans of addresses -# Values: local_bitcoind | third_party_explorer +# Values: local_bitcoind | local_indexer | third_party_explorer NODE_ACTIVE_INDEXER=local_bitcoind # FEE TYPE USED FOR FEES ESTIMATIONS BY BITCOIND diff --git a/docker/my-dojo/dojo.sh b/docker/my-dojo/dojo.sh index 9331981..b150b1b 100755 --- a/docker/my-dojo/dojo.sh +++ b/docker/my-dojo/dojo.sh @@ -14,6 +14,7 @@ source_file() { } # Source config files +source_file "$DIR/conf/docker-indexer.conf" source_file "$DIR/conf/docker-bitcoind.conf" source_file "$DIR/conf/docker-explorer.conf" source_file "$DIR/conf/docker-common.conf" @@ -38,6 +39,10 @@ select_yaml_files() { yamlFiles="$yamlFiles -f $DIR/overrides/explorer.install.yaml" fi + if [ "$INDEXER_INSTALL" == "on" ]; then + yamlFiles="$yamlFiles -f $DIR/overrides/indexer.install.yaml" + fi + # Return yamlFiles echo "$yamlFiles" } @@ -154,6 +159,7 @@ uninstall() { docker image rm samouraiwallet/dojo-nodejs:"$DOJO_NODEJS_VERSION_TAG" docker image rm samouraiwallet/dojo-nginx:"$DOJO_NGINX_VERSION_TAG" docker image rm samouraiwallet/dojo-tor:"$DOJO_TOR_VERSION_TAG" + docker image rm samouraiwallet/dojo-indexer:"$DOJO_INDEXER_VERSION_TAG" docker volume prune } @@ -177,6 +183,7 @@ clean() { del_images_for samouraiwallet/dojo-nodejs "$DOJO_NODEJS_VERSION_TAG" del_images_for samouraiwallet/dojo-nginx "$DOJO_NGINX_VERSION_TAG" del_images_for samouraiwallet/dojo-tor "$DOJO_TOR_VERSION_TAG" + del_images_for samouraiwallet/dojo-indexer "$DOJO_INDEXER_VERSION_TAG" } # Upgrade @@ -249,6 +256,7 @@ logs_explorer() { logs() { source_file "$DIR/conf/docker-bitcoind.conf" + source_file "$DIR/conf/docker-indexer.conf" source_file "$DIR/conf/docker-common.conf" case $1 in @@ -267,6 +275,14 @@ logs() { echo -e "Command not supported for your setup.\nCause: Your Dojo is using an external bitcoind" fi ;; + indexer ) + if [ "$INDEXER_INSTALL" == "on" ]; then + yamlFiles=$(select_yaml_files) + eval "docker-compose $yamlFiles logs --tail=50 --follow indexer" + else + echo -e "Command not supported for your setup.\nCause: Your Dojo is not using an internal indexer" + fi + ;; tor ) docker-compose logs --tail=50 --follow tor ;; @@ -285,6 +301,9 @@ logs() { if [ "$EXPLORER_INSTALL" == "on" ]; then services="$services explorer" fi + if [ "$INDEXER_INSTALL" == "on" ]; then + services="$services indexer" + fi eval "docker-compose $yamlFiles logs --tail=0 --follow $services" ;; esac @@ -312,6 +331,7 @@ help() { echo " dojo.sh logs bitcoind : display the logs of bitcoind" echo " dojo.sh logs db : display the logs of the MySQL database" echo " dojo.sh logs tor : display the logs of tor" + echo " dojo.sh logs indexer : display the logs of the internal indexer" echo " dojo.sh logs api : display the logs of the REST API (nodejs)" echo " dojo.sh logs tracker : display the logs of the Tracker (nodejs)" echo " dojo.sh logs pushtx : display the logs of the pushTx API (nodejs)" diff --git a/docker/my-dojo/indexer/Dockerfile b/docker/my-dojo/indexer/Dockerfile new file mode 100644 index 0000000..259f827 --- /dev/null +++ b/docker/my-dojo/indexer/Dockerfile @@ -0,0 +1,46 @@ +FROM rust:1.37.0-slim + +ENV INDEXER_HOME /home/indexer +ENV INDEXER_VERSION 0..1.0 +ENV INDEXER_URL https://github.com/Samourai-Wallet/addrindexrs.git + +RUN apt-get update && \ + apt-get install -y clang cmake git && \ + apt-get install -y libsnappy-dev + +# Create group and user indexer +RUN addgroup --system -gid 1109 indexer && \ + adduser --system --ingroup indexer -uid 1106 indexer + +# Create data directory +RUN mkdir "$INDEXER_HOME/addrindexrs" && \ + chown -h indexer:indexer "$INDEXER_HOME/addrindexrs" + +# Copy restart script +COPY ./restart.sh /restart.sh +RUN chown indexer:indexer /restart.sh && \ + chmod 777 /restart.sh + +# Copy wait-for-it script +COPY ./wait-for-it.sh /wait-for-it.sh +RUN chown indexer:indexer /wait-for-it.sh && \ + chmod u+x /wait-for-it.sh && \ + chmod g+x /wait-for-it.sh + +USER indexer + +# Install addrindexrs +RUN cd "$INDEXER_HOME" && \ + git clone "$INDEXER_URL" "$INDEXER_HOME/addrindexrs" && \ + cd addrindexrs && \ + git checkout "master" +# TODO uncomment +# git checkout "tags/v$INDEXER_VERSION" + +RUN cd "$INDEXER_HOME/addrindexrs" && \ + cargo build --release && \ + cargo install --path . + +EXPOSE 50001 + +STOPSIGNAL SIGINT diff --git a/docker/my-dojo/indexer/restart.sh b/docker/my-dojo/indexer/restart.sh new file mode 100644 index 0000000..0db33b1 --- /dev/null +++ b/docker/my-dojo/indexer/restart.sh @@ -0,0 +1,22 @@ +#!/bin/bash +set -e + +indexer_options=( + -vvvv + --index-batch-size="$INDEXER_BATCH_SIZE" + --jsonrpc-import + --db-dir="/home/indexer/db" + --indexer-rpc-addr="172.28.1.6:50001" + --daemon-rpc-addr="$BITCOIND_IP:$BITCOIND_RPC_PORT" + --cookie="$BITCOIND_RPC_USER:$BITCOIND_RPC_PASSWORD" + --txid-limit="$INDEXER_TXID_LIMIT" + --blocktxids-cache-size-mb="$INDEXER_BLK_TXIDS_CACHE_SIZE_MB" +) + +if [ "$COMMON_BTC_NETWORK" == "testnet" ]; then + bitcoind_options+=(--network="testnet") +else + bitcoind_options+=(--network="mainnet") +fi + +addrindexrs "${indexer_options[@]}" diff --git a/docker/my-dojo/indexer/wait-for-it.sh b/docker/my-dojo/indexer/wait-for-it.sh new file mode 100644 index 0000000..071c2be --- /dev/null +++ b/docker/my-dojo/indexer/wait-for-it.sh @@ -0,0 +1,178 @@ +#!/usr/bin/env bash +# Use this script to test if a given TCP host/port are available + +WAITFORIT_cmdname=${0##*/} + +echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } + +usage() +{ + cat << USAGE >&2 +Usage: + $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args] + -h HOST | --host=HOST Host or IP under test + -p PORT | --port=PORT TCP port under test + Alternatively, you specify the host and port as host:port + -s | --strict Only execute subcommand if the test succeeds + -q | --quiet Don't output any status messages + -t TIMEOUT | --timeout=TIMEOUT + Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit 1 +} + +wait_for() +{ + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + else + echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout" + fi + WAITFORIT_start_ts=$(date +%s) + while : + do + if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then + nc -z $WAITFORIT_HOST $WAITFORIT_PORT + WAITFORIT_result=$? + else + (echo > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1 + WAITFORIT_result=$? + fi + if [[ $WAITFORIT_result -eq 0 ]]; then + WAITFORIT_end_ts=$(date +%s) + echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds" + break + fi + sleep 1 + done + return $WAITFORIT_result +} + +wait_for_wrapper() +{ + # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 + if [[ $WAITFORIT_QUIET -eq 1 ]]; then + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + else + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + fi + WAITFORIT_PID=$! + trap "kill -INT -$WAITFORIT_PID" INT + wait $WAITFORIT_PID + WAITFORIT_RESULT=$? + if [[ $WAITFORIT_RESULT -ne 0 ]]; then + echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + fi + return $WAITFORIT_RESULT +} + +# process arguments +while [[ $# -gt 0 ]] +do + case "$1" in + *:* ) + WAITFORIT_hostport=(${1//:/ }) + WAITFORIT_HOST=${WAITFORIT_hostport[0]} + WAITFORIT_PORT=${WAITFORIT_hostport[1]} + shift 1 + ;; + --child) + WAITFORIT_CHILD=1 + shift 1 + ;; + -q | --quiet) + WAITFORIT_QUIET=1 + shift 1 + ;; + -s | --strict) + WAITFORIT_STRICT=1 + shift 1 + ;; + -h) + WAITFORIT_HOST="$2" + if [[ $WAITFORIT_HOST == "" ]]; then break; fi + shift 2 + ;; + --host=*) + WAITFORIT_HOST="${1#*=}" + shift 1 + ;; + -p) + WAITFORIT_PORT="$2" + if [[ $WAITFORIT_PORT == "" ]]; then break; fi + shift 2 + ;; + --port=*) + WAITFORIT_PORT="${1#*=}" + shift 1 + ;; + -t) + WAITFORIT_TIMEOUT="$2" + if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi + shift 2 + ;; + --timeout=*) + WAITFORIT_TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + WAITFORIT_CLI=("$@") + break + ;; + --help) + usage + ;; + *) + echoerr "Unknown argument: $1" + usage + ;; + esac +done + +if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then + echoerr "Error: you need to provide a host and port to test." + usage +fi + +WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15} +WAITFORIT_STRICT=${WAITFORIT_STRICT:-0} +WAITFORIT_CHILD=${WAITFORIT_CHILD:-0} +WAITFORIT_QUIET=${WAITFORIT_QUIET:-0} + +# check to see if timeout is from busybox? +WAITFORIT_TIMEOUT_PATH=$(type -p timeout) +WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH) +if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then + WAITFORIT_ISBUSY=1 + WAITFORIT_BUSYTIMEFLAG="-t" + +else + WAITFORIT_ISBUSY=0 + WAITFORIT_BUSYTIMEFLAG="" +fi + +if [[ $WAITFORIT_CHILD -gt 0 ]]; then + wait_for + WAITFORIT_RESULT=$? + exit $WAITFORIT_RESULT +else + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + wait_for_wrapper + WAITFORIT_RESULT=$? + else + wait_for + WAITFORIT_RESULT=$? + fi +fi + +if [[ $WAITFORIT_CLI != "" ]]; then + if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then + echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess" + exit $WAITFORIT_RESULT + fi + exec "${WAITFORIT_CLI[@]}" +else + exit $WAITFORIT_RESULT +fi diff --git a/docker/my-dojo/install/install-scripts.sh b/docker/my-dojo/install/install-scripts.sh index 79b2142..2f31717 100755 --- a/docker/my-dojo/install/install-scripts.sh +++ b/docker/my-dojo/install/install-scripts.sh @@ -61,6 +61,9 @@ init_config_files() { cp ./conf/docker-tor.conf.tpl ./conf/docker-tor.conf echo "Initialized docker-tor.conf" + cp ./conf/docker-indexer.conf.tpl ./conf/docker-indexer.conf + echo "Initialized docker-indexer.conf" + # Initialize config files for nginx and the maintenance tool if [ "$EXPLORER_INSTALL" == "on" ]; then cp ./nginx/explorer.conf ./nginx/dojo-explorer.conf @@ -69,6 +72,7 @@ init_config_files() { fi echo "Initialized dojo-explorer.conf (nginx)" + # Initialize config files for nginx and the maintenance tool if [ "$COMMON_BTC_NETWORK" == "testnet" ]; then cp ./nginx/testnet.conf ./nginx/dojo.conf echo "Initialized dojo.conf (nginx)" diff --git a/docker/my-dojo/install/upgrade-scripts.sh b/docker/my-dojo/install/upgrade-scripts.sh index 68554aa..f59dbcb 100755 --- a/docker/my-dojo/install/upgrade-scripts.sh +++ b/docker/my-dojo/install/upgrade-scripts.sh @@ -57,6 +57,9 @@ update_config_files() { update_config_file ./conf/docker-tor.conf ./conf/docker-tor.conf.tpl echo "Initialized docker-tor.conf" + update_config_file ./conf/docker-indexer.conf ./conf/docker-indexer.conf.tpl + echo "Initialized docker-indexer.conf" + # Initialize config files for nginx and the maintenance tool if [ "$EXPLORER_INSTALL" == "on" ]; then cp ./nginx/explorer.conf ./nginx/dojo-explorer.conf diff --git a/docker/my-dojo/node/keys.index.js b/docker/my-dojo/node/keys.index.js index 300a1a5..a6aae97 100644 --- a/docker/my-dojo/node/keys.index.js +++ b/docker/my-dojo/node/keys.index.js @@ -156,8 +156,18 @@ module.exports = { */ indexer: { // Active indexer - // Values: local_bitcoind | third_party_explorer + // Values: local_bitcoind | local_indexer | third_party_explorer active: process.env.NODE_ACTIVE_INDEXER, + // Local indexer + localIndexer: { + // IP address or hostname of the local indexer + host: process.env.INDEXER_IP, + // Port + port: parseInt(process.env.INDEXER_RPC_PORT), + // Support of batch requests by the local indexer + // Values: active | inactive + batchRequests: process.env.INDEXER_BATCH_SUPPORT + }, // Use a SOCKS5 proxy for all communications with external services // Values: null if no socks5 proxy used, otherwise the url of the socks5 proxy socks5Proxy: 'socks5h://172.28.1.4:9050', diff --git a/docker/my-dojo/overrides/indexer.install.yaml b/docker/my-dojo/overrides/indexer.install.yaml new file mode 100644 index 0000000..e130624 --- /dev/null +++ b/docker/my-dojo/overrides/indexer.install.yaml @@ -0,0 +1,27 @@ +version: "3.2" + +services: + indexer: + image: "samouraiwallet/dojo-indexer:${DOJO_INDEXER_VERSION_TAG}" + container_name: indexer + build: + context: ./indexer + env_file: + - ./.env + - ./conf/docker-common.conf + - ./conf/docker-bitcoind.conf + - ./conf/docker-indexer.conf + restart: on-failure + command: "/wait-for-it.sh tor:9050 --timeout=360 --strict -- /restart.sh" + expose: + - "50001" + volumes: + - data-indexer:/home/indexer/addrindexrs + depends_on: + - tor + networks: + dojonet: + ipv4_address: 172.28.1.6 + +volumes: + data-indexer: From ba1cd4ef64b926660a91c1385b7c57674be3cbd7 Mon Sep 17 00:00:00 2001 From: kenshin-samourai Date: Thu, 7 Nov 2019 19:47:15 +0100 Subject: [PATCH 21/29] fix path of docker volume for local indexer --- docker/my-dojo/overrides/indexer.install.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/my-dojo/overrides/indexer.install.yaml b/docker/my-dojo/overrides/indexer.install.yaml index e130624..92635d4 100644 --- a/docker/my-dojo/overrides/indexer.install.yaml +++ b/docker/my-dojo/overrides/indexer.install.yaml @@ -16,7 +16,7 @@ services: expose: - "50001" volumes: - - data-indexer:/home/indexer/addrindexrs + - data-indexer:/home/indexer depends_on: - tor networks: From 1a8f8165b321a0976fa9eb3f8310487e711cb0ea Mon Sep 17 00:00:00 2001 From: kenshin-samourai Date: Thu, 7 Nov 2019 22:30:56 +0100 Subject: [PATCH 22/29] add loading of docker-indexer.conf in nodejs container --- docker/my-dojo/docker-compose.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/my-dojo/docker-compose.yaml b/docker/my-dojo/docker-compose.yaml index b4e82f9..1c09374 100644 --- a/docker/my-dojo/docker-compose.yaml +++ b/docker/my-dojo/docker-compose.yaml @@ -31,6 +31,7 @@ services: - ./conf/docker-mysql.conf - ./conf/docker-bitcoind.conf - ./conf/docker-explorer.conf + - ./conf/docker-indexer.conf - ./conf/docker-node.conf restart: always command: "/home/node/app/wait-for-it.sh db:3306 --timeout=720 --strict -- /home/node/app/restart.sh" From 3ae248a8a1d503a86249f222f83172926a08bb06 Mon Sep 17 00:00:00 2001 From: kenshin-samourai Date: Thu, 7 Nov 2019 16:05:19 +0100 Subject: [PATCH 23/29] add new local indexer docker module to dojo (experimental) the local indexer is an instance of addrindexrs (see https://github.com/Samourai-Wallet/addrindexrs) running inside dojo --- docker/my-dojo/install/install-scripts.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/my-dojo/install/install-scripts.sh b/docker/my-dojo/install/install-scripts.sh index 2f31717..3db46a3 100755 --- a/docker/my-dojo/install/install-scripts.sh +++ b/docker/my-dojo/install/install-scripts.sh @@ -64,7 +64,6 @@ init_config_files() { cp ./conf/docker-indexer.conf.tpl ./conf/docker-indexer.conf echo "Initialized docker-indexer.conf" - # Initialize config files for nginx and the maintenance tool if [ "$EXPLORER_INSTALL" == "on" ]; then cp ./nginx/explorer.conf ./nginx/dojo-explorer.conf else From 421e136b162e36f06acccf9465ba103e6081ab0c Mon Sep 17 00:00:00 2001 From: kenshin-samourai Date: Tue, 31 Dec 2019 16:03:00 +0100 Subject: [PATCH 24/29] connect the explorer and the indexer --- docker/my-dojo/explorer/restart.sh | 6 ++++++ docker/my-dojo/overrides/explorer.install.yaml | 2 ++ 2 files changed, 8 insertions(+) diff --git a/docker/my-dojo/explorer/restart.sh b/docker/my-dojo/explorer/restart.sh index 4da82fa..985debc 100644 --- a/docker/my-dojo/explorer/restart.sh +++ b/docker/my-dojo/explorer/restart.sh @@ -18,4 +18,10 @@ explorer_options=( # Blacklist all functions provided by the RPC API explorer_options+=(--rpc-blacklist "addnode,analyzepsbt,clearbanned,combinepsbt,combinerawtransaction,converttopsbt,createmultisig,createpsbt,createrawtransaction,decodepsbt,decoderawtransaction,decodescript,deriveaddresses,disconnectnode,echo,echojson,estimaterawfee,estimatesmartfee,finalizepsbt,generatetoaddress,generatetodescriptor,getaddednodeinfo,getbestblockhash,getblock,getblockchaininfo,getblockcount,getblockfilter,getblockhash,getblockheader,getblockstats,getblocktemplate,getchaintips,getchaintxstats,getconnectioncount,getdescriptorinfo,getdifficulty,getmemoryinfo,getmempoolancestors,getmempooldescendants,getmempoolentry,getmempoolinfo,getmininginfo,getnettotals,getnetworkhashps,getnetworkinfo,getnodeaddresses,getpeerinfo,getrawmempool,getrawtransaction,getrpcinfo,gettxout,gettxoutproof,gettxoutsetinfo,help,invalidateblock,joinpsbts,listbanned,logging,ping,preciousblock,prioritisetransaction,pruneblockchain,reconsiderblock,savemempool,scantxoutset,sendrawtransaction,setban,setmocktime,setnetworkactive,signmessagewithprivkey,signrawtransactionwithkey,stop,submitblock,submitheader,syncwithvalidationinterfacequeue,testmempoolaccept,uptime,utxoupdatepsbt,validateaddress,verifychain,verifymessage,verifytxoutproof,waitforblock,waitforblockheight,waitfornewblock") +# Use the local indexer if one is defined for Dojo +if [ "$NODE_ACTIVE_INDEXER" == "local_indexer" ]; then + explorer_options+=(--address-api electrumx) + explorer_options+=(--electrumx-servers "tcp://$INDEXER_IP:$INDEXER_RPC_PORT") +fi + node ./bin/cli.js "${explorer_options[@]}" > /data/logs/explorer-error.log 2> /data/logs/explorer-output.log diff --git a/docker/my-dojo/overrides/explorer.install.yaml b/docker/my-dojo/overrides/explorer.install.yaml index 06b97a1..44ed011 100644 --- a/docker/my-dojo/overrides/explorer.install.yaml +++ b/docker/my-dojo/overrides/explorer.install.yaml @@ -9,6 +9,8 @@ services: env_file: - ./.env - ./conf/docker-bitcoind.conf + - ./conf/docker-node.conf + - ./conf/docker-indexer.conf - ./conf/docker-explorer.conf restart: always command: "/home/node/app/restart.sh" From 57ad6ef34059fc2c0e0033ba41762b1f388f6433 Mon Sep 17 00:00:00 2001 From: kenshin-samourai Date: Wed, 1 Jan 2020 17:48:15 +0100 Subject: [PATCH 25/29] optimize build --- docker/my-dojo/indexer/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/my-dojo/indexer/Dockerfile b/docker/my-dojo/indexer/Dockerfile index 259f827..8c9519e 100644 --- a/docker/my-dojo/indexer/Dockerfile +++ b/docker/my-dojo/indexer/Dockerfile @@ -38,7 +38,6 @@ RUN cd "$INDEXER_HOME" && \ # git checkout "tags/v$INDEXER_VERSION" RUN cd "$INDEXER_HOME/addrindexrs" && \ - cargo build --release && \ cargo install --path . EXPOSE 50001 From c22a8f6b1085a032f3d4ab2ab1f1dbe645f06cbe Mon Sep 17 00:00:00 2001 From: kenshin-samourai Date: Wed, 1 Jan 2020 17:49:14 +0100 Subject: [PATCH 26/29] force the explorer to wait for the local indexer --- docker/my-dojo/explorer/Dockerfile | 5 +++++ docker/my-dojo/explorer/restart.sh | 17 +++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/docker/my-dojo/explorer/Dockerfile b/docker/my-dojo/explorer/Dockerfile index 081460a..b4c60b4 100644 --- a/docker/my-dojo/explorer/Dockerfile +++ b/docker/my-dojo/explorer/Dockerfile @@ -7,6 +7,11 @@ ENV EXPLORER_URL https://github.com/janoside/btc-rpc-explorer/archiv ENV EXPLORER_VERSION 1.1.5 +# Install netcat +RUN set -ex && \ + apt-get update && \ + apt-get install -y netcat + # Create logs and apps directory RUN mkdir -p "$LOGS_DIR" && \ chown -R node:node "$LOGS_DIR" && \ diff --git a/docker/my-dojo/explorer/restart.sh b/docker/my-dojo/explorer/restart.sh index 985debc..46a07d3 100644 --- a/docker/my-dojo/explorer/restart.sh +++ b/docker/my-dojo/explorer/restart.sh @@ -22,6 +22,23 @@ explorer_options+=(--rpc-blacklist "addnode,analyzepsbt,clearbanned,combinepsbt, if [ "$NODE_ACTIVE_INDEXER" == "local_indexer" ]; then explorer_options+=(--address-api electrumx) explorer_options+=(--electrumx-servers "tcp://$INDEXER_IP:$INDEXER_RPC_PORT") + + # Wait for the local indexer + timeout="720" + i="0" + while [ $i -lt $timeout ] + do + nc -z "$INDEXER_IP" "$INDEXER_RPC_PORT" > /dev/null + if [ $? -eq 0 ] ; then + break + fi + sleep 1 + i=$[$i+1] + done + if [ $i -eq $timeout ]; then + echo "Operation timed out" + exit 1 + fi fi node ./bin/cli.js "${explorer_options[@]}" > /data/logs/explorer-error.log 2> /data/logs/explorer-output.log From 219613b583ead7f9471b0e41920b3e3dacd87662 Mon Sep 17 00:00:00 2001 From: kenshin-samourai Date: Mon, 20 Jan 2020 16:24:05 +0100 Subject: [PATCH 27/29] update doc with new sections about the local indexer and personal electrum server --- README.md | 17 ++++-- doc/DOCKER_advanced_setups.md | 99 ++++++++++++++++++++++++++++++++++- doc/DOCKER_setup.md | 54 ++++++++++++------- 3 files changed, 144 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 46facf4..786e3f3 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,8 @@ It provides in a single command the setup of a full Samourai backend composed of * the backend database, * the backend modules with an API accessible as a static Tor hidden service, * a maintenance tool accessible through a Tor web browser, -* a block explorer ([BTC RPC Explorer](https://github.com/janoside/btc-rpc-explorer)) accessible through a Tor web browser. +* a block explorer ([BTC RPC Explorer](https://github.com/janoside/btc-rpc-explorer)) accessible through a Tor web browser, +* an optional indexer of Bitcoin addresses ([addrindexrs](https://github.com/Samourai-Wallet/addrindexrs)) providing fast and private rescans of HD accounts and loose addresses. See [the documentation](./doc/DOCKER_setup.md) for detailed setup instructions. @@ -61,13 +62,16 @@ Authentication is enforced by an API key and Json Web Tokens. **Import of HD Accounts and data sources** -* First import of an unknown HD account relies on a data source (local bitcoind or OXT). After that, the tracker will keep everything current. +* First import of an unknown HD account relies on a data source (local bitcoind, local indexer or OXT). After that, the tracker will keep everything current. -* Default option relies on the local bitcoind and makes you 100% independent of Samourai Wallet's infrastructure. This option is recommended for better privacy. +* Using the local bitcoind (default option) or the local indexer makes you 100% independent of Samourai Wallet's infrastructure and is recommended for better privacy. * Activation of bitcoind as the data source: * Edit /keys/index.js and set "indexer.active" to "local_bitcoind". OXT API will be ignored. +* Activation of the local indexer as the data source: + * Edit /keys/index.js and set "indexer.active" to "local_indexer". OXT API will be ignored. + * Activation of OXT as the data source (through socks5): * Edit /keys/index.js and set "indexer.active" to "third_party_explorer". @@ -77,5 +81,8 @@ Authentication is enforced by an API key and Json Web Tokens. * It's slightly slower than using the option relying on the OXT API. * It may fail to correctly import an existing wallet if this wallet had a large activity. * If you use bitcoind and if the import seems to return an invalid balance, you can use the "XPUB rescan" function provided by the maintenance tool. This function allows you to force the minimum number of addresses to be derived and the start index for the derivation. - * As a rule of thumb, we recommend to use bitcoind as the source of imports and to setup your Dojo with a new clean wallet. It increases your privacy and it removes all potential issues with the import of a large wallet. - + +* Main drawbacks of using your local indexer for these imports: + * It requires 120GB of additional disk space during its initialization. + +As a rule of thumb, we recommend to use the local indexer as the source of imports and to setup your Dojo with a new clean wallet. It increases your privacy and it removes all potential issues with the import of a large wallet. diff --git a/doc/DOCKER_advanced_setups.md b/doc/DOCKER_advanced_setups.md index 215bb69..1810cde 100644 --- a/doc/DOCKER_advanced_setups.md +++ b/doc/DOCKER_advanced_setups.md @@ -3,10 +3,12 @@ The configuration files of Dojo provide a few advanced options allowing to tune your setup. -A word of caution, though, the default values of these options try to maximize your privacy at a network level. Most of the advanced setups described in this document may damage your privacy. Use at your own risk! +A word of caution, though, the default values of these options try to maximize your privacy at a network level. Some of the advanced setups described in this document may damage your privacy. Use at your own risk! ## Table of Content ## +- [Local indexer of Bitcoin addresses](#local_indexer) +- [Local Electrum server used as data source for imports/rescans](#local_electrum) - [External Bitcoin full node](#external_bitcoind) - [bitcoind RPC API ans ZMQ notifications exposed to external apps](#exposed_rpc_zmq) - [Static onion address for bitcoind hidden service](#static_onion) @@ -14,6 +16,101 @@ A word of caution, though, the default values of these options try to maximize y - [Support of testnet](#testnet) + + +## Local indexer of Bitcoin addresses ## + +By default, Dojo uses the local full node as its data source for imports and rescans of HD accounts and addresses. While private, this default option has many limitations. MyDojo allows to install a local indexer ([addrindexrs](https://github.com/Samourai-Wallet/addrindexrs)) providing the best of both worlds (no request sent to a third party, fast and real time rescans, complete transactional history is retrieved). + + +### Requirements ### + +To date, the initial installation of the indexer requires 120GB of additionnal disk space. + + +### Main benefits ### + +- Fast, private and exhaustive real time rescans, +- Allows the block explorer to display the detailed activity of Bitcoin addresses + + +### Known drawbacks ### + +* Additionnal disk space consumed by the index, +* Increased duration of upgrades (from 5 to 20 minutes depending on the machine hosting your Dojo), +* Slight increase of startup duration, +* First indexation will require a few hours. + + +### Procedure ### + +``` +# Edit the indexer config template file +nano ./conf/docker-indexer.conf.tpl + +# +# Set the value of INDEXER_INSTALL to "on" +# Save and exit nano +# + +# Edit the nodejs config file (or the corresponding template file if it's your first installation of Dojo) +nano ./conf/docker-node.conf + +# +# Set the value of NODE_ACTIVE_INDEXER to "local_indexer" +# Save and exit nano +# + +# +# Launch the installation or the upgrade of your Dojo +# with the commands `dojo.sh install` or `dojo.sh upgrade` +# + +# +# Be patient! +# First indexation of all Bitcoin addresses will require a few hours. +# Let the indexer complete all these operations before trying to use it for an import or a rescan. +# You can follow the progress made by the indexer with the commands: +# `dojo.sh logs` +# or +# `dojo.sh logs indexer` +# +``` + + + + +## Local Electrum server used as data source for imports/rescans ## + +If you're running an instance of ElectrumX or Electrs on your local network, Dojo allows you to define this instance as the data source used for imports and rescans. This setup is an alternative to the local indexer provided by MyDojo. + +Important: Do not use an Electrum server operated by a third party or hosted on a different local network. + + +### Procedure ### + +``` +# Edit the indexer config template file +nano ./conf/docker-indexer.conf.tpl + +# +# Set the value of INDEXER_INSTALL to "off" +# Set the value of INDEXER_IP with the IP address of your Electrum server +# Set the value of INDEXER_RPC_PORT with the port used by the RPC API of your Electrum server (default= 50001) +# Set the value of INDEXER_BATCH_SUPPORT to "active" if your Electrum server is ElectrumX, otherwise set the value to "inactive" +# Save and exit nano +# + +# Edit the nodejs config file (or the corresponding template file if it's your first installation of Dojo) +nano ./conf/docker-node.conf + +# +# Set the value of NODE_ACTIVE_INDEXER to "local_indexer" +# Save and exit nano +# +``` + + ## External Bitcoin full node ## diff --git a/doc/DOCKER_setup.md b/doc/DOCKER_setup.md index 5bca766..b44c8b3 100644 --- a/doc/DOCKER_setup.md +++ b/doc/DOCKER_setup.md @@ -6,6 +6,7 @@ MyDojo is a set of Docker containers providing a full Samourai backend composed * backend modules with an API accessible as a static Tor hidden service, * a maintenance tool accessible through a Tor web browser, * a block explorer ([BTC RPC Explorer](https://github.com/janoside/btc-rpc-explorer)) accessible as a static Tor hidden service. +* an optional indexer of Bitcoin addresses ([addrindexrs](https://github.com/Samourai-Wallet/addrindexrs)) providing fast and private rescans of HD accounts and loose addresses. ## Table of Content ## @@ -40,33 +41,39 @@ MyDojo is a set of Docker containers providing a full Samourai backend composed Host machine | (Tor hidden services) ______________________________ | _____________________________ | | | - | ------------------- dmznet | - | | Tor Container | | - | ------------------- | - | | | | - | ------------------- | | - | | Nginx Container | | | - | ------------------- | | - |- - - - - - - - - - - | - - -|- - - - | - - - - - - - - - - - | - | -------------------- | -------------------- | - | | Nodejs Container | ------ | Bitcoind Container | | - | -------------------- | -------------------- | - | | | | | - | -------------------- | -------------------- | - | | MySQL Container | ---- | BTC RPC Explorer | | - | -------------------- -------------------- | - | | - | dojonet | + | --------- dmznet | + | --------| Tor |------------ | + | | --------- | | + | | | | + | --------- | | + | --| Nginx |-------- | | + | | --------- | | | + |- - - - -|- - - - - - - - - - -|- - - - - - - - -|- - - - - - | + | | | | | + | ---------- ---------- ---------- | + | | Nodejs |----------| Explorer |------| Bitcoind | | + | ---------- ---------- ---------- | + | | | | | | + | | ------- | | | + | | | | | | + | ---------- | ---------- | | + | | MySQL | ----| Indexer |----------- | + | ---------- ---------- | + | dojonet | |______________________________________________________________| + + + + ## Requirements ## * A dedicated computer (host machine) connected 24/7 to internet * OS: Linux is recommended -* Disk: 500GB (minimal) / 1TB (recommended) - SSD is recommended +* Disk: 600GB (minimal) / 1TB (recommended) - SSD is recommended * RAM: 4GB (minimal) * Docker and Docker Compose installed on the host machine (be sure to run a recent version supporting v3.2 of docker-compose files, i.e. Docker Engine v17.04.0+) * Check that the clock of your computer is properly set (required for Tor) @@ -77,10 +84,11 @@ MyDojo is a set of Docker containers providing a full Samourai backend composed ## Configuration files ## -Each new release of Dojo is packaged with 6 template files stored in the `/docker/my-dojo/conf` directory: +Each new release of Dojo is packaged with 7 template files stored in the `/docker/my-dojo/conf` directory: - docker-common.conf.tpl - docker-bitcoin.conf.tpl - docker-explorer.conf.tpl +- docker-indexer.conf.tpl - docker-mysql.conf.tpl - docker-node.conf.tpl - docker-tor.conf.tpl @@ -106,6 +114,9 @@ For MacOS, see this detailed [installation guide](./DOCKER_mac_setup.MD). For Synology, see this detailed [installation guide](./DOCKER_synology_setup.md). +For Raspberry Pi4 and Odroid N2, see the [Ronin Dojo Project](https://github.com/RoninDojo/RoninDojo) + + This procedure allows to install a new Dojo from scratch. * Install [Docker and Docker Compose](https://docs.docker.com/compose/install/) on the host machine and check that your installation is working. @@ -146,7 +157,9 @@ This procedure allows to install a new Dojo from scratch. * If you want to deactivate the block explorer, set the value of `EXPLORER_INSTALL` to `off`. See this [section](#explorer) for more details about the block explorer. -* Dojo provides a few additional settings for advanced setups: +* Dojo provides a few additional settings for advanced setups: + * installation of an address indexer used for fast imports and rescans, + * support of an external electrum server (ElectrumX or electrs) used for fast imports and rescans, * static onion address for your full node, * bitcoind RPC API exposed to external apps, * use of an external full node, @@ -239,6 +252,7 @@ Available commands: dojo.sh logs bitcoind : display the logs of bitcoind dojo.sh logs db : display the logs of the MySQL database dojo.sh logs tor : display the logs of tor + dojo.sh logs indexer : display the logs of the internal indexer dojo.sh logs api : display the logs of the REST API (nodejs) dojo.sh logs tracker : display the logs of the Tracker (nodejs) dojo.sh logs pushtx : display the logs of the pushTx API (nodejs) From a35896e957f516547e75f2ee11acc0a0a5e67bdb Mon Sep 17 00:00:00 2001 From: kenshin-samourai Date: Mon, 20 Jan 2020 16:37:07 +0100 Subject: [PATCH 28/29] build indexer container with addrindexrs v0.1.0 --- docker/my-dojo/indexer/Dockerfile | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docker/my-dojo/indexer/Dockerfile b/docker/my-dojo/indexer/Dockerfile index 8c9519e..1deac0b 100644 --- a/docker/my-dojo/indexer/Dockerfile +++ b/docker/my-dojo/indexer/Dockerfile @@ -1,7 +1,7 @@ FROM rust:1.37.0-slim ENV INDEXER_HOME /home/indexer -ENV INDEXER_VERSION 0..1.0 +ENV INDEXER_VERSION 0.1.0 ENV INDEXER_URL https://github.com/Samourai-Wallet/addrindexrs.git RUN apt-get update && \ @@ -33,9 +33,7 @@ USER indexer RUN cd "$INDEXER_HOME" && \ git clone "$INDEXER_URL" "$INDEXER_HOME/addrindexrs" && \ cd addrindexrs && \ - git checkout "master" -# TODO uncomment -# git checkout "tags/v$INDEXER_VERSION" + git checkout "tags/v$INDEXER_VERSION" RUN cd "$INDEXER_HOME/addrindexrs" && \ cargo install --path . From 81fc94888ae4a925534c789b54d588a2ff9043e5 Mon Sep 17 00:00:00 2001 From: kenshin-samourai Date: Mon, 17 Feb 2020 18:39:25 +0100 Subject: [PATCH 29/29] bump version of explorer container --- docker/my-dojo/.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/my-dojo/.env b/docker/my-dojo/.env index 27b0332..d192fec 100644 --- a/docker/my-dojo/.env +++ b/docker/my-dojo/.env @@ -16,7 +16,7 @@ DOJO_BITCOIND_VERSION_TAG=1.4.1 DOJO_NODEJS_VERSION_TAG=1.4.0 DOJO_NGINX_VERSION_TAG=1.4.0 DOJO_TOR_VERSION_TAG=1.3.0 -DOJO_EXPLORER_VERSION_TAG=1.0.0 +DOJO_EXPLORER_VERSION_TAG=1.1.0 DOJO_INDEXER_VERSION_TAG=1.0.0