From c16604d60258ce0e7e4d0e20e939e72b204db9ea Mon Sep 17 00:00:00 2001 From: kenshin-samourai Date: Thu, 7 Nov 2019 16:05:19 +0100 Subject: [PATCH] add new local indexer docker module to dojo (experimental) the local indexer is an instance of addrindexrs (see https://github.com/Samourai-Wallet/addrindexrs) running inside dojo --- docker/my-dojo/.env | 8 + docker/my-dojo/conf/docker-indexer.conf.tpl | 42 +++++ docker/my-dojo/conf/docker-node.conf.tpl | 2 +- docker/my-dojo/dojo.sh | 20 ++ docker/my-dojo/indexer/Dockerfile | 46 +++++ docker/my-dojo/indexer/restart.sh | 22 +++ docker/my-dojo/indexer/wait-for-it.sh | 178 ++++++++++++++++++ docker/my-dojo/install/install-scripts.sh | 3 + docker/my-dojo/install/upgrade-scripts.sh | 3 + docker/my-dojo/node/keys.index.js | 12 +- docker/my-dojo/overrides/indexer.install.yaml | 27 +++ 11 files changed, 361 insertions(+), 2 deletions(-) create mode 100644 docker/my-dojo/conf/docker-indexer.conf.tpl create mode 100644 docker/my-dojo/indexer/Dockerfile create mode 100644 docker/my-dojo/indexer/restart.sh create mode 100644 docker/my-dojo/indexer/wait-for-it.sh create mode 100644 docker/my-dojo/overrides/indexer.install.yaml diff --git a/docker/my-dojo/.env b/docker/my-dojo/.env index ea9f6ef..8841053 100644 --- a/docker/my-dojo/.env +++ b/docker/my-dojo/.env @@ -16,6 +16,7 @@ DOJO_BITCOIND_VERSION_TAG=1.2.0 DOJO_NODEJS_VERSION_TAG=1.3.0 DOJO_NGINX_VERSION_TAG=1.3.0 DOJO_TOR_VERSION_TAG=1.2.0 +DOJO_INDEXER_VERSION_TAG=1.0.0 ######################################### @@ -57,3 +58,10 @@ NODE_PREFIX_STATUS_PUSHTX=status NODE_TRACKER_MEMPOOL_PERIOD=10000 NODE_TRACKER_UNCONF_TXS_PERIOD=300000 + + +######################################### +# INDEXER +######################################### + +INDEXER_BATCH_SIZE=10 diff --git a/docker/my-dojo/conf/docker-indexer.conf.tpl b/docker/my-dojo/conf/docker-indexer.conf.tpl new file mode 100644 index 0000000..d7ad83c --- /dev/null +++ b/docker/my-dojo/conf/docker-indexer.conf.tpl @@ -0,0 +1,42 @@ +######################################### +# CONFIGURATION OF A LOCAL INDEXER +######################################### + +# Install and run a local indexer inside Docker +# Set this option to 'off' for using an indexer hosted outside of Docker +# or when using a different data source (local bitcoind, OXT) +# Value: on | off +INDEXER_INSTALL=off + +# IP address of the local indexer used by Dojo +# Set value to 172.28.1.6 if INDEXER_INSTALL is set to 'on' +# Type: string +INDEXER_IP=172.28.1.6 + +# Port of the RPC API +# Set value to 50001 if INDEXER_INSTALL is set to 'on' +# Type: integer +INDEXER_RPC_PORT=50001 + +# Support of batch requests by the local indexer +# Set value to inactive if INDEXER_INSTALL is set to 'on' +# Value: active | inactive +INDEXER_BATCH_SUPPORT=inactive + + +# +# EXPERT SETTINGS +# (ACTIVE IF INDEXER_INSTALL IS SET TO ON) +# + +# Number of blocks to get in one JSONRPC request from bitcoind +# Type: integer +INDEXER_BATCH_SIZE=10 + +# Total size of block txids to cache (in MB) +# Type: integer +INDEXER_BLK_TXIDS_CACHE_SIZE_MB=10 + +# Number of transactions to lookup before returning an error +# Type: integer +INDEXER_TXID_LIMIT=501 \ No newline at end of file diff --git a/docker/my-dojo/conf/docker-node.conf.tpl b/docker/my-dojo/conf/docker-node.conf.tpl index 403ed36..e7adcfa 100644 --- a/docker/my-dojo/conf/docker-node.conf.tpl +++ b/docker/my-dojo/conf/docker-node.conf.tpl @@ -21,7 +21,7 @@ NODE_ADMIN_KEY=myAdminKey NODE_JWT_SECRET=myJwtSecret # Indexer or third-party service used for imports and rescans of addresses -# Values: local_bitcoind | third_party_explorer +# Values: local_bitcoind | local_indexer | third_party_explorer NODE_ACTIVE_INDEXER=local_bitcoind # FEE TYPE USED FOR FEES ESTIMATIONS BY BITCOIND diff --git a/docker/my-dojo/dojo.sh b/docker/my-dojo/dojo.sh index d5f5620..b0e280c 100755 --- a/docker/my-dojo/dojo.sh +++ b/docker/my-dojo/dojo.sh @@ -14,6 +14,7 @@ source_file() { } # Source config files +source_file "$DIR/conf/docker-indexer.conf" source_file "$DIR/conf/docker-bitcoind.conf" source_file "$DIR/conf/docker-common.conf" source_file "$DIR/.env" @@ -33,6 +34,10 @@ select_yaml_files() { fi fi + if [ "$INDEXER_INSTALL" == "on" ]; then + yamlFiles="$yamlFiles -f $DIR/overrides/indexer.install.yaml" + fi + # Return yamlFiles echo "$yamlFiles" } @@ -114,6 +119,7 @@ uninstall() { docker image rm samouraiwallet/dojo-nodejs:"$DOJO_NODEJS_VERSION_TAG" docker image rm samouraiwallet/dojo-nginx:"$DOJO_NGINX_VERSION_TAG" docker image rm samouraiwallet/dojo-tor:"$DOJO_TOR_VERSION_TAG" + docker image rm samouraiwallet/dojo-indexer:"$DOJO_INDEXER_VERSION_TAG" docker volume prune } @@ -136,6 +142,7 @@ clean() { del_images_for samouraiwallet/dojo-nodejs "$DOJO_NODEJS_VERSION_TAG" del_images_for samouraiwallet/dojo-nginx "$DOJO_NGINX_VERSION_TAG" del_images_for samouraiwallet/dojo-tor "$DOJO_TOR_VERSION_TAG" + del_images_for samouraiwallet/dojo-indexer "$DOJO_INDEXER_VERSION_TAG" } # Upgrade @@ -194,6 +201,7 @@ logs_node() { logs() { source_file "$DIR/conf/docker-bitcoind.conf" + source_file "$DIR/conf/docker-indexer.conf" source_file "$DIR/conf/docker-common.conf" case $1 in @@ -212,6 +220,14 @@ logs() { echo -e "Command not supported for your setup.\nCause: Your Dojo is using an external bitcoind" fi ;; + indexer ) + if [ "$INDEXER_INSTALL" == "on" ]; then + yamlFiles=$(select_yaml_files) + eval "docker-compose $yamlFiles logs --tail=50 --follow indexer" + else + echo -e "Command not supported for your setup.\nCause: Your Dojo is not using an internal indexer" + fi + ;; tor ) docker-compose logs --tail=50 --follow tor ;; @@ -224,6 +240,9 @@ logs() { if [ "$BITCOIND_INSTALL" == "on" ]; then services="$services bitcoind" fi + if [ "$INDEXER_INSTALL" == "on" ]; then + services="$services indexer" + fi eval "docker-compose $yamlFiles logs --tail=0 --follow $services" ;; esac @@ -251,6 +270,7 @@ help() { echo " dojo.sh logs bitcoind : display the logs of bitcoind" echo " dojo.sh logs db : display the logs of the MySQL database" echo " dojo.sh logs tor : display the logs of tor" + echo " dojo.sh logs indexer : display the logs of the internal indexer" echo " dojo.sh logs api : display the logs of the REST API (nodejs)" echo " dojo.sh logs tracker : display the logs of the Tracker (nodejs)" echo " dojo.sh logs pushtx : display the logs of the pushTx API (nodejs)" diff --git a/docker/my-dojo/indexer/Dockerfile b/docker/my-dojo/indexer/Dockerfile new file mode 100644 index 0000000..259f827 --- /dev/null +++ b/docker/my-dojo/indexer/Dockerfile @@ -0,0 +1,46 @@ +FROM rust:1.37.0-slim + +ENV INDEXER_HOME /home/indexer +ENV INDEXER_VERSION 0..1.0 +ENV INDEXER_URL https://github.com/Samourai-Wallet/addrindexrs.git + +RUN apt-get update && \ + apt-get install -y clang cmake git && \ + apt-get install -y libsnappy-dev + +# Create group and user indexer +RUN addgroup --system -gid 1109 indexer && \ + adduser --system --ingroup indexer -uid 1106 indexer + +# Create data directory +RUN mkdir "$INDEXER_HOME/addrindexrs" && \ + chown -h indexer:indexer "$INDEXER_HOME/addrindexrs" + +# Copy restart script +COPY ./restart.sh /restart.sh +RUN chown indexer:indexer /restart.sh && \ + chmod 777 /restart.sh + +# Copy wait-for-it script +COPY ./wait-for-it.sh /wait-for-it.sh +RUN chown indexer:indexer /wait-for-it.sh && \ + chmod u+x /wait-for-it.sh && \ + chmod g+x /wait-for-it.sh + +USER indexer + +# Install addrindexrs +RUN cd "$INDEXER_HOME" && \ + git clone "$INDEXER_URL" "$INDEXER_HOME/addrindexrs" && \ + cd addrindexrs && \ + git checkout "master" +# TODO uncomment +# git checkout "tags/v$INDEXER_VERSION" + +RUN cd "$INDEXER_HOME/addrindexrs" && \ + cargo build --release && \ + cargo install --path . + +EXPOSE 50001 + +STOPSIGNAL SIGINT diff --git a/docker/my-dojo/indexer/restart.sh b/docker/my-dojo/indexer/restart.sh new file mode 100644 index 0000000..0db33b1 --- /dev/null +++ b/docker/my-dojo/indexer/restart.sh @@ -0,0 +1,22 @@ +#!/bin/bash +set -e + +indexer_options=( + -vvvv + --index-batch-size="$INDEXER_BATCH_SIZE" + --jsonrpc-import + --db-dir="/home/indexer/db" + --indexer-rpc-addr="172.28.1.6:50001" + --daemon-rpc-addr="$BITCOIND_IP:$BITCOIND_RPC_PORT" + --cookie="$BITCOIND_RPC_USER:$BITCOIND_RPC_PASSWORD" + --txid-limit="$INDEXER_TXID_LIMIT" + --blocktxids-cache-size-mb="$INDEXER_BLK_TXIDS_CACHE_SIZE_MB" +) + +if [ "$COMMON_BTC_NETWORK" == "testnet" ]; then + bitcoind_options+=(--network="testnet") +else + bitcoind_options+=(--network="mainnet") +fi + +addrindexrs "${indexer_options[@]}" diff --git a/docker/my-dojo/indexer/wait-for-it.sh b/docker/my-dojo/indexer/wait-for-it.sh new file mode 100644 index 0000000..071c2be --- /dev/null +++ b/docker/my-dojo/indexer/wait-for-it.sh @@ -0,0 +1,178 @@ +#!/usr/bin/env bash +# Use this script to test if a given TCP host/port are available + +WAITFORIT_cmdname=${0##*/} + +echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } + +usage() +{ + cat << USAGE >&2 +Usage: + $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args] + -h HOST | --host=HOST Host or IP under test + -p PORT | --port=PORT TCP port under test + Alternatively, you specify the host and port as host:port + -s | --strict Only execute subcommand if the test succeeds + -q | --quiet Don't output any status messages + -t TIMEOUT | --timeout=TIMEOUT + Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit 1 +} + +wait_for() +{ + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + else + echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout" + fi + WAITFORIT_start_ts=$(date +%s) + while : + do + if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then + nc -z $WAITFORIT_HOST $WAITFORIT_PORT + WAITFORIT_result=$? + else + (echo > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1 + WAITFORIT_result=$? + fi + if [[ $WAITFORIT_result -eq 0 ]]; then + WAITFORIT_end_ts=$(date +%s) + echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds" + break + fi + sleep 1 + done + return $WAITFORIT_result +} + +wait_for_wrapper() +{ + # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 + if [[ $WAITFORIT_QUIET -eq 1 ]]; then + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + else + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + fi + WAITFORIT_PID=$! + trap "kill -INT -$WAITFORIT_PID" INT + wait $WAITFORIT_PID + WAITFORIT_RESULT=$? + if [[ $WAITFORIT_RESULT -ne 0 ]]; then + echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + fi + return $WAITFORIT_RESULT +} + +# process arguments +while [[ $# -gt 0 ]] +do + case "$1" in + *:* ) + WAITFORIT_hostport=(${1//:/ }) + WAITFORIT_HOST=${WAITFORIT_hostport[0]} + WAITFORIT_PORT=${WAITFORIT_hostport[1]} + shift 1 + ;; + --child) + WAITFORIT_CHILD=1 + shift 1 + ;; + -q | --quiet) + WAITFORIT_QUIET=1 + shift 1 + ;; + -s | --strict) + WAITFORIT_STRICT=1 + shift 1 + ;; + -h) + WAITFORIT_HOST="$2" + if [[ $WAITFORIT_HOST == "" ]]; then break; fi + shift 2 + ;; + --host=*) + WAITFORIT_HOST="${1#*=}" + shift 1 + ;; + -p) + WAITFORIT_PORT="$2" + if [[ $WAITFORIT_PORT == "" ]]; then break; fi + shift 2 + ;; + --port=*) + WAITFORIT_PORT="${1#*=}" + shift 1 + ;; + -t) + WAITFORIT_TIMEOUT="$2" + if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi + shift 2 + ;; + --timeout=*) + WAITFORIT_TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + WAITFORIT_CLI=("$@") + break + ;; + --help) + usage + ;; + *) + echoerr "Unknown argument: $1" + usage + ;; + esac +done + +if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then + echoerr "Error: you need to provide a host and port to test." + usage +fi + +WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15} +WAITFORIT_STRICT=${WAITFORIT_STRICT:-0} +WAITFORIT_CHILD=${WAITFORIT_CHILD:-0} +WAITFORIT_QUIET=${WAITFORIT_QUIET:-0} + +# check to see if timeout is from busybox? +WAITFORIT_TIMEOUT_PATH=$(type -p timeout) +WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH) +if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then + WAITFORIT_ISBUSY=1 + WAITFORIT_BUSYTIMEFLAG="-t" + +else + WAITFORIT_ISBUSY=0 + WAITFORIT_BUSYTIMEFLAG="" +fi + +if [[ $WAITFORIT_CHILD -gt 0 ]]; then + wait_for + WAITFORIT_RESULT=$? + exit $WAITFORIT_RESULT +else + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + wait_for_wrapper + WAITFORIT_RESULT=$? + else + wait_for + WAITFORIT_RESULT=$? + fi +fi + +if [[ $WAITFORIT_CLI != "" ]]; then + if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then + echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess" + exit $WAITFORIT_RESULT + fi + exec "${WAITFORIT_CLI[@]}" +else + exit $WAITFORIT_RESULT +fi diff --git a/docker/my-dojo/install/install-scripts.sh b/docker/my-dojo/install/install-scripts.sh index 88b6bcd..d6ee261 100755 --- a/docker/my-dojo/install/install-scripts.sh +++ b/docker/my-dojo/install/install-scripts.sh @@ -52,6 +52,9 @@ init_config_files() { cp ./conf/docker-tor.conf.tpl ./conf/docker-tor.conf echo "Initialized docker-tor.conf" + cp ./conf/docker-indexer.conf.tpl ./conf/docker-indexer.conf + echo "Initialized docker-indexer.conf" + # Initialize config files for nginx and the maintenance tool if [ "$COMMON_BTC_NETWORK" == "testnet" ]; then cp ./nginx/testnet.conf ./nginx/dojo.conf diff --git a/docker/my-dojo/install/upgrade-scripts.sh b/docker/my-dojo/install/upgrade-scripts.sh index 40e735a..bd84026 100755 --- a/docker/my-dojo/install/upgrade-scripts.sh +++ b/docker/my-dojo/install/upgrade-scripts.sh @@ -48,6 +48,9 @@ update_config_files() { update_config_file ./conf/docker-tor.conf ./conf/docker-tor.conf.tpl echo "Initialized docker-tor.conf" + update_config_file ./conf/docker-indexer.conf ./conf/docker-indexer.conf.tpl + echo "Initialized docker-indexer.conf" + # Initialize config files for nginx and the maintenance tool if [ "$COMMON_BTC_NETWORK" == "testnet" ]; then cp ./nginx/testnet.conf ./nginx/dojo.conf diff --git a/docker/my-dojo/node/keys.index.js b/docker/my-dojo/node/keys.index.js index 300a1a5..a6aae97 100644 --- a/docker/my-dojo/node/keys.index.js +++ b/docker/my-dojo/node/keys.index.js @@ -156,8 +156,18 @@ module.exports = { */ indexer: { // Active indexer - // Values: local_bitcoind | third_party_explorer + // Values: local_bitcoind | local_indexer | third_party_explorer active: process.env.NODE_ACTIVE_INDEXER, + // Local indexer + localIndexer: { + // IP address or hostname of the local indexer + host: process.env.INDEXER_IP, + // Port + port: parseInt(process.env.INDEXER_RPC_PORT), + // Support of batch requests by the local indexer + // Values: active | inactive + batchRequests: process.env.INDEXER_BATCH_SUPPORT + }, // Use a SOCKS5 proxy for all communications with external services // Values: null if no socks5 proxy used, otherwise the url of the socks5 proxy socks5Proxy: 'socks5h://172.28.1.4:9050', diff --git a/docker/my-dojo/overrides/indexer.install.yaml b/docker/my-dojo/overrides/indexer.install.yaml new file mode 100644 index 0000000..e130624 --- /dev/null +++ b/docker/my-dojo/overrides/indexer.install.yaml @@ -0,0 +1,27 @@ +version: "3.2" + +services: + indexer: + image: "samouraiwallet/dojo-indexer:${DOJO_INDEXER_VERSION_TAG}" + container_name: indexer + build: + context: ./indexer + env_file: + - ./.env + - ./conf/docker-common.conf + - ./conf/docker-bitcoind.conf + - ./conf/docker-indexer.conf + restart: on-failure + command: "/wait-for-it.sh tor:9050 --timeout=360 --strict -- /restart.sh" + expose: + - "50001" + volumes: + - data-indexer:/home/indexer/addrindexrs + depends_on: + - tor + networks: + dojonet: + ipv4_address: 172.28.1.6 + +volumes: + data-indexer: