Browse Source

Merge pull request #114 from Samourai-Wallet/develop

merge develop into master for v1.4.0
umbrel v1.4.0
kenshin samourai 5 years ago
committed by GitHub
parent
commit
1db12e6a96
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
  1. 3
      README.md
  2. 74
      RELEASES.md
  3. 41
      accounts/support-rest-api.js
  4. 81
      doc/DOCKER_setup.md
  5. 57
      doc/DOCKER_synology_setup.md
  6. BIN
      doc/static/synology_containers.png
  7. BIN
      doc/static/synology_docker-package.png
  8. BIN
      doc/static/synology_install_complete.png
  9. 11
      docker/my-dojo/.env
  10. 3
      docker/my-dojo/bitcoin/Dockerfile
  11. 10
      docker/my-dojo/bitcoin/restart.sh
  12. 16
      docker/my-dojo/conf/docker-explorer.conf.tpl
  13. 12
      docker/my-dojo/docker-compose.yaml
  14. 105
      docker/my-dojo/dojo.sh
  15. 33
      docker/my-dojo/explorer/Dockerfile
  16. 21
      docker/my-dojo/explorer/restart.sh
  17. 18
      docker/my-dojo/install/install-scripts.sh
  18. 22
      docker/my-dojo/install/upgrade-scripts.sh
  19. 20
      docker/my-dojo/nginx/Dockerfile
  20. 15
      docker/my-dojo/nginx/explorer.conf
  21. 4
      docker/my-dojo/node/Dockerfile
  22. 2
      docker/my-dojo/overrides/bitcoind.install.yaml
  23. 28
      docker/my-dojo/overrides/explorer.install.yaml
  24. 9
      docker/my-dojo/tor/Dockerfile
  25. 7
      docker/my-dojo/tor/restart.sh
  26. 2
      keys/index-example.js
  27. 4
      lib/bitcoind-rpc/rpc-client.js
  28. 134
      lib/bitcoind-rpc/transactions.js
  29. 58
      lib/db/mysql-db-wrapper.js
  30. 156
      lib/remote-importer/remote-importer.js
  31. 2
      lib/remote-importer/sources.js
  32. 18
      lib/util.js
  33. 2
      package-lock.json
  34. 2
      package.json
  35. 17
      static/admin/css/style.css
  36. 9
      static/admin/lib/api-wrapper.js
  37. 18
      static/admin/tool/index.html
  38. 55
      static/admin/tool/index.js

3
README.md

@ -16,7 +16,8 @@ It provides in a single command the setup of a full Samourai backend composed of
* a bitcoin full node only accessible as an ephemeral Tor hidden service,
* the backend database,
* the backend modules with an API accessible as a static Tor hidden service,
* a maintenance tool accessible through a Tor web browser.
* a maintenance tool accessible through a Tor web browser,
* a block explorer ([BTC RPC Explorer](https://github.com/janoside/btc-rpc-explorer)) accessible through a Tor web browser.
See [the documentation](./doc/DOCKER_setup.md) for detailed setup instructions.

74
RELEASES.md

@ -2,11 +2,85 @@
## Releases ##
- [v1.4.0](#1_4_0)
- [v1.3.0](#1_3_0)
- [v1.2.0](#1_2_0)
- [v1.1.0](#1_1_0)
<a name="1_4_0"/>
## Samourai Dojo v1.4.0 ##
### Notable changes ###
#### Local block explorer ####
This release adds a new docker container hosting a local block explorer ([BTC RPC Explorer](https://github.com/janoside/btc-rpc-explorer)).
Access to the block explorer is secured by a password defined in /docker/my-dojo/conf/docker-explorer.conf (see `EXPLORER_KEY` configuration parameter).
*Upgrade procedure*
```
# Stop your Dojo
# Download the Dojo archive for this release
# Override the content of your <dojo_dir> with the content of the Dojo archive
# Edit <dojo_dir>/docker/my-dojo/conf/docker-explorer.conf.tpl and set the value of `EXPLORER_KEY` with a custom password.
# Launch the upgrade of your Dojo with: dojo.sh upgrade
```
This local block explorer is available as a Tor hidden service. Its static onion address can be retrieved with the command
```
dojo.sh onion
```
#### Autostart of Dojo ####
Starting with this release, Dojo is automatically launched when the docker daemon starts.
### Change log ###
#### MyDojo ####
- [#101](https://github.com/Samourai-Wallet/samourai-dojo/pull/101) add --auto and --nolog options to install and upgrade commands
- [#102](https://github.com/Samourai-Wallet/samourai-dojo/pull/102) improve performances of transactions imports
- [#107](https://github.com/Samourai-Wallet/samourai-dojo/pull/107) add optional block explorer
- [#108](https://github.com/Samourai-Wallet/samourai-dojo/pull/108) switch restart policies of containers to always
- [#109](https://github.com/Samourai-Wallet/samourai-dojo/pull/109) use port 80 of keyservers
- [#110](https://github.com/Samourai-Wallet/samourai-dojo/pull/110) replace keyserver
- [#111](https://github.com/Samourai-Wallet/samourai-dojo/pull/111) enable autostart of dojo
- [#113](https://github.com/Samourai-Wallet/samourai-dojo/pull/113) check if dojo is running (start and stop commands)
#### Bug fixes ####
- [#100](https://github.com/Samourai-Wallet/samourai-dojo/pull/100) fix issue caused by sed -i on osx
#### Documentation ####
- [#99](https://github.com/Samourai-Wallet/samourai-dojo/pull/99) doc: installation of dojo on synology
- [b12d24d](https://github.com/Samourai-Wallet/samourai-dojo/commit/b12d24d088a95023a8e1c9e8a1b1c4b40491d4a7) update readme
### Credits ###
- anwfr
- jochemin
- kenshin-samourai
- LaurentMT
<a name="1_3_0"/>
## Samourai Dojo v1.3.0 ##

41
accounts/support-rest-api.js

@ -4,6 +4,7 @@
*/
'use strict'
const fs = require('fs')
const validator = require('validator')
const bodyParser = require('body-parser')
const errors = require('../lib/errors')
@ -68,6 +69,13 @@ class SupportRestApi {
HttpServer.sendAuthError
)
this.httpServer.app.get(
`/${keys.prefixes.support}/pairing/explorer`,
authMgr.checkHasAdminProfile.bind(authMgr),
this.getPairingExplorer.bind(this),
HttpServer.sendAuthError
)
this.httpServer.app.get(
`/${keys.prefixes.support}/pairing`,
authMgr.checkHasAdminProfile.bind(authMgr),
@ -299,6 +307,39 @@ class SupportRestApi {
}
}
/**
* Get pairing info for the local block explorer
*/
async getPairingExplorer(req, res) {
try {
let url = ''
if (process.env.EXPLORER_INSTALL == 'on') {
try {
url = fs.readFileSync('/var/lib/tor/hsv3explorer/hostname', 'utf8')
url = url.replace('\n', '')
} catch(e) {
Logger.error(e, 'SupportRestApi.getPairing() : Cannot read explorer onion address')
}
}
const ret = {
'pairing': {
'type': 'explorer.btcRpcExplorer',
'url': url,
'key': process.env.EXPLORER_KEY
}
}
HttpServer.sendRawData(res, JSON.stringify(ret, null, 2))
} catch(e) {
const ret = {
status: 'error'
}
Logger.error(e, 'SupportRestApi.getPairingExplorer() : Support pairing error')
HttpServer.sendError(res, JSON.stringify(ret, null, 2))
} finally {
debugApi && Logger.info(`Completed GET /pairing/explorer`)
}
}
/**
* Validate arguments related to GET xpub info requests
* @param {object} req - http request object

81
doc/DOCKER_setup.md

@ -3,8 +3,9 @@
MyDojo is a set of Docker containers providing a full Samourai backend composed of:
* a bitcoin full node accessible as an ephemeral Tor hidden service,
* a backend database,
* a backend modules with an API accessible as a static Tor hidden service,
* a maintenance tool accessible through a Tor web browser.
* backend modules with an API accessible as a static Tor hidden service,
* a maintenance tool accessible through a Tor web browser,
* a block explorer ([BTC RPC Explorer](https://github.com/janoside/btc-rpc-explorer)) accessible as a static Tor hidden service.
## Table of Content ##
@ -15,6 +16,7 @@ MyDojo is a set of Docker containers providing a full Samourai backend composed
- [Upgrade procedure](#upgrade)
- [Dojo shell script](#shell_script)
- [Dojo maintenance tool](#maintenance_tool)
- [Block explorer](#explorer)
- [Pairing your wallet to your Dojo](#pairing)
- [Network connections](#network)
@ -38,21 +40,23 @@ MyDojo is a set of Docker containers providing a full Samourai backend composed
Host machine | (Tor hidden services)
______________________________ | _____________________________
| | |
| ------------------- |
| ------------------- dmznet |
| | Tor Container | |
| ------------------- |
| | | |
| ------------------- | |
| | Nginx Container | | dmznet |
| | Nginx Container | | |
| ------------------- | |
|- - - - - - - - - - - | - - - - - - - | - - - - - - - - - - - |
| -------------------- -------------------- |
|- - - - - - - - - - - | - - -|- - - - | - - - - - - - - - - - |
| -------------------- | -------------------- |
| | Nodejs Container | ------ | Bitcoind Container | |
| -------------------- | -------------------- |
| | | | |
| -------------------- | -------------------- |
| | MySQL Container | ---- | BTC RPC Explorer | |
| -------------------- -------------------- |
| | |
| ------------------- |
| | MySQL Container | dojonet |
| ------------------- |
| |
| dojonet |
|______________________________________________________________|
@ -73,11 +77,13 @@ MyDojo is a set of Docker containers providing a full Samourai backend composed
## Configuration files ##
Each new release of Dojo is packaged with 4 template files stored in the `<dojo_dir>/docker/my-dojo/conf` directory:
Each new release of Dojo is packaged with 6 template files stored in the `<dojo_dir>/docker/my-dojo/conf` directory:
- docker-common.conf.tpl
- docker-bitcoin.conf.tpl
- docker-explorer.conf.tpl
- docker-mysql.conf.tpl
- docker-node.conf.tpl
- docker-tor.conf.tpl
These template files define default values for configuration options of your Dojo.
@ -98,6 +104,8 @@ Most options provided in the configuration files can be later modified. New valu
For MacOS, see this detailed [installation guide](./DOCKER_mac_setup.MD).
For Synology, see this detailed [installation guide](./DOCKER_synology_setup.md).
This procedure allows to install a new Dojo from scratch.
* Install [Docker and Docker Compose](https://docs.docker.com/compose/install/) on the host machine and check that your installation is working.
@ -125,6 +133,7 @@ This procedure allows to install a new Dojo from scratch.
* `MYSQL_ROOT_PASSWORD` = password protecting the root account of MySQL,
* `MYSQL_USER` = login of the account used to access the database of your Dojo,
* `MYSQL_PASSWORD` = password of the account used to access the database of your Dojo.
Note: These values can't be changed after the first installation.
* Edit docker-node.conf.tpl and provide a new value for the following parameters:
* `NODE_API_KEY` = API key which will be required from your Samourai Wallet / Sentinel for its interactions with the API of your Dojo,
@ -132,6 +141,11 @@ This procedure allows to install a new Dojo from scratch.
* `NODE_JWT_SECRET` = secret used by your Dojo for the initialization of a cryptographic key signing Json Web Tokens.
These parameters will protect the access to your Dojo. Be sure to provide alphanumeric values with enough entropy.
* Edit docker-explorer.conf.tpl and provide a new value for the following parameter:
* `EXPLORER_KEY` = password that will be required to access the block explorer,
* If you want to deactivate the block explorer, set the value of `EXPLORER_INSTALL` to `off`.
See this [section](#explorer) for more details about the block explorer.
* Dojo provides a few additional settings for advanced setups:
* static onion address for your full node,
* bitcoind RPC API exposed to external apps,
@ -149,7 +163,7 @@ This procedure allows to install a new Dojo from scratch.
./dojo.sh install
```
Docker and Docker Compose are going to build the images and containers of your Dojo. This operation will take a few minutes (download and setup of all required software components). After completion, your Dojo will be launched and will begin the initialization of the full node (Bitcoin Initial Block Download and syncing of the database). This step will take several hours/days according to the specs of your machine. Be patient. Use CTRL+C to stop the display of the full logs.
Docker and Docker Compose are going to build the images and containers of your Dojo. This operation will take several minutes (download and setup of all required software components). After completion, your Dojo will be launched and will begin the initialization of the full node (Bitcoin Initial Block Download and syncing of the database). This step will take several hours/days according to the specs of your machine. Be patient. Use CTRL+C to stop the display of the full logs.
* Monitor the progress made for the initialization of the database with this command displaying the logs of the tracker
@ -161,7 +175,7 @@ Docker and Docker Compose are going to build the images and containers of your D
Exit the logs with CTRL+C when the syncing of the database has completed.
* Retrieve the Tor onion addresses (v2 and v3) of the API of your Dojo
* Retrieve the Tor onion addresses (v3) of the API and block explorer of your Dojo
```
./dojo.sh onion
@ -196,7 +210,7 @@ This procedure allows to upgrade your Dojo with a new version.
Docker and Docker Compose are going to build new images and containers for your Dojo. After completion, the updated version of your Dojo will be launched automatically.
Note: The upgrade process will override all manual modifications of the files stored under the `<dojo_dir>` directory with an exception for the three configuration files stored in the `<dojo_dir>/docker/my-dojo/conf` directory.
Note: The upgrade process will override all manual modifications of the files stored under the `<dojo_dir>` directory with an exception for the configuration files stored in the `<dojo_dir>/docker/my-dojo/conf` directory.
<a name="shell_script"/>
@ -229,13 +243,14 @@ Available commands:
dojo.sh logs tracker : display the logs of the Tracker (nodejs)
dojo.sh logs pushtx : display the logs of the pushTx API (nodejs)
dojo.sh logs pushtx-orchest : display the logs of the Orchestrator (nodejs)
dojo.sh logs explorer : display the logs of the Explorer
Available options (for api, tracker, pushtx and pushtx-orchest modules):
Available options (for api, tracker, pushtx, pushtx-orchest and explorer modules):
-d [VALUE] : select the type of log to be displayed.
VALUE can be output (default) or error.
-n [VALUE] : display the last VALUE lines
onion Display the Tor onion address allowing your wallet to access your Dojo.
onion Display the Tor onion addresses allowing to access the API, maintenance tool and block explorer of your Dojo.
restart Restart your Dojo.
@ -262,6 +277,27 @@ The maintenance tool requires that you allow javascript for the site.
Sign in with the value entered for `NODE_ADMIN_KEY`.
<a name="explorer"/>
## Block explorer ##
A block explorer ([BTC RPC Explorer](https://github.com/janoside/btc-rpc-explorer)) is accessible through your Tor browser.
You can retrieve the onion address of the block explorer with the command
```
./dojo.sh onion
```
Sign in with a login (can be any value) and the password set in your Dojo configuration (value entered for `EXPLORER_KEY`).
Notes:
* Current version doesn't support the display of detailed information for a Bitcoin address,
* Calls to the RPC API of your bitcoind are deactivated.
<a name="pairing"/>
## Pairing your wallet to your Dojo ##
@ -270,11 +306,20 @@ Once the database has finished syncing, you can pair your Samourai Wallet with y
1. Open the maintenance tool in a Tor browser (Tor v3 onion address) and sign in with your admin key.
2. Get your smartphone and launch the Samourai Wallet app. Scan the QRCode displayed in the "Pairing" tab of the maintenance tool.
2. Get your smartphone and launch the Samourai Wallet app. Scan the first QRCode displayed in the "Pairing" tab of the maintenance tool.
If you experience any problems when pairing, try re-installing the app and select "Connect to existing Dojo" from the [⋮] menu.
## Pairing your wallet to your local block explorer (coming "soon") ##
You can pair your Samourai Wallet with your local block explorer in 2 steps:
1. Open the maintenance tool in a Tor browser (Tor v3 onion address) and sign in with your admin key.
2. Get your smartphone and launch the Samourai Wallet app. Scan the second QRCode displayed in the "Pairing" tab of the maintenance tool.
<a name="network"/>
## Network connections ##
@ -285,6 +330,8 @@ If OXT is selected as the default source for imports, OXT clearnet API is access
The maintenance tool is accessed as a Tor hidden service (static onion address).
The block explorer is accessed as a Tor hidden service (static onion address).
The Bitcoin node only allows incoming connections from Tor (ephemeral onion address).
The Bitcoin node attempts outgoing connections to both Tor and clearnet nodes (through the Tor local proxy).

57
doc/DOCKER_synology_setup.md

@ -0,0 +1,57 @@
# Installation of Dojo on Synology
This will install Dojo on your Synology hardware.
## Table of Content ##
- [Requirements](#requirements)
- [Install procedure](#install)
<a name="requirements"/>
## Requirements ##
* Synology hardware connected 24/7 to internet
* Disk: 500GB (minimal) / 1TB (recommended)
* RAM: 4GB (minimal)
<a name="install"/>
## Install procedure
- Connect to Synology web ui as administrator
- Open `Package center`, search for `Docker`, click `Install`.
![](./static/synology_docker-package.png)
- Open SSH terminal to your Synology
- Follow `first-time install procedure` in [DOCKER_setup.md](DOCKER_setup.md#install), but skip the first two steps:
```
* Install Docker and Docker Compose on the host machine and check that your installation is working.
* Install Tor Browser on the host machine.
```
You can use `/volume1/dojo` as `<dojo_dir>`.
- Install will complete with the following warnings, which you can safely ignore:
```
Attaching to nginx, nodejs, bitcoind, tor, db
nginx | WARNING: no logs are available with the 'db' log driver
nodejs | WARNING: no logs are available with the 'db' log driver
bitcoind | WARNING: no logs are available with the 'db' log driver
tor | WARNING: no logs are available with the 'db' log driver
db | WARNING: no logs are available with the 'db' log driver
```
![](./static/synology_install_complete.png)
## Dojo status & logs
- Connect to Synology web ui as administrator
- Open `Docker`, then `Container`.
![](./static/synology_containers.png)
- You will see the following containers running:
* bitcoind
* db
* nginx
* nodejs
* tor
- Select a container (ie `bitcoind`), click `Detail`, `Log` to see container's logs in real time

BIN
doc/static/synology_containers.png

Binary file not shown.

After

Width:  |  Height:  |  Size: 669 KiB

BIN
doc/static/synology_docker-package.png

Binary file not shown.

After

Width:  |  Height:  |  Size: 99 KiB

BIN
doc/static/synology_install_complete.png

Binary file not shown.

After

Width:  |  Height:  |  Size: 143 KiB

11
docker/my-dojo/.env

@ -10,12 +10,13 @@
COMPOSE_CONVERT_WINDOWS_PATHS=1
DOJO_VERSION_TAG=1.3.0
DOJO_VERSION_TAG=1.4.0
DOJO_DB_VERSION_TAG=1.1.0
DOJO_BITCOIND_VERSION_TAG=1.3.0
DOJO_NODEJS_VERSION_TAG=1.3.0
DOJO_NGINX_VERSION_TAG=1.3.0
DOJO_TOR_VERSION_TAG=1.2.0
DOJO_BITCOIND_VERSION_TAG=1.4.0
DOJO_NODEJS_VERSION_TAG=1.4.0
DOJO_NGINX_VERSION_TAG=1.4.0
DOJO_TOR_VERSION_TAG=1.3.0
DOJO_EXPLORER_VERSION_TAG=1.0.0
#########################################

3
docker/my-dojo/bitcoin/Dockerfile

@ -9,6 +9,7 @@ ENV BITCOIN_VERSION 0.19.0.1
ENV BITCOIN_URL https://bitcoincore.org/bin/bitcoin-core-0.19.0.1/bitcoin-0.19.0.1-x86_64-linux-gnu.tar.gz
ENV BITCOIN_SHA256 732cc96ae2e5e25603edf76b8c8af976fe518dd925f7e674710c6c8ee5189204
ENV BITCOIN_ASC_URL https://bitcoincore.org/bin/bitcoin-core-0.19.0.1/SHA256SUMS.asc
ENV BITCOIN_PGP_KS_URI hkp://keyserver.ubuntu.com:80
ENV BITCOIN_PGP_KEY 01EA5486DE18A882D4C2684590C8019E36C2E964
RUN set -ex && \
@ -21,7 +22,7 @@ RUN set -ex && \
cd /tmp && \
wget -qO bitcoin.tar.gz "$BITCOIN_URL" && \
echo "$BITCOIN_SHA256 bitcoin.tar.gz" | sha256sum -c - && \
gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "$BITCOIN_PGP_KEY" && \
gpg --batch --keyserver "$BITCOIN_PGP_KS_URI" --recv-keys "$BITCOIN_PGP_KEY" && \
wget -qO bitcoin.asc "$BITCOIN_ASC_URL" && \
gpg --batch --verify bitcoin.asc && \
tar -xzvf bitcoin.tar.gz -C /usr/local --strip-components=1 --exclude=*-qt && \

10
docker/my-dojo/bitcoin/restart.sh

@ -40,3 +40,13 @@ if [ "$COMMON_BTC_NETWORK" == "testnet" ]; then
fi
bitcoind "${bitcoind_options[@]}"
exitCode=$?
if [ $exitCode -eq 0 ]; then
# Loop keeping the container up
# after bitcoind has been stopped
while true
do
tail -f /dev/null > /dev/null
done
fi

16
docker/my-dojo/conf/docker-explorer.conf.tpl

@ -0,0 +1,16 @@
#########################################
# CONFIGURATION OF EXPLORER CONTAINER
#########################################
# Install and run a block explorer inside Dojo (recommended)
# Value: on | off
EXPLORER_INSTALL=on
# Password required for accessing the block explorer
# (login can be anything)
# Keep this password secret!
# Provide a value with a high entropy!
# Type: alphanumeric
EXPLORER_KEY=myExplorerPassword

12
docker/my-dojo/docker-compose.yaml

@ -10,7 +10,7 @@ services:
env_file:
- ./.env
- ./conf/docker-mysql.conf
restart: on-failure
restart: always
expose:
- "3306"
volumes:
@ -30,8 +30,9 @@ services:
- ./conf/docker-common.conf
- ./conf/docker-mysql.conf
- ./conf/docker-bitcoind.conf
- ./conf/docker-explorer.conf
- ./conf/docker-node.conf
restart: on-failure
restart: always
command: "/home/node/app/wait-for-it.sh db:3306 --timeout=720 --strict -- /home/node/app/restart.sh"
expose:
- "8080"
@ -39,6 +40,7 @@ services:
- "8082"
volumes:
- data-nodejs:/data
- data-tor:/var/lib/tor
depends_on:
- db
networks:
@ -53,10 +55,11 @@ services:
env_file:
- ./.env
- ./conf/docker-common.conf
restart: on-failure
restart: always
command: "/wait-for node:8080 --timeout=720 -- nginx"
expose:
- "80"
- "9080"
volumes:
- data-nginx:/data
depends_on:
@ -74,8 +77,9 @@ services:
context: ./tor
env_file:
- ./.env
- ./conf/docker-explorer.conf
- ./conf/docker-tor.conf
restart: on-failure
restart: always
command: /restart.sh
volumes:
- data-tor:/var/lib/tor

105
docker/my-dojo/dojo.sh

@ -15,6 +15,7 @@ source_file() {
# Source config files
source_file "$DIR/conf/docker-bitcoind.conf"
source_file "$DIR/conf/docker-explorer.conf"
source_file "$DIR/conf/docker-common.conf"
source_file "$DIR/.env"
@ -33,6 +34,10 @@ select_yaml_files() {
fi
fi
if [ "$EXPLORER_INSTALL" == "on" ]; then
yamlFiles="$yamlFiles -f $DIR/overrides/explorer.install.yaml"
fi
# Return yamlFiles
echo "$yamlFiles"
}
@ -45,35 +50,68 @@ docker_up() {
# Start
start() {
docker_up --remove-orphans
# Check if dojo is running (check the db container)
isRunning=$(docker inspect --format="{{.State.Running}}" db 2> /dev/null)
if [ $? -eq 1 ] || [ "$isRunning" == "false" ]; then
docker_up --remove-orphans
else
echo "Dojo is already running."
fi
}
# Stop
stop() {
# Check if dojo is running (check the db container)
isRunning=$(docker inspect --format="{{.State.Running}}" db 2> /dev/null)
if [ $? -eq 1 ] || [ "$isRunning" == "false" ]; then
echo "Dojo is already stopped."
exit
fi
# Shutdown the bitcoin daemon
if [ "$BITCOIND_INSTALL" == "on" ]; then
# Renewal of bitcoind onion address
if [ "$BITCOIND_EPHEMERAL_HS" = "on" ]; then
docker exec -it tor rm -rf /var/lib/tor/hsv2bitcoind
fi
# Stop the bitcoin daemon
echo "Preparing shutdown of dojo. Please wait."
docker exec -it bitcoind bitcoin-cli \
-rpcconnect=bitcoind \
--rpcport=28256 \
--rpcuser="$BITCOIND_RPC_USER" \
--rpcpassword="$BITCOIND_RPC_PASSWORD" \
stop
echo "Preparing shutdown of dojo. Please wait."
bitcoindDown=$(timeout 3m docker wait bitcoind)
if [ $bitcoindDown -eq 0 ]; then
echo "Bitcoin server stopped."
else
# Check if the bitcoin daemon is still up
# wait 3mn max
i="0"
while [ $i -lt 18 ]
do
# Check if bitcoind rpc api is responding
timeout 5 docker exec -it bitcoind bitcoin-cli \
-rpcconnect=bitcoind \
--rpcport=28256 \
--rpcuser="$BITCOIND_RPC_USER" \
--rpcpassword="$BITCOIND_RPC_PASSWORD" \
getblockchaininfo > /dev/null
# rpc api is down
if [[ $? > 0 ]]; then
echo "Bitcoin server stopped."
break
fi
# Pause before next try
sleep 5
i=$[$i+1]
done
# Bitcoin daemon is still up
# => force close
if [ $i -eq 18 ]; then
echo "Force shutdown of Bitcoin server."
fi
fi
# Stop docker containers
yamlFiles=$(select_yaml_files)
eval "docker-compose $yamlFiles down"
eval "docker-compose $yamlFiles stop"
}
# Restart dojo
@ -88,17 +126,19 @@ install() {
launchInstall=1
if [ -z "$1" ]; then
if [ "$1" = "--auto" ]; then
launchInstall=0
else
get_confirmation
launchInstall=$?
else
launchInstall=0
fi
if [ $launchInstall -eq 0 ]; then
init_config_files
docker_up --remove-orphans
logs
if [ "$1" != "--nolog" ]; then
logs
fi
fi
}
@ -111,6 +151,7 @@ uninstall() {
docker image rm samouraiwallet/dojo-db:"$DOJO_DB_VERSION_TAG"
docker image rm samouraiwallet/dojo-bitcoind:"$DOJO_BITCOIND_VERSION_TAG"
docker image rm samouraiwallet/dojo-explorer:"$DOJO_EXPLORER_VERSION_TAG"
docker image rm samouraiwallet/dojo-nodejs:"$DOJO_NODEJS_VERSION_TAG"
docker image rm samouraiwallet/dojo-nginx:"$DOJO_NGINX_VERSION_TAG"
docker image rm samouraiwallet/dojo-tor:"$DOJO_TOR_VERSION_TAG"
@ -133,6 +174,7 @@ clean() {
docker image prune
del_images_for samouraiwallet/dojo-db "$DOJO_DB_VERSION_TAG"
del_images_for samouraiwallet/dojo-bitcoind "$DOJO_BITCOIND_VERSION_TAG"
del_images_for samouraiwallet/dojo-explorer "$DOJO_EXPLORER_VERSION_TAG"
del_images_for samouraiwallet/dojo-nodejs "$DOJO_NODEJS_VERSION_TAG"
del_images_for samouraiwallet/dojo-nginx "$DOJO_NGINX_VERSION_TAG"
del_images_for samouraiwallet/dojo-tor "$DOJO_TOR_VERSION_TAG"
@ -144,11 +186,11 @@ upgrade() {
launchUpgrade=1
if [ -z "$1" ]; then
if [ "$1" = "--auto" ]; then
launchUpgrade=0
else
get_confirmation
launchUpgrade=$?
else
launchUpgrade=0
fi
if [ $launchUpgrade -eq 0 ]; then
@ -160,15 +202,21 @@ upgrade() {
eval "docker-compose $yamlFiles build --no-cache"
docker_up --remove-orphans
update_dojo_db
logs
if [ "$1" != "--nolog" ]; then
logs
fi
fi
}
# Display the onion address
onion() {
if [ "$EXPLORER_INSTALL" == "on" ]; then
V3_ADDR_EXPLORER=$( docker exec -it tor cat /var/lib/tor/hsv3explorer/hostname )
echo "Explorer hidden service address (v3) = $V3_ADDR_EXPLORER"
fi
V2_ADDR=$( docker exec -it tor cat /var/lib/tor/hsv2dojo/hostname )
V3_ADDR=$( docker exec -it tor cat /var/lib/tor/hsv3dojo/hostname )
echo "API hidden service address (v3) = $V3_ADDR"
echo "API hidden service address (v2) = $V2_ADDR"
@ -192,6 +240,14 @@ logs_node() {
fi
}
logs_explorer() {
if [ $3 -eq 0 ]; then
docker exec -ti explorer tail -f /data/logs/$1-$2.log
else
docker exec -ti explorer tail -n $3 /data/logs/$1-$2.log
fi
}
logs() {
source_file "$DIR/conf/docker-bitcoind.conf"
source_file "$DIR/conf/docker-common.conf"
@ -218,12 +274,18 @@ logs() {
api | pushtx | pushtx-orchest | tracker )
logs_node $1 $2 $3
;;
explorer )
logs_explorer $1 $2 $3
;;
* )
yamlFiles=$(select_yaml_files)
services="nginx node tor db"
if [ "$BITCOIND_INSTALL" == "on" ]; then
services="$services bitcoind"
fi
if [ "$EXPLORER_INSTALL" == "on" ]; then
services="$services explorer"
fi
eval "docker-compose $yamlFiles logs --tail=0 --follow $services"
;;
esac
@ -255,8 +317,9 @@ help() {
echo " dojo.sh logs tracker : display the logs of the Tracker (nodejs)"
echo " dojo.sh logs pushtx : display the logs of the pushTx API (nodejs)"
echo " dojo.sh logs pushtx-orchest : display the logs of the pushTx Orchestrator (nodejs)"
echo " dojo.sh logs explorer : display the logs of the Explorer"
echo " "
echo " Available options (only available for api, tracker, pushtx and pushtx-orchest modules):"
echo " Available options (only available for api, tracker, pushtx, pushtx-orchest and explorer modules):"
echo " -d [VALUE] : select the type of log to be displayed."
echo " VALUE can be output (default) or error."
echo " -n [VALUE] : display the last VALUE lines"

33
docker/my-dojo/explorer/Dockerfile

@ -0,0 +1,33 @@
FROM node:8.12.0-stretch
ENV LOGS_DIR /data/logs
ENV APP_DIR /home/node/app
ENV EXPLORER_URL https://github.com/janoside/btc-rpc-explorer/archive
ENV EXPLORER_VERSION 1.1.5
# Create logs and apps directory
RUN mkdir -p "$LOGS_DIR" && \
chown -R node:node "$LOGS_DIR" && \
mkdir "$APP_DIR"
# Download the source code and install it
RUN set -ex && \
wget -qO explorer.tar.gz "$EXPLORER_URL/v$EXPLORER_VERSION.tar.gz" && \
tar -xzvf explorer.tar.gz -C "$APP_DIR/" --strip-components 1 && \
rm explorer.tar.gz && \
cd "$APP_DIR" && \
npm install --only=prod && \
chown -R node:node "$APP_DIR"
# Copy restart script
COPY ./restart.sh "$APP_DIR/restart.sh"
RUN chown node:node "$APP_DIR/restart.sh" && \
chmod u+x "$APP_DIR/restart.sh" && \
chmod g+x "$APP_DIR/restart.sh"
EXPOSE 3002
USER node

21
docker/my-dojo/explorer/restart.sh

@ -0,0 +1,21 @@
#!/bin/bash
cd /home/node/app
explorer_options=(
--port 3002
--host 172.28.1.7
--basic-auth-password "$EXPLORER_KEY"
--coin BTC
--bitcoind-host "$BITCOIND_IP"
--bitcoind-port "$BITCOIND_RPC_PORT"
--bitcoind-user "$BITCOIND_RPC_USER"
--bitcoind-pass "$BITCOIND_RPC_PASSWORD"
--no-rates
--privacy-mode
)
# Blacklist all functions provided by the RPC API
explorer_options+=(--rpc-blacklist "addnode,analyzepsbt,clearbanned,combinepsbt,combinerawtransaction,converttopsbt,createmultisig,createpsbt,createrawtransaction,decodepsbt,decoderawtransaction,decodescript,deriveaddresses,disconnectnode,echo,echojson,estimaterawfee,estimatesmartfee,finalizepsbt,generatetoaddress,generatetodescriptor,getaddednodeinfo,getbestblockhash,getblock,getblockchaininfo,getblockcount,getblockfilter,getblockhash,getblockheader,getblockstats,getblocktemplate,getchaintips,getchaintxstats,getconnectioncount,getdescriptorinfo,getdifficulty,getmemoryinfo,getmempoolancestors,getmempooldescendants,getmempoolentry,getmempoolinfo,getmininginfo,getnettotals,getnetworkhashps,getnetworkinfo,getnodeaddresses,getpeerinfo,getrawmempool,getrawtransaction,getrpcinfo,gettxout,gettxoutproof,gettxoutsetinfo,help,invalidateblock,joinpsbts,listbanned,logging,ping,preciousblock,prioritisetransaction,pruneblockchain,reconsiderblock,savemempool,scantxoutset,sendrawtransaction,setban,setmocktime,setnetworkactive,signmessagewithprivkey,signrawtransactionwithkey,stop,submitblock,submitheader,syncwithvalidationinterfacequeue,testmempoolaccept,uptime,utxoupdatepsbt,validateaddress,verifychain,verifymessage,verifytxoutproof,waitforblock,waitforblockheight,waitfornewblock")
node ./bin/cli.js "${explorer_options[@]}" > /data/logs/explorer-error.log 2> /data/logs/explorer-output.log

18
docker/my-dojo/install/install-scripts.sh

@ -6,6 +6,12 @@ else
source ./conf/docker-bitcoind.conf.tpl
fi
if [ -f ./conf/docker-explorer.conf ]; then
source ./conf/docker-explorer.conf
else
source ./conf/docker-explorer.conf.tpl
fi
if [ -f ./conf/docker-common.conf ]; then
source ./conf/docker-common.conf
else
@ -49,10 +55,20 @@ init_config_files() {
cp ./conf/docker-node.conf.tpl ./conf/docker-node.conf
echo "Initialized docker-node.conf"
cp ./conf/docker-explorer.conf.tpl ./conf/docker-explorer.conf
echo "Initialized docker-explorer.conf"
cp ./conf/docker-tor.conf.tpl ./conf/docker-tor.conf
echo "Initialized docker-tor.conf"
# Initialize config files for nginx and the maintenance tool
# Initialize config files for nginx and the maintenance tool
if [ "$EXPLORER_INSTALL" == "on" ]; then
cp ./nginx/explorer.conf ./nginx/dojo-explorer.conf
else
cp /dev/null ./nginx/dojo-explorer.conf
fi
echo "Initialized dojo-explorer.conf (nginx)"
if [ "$COMMON_BTC_NETWORK" == "testnet" ]; then
cp ./nginx/testnet.conf ./nginx/dojo.conf
echo "Initialized dojo.conf (nginx)"

22
docker/my-dojo/install/upgrade-scripts.sh

@ -6,6 +6,12 @@ else
source ./conf/docker-common.conf.tpl
fi
if [ -f ./conf/docker-explorer.conf ]; then
source ./conf/docker-explorer.conf
else
source ./conf/docker-explorer.conf.tpl
fi
source ./conf/docker-bitcoind.conf
# Confirm upgrade operation
@ -45,10 +51,20 @@ update_config_files() {
update_config_file ./conf/docker-node.conf ./conf/docker-node.conf.tpl
echo "Initialized docker-node.conf"
update_config_file ./conf/docker-explorer.conf ./conf/docker-explorer.conf.tpl
echo "Initialized docker-explorer.conf"
update_config_file ./conf/docker-tor.conf ./conf/docker-tor.conf.tpl
echo "Initialized docker-tor.conf"
# Initialize config files for nginx and the maintenance tool
if [ "$EXPLORER_INSTALL" == "on" ]; then
cp ./nginx/explorer.conf ./nginx/dojo-explorer.conf
else
cp /dev/null ./nginx/dojo-explorer.conf
fi
echo "Initialized dojo-explorer.conf (nginx)"
if [ "$COMMON_BTC_NETWORK" == "testnet" ]; then
cp ./nginx/testnet.conf ./nginx/dojo.conf
echo "Initialized dojo.conf (nginx)"
@ -72,7 +88,11 @@ update_config_file() {
cp -p $2 $1
while IFS='=' read -r key val ; do
sed -i "s~$key=.*~$key=$val~g" "$1"
if [[ $OSTYPE == darwin* ]]; then
sed -i "" "s~$key=.*~$key=$val~g" "$1"
else
sed -i "s~$key=.*~$key=$val~g" "$1"
fi
done < ./original.lines.raw
rm ./original.keys.raw

20
docker/my-dojo/nginx/Dockerfile

@ -1,18 +1,18 @@
FROM nginx:1.15.10-alpine
FROM nginx:1.15.10-alpine
# Create data directory
ENV LOGS_DIR /data/logs
ENV LOGS_DIR /data/logs
RUN mkdir -p "$LOGS_DIR" && \
chown -R nginx:nginx "$LOGS_DIR"
RUN mkdir -p "$LOGS_DIR" && \
chown -R nginx:nginx "$LOGS_DIR"
# Copy configuration files
COPY ./nginx.conf /etc/nginx/nginx.conf
COPY ./dojo.conf /etc/nginx/sites-enabled/dojo.conf
COPY ./nginx.conf /etc/nginx/nginx.conf
COPY ./dojo.conf /etc/nginx/sites-enabled/dojo.conf
COPY ./dojo-explorer.conf /etc/nginx/sites-enabled/dojo-explorer.conf
# Copy wait-for script
COPY ./wait-for /wait-for
COPY ./wait-for /wait-for
RUN chmod u+x /wait-for && \
chmod g+x /wait-for
RUN chmod u+x /wait-for && \
chmod g+x /wait-for

15
docker/my-dojo/nginx/explorer.conf

@ -0,0 +1,15 @@
server {
listen 9080;
server_name _;
resolver 127.0.0.11 valid=30s;
location / {
set $upstream http://explorer:3002;
proxy_pass $upstream;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection 'upgrade';
proxy_set_header Host $host;
proxy_cache_bypass $http_upgrade;
}
}

4
docker/my-dojo/node/Dockerfile

@ -4,6 +4,10 @@ ENV LOGS_DIR /data/logs
ENV APP_DIR /home/node/app
# Add node user to tor group
RUN addgroup --system -gid 1107 tor && \
usermod -a -G tor node
# Install forever
RUN npm install -g forever

2
docker/my-dojo/overrides/bitcoind.install.yaml

@ -10,7 +10,7 @@ services:
- ./.env
- ./conf/docker-common.conf
- ./conf/docker-bitcoind.conf
restart: on-failure
restart: always
command: "/wait-for-it.sh tor:9050 --timeout=720 --strict -- /restart.sh"
expose:
- "8333"

28
docker/my-dojo/overrides/explorer.install.yaml

@ -0,0 +1,28 @@
version: "3.2"
services:
explorer:
image: "samouraiwallet/dojo-explorer:${DOJO_EXPLORER_VERSION_TAG}"
container_name: explorer
build:
context: ./explorer
env_file:
- ./.env
- ./conf/docker-bitcoind.conf
- ./conf/docker-explorer.conf
restart: always
command: "/home/node/app/restart.sh"
expose:
- "3002"
volumes:
- data-explorer:/data/logs
networks:
dojonet:
ipv4_address: 172.28.1.7
node:
depends_on:
- explorer
volumes:
data-explorer:

9
docker/my-dojo/tor/Dockerfile

@ -3,6 +3,7 @@ FROM debian:stretch
ENV TOR_HOME /var/lib/tor
ENV TOR_URL https://archive.torproject.org/tor-package-archive
ENV TOR_VERSION 0.3.5.8
ENV TOR_GPG_KS_URI hkp://keyserver.ubuntu.com:80
ENV TOR_GPG_KEY1 0xEB5A896A28988BF5
ENV TOR_GPG_KEY2 0xC218525819F78451
ENV TOR_GPG_KEY3 0x21194EBB165733EA
@ -24,10 +25,10 @@ RUN set -ex && \
cd /usr/local/src && \
wget -qO "tor-$TOR_VERSION.tar.gz" "$TOR_URL/tor-$TOR_VERSION.tar.gz" && \
wget -qO "tor-$TOR_VERSION.tar.gz.asc" "$TOR_URL/tor-$TOR_VERSION.tar.gz.asc" && \
gpg --keyserver ipv4.pool.sks-keyservers.net --recv-keys "$TOR_GPG_KEY1" && \
gpg --keyserver ipv4.pool.sks-keyservers.net --recv-keys "$TOR_GPG_KEY2" && \
gpg --keyserver ipv4.pool.sks-keyservers.net --recv-keys "$TOR_GPG_KEY3" && \
gpg --keyserver ipv4.pool.sks-keyservers.net --recv-keys "$TOR_GPG_KEY4" && \
gpg --keyserver "$TOR_GPG_KS_URI" --recv-keys "$TOR_GPG_KEY1" && \
gpg --keyserver "$TOR_GPG_KS_URI" --recv-keys "$TOR_GPG_KEY2" && \
gpg --keyserver "$TOR_GPG_KS_URI" --recv-keys "$TOR_GPG_KEY3" && \
gpg --keyserver "$TOR_GPG_KS_URI" --recv-keys "$TOR_GPG_KEY4" && \
gpg --verify "tor-$TOR_VERSION.tar.gz.asc" && \
tar -xzvf "tor-$TOR_VERSION.tar.gz" -C /usr/local/src && \
cd "/usr/local/src/tor-$TOR_VERSION" && \

7
docker/my-dojo/tor/restart.sh

@ -25,6 +25,13 @@ tor_options=(
--HiddenServiceDirGroupReadable 1
)
if [ "$EXPLORER_INSTALL" == "on" ]; then
tor_options+=(--HiddenServiceDir /var/lib/tor/hsv3explorer)
tor_options+=(--HiddenServiceVersion 3)
tor_options+=(--HiddenServicePort "80 172.29.1.3:9080")
tor_options+=(--HiddenServiceDirGroupReadable 1)
fi
if [ "$TOR_USE_BRIDGES" == "on" ]; then
tor_options+=(--ClientTransportPlugin "obfs4 exec /usr/local/bin/obfs4proxy")
tor_options+=(--UseBridges 1)

2
keys/index-example.js

@ -15,7 +15,7 @@ module.exports = {
/*
* Dojo version
*/
dojoVersion: '1.3.0',
dojoVersion: '1.4.0',
/*
* Bitcoind
*/

4
lib/bitcoind-rpc/rpc-client.js

@ -37,7 +37,9 @@ class RpcClient {
return async function(...args) {
const result = await origMethod.apply(target.client, args)
if (result.result) {
if (Array.isArray(result)) {
return result
} else if (result.result) {
return result.result
} else if (result.error) {
throw result.error

134
lib/bitcoind-rpc/transactions.js

@ -46,6 +46,38 @@ class Transactions {
this.rpcClient = new RpcClient()
}
/**
* Get the transactions for a given array of txids
* @param {string[]} txids - txids of the transaction to be retrieved
* @param {boolean} fees - true if fees must be computed, false otherwise
* @returns {Promise} return an array of transactions (object[])
*/
async getTransactions(txids, fees) {
try {
const rpcCalls = txids.map(txid => {
return {
'method': 'getrawtransaction',
'params': [txid, true]
}
})
const txs = await this.rpcClient.batch(rpcCalls)
return await util.seriesCall(txs, async tx => {
if (tx.result == null) {
Logger.info(` got null for ${txids[tx.id]}`)
return null
} else {
return this._prepareTxResult(tx.result, fees)
}
})
} catch(e) {
Logger.error(e, 'Transaction.getTransactions()')
return Promise.reject(errors.generic.GEN)
}
}
/**
* Get the transaction for a given txid
* @param {string} txid - txid of the transaction to be retrieved
@ -61,64 +93,73 @@ class Transactions {
try {
const tx = await this.rpcClient.getrawtransaction(txid, true)
const ret = await this._prepareTxResult(tx)
// Store the result in cache
if (ret.block && ret.block.hash)
this.txCache.set(txid, ret)
return ret
} catch(e) {
Logger.error(e, 'Transaction.getTransaction()')
return Promise.reject(errors.generic.GEN)
}
}
const ret = {
txid: tx.txid,
size: tx.size,
vsize: tx.vsize,
version: tx.version,
locktime: tx.locktime,
inputs: [],
outputs: []
}
/**
* Formats a transaction object returned by the RPC API
* @param {object} tx - transaction
* @param {boolean} fees - true if fees must be computed, false otherwise
* @returns {Promise} return an array of inputs (object[])
*/
async _prepareTxResult(tx, fees) {
const ret = {
txid: tx.txid,
size: tx.size,
vsize: tx.vsize,
version: tx.version,
locktime: tx.locktime,
inputs: [],
outputs: []
}
if (!ret.vsize)
delete ret.vsize
if (!ret.vsize)
delete ret.vsize
if (tx.time)
ret.created = tx.time
if (tx.time)
ret.created = tx.time
// Process block informations
if (tx.blockhash && tx.confirmations && tx.blocktime) {
ret.block = {
height: rpcLatestBlock.height - tx.confirmations + 1,
hash: tx.blockhash,
time: tx.blocktime
}
// Process block informations
if (tx.blockhash && tx.confirmations && tx.blocktime) {
ret.block = {
height: rpcLatestBlock.height - tx.confirmations + 1,
hash: tx.blockhash,
time: tx.blocktime
}
}
let inAmount = 0
let outAmount = 0
// Process the inputs
ret.inputs = await this._getInputs(tx, fees)
inAmount = ret.inputs.reduce((prev, cur) => prev + cur.outpoint.value, 0)
// Process the outputs
ret.outputs = await this._getOutputs(tx)
outAmount = ret.outputs.reduce((prev, cur) => prev + cur.value, 0)
let inAmount = 0
let outAmount = 0
// Process the fees (if needed)
if (fees) {
ret.fees = inAmount - outAmount
if (ret.fees > 0 && ret.size)
ret.feerate = Math.round(ret.fees / ret.size)
if (ret.fees > 0 && ret.vsize)
ret.vfeerate = Math.round(ret.fees / ret.vsize)
}
// Process the inputs
ret.inputs = await this._getInputs(tx, fees)
inAmount = ret.inputs.reduce((prev, cur) => prev + cur.outpoint.value, 0)
// Store in cache
if (ret.block && ret.block.hash)
this.txCache.set(keyCache, ret)
// Process the outputs
ret.outputs = await this._getOutputs(tx)
outAmount = ret.outputs.reduce((prev, cur) => prev + cur.value, 0)
return ret
} catch(e) {
Logger.error(e, 'Transaction.getTransaction()')
return Promise.reject(errors.generic.GEN)
// Process the fees (if needed)
if (fees) {
ret.fees = inAmount - outAmount
if (ret.fees > 0 && ret.size)
ret.feerate = Math.round(ret.fees / ret.size)
if (ret.fees > 0 && ret.vsize)
ret.vfeerate = Math.round(ret.fees / ret.vsize)
}
return ret
}
/**
* Extract information about the inputs of a transaction
* @param {object} tx - transaction
@ -180,7 +221,6 @@ class Transactions {
/**
* Extract information about the outputs of a transaction
* @param {object} tx - transaction
* @param {boolean} fees - true if fees must be computed, false otherwise
* @returns {Promise} return an array of outputs (object[])
*/
async _getOutputs(tx) {

58
lib/db/mysql-db-wrapper.js

@ -1358,6 +1358,26 @@ class MySqlDbWrapper {
return (result.length == 0) ? null : result[0].txnID
}
/**
* Get the mysql IDs of a collection of transactions
* @param {string[]} txids - txids of the transactions
* @returns {object[]} returns an array of {txnTxid: txnId}
*/
async getTransactionsIds(txids) {
if (txids.length == 0)
return []
const sqlQuery = 'SELECT `txnID`, `txnTxid` FROM `transactions` WHERE `txnTxid` IN (?)'
const params = [txids]
const query = mysql.format(sqlQuery, params)
const results = await this._query(query)
const ret = {}
for (let r of results)
ret[r.txnTxid] = r.txnID
return ret
}
/**
* Get the mysql IDs of a set of transactions
* @param {string[]} txid - array of transactions txids
@ -1396,6 +1416,29 @@ class MySqlDbWrapper {
return this._query(query)
}
/**
* Insert a collection of transactions in db
* @param {object[]} txs - array of {txid, version, locktime}
*/
async addTransactions(txs) {
if (txs.length == 0)
return
const sqlQuery = 'INSERT INTO `transactions` \
(txnTxid, txnCreated, txnVersion, txnLocktime) VALUES ? \
ON DUPLICATE KEY UPDATE txnVersion = VALUES(txnVersion)'
const params = [txs.map(tx => [
tx.txid,
tx.created,
tx.version,
tx.locktime
])]
const query = mysql.format(sqlQuery, params)
return this._query(query)
}
/**
* Get a transaction for a given txid
* @param {string} txid - txid of the transaction
@ -1773,6 +1816,21 @@ class MySqlDbWrapper {
return (result.length == 1) ? result[0] : null
}
/**
* Get a collection of blocks identified by the blocks hashes
* @param {string[]} hashes - blocks hashes
* @returns {object[]} returns the blocks
*/
async getBlocksByHashes(hashes) {
if (hashes.length == 0)
return []
const sqlQuery = 'SELECT * FROM `blocks` WHERE `blockHash` IN (?)'
const params = [hashes]
const query = mysql.format(sqlQuery, params)
return await this._query(query)
}
/**
* Get details about all blocks at a given block height
* @param {integer} height - block height

156
lib/remote-importer/remote-importer.js

@ -115,7 +115,10 @@ class RemoteImporter {
if (txMaps.txMap[txid])
aTxs.push(txMaps.txMap[txid])
return util.seriesCall(aTxs, tx => this.addTransaction(tx, addrIdMap))
// Store the transactions by batches of 200 transactions
const txsChunks = util.splitList(aTxs, 200)
for (let txsChunk of txsChunks)
await this.addTransactions(txsChunk, addrIdMap)
}
/**
@ -260,19 +263,21 @@ class RemoteImporter {
Logger.info(` Got ${scanTx.length} transactions`)
await util.seriesCall(scanTx, async txid => {
try {
const tx = await rpcTxns.getTransaction(txid, false)
if (tx == null) {
Logger.info(` got null for ${txid}`)
return null
// Retrieve the transactions by batches of 200 transactions
const txsChunks = util.splitList(scanTx, 200)
try {
for (let txsChunk of txsChunks) {
const txs = await rpcTxns.getTransactions(txsChunk, false)
for (let tx of txs) {
if (tx != null) {
ret.transactions.push(tx)
txids[tx.txid] = true
}
}
ret.transactions.push(tx)
txids[tx.txid] = true
} catch(e) {
Logger.error(e, `RemoteImporter.xpubScan() : rawTransaction error, txid ${txid}`)
}
})
} catch(e) {
Logger.error(e, `RemoteImporter.xpubScan() : getTransactions error`)
}
if (gotTransactions) {
// We must go deeper
@ -335,15 +340,14 @@ class RemoteImporter {
Logger.info(` Got ${scanTx.length} transactions`)
// Get transaction s data from bitcoind
await util.seriesCall(scanTx, async txid => {
const tx = await rpcTxns.getTransaction(txid, false)
if (tx == null) {
Logger.info(` got null for ${txid}`)
return null
}
txns.push(tx)
})
// Retrieve the transactions by batches of 100 transactions
const txsChunks = util.splitList(scanTx, 100)
for (let txsChunk of txsChunks) {
const txs = await rpcTxns.getTransactions(txsChunk, false)
for (let tx of txs)
if (tx != null)
txns.push(tx)
}
// Import addresses and transactions into the database
await db.addAddresses(imported)
@ -368,70 +372,86 @@ class RemoteImporter {
}
/**
* Add a transaction to the database.
* @param {object} tx - transaction object
* @params {Promise}
* Add a collection of transactions to the database.
* @param {object[]} txs - array of transaction objects
* @params {object} addrIdMap - map address => addrId
* @returns {Promise}
*/
async addTransaction(tx, addrIdMap) {
const outputs = []
async addTransactions(txs, addrIdMap) {
try {
// Store the transaction into the database
await db.addTransaction(tx)
// Confirm the transaction
if (tx.block) {
const block = await db.getBlockByHash(tx.block.hash)
if (block)
await db.confirmTransactions([tx.txid], block.blockID)
// Store the transactions into the database
await db.addTransactions(txs)
// Confirm the transactions if needed
const blocksHashes = new Set()
for (let tx of txs)
if (tx.block)
blocksHashes.add(tx.block.hash)
const blocks = await db.getBlocksByHashes(Array.from(blocksHashes))
for (let block of blocks) {
// Filter the transactions by blockHash
const filteredTxs = txs.filter(tx => (tx.block && tx.block.hash == block.blockHash))
if (filteredTxs.length > 0) {
const txids = filteredTxs.map(tx => tx.txid)
// Asynchronous confirmations
db.confirmTransactions(txids, block.blockID)
}
}
// Retrieve the database id for the transaction
let txnID = await db.ensureTransactionId(tx.txid)
// Process the outputs
for (let output of tx.outputs) {
if (addrIdMap[output.address]) {
outputs.push({
txnID,
addrID: addrIdMap[output.address],
outIndex: output.n,
outAmount: output.value,
outScript: output.scriptpubkey,
})
// Retrieve the database ids for the transactions
const txids = txs.map(tx => tx.txid)
const mapTxsIds = await db.getTransactionsIds(txids)
// Store the outputs in db
const outputs = []
for (let tx of txs) {
for (let output of tx.outputs) {
if (addrIdMap[output.address]) {
outputs.push({
txnID: mapTxsIds[tx.txid],
addrID: addrIdMap[output.address],
outIndex: output.n,
outAmount: output.value,
outScript: output.scriptpubkey,
})
}
}
}
await db.addOutputs(outputs)
// Process the inputs
// Get any outputs spent by the inputs of this transaction, add those
// database outIDs to the corresponding transaction inputs, and store.
const res = await db.getOutputIds(tx.inputs.map(input => input.outpoint))
const spent = {}
// Store the inputs in db
const inputs = []
const spent = {}
// Get any outputs spent by the inputs of this transaction,
// add those database outIDs to the corresponding inputs, and store.
let outpoints = []
for (let tx of txs)
outpoints = outpoints.concat(tx.inputs.map(input => input.outpoint))
const res = await db.getOutputIds(outpoints)
for (let r of res)
spent[`${r.txnTxid}-${r.outIndex}`] = r.outID
for (let input of tx.inputs) {
let key = `${input.outpoint.txid}-${input.outpoint.vout}`
if (spent[key]) {
inputs.push({
outID: spent[key],
txnID,
inIndex: input.n,
inSequence: input.seq
})
for (let tx of txs) {
for (let input of tx.inputs) {
const key = `${input.outpoint.txid}-${input.outpoint.vout}`
if (spent[key]) {
inputs.push({
outID: spent[key],
txnID: mapTxsIds[tx.txid],
inIndex: input.n,
inSequence: input.seq
})
}
}
}
await db.addInputs(inputs)
} catch(e) {
Logger.error(e, `RemoteImporter.addTransaction() : xpub ${tx.txid}`)
Logger.error(null, JSON.stringify(tx,null,2))
Logger.error(e, `RemoteImporter.addTransactions() :`)
}
}

2
lib/remote-importer/sources.js

@ -75,7 +75,7 @@ class Sources {
}
} catch(e) {
Logger.error(e, `Sources.getAddresses() : ${addresses} from ${this.source.base}`)
Logger.error(null, `Sources.getAddresses() : Error while requesting ${addresses} from ${this.source.base}`)
} finally {
return ret
}

18
lib/util.js

@ -101,20 +101,14 @@ class Util {
* Splits a list into a list of lists each with maximum length LIMIT
*/
static splitList(list, limit) {
if (list.length <= limit) {
if (list.length <= limit)
return [list]
} else {
const lists = []
// How many lists to create?
const count = Math.ceil(list.length / limit)
// How many elements per list (max)?
const els = Math.ceil(list.length / count)
for (let i=0; i < count; i++) {
lists.push(list.slice(i * els, (i+1) * els))
}
return lists
const lists = []
while (list.length) {
lists.push(list.splice(0, limit))
}
return lists
}
/**

2
package-lock.json

@ -1,6 +1,6 @@
{
"name": "samourai-dojo",
"version": "1.3.0",
"version": "1.4.0",
"lockfileVersion": 1,
"requires": true,
"dependencies": {

2
package.json

@ -1,6 +1,6 @@
{
"name": "samourai-dojo",
"version": "1.3.0",
"version": "1.4.0",
"description": "Backend server for Samourai Wallet",
"main": "accounts/index.js",
"scripts": {

17
static/admin/css/style.css

@ -370,11 +370,20 @@ table.spaced tr td {
}
/* PAIRING */
#qr-label {
#qr-label,
#qr-explorer-label {
margin: 0 0 20px 0;
text-align: center;
display: inline-block;
}
#qr-container,
#qr-explorer-container {
display: inline-block;
}
#qr-pairing {
#qr-pairing,
#qr-explorer-pairing {
width: 276px;
height: 276px;
padding: 10px;
@ -390,8 +399,8 @@ table.spaced tr td {
}
.halfwidth {
width: 50%;
min-width: 50%;
width: 49%;
min-width: 49%;
}
.fullwidth {

9
static/admin/lib/api-wrapper.js

@ -39,6 +39,15 @@ var lib_api = {
return this.sendGetUriEncoded(uri, {});
},
/**
* Get block explorer pairing info
*/
getExplorerPairingInfo: function() {
let prefix = conf['prefixes']['support'];
let uri = this.baseUri + '/' + prefix + '/pairing/explorer';
return this.sendGetUriEncoded(uri, {});
},
/**
* PushTx Status
*/

18
static/admin/tool/index.html

@ -86,10 +86,22 @@
<div class="col-xs-10 json-data-container">
<!-- PAIRING -->
<div id="screen-pairing">
<div class="center" id="qr-label">
PAIR YOUR SAMOURAI WALLET WITH YOUR DOJO BY SCANNING THIS QRCODE
<div class="row">
<div id="qr-label" class="halfwidth">
PAIR YOUR WALLET WITH YOUR DOJO
</div>
<div id="qr-explorer-label" class="halfwidth">
PAIR YOUR WALLET WITH YOUR BLOCK EXPLORER
</div>
</div>
<div class="row">
<div id="qr-container" class="halfwidth">
<div id="qr-pairing"></div>
</div>
<div id="qr-explorer-container" class="halfwidth">
<div id="qr-explorer-pairing"></div>
</div>
</div>
<div class="center" id="qr-pairing"></div>
</div>
<!-- MAINTENANCE -->
<div id="form-maintenance">

55
static/admin/tool/index.js

@ -16,12 +16,25 @@ function displayQRPairing() {
const activeTab = sessionStorage.getItem('activeTab');
processAction(activeTab).then(
function (result) {
if (!result) {return;}
const url = window.location.protocol + '//' + window.location.host + conf['api']['baseUri'];
result['pairing']['url'] = url;
const textJson = JSON.stringify(result, null, 4);
$("#qr-pairing").html('') // clear qrcode first
$('#qr-pairing').qrcode({width: 256, height: 256, text: textJson});
if (result) {
if (result['api']) {
const textJson = JSON.stringify(result['api'], null, 4);
$("#qr-pairing").html('') // clear qrcode first
$('#qr-pairing').qrcode({width: 256, height: 256, text: textJson});
}
if (result['explorer'] && result['explorer']['pairing']['url']) {
const textJson = JSON.stringify(result['explorer'], null, 4);
$("#qr-explorer-pairing").html('') // clear qrcode first
$('#qr-explorer-pairing').qrcode({width: 256, height: 256, text: textJson});
} else {
$("#qr-label").removeClass('halfwidth');
$("#qr-label").addClass('fullwidth');
$("#qr-container").removeClass('halfwidth');
$("#qr-container").addClass('fullwidth');
$("#qr-explorer-label").hide();
$("#qr-explorer-container").hide();
}
}
},
function (jqxhr) {}
);
@ -147,14 +160,34 @@ function preparePage() {
* Process action (api calls)
*/
function processAction(activeTab, args, args2, args3) {
if (activeTab == '#link-pairing')
return lib_api.getPairingInfo();
else if (activeTab == '#link-status-api')
if (activeTab == '#link-pairing') {
//return lib_api.getPairingInfo();
let result = {
'api': null,
'explorer': null
};
return lib_api.getPairingInfo().then(apiInfo => {
if (apiInfo) {
apiInfo['pairing']['url'] = window.location.protocol + '//' + window.location.host + conf['api']['baseUri'];
result['api'] = apiInfo;
}
}).then(() => {
return lib_api.getExplorerPairingInfo();
}).then(explorerInfo => {
if (explorerInfo)
result['explorer'] = explorerInfo;
return result
}).catch(e => {
console.log(e);
return result;
});
} else if (activeTab == '#link-status-api') {
return lib_api.getApiStatus();
else if (activeTab == '#link-status-pushtx')
} else if (activeTab == '#link-status-pushtx') {
return lib_api.getPushtxStatus();
else if (activeTab == '#link-orchestrator')
} else if (activeTab == '#link-orchestrator') {
return lib_api.getOrchestratorStatus();
}
if (args == '') {
alert('Argument is mandatory');

Loading…
Cancel
Save