Browse Source

Merge pull request #44 from Samourai-Wallet/develop

merge develop into master for v1.1.0
1.1 v1.1.0
kenshin samourai 5 years ago
committed by GitHub
parent
commit
a5a4d429df
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
  1. 6
      .gitignore
  2. 9
      README.md
  3. 114
      RELEASES.md
  4. 6
      accounts/api-helper.js
  5. 27
      db-scripts/1_db.sql.tpl
  6. 7
      db-scripts/2_update.sql.tpl
  7. 161
      doc/DOCKER_advanced_setups.md
  8. 54
      doc/DOCKER_mac_setup.MD
  9. 150
      doc/DOCKER_setup.md
  10. 33
      doc/GET_multiaddr.md
  11. 33
      doc/GET_unspent.md
  12. 8
      docker/my-dojo/.env
  13. 11
      docker/my-dojo/bitcoin/Dockerfile
  14. 22
      docker/my-dojo/bitcoin/bitcoin.conf
  15. 42
      docker/my-dojo/bitcoin/restart.sh
  16. 32
      docker/my-dojo/conf/docker-bitcoind.conf
  17. 103
      docker/my-dojo/conf/docker-bitcoind.conf.tpl
  18. 0
      docker/my-dojo/conf/docker-mysql.conf.tpl
  19. 0
      docker/my-dojo/conf/docker-node.conf.tpl
  20. 38
      docker/my-dojo/docker-compose.yaml
  21. 221
      docker/my-dojo/dojo.sh
  22. 34
      docker/my-dojo/install/install-scripts.sh
  23. 67
      docker/my-dojo/install/upgrade-scripts.sh
  24. 12
      docker/my-dojo/mysql/Dockerfile
  25. 14
      docker/my-dojo/mysql/update-db.sh
  26. 5
      docker/my-dojo/nginx/dojo.conf
  27. 2
      docker/my-dojo/nginx/nginx.conf
  28. 2
      docker/my-dojo/node/Dockerfile
  29. 8
      docker/my-dojo/node/keys.index.js
  30. 34
      docker/my-dojo/overrides/bitcoind.install.yaml
  31. 10
      docker/my-dojo/overrides/bitcoind.rpc.expose.yaml
  32. 14
      docker/my-dojo/tor/Dockerfile
  33. 9
      docker/my-dojo/tor/restart.sh
  34. 17
      docker/my-dojo/tor/torrc
  35. 2
      keys/index-example.js
  36. 30
      lib/db/mysql-db-wrapper.js
  37. 73
      lib/remote-importer/btccom-wrapper.js
  38. 66
      lib/remote-importer/sources-mainnet.js
  39. 124
      lib/remote-importer/sources-testnet.js
  40. 54
      lib/remote-importer/sources.js
  41. 2302
      package-lock.json
  42. 3
      package.json
  43. 28
      pushtx/orchestrator.js
  44. 20
      static/admin/index.js
  45. 33
      static/admin/lib/auth-utils.js
  46. 12
      test/lib/bitcoin/addresses-helper-test.js
  47. 8
      tracker/blockchain-processor.js
  48. 21
      tracker/transaction.js

6
.gitignore

@ -1,7 +1,11 @@
db-scripts/updates/
db-scripts/1_db.sql
db-scripts/2_update.sql
docker/my-dojo/conf/docker-bitcoind.conf
docker/my-dojo/conf/docker-mysql.conf
docker/my-dojo/conf/docker-node.conf
keys/index.js
keys/sslcert/
node_modules/
private-tests/
*.log
package-lock.json

9
README.md

@ -71,7 +71,10 @@ Authentication is enforced by an API key and Json Web Tokens.
* Edit /keys/index.js and set "explorers.bitcoind" to "inactive".
* Main drawbacks of using your local bitcoind for these imports:
* It doesn't return the full transactional history associated to the HD account but only transactions having an unspent output controlled by the HD account.
* It's slightly slower than using the option relying on the OXT API.
* In some specific cases, the importer might miss the most recent unspent outputs. Higher values of gap.external and gap.internal in /keys/index.js should help to mitigate this issue. Another workaround is to request the endpoint /support/xpub/.../rescan provided by the REST API with the optional gap parameter.
* This option is considered as experimental.
* It doesn't return the full transactional history associated to an HD account or to an address but only transactions having an unspent output controlled by the HD account or the address.
* It's slightly slower than using the option relying on the OXT API.
* It may fail to correctly import an existing wallet if this wallet had a large activity.
* If you use bitcoind and if the import seems to return an invalid balance, you can use the "XPUB rescan" function provided by the maintenance tool. This function allows you to force the minimum number of addresses to be derived and the start index for the derivation.
* As a rule of thumb, we recommend to use bitcoind as the source of imports and to setup your Dojo with a new clean wallet. It increases your privacy and it removes all potential issues with the import of a large wallet.

114
RELEASES.md

@ -0,0 +1,114 @@
# Release Notes
## Samourai Dojo v1.1.0 ##
### Notable changes ###
#### Upgrade mechanism ####
An upgrade mechanism for MyDojo has been added.
See this [doc](./doc/DOCKER_setup.md#upgrade) for more details.
#### Optional support of an external bitcoin full node ####
Optional support of an existing Bitcoin full node running outside of Docker has been added.
This setup can be configured thanks to new options defined in ./docker/my-dojo/conf/docker-bitcoind.conf. When this option is activated, the install command skips the installation of bitcoind in Docker.
Note: The Bitcoin full node installed by MyDojo is configured for taking care of your privacy at a network level. You may lose the benefits provided by the default setup if your external full node isn't properly configured. Use at your own risk.
See this [doc](./doc/DOCKER_advanced_setups.md#external_bitcoind) for more details.
#### Optional support of external apps ####
New options defined in ./docker/my-dojo/conf/docker-bitcoind.conf allow to expose the RPC API and ZMQ notifications provided by the full node of MyDojo to applications runnnig outside of Docker.
Note: Exposing the full node of MyDojo to external applications may damage your privacy. Use at your own risk.
See this [doc](./doc/DOCKER_advanced_setups.md#exposed_rpc_zmq) for more details.
#### Optional support of a static onion address for the full node ####
A new option defined in ./docker/my-dojo/conf/docker-bitcoind.conf allows to keep a static onion address for your full node.
By default, MyDojo generates a new onion address at each startup. We recommend to keep this default setup for better privacy.
See this [doc](./doc/DOCKER_advanced_setups.md#static_onion) for more details.
#### Clean-up of Docker images ####
A new "clean" command has been added to Dojo shell script for deleting old Docker images of MyDojo.
This command allows to free disk space on the Docker host.
#### Documentation ####
Added a new [doc](./doc/DOCKER_advanced_setups.md) for advanced setups.
Added a new [doc](./doc/DOCKER_mac_setup.MD) for MacOS users.
### Change log ###
#### MyDojo ####
- [#1](https://github.com/Samourai-Wallet/samourai-dojo/pull/1) my-dojo upgrade mechanism
- [#7](https://github.com/Samourai-Wallet/samourai-dojo/pull/7) support of inbound connections through Tor
- [#8](https://github.com/Samourai-Wallet/samourai-dojo/pull/8) add config option exposing the rpc api and zmq notifications to external apps
- [#10](https://github.com/Samourai-Wallet/samourai-dojo/pull/10) add an option allowing to run dojo on top of an external bitcoind
- [#11](https://github.com/Samourai-Wallet/samourai-dojo/pull/11) clean-up
- [#12](https://github.com/Samourai-Wallet/samourai-dojo/pull/12) extend support of external apps
- [#15](https://github.com/Samourai-Wallet/samourai-dojo/pull/15) fix issue introduced by #10
- [#19](https://github.com/Samourai-Wallet/samourai-dojo/pull/19) fix bitcoind port in torrc
- [#20](https://github.com/Samourai-Wallet/samourai-dojo/pull/20) increase nginx timeout
- [#25](https://github.com/Samourai-Wallet/samourai-dojo/pull/25) force the tracker to derive next indices if a hole is detected
- [#27](https://github.com/Samourai-Wallet/samourai-dojo/pull/27) rework external loop of Orchestrator
- [#28](https://github.com/Samourai-Wallet/samourai-dojo/pull/28) rework RemoteImporter
- [#32](https://github.com/Samourai-Wallet/samourai-dojo/pull/32) change the conditions switching the startup mode of the tracker
- [#33](https://github.com/Samourai-Wallet/samourai-dojo/pull/33) check authentication with admin key
- [#37](https://github.com/Samourai-Wallet/samourai-dojo/pull/37) automatic redirect of onion address to maintenance tool
- [#38](https://github.com/Samourai-Wallet/samourai-dojo/pull/38) dojo shutdown - replace sleep with static delay by docker wait
#### Security ####
- [#5](https://github.com/Samourai-Wallet/samourai-dojo/pull/5) mydojo - install nodejs
- [#6](https://github.com/Samourai-Wallet/samourai-dojo/pull/6) remove deprecated "new Buffer" in favor of "Buffer.from"
- [#41](https://github.com/Samourai-Wallet/samourai-dojo/pull/41) update nodejs packages
#### Documentation ####
- [#13](https://github.com/Samourai-Wallet/samourai-dojo/pull/13) included Mac instructions
- [92097d8](https://github.com/Samourai-Wallet/samourai-dojo/commit/92097d8ec7f9488ce0318c452356994315f4be72) doc
- [de4c9b5](https://github.com/Samourai-Wallet/samourai-dojo/commit/de4c9b5e5078b673c7b199503d48e7ceca328285) doc - minor updates
- [fead0bb](https://github.com/Samourai-Wallet/samourai-dojo/commit/fead0bb4b2b6174e637f5cb8c57edd9b55c3a1c7) doc - add link to MacOS install doc
- [#42](https://github.com/Samourai-Wallet/samourai-dojo/pull/42) fix few typos, add backticks for config values
- [#43](https://github.com/Samourai-Wallet/samourai-dojo/pull/43) add missing `d` in `docker-bitcoind.conf`
#### Misc ####
- [a382e42](https://github.com/Samourai-Wallet/samourai-dojo/commit/a382e42469b884d2eda9fa6f5a3c8ce93a7cd39a) add sql scripts and config files to gitignore
### Credits ###
- 05nelsonm
- clarkmoody
- dergigi
- hkjn
- kenshin-samourai
- LaurentMT
- michel-foucault
- pxsocs
- Technifocal

6
accounts/api-helper.js

@ -28,10 +28,10 @@ class ApiHelper {
*/
parseEntities(str) {
const ret = new WalletEntities()
if (typeof str !== 'string')
return ret
for (let item of str.split('|')) {
try {
@ -47,7 +47,7 @@ class ApiHelper {
} else if (addrHelper.isSupportedPubKey(item) && !ret.hasPubKey(item)) {
// Derive pubkey as 3 addresses (P1PKH, P2WPKH/P2SH, BECH32)
const bufItem = new Buffer(item, 'hex')
const bufItem = Buffer.from(item, 'hex')
const funcs = [
addrHelper.p2pkhAddress,

27
db-scripts/1_db.sql → db-scripts/1_db.sql.tpl

@ -25,10 +25,9 @@
-- Table structure for table `addresses`
--
DROP TABLE IF EXISTS `addresses`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `addresses` (
CREATE TABLE IF NOT EXISTS `addresses` (
`addrID` int(10) unsigned NOT NULL AUTO_INCREMENT,
`addrAddress` varchar(74) DEFAULT NULL,
PRIMARY KEY (`addrID`),
@ -40,10 +39,9 @@ CREATE TABLE `addresses` (
-- Table structure for table `banned_addresses`
--
DROP TABLE IF EXISTS `banned_addresses`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `banned_addresses` (
CREATE TABLE IF NOT EXISTS `banned_addresses` (
`bannedAddressId` int(11) NOT NULL AUTO_INCREMENT,
`addrAddress` varchar(35) NOT NULL,
PRIMARY KEY (`bannedAddressId`),
@ -55,10 +53,9 @@ CREATE TABLE `banned_addresses` (
-- Table structure for table `blocks`
--
DROP TABLE IF EXISTS `blocks`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `blocks` (
CREATE TABLE IF NOT EXISTS `blocks` (
`blockID` int(10) unsigned NOT NULL AUTO_INCREMENT,
`blockHash` char(64) NOT NULL DEFAULT '',
`blockParent` int(10) unsigned DEFAULT NULL,
@ -76,10 +73,9 @@ CREATE TABLE `blocks` (
-- Table structure for table `hd`
--
DROP TABLE IF EXISTS `hd`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `hd` (
CREATE TABLE IF NOT EXISTS `hd` (
`hdID` int(10) unsigned NOT NULL AUTO_INCREMENT,
`hdXpub` char(112) DEFAULT NULL,
`hdCreated` int(10) unsigned NOT NULL DEFAULT '0',
@ -94,10 +90,9 @@ CREATE TABLE `hd` (
-- Table structure for table `hd_addresses`
--
DROP TABLE IF EXISTS `hd_addresses`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `hd_addresses` (
CREATE TABLE IF NOT EXISTS `hd_addresses` (
`hdAddrID` int(10) unsigned NOT NULL AUTO_INCREMENT,
`hdID` int(10) unsigned NOT NULL DEFAULT '0',
`addrID` int(10) unsigned NOT NULL DEFAULT '0',
@ -116,10 +111,9 @@ CREATE TABLE `hd_addresses` (
-- Table structure for table `inputs`
--
DROP TABLE IF EXISTS `inputs`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `inputs` (
CREATE TABLE IF NOT EXISTS `inputs` (
`inID` int(10) unsigned NOT NULL AUTO_INCREMENT,
`outID` int(10) unsigned NOT NULL DEFAULT '0',
`txnID` int(10) unsigned NOT NULL DEFAULT '0',
@ -138,10 +132,9 @@ CREATE TABLE `inputs` (
-- Table structure for table `outputs`
--
DROP TABLE IF EXISTS `outputs`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `outputs` (
CREATE TABLE IF NOT EXISTS `outputs` (
`outID` int(10) unsigned NOT NULL AUTO_INCREMENT,
`txnID` int(10) unsigned NOT NULL DEFAULT '0',
`addrID` int(10) unsigned NOT NULL DEFAULT '0',
@ -161,10 +154,9 @@ CREATE TABLE `outputs` (
-- Table structure for table `transactions`
--
DROP TABLE IF EXISTS `transactions`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `transactions` (
CREATE TABLE IF NOT EXISTS `transactions` (
`txnID` int(10) unsigned NOT NULL AUTO_INCREMENT,
`txnTxid` char(64) DEFAULT NULL,
`txnCreated` int(10) unsigned NOT NULL DEFAULT '0',
@ -183,10 +175,9 @@ CREATE TABLE `transactions` (
-- Table structure for table `scheduled_transactions`
--
DROP TABLE IF EXISTS `scheduled_transactions`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `scheduled_transactions` (
CREATE TABLE IF NOT EXISTS `scheduled_transactions` (
`schID` int(10) unsigned NOT NULL AUTO_INCREMENT,
`schTxid` char(64) NOT NULL DEFAULT '',
`schCreated` int(10) unsigned NOT NULL DEFAULT '0',

7
db-scripts/2_update.sql.tpl

@ -0,0 +1,7 @@
# SQL scripts for incremental updates
# Copyright © 2019 – Katana Cryptographic Ltd. All Rights Reserved.
--
-- UPDATES vX.Y.Z
--

161
doc/DOCKER_advanced_setups.md

@ -0,0 +1,161 @@
# MyDojo - Advanced Setups
The 3 configuration files of Dojo provide a few advanced options allowing to tune your setup.
A word of caution, though, the default values of these options try to maximize your privacy at a network level. All the advanced setups described in this document may damage your privacy. Use at your own risk!
<a name="external_bitcoind"/>
## External Bitcoin full node ##
By default, Dojo installs and runs a Bitcoin full node in Docker.
The following procedure allows to bypass the installation of this full node by telling Dojo to rely on an external bitcoind running on your host machine.
### Requirements ###
The external full node mustn't be pruned.
The external full node must be configured for the support of Dojo. Edit the bitcoin.conf file of your external full node and check that the following lines are properly initialized.
```
# Force bitcoind to accept JSON-RPC commands
server=1
# Force bitcoind to index all the transactions
txindex=1
# Check that bitcoind accepts connections from 127.0.0.1 (linux)
# or from the IP address of the Docker Virtual Machine (MacOS, Windows)
rpcallowip=...
# Check that a port is defined for the RPC API (or 8332 will be used as default value)
rpcport=...
# Check that the RPC API listens on an IP address accessible from the nodejs container
rpcbind=...
# Check that the RPC user is set
rpcuser=...
# Check that the RPC password is set
rpcpassword=
# Enable publish hash block on an IP address accessible from the nodejs container
zmqpubhashblock=...
# Enable publish raw transaction on an IP address accessible from the nodejs container
zmqpubrawtx=...
```
### Procedure ###
#### Configuration of Dojo ####
```
# Edit the bitcoin config template file
nano ./conf/docker-bitcoind.conf.tpl
#
# Set the value of BITCOIND_INSTALL to "off"
# Set the value of BITCOIND_IP with the IP address of you bitcoin full node
# Set the value of BITCOIND_RPC_PORT with the port used by your bitcoin full node for the RPC API
# Set the value of BITCOIND_ZMQ_RAWTXS with the port used by your bitcoin full node for ZMQ notifications of raw transactions
# (i.e. port defined for -zmqpubrawtx in the bitcoin.conf of your full node)
# Set the value of BITCOIND_ZMQ_BLK_HASH with the port used by your bitcoin full node for ZMQ notifications of block hashes
# (i.e. port defined for -zmqpubhashblock in the bitcoin.conf of your full node)
#
# Save and exit nano
#
```
### Fast import of block headers in Dojo (optional) ###
When Dojo is installed for the first time, the Tracker imports the block headers in the database.
Follow these steps if you want to speed up this operation by preloading an archive of the block headers.
```
# Download the archive [https://samouraiwallet.com/static/share/2_blocks.sql.gz](https://samouraiwallet.com/static/share/2_blocks.sql.gz) to the "<dojo_dir>/db-scripts/" directory. Don't modify the name of the archive.
```
#### Start the installation of Dojo ####
```
./dojo.sh install
```
<a name="exposed_rpc_zmq"/>
## bitcoind RPC API ans ZMQ notifications exposed to external apps ##
By default, access to the RPC API of your bitcoind is restricted to Docker containers hosted on the "dojonet" network.
The following steps allow to expose the RPC API and ZMQ notifications to applications running on your local machine but outside of Docker.
```
#
# If your Docker runs on macos or windows,
# retrieve the local IP address of the VM
# hosting your Docker containers
#
# Stop your Dojo
./dojo.sh stop
# Edit the bitcoin config file
nano ./conf/docker-bitcoind.conf
#
# Set the value of BITCOIND_RPC_EXTERNAL to "on"
#
# If your Docker runs on macos or windows,
# set the value of BITCOIND_RPC_EXTERNAL_IP to the IP address of the VM
#
# Save and exit nano
#
# Start your Dojo
./dojo.sh start
```
With this setting, external applications running on your local machine should be able to access the following ports:
* 9500: bitcoind zmqpubhashtx notifications
* 9501: bitcoind zmqpubrawtx notifications
* 9502: bitcoind zmqpubhashblock notifications
* 9503: bitcoind zmqpubrawblock notifications
* 28256: bitcoind RPC API
Note: this option has no effect if your setup relies on a external full node (i.e. if BITCOIND_INSTALL is set to "off").
<a name="static_onion"/>
## Static onion address for bitcoind hidden service ##
By default, Dojo creates a new onion address for your bitcoind at each startup.
The following steps allow to keep a static onion address (not recommended).
```
# Stop your Dojo
./dojo.sh stop
# Edit the bitcoin config file
nano ./conf/docker-bitcoind.conf
#
# Set the value of BITCOIND_EPHEMERAL_HS to "off"
#
# Start your Dojo
./dojo.sh start
```
Note: this option has no effect if your setup relies on a external full node (i.e. if BITCOIND_INSTALL is set to "off").

54
doc/DOCKER_mac_setup.MD

@ -0,0 +1,54 @@
# MacOS Installation
This installation was tested on an iMac (mid 2011) with a 2.7GHz i5 processor with 8GB RAM and 1TB external Hard Drive.
## Getting Started
#### Create a new user:
1. Launch System Preferences by clicking the **System Preferences** icon in the **Dock**, or selecting **System Preferences** from the Apple menu.
2. Click on __Users & Groups__
3. If settings are locked, click on the __Lock__ at the bottom of window and enter your password.
4. Click on __+__ to add a new User
5. Under __New Account__ select __Administrator__
6. Fill the remaining fields with your choice of User Name and Password
#### Move the __New User Folder__ into the __External HD__
Note: _This is an important step, otherwise, it's probable that when you run the container, it will be installed in your main OS Hard Drive and will run out of space as it validates the Bitcoin blockchain._
1. Open **Finder** and navigate to your startup drive's **/Users** folder. For most people, this is **/Macintosh HD/Users**. In the **Users** **folder**, you'll find your user's folder.
2. On your external Hard Drive, create a folder named **Users**.
3. Select your user folder and drag it to the external HD **/Users** folder you created. _Because you're using a different drive for the destination, the operating system will copy the data rather than move it. This ok for now but delete it later._
4. Launch System Preferences again.
5. In the **Users & Groups** click the lock icon in the bottom left corner, then provide an administrator name and password.
6. From the list of user accounts, right-click on the account whose home folder you moved, and select **Advanced Options** from the pop-up menu.
_Do not make any changes to Advanced Options except for those noted here. Doing so can cause quite a few unforeseen problems that could lead to data loss or the need to reinstall the operating system._
7. In the **Advanced Options** sheet, click **Choose**, located to the right of the **Home directory** field.
8. Navigate to the location you moved your home folder to, select the new home folder, and click **OK**.
9. Click **OK** to dismiss the **Advanced Options** sheet, and then close **System Preferences**.
10. __Restart your Mac__
#### Download and install Docker, Kitematic and TOR
1. Make sure your system fills the [requirements]([https://docs.docker.com/docker-for-mac/install/](https://docs.docker.com/docker-for-mac/install/)) (particularly MacOS Sierra 10.12 or higher. If not, upgrade before proceeding).
2. [Download Docker]([https://docs.docker.com/docker-for-mac/install/](https://docs.docker.com/docker-for-mac/install/)) and follow the installation steps.
3. _Optional_: Download [Kitematic]([https://kitematic.com/) and follow installation instructions.
(_This may be system specific but I've found that monitoring the logs with Kitematic was more stable than using the Terminal_).
4. Install [Tor Browser](https://www.torproject.org/projects/torbrowser.html.en) on the host machine.
## Adjust Docker Settings
1. Click on the Docker icon (![whale menu](https://docs.docker.com/docker-for-mac/images/whale-x.png)) at the status bar and select __Preferences__.
2. Under Disk, click on __Reveal in Finder__ and double check that the disk image is saved under the external HD.
3. __Adjust Disk__ Image size to 400GB+ and click Apply.
4. Click __Advanced__ and increase the CPU count, Memory and Swap sizes. Adjusting these will speed up the blockchain validation process
(_At 4 CPUs, 8GB of RAM and a 4GiB Swap - the initial block download took 4.5 days at the time of writing_).
## Install the DOJO
Follow the instructions [here](https://github.com/Samourai-Wallet/samourai-dojo/blob/develop/doc/DOCKER_setup.md) starting at the step:
__"Download the most recent release of Dojo from Github"__
_Note: For tracking progress, open Kitematic and follow the bitcoind logs. You'll be able to see the Blockchain verification process under the _progress_ log variable (1.00 = fully validated). This process takes a long time. Just let it do its thing. In my system it took 3 days._
__Some possible optimization tips:__
. If you notice that progress has stopped. Click the whale icon and select Restart. Check Kitematic logs of bitcoind to confirm that progress has resumed.
. This may optimize speed: open __Activity Monitor__, check the PID (Process ID) of your docker process. Open Terminal and type:
`sudo renice-20 -p [enter your PID]`

150
doc/DOCKER_setup.md

@ -2,11 +2,25 @@
MyDojo is a set of Docker containers providing a full Samourai backend composed of:
* a bitcoin full node accessible as an ephemeral Tor hidden service,
* the backend database,
* the backend modules with an API accessible as a static Tor hidden service,
* a backend database,
* a backend modules with an API accessible as a static Tor hidden service,
* a maintenance tool accessible through a Tor web browser.
## Table of Content ##
- [Architecture](#architecture)
- [Requirements](#requirements)
- [First-time install procedure](#install)
- [Upgrade procedure](#upgrade)
- [Configuration files](#config_files)
- [Dojo shell script](#shell_script)
- [Dojo maintenance tool](#maintenance_tool)
- [Pairing your wallet to your Dojo](#pairing)
- [Network connections](#network)
<a name="architecture"/>
## Architecture ##
@ -21,7 +35,7 @@ MyDojo is a set of Docker containers providing a full Samourai backend composed
------------
|
Host machine | (Tor - port 80)
Host machine | (Tor hidden services)
______________________________ | _____________________________
| | |
| ------------------- |
@ -42,6 +56,7 @@ MyDojo is a set of Docker containers providing a full Samourai backend composed
|______________________________________________________________|
<a name="requirements"/>
## Requirements ##
@ -54,41 +69,52 @@ MyDojo is a set of Docker containers providing a full Samourai backend composed
* Tor Browser installed on the host machine (or on another machine if your host is a headless server)
## Setup ##
<a name="install"/>
* Install [Docker and Docker Compose](https://docs.docker.com/compose/install/) on the host machine and check that your installation is working.
## First-time Setup ##
* Install [Tor Browser](https://www.torproject.org/projects/torbrowser.html.en) on the host machine.
For MacOS, see this detailed [installation guide](./DOCKER_mac_setup.MD).
* Download the most recent version of Dojo from [Github](https://github.com/Samourai-Wallet/samourai-dojo/archive/master.zip)
This procedure allows to install a new Dojo from scratch.
* Uncompress the archive on the host machine in a temporary directory of your choice (named /tmp_dir in this doc)
* Install [Docker and Docker Compose](https://docs.docker.com/compose/install/) on the host machine and check that your installation is working.
* Create a directory for Dojo (named /dojo_dir in this doc)
* Install [Tor Browser](https://www.torproject.org/projects/torbrowser.html.en) on the host machine.
* Copy the content of the "/tmp_dir/samourai-dojo-master" directory into the "/dojo_dir" directory
* Download the most recent release of Dojo from [Github](https://github.com/Samourai-Wallet/samourai-dojo/archive/master.zip)
* Customize the configuration of your Dojo
* Uncompress the archive on the host machine in a temporary directory of your choice (named `<tmp_dir>` in this doc)
* Go to the "/dojo_dir/docker/my_dojo/conf" directory
* Create a directory for Dojo (named `<dojo_dir>` in this doc)
* Edit docker-bitcoind.conf and provide a new value for the following parameters:
* BITCOIND_RPC_USER = login protecting the access to the RPC API of your full node,
* BITCOIND_RPC_PASSWORD = password protecting the access to the RPC API of your full node.
* If your machine has a lot of RAM, it's recommended that you increase the value of BITCOIND_DB_CACHE for a faster Initial Block Download.
* Copy the content of the `<tmp_dir>/samourai-dojo-master` directory into the `<dojo_dir>` directory
* Edit docker-mysql.conf and provide a new value for the following parameters:
* MYSQL_ROOT_PASSWORD = password protecting the root account of MySQL,
* MYSQL_USER = login of the account used to access the database of your Dojo,
* MYSQL_PASSWORD = password of the account used to access the database of your Dojo.
* Customize the configuration of your Dojo
* Edit docker-node.conf and provide a new value for the following parameters:
* NODE_API_KEY = API key which will be required from your Samourai Wallet / Sentinel for its interactions with the API of your Dojo,
* NODE_ADMIN_KEY = API key which will be required from the maintenance tool for accessing a set of advanced features provided by the API of your Dojo,
* NODE_JWT_SECRET = secret used by your Dojo for the initialization of a cryptographic key signing Json Web Tokens.
* Go to the `<dojo_dir>/docker/my-dojo/conf` directory
* Edit docker-bitcoind.conf.tpl and provide a new value for the following parameters:
* `BITCOIND_RPC_USER` = login protecting the access to the RPC API of your full node,
* `BITCOIND_RPC_PASSWORD` = password protecting the access to the RPC API of your full node.
* If your machine has a lot of RAM, it's recommended that you increase the value of `BITCOIND_DB_CACHE` for a faster Initial Block Download.
* This file also provides a few additional settings for advanced setups:
* static onion address for your full node,
* bitcoind RPC API exposed to external apps,
* use of an external full node.
See this [doc](./DOCKER_advanced_setups.md) for more details.
* Edit docker-mysql.conf.tpl and provide a new value for the following parameters:
* `MYSQL_ROOT_PASSWORD` = password protecting the root account of MySQL,
* `MYSQL_USER` = login of the account used to access the database of your Dojo,
* `MYSQL_PASSWORD` = password of the account used to access the database of your Dojo.
* Edit docker-node.conf.tpl and provide a new value for the following parameters:
* `NODE_API_KEY` = API key which will be required from your Samourai Wallet / Sentinel for its interactions with the API of your Dojo,
* `NODE_ADMIN_KEY` = API key which will be required from the maintenance tool for accessing a set of advanced features provided by the API of your Dojo,
* `NODE_JWT_SECRET` = secret used by your Dojo for the initialization of a cryptographic key signing Json Web Tokens.
These parameters will protect the access to your Dojo. Be sure to provide alphanumeric values with enough entropy.
* Open the docker quickstart terminal or a terminal console and go to the "/dojo_dir/docker/my_dojo" directory. This directory contains a script named dojo.sh which will be your entrypoint for all operations related to the management of your Dojo.
* Open the docker quickstart terminal or a terminal console and go to the `<dojo_dir>/docker/my-dojo` directory. This directory contains a script named dojo.sh which will be your entrypoint for all operations related to the management of your Dojo.
* Launch the installation of your Dojo with
@ -118,6 +144,59 @@ Exit the logs with CTRL+C when the syncing of the database has completed.
* Restrict the access to your host machine as much as possible by configuring its firewall.
<a name="upgrade"/>
## Upgrade ##
This procedure allows to upgrade your Dojo with a new version.
* Stop your Dojo with
```
./dojo.sh stop
```
* Download the most recent release of Dojo from [Github](https://github.com/Samourai-Wallet/samourai-dojo/releases)
* Uncompress the archive on the host machine in a temporary directory of your choice (named `<tmp_dir>` in this doc)
* Copy the content of the `<tmp_dir>/samourai-dojo-1.x.y` directory into the `<dojo_dir>` directory (overwrite).
* Launch the upgrade of your Dojo with
```
./dojo.sh upgrade
```
Docker and Docker Compose are going to build new images and containers for your Dojo. After completion, the updated version of your Dojo will be launched automatically.
Note: The upgrade process will override all manual modifications of the files stored under the `<dojo_dir>` directory with an exception for the three configuration files stored in the `<dojo_dir>/docker/my-dojo/conf` directory.
<a name="config_files"/>
## Configuration files ##
Each new release of Dojo is packaged with 3 template files stored in the `<dojo_dir>/docker/my-dojo/conf` directory:
- docker-bitcoin.conf.tpl
- docker-mysql.conf.tpl
- docker-node.conf.tpl
These templates files define default values for configuration options of your Dojo.
During the first-time installation (dojo.sh install) these templates are used to initialize the configuration files (files with .conf extension) that will be used by your Dojo.
During an upgrade (dojo.sh upgrade), the content of the templates files is merged with the content of the configuration files, preserving the values that you may have modified in the configuration files. A backup of the configuration files is saved in the same directory (files with .save extension).
Most options provided in the configuration files can be later modified. New values will become active after a call to
```
./dojo.sh restart
```
<a name="shell_script"/>
## Dojo shell script ##
dojo.sh is a multifeature tool allowing to interact with your Dojo.
@ -131,6 +210,8 @@ Available commands:
bitcoin-cli Launch a bitcoin-cli console for interacting with bitcoind RPC API.
clean Free disk space by deleting docker dangling images and images of previous versions.
install Install your Dojo.
logs [module] [options] Display the logs of your Dojo. Use CTRL+C to stop the logs.
@ -160,28 +241,35 @@ Available commands:
uninstall Delete your Dojo. Be careful! This command will also remove all data.
upgrade Upgrade your Dojo.
version Display the version of dojo.
```
<a name="maintenance_tool"/>
## Dojo maintenance tool ##
A maintenance tool is accessible through your Tor browser at the url: <v2_onion_address>/admin
A maintenance tool is accessible through your Tor browser at the url: <v3_onion_address>/admin
The maintenance tool requires that you allow javascript for the site.
Sign in with the value entered for NODE_ADMIN_KEY.
Sign in with the value entered for `NODE_ADMIN_KEY`.
<a name="pairing"/>
## Pairing ##
## Pairing your wallet to your Dojo ##
Once the database has finished syncing, you can pair your Samourai Wallet with your Dojo in 2 steps:
* Open the maintenance tool in a Tor browser and sign in with your admin key.
* Open the maintenance tool in a Tor browser (Tor v3 onion address) and sign in with your admin key.
* Get your smartphone and launch the Samourai Wallet app. Scan the QRCode displayed in the "Pairing" tab of the maintenance tool.
<a name="network"/>
## Network connections ##
@ -191,6 +279,6 @@ If OXT is selected as the default source for imports, OXT clearnet API is access
The maintenance tool is accessed as a Tor hidden service (static onion address).
The Bitcoin node only allows incoming connections from Tor (dynamic onion address).
The Bitcoin node only allows incoming connections from Tor (ephemeral onion address).
The Bitcoin node attempts outgoing connections to both Tor and clearrnet nodes (through the Tor local proxy).
The Bitcoin node attempts outgoing connections to both Tor and clearnet nodes (through the Tor local proxy).

33
doc/GET_multiaddr.md

@ -1,10 +1,31 @@
# Get Multiaddr
Request details about a collection of HD accounts and/or loose addresses and/or public keys. If accounts do not exist, they will be created with a relayed call to the [POST /xpub](./POST_xpub.md) mechanics if new or will be imported from external data sources. Instruct the server that entities are new with `?new=xpub1|addr2|addr3` in the query parameters, and the server will skip importing for those entities. SegWit support via BIP49 is activated for new xpubs with `?bip49=xpub3|xpub4`. SegWit support via BIP84 is activated for new xpubs with `?bip84=xpub3|xpub4`. Pass xpubs to `?bip49` or `?bip84` only for newly-created accounts. Support of BIP47 (with addresses derived in 3 formats (P2PKH, P2WPKH/P2SH, P2WPKH Bech32)) is activated for new pubkeys with `?pubkey=pubkey1|pubkey2`.
Request details about a collection of HD accounts and/or loose addresses and/or pubkeys (derived in 3 formats P2PKH, P2WPKH/P2SH, P2WPKH Bech32).
## Behavior of the active parameter
If accounts passed to `?active` do not exist, they will be created with a relayed call to the [POST /xpub](./POST_xpub.md) mechanics if new or will be imported from external data sources.
If loose addresses passed to `?active` do not exist, they will be imported from external data sources.
If addresses derived from pubkeys passed to `?active` do not exist, they will be imported from external data sources.
## Declaration of new entities
Instruct the server that [BIP44](https://github.com/bitcoin/bips/blob/master/bip-0044.mediawiki) entities are new with `?new=xpub1|addr2|addr3` in the query parameters, and the server will skip importing for those entities.
SegWit support via [BIP49](https://github.com/bitcoin/bips/blob/master/bip-0049.mediawiki) is activated for new ypubs and new P2WPKH/P2SH loose addresses with `?bip49=xpub3|xpub4`.
SegWit support via [BIP84](https://github.com/bitcoin/bips/blob/master/bip-0084.mediawiki) is activated for new zpubs and new P2WPKH Bech32 loose addresses with `?bip84=xpub3|xpub4`.
Support of [BIP47](https://github.com/bitcoin/bips/blob/master/bip-0047.mediawiki) with addresses derived in 3 formats (P2PKH, P2WPKH/P2SH, P2WPKH Bech32) is activated for new pubkeys with `?pubkey=pubkey1|pubkey2`.
Note that loose addresses that are also part of one of the HD accounts requested will be ignored. Their balances and transactions are listed as part of the HD account result.
The `POST` version of multiaddr is identical, except the `active` and `new` parameters are in the POST body.
The `POST` version of multiaddr is identical, except the parameters are in the POST body.
```
@ -13,10 +34,10 @@ GET /multiaddr?active=...[&new=...][&bip49=...][&bip84=...][&pubkey=...]
## Parameters
* **active** - `string` - A pipe-separated list of extended public keys and/or loose addresses and/or pubkeys (`xpub1|address1|address2|pubkey1|...`)
* **new** - `string` - A pipe-separated list of extended public keys and/or loose addresses that need no import from external services
* **bip49** - `string` - A pipe-separated list of new extended public keys to be derived via [BIP49](https://github.com/bitcoin/bips/blob/master/bip-0049.mediawiki)
* **bip84** - `string` - A pipe-separated list of new extended public keys to be derived via [BIP84](https://github.com/bitcoin/bips/blob/master/bip-0084.mediawiki)
* **pubkey** - `string` - A pipe-separated list of public keys to be derived as P2PKH, P2WPKH/P2SH, P2WPKH Bech32 addresses.
* **new** - `string` - A pipe-separated list of **new** extended public keys to be derived via [BIP44](https://github.com/bitcoin/bips/blob/master/bip-0044.mediawiki) and/or new P2PKH loose addresses
* **bip49** - `string` - A pipe-separated list of **new** extended public keys to be derived via [BIP49](https://github.com/bitcoin/bips/blob/master/bip-0049.mediawiki) and/or new P2WPKH/P2SH loose addresses
* **bip84** - `string` - A pipe-separated list of **new** extended public keys to be derived via [BIP84](https://github.com/bitcoin/bips/blob/master/bip-0084.mediawiki) and/or new P2WPKH Bech32 loose addresses
* **pubkey** - `string` - A pipe-separated list of **new** public keys to be derived as P2PKH, P2WPKH/P2SH, P2WPKH Bech32 addresses
* **at** - `string` (optional) - Access Token (json web token). Required if authentication is activated.
### Examples

33
doc/GET_unspent.md

@ -1,6 +1,27 @@
# Get Unspent
Request a list of unspent transaction outputs from a collection of HD accounts and/or loose addresses and or public keys. If accounts do not exist, they will be created with a relayed call to the [POST /xpub](./POST_xpub.md) mechanics if new or will be imported from external data sources. Instruct the server that entities are new with `?new=xpub1|addr2|addr3` in the query parameters. SegWit support via BIP49 is activated for new xpubs with `?bip49=xpub3|xpub4`. SegWit support via BIP84 is activated for new xpubs with `?bip84=xpub3|xpub4`. Pass xpubs to `?bip49` or `?bip84` only for newly-created accounts. Support of BIP47 (with addresses derived in 3 formats (P2PKH, P2WPKH/P2SH, P2WPKH Bech32)) is activated for new pubkeys with `?pubkey=pubkey1|pubkey2`.
Request a list of unspent transaction outputs from a collection of HD accounts and/or loose addresses and/or pubkeys (derived in 3 formats P2PKH, P2WPKH/P2SH, P2WPKH Bech32).
## Behavior of the active parameter
If accounts passed to `?active` do not exist, they will be created with a relayed call to the [POST /xpub](./POST_xpub.md) mechanics if new or will be imported from external data sources.
If loose addresses passed to `?active` do not exist, they will be imported from external data sources.
If addresses derived from pubkeys passed to `?active` do not exist, they will be imported from external data sources.
## Declaration of new entities
Instruct the server that [BIP44](https://github.com/bitcoin/bips/blob/master/bip-0044.mediawiki) entities are new with `?new=xpub1|addr2|addr3` in the query parameters, and the server will skip importing for those entities.
SegWit support via [BIP49](https://github.com/bitcoin/bips/blob/master/bip-0049.mediawiki) is activated for new ypubs and new P2WPKH/P2SH loose addresses with `?bip49=xpub3|xpub4`.
SegWit support via [BIP84](https://github.com/bitcoin/bips/blob/master/bip-0084.mediawiki) is activated for new zpubs and new P2WPKH Bech32 loose addresses with `?bip84=xpub3|xpub4`.
Support of [BIP47](https://github.com/bitcoin/bips/blob/master/bip-0047.mediawiki) with addresses derived in 3 formats (P2PKH, P2WPKH/P2SH, P2WPKH Bech32) is activated for new pubkeys with `?pubkey=pubkey1|pubkey2`.
The `POST` version of unspent is identical, except the parameters are in the POST body.
@ -10,11 +31,11 @@ GET /unspent?active=...&new=...&bip49=...&bip84=...&pubkey=...
```
## Parameters
* **active** - `string` - A pipe-separated list of extended public keys and/or loose addresses (`xpub1|address1|address2|...`)
* **new** - `string` - A pipe-separated list of extended public keys and/or loose addresses that need no import from external services
* **bip49** - `string` - A pipe-separated list of new extended public keys to be derived via [BIP49](https://github.com/bitcoin/bips/blob/master/bip-0049.mediawiki)
* **bip84** - `string` - A pipe-separated list of new extended public keys to be derived via [BIP84](https://github.com/bitcoin/bips/blob/master/bip-0084.mediawiki)
* **pubkey** - `string` - A pipe-separated list of public keys to be derived as P2PKH, P2WPKH/P2SH, P2WPKH Bech32 addresses.
* **active** - `string` - A pipe-separated list of extended public keys and/or loose addresses and/or pubkeys (`xpub1|address1|address2|pubkey1|...`)
* **new** - `string` - A pipe-separated list of **new** extended public keys to be derived via [BIP44](https://github.com/bitcoin/bips/blob/master/bip-0044.mediawiki) and/or new P2PKH loose addresses
* **bip49** - `string` - A pipe-separated list of **new** extended public keys to be derived via [BIP49](https://github.com/bitcoin/bips/blob/master/bip-0049.mediawiki) and/or new P2WPKH/P2SH loose addresses
* **bip84** - `string` - A pipe-separated list of **new** extended public keys to be derived via [BIP84](https://github.com/bitcoin/bips/blob/master/bip-0084.mediawiki) and/or new P2WPKH Bech32 loose addresses
* **pubkey** - `string` - A pipe-separated list of **new** public keys to be derived as P2PKH, P2WPKH/P2SH, P2WPKH Bech32 addresses
* **at** - `string` (optional) - Access Token (json web token). Required if authentication is activated.
### Examples

8
docker/my-dojo/.env

@ -9,7 +9,13 @@
#########################################
COMPOSE_CONVERT_WINDOWS_PATHS=1
DOJO_VERSION_TAG=1.0.0
DOJO_VERSION_TAG=1.1.0
DOJO_DB_VERSION_TAG=1.1.0
DOJO_BITCOIND_VERSION_TAG=1.1.0
DOJO_NODEJS_VERSION_TAG=1.1.0
DOJO_NGINX_VERSION_TAG=1.1.0
DOJO_TOR_VERSION_TAG=1.1.0
#########################################

11
docker/my-dojo/bitcoin/Dockerfile

@ -27,18 +27,17 @@ RUN set -ex && \
tar -xzvf bitcoin.tar.gz -C /usr/local --strip-components=1 --exclude=*-qt && \
rm -rf /tmp/*
# Create group & user bitcoin
# Create groups bitcoin & tor
# Create user bitcoin and add it to groups
RUN addgroup --system -gid 1108 bitcoin && \
adduser --system --ingroup bitcoin -uid 1105 bitcoin
addgroup --system -gid 1107 tor && \
adduser --system --ingroup bitcoin -uid 1105 bitcoin && \
usermod -a -G tor bitcoin
# Create data directory
RUN mkdir "$BITCOIN_HOME/.bitcoin" && \
chown -h bitcoin:bitcoin "$BITCOIN_HOME/.bitcoin"
# Copy bitcoin config file
COPY ./bitcoin.conf "$BITCOIN_HOME/.bitcoin/bitcoin.conf"
RUN chown bitcoin:bitcoin "$BITCOIN_HOME/.bitcoin/bitcoin.conf"
# Copy restart script
COPY ./restart.sh /restart.sh
RUN chown bitcoin:bitcoin /restart.sh && \

22
docker/my-dojo/bitcoin/bitcoin.conf

@ -1,22 +0,0 @@
# Bitcoin Configuration
server=1
listen=1
bind=127.0.0.1
# Tor proxy through dojonet
proxy=172.28.1.4:9050
# Non-default RPC Port
rpcport=28256
rpcallowip=::/0
rpcbind=bitcoind
# Store transaction information for fully-spent txns
txindex=1
# No wallet
disablewallet=1
# ZeroMQ Notification Settings
zmqpubhashblock=tcp://0.0.0.0:9502
zmqpubrawtx=tcp://0.0.0.0:9501

42
docker/my-dojo/bitcoin/restart.sh

@ -2,13 +2,37 @@
set -e
echo "## Start bitcoind #############################"
bitcoind -datadir=/home/bitcoin/.bitcoin \
-dbcache=$BITCOIND_DB_CACHE \
-dnsseed=$BITCOIND_DNSSEED \
-dns=$BITCOIND_DNS \
-rpcuser=$BITCOIND_RPC_USER \
-rpcpassword=$BITCOIND_RPC_PASSWORD \
-maxconnections=$BITCOIND_MAX_CONNECTIONS \
-maxmempool=$BITCOIND_MAX_MEMPOOL \
-mempoolexpiry=$BITCOIND_MEMPOOL_EXPIRY \
bitcoind_options=(
-bind=172.28.1.5
-datadir=/home/bitcoin/.bitcoin
-dbcache=$BITCOIND_DB_CACHE
-disablewallet=1
-dns=$BITCOIND_DNS
-dnsseed=$BITCOIND_DNSSEED
-externalip=$(cat /var/lib/tor/hsv2bitcoind/hostname)
-listen=1
-maxconnections=$BITCOIND_MAX_CONNECTIONS
-maxmempool=$BITCOIND_MAX_MEMPOOL
-mempoolexpiry=$BITCOIND_MEMPOOL_EXPIRY
-minrelaytxfee=$BITCOIND_MIN_RELAY_TX_FEE
-port=8333
-proxy=172.28.1.4:9050
-rpcallowip=::/0
-rpcbind=172.28.1.5
-rpcpassword=$BITCOIND_RPC_PASSWORD
-rpcport=28256
-rpcthreads=$BITCOIND_RPC_THREADS
-rpcuser=$BITCOIND_RPC_USER
-server=1
-txindex=1
-zmqpubhashblock=tcp://0.0.0.0:9502
-zmqpubrawtx=tcp://0.0.0.0:9501
)
if [ "$BITCOIND_RPC_EXTERNAL" == "on" ]; then
bitcoind_options+=(-zmqpubhashtx=tcp://0.0.0.0:9500)
bitcoind_options+=(-zmqpubrawblock=tcp://0.0.0.0:9503)
fi
bitcoind "${bitcoind_options[@]}"

32
docker/my-dojo/conf/docker-bitcoind.conf

@ -1,32 +0,0 @@
#########################################
# CONFIGURATION OF BITCOIND CONTAINER
#########################################
# User account used for rpc access to bitcoind
# Type: alphanumeric
BITCOIND_RPC_USER=dojorpc
# Password of user account used for rpc access to bitcoind
# Type: alphanumeric
BITCOIND_RPC_PASSWORD=dojorpcpassword
# Max number of connections to network peers
# Type: integer
BITCOIND_MAX_CONNECTIONS=16
# Mempool maximum size in MB
# Type: integer
BITCOIND_MAX_MEMPOOL=1024
# Db cache size in MB
# Type: integer
BITCOIND_DB_CACHE=1024
# Mempool expiry in hours
# Defines how long transactions stay in your local mempool before expiring
# Type: integer
BITCOIND_MEMPOOL_EXPIRY=72
# Min relay tx fee in BTC
# Type: numeric
BITCOIND_MIN_RELAY_TX_FEE=0.00001

103
docker/my-dojo/conf/docker-bitcoind.conf.tpl

@ -0,0 +1,103 @@
#########################################
# CONFIGURATION OF BITCOIND CONTAINER
#########################################
# User account used for rpc access to bitcoind
# Type: alphanumeric
BITCOIND_RPC_USER=dojorpc
# Password of user account used for rpc access to bitcoind
# Type: alphanumeric
BITCOIND_RPC_PASSWORD=dojorpcpassword
# Max number of connections to network peers
# Type: integer
BITCOIND_MAX_CONNECTIONS=16
# Mempool maximum size in MB
# Type: integer
BITCOIND_MAX_MEMPOOL=1024
# Db cache size in MB
# Type: integer
BITCOIND_DB_CACHE=1024
# Number of threads to service RPC calls
# Type: integer
BITCOIND_RPC_THREADS=6
# Mempool expiry in hours
# Defines how long transactions stay in your local mempool before expiring
# Type: integer
BITCOIND_MEMPOOL_EXPIRY=72
# Min relay tx fee in BTC
# Type: numeric
BITCOIND_MIN_RELAY_TX_FEE=0.00001
#
# EXPERT SETTINGS
#
#
# EPHEMERAL ONION ADDRESS FOR BITCOIND
# THIS PARAMETER HAS NO EFFECT IF BITCOIND_INSTALL IS SET TO OFF
#
# Generate a new onion address for bitcoind when Dojo is launched
# Activation of this option is recommended for improved privacy.
# Values: on | off
BITCOIND_EPHEMERAL_HS=on
#
# EXPOSE BITCOIND RPC API AND ZMQ NOTIFICATIONS TO EXTERNAL APPS
# THESE PARAMETERS HAVE NO EFFECT IF BITCOIND_INSTALL IS SET TO OFF
#
# Expose the RPC API to external apps
# Warning: Do not expose your RPC API to internet!
# See BITCOIND_RPC_EXTERNAL_IP
# Value: on | off
BITCOIND_RPC_EXTERNAL=off
# IP address used to expose the RPC API to external apps
# This parameter is inactive if BITCOIND_RPC_EXTERNAL isn't set to 'on'
# Warning: Do not expose your RPC API to internet!
# Recommended value:
# linux: 127.0.0.1
# macos or windows: IP address of the VM running the docker host
# Type: string
BITCOIND_RPC_EXTERNAL_IP=127.0.0.1
#
# INSTALL AND RUN BITCOIND INSIDE DOCKER
#
# Install and run bitcoind inside Docker
# Set this option to 'off' for using a bitcoind hosted outside of Docker (not recommended)
# Value: on | off
BITCOIND_INSTALL=on
# IP address of bitcoind used by Dojo
# Set value to 172.28.1.5 if BITCOIND_INSTALL is set to 'on'
# Type: string
BITCOIND_IP=172.28.1.5
# Port of the RPC API
# Set value to 28256 if BITCOIND_INSTALL is set to 'on'
# Type: integer
BITCOIND_RPC_PORT=28256
# Port exposing ZMQ notifications for raw transactions
# Set value to 9501 if BITCOIND_INSTALL is set to 'on'
# Type: integer
BITCOIND_ZMQ_RAWTXS=9501
# Port exposing ZMQ notifications for block hashes
# Set value to 9502 if BITCOIND_INSTALL is set to 'on'
# Type: integer
BITCOIND_ZMQ_BLK_HASH=9502

0
docker/my-dojo/conf/docker-mysql.conf → docker/my-dojo/conf/docker-mysql.conf.tpl

0
docker/my-dojo/conf/docker-node.conf → docker/my-dojo/conf/docker-node.conf.tpl

38
docker/my-dojo/docker-compose.yaml

@ -2,7 +2,7 @@ version: "3.2"
services:
db:
image: "samouraiwallet/dojo-db:1.0.0"
image: "samouraiwallet/dojo-db:${DOJO_DB_VERSION_TAG}"
container_name: db
build:
context: ./../..
@ -19,31 +19,8 @@ services:
dojonet:
ipv4_address: 172.28.1.1
bitcoind:
image: "samouraiwallet/dojo-bitcoind:1.0.0"
container_name: bitcoind
build:
context: ./bitcoin
env_file:
- ./.env
- ./conf/docker-bitcoind.conf
restart: on-failure
command: "/wait-for-it.sh tor:9050 --timeout=360 --strict -- /restart.sh"
expose:
- "28256"
- "9501"
- "9502"
volumes:
- data-bitcoind:/home/bitcoin/.bitcoin
depends_on:
- db
- tor
networks:
dojonet:
ipv4_address: 172.28.1.5
node:
image: "samouraiwallet/dojo-nodejs:1.0.0"
image: "samouraiwallet/dojo-nodejs:${DOJO_NODEJS_VERSION_TAG}"
container_name: nodejs
build:
context: ./../..
@ -61,14 +38,13 @@ services:
volumes:
- data-nodejs:/data
depends_on:
- bitcoind
- db
networks:
dojonet:
ipv4_address: 172.28.1.2
nginx:
image: "samouraiwallet/dojo-nginx:1.0.0"
image: "samouraiwallet/dojo-nginx:${DOJO_NGINX_VERSION_TAG}"
container_name: nginx
build:
context: ./nginx
@ -89,16 +65,14 @@ services:
ipv4_address: 172.28.1.3
tor:
image: "samouraiwallet/dojo-tor:1.0.0"
image: "samouraiwallet/dojo-tor:${DOJO_TOR_VERSION_TAG}"
container_name: tor
build:
context: ./tor
env_file:
- ./.env
restart: on-failure
command: tor
ports:
- "80:80"
command: /restart.sh
volumes:
- data-tor:/var/lib/tor
networks:
@ -123,8 +97,6 @@ networks:
volumes:
data-mysql:
data-bitcoind:
data-bitcoind-tor:
data-nodejs:
data-nginx:
data-tor:

221
docker/my-dojo/dojo.sh

@ -1,72 +1,179 @@
#!/bin/bash
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
source "$DIR/conf/docker-bitcoind.conf"
# Source a file
source_file() {
if [ -f $1 ]; then
source $1
fi
}
source_file "$DIR/conf/docker-bitcoind.conf"
source_file "$DIR/.env"
# Select YAML files
select_yaml_files() {
source_file "$DIR/conf/docker-bitcoind.conf"
yamlFiles="-f $DIR/docker-compose.yaml"
if [ "$BITCOIND_INSTALL" == "on" ]; then
yamlFiles="$yamlFiles -f $DIR/overrides/bitcoind.install.yaml"
if [ "$BITCOIND_RPC_EXTERNAL" == "on" ]; then
yamlFiles="$yamlFiles -f $DIR/overrides/bitcoind.rpc.expose.yaml"
export BITCOIND_RPC_EXTERNAL_IP
fi
fi
# Return yamlFiles
echo "$yamlFiles"
}
# Docker up
docker_up() {
yamlFiles=$(select_yaml_files)
eval "docker-compose $yamlFiles up $1 -d"
}
# Start
start() {
docker-compose up --remove-orphans -d
docker_up --remove-orphans
}
# Stop
stop() {
docker exec -it bitcoind bitcoin-cli \
-rpcconnect=bitcoind \
--rpcport=28256 \
--rpcuser="$BITCOIND_RPC_USER" \
--rpcpassword="$BITCOIND_RPC_PASSWORD" \
stop
if [ "$BITCOIND_INSTALL" == "on" ]; then
if [ "$BITCOIND_EPHEMERAL_HS" = "on" ]; then
docker exec -it tor rm -rf /var/lib/tor/hsv2bitcoind
fi
echo "Preparing shutdown of dojo. Please wait."
sleep 15s
docker exec -it bitcoind bitcoin-cli \
-rpcconnect=bitcoind \
--rpcport=28256 \
--rpcuser="$BITCOIND_RPC_USER" \
--rpcpassword="$BITCOIND_RPC_PASSWORD" \
stop
docker-compose down
echo "Preparing shutdown of dojo. Please wait."
bitcoindDown=$(timeout 3m docker wait bitcoind)
if [ $bitcoindDown -eq 0 ]; then
echo "Bitcoin server stopped."
else
echo "Force shutdown of Bitcoin server."
fi
fi
yamlFiles=$(select_yaml_files)
eval "docker-compose $yamlFiles down"
}
# Restart dojo
restart() {
docker exec -it bitcoind bitcoin-cli \
-rpcconnect=bitcoind \
--rpcport=28256 \
--rpcuser="$BITCOIND_RPC_USER" \
--rpcpassword="$BITCOIND_RPC_PASSWORD" \
stop
echo "Preparing shutdown of dojo. Please wait."
sleep 15s
docker-compose down
docker-compose up -d
stop
docker_up
}
# Install
install() {
docker-compose up -d --remove-orphans
docker-compose logs --tail=0 --follow
source "$DIR/install/install-scripts.sh"
launchInstall=1
if [ -z "$1" ]; then
get_confirmation
launchInstall=$?
else
launchInstall=0
fi
if [ $launchInstall -eq 0 ]; then
init_config_files
docker_up --remove-orphans
logs
fi
}
# Delete everything
uninstall() {
docker-compose rm
docker-compose down
docker image rm samouraiwallet/dojo-db:1.0.0
docker image rm samouraiwallet/dojo-bitcoind:1.0.0
docker image rm samouraiwallet/dojo-nodejs:1.0.0
docker image rm samouraiwallet/dojo-nginx:1.0.0
docker image rm samouraiwallet/dojo-tor:1.0.0
yamlFiles=$(select_yaml_files)
eval "docker-compose $yamlFiles down"
docker image rm samouraiwallet/dojo-db:"$DOJO_DB_VERSION_TAG"
docker image rm samouraiwallet/dojo-bitcoind:"$DOJO_BITCOIND_VERSION_TAG"
docker image rm samouraiwallet/dojo-nodejs:"$DOJO_NODEJS_VERSION_TAG"
docker image rm samouraiwallet/dojo-nginx:"$DOJO_NGINX_VERSION_TAG"
docker image rm samouraiwallet/dojo-tor:"$DOJO_TOR_VERSION_TAG"
docker volume prune
}
# Clean-up (remove old docker images)
del_images_for() {
# $1: image name
# $2: most recent version of the image (do not delete this one)
docker image ls | grep "$1" | sed "s/ \+/,/g" | cut -d"," -f2 | while read -r version ; do
if [ "$2" != "$version" ]; then
docker image rm "$1:$version"
fi
done
}
clean() {
docker image prune
del_images_for samouraiwallet/dojo-db "$DOJO_DB_VERSION_TAG"
del_images_for samouraiwallet/dojo-bitcoind "$DOJO_BITCOIND_VERSION_TAG"
del_images_for samouraiwallet/dojo-nodejs "$DOJO_NODEJS_VERSION_TAG"
del_images_for samouraiwallet/dojo-nginx "$DOJO_NGINX_VERSION_TAG"
del_images_for samouraiwallet/dojo-tor "$DOJO_TOR_VERSION_TAG"
}
# Upgrade
upgrade() {
source "$DIR/install/upgrade-scripts.sh"
launchUpgrade=1
if [ -z "$1" ]; then
get_confirmation
launchUpgrade=$?
else
launchUpgrade=0
fi
if [ $launchUpgrade -eq 0 ]; then
yamlFiles=$(select_yaml_files)
update_config_files
cleanup
eval "docker-compose $yamlFiles build --no-cache"
docker_up --remove-orphans
update_dojo_db
logs
fi
}
# Display the onion address
onion() {
V2_ADDR=$( docker exec -it tor cat /var/lib/tor/hsv2dojo/hostname )
V3_ADDR=$( docker exec -it tor cat /var/lib/tor/hsv3dojo/hostname )
echo "API hidden service address (v3) = $V3_ADDR"
echo "API hidden service address (v2) = $V2_ADDR"
echo "API Hidden Service address (v3) = $V3_ADDR"
echo "API Hidden Service address (v2) = $V2_ADDR"
if [ "$BITCOIND_INSTALL" == "on" ]; then
V2_ADDR_BTCD=$( docker exec -it tor cat /var/lib/tor/hsv2bitcoind/hostname )
echo "bitcoind hidden service address (v2) = $V2_ADDR_BTCD"
fi
}
# Display the version of this dojo
version() {
echo "Dojo v$DOJO_VERSION_TAG"
}
# Display logs
@ -79,12 +186,18 @@ logs_node() {
}
logs() {
source_file "$DIR/conf/docker-bitcoind.conf"
case $1 in
db )
docker-compose logs --tail=50 --follow db
;;
bitcoind )
docker exec -ti bitcoind tail -f /home/bitcoin/.bitcoin/debug.log
if [ "$BITCOIND_INSTALL" == "on" ]; then
docker exec -ti bitcoind tail -f /home/bitcoin/.bitcoin/debug.log
else
echo -e "Command not supported for your setup.\nCause: Your Dojo is using an external bitcoind"
fi
;;
tor )
docker-compose logs --tail=50 --follow tor
@ -93,7 +206,12 @@ logs() {
logs_node $1 $2 $3
;;
* )
docker-compose logs --tail=0 --follow
yamlFiles=$(select_yaml_files)
services="nginx node tor db"
if [ "$BITCOIND_INSTALL" == "on" ]; then
services="$services bitcoind"
fi
eval "docker-compose $yamlFiles logs --tail=0 --follow $services"
;;
esac
}
@ -109,6 +227,8 @@ help() {
echo " "
echo " bitcoin-cli Launch a bitcoin-cli console allowing to interact with your full node through its RPC API."
echo " "
echo " clean Free disk space by deleting docker dangling images and images of previous versions."
echo " "
echo " install Install your dojo."
echo " "
echo " logs [module] [options] Display the logs of your dojo. Use CTRL+C to stop the logs."
@ -137,6 +257,10 @@ help() {
echo " stop Stop your dojo."
echo " "
echo " uninstall Delete your dojo. Be careful! This command will also remove all data."
echo " "
echo " upgrade Upgrade your dojo."
echo " "
echo " version Display the version of dojo"
}
@ -163,18 +287,25 @@ subcommand=$1; shift
case "$subcommand" in
bitcoin-cli )
docker exec -it bitcoind bitcoin-cli \
-rpcconnect=bitcoind \
--rpcport=28256 \
--rpcuser="$BITCOIND_RPC_USER" \
--rpcpassword="$BITCOIND_RPC_PASSWORD" \
$1 $2 $3 $4 $5
if [ "$BITCOIND_INSTALL" == "on" ]; then
docker exec -it bitcoind bitcoin-cli \
-rpcconnect=bitcoind \
--rpcport=28256 \
--rpcuser="$BITCOIND_RPC_USER" \
--rpcpassword="$BITCOIND_RPC_PASSWORD" \
$1 $2 $3 $4 $5
else
echo -e "Command not supported for your setup.\nCause: Your Dojo is using an external bitcoind"
fi
;;
help )
help
;;
clean )
clean
;;
install )
install
install $1
;;
logs )
module=$1; shift
@ -219,4 +350,10 @@ case "$subcommand" in
uninstall )
uninstall
;;
upgrade )
upgrade $1
;;
version )
version
;;
esac

34
docker/my-dojo/install/install-scripts.sh

@ -0,0 +1,34 @@
#!/bin/bash
# Confirm installation
get_confirmation() {
while true; do
echo "This operation is going to install Dojo v$DOJO_VERSION_TAG on your computer."
read -p "Do you wish to continue? [y/n]" yn
case $yn in
[Yy]* ) return 0;;
[Nn]* ) echo "Installation was cancelled."; return 1;;
* ) echo "Please answer yes or no.";;
esac
done
}
# Initialize configuration files from templates
init_config_files() {
cp ../../db-scripts/1_db.sql.tpl ../../db-scripts/1_db.sql
echo "Initialized 1_db.sql"
if [ -f ../../db-scripts/2_update.sql ]; then
rm ../../db-scripts/2_update.sql
echo "Deleted 2_update.sql"
fi
cp ./conf/docker-bitcoind.conf.tpl ./conf/docker-bitcoind.conf
echo "Initialized docker-bitcoind.conf"
cp ./conf/docker-mysql.conf.tpl ./conf/docker-mysql.conf
echo "Initialized docker-mysql.conf"
cp ./conf/docker-node.conf.tpl ./conf/docker-node.conf
echo "Initialized docker-node.conf"
}

67
docker/my-dojo/install/upgrade-scripts.sh

@ -0,0 +1,67 @@
#!/bin/bash
# Confirm upgrade operation
get_confirmation() {
while true; do
echo "This operation is going to upgrade your Dojo to v$DOJO_VERSION_TAG."
read -p "Do you wish to continue? [y/n]" yn
case $yn in
[Yy]* ) return 0;;
[Nn]* ) echo "Upgrade was cancelled."; return 1;;
* ) echo "Please answer yes or no.";;
esac
done
}
# Update configuration files from templates
update_config_files() {
if [ -f ../../db-scripts/1_db.sql ]; then
rm ../../db-scripts/1_db.sql
echo "Deleted 1_db.sql"
fi
cp ../../db-scripts/2_update.sql.tpl ../../db-scripts/2_update.sql
echo "Initialized 2_update.sql"
update_config_file ./conf/docker-bitcoind.conf ./conf/docker-bitcoind.conf.tpl
echo "Initialized docker-bitcoind.conf"
update_config_file ./conf/docker-mysql.conf ./conf/docker-mysql.conf.tpl
echo "Initialized docker-mysql.conf"
update_config_file ./conf/docker-node.conf ./conf/docker-node.conf.tpl
echo "Initialized docker-node.conf"
}
# Update a configuration file from template
update_config_file() {
sed "s/^#.*//g;s/=.*//g;/^$/d" $1 > ./original.keys.raw
grep -f ./original.keys.raw $1 > ./original.lines.raw
cp -p $1 "$1.save"
cp -p $2 $1
while IFS='=' read -r key val ; do
sed -i "s/$key=.*/$key=$val/g" "$1"
done < ./original.lines.raw
rm ./original.keys.raw
rm ./original.lines.raw
}
# Update dojo database
update_dojo_db() {
docker exec -d db /update-db.sh
}
# Clean-up
cleanup() {
#################
# Clean-up v1.1.0
#################
# Remove deprecated bitcoin.conf file
if [ -f ./bitcoin/bitcoin.conf ]; then
rm ./bitcoin/bitcoin.conf
fi
}

12
docker/my-dojo/mysql/Dockerfile

@ -1,7 +1,13 @@
FROM mysql:5.7.25
FROM mysql:5.7.25
# Copy mysql config
COPY ./docker/my-dojo/mysql/mysql-dojo.cnf /etc/mysql/conf.d/mysql-dojo.cnf
COPY ./docker/my-dojo/mysql/mysql-dojo.cnf /etc/mysql/conf.d/mysql-dojo.cnf
# Copy update-db script
COPY ./docker/my-dojo/mysql/update-db.sh /update-db.sh
RUN chmod u+x /update-db.sh && \
chmod g+x /update-db.sh
# Copy content of mysql scripts into /docker-entrypoint-initdb.d
COPY ./db-scripts/ /docker-entrypoint-initdb.d
COPY ./db-scripts/ /docker-entrypoint-initdb.d

14
docker/my-dojo/mysql/update-db.sh

@ -0,0 +1,14 @@
#!/bin/bash
for i in {30..0}; do
if echo "SELECT 1" | mysql -h"db" -u"root" -p"$MYSQL_ROOT_PASSWORD" &> /dev/null; then
break
fi
echo "MySQL init process in progress..."
sleep 1
done
if [ -f /docker-entrypoint-initdb.d/2_update.sql ]; then
mysql -h"db" -u"root" -p"$MYSQL_ROOT_PASSWORD" "$MYSQL_DATABASE" < /docker-entrypoint-initdb.d/2_update.sql
echo "Updated database with 2_update.sql"
fi

5
docker/my-dojo/nginx/dojo.conf

@ -44,6 +44,11 @@ server {
proxy_pass http://node:8080/;
}
# Redirect onion address to maintenance tool
location = / {
return 301 /admin;
}
# Serve remaining requests
location / {
return 200 '{"status":"ok"}';

2
docker/my-dojo/nginx/nginx.conf

@ -24,7 +24,7 @@ http {
sendfile on;
keepalive_timeout 65;
keepalive_timeout 95;
# Enable response compression
gzip on;

2
docker/my-dojo/node/Dockerfile

@ -20,7 +20,7 @@ COPY . "$APP_DIR"
# Install node modules required by the app
RUN cd "$APP_DIR" && \
npm install --only=prod
npm ci --only=production
# Copy config file
COPY ./docker/my-dojo/node/keys.index.js "$APP_DIR/keys/index.js"

8
docker/my-dojo/node/keys.index.js

@ -27,14 +27,14 @@ module.exports = {
// Password
pass: process.env.BITCOIND_RPC_PASSWORD,
// IP address
host: 'bitcoind',
host: process.env.BITCOIND_IP,
// TCP port
port: 28256
port: parseInt(process.env.BITCOIND_RPC_PORT)
},
// ZMQ Tx notifications
zmqTx: 'tcp://bitcoind:9501',
zmqTx: `tcp://${process.env.BITCOIND_IP}:${process.env.BITCOIND_ZMQ_RAWTXS}`,
// ZMQ Block notifications
zmqBlk: 'tcp://bitcoind:9502',
zmqBlk: `tcp://${process.env.BITCOIND_IP}:${process.env.BITCOIND_ZMQ_BLK_HASH}`,
// Fee type (estimatesmartfee)
feeType: process.env.NODE_FEE_TYPE
},

34
docker/my-dojo/overrides/bitcoind.install.yaml

@ -0,0 +1,34 @@
version: "3.2"
services:
bitcoind:
image: "samouraiwallet/dojo-bitcoind:${DOJO_BITCOIND_VERSION_TAG}"
container_name: bitcoind
build:
context: ./bitcoin
env_file:
- ./.env
- ./conf/docker-bitcoind.conf
restart: on-failure
command: "/wait-for-it.sh tor:9050 --timeout=360 --strict -- /restart.sh"
expose:
- "8333"
- "28256"
- "9501"
- "9502"
volumes:
- data-bitcoind:/home/bitcoin/.bitcoin
- data-tor:/var/lib/tor
depends_on:
- db
- tor
networks:
dojonet:
ipv4_address: 172.28.1.5
node:
depends_on:
- bitcoind
volumes:
data-bitcoind:

10
docker/my-dojo/overrides/bitcoind.rpc.expose.yaml

@ -0,0 +1,10 @@
version: "3.2"
services:
bitcoind:
ports:
- "${BITCOIND_RPC_EXTERNAL_IP}:28256:28256"
- "${BITCOIND_RPC_EXTERNAL_IP}:9500:9500"
- "${BITCOIND_RPC_EXTERNAL_IP}:9501:9501"
- "${BITCOIND_RPC_EXTERNAL_IP}:9502:9502"
- "${BITCOIND_RPC_EXTERNAL_IP}:9503:9503"

14
docker/my-dojo/tor/Dockerfile

@ -23,11 +23,6 @@ RUN set -ex && \
RUN addgroup --system -gid 1107 tor && \
adduser --system --ingroup tor -uid 1104 tor
# Create group & user bitcoin and add user to tor group
RUN addgroup --system -gid 1108 bitcoin && \
adduser --system --ingroup bitcoin -uid 1105 bitcoin && \
usermod -a -G tor bitcoin
# Create /etc/tor directory
RUN mkdir -p /etc/tor/ && \
chown -Rv tor:tor /etc/tor
@ -35,12 +30,19 @@ RUN mkdir -p /etc/tor/ && \
# Create .tor subdirectory of TOR_HOME
RUN mkdir -p "$TOR_HOME/.tor" && \
chown -Rv tor:tor "$TOR_HOME" && \
chmod -R 700 "$TOR_HOME"
chmod -R 750 "$TOR_HOME"
# Copy Tor configuration file
COPY ./torrc /etc/tor/torrc
RUN chown tor:tor /etc/tor/torrc
# Copy restart script
COPY ./restart.sh /restart.sh
RUN chown tor:tor /restart.sh && \
chmod u+x /restart.sh && \
chmod g+x /restart.sh
# Copy wait-for-it script
COPY ./wait-for-it.sh /wait-for-it.sh

9
docker/my-dojo/tor/restart.sh

@ -0,0 +1,9 @@
#!/bin/bash
set -e
# Set permissions for bitcoind
echo "## Set permissions on /var/lib/tor dir ###"
chmod 750 /var/lib/tor
echo "## Start tor #############################"
tor

17
docker/my-dojo/tor/torrc

@ -19,17 +19,7 @@ SocksPolicy reject *
## things in $HOME/.tor on Unix, and in Application Data\tor on Windows.
DataDirectory /var/lib/tor/.tor
## The port on which Tor will listen for local connections from Tor
## controller applications, as documented in control-spec.txt.
ControlPort 9051
## If you enable the controlport, be sure to enable one of these
## authentication methods, to prevent attackers from accessing it.
CookieAuthentication 1
CookieAuthFileGroupReadable 1
DataDirectoryGroupReadable 1
############### This section is just for location-hidden services ###
@ -47,3 +37,8 @@ HiddenServicePort 80 172.29.1.3:80
HiddenServiceDir /var/lib/tor/hsv3dojo
HiddenServiceVersion 3
HiddenServicePort 80 172.29.1.3:80
HiddenServiceDir /var/lib/tor/hsv2bitcoind
HiddenServiceVersion 2
HiddenServicePort 8333 172.28.1.5:8333
HiddenServiceDirGroupReadable 1

2
keys/index-example.js

@ -15,7 +15,7 @@ module.exports = {
/*
* Dojo version
*/
dojoVersion: '1.0.0',
dojoVersion: '1.1.0',
/*
* Bitcoind
*/

30
lib/db/mysql-db-wrapper.js

@ -931,6 +931,36 @@ class MySqlDbWrapper {
return ret
}
/**
* Get the number of indices derived in an interval for a HD chain
* @param {string} xpub - xpub
* @param {integer} chain - HD chain (0 or 1)
* @param {integer} minIdx - min index of derivation
* @param {integer} maxIdx - max index of derivation
* @returns {integer[]} returns an array of number of derived indices
*/
async getHDAccountNbDerivedIndices(xpub, chain, minIdx, maxIdx) {
const sqlQuery = 'SELECT \
COUNT(`hd_addresses`.`hdAddrIndex`) AS `nbDerivedIndices` \
FROM `hd_addresses` \
INNER JOIN `hd` ON `hd_addresses`.`hdID` = `hd`.`hdID` \
WHERE `hd`.`hdXpub` = ? \
AND `hd_addresses`.`hdAddrChain` = ? \
AND `hd_addresses`.`hdAddrIndex` >= ? \
AND `hd_addresses`.`hdAddrIndex` <= ?'
const params = [xpub, chain, minIdx, maxIdx]
const query = mysql.format(sqlQuery, params)
const results = await this._query(query)
if (results.length == 1) {
const nbDerivedIndices = results[0].nbDerivedIndices
return (nbDerivedIndices == null) ? 0 : nbDerivedIndices
}
return 0
}
/**
* Get the number of transactions for an HD account
* @param {string} xpub - xpub

73
lib/remote-importer/btccom-wrapper.js

@ -66,11 +66,11 @@ class BtcComWrapper extends Wrapper {
* { address: <bitcoin_address>, txids: <txids>, ntx: <total_nb_txs>}
*/
async getAddress(address, filterAddr) {
// Extracts the scripthash from the bech32 address
// (btc.com api manages scripthashes, not bech32 addresses)
const scripthash = addrHelper.getScriptHashFromBech32(address)
const reqAddr = addrHelper.isBech32(address)
? addrHelper.getScriptHashFromBech32(address)
: address
const uri = `/address/${scripthash}`
const uri = `/address/${reqAddr}`
const result = await this._get(uri)
const ret = {
@ -92,7 +92,7 @@ class BtcComWrapper extends Wrapper {
const listPages = Array.from(aPages, (val, idx) => idx + 1)
const results = await util.seriesCall(listPages, idx => {
return this._getTxsForAddress(scripthash, idx)
return this._getTxsForAddress(reqAddr, idx)
})
for (let txids of results)
@ -103,15 +103,72 @@ class BtcComWrapper extends Wrapper {
/**
* Retrieve information for a given list of addresses
* @param {string} addresses - array of bitcoin addresses
* @param {string[]} addresses - array of bitcoin addresses
* @param {boolean} filterAddr - True if an upper bound should be used
* for #transactions associated to the address, False otherwise
* @returns {Promise} returns an array of objects
* { address: <bitcoin_address>, txids: <txids>, ntx: <total_nb_txs>}
*/
async getAddresses(addresses, filterAddr) {
// Not implemented for this api
throw "Not implemented"
const ret = []
const reqAddresses = []
const xlatedBech32Addr = {}
for (let a of addresses) {
if (addrHelper.isBech32(a)) {
const scriptHash = addrHelper.getScriptHashFromBech32(a)
reqAddresses.push(scriptHash)
xlatedBech32Addr[scriptHash] = a
} else {
reqAddresses.push(a)
}
}
// Send a batch request for all the addresses
const strReqAddresses = reqAddresses.join(',')
const uri = `/address/${strReqAddresses}`
const results = await this._get(uri)
const foundAddresses = Array.isArray(results.data)
? results.data
: [results.data]
for (let a of foundAddresses) {
if (a && a.tx_count > 0) {
// Translate bech32 address
const address = xlatedBech32Addr.hasOwnProperty(a.address)
? xlatedBech32Addr[a.address]
: a.address
if (a.tx_count <= 2) {
// Less than 3 transactions for this address
// all good
const retAddr = {
address: address,
ntx: a.tx_count,
txids: []
}
retAddr.txids = (a.tx_count == 1)
? [a.first_tx]
: [a.first_tx, a.last_tx]
ret.push(retAddr)
} else {
// More than 2 transactions for this address
// We need more requests to the API
if (filterAddr && a.tx_count > keys.addrFilterThreshold) {
Logger.info(` import of ${address} rejected (too many transactions - ${a.tx_count})`)
} else {
const retAddr = await this.getAddress(address)
ret.push(retAddr)
}
}
}
}
return ret
}
}

66
lib/remote-importer/sources-mainnet.js

@ -4,9 +4,7 @@
*/
'use strict'
const addrHelper = require('../bitcoin/addresses-helper')
const network = require('../bitcoin/network')
const util = require('../util')
const Logger = require('../logger')
const keys = require('../../keys')[network.key]
const Sources = require('./sources')
@ -24,8 +22,6 @@ class SourcesMainnet extends Sources {
*/
constructor() {
super()
// Initializes external source
this.source = null
this._initSource()
}
@ -45,68 +41,6 @@ class SourcesMainnet extends Sources {
}
}
/**
* Retrieve information for a given address
* @param {string} address - bitcoin address
* @param {boolean} filterAddr - True if an upper bound should be used
* for #transactions associated to the address, False otherwise
* @returns {Promise} returns an object
* { address: <bitcoin_address>, txids: <txids>, ntx: <total_nb_txs>}
*/
async getAddress(address, filterAddr) {
const ret = {
address,
txids: [],
ntx: 0
}
try {
const result = await this.source.getAddress(address, filterAddr)
if (result.ntx)
ret.ntx = result.ntx
else if (result.txids)
ret.ntx = result.txids.length
if (result.txids)
ret.txids = result.txids
} catch(e) {
//Logger.error(e, `SourcesMainnet.getAddress() : ${address} from ${this.source.base}`)
Logger.error(null, `SourcesMainnet.getAddress() : ${address} from ${this.source.base}`)
} finally {
return ret
}
}
/**
* Retrieve information for a list of addresses
* @param {string[]} addresses - array of bitcoin address
* @param {boolean} filterAddr - True if an upper bound should be used
* for #transactions associated to the address, False otherwise
* @returns {Promise} returns an object
* { address: <bitcoin_address>, txids: <txids>, ntx: <total_nb_txs>}
*/
async getAddresses(addresses, filterAddr) {
const ret = []
try {
const results = await this.source.getAddresses(addresses, filterAddr)
for (let r of results) {
// Filter addresses with too many txs
if (!filterAddr || (r.ntx <= keys.addrFilterThreshold))
ret.push(r)
}
} catch(e) {
//Logger.error(e, `SourcesMainnet.getAddresses() : ${addresses} from ${this.source.base}`)
Logger.error(null, `SourcesMainnet.getAddresses() : ${addresses} from ${this.source.base}`)
} finally {
return ret
}
}
}
module.exports = SourcesMainnet

124
lib/remote-importer/sources-testnet.js

@ -4,19 +4,17 @@
*/
'use strict'
const addrHelper = require('../bitcoin/addresses-helper')
const network = require('../bitcoin/network')
const util = require('../util')
const Logger = require('../logger')
const keys = require('../../keys')[network.key]
const Sources = require('./sources')
const BitcoindWrapper = require('./bitcoind-wrapper')
const InsightWrapper = require('./insight-wrapper')
const BtcComWrapper = require('./btccom-wrapper')
/**
* Remote data sources for testnet polled round-robin to spread load
* Remote data sources for testnet
*/
class SourcesTestnet extends Sources {
@ -25,126 +23,22 @@ class SourcesTestnet extends Sources {
*/
constructor() {
super()
this.sources = []
this.index = 0
this.sourceBech32 = null
this.isBitcoindActive = false
// Initializes external sources
this._initSources()
this._initSource()
}
/**
* Initialize the external data sources
* Initialize the external data source
*/
_initSources() {
_initSource() {
if (keys.explorers.bitcoind == 'active') {
// If local bitcoind option is activated
// we'll use the local node as our unique source
this.sourceBech32 = new BitcoindWrapper()
this.sources.push(this.sourceBech32)
this.isBitcoindActive = true
this.source = new BitcoindWrapper()
Logger.info('Activated Bitcoind as the data source for imports')
} else {
// Otherwise, we use a set of insight servers + btc.com for bech32 addresses
this.sourceBech32 = new BtcComWrapper(keys.explorers.btccom)
for (let url of keys.explorers.insight)
this.sources.push(new InsightWrapper(url))
this.isBitcoindActive = false
}
}
/**
* Get the next source index
* @returns {integer} returns the next source index
*/
nextIndex() {
this.index++
if (this.index >= this.sources.length)
this.index = 0
return this.index
}
/**
* Retrieve information for a given address
* @param {string} address - bitcoin address
* @param {boolean} filterAddr - True if an upper bound should be used
* for #transactions associated to the address, False otherwise
* @returns {Promise} returns an object
* { address: <bitcoin_address>, txids: <txids>, ntx: <total_nb_txs>}
*/
async getAddress(address, filterAddr) {
let source = ''
const isBech32 = addrHelper.isBech32(address)
const ret = {
address,
txids: [],
ntx: 0
}
try {
source = isBech32 ? this.sourceBech32 : this.sources[this.nextIndex()]
const result = await source.getAddress(address, filterAddr)
if (result.ntx)
ret.ntx = result.ntx
else if (result.txids)
ret.ntx = result.txids.length
if (result.txids)
ret.txids = result.txids
return ret
} catch(e) {
Logger.error(e, `SourcesTestnet.getAddress() : ${address} from ${source.base}`)
if (!isBech32 && this.sources.length > 1) {
// Try again with another source
return this.getAddress(address, filterAddr)
} else {
return ret
}
}
}
/**
* Retrieve information for a list of addresses
* @param {string[]} addresses - array of bitcoin address
* @param {boolean} filterAddr - True if an upper bound should be used
* for #transactions associated to the address, False otherwise
* @returns {Promise} returns an object
* { address: <bitcoin_address>, txids: <txids>, ntx: <total_nb_txs>}
*/
async getAddresses(addresses, filterAddr) {
const ret = []
try {
if (this.isBitcoindActive) {
const source = this.sources[0]
const results = await source.getAddresses(addresses, filterAddr)
for (let r of results) {
// Filter addresses with too many txs
if (!filterAddr || (r.ntx <= keys.addrFilterThreshold))
ret.push(r)
}
} else {
const lists = util.splitList(addresses, this.sources.length)
await util.seriesCall(lists, async list => {
const results = await Promise.all(list.map(a => {
return this.getAddress(a, filterAddr)
}))
for (let r of results) {
// Filter addresses with too many txs
if (!filterAddr || (r.ntx <= keys.addrFilterThreshold))
ret.push(r)
}
})
}
} catch (e) {
Logger.error(e, `SourcesTestnet.getAddresses() : Addr list = ${addresses}`)
} finally {
return ret
// Otherwise, we'll use the rest api provided by OXT
this.source = new BtcComWrapper(keys.explorers.btccom)
Logger.info('Activated BTC.COM API as the data source for imports')
}
}

54
lib/remote-importer/sources.js

@ -4,16 +4,22 @@
*/
'use strict'
const network = require('../bitcoin/network')
const Logger = require('../logger')
const keys = require('../../keys')[network.key]
/**
* Abstract class defining a list of blockchain explorer providing a remote API
* Base class defining data source for imports/rescans of HD accounts and addresses
*/
class Sources {
/**
* Constructor
*/
constructor() {}
constructor() {
this.source = null
}
/**
* Retrieve information for a given address
@ -23,7 +29,30 @@ class Sources {
* @returns {Promise} returns an object
* { address: <bitcoin_address>, txids: <txids>, ntx: <total_nb_txs>}
*/
async getAddress(address, filterAddr) {}
async getAddress(address, filterAddr) {
const ret = {
address,
txids: [],
ntx: 0
}
try {
const result = await this.source.getAddress(address, filterAddr)
if (result.ntx)
ret.ntx = result.ntx
else if (result.txids)
ret.ntx = result.txids.length
if (result.txids)
ret.txids = result.txids
} catch(e) {
Logger.error(null, `Sources.getAddress() : ${address} from ${this.source.base}`)
} finally {
return ret
}
}
/**
* Retrieve information for a list of addresses
@ -33,7 +62,24 @@ class Sources {
* @returns {Promise} returns an object
* { address: <bitcoin_address>, txids: <txids>, ntx: <total_nb_txs>}
*/
async getAddresses(addresses, filterAddr) {}
async getAddresses(addresses, filterAddr) {
const ret = []
try {
const results = await this.source.getAddresses(addresses, filterAddr)
for (let r of results) {
// Filter addresses with too many txs
if (!filterAddr || (r.ntx <= keys.addrFilterThreshold))
ret.push(r)
}
} catch(e) {
Logger.error(e, `Sources.getAddresses() : ${addresses} from ${this.source.base}`)
} finally {
return ret
}
}
}

2302
package-lock.json

File diff suppressed because it is too large

3
package.json

@ -1,6 +1,6 @@
{
"name": "samourai-dojo",
"version": "1.0.0",
"version": "1.1.0",
"description": "Backend server for Samourai Wallet",
"main": "accounts/index.js",
"scripts": {
@ -25,6 +25,7 @@
"generic-pool": "3.4.2",
"heapdump": "0.3.9",
"helmet": "3.12.1",
"lodash": "4.17.14",
"lru-cache": "4.0.2",
"mysql": "2.16.0",
"passport": "0.4.0",

28
pushtx/orchestrator.js

@ -82,16 +82,21 @@ class Orchestrator {
Logger.info(`Block ${height} ${blockHash}`)
// Retrieve the transactions triggered by this block
let txs = await db.getActivatedScheduledTransactions(height)
while (txs && txs.length > 0) {
let rpcConnOk = true
let nbTxsPushed
let rpcConnOk = true
do {
nbTxsPushed = 0
// Retrieve the transactions triggered by this block
let txs = await db.getActivatedScheduledTransactions(height)
if (!(txs && txs.length > 0))
break
for (let tx of txs) {
let hasParentTx = (tx.schParentTxid != null) && (tx.schParentTxid != '')
let parentTx = null
// Check if previous transaction has been confirmed
if (hasParentTx) {
try {
@ -132,20 +137,15 @@ class Orchestrator {
// Delete the transaction
try {
await db.deleteScheduledTransaction(tx.schTxid)
// Count the transaction as successfully processed
nbTxsPushed++
} catch(e) {
const msg = 'A problem was met while trying to delete a scheduled transaction'
Logger.error(e, `Orchestrator.onBlockHash() : ${msg}`)
}
}
}
// If a connection issue was detected, then stop the loop
if (!rpcConnOk)
break
// Check if more transactions have to be pushed
txs = await db.getActivatedScheduledTransactions(height)
}
} while (rpcConnOk && nbTxsPushed > 0)
} catch(e) {
Logger.error(e, 'Orchestrator.onBlockHash() : Error')

20
static/admin/index.js

@ -9,7 +9,7 @@ function login() {
// Checks input fields
if (!apiKey) {
lib_msg.displayErrors('API key is mandatory');
lib_msg.displayErrors('Admin key is mandatory');
return;
}
@ -20,13 +20,17 @@ function login() {
function (result) {
const auth = result['authorizations'];
const accessToken = auth['access_token'];
lib_auth.setAccessToken(accessToken);
const refreshToken = auth['refresh_token'];
lib_auth.setRefreshToken(refreshToken);
sessionStorage.setItem('activeTab', '');
lib_msg.displayInfo('Successfully connected to your backend');
// Redirection to default page
lib_cmn.goToDefaultPage();
if (lib_auth.isAdmin(accessToken)) {
lib_auth.setAccessToken(accessToken);
const refreshToken = auth['refresh_token'];
lib_auth.setRefreshToken(refreshToken);
sessionStorage.setItem('activeTab', '');
lib_msg.displayInfo('Successfully connected to your backend');
// Redirection to default page
lib_cmn.goToDefaultPage();
} else {
lib_msg.displayErrors('You must sign in with the admin key');
}
},
function (jqxhr) {
let msg = lib_msg.extractJqxhrErrorMsg(jqxhr);

33
static/admin/lib/auth-utils.js

@ -12,6 +12,9 @@ var lib_auth = {
/* JWT Scheme */
JWT_SCHEME: 'Bearer',
/* Admin profile */
TOKEN_PROFILE_ADMIN: 'admin',
/*
* Retrieves access token from session storage
@ -87,6 +90,36 @@ var lib_auth = {
return (token && (token != 'null')) ? true : false;
},
/*
* Extract the payload of an access token
* in json format
*/
getPayloadAccessToken: function(token) {
if (!token)
token = this.getAccessToken();
if (!token)
return null;
try {
const payloadBase64 = token.split('.')[1];
const payloadUtf8 = atob(payloadBase64);
return JSON.parse(payloadUtf8);
} catch {
return null;
}
},
/*
* Check if user has admin profile
*/
isAdmin: function(token) {
const payload = this.getPayloadAccessToken(token);
if (!payload)
return false;
return (('prf' in payload) && (payload['prf'] == this.TOKEN_PROFILE_ADMIN));
},
/*
* Local logout
*/

12
test/lib/bitcoin/addresses-helper-test.js

@ -80,21 +80,21 @@ const VECTOR_5 = [
describe('AddressesHelper', function() {
describe('p2pkhAddress()', function() {
it('should successfully derive P2PKH addresses from pubkeys', function() {
for (const v of VECTOR_1) {
const pkb = new Buffer(v[0], 'hex')
const pkb = Buffer.from(v[0], 'hex')
const addr = addrHelper.p2pkhAddress(pkb)
assert(addr == v[1])
}
})
})
describe('p2wpkhP2shAddress()', function() {
it('should successfully derive P2WPKH-P2SH addresses from pubkeys', function() {
for (const v of VECTOR_1) {
const pkb = new Buffer(v[0], 'hex')
const pkb = Buffer.from(v[0], 'hex')
const addr = addrHelper.p2wpkhP2shAddress(pkb)
assert(addr == v[2])
}
@ -104,7 +104,7 @@ describe('AddressesHelper', function() {
describe('p2wpkhAddress()', function() {
it('should successfully derive bech32 addresses from pubkeys', function() {
for (const v of VECTOR_1) {
const pkb = new Buffer(v[0], 'hex')
const pkb = Buffer.from(v[0], 'hex')
const addr = addrHelper.p2wpkhAddress(pkb)
assert(addr == v[3])
}
@ -150,7 +150,7 @@ describe('AddressesHelper', function() {
const expectedResult = stc[2]
const sig = btcMessage.sign(msg, prefix, privKey, true)
// Check that library returns valid result
assert((sig.compare(targetSig) == 0) == expectedResult)

8
tracker/blockchain-processor.js

@ -55,11 +55,9 @@ class BlockchainProcessor extends AbstractProcessor {
* @returns {Promise}
*/
async catchup() {
// Get highest block in the blockchain
const info = await this.client.getblockchaininfo()
// Consider that we are in IBD mode if bitcoind is far in the past
this.isIBD = info.initialblockdownload && (info.blocks < 550000)
// Consider that we are in IBD mode if Dojo is far in the past
const highest = await db.getHighestBlock()
this.isIBD = highest.blockHeight < 570000
if (this.isIBD)
return this.catchupIBDMode()

21
tracker/transaction.js

@ -333,9 +333,24 @@ class Transaction {
if (chainMaxUsedIndex < unusedIndices[chain])
continue
// If max derived index is beyond max used index plus gap limit, move on
if (derivedIndices[chain] >= chainMaxUsedIndex + gapLimit[chain])
continue
// If max derived index is beyond max used index plus gap limit.
if (derivedIndices[chain] >= chainMaxUsedIndex + gapLimit[chain]) {
// Check that we don't have a hole in the next <gapLimit> indices
const nbDerivedIndicesForward = await db.getHDAccountNbDerivedIndices(
xpub,
chain,
chainMaxUsedIndex,
chainMaxUsedIndex + gapLimit[chain]
)
if (nbDerivedIndicesForward < gapLimit[chain] + 1) {
// Hole detected. Force derivation.
derivedIndices[chain] = chainMaxUsedIndex
} else {
// Move on
continue
}
}
let done

Loading…
Cancel
Save