diff --git a/.gitignore b/.gitignore index 28e5ac1..eaac5fa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,14 +1,13 @@ db-scripts/updates/ db-scripts/1_db.sql db-scripts/2_update.sql -docker/my-dojo/conf/docker-bitcoind.conf -docker/my-dojo/conf/docker-mysql.conf -docker/my-dojo/conf/docker-node.conf +docker/my-dojo/conf/*.conf +docker/my-dojo/conf/*.conf.save keys/index.js keys/sslcert/ node_modules/ private-tests/ static/admin/conf/index.js +static/admin-legacy static/admin-legacy/ *.log -static/admin-legacy diff --git a/RELEASES.md b/RELEASES.md index 9560794..5078ee5 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -3,6 +3,7 @@ ## Releases ## +- [v1.9.0](#1_9_0) - [v1.8.1](#1_8_1) - [v1.8.0](#1_8_0) - [v1.7.0](#1_7_0) @@ -15,6 +16,151 @@ - [v1.1.0](#1_1_0) + + +## Samourai Dojo v1.9.0 ## + + +### Notable changes ### + + +#### Maintenance Tool: multiple UX improvements #### + +*Status screen* + +The status screen now displays information related to the Dojo database and to the data source used by Dojo for its imports and rescans. This screen provides a high level view of the state of the Dojo instance, that can be shared for support. + +*XPUB Tool* + +- Progress made is now displayed during an import or a rescan. +- New feature allowing to delete a XPUB tracked by Dojo. +- Improved management of timeouts by the authentication system. + + +#### Dojo Shell Script: improvements #### + +- Script automatically stops if build fails during install/upgrade operation. +- Script returns a not null exit code if build fails or if install/upgrade operation is cancelled. +- Dojo is automatically stopped if an upgrade operation is launched with Dojo up and running. +- A cleanup of old Dojo versions is automatically processed at the end of successful upgrade operations. + + +#### New configuration options #### + +Addition of two new configuration options: + +- BITCOIND_LISTEN_MODE (in docker-bitcoind.conf): When set to `off`, the fullnode will refuse incoming connections. Default = `on`. +- WHIRLPOOL_COORDINATOR_ONION (in docker-whirlpool.conf): When set to `on`, whirlpool-cli will contact the coordinator through its onion address. When set to `off`, clearnet address will be used (through Tor). Default = `on`. + + +#### Extended support Tor hidden services #### + +Dojo now provides a v2 and v3 hidden service for: + +- Dojo Maintenance Tool and API +- Whirlpool CLI +- Bitcoind +- Explorer + +Tor v3 onion addresses are recommended but v2 addresses can be used in the case of new attacks disrupting v3 hidden services. + +These onion addresses can be retrieved thanks to the `onion` command of the Dojo Shell Script + +''' +# Display Tor v3 onion addresses (default) +> ./dojo.h onion + +# Display Tor v3 onion addresses +> ./dojo.h onion v3 + +# Display Tor v2 onion addresses +> ./dojo.h onion v2 +''' + + +#### Upgrade of bitcoind to v0.21.0 #### + +Upgrade to Bitcoin Core v0.21.0 + + +#### Upgrade of whirlpool to v0.10.9 #### + +Upgrade to whirlpool-cli 0.10.9 + + +#### Upgrade of explorer to v2.1.0 #### + +Upgrade to btc-rpc-explorer 2.1.0 + + +#### Upgrade of tor to v0.4.4.7 #### + +Tor 0.4.4.7 fixes and mitigates multiple issues, including one that made v3 onion services more susceptible to denial-of-service attacks. + + +#### Upgrade of indexer to v0.4.0 #### + +Upgrade to addrindexrs v0.4.0 + + +### Change log ### + + +#### MyDojo #### + +- [#mr165](https://code.samourai.io/dojo/samourai-dojo/-/merge_requests/165) improve dmt ux +- [#mr166](https://code.samourai.io/dojo/samourai-dojo/-/merge_requests/166) add new configuration property BITCOIND_LISTEN_MODE +- [#mr167](https://code.samourai.io/dojo/samourai-dojo/-/merge_requests/167) upgrade explorer to btc-rpc-explorer 2.0.2 +- [#mr168](https://code.samourai.io/dojo/samourai-dojo/-/merge_requests/168) add new getChaintipHeight() method to remote importer and data sources +- [#mr170](https://code.samourai.io/dojo/samourai-dojo/-/merge_requests/170) add indexer info to /status endpoint +- [#mr171](https://code.samourai.io/dojo/samourai-dojo/-/merge_requests/171) add db and indexer blocks to status screen of dmt +- [#mr172](https://code.samourai.io/dojo/samourai-dojo/-/merge_requests/172) upgrade indexer to addrindexrs 0.4.0 +- [#mr174](https://code.samourai.io/dojo/samourai-dojo/-/merge_requests/174) upgrade whirlpool to whirlpool-cli 0.10.9 +- [#mr175](https://code.samourai.io/dojo/samourai-dojo/-/merge_requests/175) track and display progress of import/rescan +- [#mr178](https://code.samourai.io/dojo/samourai-dojo/-/merge_requests/178) improve dojo shell script +- [#mr179](https://code.samourai.io/dojo/samourai-dojo/-/merge_requests/179) update samourai logo +- [#mr181](https://code.samourai.io/dojo/samourai-dojo/-/merge_requests/181) add support of xpub deletion from the dmt +- [#mr182](https://code.samourai.io/dojo/samourai-dojo/-/merge_requests/182) upgrade bitcoin container with bitcoin core 0.21.0 +- [#mr183](https://code.samourai.io/dojo/samourai-dojo/-/merge_requests/183) upgrade explorer to btc-rpc-explorer 2.1.0 +- [#mr184](https://code.samourai.io/dojo/samourai-dojo/-/merge_requests/184) upgrade tor to v0.4.4.6 +- [#mr186](https://code.samourai.io/dojo/samourai-dojo/-/merge_requests/186) upgrade tor to v0.4.4.6 +- [#mr188](https://code.samourai.io/dojo/samourai-dojo/-/merge_requests/188) return exit code 2 if install or upgrade is cancelled +- [#mr190](https://code.samourai.io/dojo/samourai-dojo/-/merge_requests/190) add new WHIRLPOOL_COORDINATOR_ONION config option +- [#mr191](https://code.samourai.io/dojo/samourai-dojo/-/merge_requests/191) add v2 onion addresses for explorer and whirlpool +- [#mr192](https://code.samourai.io/dojo/samourai-dojo/-/merge_requests/192) return exit code 1 instead of 2 for aborted install & upgrade +- [#mr193](https://code.samourai.io/dojo/samourai-dojo/-/merge_requests/193) reactivate tor v2 hidden service for bitcoind +- [#mr194](https://code.samourai.io/dojo/samourai-dojo/-/merge_requests/194) upgrade tor to v0.4.4.7 +- [#mr195](https://code.samourai.io/dojo/samourai-dojo/-/merge_requests/195) bump block height defining ibd mode +- [9fe22a35](https://code.samourai.io/dojo/samourai-dojo/-/commit/9fe22a356625e0c1aeb18691d617af9118990c84) update .gitignore + + +#### Bug fixes #### + +- [#mr176](https://code.samourai.io/dojo/samourai-dojo/-/merge_requests/176) check that jqxhr['responseJSON']['error'] is a string +- [#mr177](https://code.samourai.io/dojo/samourai-dojo/-/merge_requests/177) prevent restart of bitcoin container if bitcoind fails +- [#mr185](https://code.samourai.io/dojo/samourai-dojo/-/merge_requests/185) build addrindexrs with --locked argument +- [#mr189](https://code.samourai.io/dojo/samourai-dojo/-/merge_requests/189) return a 0 feerate if bitcoind doesn't return an estimate + + +#### Security #### + +- [#mr173](https://code.samourai.io/dojo/samourai-dojo/-/merge_requests/173) replace request-promise-native by axios + + +#### Documentation #### + +- [#mr180](https://code.samourai.io/dojo/samourai-dojo/-/merge_requests/180) add a detailed installation and upgrade guide for ubuntu + + +#### Credits ### + +- BTCxZelko +- flatcloud0b3 +- kenshin-samourai +- LaurentMT +- likewhoa + + ## Samourai Dojo v1.8.1 ## diff --git a/accounts/status.js b/accounts/status.js index 5c5fffa..f07e0b0 100644 --- a/accounts/status.js +++ b/accounts/status.js @@ -4,8 +4,12 @@ */ 'use strict' +const network = require('../lib/bitcoin/network') +const keys = require('../keys')[network.key] const util = require('../lib/util') +const Logger = require('../lib/logger') const db = require('../lib/db/mysql-db-wrapper') +const remote = require('../lib/remote-importer/remote-importer') /** @@ -32,8 +36,31 @@ class Status { const memory = `${util.toMb(process.memoryUsage().rss)} MiB` // Get highest block processed by the tracker - const highest = await db.getHighestBlock() - const dbMaxHeight = highest.blockHeight + let dbMaxHeight = null + try { + const highest = await db.getHighestBlock() + dbMaxHeight = highest.blockHeight + } catch(e) { + Logger.error(e, 'API : Status.getCurrent() :') + } + + // Get info about the indexer + const indexerType = keys.indexer.active + let indexerMaxHeight = null + let indexerUrl = null + + if (indexerType == 'third_party_explorer') { + indexerUrl = (network.key == 'bitcoin') + ? keys.indexer.oxt + : keys.indexer.esplora + } + + try { + const chaintip = await remote.getChainTipHeight() + indexerMaxHeight = chaintip['chainTipHeight'] + } catch(e) { + Logger.error(e, 'API : Status.getCurrent() :') + } return { uptime: uptime, @@ -43,7 +70,12 @@ class Status { sessions: this.sessions, max: this.maxConn }, - blocks: dbMaxHeight + blocks: dbMaxHeight, + indexer: { + type: indexerType, + url: indexerUrl, + maxHeight: indexerMaxHeight + } } } diff --git a/accounts/support-rest-api.js b/accounts/support-rest-api.js index 38e9df9..5bd293d 100644 --- a/accounts/support-rest-api.js +++ b/accounts/support-rest-api.js @@ -69,6 +69,14 @@ class SupportRestApi { HttpServer.sendAuthError ) + this.httpServer.app.get( + `/${keys.prefixes.support}/xpub/:xpub/delete`, + authMgr.checkHasAdminProfile.bind(authMgr), + this.validateArgsGetXpubDelete.bind(this), + this.getXpubDelete.bind(this), + HttpServer.sendAuthError + ) + this.httpServer.app.get( `/${keys.prefixes.support}/pairing/explorer`, authMgr.checkHasAdminProfile.bind(authMgr), @@ -120,32 +128,9 @@ class SupportRestApi { */ _formatAddressInfoResult(info) { const res = info.toPojoExtended() - /*res._endpoints = [] - - if (info.tracked) { - res._endpoints.push({ - task: 'Rescan this address from remote sources', - url: `/${keys.prefixes.support}/address/${info.address}/rescan` - }) - } - - if (info.xpub != null) { - res._endpoints.push({ - task: 'Get information about the HD account that owns this address', - url: `/${keys.prefixes.support}/xpub/${info.xpub}/info` - }) - - res._endpoints.push({ - task: 'Rescan the whole HD account that owns this address', - url: `/${keys.prefixes.support}/xpub/${info.xpub}/rescan` - }) - }*/ - return JSON.stringify(res, null, 2) } - - /** * Rescan the blockchain for a given address * @param {object} req - http request object @@ -162,10 +147,6 @@ class SupportRestApi { const ret = { status: 'Rescan complete', - /*_endpoints: [{ - task: 'Get updated information about this address', - url: `/${keys.prefixes.support}/address/${address}/info` - }]*/ } await addrService.rescan(address) @@ -224,12 +205,6 @@ class SupportRestApi { */ _formatXpubInfoResult(info) { const res = info.toPojoExtended() - - /*res._endpoints = [{ - task: 'Rescan the whole HD account from remote sources', - url: `/${keys.prefixes.support}/xpub/${info.xpub}/rescan` - }]*/ - return JSON.stringify(res, null, 2) } @@ -249,10 +224,6 @@ class SupportRestApi { const ret = { status: 'Rescan complete', - /*_endpoints: [{ - task: 'Get updated information about this HD account', - url: `/${keys.prefixes.support}/xpub/${xpub}/info` - }]*/ } const gapLimit = req.query.gap != null ? parseInt(req.query.gap) : 0 @@ -283,6 +254,36 @@ class SupportRestApi { } } + /** + * Delete all data related to a hd account + * @param {object} req - http request object + * @param {object} res - http response object + */ + async getXpubDelete(req, res) { + try { + // Parse the entities passed as url params + const entities = apiHelper.parseEntities(req.params.xpub).xpubs + if (entities.length == 0) + return HttpServer.sendError(res, errors.xpub.INVALID) + + const xpub = entities[0] + + try { + await hdaService.deleteHdAccount(xpub) + HttpServer.sendOk(res) + } catch(e) { + HttpServer.sendError(res, e) + } + + } catch(e) { + HttpServer.sendError(res, errors.generic.GEN) + + } finally { + debugApi && Logger.info(`API : Completed GET /support/xpub/${req.params.xpub}/delete`) + } + } + + /** * Get pairing info */ @@ -366,6 +367,23 @@ class SupportRestApi { } } + /** + * Validate arguments related to GET xpub delete requests + * @param {object} req - http request object + * @param {object} res - http response object + * @param {function} next - next express middleware + */ + validateArgsGetXpubDelete(req, res, next) { + const isValidXpub = validator.isAlphanumeric(req.params.xpub) + + if (!isValidXpub) { + HttpServer.sendError(res, errors.body.INVDATA) + Logger.error(null, `API : SupportRestApi.validateArgsGetXpubDelete() : Invalid xpub ${req.params.xpub}`) + } else { + next() + } + } + /** * Validate arguments related to addresses requests * @param {object} req - http request object diff --git a/accounts/xpub-rest-api.js b/accounts/xpub-rest-api.js index b612a5b..cce81d8 100644 --- a/accounts/xpub-rest-api.js +++ b/accounts/xpub-rest-api.js @@ -16,6 +16,7 @@ const RpcClient = require('../lib/bitcoind-rpc/rpc-client') const HdAccountInfo = require('../lib/wallet/hd-account-info') const authMgr = require('../lib/auth/authorizations-manager') const HttpServer = require('../lib/http-server/http-server') +const remoteImporter = require('../lib/remote-importer/remote-importer') const debugApi = !!(process.argv.indexOf('api-debug') > -1) const gap = require('../keys/')[network.key].gap @@ -230,8 +231,18 @@ class XPubRestApi { return HttpServer.sendError(res, e) } - const ret = { - import_in_progress: hdaService.importInProgress(xpub) + let ret = { + import_in_progress: false + } + + const status = hdaService.importInProgress(xpub) + if (status != null) { + ret['import_in_progress'] = true + ret['status'] = status['status'] + if (ret['status'] == remoteImporter.STATUS_RESCAN) + ret['hits'] = status['txs_int'] + status['txs_ext'] + else + ret['hits'] = status['txs'] } HttpServer.sendOkData(res, ret) @@ -275,7 +286,7 @@ class XPubRestApi { const argAddr = req.body.address const argSig = req.body.signature const argMsg = req.body.message - + // Translate xpub if needed try { const ret = this.xlatHdAccount(argXpub) @@ -323,7 +334,7 @@ class XPubRestApi { const argXpub = req.params.xpub const argAddr = req.body.address const argSig = req.body.signature - + // Translate xpub if needed try { const ret = this.xlatHdAccount(argXpub) @@ -398,15 +409,15 @@ class XPubRestApi { validateArgsPostXpub(req, res, next) { const isValidXpub = validator.isAlphanumeric(req.body.xpub) - const isValidSegwit = + const isValidSegwit = !req.body.segwit || validator.isAlphanumeric(req.body.segwit) - const isValidType = + const isValidType = !req.body.type || validator.isAlphanumeric(req.body.type) - const isValidForce = + const isValidForce = !req.body.force || validator.isAlphanumeric(req.body.force) diff --git a/doc/DOCKER_setup.md b/doc/DOCKER_setup.md index 150d9aa..9217bc6 100644 --- a/doc/DOCKER_setup.md +++ b/doc/DOCKER_setup.md @@ -61,7 +61,7 @@ MyDojo is a set of Docker containers providing a full Samourai backend composed | | ---------- ---------- | | whirlnet | dojonet | |_________________|______________________________________________________________| - Host machine + Host machine @@ -109,6 +109,8 @@ Most options provided in the configuration files can be later modified. New valu ## First-time Setup ## +For Ubuntu 16, see this detailed [installation and upgrade guide](./DOCKER_ubuntu_setup.MD). + For MacOS, see this detailed [installation guide](./DOCKER_mac_setup.MD). For Synology, see this detailed [installation guide](./DOCKER_synology_setup.md). @@ -325,7 +327,7 @@ Once the database has finished syncing, you can pair your Samourai Wallet with y 1. Open the maintenance tool in a Tor browser (Tor v3 onion address) and sign in with your admin key. -2. Get your smartphone and launch the Samourai Wallet app. Scan the first QRCode displayed in the "Pairing" tab of the maintenance tool. +2. Get your smartphone and launch the Samourai Wallet app. Scan the first QRCode displayed in the "Pairing" tab of the maintenance tool. If you experience any problems when pairing, try re-installing the app and select "Connect to existing Dojo" from the [⋮] menu. @@ -336,7 +338,7 @@ You can pair your Samourai Wallet with your local block explorer in 2 steps: 1. Open the maintenance tool in a Tor browser (Tor v3 onion address) and sign in with your admin key. -2. Get your smartphone and launch the Samourai Wallet app. Scan the second QRCode displayed in the "Pairing" tab of the maintenance tool. +2. Get your smartphone and launch the Samourai Wallet app. Scan the second QRCode displayed in the "Pairing" tab of the maintenance tool. @@ -353,7 +355,7 @@ The block explorer is accessed as a Tor hidden service (static onion address). The Whirlpool API is accessed as a Tor hidden service (static onion address). -The Whirlpool client connects to the Whirlpool Coordinator hidden service. +The Whirlpool client connects to the Whirlpool Coordinator hidden service. The Bitcoin node only allows incoming connections from Tor (ephemeral onion address). diff --git a/doc/DOCKER_ubuntu_setup.md b/doc/DOCKER_ubuntu_setup.md new file mode 100644 index 0000000..dc71d34 --- /dev/null +++ b/doc/DOCKER_ubuntu_setup.md @@ -0,0 +1,509 @@ +# Installation and Upgrade of Dojo on Ubuntu 16 + +This procedure is written for the installation of MyDojo on a host machine running Ubuntu 16 but it should be easy to adapt it for Linux distributions running on a x86-64 architecture. + + +## Table of Content ## + +- [Requirements](#requirements) +- [Storage locations](#storage) +- [Installation procedure](#install) +- [Upgrade procedure](#upgrade) + + + + +## 1/ Requirements ## + +The main requirements for the host machine running MyDojo are: + +* Processor: x86-64 +* OS: Linux 64 bits +* Disk Type: SSD (recommended) +* Disk Size: 600GB (min)/ 1TB (recommended) +* RAM: 2GB (min) / 4GB (recommended) + +__Additional Considerations__ + +* While MyDojo will work fine on a multipurpose computer, a dedicated host machine connected 24/7 to the network is recommended. MyDojo was primarily designed as a server always ready for use +* The clock of the host machine MUST be set properly (required for Tor) + + + + +## 2/ Storage Locations ## + +By default, MyDojo stores its code and data in 2 different locations: + +* __MyDojo application code__: source code of MyDojo + Dojo Shell Script + Configuration files + +* __MyDojo and Docker data__: persistent data required for a functional MyDojo (blockchain data, Dojo database, logs, Docker images) + +By default, MyDojo and Docker data will be stored under `/var/lib/docker` directory but the directory can be modified (e.g.: all data stored on an dedicated disk). + + + + +## 3/ First Installation + +The procedure described in this section will configure and install MyDojo with its own Indexer for fast rescans. See the [Advanced Setups](./DOCKER_advanced_setups.md) for more information about this module or for the additional installation of a Whirlpool client on your Dojo. + +### 3.1/ Prepare Host System + +First, we must prepare our host system for MyDojo by installing required operating system dependencies and configuring our linux system with privacy and security in mind. + +#### 3.1.1/ Install OS dependencies + +``` +> cd ~ +> sudo apt-get update +> sudo apt-get install apt-transport-https ca-certificates curl gnupg-agent software-properties-common unzip +``` + +#### 3.1.2/ Install Docker & Docker-Compose + +For an installation of Docker and Docker Compose with a different Linux distribution, please refer to the official [Docker](https://docs.docker.com/install/) and [Docker Compose](https://docs.docker.com/compose/install/) documentations. + +If Docker is already installed on the host machine remove old Docker versions installed on the computer + +``` +> sudo apt-get remove docker docker-engine docker.io containerd runc +``` + +__Download Docker's official PGP key__ + +``` +> curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - +> sudo apt-key fingerprint 0EBFCD88 +``` + +Verify that you now have the key with the fingerprint `9DC8 5822 9FC7 DD38 854A E2D8 8D81 803C 0EBF CD88` + +__Install Docker__ + +``` +> sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" +> sudo apt-get update +> sudo apt-get install docker-ce docker-ce-cli containerd.io +``` + +__Test the installation of Docker__ + +``` +> sudo docker --version +``` + +This command should return the version of Docker if installation was successful. + + +__Install Docker Compose__ + +``` +> sudo curl -L "https://github.com/docker/compose/releases/download/1.25.3/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose +> sudo chmod +x /usr/local/bin/docker-compose +``` + +__Test the installation of Docker Compose__ + +``` +> sudo docker-compose --version +``` + +This command should return the version of Docker Compose if installation was successful. + +__Create a user account for MyDojo__ + +Creating a segregated user account for MyDojo is a good idea for security reasons. + + +#### 3.1.3/ Create a dojo user + +``` +> sudo useradd -s /bin/bash -d /home/dojo -m -G sudo dojo +> sudo passwd dojo +``` + +Enter and confirm the password for the `dojo` user. + +__Add the user to the docker group__ + +``` +> sudo usermod -aG docker dojo +``` + +__Restart Host System__ + +``` +> sudo shutdown -r now +``` + +Log back into the Host System with the `dojo` user and test the Docker installation + +``` +> docker run hello-world +``` + +This command should display a hello message if Docker can be run with the `dojo` user. + + +#### 3.1.4/ Configure Docker data storage directory (optional) + +This step should be applied if you don't want to store MyDojo and Docker data under the default `/var/lib/docker` directory (e.g.: all data will be stored on an external SSD). + +__Stop the Docker Service__ + +``` +> sudo systemctl stop docker +``` + +Create the directory that will store Docker data (replace `/path/to/target/directory/` by the correct path). + +``` +> sudo mkdir /path/to/target/directory/ +``` + +Temporarily switch to root and create the daemon.json file storing the path to your Docker direct (replace `/path/to/target/directory/` by the correct path). + +``` +> sudo su - root +> sudo echo '{ "data-root": "/path/to/target/directory/" }' > /etc/docker/daemon.json +> exit +``` + + +### 3.2/ Downloading MyDojo + +Now that the Host System has been prepared, we will download the latest version of MyDojo source code and configure it before proceeding with install. + + +#### 3.2.1/ Initialize the dojo app directory + +We first create a directory for housing our MyDojo files. In this guide we are naming this directory `dojo-app` and it will be located in the home directory of the `dojo` user. + +``` +> mkdir ~/dojo-app +``` + +#### 3.2.2/ Download latest MyDojo files + +Download and unpack the source archive for the latest version of MyDojo and copy these files to the newly created `dojo-app` directory with the following commands. + +``` +> cd ~ +> wget https://code.samourai.io/dojo/samourai-dojo/-/archive/master/samourai-dojo-master.zip +> unzip samourai-dojo-master.zip -d . +> cp -a samourai-dojo-master/. dojo-app/ +``` + +Delete the source archive now that we have copied the files to our directory. + +``` +> rm -rf samourai-dojo-master +> rm samourai-dojo-master.zip +``` + + +### 3.3/ Configuring MyDojo + +Change the working directory to the MyDojo configuration directory. + +``` +> cd ~/dojo-app/docker/my-dojo/conf +``` + +__Note:__ + +You will be required to generate various random alphanumeric passwords to secure various aspects of your Dojo installation. You can generate these in any way you wish, but you may wish to use the following command in a terminal session to generate these passwords with sufficient entropy: + +``` +> cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 64 | head -n 1 +``` + +#### 3.3.1/ Bitcoin configuration + +Edit the `docker-bitcoind.conf.tpl` file. + +``` +> nano docker-bitcoind.conf.tpl +``` + +Customize the content of the file + +``` +BITCOIND_RPC_USER= +BITCOIND_RPC_PASSWORD= +``` + +__Note:__ + +If your machine has a lot of RAM, it's recommended that you increase the value of `BITCOIND_DB_CACHE` (e.g 2048) for a faster Initial Block Download. + +Save and exit the file with `CTRL+X`, `Y` and `ENTER`. + + +#### 3.3.2/ Database configuration + +Edit the ``docker-mysql.conf.tpl`` file. + +``` +> nano docker-mysql.conf.tpl +``` + +Customize the content of the file + +``` +MYSQL_ROOT_PASSWORD= +MYSQL_USER= +MYSQL_PASSWORD= +``` + +Save and exit the file with `CTRL+X`, `Y` and `ENTER`. + + +#### 3.3.3/ NodeJS configuration + +Edit the `docker-node.conf.tpl` file. + +``` +> nano docker-node.conf.tpl +``` + +Customize the content of the file + +``` +NODE_API_KEY= +NODE_ADMIN_KEY= +NODE_JWT_SECRET= +NODE_ACTIVE_INDEXER=local_indexer +``` + +Save and exit the file with `CTRL+X`, `Y` and `ENTER`. + + +#### 3.3.4/ Indexer configuration + +Edit the `docker-indexer.conf.tpl` file. + +``` +> nano docker-indexer.conf.tpl +``` + +Customize the content of the file + +``` +INDEXER_INSTALL=on +``` + +Save and exit the file with `CTRL+X`, `Y` and `ENTER`. + + +#### 3.3.5/ Explorer configuration + +Edit the `docker-explorer.conf.tpl` file. + +``` +> nano docker-explorer.conf.tpl +``` + +Customize the content of the file + +``` +EXPLORER_KEY= +``` + +Save and exit the file with `CTRL+X`, `Y` and `ENTER`. + + +### 3.6/ Execute the first installation of MyDojo + +From this point on the install process is automatic. Launch the installation of MyDojo + +``` +> cd ~/dojo-app/docker/my-dojo +> ./dojo.sh install +``` + +Confirm the installation with `Y` and `ENTER`. + + +#### 3.6.1/ Understanding the Automatic Install + +At this point, your job is done. The Dojo Shell script and Docker are going to take over and a lot of things are going to happen automatically. + +__Building of MyDojo containers__ + +First, the shell script is going to initialize a set of configuration files. Then Docker is going to +initialize the dedicated networks and data volumes that will be used by MyDojo. At last, Docker is going +to build the Docker containers composing MyDojo. + +This latest operation may last from a dozen to a few tens of minutes, depending on the specs of the hardware. A lot of logs are displayed and you may be concerned that some of these logs (displayed in red) are the sign of a problem. Don't worry about this. Some of these logs are just informational or warnings. Just be aware that if a blocking error occurs, Docker will stop building the containers and you won't reach the next phase. + +__First launch of MyDojo__ + +If all Docker containers have been successfully built, the Shell Script is going to launch MyDojo (equivalent of the `dojo.sh start` command) and display its logs. + +Here you may be able to observe a few things in the logs: + +* The schema of the local database is initialized (done once during first installation) +* The Tor container is connecting to the Tor network +* Dojo application modules are started in the NodeJS container +* The Bitcoin daemon is launched, establishes connections to remote full nodes and starts to download block headers. +* The indexer is launched and waits for the completion of the Initial Blocks Download by the Bitcoin daemon. + +__Initial Blocks Download__ + +When all blocks headers have been successfully downloaded and processed, the Bitcoin daemon is going to process its Initial Blocks Download (IBD). In parallel, the `Tracker` module of Dojo is going to process its own IBD thanks to data provided by the Bitcoin daemon. + +This phase is going to last from 1 to several days, depending on the specs of the hardware. At this point, all you have to do is wait. The Bitcoin daemon is processing the IBD while the Tracker is importing block headers in parallel. + +Note: You can exit the logs with `CTRL+C`. Don't worry, MyDojo is still running in background. + +__Address Indexing__ + +When the Bitcoin daemon has completed its IBD, the Indexer will automatically start the indexation of all Bitcoin addresses. When the indexation is complete, it will compact the database. + +These operations should last a few more hours, depending on the specs of the hardware. + +Note: You may notice errors returned by the Block Explorer during all these operations. Don't worry. The Block Explorer should be available as soon as the Indexer has completed its tasks. + + +### 3.7/ Monitor Install Progress with the Maintenance Tool (DMT) + +Retrieve the onion address of the DMT with the commands + +``` +> cd ~/dojo-app/docker/my-dojo +> ./dojo.sh onion +``` + +Open the DMT with the Tor browser and check the `Status` page. + +If a green check is displayed for all modules and if the chaintip displayed for all modules match with the chaintip displayed by a third-party Block Explorer, then your Dojo is ready. + + + + + +## 4/ Upgrade + +The procedures described in this section will upgrade your Dojo to the most recent version or to a specific version of Dojo. + + +### 4.1/ Upgrade to latest version + +This procedure allows to upgrade MyDojo to the latest version. + +#### 4.1.1/ Stop MyDojo + +``` +> cd ~/dojo-app/docker/my-dojo +> ./dojo.sh stop +``` + +#### 4.1.2/ Update the code of MyDojo + +Download the archive of latest version + +``` +> cd ~ +> wget https://code.samourai.io/dojo/samourai-dojo/-/archive/master/samourai-dojo-master.zip +``` + +Uncompress the archive + +``` +> unzip samourai-dojo-master.zip -d . +``` + +Overwrite the dojo-app directory with the content of the archive + +``` +> cp -a samourai-dojo-master/. dojo-app/ +``` + +#### 4.1.3/ Update Configuration (optional) + +Check the [release notes](https://code.samourai.io/dojo/samourai-dojo/-/blob/master/RELEASES.md) of the new vesion for a list of new features that may require to tune the value of new configuration options. + +If applicable, edit the templates files stored in `~/dojo-app/docker/my-dojo/conf/` and modify the values set for new configuration options. + + +#### 4.1.4/ Start Upgrade + +``` +> cd ~/dojo-app/docker/my-dojo +> ./dojo.sh upgrade +``` + +Confirm that you want to upgrade MyDojo with `Y` and `ENTER`. + +The shell script is going to rebuild the Docker containers. MyDojo will be automatically restarted after the containers have been rebuilt. + + +#### 4.1.5/ Cleanup + +``` +> cd ~ +> rm -rf samourai-dojo-master +> rm samourai-dojo-master.zip +``` + + +### 4.2/ Upgrade to a specific version + +This procedure allows to upgrade MyDojo to a specific version `X.Y.Z` + +#### 4.2.1/ Stop MyDojo + +``` +> cd ~/dojo-app/docker/my-dojo +> ./dojo.sh stop +``` + +#### 4.2.2/ Update the code of MyDojo + +Download the archive of version `X.Y.Z` + +``` +> cd ~ +> wget https://code.samourai.io/dojo/samourai-dojo/-/archive/vX.Y.Z/samourai-dojo-vX.Y.Z.zip +``` + +Uncompress the archive + +``` +> unzip samourai-dojo-vX.Y.Z.zip -d . +``` + +Overwrite the dojo-app directory with the content of the archive + +``` +> cp -a samourai-dojo-vX.Y.Z/. dojo-app/ +``` + +#### 4.2.3/ Update Configuration (optional) + +Check the [release notes](https://code.samourai.io/dojo/samourai-dojo/-/blob/master/RELEASES.md) for a list of new features that may require to tune the value of new configuration options. + +If applicable, edit the templates files stored in `~/dojo-app/docker/my-dojo/conf/` and modify the values set for new configuration options. + + +#### 4.2.4/ Start Upgrade + +``` +> cd ~/dojo-app/docker/my-dojo +> ./dojo.sh upgrade +``` + +Confirm that you want to upgrade MyDojo with `Y` and `ENTER`. + +The shell script is going to rebuild the Docker containers. MyDojo will be automatically restarted after the containers have been rebuilt. + + +#### 4.2.5/ Cleanup + +``` +> cd ~ +> rm -rf samourai-dojo-vX.Y.Z +> rm samourai-dojo-vX.Y.Z.zip +``` diff --git a/doc/GET_xpub_import_status.md b/doc/GET_xpub_import_status.md index 4c9849a..8d0d45e 100644 --- a/doc/GET_xpub_import_status.md +++ b/doc/GET_xpub_import_status.md @@ -27,6 +27,17 @@ Status code 200 with JSON response: } ``` +```json +{ + "status": "ok", + "data": { + "import_in_progress": true, + "status": "rescan", + "hits": 1143 + } +} +``` + #### Failure Status code 400 with JSON response: ```json diff --git a/docker/my-dojo/.env b/docker/my-dojo/.env index e0daa61..cd27eb6 100644 --- a/docker/my-dojo/.env +++ b/docker/my-dojo/.env @@ -10,15 +10,15 @@ COMPOSE_CONVERT_WINDOWS_PATHS=1 -DOJO_VERSION_TAG=1.8.1 +DOJO_VERSION_TAG=1.9.0 DOJO_DB_VERSION_TAG=1.2.0 -DOJO_BITCOIND_VERSION_TAG=1.8.0 -DOJO_NODEJS_VERSION_TAG=1.8.0 +DOJO_BITCOIND_VERSION_TAG=1.11.0 +DOJO_NODEJS_VERSION_TAG=1.9.0 DOJO_NGINX_VERSION_TAG=1.5.0 -DOJO_TOR_VERSION_TAG=1.5.1 -DOJO_EXPLORER_VERSION_TAG=1.3.0 -DOJO_INDEXER_VERSION_TAG=1.1.0 -DOJO_WHIRLPOOL_VERSION_TAG=1.2.1 +DOJO_TOR_VERSION_TAG=1.7.0 +DOJO_EXPLORER_VERSION_TAG=1.4.0 +DOJO_INDEXER_VERSION_TAG=1.2.0 +DOJO_WHIRLPOOL_VERSION_TAG=1.3.0 ######################################### diff --git a/docker/my-dojo/bitcoin/Dockerfile b/docker/my-dojo/bitcoin/Dockerfile index 2be84c8..160ea50 100644 --- a/docker/my-dojo/bitcoin/Dockerfile +++ b/docker/my-dojo/bitcoin/Dockerfile @@ -5,10 +5,10 @@ FROM debian:buster # INSTALL BITCOIN ################################################################# ENV BITCOIN_HOME /home/bitcoin -ENV BITCOIN_VERSION 0.20.1 -ENV BITCOIN_URL https://bitcoincore.org/bin/bitcoin-core-0.20.1/bitcoin-0.20.1-x86_64-linux-gnu.tar.gz -ENV BITCOIN_SHA256 376194f06596ecfa40331167c39bc70c355f960280bd2a645fdbf18f66527397 -ENV BITCOIN_ASC_URL https://bitcoincore.org/bin/bitcoin-core-0.20.1/SHA256SUMS.asc +ENV BITCOIN_VERSION 0.21.0 +ENV BITCOIN_URL https://bitcoincore.org/bin/bitcoin-core-0.21.0/bitcoin-0.21.0-x86_64-linux-gnu.tar.gz +ENV BITCOIN_SHA256 da7766775e3f9c98d7a9145429f2be8297c2672fe5b118fd3dc2411fb48e0032 +ENV BITCOIN_ASC_URL https://bitcoincore.org/bin/bitcoin-core-0.21.0/SHA256SUMS.asc ENV BITCOIN_PGP_KS_URI hkp://keyserver.ubuntu.com:80 ENV BITCOIN_PGP_KEY 01EA5486DE18A882D4C2684590C8019E36C2E964 diff --git a/docker/my-dojo/bitcoin/restart.sh b/docker/my-dojo/bitcoin/restart.sh index 751bd6b..db14dd3 100644 --- a/docker/my-dojo/bitcoin/restart.sh +++ b/docker/my-dojo/bitcoin/restart.sh @@ -4,22 +4,19 @@ set -e echo "## Start bitcoind #############################" bitcoind_options=( - -bind=172.28.1.5 -datadir=/home/bitcoin/.bitcoin -printtoconsole=1 -dbcache=$BITCOIND_DB_CACHE -disablewallet=1 -dns=$BITCOIND_DNS -dnsseed=$BITCOIND_DNSSEED - -externalip=$(cat /var/lib/tor/hsv2bitcoind/hostname) - -listen=1 -maxconnections=$BITCOIND_MAX_CONNECTIONS -maxmempool=$BITCOIND_MAX_MEMPOOL -mempoolexpiry=$BITCOIND_MEMPOOL_EXPIRY -minrelaytxfee=$BITCOIND_MIN_RELAY_TX_FEE -port=8333 -proxy=172.28.1.4:9050 - -rpcallowip=::/0 + -rpcallowip=0.0.0.0/0 -rpcbind=172.28.1.5 -rpcpassword=$BITCOIND_RPC_PASSWORD -rpcport=28256 @@ -32,6 +29,13 @@ bitcoind_options=( -zmqpubrawtx=tcp://0.0.0.0:9501 ) +if [ "$BITCOIND_LISTEN_MODE" == "on" ]; then + bitcoind_options+=(-listen=1) + bitcoind_options+=(-bind=172.28.1.5) + bitcoind_options+=(-externalip=$(cat /var/lib/tor/hsv2bitcoind/hostname)) + bitcoind_options+=(-externalip=$(cat /var/lib/tor/hsv3bitcoind/hostname)) +fi + if [ "$BITCOIND_RPC_EXTERNAL" == "on" ]; then bitcoind_options+=(-zmqpubhashtx=tcp://0.0.0.0:9500) bitcoind_options+=(-zmqpubrawblock=tcp://0.0.0.0:9503) @@ -41,7 +45,7 @@ if [ "$COMMON_BTC_NETWORK" == "testnet" ]; then bitcoind_options+=(-testnet) fi -bitcoind "${bitcoind_options[@]}" +bitcoind "${bitcoind_options[@]}" || true # Keep the container up while true diff --git a/docker/my-dojo/conf/docker-bitcoind.conf.tpl b/docker/my-dojo/conf/docker-bitcoind.conf.tpl index 2695e14..6a19192 100644 --- a/docker/my-dojo/conf/docker-bitcoind.conf.tpl +++ b/docker/my-dojo/conf/docker-bitcoind.conf.tpl @@ -39,6 +39,11 @@ BITCOIND_MEMPOOL_EXPIRY=72 # Type: numeric BITCOIND_MIN_RELAY_TX_FEE=0.00001 +# Allow incoming connections +# This parameter is inactive if BITCOIND_INSTALL is set to 'off' +# Values: on | off +BITCOIND_LISTEN_MODE=on + # # EXPERT SETTINGS diff --git a/docker/my-dojo/conf/docker-whirlpool.conf.tpl b/docker/my-dojo/conf/docker-whirlpool.conf.tpl index 762681f..dbf5e9b 100644 --- a/docker/my-dojo/conf/docker-whirlpool.conf.tpl +++ b/docker/my-dojo/conf/docker-whirlpool.conf.tpl @@ -10,6 +10,10 @@ WHIRLPOOL_INSTALL=off # Value: on | off WHIRLPOOL_RESYNC=off +# Contact coordinator through its onion address or clearnet address +# Set to on for onion address, set to off for clearnet address +# Value: on | off +WHIRLPOOL_COORDINATOR_ONION=on # # EXPERT SETTINGS diff --git a/docker/my-dojo/docker-compose.yaml b/docker/my-dojo/docker-compose.yaml index 1f1638b..197ce65 100644 --- a/docker/my-dojo/docker-compose.yaml +++ b/docker/my-dojo/docker-compose.yaml @@ -92,6 +92,7 @@ services: context: ./tor env_file: - ./.env + - ./conf/docker-bitcoind.conf - ./conf/docker-explorer.conf - ./conf/docker-whirlpool.conf - ./conf/docker-tor.conf diff --git a/docker/my-dojo/dojo.sh b/docker/my-dojo/dojo.sh index cda0d74..3798f38 100755 --- a/docker/my-dojo/dojo.sh +++ b/docker/my-dojo/dojo.sh @@ -55,7 +55,7 @@ select_yaml_files() { # Docker up docker_up() { yamlFiles=$(select_yaml_files) - eval "docker-compose $yamlFiles up $1 -d" + eval "docker-compose $yamlFiles up $@ -d" } # Start @@ -74,7 +74,7 @@ start() { # Stop stop() { echo "Preparing shutdown of Dojo. Please wait." - # Check if dojo is running (check the db container) + # Check if dojo is running (check the db container) isRunning=$(docker inspect --format="{{.State.Running}}" db 2> /dev/null) if [ $? -eq 1 ] || [ "$isRunning" == "false" ]; then echo "Dojo is already stopped." @@ -83,8 +83,11 @@ stop() { # Shutdown the bitcoin daemon if [ "$BITCOIND_INSTALL" == "on" ]; then # Renewal of bitcoind onion address - if [ "$BITCOIND_EPHEMERAL_HS" = "on" ]; then - $( docker exec -it tor rm -rf /var/lib/tor/hsv2bitcoind ) &> /dev/null + if [ "$BITCOIND_LISTEN_MODE" == "on" ]; then + if [ "$BITCOIND_EPHEMERAL_HS" = "on" ]; then + $( docker exec -it tor rm -rf /var/lib/tor/hsv2bitcoind ) &> /dev/null + $( docker exec -it tor rm -rf /var/lib/tor/hsv3bitcoind ) &> /dev/null + fi fi # Stop the bitcoin daemon $( docker exec -it bitcoind bitcoin-cli \ @@ -163,7 +166,7 @@ install() { if [ $launchInstall -eq 0 ]; then pastInstallsfound=$(docker image ls | grep samouraiwallet/dojo-db | wc -l) if [ $pastInstallsfound -ne 0 ]; then - # Past installation found. Ask confirmation forreinstall + # Past installation found. Ask confirmation for reinstall echo -e "\nWarning: Found traces of a previous installation of Dojo on this machine." echo "A new installation requires to remove these elements first." if [ $auto -eq 0 ]; then @@ -198,10 +201,19 @@ install() { init_config_files # Build and start Dojo docker_up --remove-orphans - # Display the logs - if [ $noLog -eq 1 ]; then - logs "" 0 + buildResult=$? + if [ $buildResult -eq 0 ]; then + # Display the logs + if [ $noLog -eq 1 ]; then + logs "" 0 + fi + else + # Return an error + echo -e "\nInstallation of Dojo failed. See the above error message." + exit $buildResult fi + else + exit 1 fi } @@ -231,20 +243,8 @@ uninstall() { fi if [ $launchUninstall -eq 0 ]; then - docker-compose rm -f - yamlFiles=$(select_yaml_files) - eval "docker-compose $yamlFiles down" - - docker image rm -f samouraiwallet/dojo-db:"$DOJO_DB_VERSION_TAG" - docker image rm -f samouraiwallet/dojo-bitcoind:"$DOJO_BITCOIND_VERSION_TAG" - docker image rm -f samouraiwallet/dojo-explorer:"$DOJO_EXPLORER_VERSION_TAG" - docker image rm -f samouraiwallet/dojo-nodejs:"$DOJO_NODEJS_VERSION_TAG" - docker image rm -f samouraiwallet/dojo-nginx:"$DOJO_NGINX_VERSION_TAG" - docker image rm -f samouraiwallet/dojo-tor:"$DOJO_TOR_VERSION_TAG" - docker image rm -f samouraiwallet/dojo-indexer:"$DOJO_INDEXER_VERSION_TAG" - docker image rm -f samouraiwallet/dojo-whirlpool:"$DOJO_WHIRLPOOL_VERSION_TAG" - + eval "docker-compose $yamlFiles down --rmi all" docker volume prune -f return 0 else @@ -258,13 +258,12 @@ del_images_for() { # $2: most recent version of the image (do not delete this one) docker image ls | grep "$1" | sed "s/ \+/,/g" | cut -d"," -f2 | while read -r version ; do if [ "$2" != "$version" ]; then - docker image rm "$1:$version" + docker image rm -f "$1:$version" fi done } clean() { - docker image prune del_images_for samouraiwallet/dojo-db "$DOJO_DB_VERSION_TAG" del_images_for samouraiwallet/dojo-bitcoind "$DOJO_BITCOIND_VERSION_TAG" del_images_for samouraiwallet/dojo-explorer "$DOJO_EXPLORER_VERSION_TAG" @@ -273,6 +272,7 @@ clean() { del_images_for samouraiwallet/dojo-tor "$DOJO_TOR_VERSION_TAG" del_images_for samouraiwallet/dojo-indexer "$DOJO_INDEXER_VERSION_TAG" del_images_for samouraiwallet/dojo-whirlpool "$DOJO_WHIRLPOOL_VERSION_TAG" + docker image prune -f } # Upgrade @@ -309,7 +309,16 @@ upgrade() { if [ $launchUpgrade -eq 0 ]; then # Select yaml files yamlFiles=$(select_yaml_files) + # Check if dojo is running (check the db container) + isRunning=$(docker inspect --format="{{.State.Running}}" db 2> /dev/null) + if [ $? -eq 1 ] || [ "$isRunning" == "false" ]; then + echo -e "\nChecked that Dojo isn't running." + else + echo " " + stop + fi # Update config files + echo -e "\nPreparing the upgrade of Dojo.\n" update_config_files # Cleanup cleanup @@ -318,51 +327,105 @@ upgrade() { export BITCOIND_RPC_EXTERNAL_IP # Rebuild the images (with or without cache) if [ $noCache -eq 0 ]; then - eval "docker-compose $yamlFiles build --no-cache" - else - eval "docker-compose $yamlFiles build" + echo -e "\nDeleting Dojo containers and images." + eval "docker-compose $yamlFiles down --rmi all" fi - # Start Dojo - docker_up --remove-orphans - # Post start clean-up - post_start_cleanup - # Update the database - update_dojo_db - # Display the logs - if [ $noLog -eq 1 ]; then - logs "" 0 + echo -e "\nStarting the upgrade of Dojo.\n" + docker_up --build --force-recreate --remove-orphans + buildResult=$? + if [ $buildResult -eq 0 ]; then + # Post start clean-up + clean + post_start_cleanup + # Update the database + update_dojo_db + # Display the logs + if [ $noLog -eq 1 ]; then + logs "" 0 + fi + else + # Return an error + echo -e "\nUpgrade of Dojo failed. See the above error message." + exit $buildResult fi + else + exit 1 fi } -# Display the onion address +# Display the onion addresses onion() { + version=3 + + # Extract version arguments + if [ $# -gt 0 ]; then + for option in $@ + do + case "$option" in + v2 ) version=2 ;; + v3 ) version=3 ;; + * ) break ;; + esac + done + fi + echo " " echo "WARNING: Do not share these onion addresses with anyone!" echo " To allow another person to use this Dojo with their Samourai Wallet," echo " you should share the QRCodes provided by the Maintenance Tool." echo " " - V3_ADDR=$( docker exec -it tor cat /var/lib/tor/hsv3dojo/hostname ) - echo "Dojo API and Maintenance Tool = $V3_ADDR" - echo " " - - if [ "$EXPLORER_INSTALL" == "on" ]; then - V3_ADDR_EXPLORER=$( docker exec -it tor cat /var/lib/tor/hsv3explorer/hostname ) - echo "Block Explorer = $V3_ADDR_EXPLORER" + if [ $version -eq 3 ]; then + # V3 onion addresses + V3_ADDR=$( docker exec -it tor cat /var/lib/tor/hsv3dojo/hostname ) + echo "Dojo API and Maintenance Tool = $V3_ADDR" echo " " - fi - if [ "$WHIRLPOOL_INSTALL" == "on" ]; then - V3_ADDR_WHIRLPOOL=$( docker exec -it tor cat /var/lib/tor/hsv3whirlpool/hostname ) - echo "Your private Whirlpool client (do not share) = $V3_ADDR_WHIRLPOOL" - echo " " - fi + if [ "$EXPLORER_INSTALL" == "on" ]; then + V3_ADDR_EXPLORER=$( docker exec -it tor cat /var/lib/tor/hsv3explorer/hostname ) + echo "Block Explorer = $V3_ADDR_EXPLORER" + echo " " + fi - if [ "$BITCOIND_INSTALL" == "on" ]; then - V2_ADDR_BTCD=$( docker exec -it tor cat /var/lib/tor/hsv2bitcoind/hostname ) - echo "Your local bitcoind (do not share) = $V2_ADDR_BTCD" + if [ "$WHIRLPOOL_INSTALL" == "on" ]; then + V3_ADDR_WHIRLPOOL=$( docker exec -it tor cat /var/lib/tor/hsv3whirlpool/hostname ) + echo "Your private Whirlpool client (do not share) = $V3_ADDR_WHIRLPOOL" + echo " " + fi + + if [ "$BITCOIND_INSTALL" == "on" ]; then + if [ "$BITCOIND_LISTEN_MODE" == "on" ]; then + V3_ADDR_BTCD=$( docker exec -it tor cat /var/lib/tor/hsv3bitcoind/hostname ) + echo "Your local bitcoind (do not share) = $V3_ADDR_BTCD" + echo " " + fi + fi + + else + # v2 onion addresses + V2_ADDR=$( docker exec -it tor cat /var/lib/tor/hsv2dojo/hostname ) + echo "Dojo API and Maintenance Tool = $V2_ADDR" echo " " + + if [ "$EXPLORER_INSTALL" == "on" ]; then + V2_ADDR_EXPLORER=$( docker exec -it tor cat /var/lib/tor/hsv2explorer/hostname ) + echo "Block Explorer = $V2_ADDR_EXPLORER" + echo " " + fi + + if [ "$WHIRLPOOL_INSTALL" == "on" ]; then + V2_ADDR_WHIRLPOOL=$( docker exec -it tor cat /var/lib/tor/hsv2whirlpool/hostname ) + echo "Your private Whirlpool client (do not share) = $V2_ADDR_WHIRLPOOL" + echo " " + fi + + if [ "$BITCOIND_INSTALL" == "on" ]; then + if [ "$BITCOIND_LISTEN_MODE" == "on" ]; then + V2_ADDR_BTCD=$( docker exec -it tor cat /var/lib/tor/hsv2bitcoind/hostname ) + echo "Your local bitcoind (do not share) = $V2_ADDR_BTCD" + echo " " + fi + fi fi } @@ -498,7 +561,11 @@ help() { echo " Available options:" echo " -n [VALUE] : display the last VALUE lines" echo " " - echo " onion Display the Tor onion address allowing your wallet to access your dojo." + echo " onion [version] Display the Tor onion addresses allowing your wallet to access your dojo." + echo " " + echo " Available versions:" + echo " v2: display Tor v2 onion addresses" + echo " v3 (default): display Tor v3 onion addresses" echo " " echo " restart Restart your dojo." echo " " @@ -592,7 +659,7 @@ case "$subcommand" in logs "$module" $numlines ;; onion ) - onion + onion "$@" ;; restart ) restart diff --git a/docker/my-dojo/explorer/Dockerfile b/docker/my-dojo/explorer/Dockerfile index 8f32102..c41c689 100644 --- a/docker/my-dojo/explorer/Dockerfile +++ b/docker/my-dojo/explorer/Dockerfile @@ -3,7 +3,7 @@ FROM node:12-buster ENV APP_DIR /home/node/app ENV EXPLORER_URL https://github.com/janoside/btc-rpc-explorer/archive -ENV EXPLORER_VERSION 2.0.0 +ENV EXPLORER_VERSION 2.1.0 # Install netcat RUN set -ex && \ diff --git a/docker/my-dojo/indexer/Dockerfile b/docker/my-dojo/indexer/Dockerfile index b838a13..ea3b26a 100644 --- a/docker/my-dojo/indexer/Dockerfile +++ b/docker/my-dojo/indexer/Dockerfile @@ -1,7 +1,7 @@ FROM rust:1.42.0-slim-buster ENV INDEXER_HOME /home/indexer -ENV INDEXER_VERSION 0.3.0 +ENV INDEXER_VERSION 0.4.0 ENV INDEXER_URL https://code.samourai.io/dojo/addrindexrs.git RUN apt-get update && \ @@ -36,7 +36,7 @@ RUN cd "$INDEXER_HOME" && \ git checkout "tags/v$INDEXER_VERSION" RUN cd "$INDEXER_HOME/addrindexrs" && \ - cargo install --path . + cargo install --locked --path . EXPOSE 50001 diff --git a/docker/my-dojo/install/upgrade-scripts.sh b/docker/my-dojo/install/upgrade-scripts.sh index c342ac1..9669cd1 100755 --- a/docker/my-dojo/install/upgrade-scripts.sh +++ b/docker/my-dojo/install/upgrade-scripts.sh @@ -69,7 +69,7 @@ update_config_files() { update_config_file ./conf/docker-whirlpool.conf ./conf/docker-whirlpool.conf.tpl echo "Initialized docker-whirlpool.conf" - # Initialize config files for nginx and the maintenance tool + # Initialize config files for nginx and the maintenance tool if [ "$EXPLORER_INSTALL" == "on" ]; then cp ./nginx/explorer.conf ./nginx/dojo-explorer.conf else @@ -106,7 +106,7 @@ update_config_file() { cp -p $1 "$1.save" cp -p $2 $1 - while IFS='=' read -r key val ; do + while IFS='=' read -r key val ; do if [[ $OSTYPE == darwin* ]]; then sed -i "" "s~$key=.*~$key=$val~g" "$1" else @@ -179,7 +179,7 @@ cleanup() { if [ -f ./bitcoin/bitcoin.conf ]; then rm ./bitcoin/bitcoin.conf fi - + } # Post start clean-up diff --git a/docker/my-dojo/tor/Dockerfile b/docker/my-dojo/tor/Dockerfile index 4dda9eb..f21ce2c 100644 --- a/docker/my-dojo/tor/Dockerfile +++ b/docker/my-dojo/tor/Dockerfile @@ -1,9 +1,9 @@ FROM debian:buster ENV TOR_HOME /var/lib/tor -ENV TOR_URL https://archive.torproject.org/tor-package-archive +ENV TOR_URL https://dist.torproject.org ENV TOR_MIRROR_URL https://tor.eff.org/dist -ENV TOR_VERSION 0.4.5.4-rc +ENV TOR_VERSION 0.4.4.7 ENV TOR_GPG_KS_URI hkp://keyserver.ubuntu.com:80 ENV TOR_GPG_KEY1 0xEB5A896A28988BF5 ENV TOR_GPG_KEY2 0xC218525819F78451 diff --git a/docker/my-dojo/tor/restart.sh b/docker/my-dojo/tor/restart.sh index 1cf4458..ef1392b 100644 --- a/docker/my-dojo/tor/restart.sh +++ b/docker/my-dojo/tor/restart.sh @@ -19,13 +19,28 @@ tor_options=( --HiddenServiceDir /var/lib/tor/hsv3dojo --HiddenServiceVersion 3 --HiddenServicePort "80 172.29.1.3:80" - --HiddenServiceDir /var/lib/tor/hsv2bitcoind - --HiddenServiceVersion 2 - --HiddenServicePort "8333 172.28.1.5:8333" - --HiddenServiceDirGroupReadable 1 ) +if [ "$BITCOIND_INSTALL" == "on" ]; then + if [ "$BITCOIND_LISTEN_MODE" == "on" ]; then + tor_options+=(--HiddenServiceDir /var/lib/tor/hsv2bitcoind) + tor_options+=(--HiddenServiceVersion 2) + tor_options+=(--HiddenServicePort "8333 172.28.1.5:8333") + tor_options+=(--HiddenServiceDirGroupReadable 1) + + tor_options+=(--HiddenServiceDir /var/lib/tor/hsv3bitcoind) + tor_options+=(--HiddenServiceVersion 3) + tor_options+=(--HiddenServicePort "8333 172.28.1.5:8333") + tor_options+=(--HiddenServiceDirGroupReadable 1) + fi +fi + if [ "$EXPLORER_INSTALL" == "on" ]; then + tor_options+=(--HiddenServiceDir /var/lib/tor/hsv2explorer) + tor_options+=(--HiddenServiceVersion 2) + tor_options+=(--HiddenServicePort "80 172.29.1.3:9080") + tor_options+=(--HiddenServiceDirGroupReadable 1) + tor_options+=(--HiddenServiceDir /var/lib/tor/hsv3explorer) tor_options+=(--HiddenServiceVersion 3) tor_options+=(--HiddenServicePort "80 172.29.1.3:9080") @@ -33,6 +48,11 @@ if [ "$EXPLORER_INSTALL" == "on" ]; then fi if [ "$WHIRLPOOL_INSTALL" == "on" ]; then + tor_options+=(--HiddenServiceDir /var/lib/tor/hsv2whirlpool) + tor_options+=(--HiddenServiceVersion 2) + tor_options+=(--HiddenServicePort "80 172.29.1.3:8898") + tor_options+=(--HiddenServiceDirGroupReadable 1) + tor_options+=(--HiddenServiceDir /var/lib/tor/hsv3whirlpool) tor_options+=(--HiddenServiceVersion 3) tor_options+=(--HiddenServicePort "80 172.29.1.3:8898") diff --git a/docker/my-dojo/whirlpool/Dockerfile b/docker/my-dojo/whirlpool/Dockerfile index bfa319e..2414ecd 100644 --- a/docker/my-dojo/whirlpool/Dockerfile +++ b/docker/my-dojo/whirlpool/Dockerfile @@ -19,9 +19,9 @@ RUN set -ex && \ mkdir -p "$WHIRLPOOL_DIR" # Install Tor -ENV WHIRLPOOL_TOR_URL https://archive.torproject.org/tor-package-archive +ENV WHIRLPOOL_TOR_URL https://dist.torproject.org ENV WHIRLPOOL_TOR_MIRROR_URL https://tor.eff.org/dist -ENV WHIRLPOOL_TOR_VERSION 0.4.5.4-rc +ENV WHIRLPOOL_TOR_VERSION 0.4.4.7 ENV WHIRLPOOL_TOR_GPG_KS_URI hkp://keyserver.ubuntu.com:80 ENV WHIRLPOOL_TOR_GPG_KEY1 0xEB5A896A28988BF5 ENV WHIRLPOOL_TOR_GPG_KEY2 0xC218525819F78451 @@ -60,10 +60,10 @@ RUN set -ex && \ # Install whirlpool-cli ENV WHIRLPOOL_URL https://code.samourai.io/whirlpool/whirlpool-client-cli/uploads -ENV WHIRLPOOL_VERSION 0.10.8 -ENV WHIRLPOOL_VERSION_HASH 7998ea5a9bb180451616809bc346b9ac +ENV WHIRLPOOL_VERSION 0.10.9 +ENV WHIRLPOOL_VERSION_HASH 602666c59f95ce72f1466f72d9c853e3 ENV WHIRLPOOL_JAR "whirlpool-client-cli-$WHIRLPOOL_VERSION-run.jar" -ENV WHIRLPOOL_SHA256 62e17b6020d0821a98e99ebb773b46191770ec186ceaa3e616a428f5cafe9f49 +ENV WHIRLPOOL_SHA256 9de3ceaff6e8cc0849bde58bc9e17b9c602352df8659adc67ab95b39cf046e4c RUN set -ex && \ cd "$WHIRLPOOL_DIR" && \ diff --git a/docker/my-dojo/whirlpool/restart.sh b/docker/my-dojo/whirlpool/restart.sh index df9715d..155cdbf 100644 --- a/docker/my-dojo/whirlpool/restart.sh +++ b/docker/my-dojo/whirlpool/restart.sh @@ -9,9 +9,9 @@ whirlpool_options=( --cli.tor=true --cli.torConfig.executable=/usr/local/bin/tor --cli.torConfig.coordinator.enabled=true - --cli.torConfig.coordinator.onion=true --cli.torConfig.backend.enabled=false --cli.torConfig.backend.onion=false + --cli.mix.liquidityClient=false ) if [ "$COMMON_BTC_NETWORK" == "testnet" ]; then @@ -22,6 +22,12 @@ else whirlpool_options+=(--cli.dojo.url="http://${NGINX_IP:-172.30.1.3}:80/v2/") fi +if [ "$WHIRLPOOL_COORDINATOR_ONION" == "on" ]; then + whirlpool_options+=(--cli.torConfig.coordinator.onion=true) +else + whirlpool_options+=(--cli.torConfig.coordinator.onion=false) +fi + if [ "$WHIRLPOOL_RESYNC" == "on" ]; then whirlpool_options+=(--resync) fi diff --git a/keys/index-example.js b/keys/index-example.js index 7e6c98a..902b4ad 100644 --- a/keys/index-example.js +++ b/keys/index-example.js @@ -15,7 +15,7 @@ module.exports = { /* * Dojo version */ - dojoVersion: '1.8.0', + dojoVersion: '1.9.0', /* * Bitcoind */ @@ -232,7 +232,7 @@ module.exports = { * Testnet parameters */ testnet: { - dojoVersion: '1.8.0', + dojoVersion: '1.9.0', bitcoind: { rpc: { user: 'user', diff --git a/lib/bitcoind-rpc/fees.js b/lib/bitcoind-rpc/fees.js index 386ecd2..fb4174d 100644 --- a/lib/bitcoind-rpc/fees.js +++ b/lib/bitcoind-rpc/fees.js @@ -14,7 +14,7 @@ const latestBlock = require('./latest-block') /** - * A singleton providing information about network fees + * A singleton providing information about network fees */ class Fees { @@ -56,10 +56,10 @@ class Fees { await util.seriesCall(this.targets, async tgt => { try { const level = await this.rpcClient.cmd('estimatesmartfee', tgt, this.feeType) - this.fees[tgt] = Math.round(level.feerate * 1e5) + this.fees[tgt] = (level.errors && level.errors.length > 0) ? 0 : Math.round(level.feerate * 1e5) } catch(e) { Logger.error(e, 'Bitcoind RPC : Fees.refresh()') - delete this.fees[tgt] + this.fees[tgt] = 0 } }) diff --git a/lib/indexer-rpc/rpc-client.js b/lib/indexer-rpc/rpc-client.js index 6982d3c..eb324ea 100644 --- a/lib/indexer-rpc/rpc-client.js +++ b/lib/indexer-rpc/rpc-client.js @@ -181,14 +181,14 @@ class RpcClient { throw new Error(JSON.stringify(parsed.error)) // Add the parsed reponse to the array of responses if (batched) { - responses = parsed.map(p => { return {idxAddr: p.id, txs: p.result} }) + responses = parsed.map(p => { return {id: p.id, response: p.result} }) } else { - responses.push({idxAddr: parsed.id, txs: parsed.result}) + responses.push({id: parsed.id, response: parsed.result}) } // Reset the response response = '' // If all responses have been received - // close the connection + // close the connection if (responses.length == data.length) conn.end() } catch (err) { diff --git a/lib/remote-importer/bitcoind-wrapper.js b/lib/remote-importer/bitcoind-wrapper.js index e578bbf..c66e52d 100644 --- a/lib/remote-importer/bitcoind-wrapper.js +++ b/lib/remote-importer/bitcoind-wrapper.js @@ -6,6 +6,7 @@ const bitcoin = require('bitcoinjs-lib') const RpcClient = require('../bitcoind-rpc/rpc-client') +const rpcLatestBlock = require('../bitcoind-rpc/latest-block') const Logger = require('../logger') const network = require('../bitcoin/network') const activeNet = network.network @@ -77,7 +78,7 @@ class BitcoindWrapper extends Wrapper { txids: [] } } - + return ret } @@ -113,7 +114,7 @@ class BitcoindWrapper extends Wrapper { } const aRet = Object.values(ret) - + for (let i in aRet) { if (filterAddr && aRet[i].ntx > keys.addrFilterThreshold) { Logger.info(`Importer : Import of ${aRet[i].address} rejected (too many transactions - ${aRet[i].ntx})`) @@ -124,6 +125,15 @@ class BitcoindWrapper extends Wrapper { return aRet } + /** + * Retrieve the height of the chaintip for the remote source + * @returns {Promise} returns an object + * {chainTipHeight: } + */ + async getChainTipHeight() { + return {'chainTipHeight': rpcLatestBlock.height} + } + } module.exports = BitcoindWrapper diff --git a/lib/remote-importer/esplora-wrapper.js b/lib/remote-importer/esplora-wrapper.js index 8376179..1a82039 100644 --- a/lib/remote-importer/esplora-wrapper.js +++ b/lib/remote-importer/esplora-wrapper.js @@ -4,7 +4,7 @@ */ 'use strict' -const rp = require('request-promise-native') +const axios = require('axios') const addrHelper = require('../bitcoin/addresses-helper') const util = require('../util') const Logger = require('../logger') @@ -34,15 +34,21 @@ class EsploraWrapper extends Wrapper { const params = { url: `${this.base}${route}`, method: 'GET', - json: true, - timeout: 15000 + responseType: 'json', + timeout: 15000, + headers: { + 'User-Agent': 'Dojo' + } } - + // Sets socks proxy agent if required - if (keys.indexer.socks5Proxy != null) - params['agent'] = this.socksProxyAgent + if (keys.indexer.socks5Proxy != null) { + params['httpAgent'] = this.socksProxyAgent + params['httpsAgent'] = this.socksProxyAgent + } - return rp(params) + const result = await axios(params) + return result.data } /** @@ -123,6 +129,19 @@ class EsploraWrapper extends Wrapper { return ret } + /** + * Retrieve the height of the chaintip for the remote source + * @returns {Promise} returns an object + * {chainTipHeight: } + */ + async getChainTipHeight() { + let chainTipHeight = null + const result = await this._get(`/api/blocks/tip/height`) + if (result != null) + chainTipHeight = parseInt(result) + return {'chainTipHeight': chainTipHeight} + } + } // Esplora returns a max of 25 txs per page diff --git a/lib/remote-importer/local-indexer-wrapper.js b/lib/remote-importer/local-indexer-wrapper.js index 5e706ae..dbf480f 100644 --- a/lib/remote-importer/local-indexer-wrapper.js +++ b/lib/remote-importer/local-indexer-wrapper.js @@ -16,7 +16,7 @@ const Wrapper = require('./wrapper') /** * Wrapper for a local indexer * Currently supports indexers - * providing a RPC API compliant + * providing a RPC API compliant * with a subset of the electrum protocol */ class LocalIndexerWrapper extends Wrapper { @@ -64,7 +64,7 @@ class LocalIndexerWrapper extends Wrapper { scriptHash ) - for (let r of results.txs) { + for (let r of results.response) { ret.txids.push(r.tx_hash) ret.ntx++ } @@ -77,7 +77,7 @@ class LocalIndexerWrapper extends Wrapper { txids: [] } } - + return ret } @@ -109,8 +109,8 @@ class LocalIndexerWrapper extends Wrapper { : await this.client.sendRequests(commands) for (let r of results) { - const addr = addresses[r.idxAddr] - const txids = r.txs.map(t => t.tx_hash) + const addr = addresses[r.id] + const txids = r.response.map(t => t.tx_hash) ret[addr] = { address: addr, @@ -131,12 +131,31 @@ class LocalIndexerWrapper extends Wrapper { return aRet } + /** + * Retrieve the height of the chaintip for the remote source + * @returns {Promise} returns an object + * {chainTipHeight: } + */ + async getChainTipHeight() { + let chainTipHeight = null + const result = await this.client.sendRequest( + LocalIndexerWrapper.HEADERS_SUBSCRIBE_RPC_CMD, + null + ) + if (result != null && result['response'] != null && result['response']['height'] != null) + chainTipHeight = parseInt(result['response']['height']) + return {'chainTipHeight': chainTipHeight} + } } /** * Get history RPC command (Electrum protocol) */ LocalIndexerWrapper.GET_HISTORY_RPC_CMD = 'blockchain.scripthash.get_history' +/** + * Get history RPC command (Electrum protocol) + */ +LocalIndexerWrapper.HEADERS_SUBSCRIBE_RPC_CMD = 'blockchain.headers.subscribe' module.exports = LocalIndexerWrapper diff --git a/lib/remote-importer/oxt-wrapper.js b/lib/remote-importer/oxt-wrapper.js index f1c482e..6bc66fa 100644 --- a/lib/remote-importer/oxt-wrapper.js +++ b/lib/remote-importer/oxt-wrapper.js @@ -4,7 +4,7 @@ */ 'use strict' -const rp = require('request-promise-native') +const axios = require('axios') const Logger = require('../logger') const network = require('../bitcoin/network') const keys = require('../../keys')[network.key] @@ -20,7 +20,7 @@ class OxtWrapper extends Wrapper { * Constructor */ constructor(url) { - super(url, keys.indexer.socks5Proxy) + super(url, keys.indexer.socks5Proxy) } /** @@ -32,15 +32,21 @@ class OxtWrapper extends Wrapper { const params = { url: `${this.base}${route}`, method: 'GET', - json: true, - timeout: 15000 + responseType: 'json', + timeout: 15000, + headers: { + 'User-Agent': 'Dojo' + } } // Sets socks proxy agent if required - if (keys.indexer.socks5Proxy != null) - params['agent'] = this.socksProxyAgent + if (keys.indexer.socks5Proxy != null) { + params['httpAgent'] = this.socksProxyAgent + params['httpsAgent'] = this.socksProxyAgent + } - return rp(params) + const result = await axios(params) + return result.data } /** @@ -55,7 +61,7 @@ class OxtWrapper extends Wrapper { // Try to retrieve more txs than the 1000 managed by the backend const uri = `/addresses/${address}/txids?count=${keys.addrFilterThreshold + 1}` const result = await this._get(uri) - + const ret = { address: address, ntx: result.count, @@ -109,6 +115,19 @@ class OxtWrapper extends Wrapper { return ret } + /** + * Retrieve the height of the chaintip for the remote source + * @returns {Promise} returns an object + * {chainTipHeight: } + */ + async getChainTipHeight() { + let chainTipHeight = null + const result = await this._get(`/lastblock`) + if (result != null && result['data'].length == 1) + chainTipHeight = parseInt(result['data'][0]['height']) + return {'chainTipHeight': chainTipHeight} + } + } module.exports = OxtWrapper diff --git a/lib/remote-importer/remote-importer.js b/lib/remote-importer/remote-importer.js index 3c4be3b..a46aa7e 100644 --- a/lib/remote-importer/remote-importer.js +++ b/lib/remote-importer/remote-importer.js @@ -25,7 +25,7 @@ if (network.key == 'bitcoin') { /** - * A singleton providing tools + * A singleton providing tools * for importing HD and loose addresses from remote sources */ class RemoteImporter { @@ -34,6 +34,8 @@ class RemoteImporter { * Constructor */ constructor() { + this.STATUS_RESCAN = 'rescan' + this.STATUS_IMPORT = 'import' // Guard against overlapping imports this.importing = {} this.sources = new Sources() @@ -50,12 +52,12 @@ class RemoteImporter { /** * Check if a xpub is currently being imported or rescanned by Dojo - * Returns true if import/rescan is in progress, otherwise returns false + * Returns infor about the operation if import/rescan is in progress, otherwise returns null * @param {string} xpub - xpub - * @returns {boolean} + * @returns {object} */ importInProgress(xpub) { - return this.importing[xpub] ? true : false + return this.importing[xpub] ? this.importing[xpub] : null } /** @@ -81,7 +83,7 @@ class RemoteImporter { if (!txParents[txid]) txParents[txid] = [] - + for (let i in tx.inputs) { const input = tx.inputs[i] let prev = input.outpoint.txid @@ -109,13 +111,13 @@ class RemoteImporter { * Import a list of transactions associated to a list of addresses * @param {object[]} addresses - array of addresses objects * @param {object[]} txns - array of transaction objects - * @returns {Promise} + * @returns {Promise} */ async _importTransactions(addresses, txns) { const addrIdMap = await db.getAddressesIds(addresses) - // The transactions array must be topologically ordered, such that - // entries earlier in the array MUST NOT depend upon any entry later + // The transactions array must be topologically ordered, such that + // entries earlier in the array MUST NOT depend upon any entry later // in the array. const txMaps = this._processTxsRelations(txns) const txOrdered = util.topologicalOrdering(txMaps.txParents, txMaps.txChildren) @@ -147,7 +149,11 @@ class RemoteImporter { return Promise.reject(errors.xpub.OVERLAP) } - this.importing[xpub] = true + this.importing[xpub] = { + 'status': this.STATUS_RESCAN, + 'txs_ext': 0, + 'txs_int': 0 + } const ts = hdaHelper.typeString(type) Logger.info(`Importer : Importing ${xpub} ${ts}`) @@ -161,7 +167,7 @@ class RemoteImporter { if (gapLimit && ((keys.indexer.active == 'local_bitcoind') || (keys.indexer.active == 'local_indexer')) - ) { + ) { gaps = [gapLimit, gapLimit] } @@ -182,6 +188,11 @@ class RemoteImporter { addresses = addresses.concat(result.addresses) } + this.importing[xpub] = { + 'status': this.STATUS_IMPORT, + 'txs': txns.length + } + // Store the hdaccount and the addresses into the database await db.ensureHDAccountId(xpub, type) await db.addAddressesToHDAccount(xpub, addresses) @@ -212,12 +223,12 @@ class RemoteImporter { * 4. Set u = highest chain index of used address, go to 1 * 5. Store all in database * - * @returns {object} returns + * @returns {object} returns * { * addresses: [{address, chain, index}], * transactions: [{ - * txid, - * version, + * txid, + * version, * locktime, * created, // if known * block: 'abcdef', // if confirmed @@ -290,6 +301,10 @@ class RemoteImporter { } if (gotTransactions) { + if (c == 0) + this.importing[xpub]['txs_ext'] = Object.keys(txids).length + else + this.importing[xpub]['txs_int'] = Object.keys(txids).length // We must go deeper const result = await this.xpubScan(xpub, c, d, u, G, type, txids) // Accumulate results from further down the rabbit hole @@ -331,7 +346,7 @@ class RemoteImporter { return true Logger.info(`Importer : Importing ${addresses.join(',')}`) - + try { const scanTx = [] const results = await this.sources.getAddresses(addresses, filterAddr) @@ -373,7 +388,7 @@ class RemoteImporter { if (N > 0) Logger.info(`Importer : Imported ${N} addresses in ${ts}s (${(dt/N).toFixed(0)} ms/addr)`) - + for (let address of addresses) delete this.importing[address] @@ -405,7 +420,7 @@ class RemoteImporter { const filteredTxs = txs.filter(tx => (tx.block && tx.block.hash == block.blockHash)) if (filteredTxs.length > 0) { const txids = filteredTxs.map(tx => tx.txid) - // Asynchronous confirmations + // Asynchronous confirmations db.confirmTransactions(txids, block.blockID) } } @@ -430,12 +445,12 @@ class RemoteImporter { } } await db.addOutputs(outputs) - + // Store the inputs in db const inputs = [] const spent = {} - // Get any outputs spent by the inputs of this transaction, + // Get any outputs spent by the inputs of this transaction, // add those database outIDs to the corresponding inputs, and store. let outpoints = [] for (let tx of txs) @@ -465,6 +480,15 @@ class RemoteImporter { } } + /** + * Retrieve the height of the chaintip for the remote source + * @returns {Promise} returns an object + * {chainTipHeight: } + */ + async getChainTipHeight() { + return this.sources.getChainTipHeight() + } + } module.exports = new RemoteImporter() diff --git a/lib/remote-importer/sources.js b/lib/remote-importer/sources.js index 8e8cef4..ee566cc 100644 --- a/lib/remote-importer/sources.js +++ b/lib/remote-importer/sources.js @@ -38,7 +38,7 @@ class Sources { try { const result = await this.source.getAddress(address, filterAddr) - + if (result.ntx) ret.ntx = result.ntx else if (result.txids) @@ -81,6 +81,22 @@ class Sources { } } + /** + * Retrieve the height of the chaintip + * @returns {Promise} returns an object + * {chainTipHeight: } + */ + async getChainTipHeight() { + let ret = {'chainTipHeight': null} + try { + ret = await this.source.getChainTipHeight() + } catch(e) { + Logger.error(e, `Importer : Sources.getChainTipHeight() : Error while retrieving the chaintip`) + } finally { + return ret + } + } + } module.exports = Sources diff --git a/package-lock.json b/package-lock.json index da87e85..a7451db 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "samourai-dojo", - "version": "1.8.0", + "version": "1.9.0", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -26,17 +26,6 @@ "es6-promisify": "5.0.0" } }, - "ajv": { - "version": "6.12.3", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.3.tgz", - "integrity": "sha512-4K0cK3L1hsqk9xIb2z9vs/XU+PGJZ9PNpJRDS9YLzmNdX6jmVPfamLvTJr0aDAusnHyCHO6MjzlkAsgtqp9teA==", - "requires": { - "fast-deep-equal": "3.1.3", - "fast-json-stable-stringify": "2.1.0", - "json-schema-traverse": "0.4.1", - "uri-js": "4.2.2" - } - }, "ansi-colors": { "version": "3.2.3", "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-3.2.3.tgz", @@ -124,19 +113,6 @@ "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" }, - "asn1": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", - "integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==", - "requires": { - "safer-buffer": "2.1.2" - } - }, - "assert-plus": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", - "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" - }, "async": { "version": "1.5.2", "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", @@ -150,20 +126,13 @@ "double-ended-queue": "2.1.0-0" } }, - "asynckit": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" - }, - "aws-sign2": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", - "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=" - }, - "aws4": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.10.0.tgz", - "integrity": "sha512-3YDiu347mtVtjpyV3u5kVqQLP242c06zwDOgpeRnybmXlYYsLbtTrUBUm8i8srONt+FWobl5aibnU1030PeeuA==" + "axios": { + "version": "0.20.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.20.0.tgz", + "integrity": "sha512-ANA4rr2BDcmmAQLOKft2fufrtuvlqR+cXNNinUmvfeSNCOF98PZL+7M/v1zIdGo7OLjEA9J2gXJL+j4zGsl0bA==", + "requires": { + "follow-redirects": "1.13.0" + } }, "babel-runtime": { "version": "5.8.38", @@ -187,14 +156,6 @@ "safe-buffer": "5.2.1" } }, - "bcrypt-pbkdf": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", - "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=", - "requires": { - "tweetnacl": "0.14.5" - } - }, "bech32": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/bech32/-/bech32-1.1.4.tgz", @@ -517,11 +478,6 @@ "resolved": "https://registry.npmjs.org/camelize/-/camelize-1.0.0.tgz", "integrity": "sha1-FkpUg+Yw+kMh5a8HAg5TGDGyYJs=" }, - "caseless": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", - "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" - }, "chalk": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", @@ -639,14 +595,6 @@ "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", "dev": true }, - "combined-stream": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", - "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "requires": { - "delayed-stream": "1.0.0" - } - }, "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -718,14 +666,6 @@ "sha.js": "2.4.11" } }, - "dashdash": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", - "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", - "requires": { - "assert-plus": "1.0.0" - } - }, "dasherize": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/dasherize/-/dasherize-2.0.0.tgz", @@ -767,11 +707,6 @@ "object-keys": "1.1.1" } }, - "delayed-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" - }, "delegates": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", @@ -818,15 +753,6 @@ "create-hmac": "1.1.7" } }, - "ecc-jsbn": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", - "integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=", - "requires": { - "jsbn": "0.1.1", - "safer-buffer": "2.1.2" - } - }, "ecdsa-sig-formatter": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", @@ -1091,26 +1017,6 @@ "resolved": "https://registry.npmjs.org/express-unless/-/express-unless-0.3.1.tgz", "integrity": "sha1-JVfBRudb65A+LSR/m1ugFFJpbiA=" }, - "extend": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", - "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" - }, - "extsprintf": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", - "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=" - }, - "fast-deep-equal": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", - "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" - }, - "fast-json-stable-stringify": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", - "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==" - }, "feature-policy": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/feature-policy/-/feature-policy-0.3.0.tgz", @@ -1169,20 +1075,10 @@ "is-buffer": "2.0.4" } }, - "forever-agent": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", - "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=" - }, - "form-data": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", - "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", - "requires": { - "asynckit": "0.4.0", - "combined-stream": "1.0.8", - "mime-types": "2.1.27" - } + "follow-redirects": { + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.13.0.tgz", + "integrity": "sha512-aq6gF1BEKje4a9i9+5jimNFIpq4Q1WiwBToeRK5NvZBd/TRsmW8BsJfOEGkr76TbOyPVD3OVDN910EcUNtRYEA==" }, "forwarded": { "version": "0.1.2", @@ -1244,14 +1140,6 @@ "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", "dev": true }, - "getpass": { - "version": "0.1.7", - "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", - "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", - "requires": { - "assert-plus": "1.0.0" - } - }, "github-from-package": { "version": "0.0.0", "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", @@ -1286,20 +1174,6 @@ "integrity": "sha512-qBr4OuELkhPenW6goKVXiv47US3clb3/IbuWF9KNKEijAy9oeHxU9IgzjvJhHkUzhaj7rOUD7+YGWqUjLp5oSA==", "dev": true }, - "har-schema": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", - "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=" - }, - "har-validator": { - "version": "5.1.3", - "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz", - "integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==", - "requires": { - "ajv": "6.12.3", - "har-schema": "2.0.0" - } - }, "has": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", @@ -1445,16 +1319,6 @@ } } }, - "http-signature": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", - "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", - "requires": { - "assert-plus": "1.0.0", - "jsprim": "1.4.1", - "sshpk": "1.16.1" - } - }, "iconv-lite": { "version": "0.4.23", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.23.tgz", @@ -1583,11 +1447,6 @@ "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=", "dev": true }, - "isstream": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", - "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" - }, "js-yaml": { "version": "3.13.1", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.13.1.tgz", @@ -1598,26 +1457,6 @@ "esprima": "4.0.1" } }, - "jsbn": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", - "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=" - }, - "json-schema": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", - "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=" - }, - "json-schema-traverse": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" - }, - "json-stringify-safe": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", - "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" - }, "jsonwebtoken": { "version": "8.5.1", "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-8.5.1.tgz", @@ -1642,17 +1481,6 @@ } } }, - "jsprim": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", - "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", - "requires": { - "assert-plus": "1.0.0", - "extsprintf": "1.3.0", - "json-schema": "0.2.3", - "verror": "1.10.0" - } - }, "jwa": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.4.1.tgz", @@ -1999,11 +1827,6 @@ "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=" }, - "oauth-sign": { - "version": "0.9.0", - "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", - "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==" - }, "object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", @@ -2150,11 +1973,6 @@ "sha.js": "2.4.11" } }, - "performance-now": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", - "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" - }, "picomatch": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.2.2.tgz", @@ -2216,11 +2034,6 @@ "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=" }, - "psl": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/psl/-/psl-1.8.0.tgz", - "integrity": "sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ==" - }, "pump": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/pump/-/pump-2.0.1.tgz", @@ -2230,11 +2043,6 @@ "once": "1.4.0" } }, - "punycode": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", - "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" - }, "pushdata-bitcoin": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/pushdata-bitcoin/-/pushdata-bitcoin-1.0.1.tgz", @@ -2307,51 +2115,6 @@ "resolved": "https://registry.npmjs.org/referrer-policy/-/referrer-policy-1.2.0.tgz", "integrity": "sha512-LgQJIuS6nAy1Jd88DCQRemyE3mS+ispwlqMk3b0yjZ257fI1v9c+/p6SD5gP5FGyXUIgrNOAfmyioHwZtYv2VA==" }, - "request": { - "version": "2.88.0", - "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz", - "integrity": "sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==", - "requires": { - "aws-sign2": "0.7.0", - "aws4": "1.10.0", - "caseless": "0.12.0", - "combined-stream": "1.0.8", - "extend": "3.0.2", - "forever-agent": "0.6.1", - "form-data": "2.3.3", - "har-validator": "5.1.3", - "http-signature": "1.2.0", - "is-typedarray": "1.0.0", - "isstream": "0.1.2", - "json-stringify-safe": "5.0.1", - "mime-types": "2.1.27", - "oauth-sign": "0.9.0", - "performance-now": "2.1.0", - "qs": "6.5.2", - "safe-buffer": "5.2.1", - "tough-cookie": "2.4.3", - "tunnel-agent": "0.6.0", - "uuid": "3.4.0" - } - }, - "request-promise-core": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/request-promise-core/-/request-promise-core-1.1.1.tgz", - "integrity": "sha1-Pu4AssWqgyOc+wTFcA2jb4HNCLY=", - "requires": { - "lodash": "4.17.19" - } - }, - "request-promise-native": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/request-promise-native/-/request-promise-native-1.0.5.tgz", - "integrity": "sha1-UoF3D2jgyXGeUWP9P6tIIhX0/aU=", - "requires": { - "request-promise-core": "1.1.1", - "stealthy-require": "1.1.1", - "tough-cookie": "2.4.3" - } - }, "require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -2514,32 +2277,11 @@ "resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.1.tgz", "integrity": "sha1-R1OT/56RR5rqYtyvDKPRSYOn+0A=" }, - "sshpk": { - "version": "1.16.1", - "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz", - "integrity": "sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg==", - "requires": { - "asn1": "0.2.4", - "assert-plus": "1.0.0", - "bcrypt-pbkdf": "1.0.2", - "dashdash": "1.14.1", - "ecc-jsbn": "0.1.2", - "getpass": "0.1.7", - "jsbn": "0.1.1", - "safer-buffer": "2.1.2", - "tweetnacl": "0.14.5" - } - }, "statuses": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" }, - "stealthy-require": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz", - "integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks=" - }, "string-width": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", @@ -2691,22 +2433,6 @@ "is-number": "7.0.0" } }, - "tough-cookie": { - "version": "2.4.3", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz", - "integrity": "sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ==", - "requires": { - "psl": "1.8.0", - "punycode": "1.4.1" - }, - "dependencies": { - "punycode": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", - "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" - } - } - }, "tunnel-agent": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", @@ -2715,11 +2441,6 @@ "safe-buffer": "5.2.1" } }, - "tweetnacl": { - "version": "0.14.5", - "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", - "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" - }, "type-is": { "version": "1.6.18", "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", @@ -2752,14 +2473,6 @@ "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" }, - "uri-js": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", - "integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==", - "requires": { - "punycode": "2.1.1" - } - }, "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", @@ -2770,11 +2483,6 @@ "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" }, - "uuid": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", - "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" - }, "validator": { "version": "10.8.0", "resolved": "https://registry.npmjs.org/validator/-/validator-10.8.0.tgz", @@ -2793,16 +2501,6 @@ "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=" }, - "verror": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", - "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", - "requires": { - "assert-plus": "1.0.0", - "core-util-is": "1.0.2", - "extsprintf": "1.3.0" - } - }, "websocket": { "version": "1.0.28", "resolved": "https://registry.npmjs.org/websocket/-/websocket-1.0.28.tgz", diff --git a/package.json b/package.json index d2b4d14..4f6aa28 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "samourai-dojo", - "version": "1.8.0", + "version": "1.9.0", "description": "Backend server for Samourai Wallet", "main": "accounts/index.js", "scripts": { @@ -15,6 +15,7 @@ "homepage": "https://code.samourai.io/dojo/samourai-dojo", "dependencies": { "async-sema": "2.1.2", + "axios": "0.20.0", "bip39": "2.4.0", "bitcoind-rpc-client": "0.3.1", "bitcoinjs-lib": "5.1.4", @@ -30,8 +31,6 @@ "mysql": "2.16.0", "passport": "0.4.0", "passport-localapikey-update": "0.6.0", - "request": "2.88.0", - "request-promise-native": "1.0.5", "socks-proxy-agent": "4.0.1", "validator": "10.8.0", "websocket": "1.0.28", diff --git a/static/admin/css/style.css b/static/admin/css/style.css index 3526b53..a1da6e6 100644 --- a/static/admin/css/style.css +++ b/static/admin/css/style.css @@ -265,7 +265,7 @@ td.table-value { /* PAGES - COMMONS */ body.dmt { min-height: 100vh; - background-image: url("../icons/samourai-logo-loading.png"); + background-image: url("../icons/samourai-logo.png"); background-repeat: no-repeat; background-position: center; } @@ -453,6 +453,10 @@ button { color: #76d776; } +#indexer-status { + padding-bottom: 12px; +} + /* PAGES - PAIRING */ #qr-label, #qr-explorer-label { @@ -530,6 +534,17 @@ button { display: inline-block; } +#xpubs-deletion-actions span { + display: inline; +} + +#xpubs-deletion-actions input { + width: 50px; + margin-left: 5px; + margin-right: 5px; + display: inline-block; +} + #xpubs-tool-details #xpub-value { overflow: hidden; } diff --git a/static/admin/dmt/addresses-tools/addresses-tools.js b/static/admin/dmt/addresses-tools/addresses-tools.js index ccc302c..c8c4dbb 100644 --- a/static/admin/dmt/addresses-tools/addresses-tools.js +++ b/static/admin/dmt/addresses-tools/addresses-tools.js @@ -30,8 +30,7 @@ const screenAddressesToolsScript = { lib_api.getExplorerPairingInfo().then(explorerInfo => { this.explorerInfo = explorerInfo }).catch(e => { - lib_msg.displayErrors(lib_msg.extractJqxhrErrorMsg(e)) - console.log(e) + lib_errors.processError(e) }) }, @@ -68,14 +67,13 @@ const screenAddressesToolsScript = { this.showImportForm(false) } }).catch(e => { - lib_msg.displayErrors(lib_msg.extractJqxhrErrorMsg(e)) - console.log(e) + lib_errors.processError(e) throw e }) }, importAddress: function() { - lib_msg.displayMessage('Processing address import...'); + lib_msg.displayMessage('Processing address import. Please wait...'); const jsonData = {'active': this.currentAddress} return lib_api.getWallet(jsonData) .then(result => { @@ -83,13 +81,12 @@ const screenAddressesToolsScript = { lib_msg.displayInfo('Import complete') }) }).catch(e => { - lib_msg.displayErrors(lib_msg.extractJqxhrErrorMsg(e)) - console.log(e) + lib_errors.processError(e) }) }, rescanAddress: function() { - lib_msg.displayMessage('Processing address rescan...'); + lib_msg.displayMessage('Processing address rescan. Please wait...'); return lib_api.getAddressRescan(this.currentAddress) .then(result => { this.hideRescanForm() @@ -97,8 +94,7 @@ const screenAddressesToolsScript = { lib_msg.displayInfo('Rescan complete') }) }).catch(e => { - lib_msg.displayErrors(lib_msg.extractJqxhrErrorMsg(e)) - console.log(e) + lib_errors.processError(e) }) }, diff --git a/static/admin/dmt/blocks-rescan/blocks-rescan.html b/static/admin/dmt/blocks-rescan/blocks-rescan.html index b99a52f..80738b3 100644 --- a/static/admin/dmt/blocks-rescan/blocks-rescan.html +++ b/static/admin/dmt/blocks-rescan/blocks-rescan.html @@ -1,7 +1,7 @@

BLOCKS RESCAN

-
Force the Tracker to rescan a range of blocks.
+
Force the Tracker to rescan a small range of blocks.
diff --git a/static/admin/dmt/blocks-rescan/blocks-rescan.js b/static/admin/dmt/blocks-rescan/blocks-rescan.js index 0d47447..19b9bb8 100644 --- a/static/admin/dmt/blocks-rescan/blocks-rescan.js +++ b/static/admin/dmt/blocks-rescan/blocks-rescan.js @@ -32,8 +32,7 @@ const screenBlocksRescanScript = { const msg = `successfully rescanned blocks between height ${fromHeightRes} and height ${toHeightRes}` lib_msg.displayInfo(msg) }).catch(e => { - lib_msg.displayErrors(lib_msg.extractJqxhrErrorMsg(e)) - console.log(e) + lib_errors.processError(e) }).then(() => { $('#rescan-from-height').val('') $('#rescan-to-height').val('') diff --git a/static/admin/dmt/index.html b/static/admin/dmt/index.html index 3140759..0237e01 100644 --- a/static/admin/dmt/index.html +++ b/static/admin/dmt/index.html @@ -15,6 +15,7 @@ + diff --git a/static/admin/dmt/pairing/pairing.js b/static/admin/dmt/pairing/pairing.js index 56d2ef6..4073473 100644 --- a/static/admin/dmt/pairing/pairing.js +++ b/static/admin/dmt/pairing/pairing.js @@ -27,8 +27,7 @@ const screenPairingScript = { lib_msg.cleanMessagesUi() return result }).catch(e => { - lib_msg.displayErrors(lib_msg.extractJqxhrErrorMsg(e)) - console.log(e) + lib_errors.processError(e) return result }) }, diff --git a/static/admin/dmt/pushtx/pushtx.js b/static/admin/dmt/pushtx/pushtx.js index 514d44d..6472e3b 100644 --- a/static/admin/dmt/pushtx/pushtx.js +++ b/static/admin/dmt/pushtx/pushtx.js @@ -15,7 +15,7 @@ const pushtxScript = { }, refreshPushTxStatus: function() { - lib_msg.displayMessage('Loading PushTx status info...'); + //lib_msg.displayMessage('Loading PushTx status info...'); lib_api.getPushtxStatus().then(pushTxStatus => { if (pushTxStatus) { const data = pushTxStatus['data'] @@ -23,19 +23,18 @@ const pushtxScript = { $('#pushed-uptime').text(uptime) $('#pushed-count').text(data['push']['count']) $('#pushed-amount').text(data['push']['amount']) - lib_msg.cleanMessagesUi() + //lib_msg.cleanMessagesUi() } }).catch(e => { $('#pushed-uptime').text('-') $('#pushed-count').text('-') $('#pushed-amount').text('-') - lib_msg.displayErrors(lib_msg.extractJqxhrErrorMsg(e)) - console.log(e) + lib_errors.processError(e) }) }, refreshScheduledTxsList: function() { - lib_msg.displayMessage('Loading PushTx orchestrator status info...'); + //lib_msg.displayMessage('Loading PushTx orchestrator status info...'); lib_api.getOrchestratorStatus().then(orchestrStatus => { if(orchestrStatus) { const data = orchestrStatus['data'] @@ -45,11 +44,10 @@ const pushtxScript = { this.processedSchedTxs.add(tx['schTxid']) } } - lib_msg.cleanMessagesUi() + //lib_msg.cleanMessagesUi() } }).catch(e => { - lib_msg.displayErrors(lib_msg.extractJqxhrErrorMsg(e)) - console.log(e) + lib_errors.processError(e) }) }, diff --git a/static/admin/dmt/status/status.html b/static/admin/dmt/status/status.html index 0377afd..e2b1ea4 100644 --- a/static/admin/dmt/status/status.html +++ b/static/admin/dmt/status/status.html @@ -5,6 +5,7 @@
+
FULL NODE
@@ -41,8 +42,36 @@
+ +
+
INDEXER
+
+
+ + + + + + + + + + + + + + + + + +
Status
Latest block
Indexer type
URL
+
+
+
+
+
TRACKER
@@ -64,6 +93,23 @@
+
+
DOJO DB
+
+
+ + + + + + + + + +
Status
Latest block
+
+
+
WEB
@@ -84,6 +130,7 @@
+
diff --git a/static/admin/dmt/status/status.js b/static/admin/dmt/status/status.js index e893b97..f427e6c 100644 --- a/static/admin/dmt/status/status.js +++ b/static/admin/dmt/status/status.js @@ -1,6 +1,9 @@ const statusScript = { initPage: function() { + this.chaintipBitcoind = 0 + this.chaintipIndexer = 0 + this.chaintipDb = 0 // Refresh API status setInterval(() => {this.refreshApiStatus()}, 60000) // Refresh PushTx status @@ -13,56 +16,126 @@ const statusScript = { }, refreshApiStatus: function() { - lib_msg.displayMessage('Loading API status info...'); + // Set default values displayed + this.setStatusIndicator('#db-status-ind', 'idle') + this.setStatusIndicator('#tracker-status-ind', 'idle') + this.setStatusIndicator('#indexer-status-ind', 'idle') + $('#tracker-uptime').text('-') + $('#tracker-chaintip').text('-') + $('#db-chaintip').text('-') + $('#indexer-chaintip').text('-') + $('#indexer-type').text('-') + $('#indexer-url').text('-') + + //lib_msg.displayMessage('Loading API status info...'); return lib_api.getApiStatus().then(apiStatus => { if (apiStatus) { - $('#tracker-status-ind').html('✓') - $('#tracker-status-ind').css('color', '#76d776') $('#tracker-uptime').text(apiStatus['uptime']) - $('#tracker-chaintip').text(apiStatus['blocks']) - lib_msg.cleanMessagesUi() + + const blocks = apiStatus['blocks'] + if (blocks) { + this.chaintipBitcoind = blocks + this.chaintipDb = blocks + $('#db-chaintip').text(blocks) + $('#tracker-chaintip').text(blocks) + this.setStatusIndicator('#db-status-ind', 'ok') + this.setStatusIndicator('#tracker-status-ind', 'ok') + } else { + this.setStatusIndicator('#db-status-ind', 'ko') + this.setStatusIndicator('#tracker-status-ind', 'ko') + } + + if (apiStatus['indexer']) { + const indexerMaxHeight = apiStatus['indexer']['maxHeight'] + if (indexerMaxHeight) { + this.chaintipIndexer = indexerMaxHeight + $('#indexer-chaintip').text(indexerMaxHeight) + this.setStatusIndicator('#indexer-status-ind', 'ok') + } else { + this.setStatusIndicator('#indexer-status-ind', 'ko') + } + const indexerType = apiStatus['indexer']['type'] + if (indexerType) + $('#indexer-type').text(indexerType.replace(/_/g, ' ')) + const indexerUrl = apiStatus['indexer']['url'] + if (indexerUrl) + $('#indexer-url').text(indexerUrl) + } + + this.checkChaintips() + //lib_msg.cleanMessagesUi() } }).catch(e => { - $('#tracker-status-ind').text('X') - $('#tracker-status-ind').css('color', '#f77c7c') - $('#tracker-uptime').text('-') - $('#tracker-chaintip').text('-') - lib_msg.displayErrors(lib_msg.extractJqxhrErrorMsg(e)) - console.log(e) + this.setStatusIndicator('#db-status-ind', 'ko') + this.setStatusIndicator('#tracker-status-ind', 'ko') + this.setStatusIndicator('#indexer-status-ind', 'ko') + lib_errors.processError(e) }) }, refreshPushTxStatus: function() { - lib_msg.displayMessage('Loading Tracker status info...'); + // Set default values displayed + this.setStatusIndicator('#node-status-ind', 'idle') + $('#node-uptime').text('-') + $('#node-chaintip').text('-') + $('#node-version').text('-') + $('#node-network').text('-') + $('#node-conn').text('-') + $('#node-relay-fee').text('-') + + //lib_msg.displayMessage('Loading Tracker status info...'); lib_api.getPushtxStatus().then(pushTxStatus => { if (pushTxStatus) { const data = pushTxStatus['data'] - $('#node-status-ind').html('✓') - $('#node-status-ind').css('color', '#76d776') + this.setStatusIndicator('#node-status-ind', 'ok') const uptime = lib_cmn.timePeriod(data['uptime']) $('#node-uptime').text(uptime) + this.chaintipBitcoind = data['bitcoind']['blocks'] $('#node-chaintip').text(data['bitcoind']['blocks']) $('#node-version').text(data['bitcoind']['version']) const network = data['bitcoind']['testnet'] == true ? 'testnet' : 'mainnet' $('#node-network').text(network) $('#node-conn').text(data['bitcoind']['conn']) $('#node-relay-fee').text(data['bitcoind']['relayfee']) - lib_msg.cleanMessagesUi() + this.checkChaintips() + //lib_msg.cleanMessagesUi() } }).catch(e => { - $('#node-status-ind').text('-') - $('#node-status-ind').css('color', '#f77c7c') - $('#node-uptime').text('-') - $('#node-chaintip').text('-') - $('#node-version').text('-') - $('#node-network').text('-') - $('#node-conn').text('-') - $('#node-relay-fee').text('-') - lib_msg.displayErrors(lib_msg.extractJqxhrErrorMsg(e)) - console.log(e) + this.setStatusIndicator('#node-status-ind', 'ko') + lib_errors.processError(e) }) }, + checkChaintips: function() { + if (this.chaintipBitcoind > this.chaintipDb) { + this.setStatusIndicator('#db-status-ind', 'desynchronized') + this.setStatusIndicator('#tracker-status-ind', 'desynchronized') + } + if (this.chaintipBitcoind > this.chaintipIndexer) { + this.setStatusIndicator('#indexer-status-ind', 'desynchronized') + } else if (this.chaintipBitcoind < this.chaintipIndexer) { + this.setStatusIndicator('#node-status-ind', 'desynchronized') + this.setStatusIndicator('#db-status-ind', 'desynchronized') + this.setStatusIndicator('#tracker-status-ind', 'desynchronized') + } + }, + + setStatusIndicator: function(id, status) { + if (status == 'ok') { + $(id).html('✓') + $(id).css('color', '#76d776') + } else if (status == 'ko') { + $(id).html('X') + $(id).css('color', '#f77c7c') + } else if (status == 'desynchronized') { + $(id).html('✓') + $(id).css('color', '#f0c649') + } else { + $(id).html('-') + $(id).css('color', '#efefef') + } + }, + } screenScripts.set('#screen-status', statusScript) \ No newline at end of file diff --git a/static/admin/dmt/txs-tools/txs-tools.js b/static/admin/dmt/txs-tools/txs-tools.js index b6e992e..6bf7f0e 100644 --- a/static/admin/dmt/txs-tools/txs-tools.js +++ b/static/admin/dmt/txs-tools/txs-tools.js @@ -24,8 +24,7 @@ const screenTxsToolsScript = { lib_api.getExplorerPairingInfo().then(explorerInfo => { this.explorerInfo = explorerInfo }).catch(e => { - lib_msg.displayErrors(lib_msg.extractJqxhrErrorMsg(e)) - console.log(e) + lib_errors.processError(e) }) }, diff --git a/static/admin/dmt/welcome/welcome.html b/static/admin/dmt/welcome/welcome.html index 4584cda..ac78347 100644 --- a/static/admin/dmt/welcome/welcome.html +++ b/static/admin/dmt/welcome/welcome.html @@ -26,7 +26,7 @@ Check if a transaction is found in a block or in the mempool of your full node. BLOCKS RESCAN - Rescan the transactions confirmed by the blocks in a given range. + Force the Tracker to rescan a small range of blocks.
For large rescans you should prefer XPUB or address rescans.
HELP diff --git a/static/admin/dmt/xpubs-tools/xpubs-tools.html b/static/admin/dmt/xpubs-tools/xpubs-tools.html index 2eeb6b9..fd7017e 100644 --- a/static/admin/dmt/xpubs-tools/xpubs-tools.html +++ b/static/admin/dmt/xpubs-tools/xpubs-tools.html @@ -61,6 +61,7 @@
+
@@ -77,6 +78,14 @@ +
+
+ Do you want to delete this xpub? + + +
+
+
diff --git a/static/admin/dmt/xpubs-tools/xpubs-tools.js b/static/admin/dmt/xpubs-tools/xpubs-tools.js index b8967d5..2ada7e5 100644 --- a/static/admin/dmt/xpubs-tools/xpubs-tools.js +++ b/static/admin/dmt/xpubs-tools/xpubs-tools.js @@ -3,6 +3,7 @@ const screenXpubsToolsScript = { explorerInfo: null, currentXpub: null, isReimport: false, + rescanStatusTimerId: null, initPage: function() { this.getExplorerInfo() @@ -10,8 +11,11 @@ const screenXpubsToolsScript = { $('#btn-xpub-search-go').click(() => {this.searchXpub()}) $('#btn-xpub-details-reset').click(() => {this.showSearchForm()}) $('#btn-xpub-details-rescan').click(() => {this.showRescanForm()}) + $('#btn-xpub-details-delete').click(() => {this.showDeletionForm()}) $('#btn-xpub-rescan-go').click(() => {this.rescanXpub()}) $('#btn-xpub-rescan-cancel').click(() => {this.hideRescanForm()}) + $('#btn-xpub-delete-go').click(() => {this.deleteXpub()}) + $('#btn-xpub-delete-cancel').click(() => {this.hideDeletionForm()}) $('#btn-xpub-import-go').click(() => {this.importXpub()}) $('#btn-xpub-details-retype').click(() => {this.showImportForm(true)}) $('#btn-xpub-import-cancel').click(() => {this.hideImportForm(this.isReimport)}) @@ -24,6 +28,7 @@ const screenXpubsToolsScript = { preparePage: function() { this.hideRescanForm() + this.hideDeletionForm() this.showSearchForm() $("#xpub").focus() }, @@ -32,8 +37,7 @@ const screenXpubsToolsScript = { lib_api.getExplorerPairingInfo().then(explorerInfo => { this.explorerInfo = explorerInfo }).catch(e => { - lib_msg.displayErrors(lib_msg.extractJqxhrErrorMsg(e)) - console.log(e) + lib_errors.processError(e) }) }, @@ -70,14 +74,13 @@ const screenXpubsToolsScript = { this.showImportForm(false) } }).catch(e => { - lib_msg.displayErrors(lib_msg.extractJqxhrErrorMsg(e)) - console.log(e) + lib_errors.processError(e) throw e }) }, importXpub: function() { - lib_msg.displayMessage('Processing xpub import...'); + lib_msg.displayMessage('Processing xpub import. Please wait...'); const jsonData = { 'xpub': this.currentXpub, @@ -95,35 +98,75 @@ const screenXpubsToolsScript = { jsonData['segwit'] = 'bip84' } - return lib_api.postXpub(jsonData) - .then(result => { + try { + lib_api.postXpub(jsonData) + // Wait for import completion and display progress + this.checkRescanStatus(() => { this._searchXpub(this.currentXpub).then(() => { lib_msg.displayInfo('Import complete') }) - }).catch(e => { - lib_msg.displayErrors(lib_msg.extractJqxhrErrorMsg(e)) - console.log(e) }) + } catch(e) { + lib_errors.processError(e) + } }, rescanXpub: function() { - lib_msg.displayMessage('Processing xpub rescan...'); + lib_msg.displayMessage('Processing xpub rescan. Please wait...'); let startIdx = $('#rescan-start-idx').val() startIdx = (startIdx == null) ? 0 : parseInt(startIdx) let lookahead = $('#rescan-lookahead').val() lookahead = (lookahead == null) ? 100 : parseInt(lookahead) - return lib_api.getXpubRescan(this.currentXpub, lookahead, startIdx) - .then(result => { + + try { + lib_api.getXpubRescan(this.currentXpub, lookahead, startIdx) + // Wait for rescan completion and display progress + this.checkRescanStatus(() => { this.hideRescanForm() this._searchXpub(this.currentXpub).then(() => { lib_msg.displayInfo('Rescan complete') }) + }) + } catch(e) { + lib_errors.processError(e) + } + }, + + deleteXpub: function() { + lib_msg.displayMessage('Deleting a xpub. Please wait...') + return lib_api.getXpubDelete(this.currentXpub) + .then(result => { + this.currentXpub = null + this.preparePage() + lib_msg.displayInfo('Xpub successfully deleted') }).catch(e => { - lib_msg.displayErrors(lib_msg.extractJqxhrErrorMsg(e)) - console.log(e) + lib_errors.processError(e) }) }, + checkRescanStatus: function(callback) { + this.rescanStatusTimerId = setTimeout(() => { + lib_api.getXpubRescanStatus(this.currentXpub) + .then(result => { + const data = result['data'] + if (data['import_in_progress']) { + const lblOp = (data['status'] == 'rescan') ? 'Rescan' : 'Import' + const lblHits = (data['status'] == 'rescan') ? 'hits detected' : 'transactions imported' + const msg = `${lblOp} in progress (${data['hits']} ${lblHits})` + lib_msg.displayMessage(msg) + return this.checkRescanStatus(callback) + } else { + clearTimeout(this.rescanStatusTimerId) + return callback() + } + }).catch(e => { + lib_errors.processError(e) + lib_msg.displayMessage('Rescan in progress. Please wait...') + return this.checkRescanStatus(callback) + }) + }, 1000) + }, + setXpubDetails: function(xpubInfo) { $('tr.tx-row').remove() $('tr.utxo-row').remove() @@ -254,6 +297,17 @@ const screenXpubsToolsScript = { $('#xpubs-tool-actions').show() }, + showDeletionForm: function() { + $('#xpubs-tool-actions').hide() + $('#xpubs-deletion-actions').show() + lib_msg.cleanMessagesUi() + }, + + hideDeletionForm: function() { + $('#xpubs-deletion-actions').hide() + $('#xpubs-tool-actions').show() + }, + } screenScripts.set('#screen-xpubs-tools', screenXpubsToolsScript) diff --git a/static/admin/icons/samourai-logo-loading.png b/static/admin/icons/samourai-logo-loading.png deleted file mode 100644 index 8df79af..0000000 Binary files a/static/admin/icons/samourai-logo-loading.png and /dev/null differ diff --git a/static/admin/icons/samourai-logo-trans@2x.png b/static/admin/icons/samourai-logo-trans@2x.png deleted file mode 100644 index 50a599c..0000000 Binary files a/static/admin/icons/samourai-logo-trans@2x.png and /dev/null differ diff --git a/static/admin/icons/samourai-logo.png b/static/admin/icons/samourai-logo.png new file mode 100644 index 0000000..0a0c108 Binary files /dev/null and b/static/admin/icons/samourai-logo.png differ diff --git a/static/admin/index.html b/static/admin/index.html index f66f05d..adb5ff0 100644 --- a/static/admin/index.html +++ b/static/admin/index.html @@ -14,6 +14,7 @@ + @@ -22,7 +23,7 @@
- +

DOJO // MAINTENANCE TOOL beta

diff --git a/static/admin/index.js b/static/admin/index.js index fd8980d..1349deb 100644 --- a/static/admin/index.js +++ b/static/admin/index.js @@ -33,8 +33,7 @@ function login() { } }, function (jqxhr) { - let msg = lib_msg.extractJqxhrErrorMsg(jqxhr) - lib_msg.displayErrors(msg) + lib_errors.processError(jqxhr) } ) } diff --git a/static/admin/lib/api-wrapper.js b/static/admin/lib/api-wrapper.js index 9a3b43b..2fdae1d 100644 --- a/static/admin/lib/api-wrapper.js +++ b/static/admin/lib/api-wrapper.js @@ -110,6 +110,23 @@ const lib_api = { ) }, + /** + * Deletes a xpub + */ + getXpubDelete: function(xpub) { + let prefix = conf['prefixes']['support'] + let uri = this.baseUri + '/' + prefix + '/xpub/' + xpub + '/delete' + return this.sendGetUriEncoded(uri, {}) + }, + + /** + * Gets the status of a xpub rescan + */ + getXpubRescanStatus: function(xpub) { + let uri = this.baseUri + '/xpub/' + xpub + '/import/status' + return this.sendGetUriEncoded(uri, {}) + }, + /** * Notifies the server of the new HD account for tracking. */ diff --git a/static/admin/lib/auth-utils.js b/static/admin/lib/auth-utils.js index d357971..656a3d7 100644 --- a/static/admin/lib/auth-utils.js +++ b/static/admin/lib/auth-utils.js @@ -9,6 +9,9 @@ const lib_auth = { /* SessionStorage Key used for refresh token */ SESSION_STORE_REFRESH_TOKEN: 'refresh_token', + /* SessionStorage Key used for the timestamp of the refresh token */ + SESSION_STORE_REFRESH_TOKEN_TS: 'refresh_token_ts', + /* JWT Scheme */ JWT_SCHEME: 'Bearer', @@ -43,6 +46,8 @@ const lib_auth = { * Stores refresh token in session storage */ setRefreshToken: function(token) { + const now = new Date(); + sessionStorage.setItem(this.SESSION_STORE_REFRESH_TOKEN_TS, now.getTime()) sessionStorage.setItem(this.SESSION_STORE_REFRESH_TOKEN, token) }, @@ -56,17 +61,23 @@ const lib_auth = { const now = new Date(); const atts = sessionStorage.getItem(this.SESSION_STORE_ACCESS_TOKEN_TS) - const timeElapsed = (now.getTime() - atts) / 1000 + let timeElapsed = (now.getTime() - atts) / 1000 // Refresh the access token if more than 5mn if (timeElapsed > 300) { - const dataJson = { - 'rt': this.getRefreshToken() + // Check if refresh token has expired or is about to expire + const rtts = sessionStorage.getItem(this.SESSION_STORE_REFRESH_TOKEN_TS) + if ((now.getTime() - rtts) / 1000 > 7200 - 60) { + // Force user to sign in again + this.logout() + return } let self = this - let deferred = lib_api.refreshToken(dataJson) + let deferred = lib_api.refreshToken({ + 'rt': this.getRefreshToken() + }) deferred.then( function (result) { diff --git a/static/admin/lib/errors-utils.js b/static/admin/lib/errors-utils.js new file mode 100644 index 0000000..106cd16 --- /dev/null +++ b/static/admin/lib/errors-utils.js @@ -0,0 +1,25 @@ +const lib_errors = { + + // Extract jqxhr error message + extractJqxhrErrorMsg: function(jqxhr) { + let hasErrorMsg = ('responseJSON' in jqxhr) && + (jqxhr['responseJSON'] != null) && + ('error' in jqxhr['responseJSON']) && + (typeof jqxhr['responseJSON']['error'] == 'string') + + return hasErrorMsg ? jqxhr['responseJSON']['error'] : jqxhr.statusText + }, + + // Manage errors + processError: function(e) { + const errorMsg = this.extractJqxhrErrorMsg(e) + // Redirect to sign in page if authentication error + if (errorMsg == 'Invalid JSON Web Token' || errorMsg == 'Missing JSON Web Token') { + lib_auth.logout() + } else { + lib_msg.displayErrors(errorMsg) + console.log(e) + } + }, + +} diff --git a/static/admin/lib/messages.js b/static/admin/lib/messages.js index 8518491..b08adbb 100644 --- a/static/admin/lib/messages.js +++ b/static/admin/lib/messages.js @@ -1,14 +1,5 @@ const lib_msg = { - // Extracts jqxhr error message - extractJqxhrErrorMsg: function(jqxhr) { - let hasErrorMsg = ('responseJSON' in jqxhr) && - (jqxhr['responseJSON'] != null) && - ('error' in jqxhr['responseJSON']) - - return hasErrorMsg ? jqxhr['responseJSON']['error'] : jqxhr.statusText - }, - // UI functions addTextinID: function(text, id){ $(id).html(text.toUpperCase()) diff --git a/tracker/blockchain-processor.js b/tracker/blockchain-processor.js index b5f7069..f0ee571 100644 --- a/tracker/blockchain-processor.js +++ b/tracker/blockchain-processor.js @@ -60,7 +60,7 @@ class BlockchainProcessor extends AbstractProcessor { const daemonNbHeaders = info.headers // Consider that we are in IBD mode if Dojo is far in the past (> 13,000 blocks) - this.isIBD = (highest.blockHeight < 612000) || (highest.blockHeight < daemonNbHeaders - 13000) + this.isIBD = (highest.blockHeight < 655000) || (highest.blockHeight < daemonNbHeaders - 13000) if (this.isIBD) return this.catchupIBDMode() @@ -169,7 +169,7 @@ class BlockchainProcessor extends AbstractProcessor { try { const hash = await this.client.getblockhash(height) const header = await this.client.getblockheader(hash) - return this.processBlock(header) + return this.processBlock(header) } catch(e) { Logger.error(e, 'Tracker : BlockchainProcessor.catchupNormalMode()') process.exit() @@ -206,25 +206,25 @@ class BlockchainProcessor extends AbstractProcessor { /** * Upon receipt of a new block hash, retrieve the block header from bitcoind via * RPC. Continue pulling block headers back through the chain until the database - * contains header.previousblockhash, adding the headers to a stack. If the - * previousblockhash is not found on the first call, this is either a chain + * contains header.previousblockhash, adding the headers to a stack. If the + * previousblockhash is not found on the first call, this is either a chain * re-org or the tracker missed blocks during a shutdown. * * Once the chain has bottomed out with a known block in the database, delete * all known database transactions confirmed in blocks at heights greater than - * the last known block height. These transactions are orphaned but may reappear - * in the new chain. Notify relevant accounts of balance updates / + * the last known block height. These transactions are orphaned but may reappear + * in the new chain. Notify relevant accounts of balance updates / * transaction confirmation counts. * * Delete block entries not on the main chain. * - * Forward-scan through the block headers, pulling the full raw block hex via + * Forward-scan through the block headers, pulling the full raw block hex via * RPC. The raw block contains all transactions and is parsed by bitcoinjs-lib. - * Add the block to the database. Run checkTransaction for each transaction in + * Add the block to the database. Run checkTransaction for each transaction in * the block that is not in the database. Confirm all transactions in the block. * * After each block, query bitcoin against all database unconfirmed outputs - * to see if they remain in the mempool or have been confirmed in blocks. + * to see if they remain in the mempool or have been confirmed in blocks. * Malleated transactions entering the wallet will disappear from the mempool on * block confirmation. * @@ -247,7 +247,7 @@ class BlockchainProcessor extends AbstractProcessor { } catch(err) { Logger.error(err, `Tracker : BlockchainProcessor.onBlockHash() : error in getblockheader(${blockHash})`) } - + if(headers == null) return null @@ -263,7 +263,7 @@ class BlockchainProcessor extends AbstractProcessor { // Process the blocks return await util.seriesCall(headers, header => { - return this.processBlock(header) + return this.processBlock(header) }) } catch(e) { @@ -302,7 +302,7 @@ class BlockchainProcessor extends AbstractProcessor { } /** - * Cancel confirmation of transactions + * Cancel confirmation of transactions * and delete blocks after a given height * @param {integer} height - height of last block maintained * @returns {Promise} @@ -348,7 +348,7 @@ class BlockchainProcessor extends AbstractProcessor { Logger.info(`Tracker : Rescanning block ${height}`) const hash = await this.client.getblockhash(height) const header = await this.client.getblockheader(hash) - return this.processBlock(header) + return this.processBlock(header) } catch(e) { Logger.error(e, 'Tracker : BlockchainProcessor.rescan()') throw e @@ -367,7 +367,7 @@ class BlockchainProcessor extends AbstractProcessor { const hex = await this.client.getblock(header.hash, false) const block = new Block(hex, header) - + const txsForBroadcast = await block.checkBlock() // Send notifications