diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..b21a32b --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,38 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: '' +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. +Do not submit screenshots containing PII (logins, passwords, keys, IP addresses, etc) + +**Desktop (please complete the following information):** + - OS: [e.g. Ubuntu vX.X] + - Dojo Version [e.g. 22] + - Dojo Advanced Setups used (external bitcoind, bitcoind exposed to external apps, etc) + +**Smartphone (please complete the following information):** + - Device: [e.g. Samsung Galaxy X] + - OS: [e.g. AndroidX.X] + - Wallet Version [e.g. X.X.X] + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..bbcbbe7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: '' +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/ISSUE_TEMPLATE/support-request.md b/.github/ISSUE_TEMPLATE/support-request.md new file mode 100644 index 0000000..122aa31 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/support-request.md @@ -0,0 +1,23 @@ +--- +name: Support Request +about: Describe your issue +title: '' +labels: '' +assignees: '' + +--- + +**Describe the issue** +A clear and concise description of what the issue is. + +**Screenshots** +If applicable, add screenshots to help explain your problem. +Do not submit screenshots containing PII (logins, passwords, keys, IP addresses, etc) + +**Desktop (please complete the following information):** + - OS: [e.g. Ubuntu vX.X] + - Dojo Version [e.g. 22] + - Dojo Advanced Setups used (external bitcoind, bitcoind exposed to external apps, etc) + +**Additional context** +Add any other context about the problem here. diff --git a/.gitignore b/.gitignore index 340e537..d8b576a 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,5 @@ keys/index.js keys/sslcert/ node_modules/ private-tests/ +static/admin/conf/index.js *.log diff --git a/README.md b/README.md index d5fd07c..3e769d6 100644 --- a/README.md +++ b/README.md @@ -65,10 +65,10 @@ Authentication is enforced by an API key and Json Web Tokens. * Default option relies on the local bitcoind and makes you 100% independent of Samourai Wallet's infrastructure. This option is recommended for better privacy. * Activation of bitcoind as the data source: - * Edit /keys/index.js and set "explorers.bitcoind" to "active". OXT API will be ignored. + * Edit /keys/index.js and set "indexer.active" to "local_bitcoind". OXT API will be ignored. * Activation of OXT as the data source (through socks5): - * Edit /keys/index.js and set "explorers.bitcoind" to "inactive". + * Edit /keys/index.js and set "indexer.active" to "third_party_explorer". * Main drawbacks of using your local bitcoind for these imports: * This option is considered as experimental. diff --git a/RELEASES.md b/RELEASES.md index d4a1c6f..e4a4c30 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -2,10 +2,117 @@ ## Releases ## +- [v1.3.0](#1_3_0) - [v1.2.0](#1_2_0) - [v1.1.0](#1_1_0) + + +## Samourai Dojo v1.3.0 ## + + +### Notable changes ### + + +#### Update of configuration parameters #### + +Configuration parameter ```NODE_IMPORT_FROM_BITCOIND``` is replaced by ```NODE_ACTIVE_INDEXER```. + +The supported values for the new parameter are: +- ```local_bitcoind``` (equivalent to former ```NODE_IMPORT_FROM_BITCOIND=active```) +- ```third_party_explorer``` (equivalent to former ```NODE_IMPORT_FROM_BITCOIND=inactive```) + +**Upgrade of Dojo to v1.3.0 automatically sets the parameter to the default value** ```local_bitcoind```. + + +#### Installation of Tor from source code archives #### + +Previous versions of Dojo used the git repository operated by the Tor Project during the build of the Tor container. Starting with this version, Dojo will download an archive of the source code. + +Users living in countries blocking the access to resources provided by the Tor Project can easily switch to a mirror site by editing this [line](https://github.com/Samourai-Wallet/samourai-dojo/blob/develop/docker/my-dojo/tor/Dockerfile#L4) before installing or upgrading their Dojo. + +The default source used by Dojo is the archive provided by the [Tor Project](https://archive.torproject.org/tor-package-archive). + + +#### Add support of Tor bridges #### + +The Tor container now supports the configuration of Tor bridges. For some users, it may be appropriate to configure Tor bridges in order to circumvent a local censorship of the Tor network. See [this section](https://github.com/Samourai-Wallet/samourai-dojo/blob/develop/doc/DOCKER_advanced_setups.md#tor_bridges) of the documentation for the activation of Tor bridges on your Dojo. + + +#### Add Blocks rescan feature to the maintenance tool #### + +This version introduces a new "Blocks Rescan" feature accessible from the Maintenance Tool. + +"Blocks Rescan" allows to rescan a range of blocks for all the addresses currently tracked by your Dojo (loose addresses or addresses derived for your xpubs). This feature comes in handy when the block confirming a missing transaction is known by the user. + + +#### Add Esplora as the new external data source for testnet #### + +The testnet version of Dojo now relies on the Esplora API as its external data source for imports and rescans. + +Previously used API (BTC.COM and Insight) have been removed. + +Default URL used for the Esplora API is https://blockstream.info/testnet. A local Esplora instance can be used by editing this [line](https://github.com/Samourai-Wallet/samourai-dojo/blob/develop/docker/my-dojo/.env#L44). + + +#### Remove support of HTTPS by NodeJS #### + +Support of HTTPS by the NodeJS server has been removed. + + +#### Upgrade of bitcoind to v0.19.0.1 #### + +Upgrade to Bitcoin Core v0.19.0.1. + + +#### Update bitcoinjs to v5.1.4 #### + +The bitcoinjs library has been updated to v5.1.4. + + +### Change log ### + +#### MyDojo #### + +- [#71](https://github.com/Samourai-Wallet/samourai-dojo/pull/71) update to use latest bitcoinjs +- [#74](https://github.com/Samourai-Wallet/samourai-dojo/pull/74) adding bridge support to tor-container +- [#80](https://github.com/Samourai-Wallet/samourai-dojo/pull/80) add support of blocks rescans in the maintenance tool +- [#83](https://github.com/Samourai-Wallet/samourai-dojo/pull/83) removed unused support of https by nodejs apps +- [#84](https://github.com/Samourai-Wallet/samourai-dojo/pull/84) install tor from source code archive +- [#85](https://github.com/Samourai-Wallet/samourai-dojo/pull/85) add esplora as a data source for testnet imports and rescans +- [#90](https://github.com/Samourai-Wallet/samourai-dojo/pull/90) update the remote importer +- [#91](https://github.com/Samourai-Wallet/samourai-dojo/pull/91) improve the tracking of loose addresses +- [#93](https://github.com/Samourai-Wallet/samourai-dojo/pull/93) increase timeouts defined in docker-compose files (for raspi hardwares) +- [#93](https://github.com/Samourai-Wallet/samourai-dojo/pull/93) upgrade bitcoind to bitcoin core 0.19.0.1 + + +#### Bug fixes #### + +- [#73](https://github.com/Samourai-Wallet/samourai-dojo/pull/73) remove unhandled promise error +- [#79](https://github.com/Samourai-Wallet/samourai-dojo/pull/79) retry to send sql requests on detection of a lock +- [#94](https://github.com/Samourai-Wallet/samourai-dojo/pull/94) improve the transaction cache implemented for bitcoind rpc client + + +#### Documentation #### + +- [b5dd967](https://github.com/Samourai-Wallet/samourai-dojo/commit/b5dd9673c159b469fb19f43c33a0c0dd21b2fe5a) update api doc (see #75) +- [16926a8](https://github.com/Samourai-Wallet/samourai-dojo/commit/16926a86fb637fb06510d1418474f62d3570cfd3) update docker doc + + +#### Misc #### + +- [#76](https://github.com/Samourai-Wallet/samourai-dojo/pull/76) pin versions in package-lock.json + + +### Credits ### + +- junderw +- kenshin-samourai +- LaurentMT +- nickodev + + ## Samourai Dojo v1.2.0 ## diff --git a/accounts/index.js b/accounts/index.js index 573a292..e04e5c8 100644 --- a/accounts/index.js +++ b/accounts/index.js @@ -53,8 +53,7 @@ // Initialize the http server const port = keys.ports.account - const httpsOptions = keys.https.account - const httpServer = new HttpServer(port, httpsOptions) + const httpServer = new HttpServer(port) // Initialize the rest api endpoints const authRestApi = new AuthRestApi(httpServer) @@ -73,4 +72,7 @@ // Attach the web sockets server to the web server notifServer.attach(httpServer) -})() +})().catch(err => { + console.error(err) + process.exit(1) +}) diff --git a/doc/DOCKER_advanced_setups.md b/doc/DOCKER_advanced_setups.md index 3825b57..215bb69 100644 --- a/doc/DOCKER_advanced_setups.md +++ b/doc/DOCKER_advanced_setups.md @@ -6,6 +6,14 @@ The configuration files of Dojo provide a few advanced options allowing to tune A word of caution, though, the default values of these options try to maximize your privacy at a network level. Most of the advanced setups described in this document may damage your privacy. Use at your own risk! +## Table of Content ## +- [External Bitcoin full node](#external_bitcoind) +- [bitcoind RPC API ans ZMQ notifications exposed to external apps](#exposed_rpc_zmq) +- [Static onion address for bitcoind hidden service](#static_onion) +- [Configure Tor Bridges](#tor_bridges) +- [Support of testnet](#testnet) + + ## External Bitcoin full node ## @@ -161,6 +169,43 @@ nano ./conf/docker-bitcoind.conf Note: this option has no effect if your setup relies on a external full node (i.e. if BITCOIND_INSTALL is set to "off"). + + +## Configure Tor Bridges ## + +By default, Dojo doesn't try to hide that Tor is being used. For the majority of Dojo users, connecting to Tor with the default configuration is appropriate and will work successfully. For some users, it may be appropriate to configure Tor Bridges in order to circumvent censorship enforced by ISP, censorship enforcement bodies and other interested parties. + +The following steps allow to activate the use of Tor bridges by Dojo. + +``` +# Stop your Dojo +./dojo.sh stop + +# Head over to https://bridges.torproject.org +# Click on "Get bridges", then you will see a form with "Advanced Options" header +# Leave the Pluggable Transport as "obfs4" and click on "Get Bridges" button +# Solve the captcha, you will get the bridge addresses, usually three lines: +# obfs4 24.106.248.94:65531 B9EFBC5... cert=yrX... iat-mode=0 +# obfs4 ... +# obfs4 ... + +# Edit the tor config file +nano ./conf/docker-tor.conf + +# +# Set the value of TOR_USE_BRIDGES to "on" +# +# Set the values of TOR_BRIDGE_n properties with info returned by the website +# For instance, if the first line generated by the website is: +# obfs4 24.106.248.94:65531 B9EFBC5... cert=yrX... iat-mode=0 +# You will have to set: +# TOR_BRIDGE_1=obfs4 24.106.248.94:65531 B9EFBC5... cert=yrX... iat-mode=0 +# +# Save and exit nano +# +``` + + ## Support of testnet ## diff --git a/doc/DOCKER_setup.md b/doc/DOCKER_setup.md index 63fcd07..2e1313e 100644 --- a/doc/DOCKER_setup.md +++ b/doc/DOCKER_setup.md @@ -10,9 +10,9 @@ MyDojo is a set of Docker containers providing a full Samourai backend composed ## Table of Content ## - [Architecture](#architecture) - [Requirements](#requirements) +- [Configuration files](#config_files) - [First-time install procedure](#install) - [Upgrade procedure](#upgrade) -- [Configuration files](#config_files) - [Dojo shell script](#shell_script) - [Dojo maintenance tool](#maintenance_tool) - [Pairing your wallet to your Dojo](#pairing) @@ -69,6 +69,29 @@ MyDojo is a set of Docker containers providing a full Samourai backend composed * Tor Browser installed on the host machine (or on another machine if your host is a headless server) + + +## Configuration files ## + +Each new release of Dojo is packaged with 4 template files stored in the `/docker/my-dojo/conf` directory: +- docker-common.conf.tpl +- docker-bitcoin.conf.tpl +- docker-mysql.conf.tpl +- docker-node.conf.tpl + +These template files define default values for configuration options of your Dojo. + +During the first-time installation (dojo.sh install) these templates are used to initialize the configuration files (files with .conf extension) that will be used by your Dojo. + +During an upgrade (dojo.sh upgrade), the content of the template files is merged with the content of the configuration files, preserving the values that you may have modified in the configuration files. A backup of the configuration files is saved in the same directory (files with .save extension). + +Most options provided in the configuration files can be later modified. New values will become active after a call to + +``` +./dojo.sh restart +``` + + ## First-time Setup ## @@ -97,11 +120,6 @@ This procedure allows to install a new Dojo from scratch. * `BITCOIND_RPC_USER` = login protecting the access to the RPC API of your full node, * `BITCOIND_RPC_PASSWORD` = password protecting the access to the RPC API of your full node. * If your machine has a lot of RAM, it's recommended that you increase the value of `BITCOIND_DB_CACHE` for a faster Initial Block Download. - * This file also provides a few additional settings for advanced setups: - * static onion address for your full node, - * bitcoind RPC API exposed to external apps, - * use of an external full node. - See this [doc](./DOCKER_advanced_setups.md) for more details. * Edit docker-mysql.conf.tpl and provide a new value for the following parameters: * `MYSQL_ROOT_PASSWORD` = password protecting the root account of MySQL, @@ -114,6 +132,14 @@ This procedure allows to install a new Dojo from scratch. * `NODE_JWT_SECRET` = secret used by your Dojo for the initialization of a cryptographic key signing Json Web Tokens. These parameters will protect the access to your Dojo. Be sure to provide alphanumeric values with enough entropy. +* Dojo provides a few additional settings for advanced setups: + * static onion address for your full node, + * bitcoind RPC API exposed to external apps, + * use of an external full node, + * use of Tor Bridges, + * support of testnet. + See this [doc](./DOCKER_advanced_setups.md) for more details. + * Open the docker quickstart terminal or a terminal console and go to the `/docker/my-dojo` directory. This directory contains a script named dojo.sh which will be your entrypoint for all operations related to the management of your Dojo. @@ -173,29 +199,6 @@ Docker and Docker Compose are going to build new images and containers for your Note: The upgrade process will override all manual modifications of the files stored under the `` directory with an exception for the three configuration files stored in the `/docker/my-dojo/conf` directory. - - -## Configuration files ## - -Each new release of Dojo is packaged with 4 template files stored in the `/docker/my-dojo/conf` directory: -- docker-common.conf.tpl -- docker-bitcoin.conf.tpl -- docker-mysql.conf.tpl -- docker-node.conf.tpl - -These template files define default values for configuration options of your Dojo. - -During the first-time installation (dojo.sh install) these templates are used to initialize the configuration files (files with .conf extension) that will be used by your Dojo. - -During an upgrade (dojo.sh upgrade), the content of the template files is merged with the content of the configuration files, preserving the values that you may have modified in the configuration files. A backup of the configuration files is saved in the same directory (files with .save extension). - -Most options provided in the configuration files can be later modified. New values will become active after a call to - -``` -./dojo.sh restart -``` - - ## Dojo shell script ## diff --git a/doc/POST_auth_login.md b/doc/POST_auth_login.md index 1899168..873357a 100644 --- a/doc/POST_auth_login.md +++ b/doc/POST_auth_login.md @@ -2,7 +2,7 @@ Authenticate to the backend by providing the API key expected by the server. If authentication succeeds, the endpoint returns a json embedding an access token and a refresh token (JSON Web Tokens). The access token must be passed as an argument or in the `Authorization` HTTP header for all later calls to the backend (account & pushtx REST API + websockets). The refresh token must be passed as an argument or in the `Authorization` HTTP header for later calls to /auth/refresh allowing to generate a new access token. -Authentication is activated in /keys/inndex.js configuration file +Authentication is activated in /keys/index.js configuration file ``` auth: { @@ -44,6 +44,9 @@ auth: { POST /auth/login ``` +The API Key must be passed in the body of the request as an url encoded argument. + + ## Parameters * **apikey** - `string` - The API key securing access to the backend @@ -51,7 +54,9 @@ POST /auth/login ### Example ``` -POST /auth/login?apikey=myAPIKey +POST /auth/login + +apikey=myAPIKey ``` #### Success diff --git a/doc/POST_auth_refresh.md b/doc/POST_auth_refresh.md index 62ca803..c203a64 100644 --- a/doc/POST_auth_refresh.md +++ b/doc/POST_auth_refresh.md @@ -1,12 +1,15 @@ # Refresh the access token -Request a new access token from the backend. A valid refresh token must be passed as an argument or through the `Authorization` HTTP header (with the `Bearer` scheme). +Request a new access token from the backend. ``` POST /auth/refresh ``` +The Refresh Token must be passed in the body of the request as an url encoded argument or through the `Authorization` HTTP header (with the `Bearer` scheme). + + ## Parameters * **rt** - `string` - A valid refresh token @@ -14,7 +17,9 @@ POST /auth/refresh ### Example ``` -POST /auth/refresh?rt=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJTYW1vdXJhaSBXYWxsZXQgYmFja2VuZCIsInR5cGUiOiJyZWZyZXNoLXRva2VuIiwiaWF0IjoxNTQ0MTAzOTI5LCJleHAiOjE1NDQxMTExMjl9.6gykKq31WL4Jq7hfmoTwi1fpmBTtAeFb4KjfmSO6l00 +POST /auth/refresh + +rt=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJTYW1vdXJhaSBXYWxsZXQgYmFja2VuZCIsInR5cGUiOiJyZWZyZXNoLXRva2VuIiwiaWF0IjoxNTQ0MTAzOTI5LCJleHAiOjE1NDQxMTExMjl9.6gykKq31WL4Jq7hfmoTwi1fpmBTtAeFb4KjfmSO6l00 ``` #### Success diff --git a/doc/POST_pushtx.md b/doc/POST_pushtx.md index 0ead767..e52d50d 100644 --- a/doc/POST_pushtx.md +++ b/doc/POST_pushtx.md @@ -10,6 +10,7 @@ POST /pushtx/ * **tx** - `hex string` - The raw transaction hex * **at** - `string` (optional) - Access Token (json web token). Required if authentication is activated. Alternatively, the access token can be passed through the `Authorization` HTTP header (with the `Bearer` scheme). + ### Example ``` diff --git a/doc/POST_pushtx_schedule.md b/doc/POST_pushtx_schedule.md index 6da9d39..69d56b1 100644 --- a/doc/POST_pushtx_schedule.md +++ b/doc/POST_pushtx_schedule.md @@ -7,6 +7,9 @@ Schedule the delayed push of an ordered list of transactions (used for programma POST /pushtx/schedule ``` +Parameters must be passed in the body of the request as json encoded arguments. + + ## Parameters * **script** - `ScriptStep[]` - An array of ScriptStep objects defining the script. diff --git a/doc/POST_xpub.md b/doc/POST_xpub.md index 2d900a6..816a8ef 100644 --- a/doc/POST_xpub.md +++ b/doc/POST_xpub.md @@ -4,10 +4,14 @@ Notify the server of the new HD account for tracking. When new accounts are sent Response time for restored accounts might be long if there is much previous activity. + ``` POST /xpub ``` +Parameters must be passed in the body of the request as url encoded arguments. + + ## Parameters * **xpub** - `string` - The extended public key for the HD Account * **type** - `string` - Whether this is a newly-created account or one being restored. Recognized values are `'new'` and `'restore'`. @@ -15,12 +19,13 @@ POST /xpub * **force** - `boolean` (optional) - Force an override of derivation scheme even if xpub is locked. Used for `'restore'` operation. * **at** - `string` (optional) - Access Token (json web token). Required if authentication is activated. Alternatively, the access token can be passed through the `Authorization` HTTP header (with the `Bearer` scheme). + ### Example ``` -POST /xpub?xpub=xpub0123456789&type=restore -POST /xpub?xpub=xpub0123456789&type=new&segwit=bip49 -POST /xpub?xpub=xpub0123456789&type=restore&segwit=bip84 +POST /xpub + +xpub=xpub0123456789&type=restore&segwit=bip84 ``` #### Success diff --git a/doc/POST_xpub_lock.md b/doc/POST_xpub_lock.md index 8a61a05..74504b6 100644 --- a/doc/POST_xpub_lock.md +++ b/doc/POST_xpub_lock.md @@ -2,20 +2,27 @@ To avoid errors related to `POST xpub` and SegWit derivation type, this endpoint allows locking of the type of an xpub in the database. + ``` POST /xpub/:xpub/lock ``` +Parameters must be passed in the body of the request as url encoded arguments. + + ## Parameters * **address** - `string` - The first address of the internal chain for this `xpub`, derivation path `M/1/0`. Use compressed P2PHK address regardless of HD derivation scheme. * **message** - `string` - Either `"lock"` or `"unlock"` * **signature** - `string` - The base64-encoded signature of the double SHA256 hash of `[varuint length of message string, message string]`. Signature scheme follows [bitcoinjs-message](https://github.com/bitcoinjs/bitcoinjs-message/blob/master/index.js) with a message prefix matching the [coin type](https://github.com/bitcoinjs/bitcoinjs-lib/blob/v3.1.1/src/networks.js). Use the ECPair associated with the `M/1/0` address to sign. * **at** - `string` (optional) - Access Token (json web token). Required if authentication is activated. Alternatively, the access token can be passed through the `Authorization` HTTP header (with the `Bearer` scheme). + ### Example ``` -POST /xpub/xpub0123456789/lock?address=1address&message=lock&signature=Base64X== +POST /xpub/xpub0123456789/lock + +address=1address&message=lock&signature=Base64X== ``` #### Success diff --git a/docker/my-dojo/.env b/docker/my-dojo/.env index f24f787..57b9ff1 100644 --- a/docker/my-dojo/.env +++ b/docker/my-dojo/.env @@ -10,12 +10,12 @@ COMPOSE_CONVERT_WINDOWS_PATHS=1 -DOJO_VERSION_TAG=1.2.0 +DOJO_VERSION_TAG=1.3.0 DOJO_DB_VERSION_TAG=1.1.0 -DOJO_BITCOIND_VERSION_TAG=1.2.0 -DOJO_NODEJS_VERSION_TAG=1.2.0 -DOJO_NGINX_VERSION_TAG=1.2.0 -DOJO_TOR_VERSION_TAG=1.1.0 +DOJO_BITCOIND_VERSION_TAG=1.3.0 +DOJO_NODEJS_VERSION_TAG=1.3.0 +DOJO_NGINX_VERSION_TAG=1.3.0 +DOJO_TOR_VERSION_TAG=1.2.0 ######################################### @@ -41,7 +41,7 @@ NODE_GAP_EXTERNAL=100 NODE_GAP_INTERNAL=100 NODE_ADDR_FILTER_THRESHOLD=1000 NODE_URL_OXT_API=https://api.oxt.me -NODE_URL_BTCCOM_API=https://tchain.api.btc.com/v3 +NODE_URL_ESPLORA_API=https://blockstream.info/testnet NODE_ADDR_DERIVATION_MIN_CHILD=2 NODE_ADDR_DERIVATION_MAX_CHILD=2 NODE_ADDR_DERIVATION_THRESHOLD=10 diff --git a/docker/my-dojo/bitcoin/Dockerfile b/docker/my-dojo/bitcoin/Dockerfile index 9d638de..db88639 100644 --- a/docker/my-dojo/bitcoin/Dockerfile +++ b/docker/my-dojo/bitcoin/Dockerfile @@ -5,10 +5,10 @@ FROM debian:stretch # INSTALL BITCOIN ################################################################# ENV BITCOIN_HOME /home/bitcoin -ENV BITCOIN_VERSION 0.18.1 -ENV BITCOIN_URL https://bitcoincore.org/bin/bitcoin-core-0.18.1/bitcoin-0.18.1-x86_64-linux-gnu.tar.gz -ENV BITCOIN_SHA256 600d1db5e751fa85903e935a01a74f5cc57e1e7473c15fd3e17ed21e202cfe5a -ENV BITCOIN_ASC_URL https://bitcoincore.org/bin/bitcoin-core-0.18.1/SHA256SUMS.asc +ENV BITCOIN_VERSION 0.19.0.1 +ENV BITCOIN_URL https://bitcoincore.org/bin/bitcoin-core-0.19.0.1/bitcoin-0.19.0.1-x86_64-linux-gnu.tar.gz +ENV BITCOIN_SHA256 732cc96ae2e5e25603edf76b8c8af976fe518dd925f7e674710c6c8ee5189204 +ENV BITCOIN_ASC_URL https://bitcoincore.org/bin/bitcoin-core-0.19.0.1/SHA256SUMS.asc ENV BITCOIN_PGP_KEY 01EA5486DE18A882D4C2684590C8019E36C2E964 RUN set -ex && \ diff --git a/docker/my-dojo/conf/docker-node.conf.tpl b/docker/my-dojo/conf/docker-node.conf.tpl index 55dbfe4..403ed36 100644 --- a/docker/my-dojo/conf/docker-node.conf.tpl +++ b/docker/my-dojo/conf/docker-node.conf.tpl @@ -20,10 +20,9 @@ NODE_ADMIN_KEY=myAdminKey # Type: alphanumeric NODE_JWT_SECRET=myJwtSecret -# Data source used for imports and rescans (bitcoind or OXT) -# Note: support of local bitcoind is an experimental feature -# Values: active | inactive -NODE_IMPORT_FROM_BITCOIND=active +# Indexer or third-party service used for imports and rescans of addresses +# Values: local_bitcoind | third_party_explorer +NODE_ACTIVE_INDEXER=local_bitcoind # FEE TYPE USED FOR FEES ESTIMATIONS BY BITCOIND # Allowed values are ECONOMICAL or CONSERVATIVE diff --git a/docker/my-dojo/conf/docker-tor.conf.tpl b/docker/my-dojo/conf/docker-tor.conf.tpl new file mode 100644 index 0000000..857ef20 --- /dev/null +++ b/docker/my-dojo/conf/docker-tor.conf.tpl @@ -0,0 +1,34 @@ +######################################### +# CONFIGURATION OF TOR CONTAINER +######################################### + +# +# USE TOR BRIDGES +# +# To get Tor bridges head over to https://bridges.torproject.org and click on +# Get bridges, then you will see a form with "Advanced Options" header +# leave the Pluggable Transport as obfs4 and click on Get Bridges button +# solve the captcah, you will get the bridge addresses (usually 3) +# +# Then, set TOR_USE_BRIDGES to "on" and initialize the TOR_BRIDGE_n options +# with the 3 lines generated by the online tool. +# +# For instance, if the first line generated by the tool is: +# obfs4 24.106.248.94:65531 B9EFBC5... cert=yrX... iat-mode=0 +# You will have to set: +# TOR_BRIDGE_1=obfs4 24.106.248.94:65531 B9EFBC5... cert=yrX... iat-mode=0 +# + + +# Activate the use of Tor bridges +# Value: on | off +TOR_USE_BRIDGES=off + +# Bridge 1 +TOR_BRIDGE_1=ToBeDefined + +# Bridge 2 +TOR_BRIDGE_2=ToBeDefined + +# Bridge 3 +TOR_BRIDGE_3=ToBeDefined diff --git a/docker/my-dojo/docker-compose.yaml b/docker/my-dojo/docker-compose.yaml index e59c762..eb2ef0e 100644 --- a/docker/my-dojo/docker-compose.yaml +++ b/docker/my-dojo/docker-compose.yaml @@ -32,10 +32,11 @@ services: - ./conf/docker-bitcoind.conf - ./conf/docker-node.conf restart: on-failure - command: "/home/node/app/wait-for-it.sh db:3306 --timeout=360 --strict -- /home/node/app/restart.sh" + command: "/home/node/app/wait-for-it.sh db:3306 --timeout=720 --strict -- /home/node/app/restart.sh" expose: - "8080" - "8081" + - "8082" volumes: - data-nodejs:/data depends_on: @@ -53,7 +54,7 @@ services: - ./.env - ./conf/docker-common.conf restart: on-failure - command: "/wait-for node:8080 --timeout=360 -- nginx" + command: "/wait-for node:8080 --timeout=720 -- nginx" expose: - "80" volumes: @@ -73,6 +74,7 @@ services: context: ./tor env_file: - ./.env + - ./conf/docker-tor.conf restart: on-failure command: /restart.sh volumes: diff --git a/docker/my-dojo/install/install-scripts.sh b/docker/my-dojo/install/install-scripts.sh index 3fb6217..88b6bcd 100755 --- a/docker/my-dojo/install/install-scripts.sh +++ b/docker/my-dojo/install/install-scripts.sh @@ -49,6 +49,9 @@ init_config_files() { cp ./conf/docker-node.conf.tpl ./conf/docker-node.conf echo "Initialized docker-node.conf" + cp ./conf/docker-tor.conf.tpl ./conf/docker-tor.conf + echo "Initialized docker-tor.conf" + # Initialize config files for nginx and the maintenance tool if [ "$COMMON_BTC_NETWORK" == "testnet" ]; then cp ./nginx/testnet.conf ./nginx/dojo.conf diff --git a/docker/my-dojo/install/upgrade-scripts.sh b/docker/my-dojo/install/upgrade-scripts.sh index 34e3f2e..40e735a 100755 --- a/docker/my-dojo/install/upgrade-scripts.sh +++ b/docker/my-dojo/install/upgrade-scripts.sh @@ -33,11 +33,7 @@ update_config_files() { echo "Initialized 2_update.sql" # Initialize config files for MyDojo - if [ -f ./conf/docker-common.conf ]; then - update_config_file ./conf/docker-common.conf ./conf/docker-common.conf.tpl - else - cp ./conf/docker-common.conf.tpl ./conf/docker-common.conf - fi + update_config_file ./conf/docker-common.conf ./conf/docker-common.conf.tpl echo "Initialized docker-common.conf" update_config_file ./conf/docker-bitcoind.conf ./conf/docker-bitcoind.conf.tpl @@ -49,6 +45,9 @@ update_config_files() { update_config_file ./conf/docker-node.conf ./conf/docker-node.conf.tpl echo "Initialized docker-node.conf" + update_config_file ./conf/docker-tor.conf ./conf/docker-tor.conf.tpl + echo "Initialized docker-tor.conf" + # Initialize config files for nginx and the maintenance tool if [ "$COMMON_BTC_NETWORK" == "testnet" ]; then cp ./nginx/testnet.conf ./nginx/dojo.conf @@ -65,18 +64,22 @@ update_config_files() { # Update a configuration file from template update_config_file() { - sed "s/^#.*//g;s/=.*//g;/^$/d" $1 > ./original.keys.raw - grep -f ./original.keys.raw $1 > ./original.lines.raw + if [ -f $1 ]; then + sed "s/^#.*//g;s/=.*//g;/^$/d" $1 > ./original.keys.raw + grep -f ./original.keys.raw $1 > ./original.lines.raw - cp -p $1 "$1.save" - cp -p $2 $1 + cp -p $1 "$1.save" + cp -p $2 $1 - while IFS='=' read -r key val ; do - sed -i "s/$key=.*/$key=$val/g" "$1" - done < ./original.lines.raw + while IFS='=' read -r key val ; do + sed -i "s~$key=.*~$key=$val~g" "$1" + done < ./original.lines.raw - rm ./original.keys.raw - rm ./original.lines.raw + rm ./original.keys.raw + rm ./original.lines.raw + else + cp $2 $1 + fi } # Update dojo database @@ -86,6 +89,15 @@ update_dojo_db() { # Clean-up cleanup() { + ################# + # Clean-up v1.3.0 + ################# + + # Remove deprecated torrc file + if [ -f ./tor/torrc ]; then + rm ./tor/torrc + fi + ################# # Clean-up v1.1.0 ################# @@ -94,4 +106,5 @@ cleanup() { if [ -f ./bitcoin/bitcoin.conf ]; then rm ./bitcoin/bitcoin.conf fi + } diff --git a/docker/my-dojo/nginx/mainnet.conf b/docker/my-dojo/nginx/mainnet.conf index c7517d3..d729066 100644 --- a/docker/my-dojo/nginx/mainnet.conf +++ b/docker/my-dojo/nginx/mainnet.conf @@ -34,6 +34,11 @@ server { proxy_pass http://node:8081/; } + # Tracker server is separate, so proxy first + location /v2/tracker/ { + proxy_pass http://node:8082/; + } + # Proxy requests to maintenance tool location /admin/ { proxy_pass http://node:8080/static/admin/; diff --git a/docker/my-dojo/nginx/testnet.conf b/docker/my-dojo/nginx/testnet.conf index 4b9b71f..8c75fc6 100644 --- a/docker/my-dojo/nginx/testnet.conf +++ b/docker/my-dojo/nginx/testnet.conf @@ -34,6 +34,11 @@ server { proxy_pass http://node:8081/; } + # Tracker server is separate, so proxy first + location /test/v2/tracker/ { + proxy_pass http://node:8082/; + } + # Proxy requests to maintenance tool location /admin/ { proxy_pass http://node:8080/static/admin/; diff --git a/docker/my-dojo/node/keys.index.js b/docker/my-dojo/node/keys.index.js index 936a5bc..300a1a5 100644 --- a/docker/my-dojo/node/keys.index.js +++ b/docker/my-dojo/node/keys.index.js @@ -70,8 +70,10 @@ module.exports = { ports: { // Port used by the API account: 8080, - // Port used by pushtx + // Port used by the pushtx API pushtx: 8081, + // Port used by the tracker API + trackerApi: 8082, // Port used by the tracker for its notifications tracker: 5555, // Port used by pushtx for its notifications @@ -79,45 +81,6 @@ module.exports = { // Port used by the pushtx orchestrator for its notifications orchestrator: 5557 }, - /* - * HTTPS - * Activate only if node js is used as frontend web server - * (no nginx proxy server) - */ - https: { - // HTTPS for the API - account: { - // Activate https - active: false, - // Filepath of server private key - // (shoud be stored in keys/sslcert) - keypath: '', - // Passphrase of the private key - passphrase: '', - // Filepath of server certificate - // (shoud be stored in keys/sslcert) - certpath: '', - // Filepath of CA certificate - // (shoud be stored in keys/sslcert) - capath: '' - }, - // HTTPS for pushtx - pushtx: { - // Activate https - active: false, - // Filepath of server private key - // (shoud be stored in keys/sslcert) - keypath: '', - // Passphrase of the private key - passphrase: '', - // Filepath of server certificate - // (shoud be stored in keys/sslcert) - certpath: '', - // Filepath of CA certificate - // (shoud be stored in keys/sslcert) - capath: '' - } - }, /* * Authenticated access to the APIs (account & pushtx) */ @@ -188,21 +151,20 @@ module.exports = { transactions: 50 }, /* - * Third party explorers + * Indexer or third party service * used for fast scan of addresses */ - explorers: { - // Use local bitcoind for imports and rescans - // or use OXT as a fallback - // Values: active | inactive - bitcoind: process.env.NODE_IMPORT_FROM_BITCOIND, + indexer: { + // Active indexer + // Values: local_bitcoind | third_party_explorer + active: process.env.NODE_ACTIVE_INDEXER, // Use a SOCKS5 proxy for all communications with external services // Values: null if no socks5 proxy used, otherwise the url of the socks5 proxy socks5Proxy: 'socks5h://172.28.1.4:9050', // OXT (mainnet) oxt: process.env.NODE_URL_OXT_API, - // BTC.COM (testnet) - btccom: process.env.NODE_URL_BTCCOM_API + // Esplora (testnet) + esplora: process.env.NODE_URL_ESPLORA_API, }, /* * Max number of transactions per address diff --git a/docker/my-dojo/overrides/bitcoind.install.yaml b/docker/my-dojo/overrides/bitcoind.install.yaml index ff63912..085b7be 100644 --- a/docker/my-dojo/overrides/bitcoind.install.yaml +++ b/docker/my-dojo/overrides/bitcoind.install.yaml @@ -11,7 +11,7 @@ services: - ./conf/docker-common.conf - ./conf/docker-bitcoind.conf restart: on-failure - command: "/wait-for-it.sh tor:9050 --timeout=360 --strict -- /restart.sh" + command: "/wait-for-it.sh tor:9050 --timeout=720 --strict -- /restart.sh" expose: - "8333" - "28256" diff --git a/docker/my-dojo/tor/Dockerfile b/docker/my-dojo/tor/Dockerfile index 59578f2..6e96b2a 100644 --- a/docker/my-dojo/tor/Dockerfile +++ b/docker/my-dojo/tor/Dockerfile @@ -1,23 +1,61 @@ FROM debian:stretch -ENV TOR_HOME /var/lib/tor +ENV TOR_HOME /var/lib/tor +ENV TOR_URL https://archive.torproject.org/tor-package-archive +ENV TOR_VERSION 0.3.5.8 +ENV TOR_GPG_KEY1 0xEB5A896A28988BF5 +ENV TOR_GPG_KEY2 0xC218525819F78451 +ENV TOR_GPG_KEY3 0x21194EBB165733EA +ENV TOR_GPG_KEY4 0x6AFEE6D49E92B601 + +ENV GOLANG_DL_URL https://dl.google.com/go +ENV GOLANG_ARCHIVE go1.11.13.linux-amd64.tar.gz +ENV GOLANG_SHA256 50fe8e13592f8cf22304b9c4adfc11849a2c3d281b1d7e09c924ae24874c6daa + +ENV OBFS4_URL https://github.com/Yawning/obfs4.git +ENV OBFS4_VERSION 0.0.11 + # Install Tor RUN set -ex && \ apt-get update && \ - apt-get install -y git libevent-dev zlib1g-dev libssl-dev gcc make automake ca-certificates autoconf musl-dev coreutils && \ + apt-get install -y git libevent-dev zlib1g-dev libssl-dev gcc make automake ca-certificates autoconf musl-dev coreutils gpg wget && \ mkdir -p /usr/local/src/ && \ - git clone https://git.torproject.org/tor.git /usr/local/src/tor && \ - cd /usr/local/src/tor && \ - git checkout tor-0.3.5.8 && \ - ./autogen.sh && \ + cd /usr/local/src && \ + wget -qO "tor-$TOR_VERSION.tar.gz" "$TOR_URL/tor-$TOR_VERSION.tar.gz" && \ + wget -qO "tor-$TOR_VERSION.tar.gz.asc" "$TOR_URL/tor-$TOR_VERSION.tar.gz.asc" && \ + gpg --keyserver ipv4.pool.sks-keyservers.net --recv-keys "$TOR_GPG_KEY1" && \ + gpg --keyserver ipv4.pool.sks-keyservers.net --recv-keys "$TOR_GPG_KEY2" && \ + gpg --keyserver ipv4.pool.sks-keyservers.net --recv-keys "$TOR_GPG_KEY3" && \ + gpg --keyserver ipv4.pool.sks-keyservers.net --recv-keys "$TOR_GPG_KEY4" && \ + gpg --verify "tor-$TOR_VERSION.tar.gz.asc" && \ + tar -xzvf "tor-$TOR_VERSION.tar.gz" -C /usr/local/src && \ + cd "/usr/local/src/tor-$TOR_VERSION" && \ ./configure \ --disable-asciidoc \ --sysconfdir=/etc \ --disable-unittests && \ make && make install && \ cd .. && \ - rm -rf tor + rm -rf "tor-$TOR_VERSION" && \ + rm "tor-$TOR_VERSION.tar.gz" && \ + rm "tor-$TOR_VERSION.tar.gz.asc" + +# Install Golang & OBFS4 proxy +RUN cd /usr/local/src && \ + echo "$GOLANG_SHA256 *$GOLANG_ARCHIVE" > GO_CHECKSUMS && \ + wget "$GOLANG_DL_URL/$GOLANG_ARCHIVE" && \ + sha256sum -c GO_CHECKSUMS 2>&1 | grep OK && \ + tar -C /usr/local/lib -xzf "$GOLANG_ARCHIVE" && \ + ln -s /usr/local/lib/go/bin/go /usr/local/bin/ && \ + git clone "$OBFS4_URL" /usr/local/src/obfs4proxy && \ + cd obfs4proxy && \ + git checkout "tags/obfs4proxy-$OBFS4_VERSION" && \ + go build -o obfs4proxy/obfs4proxy ./obfs4proxy && \ + cp ./obfs4proxy/obfs4proxy /usr/local/bin && \ + cd .. && \ + rm "$GOLANG_ARCHIVE" && \ + rm -rf obfs4proxy # Create group & user tor RUN addgroup --system -gid 1107 tor && \ @@ -32,10 +70,6 @@ RUN mkdir -p "$TOR_HOME/.tor" && \ chown -Rv tor:tor "$TOR_HOME" && \ chmod -R 750 "$TOR_HOME" -# Copy Tor configuration file -COPY ./torrc /etc/tor/torrc -RUN chown tor:tor /etc/tor/torrc - # Copy restart script COPY ./restart.sh /restart.sh diff --git a/docker/my-dojo/tor/restart.sh b/docker/my-dojo/tor/restart.sh index 24921ba..c3b29bb 100644 --- a/docker/my-dojo/tor/restart.sh +++ b/docker/my-dojo/tor/restart.sh @@ -6,4 +6,31 @@ echo "## Set permissions on /var/lib/tor dir ###" chmod 750 /var/lib/tor echo "## Start tor #############################" -tor \ No newline at end of file + +tor_options=( + --SocksPort 172.28.1.4:9050 + --SocksPolicy "accept 172.28.0.0/16" + --SocksPolicy "reject *" + --DataDirectory /var/lib/tor/.tor + --DataDirectoryGroupReadable 1 + --HiddenServiceDir /var/lib/tor/hsv2dojo + --HiddenServiceVersion 2 + --HiddenServicePort "80 172.29.1.3:80" + --HiddenServiceDir /var/lib/tor/hsv3dojo + --HiddenServiceVersion 3 + --HiddenServicePort "80 172.29.1.3:80" + --HiddenServiceDir /var/lib/tor/hsv2bitcoind + --HiddenServiceVersion 2 + --HiddenServicePort "8333 172.28.1.5:8333" + --HiddenServiceDirGroupReadable 1 +) + +if [ "$TOR_USE_BRIDGES" == "on" ]; then + tor_options+=(--ClientTransportPlugin "obfs4 exec /usr/local/bin/obfs4proxy") + tor_options+=(--UseBridges 1) + tor_options+=(--Bridge "$TOR_BRIDGE_1") + tor_options+=(--Bridge "$TOR_BRIDGE_2") + tor_options+=(--Bridge "$TOR_BRIDGE_3") +fi + +tor "${tor_options[@]}" diff --git a/docker/my-dojo/tor/torrc b/docker/my-dojo/tor/torrc deleted file mode 100644 index 30bc02e..0000000 --- a/docker/my-dojo/tor/torrc +++ /dev/null @@ -1,44 +0,0 @@ -## Tor opens a socks proxy on port 9050 by default -- even if you don't -## configure one below. Set "SocksPort 0" if you plan to run Tor only -## as a relay, and not make any local application connections yourself. - -# Socks is only available from dojonet -SocksPort 172.28.1.4:9050 - -## Entry policies to allow/deny SOCKS requests based on IP address. -## First entry that matches wins. If no SocksPolicy is set, we accept -## all (and only) requests that reach a SocksPort. Untrusted users who -## can access your SocksPort may be able to learn about the connections -## you make. - -# Socks is only available from dojonet -SocksPolicy accept 172.28.0.0/16 -SocksPolicy reject * - -## The directory for keeping all the keys/etc. By default, we store -## things in $HOME/.tor on Unix, and in Application Data\tor on Windows. - -DataDirectory /var/lib/tor/.tor -DataDirectoryGroupReadable 1 - - -############### This section is just for location-hidden services ### - -## Once you have configured a hidden service, you can look at the -## contents of the file ".../hidden_service/hostname" for the address -## to tell people. -## HiddenServicePort x y:z says to redirect requests on port x to the -## address y:z. - -HiddenServiceDir /var/lib/tor/hsv2dojo -HiddenServiceVersion 2 -HiddenServicePort 80 172.29.1.3:80 - -HiddenServiceDir /var/lib/tor/hsv3dojo -HiddenServiceVersion 3 -HiddenServicePort 80 172.29.1.3:80 - -HiddenServiceDir /var/lib/tor/hsv2bitcoind -HiddenServiceVersion 2 -HiddenServicePort 8333 172.28.1.5:8333 -HiddenServiceDirGroupReadable 1 diff --git a/keys/index-example.js b/keys/index-example.js index 3628c21..74041bd 100644 --- a/keys/index-example.js +++ b/keys/index-example.js @@ -15,7 +15,7 @@ module.exports = { /* * Dojo version */ - dojoVersion: '1.2.0', + dojoVersion: '1.3.0', /* * Bitcoind */ @@ -67,8 +67,10 @@ module.exports = { ports: { // Port used by the API account: 8080, - // Port used by pushtx + // Port used by pushtx API pushtx: 8081, + // Port used by the tracker API + trackerApi: 8082, // Port used by the tracker for its notifications tracker: 5555, // Port used by pushtx for its notifications @@ -76,45 +78,6 @@ module.exports = { // Port used by the pushtx orchestrator for its notifications orchestrator: 5557 }, - /* - * HTTPS - * Activate only if node js is used as frontend web server - * (no nginx proxy server) - */ - https: { - // HTTPS for the API - account: { - // Activate https - active: false, - // Filepath of server private key - // (shoud be stored in keys/sslcert) - keypath: '', - // Passphrase of the private key - passphrase: '', - // Filepath of server certificate - // (shoud be stored in keys/sslcert) - certpath: '', - // Filepath of CA certificate - // (shoud be stored in keys/sslcert) - capath: '' - }, - // HTTPS for pushtx - pushtx: { - // Activate https - active: false, - // Filepath of server private key - // (shoud be stored in keys/sslcert) - keypath: '', - // Passphrase of the private key - passphrase: '', - // Filepath of server certificate - // (shoud be stored in keys/sslcert) - certpath: '', - // Filepath of CA certificate - // (shoud be stored in keys/sslcert) - capath: '' - } - }, /* * Authenticated access to the APIs (account & pushtx) */ @@ -184,14 +147,23 @@ module.exports = { transactions: 50 }, /* - * Third party explorers + * Indexer or third party service * used for fast scan of addresses */ - explorers: { - // Use local bitcoind for imports and rescans - // or use OXT as a fallback - // Values: active | inactive - bitcoind: 'active', + indexer: { + // Active indexer + // Values: local_bitcoind | local_indexer | third_party_explorer + active: 'local_bitcoind', + // Local indexer + localIndexer: { + // IP address or hostname + host: '127.0.0.1', + // Port + port: 50001, + // Support of batch requests + // Values: active | inactive + batchRequests: 'inactive' + }, // Use a SOCKS5 proxy for all communications with external services // Values: null if no socks5 proxy used, otherwise the url of the socks5 proxy socks5Proxy: null, @@ -270,26 +242,11 @@ module.exports = { ports: { account: 18080, pushtx: 18081, + trackerApi: 18082, tracker: 15555, notifpushtx: 15556, orchestrator: 15557 }, - https: { - account: { - active: false, - keypath: '', - passphrase: '', - certpath: '', - capath: '' - }, - pushtx: { - active: false, - keypath: '', - passphrase: '', - certpath: '', - capath: '' - } - }, auth: { activeStrategy: null, mandatory: false, @@ -322,13 +279,15 @@ module.exports = { multiaddr: { transactions: 50 }, - explorers: { - bitcoind: 'inactive', + indexer: { + active: 'third_party_explorer', + localIndexer: { + host: '127.0.0.1', + port: 50001, + batchRequests: 'inactive' + }, socks5Proxy: null, - insight: [ - 'https://testnet-api.example.com' - ], - btccom: 'https://tchain.api.btc.com/v3' + esplora: 'https://blockstream.info/testnet' }, addrFilterThreshold: 1000, addrDerivationPool: { diff --git a/lib/bitcoin/addresses-helper.js b/lib/bitcoin/addresses-helper.js index 76eb572..9476b67 100644 --- a/lib/bitcoin/addresses-helper.js +++ b/lib/bitcoin/addresses-helper.js @@ -7,7 +7,7 @@ const bitcoin = require('bitcoinjs-lib') const btcMessage = require('bitcoinjs-message') const activeNet = require('./network').network - +const { p2pkh, p2sh, p2wpkh } = bitcoin.payments /** * A singleton providing Addresses helper functions @@ -20,8 +20,10 @@ class AddressesHelper { * @returns {string} return the derived address */ p2pkhAddress(pubKeyBuffer) { - const pubKeyHash = bitcoin.crypto.hash160(pubKeyBuffer) - return bitcoin.address.toBase58Check(pubKeyHash, activeNet.pubKeyHash) + return p2pkh({ + pubkey: pubKeyBuffer, + network: activeNet, + }).address } /** @@ -30,11 +32,13 @@ class AddressesHelper { * @returns {string} return the derived address */ p2wpkhP2shAddress(pubKeyBuffer) { - const pubKeyHash = bitcoin.crypto.hash160(pubKeyBuffer) - const witnessProgram = bitcoin.script.witnessPubKeyHash.output.encode(pubKeyHash) - const scriptPubKey = bitcoin.crypto.hash160(witnessProgram) - const outputScript = bitcoin.script.scriptHash.output.encode(scriptPubKey) - return bitcoin.address.fromOutputScript(outputScript, activeNet) + return p2sh({ + redeem: p2wpkh({ + pubkey: pubKeyBuffer, + network: activeNet, + }), + network: activeNet, + }).address } /** @@ -43,9 +47,10 @@ class AddressesHelper { * @returns {string} return the derived address */ p2wpkhAddress(pubKeyBuffer) { - const pubKeyHash = bitcoin.crypto.hash160(pubKeyBuffer) - const outputScript = bitcoin.script.witnessPubKeyHash.output.encode(pubKeyHash) - return bitcoin.address.fromOutputScript(outputScript, activeNet).toLowerCase() + return p2wpkh({ + pubkey: pubKeyBuffer, + network: activeNet, + }).address.toLowerCase() } /** diff --git a/lib/bitcoin/hd-accounts-helper.js b/lib/bitcoin/hd-accounts-helper.js index b42c19e..8ecc6fe 100644 --- a/lib/bitcoin/hd-accounts-helper.js +++ b/lib/bitcoin/hd-accounts-helper.js @@ -105,7 +105,7 @@ class HDAccountsHelper { } /** - * Translates + * Translates * - a xpub/ypub/zpub into a xpub * - a tpub/upub/vpub into a tpub * @param {string} xpub - extended public key to be translated @@ -120,7 +120,7 @@ class HDAccountsHelper { && ver != this.MAGIC_TPUB && ver != this.MAGIC_YPUB && ver != this.MAGIC_UPUB - && ver != this.MAGIC_ZPUB + && ver != this.MAGIC_ZPUB && ver != this.MAGIC_VPUB ) { //Logger.error(null, 'HdAccountsHelper.xlatXPUB() : Incorrect format') @@ -177,12 +177,12 @@ class HDAccountsHelper { } let p = v - + if (p >= this.LOCKED) { ret.locked = true p -= this.LOCKED } - + switch (p) { case this.BIP44: case this.BIP49: @@ -190,7 +190,7 @@ class HDAccountsHelper { ret.type = p break } - + return ret } @@ -201,16 +201,16 @@ class HDAccountsHelper { * @returns {integer} */ makeType(type, locked) { - let p = + let p = (type >= this.LOCKED) ? type - this.LOCKED : type - + locked = !!locked - + if (locked) p += this.LOCKED - + return p } @@ -245,7 +245,7 @@ class HDAccountsHelper { } /** - * Checks if a hd account is a valid hdnode + * Checks if a hd account is a valid bip32 * @param {string} xpub - hd account * @returns {boolean} returns true if hd account is valid, false otherwise */ @@ -258,7 +258,7 @@ class HDAccountsHelper { const xlatedXpub = this.xlatXPUB(xpub) // Parse input as an HD Node. Throws if invalid - const node = bitcoin.HDNode.fromBase58(xlatedXpub, activeNet) + const node = bitcoin.bip32.fromBase58(xlatedXpub, activeNet) // Check and see if this is a private key if (!node.isNeutered()) @@ -278,7 +278,7 @@ class HDAccountsHelper { /** * Get the hd node associated to an hd account * @param {string} xpub - hd account - * @returns {HDNode} + * @returns {bip32} */ getNode(xpub) { if (this.isValid(xpub)) @@ -291,7 +291,7 @@ class HDAccountsHelper { * Derives an address for an hd account * @param {int} chain - chain to be derived * must have a value on [0,1] for BIP44/BIP49/BIP84 derivation - * @param {HDNode} chainNode - Parent HDNode used for derivation + * @param {bip32} chainNode - Parent bip32 used for derivation * @param {int} index - index to be derived * @param {int} type - type of derivation * @returns {Promise - object} returns an object {address: '...', chain: , index: } @@ -307,13 +307,13 @@ class HDAccountsHelper { switch (type) { case this.BIP44: - addr.address = indexNode.getAddress() + addr.address = addrHelper.p2pkhAddress(indexNode.publicKey) break case this.BIP49: - addr.address = addrHelper.p2wpkhP2shAddress(indexNode.getPublicKeyBuffer()) + addr.address = addrHelper.p2wpkhP2shAddress(indexNode.publicKey) break case this.BIP84: - addr.address = addrHelper.p2wpkhAddress(indexNode.getPublicKeyBuffer()) + addr.address = addrHelper.p2wpkhAddress(indexNode.publicKey) break } @@ -382,7 +382,7 @@ class HDAccountsHelper { Logger.error(null, 'A problem was met during parallel addresses derivation') reject() } - + } catch(e) { Logger.error(e, 'A problem was met during parallel addresses derivation') reject(e) diff --git a/lib/bitcoin/parallel-address-derivation.js b/lib/bitcoin/parallel-address-derivation.js index b032762..cae5240 100644 --- a/lib/bitcoin/parallel-address-derivation.js +++ b/lib/bitcoin/parallel-address-derivation.js @@ -21,7 +21,7 @@ const BIP84 = 2 * Derives an address for an hd account * @param {int} chain - chain to be derived * must have a value on [0,1] for BIP44/BIP49/BIP84 derivation - * @param {HDNode} chainNode - Parent HDNode used for derivation + * @param {bip32} chainNode - Parent bip32 used for derivation * @param {int} index - index to be derived * @param {int} type - type of derivation * @returns {Promise - object} returns an object {address: '...', chain: , index: } @@ -37,13 +37,13 @@ const deriveAddress = async function(chain, chainNode, index, type) { switch (type) { case BIP44: - addr.address = indexNode.getAddress() + addr.address = addrHelper.p2pkhAddress(indexNode.publicKey) break case BIP49: - addr.address = addrHelper.p2wpkhP2shAddress(indexNode.getPublicKeyBuffer()) + addr.address = addrHelper.p2wpkhP2shAddress(indexNode.publicKey) break case BIP84: - addr.address = addrHelper.p2wpkhAddress(indexNode.getPublicKeyBuffer()) + addr.address = addrHelper.p2wpkhAddress(indexNode.publicKey) break } @@ -61,7 +61,7 @@ process.on('message', async (msg) => { const type = msg.type // Parse input as an HD Node. Throws if invalid - const node = bitcoin.HDNode.fromBase58(xpub, activeNet) + const node = bitcoin.bip32.fromBase58(xpub, activeNet) // Check and see if this is a private key if (!node.isNeutered()) @@ -88,5 +88,5 @@ process.on('message', async (msg) => { error: e }) } - + }) diff --git a/lib/bitcoind-rpc/transactions.js b/lib/bitcoind-rpc/transactions.js index 275dab8..0990a64 100644 --- a/lib/bitcoind-rpc/transactions.js +++ b/lib/bitcoind-rpc/transactions.js @@ -53,9 +53,11 @@ class Transactions { * @returns {Promise} */ async getTransaction(txid, fees) { + const keyCache = `${txid}-${fees ? '1' : '0'}` + // Return transaction from cache when possible - if (this.txCache.has(txid)) - return this.txCache.get(txid) + if (this.txCache.has(keyCache)) + return this.txCache.get(keyCache) try { const tx = await this.rpcClient.getrawtransaction(txid, true) @@ -107,7 +109,7 @@ class Transactions { // Store in cache if (ret.block && ret.block.hash) - this.txCache.set(txid, ret) + this.txCache.set(keyCache, ret) return ret diff --git a/lib/db/mysql-db-wrapper.js b/lib/db/mysql-db-wrapper.js index dcab303..56285da 100644 --- a/lib/db/mysql-db-wrapper.js +++ b/lib/db/mysql-db-wrapper.js @@ -378,15 +378,33 @@ class MySqlDbWrapper { /** * Send a query */ - async _query(query) { + async _query(query, retries) { queryDebug && Logger.info(query) + if (retries == null) + retries = 5 + return new Promise((resolve, reject) => { try { - this.pool.query(query, null, (err, result, fields) => { + this.pool.query(query, null, async (err, result, fields) => { if (err) { - this.queryError(err, query) - reject(err) + // Retry the request on lock errors + if ((err.code == 'ER_LOCK_DEADLOCK' || + err.code == 'ER_LOCK_TIMEOUT' || + err.code == 'ER_LOCK_WAIT_TIMEOUT') && (retries > 0) + ) { + try { + this.queryError('Lock detected. Retry request in a few ms', query) + const sleepMillis = Math.floor((Math.random() * 100) + 1) + await new Promise(resolve2 => setTimeout(resolve2, sleepMillis)) + const res = await this._query(query, retries - 1) + resolve(res) + } catch(err) { + reject(err) + } + } else { + reject(err) + } } else { queryDebug && Logger.info(result) resolve(result) diff --git a/lib/http-server/http-server.js b/lib/http-server/http-server.js index 93ccb3b..9076be6 100644 --- a/lib/http-server/http-server.js +++ b/lib/http-server/http-server.js @@ -5,7 +5,6 @@ 'use strict' const fs = require('fs') -const https = require('https') const express = require('express') const helmet = require('helmet') const Logger = require('../logger') @@ -19,15 +18,11 @@ class HttpServer { /** * Constructor * @param {int} port - port used by the http server - * @param {object} httpsOptions - https options */ - constructor(port, httpsOptions) { + constructor(port) { // Initialize server port this.port = port - // Store https options - this.httpsOptions = httpsOptions - // Listening server instance this.server = null @@ -58,30 +53,10 @@ class HttpServer { HttpServer.sendError(res, ret, 500) }) - if (this.httpsOptions == null || !this.httpsOptions.active) { - // Start a http server - this.server = this.app.listen(this.port, () => { - Logger.info('HTTP server listening on port ' + this.port) - }) - } else { - // Start a https server - const options = { - key: fs.readFileSync(this.httpsOptions.keypath), - cert: fs.readFileSync(this.httpsOptions.certpath), - requestCert: false, - rejectUnauthorized: false - } - - if (this.httpsOptions.capath) - options.ca = fs.readFileSync(this.httpsOptions.capath) - - if (this.httpsOptions.passphrase) - options.passphrase = this.httpsOptions.passphrase - - this.server = https.createServer(options, this.app).listen(this.port, () => { - Logger.info('HTTPS server listening on port ' + this.port) - }) - } + // Start a http server + this.server = this.app.listen(this.port, () => { + Logger.info('HTTP server listening on port ' + this.port) + }) this.server.timeout = 600 * 1000 // @see https://github.com/nodejs/node/issues/13391 diff --git a/lib/indexer-rpc/rpc-client.js b/lib/indexer-rpc/rpc-client.js new file mode 100644 index 0000000..704b6ab --- /dev/null +++ b/lib/indexer-rpc/rpc-client.js @@ -0,0 +1,224 @@ +/*! + * lib/indexer_rpc/rpc-client.js + * Copyright © 2019 – Katana Cryptographic Ltd. All Rights Reserved. + */ +'use strict' + +const net = require('net') +const makeConcurrent = require('make-concurrent') +const network = require('../bitcoin/network') +const keys = require('../../keys')[network.key] + + +/** + * RPC client for an indexer + * following the Electrum protocol + */ +class RpcClient { + + /** + * Constructor + */ + constructor(opts) { + this._opts = { + host: keys.indexer.localIndexer.host, + port: keys.indexer.localIndexer.port, + concurrency: Infinity, + timeout: 10000 + } + + if (this._opts.concurrency !== Infinity) { + this._call = makeConcurrent( + this._call.bind(this), + {concurrency: this._opts.concurrency} + ) + } + } + + /** + * Set an option + * @param {string} key + * @param {*} value + * @return {RpcClient} + */ + set(key, value) { + this._opts[key] = value + return this + } + + /** + * Get an option + * @param {string} key + * @return {*} + */ + get(key) { + return this._opts[key] + } + + /** + * Check if an error returned by the wrapper + * is a connection error. + * @param {string} err - error message + * @returns {boolean} returns true if message related to a connection error + */ + static isConnectionError(err) { + if (typeof err != 'string') + return false + + const isTimeoutError = (err.indexOf('connect ETIMEDOUT') != -1) + const isConnRejected = (err.indexOf('Connection Rejected') != -1) + + return (isTimeoutError || isConnRejected) + } + + /** + * Check if the rpc api is ready to process requests + * @returns {Promise} + */ + static async waitForIndexerRpcApi(opts) { + let client = new RpcClient(opts) + + try { + await client.sendRequest('server.version', 'dojo', ['1.0', '1.4']) + } catch(e) { + client = null + Logger.info('Indexer RPC API is still unreachable. New attempt in 20s.') + return util.delay(20000).then(() => { + return RpcClient.waitForIndexerRpcApi() + }) + } + } + + /** + * Send multiple requests (batch mode) + * @param {Object[]} batch - array of objects {method: ..., params: ...} + * @return {Promise} + */ + async sendBatch(batch) { + return this._call(batch, true) + } + + /** + * Send multiple requests (flood mode) + * @param {Object[]} batch - array of objects {method: ..., params: ...} + * @return {Promise} + */ + async sendRequests(batch) { + return this._call(batch, false) + } + + /** + * Send a request + * @param {string} method - called method + * @return {Promise} + */ + async sendRequest(method, ...params) { + const batch = [{method: method, params: params}] + const ret = await this._call(batch, false) + return ret[0] + } + + /** + * Send requests (internal method) + * @param {Object[]} data - array of objects {method: ..., params: ...} + * @returns {Promise} + */ + async _call(data, batched) { + return new Promise((resolve, reject) => { + let methodId = 0 + let requests = [] + let responses = [] + let response = '' + + const requestErrorMsg = `Indexer JSON-RPC: host=${this._opts.host} port=${this._opts.port}:` + + // Prepare an array of requests + requests = data.map(req => { + return JSON.stringify({ + jsonrpc: '2.0', + method: req.method, + params: req.params || [], + id: methodId++ + }) + }) + + // If batch mode + // send a single batched request + if (batched) + requests = [`[${requests.join(',')}]`] + + // Initialize the connection + const conn = net.Socket() + conn.setTimeout(this._opts.timeout) + conn.setEncoding('utf8') + conn.setKeepAlive(true, 0) + conn.setNoDelay(true) + + conn.on('connect', () => { + // Send the requests + for (let request of requests) + conn.write(request + '\n') + }) + + conn.on('timeout', () => { + const e = new Error('ETIMEDOUT') + e.errorno = 'ETIMEDOUT' + e.code = 'ETIMEDOUT' + e.connect = false + conn.emit('error', e) + }) + + conn.on('data', chunk => { + // Process the received chunk char by char + for (let c of chunk) { + response += c + // Detect the end of a response + if (c == '\n') { + try { + // Parse the response + let parsed = JSON.parse(response) + if (parsed.error) + throw new Error(JSON.stringify(parsed.error)) + // Add the parsed reponse to the array of responses + if (batched) { + responses = parsed.map(p => { return {idxAddr: p.id, txs: p.result} }) + } else { + responses.push({idxAddr: parsed.id, txs: parsed.result}) + } + // Reset the response + response = '' + // If all responses have been received + // close the connection + if (responses.length == data.length) + conn.end() + } catch (err) { + reject( + new Error(`${requestErrorMsg} Error Parsing JSON: ${err.message}, data: ${response}`) + ) + } + } + } + }) + + conn.on('end', e => { + // Connection closed + // we can return the responses + resolve(responses) + }) + + conn.on('error', e => { + reject(new Error(`${requestErrorMsg} Request error: ${e}`)) + }) + + // Connect to the RPC API + conn.connect({ + host: this._opts.host, + port: this._opts.port + }) + + }) + } + +} + +module.exports = RpcClient diff --git a/lib/remote-importer/btccom-wrapper.js b/lib/remote-importer/btccom-wrapper.js deleted file mode 100644 index 767f4b5..0000000 --- a/lib/remote-importer/btccom-wrapper.js +++ /dev/null @@ -1,179 +0,0 @@ -/*! - * lib/remote-importer\btccom-wrapper.js - * Copyright © 2019 – Katana Cryptographic Ltd. All Rights Reserved. - */ -'use strict' - -const rp = require('request-promise-native') -const addrHelper = require('../bitcoin/addresses-helper') -const util = require('../util') -const Logger = require('../logger') -const network = require('../bitcoin/network') -const keys = require('../../keys')[network.key] -const Wrapper = require('./wrapper') - - -/** - * Wrapper for the btc.com block explorer APIs - */ -class BtcComWrapper extends Wrapper { - - /** - * Constructor - */ - constructor(url) { - super(url, keys.explorers.socks5Proxy) - } - - /** - * Send a GET request to the API - * @param {string} route - * @returns {Promise} - */ - async _get(route) { - const params = { - url: `${this.base}${route}`, - method: 'GET', - json: true, - timeout: 15000 - } - - // Sets socks proxy agent if required - if (keys.explorers.socks5Proxy != null) - params['agent'] = this.socksProxyAgent - - return rp(params) - } - - /** - * Get a page of transactions related to a given address - * @param {string} address - bitcoin address - * @param {integer} page - page index - * @returns {Promise} - */ - async _getTxsForAddress(address, page) { - const uri = `/address/${address}/tx?page=${page}&verbose=1` - const results = await this._get(uri) - return results.data.list.map(tx => tx.hash) - } - - /** - * Retrieve information for a given address - * @param {string} address - bitcoin address - * @param {boolean} filterAddr - True if an upper bound should be used - * for #transactions associated to the address, False otherwise - * @returns {Promise} returns an object - * { address: , txids: , ntx: } - */ - async getAddress(address, filterAddr) { - const reqAddr = addrHelper.isBech32(address) - ? addrHelper.getScriptHashFromBech32(address) - : address - - const uri = `/address/${reqAddr}` - const result = await this._get(uri) - - const ret = { - address: address, - ntx: result.data.tx_count, - txids: [] - } - - // Check if we should filter this address - if (filterAddr && ret.ntx > keys.addrFilterThreshold) { - Logger.info(` import of ${ret.address} rejected (too many transactions - ${ret.ntx})`) - return ret - } - - const nbPagesApi = Math.ceil(ret.ntx / BtcComWrapper.NB_TXS_PER_PAGE) - const nbPages = Math.min(20, nbPagesApi) - - const aPages = new Array(nbPages) - const listPages = Array.from(aPages, (val, idx) => idx + 1) - - const results = await util.seriesCall(listPages, idx => { - return this._getTxsForAddress(reqAddr, idx) - }) - - for (let txids of results) - ret.txids = ret.txids.concat(txids) - - return ret - } - - /** - * Retrieve information for a given list of addresses - * @param {string[]} addresses - array of bitcoin addresses - * @param {boolean} filterAddr - True if an upper bound should be used - * for #transactions associated to the address, False otherwise - * @returns {Promise} returns an array of objects - * { address: , txids: , ntx: } - */ - async getAddresses(addresses, filterAddr) { - const ret = [] - const reqAddresses = [] - const xlatedBech32Addr = {} - - for (let a of addresses) { - if (addrHelper.isBech32(a)) { - const scriptHash = addrHelper.getScriptHashFromBech32(a) - reqAddresses.push(scriptHash) - xlatedBech32Addr[scriptHash] = a - } else { - reqAddresses.push(a) - } - } - - // Send a batch request for all the addresses - const strReqAddresses = reqAddresses.join(',') - const uri = `/address/${strReqAddresses}` - const results = await this._get(uri) - - const foundAddresses = Array.isArray(results.data) - ? results.data - : [results.data] - - for (let a of foundAddresses) { - if (a && a.tx_count > 0) { - // Translate bech32 address - const address = xlatedBech32Addr.hasOwnProperty(a.address) - ? xlatedBech32Addr[a.address] - : a.address - - if (a.tx_count <= 2) { - // Less than 3 transactions for this address - // all good - const retAddr = { - address: address, - ntx: a.tx_count, - txids: [] - } - - retAddr.txids = (a.tx_count == 1) - ? [a.first_tx] - : [a.first_tx, a.last_tx] - - ret.push(retAddr) - - } else { - // More than 2 transactions for this address - // We need more requests to the API - if (filterAddr && a.tx_count > keys.addrFilterThreshold) { - Logger.info(` import of ${address} rejected (too many transactions - ${a.tx_count})`) - } else { - const retAddr = await this.getAddress(address) - ret.push(retAddr) - } - } - } - } - - return ret - } - -} - -// BTC.COM acepts a max of 50txs per page -BtcComWrapper.NB_TXS_PER_PAGE = 50 - -module.exports = BtcComWrapper diff --git a/lib/remote-importer/esplora-wrapper.js b/lib/remote-importer/esplora-wrapper.js new file mode 100644 index 0000000..a962867 --- /dev/null +++ b/lib/remote-importer/esplora-wrapper.js @@ -0,0 +1,131 @@ +/*! + * lib/remote-importer\esplora-wrapper.js + * Copyright © 2019 – Katana Cryptographic Ltd. All Rights Reserved. + */ +'use strict' + +const rp = require('request-promise-native') +const addrHelper = require('../bitcoin/addresses-helper') +const util = require('../util') +const Logger = require('../logger') +const network = require('../bitcoin/network') +const keys = require('../../keys')[network.key] +const Wrapper = require('./wrapper') + + +/** + * Wrapper for the esplora block explorer APIs + */ +class EsploraWrapper extends Wrapper { + + /** + * Constructor + */ + constructor(url) { + super(url, keys.indexer.socks5Proxy) + } + + /** + * Send a GET request to the API + * @param {string} route + * @returns {Promise} + */ + async _get(route) { + const params = { + url: `${this.base}${route}`, + method: 'GET', + json: true, + timeout: 15000 + } + + // Sets socks proxy agent if required + if (keys.indexer.socks5Proxy != null) + params['agent'] = this.socksProxyAgent + + return rp(params) + } + + /** + * Get a page of transactions related to a given address + * @param {string} address - bitcoin address + * @param {string} lastSeenTxid - last seen txid + * (see https://github.com/Blockstream/esplora/blob/master/API.md) + * @returns {Promise} + */ + async _getTxsForAddress(address, lastSeenTxid) { + let uri = `/api/address/${address}/txs` + if (lastSeenTxid) + uri = uri + `/chain/${lastSeenTxid}` + + const results = await this._get(uri) + return results.map(tx => tx.txid) + } + + /** + * Retrieve information for a given address + * @param {string} address - bitcoin address + * @param {boolean} filterAddr - True if an upper bound should be used + * for #transactions associated to the address, False otherwise + * @returns {Promise} returns an object + * { address: , txids: , ntx: } + */ + async getAddress(address, filterAddr) { + const ret = { + address: address, + ntx: 0, + txids: [] + } + + let lastSeenTxid = null + + while (true) { + const txids = await this._getTxsForAddress(address, lastSeenTxid) + + if (txids.length == 0) + // we have all the transactions + return ret + + ret.txids = ret.txids.concat(txids) + ret.ntx += ret.txids.length + + if (txids.length < EsploraWrapper.NB_TXS_PER_PAGE) { + // we have all the transactions + return ret + } else if (filterAddr && ret.ntx > keys.addrFilterThreshold) { + // we have too many transactions + Logger.info(` import of ${ret.address} rejected (too many transactions - ${ret.ntx})`) + ret.txids = [] + ret.ntx = 0 + return ret + } else { + // we need a new iteration + lastSeenTxid = txids[txids.length-1] + } + } + } + + /** + * Retrieve information for a given list of addresses + * @param {string[]} addresses - array of bitcoin addresses + * @param {boolean} filterAddr - True if an upper bound should be used + * for #transactions associated to the address, False otherwise + * @returns {Promise} returns an array of objects + * { address: , txids: , ntx: } + */ + async getAddresses(addresses, filterAddr) { + const ret = [] + + for (let a of addresses) { + const retAddr = await this.getAddress(a, filterAddr) + ret.push(retAddr) + } + + return ret + } + +} + +// Esplora returns a max of 25 txs per page +EsploraWrapper.NB_TXS_PER_PAGE = 25 + +module.exports = EsploraWrapper diff --git a/lib/remote-importer/insight-wrapper.js b/lib/remote-importer/insight-wrapper.js deleted file mode 100644 index c11045f..0000000 --- a/lib/remote-importer/insight-wrapper.js +++ /dev/null @@ -1,90 +0,0 @@ -/*! - * lib/remote-importer/insight-wrapper.js - * Copyright © 2019 – Katana Cryptographic Ltd. All Rights Reserved. - */ -'use strict' - -const rp = require('request-promise-native') -const Logger = require('../logger') -const network = require('../bitcoin/network') -const keys = require('../../keys')[network.key] -const Wrapper = require('./wrapper') - - -/** - * Wrapper for the Insight block explorer APIs - */ -class InsightWrapper extends Wrapper { - - /** - * Constructor - */ - constructor(url) { - super(url, keys.explorers.socks5Proxy) - } - - /** - * Send a GET request to the API - * @param {string} route - * @returns {Promise} - */ - async _get(route) { - const params = { - url: `${this.base}${route}`, - method: 'GET', - json: true, - timeout: 15000 - } - - // Sets socks proxy agent if required - if (keys.explorers.socks5Proxy != null) - params['agent'] = this.socksProxyAgent - - return rp(params) - } - - /** - * Retrieve information for a given address - * @param {string} address - bitcoin address - * @param {boolean} filterAddr - True if an upper bound should be used - * for #transactions associated to the address, False otherwise - * @returns {Promise} returns an object - * { address: , txids: , ntx: } - */ - async getAddress(address, filterAddr) { - const uri = `/addr/${address}` - // Param filterAddr isn't used for insight - const result = await this._get(uri) - - const ret = { - address: result.addrStr, - txids: [], - ntx: result.txApperances - } - - // Check if we should filter this address - if (filterAddr && ret.ntx > keys.addrFilterThreshold) { - Logger.info(` import of ${ret.address} rejected (too many transactions - ${ret.ntx})`) - return ret - } - - ret.txids = result.transactions - return ret - } - - /** - * Retrieve information for a given list of addresses - * @param {string} addresses - array of bitcoin addresses - * @param {boolean} filterAddr - True if an upper bound should be used - * for #transactions associated to the address, False otherwise - * @returns {Promise} returns an array of objects - * { address: , txids: , ntx: } - */ - async getAddresses(addresses, filterAddr) { - // Not implemented for this api - throw "Not implemented" - } - -} - -module.exports = InsightWrapper diff --git a/lib/remote-importer/local-indexer-wrapper.js b/lib/remote-importer/local-indexer-wrapper.js new file mode 100644 index 0000000..9fcc37b --- /dev/null +++ b/lib/remote-importer/local-indexer-wrapper.js @@ -0,0 +1,143 @@ +/*! + * lib/remote-importer/local-indexer-wrapper.js + * Copyright © 2019 – Katana Cryptographic Ltd. All Rights Reserved. + */ +'use strict' + +const bitcoin = require('bitcoinjs-lib') +const Logger = require('../logger') +const util = require('../util') +const network = require('../bitcoin/network') +const activeNet = network.network +const keys = require('../../keys')[network.key] +const RpcClient = require('../indexer-rpc/rpc-client') +const Wrapper = require('./wrapper') + + +/** + * Wrapper for a local indexer + * Currently supports indexers + * providing a RPC API compliant + * with a subset of the electrum protocol + */ +class LocalIndexerWrapper extends Wrapper { + + /** + * Constructor + */ + constructor() { + super(null, null) + // RPC client + this.client = new RpcClient() + } + + /** + * Translate a bitcoin address into a script hash + * (@see https://electrumx.readthedocs.io/en/latest/protocol-basics.html#script-hashes) + * @param {string} address - bitcoin address + * @returns {string} returns the script hash associated to the address + */ + _getScriptHash(address) { + const bScriptPubKey = bitcoin.address.toOutputScript(address, activeNet) + const bScriptHash = bitcoin.crypto.sha256(bScriptPubKey) + return util.reverseBuffer(bScriptHash).toString('hex') + } + + /** + * Retrieve information for a given address + * @param {string} address - bitcoin address + * @param {boolean} filterAddr - True if an upper bound should be used + * for #transactions associated to the address, False otherwise + * @returns {Promise} returns an object + * { address: , txids: , ntx: } + */ + async getAddress(address, filterAddr) { + const ret = { + address: address, + ntx: 0, + txids: [] + } + + const scriptHash = this._getScriptHash(address) + + const results = await this.client.sendRequest( + LocalIndexerWrapper.GET_HISTORY_RPC_CMD, + scriptHash + ) + + for (let r of results.txs) { + ret.txids.push(r.tx_hash) + ret.ntx++ + } + + if (filterAddr && ret.ntx > keys.addrFilterThreshold) { + Logger.info(` import of ${address} rejected (too many transactions - ${ret.ntx})`) + return { + address: address, + ntx: 0, + txids: [] + } + } + + return ret + } + + /** + * Retrieve information for a given list of addresses + * @param {string} addresses - array of bitcoin addresses + * @param {boolean} filterAddr - True if an upper bound should be used + * for #transactions associated to the address, False otherwise + * @returns {Promise} returns an array of objects + * { address: , txids: , ntx: } + */ + async getAddresses(addresses, filterAddr) { + const ret = {} + + // Build an array of script hashes + const scriptHashes = addresses.map(a => this._getScriptHash(a)) + + // Build an array of commands + const commands = scriptHashes.map(s => { + return { + method: LocalIndexerWrapper.GET_HISTORY_RPC_CMD, + params: [s] + } + }) + + // Send the requests + const results = (keys.indexer.localIndexer.batchRequests == 'active') + ? await this.client.sendBatch(commands) + : await this.client.sendRequests(commands) + + for (let r of results) { + const addr = addresses[r.idxAddr] + const txids = r.txs.map(t => t.tx_hash) + + ret[addr] = { + address: addr, + ntx: txids.length, + txids: txids + } + } + + const aRet = Object.values(ret) + + for (let i in aRet) { + if (filterAddr && aRet[i].ntx > keys.addrFilterThreshold) { + Logger.info(` import of ${aRet[i].address} rejected (too many transactions - ${aRet[i].ntx})`) + aRet.splice(i, 1) + } + } + + return aRet + } + +} + +/** + * Get history RPC command (Electrum protocol) + */ +LocalIndexerWrapper.GET_HISTORY_RPC_CMD = 'blockchain.scripthash.get_history' + + +module.exports = LocalIndexerWrapper diff --git a/lib/remote-importer/oxt-wrapper.js b/lib/remote-importer/oxt-wrapper.js index dd63fa7..31a2bc1 100644 --- a/lib/remote-importer/oxt-wrapper.js +++ b/lib/remote-importer/oxt-wrapper.js @@ -20,7 +20,7 @@ class OxtWrapper extends Wrapper { * Constructor */ constructor(url) { - super(url, keys.explorers.socks5Proxy) + super(url, keys.indexer.socks5Proxy) } /** @@ -37,7 +37,7 @@ class OxtWrapper extends Wrapper { } // Sets socks proxy agent if required - if (keys.explorers.socks5Proxy != null) + if (keys.indexer.socks5Proxy != null) params['agent'] = this.socksProxyAgent return rp(params) diff --git a/lib/remote-importer/remote-importer.js b/lib/remote-importer/remote-importer.js index c48a8db..51df487 100644 --- a/lib/remote-importer/remote-importer.js +++ b/lib/remote-importer/remote-importer.js @@ -144,9 +144,13 @@ class RemoteImporter { let gaps = [gap.external, gap.internal] // Allow custom higher gap limits - // for local scans relying on bitcoind - if (gapLimit && keys.explorers.bitcoind) + // for local scans relying on bitcoind or on a local indexer + if (gapLimit + && ((keys.indexer.active == 'local_bitcoind') + || (keys.indexer.active == 'local_indexer')) + ) { gaps = [gapLimit, gapLimit] + } startIndex = (startIndex == null) ? -1 : startIndex - 1 diff --git a/lib/remote-importer/sources-mainnet.js b/lib/remote-importer/sources-mainnet.js index 6bc1121..6037bfa 100644 --- a/lib/remote-importer/sources-mainnet.js +++ b/lib/remote-importer/sources-mainnet.js @@ -9,6 +9,7 @@ const Logger = require('../logger') const keys = require('../../keys')[network.key] const Sources = require('./sources') const BitcoindWrapper = require('./bitcoind-wrapper') +const LocalIndexerWrapper = require('./local-indexer-wrapper') const OxtWrapper = require('./oxt-wrapper') @@ -29,14 +30,19 @@ class SourcesMainnet extends Sources { * Initialize the external data source */ _initSource() { - if (keys.explorers.bitcoind == 'active') { + if (keys.indexer.active == 'local_bitcoind') { // If local bitcoind option is activated // we'll use the local node as our unique source this.source = new BitcoindWrapper() Logger.info('Activated Bitcoind as the data source for imports') + } else if (keys.indexer.active == 'local_indexer') { + // If local indexer option is activated + // we'll use the local indexer as our unique source + this.source = new LocalIndexerWrapper() + Logger.info('Activated local indexer as the data source for imports') } else { // Otherwise, we'll use the rest api provided by OXT - this.source = new OxtWrapper(keys.explorers.oxt) + this.source = new OxtWrapper(keys.indexer.oxt) Logger.info('Activated OXT API as the data source for imports') } } diff --git a/lib/remote-importer/sources-testnet.js b/lib/remote-importer/sources-testnet.js index 588f9e0..084fdc2 100644 --- a/lib/remote-importer/sources-testnet.js +++ b/lib/remote-importer/sources-testnet.js @@ -10,7 +10,8 @@ const Logger = require('../logger') const keys = require('../../keys')[network.key] const Sources = require('./sources') const BitcoindWrapper = require('./bitcoind-wrapper') -const BtcComWrapper = require('./btccom-wrapper') +const LocalIndexerWrapper = require('./local-indexer-wrapper') +const EsploraWrapper = require('./esplora-wrapper') /** @@ -30,15 +31,20 @@ class SourcesTestnet extends Sources { * Initialize the external data source */ _initSource() { - if (keys.explorers.bitcoind == 'active') { + if (keys.indexer.active == 'local_bitcoind') { // If local bitcoind option is activated // we'll use the local node as our unique source this.source = new BitcoindWrapper() Logger.info('Activated Bitcoind as the data source for imports') + } else if (keys.indexer.active == 'local_indexer') { + // If local indexer option is activated + // we'll use the local indexer as our unique source + this.source = new LocalIndexerWrapper() + Logger.info('Activated local indexer as the data source for imports') } else { - // Otherwise, we'll use the rest api provided by OXT - this.source = new BtcComWrapper(keys.explorers.btccom) - Logger.info('Activated BTC.COM API as the data source for imports') + // Otherwise, we'll use the rest api provided by Esplora + this.source = new EsploraWrapper(keys.indexer.esplora) + Logger.info('Activated Esplora API as the data source for imports') } } diff --git a/lib/util.js b/lib/util.js index 199da0b..2e2e1fc 100644 --- a/lib/util.js +++ b/lib/util.js @@ -132,6 +132,27 @@ class Util { return Util.isHashStr(hash) && hash.length == 64 } + + /** + * Reverse buffer content (swap endiannes) + */ + static reverseBuffer(buffer) { + if (buffer.length < 1) + return buffer + + let j = buffer.length - 1 + let tmp = 0 + + for (let i = 0; i < buffer.length / 2; i++) { + tmp = buffer[i] + buffer[i] = buffer[j] + buffer[j] = tmp + j-- + } + + return buffer + } + /** * Sum an array of values */ diff --git a/lib/wallet/address-info.js b/lib/wallet/address-info.js index 53bf15a..c5a22c2 100644 --- a/lib/wallet/address-info.js +++ b/lib/wallet/address-info.js @@ -65,9 +65,12 @@ class AddressInfo { this.path = ['M', info.hdAddrChain, info.hdAddrIndex].join('/') } - if (res.loose.indexOf(this.address) > -1) { - this.tracked = true - this.type = 'loose' + for (let a of res.loose) { + if (a.addrAddress == this.address) { + this.tracked = true + this.type = 'loose' + break + } } return this.loadInfo() diff --git a/lib/wallet/wallet-service.js b/lib/wallet/wallet-service.js index 9d1745d..78f75be 100644 --- a/lib/wallet/wallet-service.js +++ b/lib/wallet/wallet-service.js @@ -9,6 +9,9 @@ const Logger = require('../logger') const db = require('../db/mysql-db-wrapper') const hdaService = require('../bitcoin/hd-accounts-service') const hdaHelper = require('../bitcoin/hd-accounts-helper') +const network = require('../bitcoin/network') +const activeNet = network.network +const keys = require('../../keys')[network.key] const WalletInfo = require('./wallet-info') @@ -60,9 +63,13 @@ class WalletService { await db.addAddresses(bip49.addrs) await db.addAddresses(bip84.addrs) await db.addAddresses(pubkeys.addrs) - // Ensure addresses exist and filter them + // Ensure addresses exist await walletInfo.ensureAddresses() - //await this._forceEnsureAddressesForActivePubkeys(active) + // Force import of addresses associated to paynyms + // if dojo relies on a local index + if (keys.indexer.active != 'third_party_explorer') + await this._forceEnsureAddressesForActivePubkeys(active) + // Filter the address and load them await walletInfo.filterAddresses() await walletInfo.loadAddressesInfo() // Load the most recent transactions @@ -136,9 +143,13 @@ class WalletService { await db.addAddresses(bip49.addrs) await db.addAddresses(bip84.addrs) await db.addAddresses(pubkeys.addrs) - // Ensure addresses exist and filter them + // Ensure addresses exist await walletInfo.ensureAddresses() - //await this._forceEnsureAddressesForActivePubkeys(active) + // Force import of addresses associated to paynyms + // if dojo relies on a local index + if (keys.indexer.active != 'third_party_explorer') + await this._forceEnsureAddressesForActivePubkeys(active) + // Filter the addresses await walletInfo.filterAddresses() // Load the utxos await walletInfo.loadUtxos() diff --git a/package-lock.json b/package-lock.json index 126bec6..c200089 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,9 +1,14 @@ { "name": "samourai-dojo", - "version": "1.2.0", + "version": "1.3.0", "lockfileVersion": 1, "requires": true, "dependencies": { + "@types/node": { + "version": "10.12.18", + "resolved": "https://registry.npmjs.org/@types/node/-/node-10.12.18.tgz", + "integrity": "sha512-fh+pAqt4xRzPfqA6eh3Z2y6fyZavRIumvjhaCL753+TVkGKGhpPeyrJG2JftD0T9q4GF00KjefsQ+PQNDdWQaQ==" + }, "accepts": { "version": "1.3.7", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz", @@ -32,11 +37,26 @@ "uri-js": "4.2.2" } }, + "ansi-colors": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-3.2.3.tgz", + "integrity": "sha512-LEHHyuhlPY3TmuUYMh2oz89lTShfvgbmzaBcxve9t/9Wuy7Dwf4yoAKcND7KFT1HAQfqZ12qtc+DUrBMeKF9nw==", + "dev": true + }, "ansi-regex": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=" }, + "ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "requires": { + "color-convert": "1.9.3" + } + }, "aproba": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", @@ -51,6 +71,15 @@ "readable-stream": "2.3.6" } }, + "argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, + "requires": { + "sprintf-js": "1.0.3" + } + }, "array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", @@ -128,14 +157,9 @@ } }, "bech32": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/bech32/-/bech32-0.0.3.tgz", - "integrity": "sha512-O+K1w8P/aAOLcYwwQ4sbiPYZ51ZIW95lnS4/6nE8Aib/z+OOddQIIPdu2qi94qGDp4HhYy/wJotttXKkak1lXg==" - }, - "bigi": { - "version": "1.4.2", - "resolved": "https://registry.npmjs.org/bigi/-/bigi-1.4.2.tgz", - "integrity": "sha1-nGZalfiLiwj8Bc/XMfVhhZ1yWCU=" + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/bech32/-/bech32-1.1.3.tgz", + "integrity": "sha512-yuVFUvrNcoJi0sv5phmqc6P+Fl1HjRDRNOOkHY2X/3LBy2bIGNSFx4fZ95HMaXHupuS7cZR15AsvtmCIF4UEyg==" }, "bignumber.js": { "version": "4.1.0", @@ -150,6 +174,25 @@ "file-uri-to-path": "1.0.0" } }, + "bip174": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/bip174/-/bip174-1.0.1.tgz", + "integrity": "sha512-Mq2aFs1TdMfxBpYPg7uzjhsiXbAtoVq44TNjEWtvuZBiBgc3m7+n55orYMtTAxdg7jWbL4DtH0MKocJER4xERQ==" + }, + "bip32": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/bip32/-/bip32-2.0.4.tgz", + "integrity": "sha512-ioPytarPDIrWckWMuK4RNUtvwhvWEc2fvuhnO0WEwu732k5OLjUXv4rXi2c/KJHw9ZMNQMkYRJrBw81RujShGQ==", + "requires": { + "@types/node": "10.12.18", + "bs58check": "2.1.2", + "create-hash": "1.2.0", + "create-hmac": "1.1.7", + "tiny-secp256k1": "1.1.3", + "typeforce": "1.18.0", + "wif": "2.0.6" + } + }, "bip39": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/bip39/-/bip39-2.4.0.tgz", @@ -186,22 +229,23 @@ } }, "bitcoinjs-lib": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/bitcoinjs-lib/-/bitcoinjs-lib-3.2.0.tgz", - "integrity": "sha512-6as6LSz/K5WbCLi8wTNxPJcQgpgeuTsfS52v/C/dM7IcmnDA5PDaZArhsUTFaKsM6NPFgsQr2ZCbWTjYyXJevw==", - "requires": { - "bech32": "0.0.3", - "bigi": "1.4.2", + "version": "5.1.4", + "resolved": "https://registry.npmjs.org/bitcoinjs-lib/-/bitcoinjs-lib-5.1.4.tgz", + "integrity": "sha512-tSgxnXC3xEXw5DxKGIHNkzz+pxYzsSViT/XfQVIn8vM/b5kXtg3BJuFg6T1h2zhgG+iClzDf+5S/5wxFIddzWg==", + "requires": { + "@types/node": "10.12.18", + "bech32": "1.1.3", + "bip174": "1.0.1", + "bip32": "2.0.4", "bip66": "1.1.5", "bitcoin-ops": "1.4.1", "bs58check": "2.1.2", "create-hash": "1.2.0", "create-hmac": "1.1.7", - "ecurve": "1.0.6", "merkle-lib": "2.0.10", "pushdata-bitcoin": "1.0.1", "randombytes": "2.1.0", - "safe-buffer": "5.2.0", + "tiny-secp256k1": "1.1.3", "typeforce": "1.18.0", "varuint-bitcoin": "1.1.0", "wif": "2.0.6" @@ -290,9 +334,9 @@ "integrity": "sha1-EsJe/kCkXjwyPrhnWgoM5XsiNx8=" }, "browser-stdout": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.0.tgz", - "integrity": "sha1-81HTKWnTL6XXpVZxVCY9korjvR8=", + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.1.tgz", + "integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==", "dev": true }, "browserify-aes": { @@ -365,6 +409,12 @@ "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", "integrity": "sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg=" }, + "camelcase": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", + "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", + "dev": true + }, "camelize": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/camelize/-/camelize-1.0.0.tgz", @@ -375,6 +425,28 @@ "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" }, + "chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "requires": { + "ansi-styles": "3.2.1", + "escape-string-regexp": "1.0.5", + "supports-color": "5.5.0" + }, + "dependencies": { + "supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "requires": { + "has-flag": "3.0.0" + } + } + } + }, "chownr": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.2.tgz", @@ -389,11 +461,70 @@ "safe-buffer": "5.2.0" } }, + "cliui": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-4.1.0.tgz", + "integrity": "sha512-4FG+RSG9DL7uEwRUZXZn3SS34DiDPfzP0VOiEwtUWlE+AR2EIg+hSyvrIgUUfhdgR/UkAeW2QHgeP+hWrXs7jQ==", + "dev": true, + "requires": { + "string-width": "2.1.1", + "strip-ansi": "4.0.0", + "wrap-ansi": "2.1.0" + }, + "dependencies": { + "ansi-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", + "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=", + "dev": true + }, + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", + "dev": true + }, + "string-width": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", + "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", + "dev": true, + "requires": { + "is-fullwidth-code-point": "2.0.0", + "strip-ansi": "4.0.0" + } + }, + "strip-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", + "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", + "dev": true, + "requires": { + "ansi-regex": "3.0.0" + } + } + } + }, "code-point-at": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=" }, + "color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "requires": { + "color-name": "1.1.3" + } + }, + "color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", + "dev": true + }, "combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -402,15 +533,6 @@ "delayed-stream": "1.0.0" } }, - "commander": { - "version": "2.9.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-2.9.0.tgz", - "integrity": "sha1-nJkJQXbhIkDLItbFFGCYQA/g99Q=", - "dev": true, - "requires": { - "graceful-readlink": "1.0.1" - } - }, "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -482,6 +604,19 @@ "sha.js": "2.4.11" } }, + "cross-spawn": { + "version": "6.0.5", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz", + "integrity": "sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==", + "dev": true, + "requires": { + "nice-try": "1.0.5", + "path-key": "2.0.1", + "semver": "5.7.1", + "shebang-command": "1.2.0", + "which": "1.3.1" + } + }, "dashdash": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", @@ -503,6 +638,12 @@ "ms": "2.0.0" } }, + "decamelize": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", + "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=", + "dev": true + }, "decompress-response": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-3.3.0.tgz", @@ -516,6 +657,15 @@ "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==" }, + "define-properties": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", + "integrity": "sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==", + "dev": true, + "requires": { + "object-keys": "1.1.1" + } + }, "delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -542,9 +692,9 @@ "integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups=" }, "diff": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/diff/-/diff-3.2.0.tgz", - "integrity": "sha1-yc45Okt8vQsFinJck98pkCeGj/k=", + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz", + "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==", "dev": true }, "dns-prefetch-control": { @@ -589,15 +739,6 @@ "safe-buffer": "5.2.0" } }, - "ecurve": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/ecurve/-/ecurve-1.0.6.tgz", - "integrity": "sha512-/BzEjNfiSuB7jIWKcS/z8FK9jNjmEWvUV2YZ4RLSmcDtP7Lq0m6FvDuSnJpBlDpGRpfRQeTLGLBI8H+kEv0r+w==", - "requires": { - "bigi": "1.4.2", - "safe-buffer": "5.2.0" - } - }, "ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", @@ -617,6 +758,12 @@ "minimalistic-crypto-utils": "1.0.1" } }, + "emoji-regex": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", + "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", + "dev": true + }, "encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", @@ -638,6 +785,31 @@ "inherits": "2.0.4" } }, + "es-abstract": { + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.13.0.tgz", + "integrity": "sha512-vDZfg/ykNxQVwup/8E1BZhVzFfBxs9NqMzGcvIJrqg5k2/5Za2bWo40dK2J1pgLngZ7c+Shh8lwYtLGyrwPutg==", + "dev": true, + "requires": { + "es-to-primitive": "1.2.0", + "function-bind": "1.1.1", + "has": "1.0.3", + "is-callable": "1.1.4", + "is-regex": "1.0.4", + "object-keys": "1.1.1" + } + }, + "es-to-primitive": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.0.tgz", + "integrity": "sha512-qZryBOJjV//LaxLTV6UC//WewneB3LcXOL9NP++ozKVXsIIIpm/2c13UDiD9Jp2eThsecw9m3jPqDwTyobcdbg==", + "dev": true, + "requires": { + "is-callable": "1.1.4", + "is-date-object": "1.0.1", + "is-symbol": "1.0.2" + } + }, "es6-promise": { "version": "4.2.8", "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.8.tgz", @@ -662,6 +834,12 @@ "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", "dev": true }, + "esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "dev": true + }, "etag": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", @@ -676,6 +854,21 @@ "safe-buffer": "5.2.0" } }, + "execa": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/execa/-/execa-1.0.0.tgz", + "integrity": "sha512-adbxcyWV46qiHyvSp50TKt05tB4tK3HcmF7/nxfAdhnox83seTDbwnaqKO4sXRy7roHAIFqJP/Rw/AuEbX61LA==", + "dev": true, + "requires": { + "cross-spawn": "6.0.5", + "get-stream": "4.1.0", + "is-stream": "1.1.0", + "npm-run-path": "2.0.2", + "p-finally": "1.0.0", + "signal-exit": "3.0.2", + "strip-eof": "1.0.0" + } + }, "expand-template": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-1.1.1.tgz", @@ -863,6 +1056,24 @@ } } }, + "find-up": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", + "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", + "dev": true, + "requires": { + "locate-path": "3.0.0" + } + }, + "flat": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/flat/-/flat-4.1.0.tgz", + "integrity": "sha512-Px/TiLIznH7gEDlPXcUD4KnBusa6kR6ayRUVcnEAbreRIuhkqow/mun59BuRXwoYk7ZQOLW1ZM05ilIvK38hFw==", + "dev": true, + "requires": { + "is-buffer": "2.0.3" + } + }, "forever-agent": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", @@ -904,6 +1115,12 @@ "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", "dev": true }, + "function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", + "dev": true + }, "gauge": { "version": "2.7.4", "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz", @@ -924,6 +1141,33 @@ "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.4.2.tgz", "integrity": "sha512-H7cUpwCQSiJmAHM4c/aFu6fUfrhWXW1ncyh8ftxEPMu6AiYkHw9K8br720TGPZJbk5eOH2bynjZD1yPvdDAmag==" }, + "get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true + }, + "get-stream": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz", + "integrity": "sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==", + "dev": true, + "requires": { + "pump": "3.0.0" + }, + "dependencies": { + "pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "dev": true, + "requires": { + "end-of-stream": "1.4.1", + "once": "1.4.0" + } + } + } + }, "getpass": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", @@ -938,9 +1182,9 @@ "integrity": "sha1-l/tdlr/eiXMxPyDoKI75oWf6ZM4=" }, "glob": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.1.tgz", - "integrity": "sha1-gFIR3wT6rxxjo2ADBs31reULLsg=", + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz", + "integrity": "sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==", "dev": true, "requires": { "fs.realpath": "1.0.0", @@ -951,16 +1195,10 @@ "path-is-absolute": "1.0.1" } }, - "graceful-readlink": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/graceful-readlink/-/graceful-readlink-1.0.1.tgz", - "integrity": "sha1-TK+tdrxi8C+gObL5Tpo906ORpyU=", - "dev": true - }, "growl": { - "version": "1.9.2", - "resolved": "https://registry.npmjs.org/growl/-/growl-1.9.2.tgz", - "integrity": "sha1-Dqd0NxXbjY3ixe3hd14bRayFwC8=", + "version": "1.10.5", + "resolved": "https://registry.npmjs.org/growl/-/growl-1.10.5.tgz", + "integrity": "sha512-qBr4OuELkhPenW6goKVXiv47US3clb3/IbuWF9KNKEijAy9oeHxU9IgzjvJhHkUzhaj7rOUD7+YGWqUjLp5oSA==", "dev": true }, "har-schema": { @@ -977,10 +1215,25 @@ "har-schema": "2.0.0" } }, + "has": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "dev": true, + "requires": { + "function-bind": "1.1.1" + } + }, "has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "dev": true + }, + "has-symbols": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-1.0.0.tgz", - "integrity": "sha1-nZ55MWXOAXoA8AQYxD+UKnsdEfo=", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.0.tgz", + "integrity": "sha1-uhqPGvKg/DllD1yFA2dwQSIGO0Q=", "dev": true }, "has-unicode": { @@ -1007,9 +1260,9 @@ } }, "he": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/he/-/he-1.1.1.tgz", - "integrity": "sha1-k0EP0hsAlzUVH4howvJx80J+I/0=", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", + "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", "dev": true }, "helmet": { @@ -1129,6 +1382,12 @@ "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==" }, + "invert-kv": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-2.0.0.tgz", + "integrity": "sha512-wPVv/y/QQ/Uiirj/vh3oP+1Ww+AWehmi1g5fFWGPF6IpCBCDVrhgHRMvrLfdYcwDh3QJbGXDW4JAuzxElLSqKA==", + "dev": true + }, "ip": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/ip/-/ip-1.1.5.tgz", @@ -1139,6 +1398,24 @@ "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.0.tgz", "integrity": "sha512-M4Sjn6N/+O6/IXSJseKqHoFc+5FdGJ22sXqnjTpdZweHK64MzEPAyQZyEU3R/KRv2GLoa7nNtg/C2Ev6m7z+eA==" }, + "is-buffer": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.3.tgz", + "integrity": "sha512-U15Q7MXTuZlrbymiz95PJpZxu8IlipAp4dtS3wOdgPXx3mqBnslrWU14kxfHB+Py/+2PVKSr37dMAgM2A4uArw==", + "dev": true + }, + "is-callable": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.1.4.tgz", + "integrity": "sha512-r5p9sxJjYnArLjObpjA4xu5EKI3CuKHkJXMhT7kwbpUyIFD1n5PMAsoPvWnvtZiNz7LjkYDRZhd7FlI0eMijEA==", + "dev": true + }, + "is-date-object": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.1.tgz", + "integrity": "sha1-mqIOtq7rv/d/vTPnTKAbM1gdOhY=", + "dev": true + }, "is-fullwidth-code-point": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", @@ -1147,6 +1424,30 @@ "number-is-nan": "1.0.1" } }, + "is-regex": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.0.4.tgz", + "integrity": "sha1-VRdIm1RwkbCTDglWVM7SXul+lJE=", + "dev": true, + "requires": { + "has": "1.0.3" + } + }, + "is-stream": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", + "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=", + "dev": true + }, + "is-symbol": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.2.tgz", + "integrity": "sha512-HS8bZ9ox60yCJLH9snBpIwv9pYUAkcuLhSA1oero1UB5y9aiQpRA8y2ex945AOtCZL1lJDeIk3G5LthswI46Lw==", + "dev": true, + "requires": { + "has-symbols": "1.0.0" + } + }, "is-typedarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", @@ -1157,11 +1458,27 @@ "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" }, + "isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=", + "dev": true + }, "isstream": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" }, + "js-yaml": { + "version": "3.13.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.13.1.tgz", + "integrity": "sha512-YfbcO7jXDdyj0DGxYVSlSeQNHbD7XPWvrVWeVUujrQEoZzWJIRrCPoyk6kL6IAjAG2IolMK4T0hNUe0HOUs5Jw==", + "dev": true, + "requires": { + "argparse": "1.0.10", + "esprima": "4.0.1" + } + }, "jsbn": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", @@ -1182,12 +1499,6 @@ "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" }, - "json3": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/json3/-/json3-3.3.2.tgz", - "integrity": "sha1-PAQ0dD35Pi9cQq7nsZvLSDV19OE=", - "dev": true - }, "jsonwebtoken": { "version": "8.5.1", "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-8.5.1.tgz", @@ -1202,7 +1513,7 @@ "lodash.isstring": "4.0.1", "lodash.once": "4.1.1", "ms": "2.1.2", - "semver": "5.7.0" + "semver": "5.7.1" }, "dependencies": { "ms": { @@ -1242,73 +1553,35 @@ "safe-buffer": "5.2.0" } }, - "lodash": { - "version": "4.17.14", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.14.tgz", - "integrity": "sha512-mmKYbW3GLuJeX+iGP+Y7Gp1AiGHGbXHCOh/jZmrawMmsE7MS4znI3RL2FsjbqOyMayHInjOeykW7PEajUk1/xw==" - }, - "lodash._baseassign": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/lodash._baseassign/-/lodash._baseassign-3.2.0.tgz", - "integrity": "sha1-jDigmVAPIVrQnlnxci/QxSv+Ck4=", + "lcid": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/lcid/-/lcid-2.0.0.tgz", + "integrity": "sha512-avPEb8P8EGnwXKClwsNUgryVjllcRqtMYa49NTsbQagYuT1DcXnl1915oxWjoyGrXR6zH/Y0Zc96xWsPcoDKeA==", "dev": true, "requires": { - "lodash._basecopy": "3.0.1", - "lodash.keys": "3.1.2" + "invert-kv": "2.0.0" } }, - "lodash._basecopy": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/lodash._basecopy/-/lodash._basecopy-3.0.1.tgz", - "integrity": "sha1-jaDmqHbPNEwK2KVIghEd08XHyjY=", - "dev": true - }, - "lodash._basecreate": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/lodash._basecreate/-/lodash._basecreate-3.0.3.tgz", - "integrity": "sha1-G8ZhYU2qf8MRt9A78WgGoCE8+CE=", - "dev": true - }, - "lodash._getnative": { - "version": "3.9.1", - "resolved": "https://registry.npmjs.org/lodash._getnative/-/lodash._getnative-3.9.1.tgz", - "integrity": "sha1-VwvH3t5G1hzc3mh9ZdPuy6o6r/U=", - "dev": true - }, - "lodash._isiterateecall": { - "version": "3.0.9", - "resolved": "https://registry.npmjs.org/lodash._isiterateecall/-/lodash._isiterateecall-3.0.9.tgz", - "integrity": "sha1-UgOte6Ql+uhCRg5pbbnPPmqsBXw=", - "dev": true - }, - "lodash.create": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/lodash.create/-/lodash.create-3.1.1.tgz", - "integrity": "sha1-1/KEnw29p+BGgruM1yqwIkYd6+c=", + "locate-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", + "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", "dev": true, "requires": { - "lodash._baseassign": "3.2.0", - "lodash._basecreate": "3.0.3", - "lodash._isiterateecall": "3.0.9" + "p-locate": "3.0.0", + "path-exists": "3.0.0" } }, + "lodash": { + "version": "4.17.14", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.14.tgz", + "integrity": "sha512-mmKYbW3GLuJeX+iGP+Y7Gp1AiGHGbXHCOh/jZmrawMmsE7MS4znI3RL2FsjbqOyMayHInjOeykW7PEajUk1/xw==" + }, "lodash.includes": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", "integrity": "sha1-YLuYqHy5I8aMoeUTJUgzFISfVT8=" }, - "lodash.isarguments": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", - "integrity": "sha1-L1c9hcaiQon/AGY7SRwdM4/zRYo=", - "dev": true - }, - "lodash.isarray": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/lodash.isarray/-/lodash.isarray-3.0.4.tgz", - "integrity": "sha1-eeTriMNqgSKvhvhEqpvNhRtfu1U=", - "dev": true - }, "lodash.isboolean": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", @@ -1334,17 +1607,6 @@ "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz", "integrity": "sha1-1SfftUVuynzJu5XV2ur4i6VKVFE=" }, - "lodash.keys": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/lodash.keys/-/lodash.keys-3.1.2.tgz", - "integrity": "sha1-TbwEcrFWvlCgsoaFXRvQsMZWCYo=", - "dev": true, - "requires": { - "lodash._getnative": "3.9.1", - "lodash.isarguments": "3.1.0", - "lodash.isarray": "3.0.4" - } - }, "lodash.once": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz", @@ -1360,6 +1622,15 @@ "resolved": "https://registry.npmjs.org/lodash.set/-/lodash.set-4.3.2.tgz", "integrity": "sha1-2HV7HagH3eJIFrDWqEvqGnYjCyM=" }, + "log-symbols": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-2.2.0.tgz", + "integrity": "sha512-VeIAFslyIerEJLXHziedo2basKbMKtTw3vfn5IzG0XTjhAVEJyNHnL2p7vc+wBDSdQuUpNw3M2u6xb9QsAY5Eg==", + "dev": true, + "requires": { + "chalk": "2.4.2" + } + }, "lru-cache": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.0.2.tgz", @@ -1377,6 +1648,15 @@ "babel-runtime": "5.8.38" } }, + "map-age-cleaner": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/map-age-cleaner/-/map-age-cleaner-0.1.3.tgz", + "integrity": "sha512-bJzx6nMoP6PDLPBFmg7+xRKeFZvFboMrGlxmNj9ClvX53KrmvM5bXFXEWjbz4cz1AFn+jWJ9z/DJSz7hrs0w3w==", + "dev": true, + "requires": { + "p-defer": "1.0.0" + } + }, "md5.js": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/md5.js/-/md5.js-1.3.5.tgz", @@ -1392,6 +1672,17 @@ "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=" }, + "mem": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/mem/-/mem-4.3.0.tgz", + "integrity": "sha512-qX2bG48pTqYRVmDB37rn/6PT7LcR8T7oAX3bf99u1Tt1nzxYfxkgqDwUwolPlXweM0XzBOBFzSx4kfp7KP1s/w==", + "dev": true, + "requires": { + "map-age-cleaner": "0.1.3", + "mimic-fn": "2.1.0", + "p-is-promise": "2.1.0" + } + }, "merge-descriptors": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", @@ -1425,6 +1716,12 @@ "mime-db": "1.40.0" } }, + "mimic-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "dev": true + }, "mimic-response": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz", @@ -1470,33 +1767,50 @@ } }, "mocha": { - "version": "3.5.3", - "resolved": "https://registry.npmjs.org/mocha/-/mocha-3.5.3.tgz", - "integrity": "sha512-/6na001MJWEtYxHOV1WLfsmR4YIynkUEhBwzsb+fk2qmQ3iqsi258l/Q2MWHJMImAcNpZ8DEdYAK72NHoIQ9Eg==", + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/mocha/-/mocha-6.2.0.tgz", + "integrity": "sha512-qwfFgY+7EKAAUAdv7VYMZQknI7YJSGesxHyhn6qD52DV8UcSZs5XwCifcZGMVIE4a5fbmhvbotxC0DLQ0oKohQ==", "dev": true, "requires": { - "browser-stdout": "1.3.0", - "commander": "2.9.0", - "debug": "2.6.8", - "diff": "3.2.0", + "ansi-colors": "3.2.3", + "browser-stdout": "1.3.1", + "debug": "3.2.6", + "diff": "3.5.0", "escape-string-regexp": "1.0.5", - "glob": "7.1.1", - "growl": "1.9.2", - "he": "1.1.1", - "json3": "3.3.2", - "lodash.create": "3.1.1", + "find-up": "3.0.0", + "glob": "7.1.3", + "growl": "1.10.5", + "he": "1.2.0", + "js-yaml": "3.13.1", + "log-symbols": "2.2.0", + "minimatch": "3.0.4", "mkdirp": "0.5.1", - "supports-color": "3.1.2" + "ms": "2.1.1", + "node-environment-flags": "1.0.5", + "object.assign": "4.1.0", + "strip-json-comments": "2.0.1", + "supports-color": "6.0.0", + "which": "1.3.1", + "wide-align": "1.1.3", + "yargs": "13.2.2", + "yargs-parser": "13.0.0", + "yargs-unparser": "1.5.0" }, "dependencies": { "debug": { - "version": "2.6.8", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.8.tgz", - "integrity": "sha1-5zFTHKLt4n0YgiJCfaF4IdaP9Pw=", + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", + "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", "dev": true, "requires": { - "ms": "2.0.0" + "ms": "2.1.1" } + }, + "ms": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", + "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==", + "dev": true } } }, @@ -1533,17 +1847,33 @@ "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", "integrity": "sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw==" }, + "nice-try": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/nice-try/-/nice-try-1.0.5.tgz", + "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==", + "dev": true + }, "nocache": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/nocache/-/nocache-2.0.0.tgz", "integrity": "sha1-ICtIAhoMTL3i34DeFaF0Q8i0OYA=" }, "node-abi": { - "version": "2.9.0", - "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-2.9.0.tgz", - "integrity": "sha512-jmEOvv0eanWjhX8dX1pmjb7oJl1U1oR4FOh0b2GnvALwSYoOdU7sj+kLDSAyjo4pfC9aj/IxkloxdLJQhSSQBA==", + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-2.11.0.tgz", + "integrity": "sha512-kuy/aEg75u40v378WRllQ4ZexaXJiCvB68D2scDXclp/I4cRq6togpbOoKhmN07tns9Zldu51NNERo0wehfX9g==", + "requires": { + "semver": "5.7.1" + } + }, + "node-environment-flags": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/node-environment-flags/-/node-environment-flags-1.0.5.tgz", + "integrity": "sha512-VNYPRfGfmZLx0Ye20jWzHUjyTW/c+6Wq+iLhDzUI4XmhrDd9l/FozXV3F2xOaXjvp0co0+v1YSR3CMP6g+VvLQ==", + "dev": true, "requires": { - "semver": "5.7.0" + "object.getownpropertydescriptors": "2.0.3", + "semver": "5.7.1" } }, "noop-logger": { @@ -1551,6 +1881,15 @@ "resolved": "https://registry.npmjs.org/noop-logger/-/noop-logger-0.1.1.tgz", "integrity": "sha1-lKKxYzxPExdVMAfYlm/Q6EG2pMI=" }, + "npm-run-path": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-2.0.2.tgz", + "integrity": "sha1-NakjLfo11wZ7TLLd8jV7GHFTbF8=", + "dev": true, + "requires": { + "path-key": "2.0.1" + } + }, "npmlog": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz", @@ -1577,6 +1916,34 @@ "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=" }, + "object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", + "dev": true + }, + "object.assign": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.0.tgz", + "integrity": "sha512-exHJeq6kBKj58mqGyTQ9DFvrZC/eR6OwxzoM9YRoGBqrXYonaFyGiFMuc9VZrXf7DarreEwMpurG3dd+CNyW5w==", + "dev": true, + "requires": { + "define-properties": "1.1.3", + "function-bind": "1.1.1", + "has-symbols": "1.0.0", + "object-keys": "1.1.1" + } + }, + "object.getownpropertydescriptors": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/object.getownpropertydescriptors/-/object.getownpropertydescriptors-2.0.3.tgz", + "integrity": "sha1-h1jIRvW0B62rDyNuCYbxSwUcqhY=", + "dev": true, + "requires": { + "define-properties": "1.1.3", + "es-abstract": "1.13.0" + } + }, "on-finished": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", @@ -1598,6 +1965,59 @@ "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", "integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M=" }, + "os-locale": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/os-locale/-/os-locale-3.1.0.tgz", + "integrity": "sha512-Z8l3R4wYWM40/52Z+S265okfFj8Kt2cC2MKY+xNi3kFs+XGI7WXu/I309QQQYbRW4ijiZ+yxs9pqEhJh0DqW3Q==", + "dev": true, + "requires": { + "execa": "1.0.0", + "lcid": "2.0.0", + "mem": "4.3.0" + } + }, + "p-defer": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-defer/-/p-defer-1.0.0.tgz", + "integrity": "sha1-n26xgvbJqozXQwBKfU+WsZaw+ww=", + "dev": true + }, + "p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=", + "dev": true + }, + "p-is-promise": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/p-is-promise/-/p-is-promise-2.1.0.tgz", + "integrity": "sha512-Y3W0wlRPK8ZMRbNq97l4M5otioeA5lm1z7bkNkxCka8HSPjR0xRWmpCmc9utiaLP9Jb1eD8BgeIxTW4AIF45Pg==", + "dev": true + }, + "p-limit": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.2.1.tgz", + "integrity": "sha512-85Tk+90UCVWvbDavCLKPOLC9vvY8OwEX/RtKF+/1OADJMVlFfEHOiMTPVyxg7mk/dKa+ipdHm0OUkTvCpMTuwg==", + "dev": true, + "requires": { + "p-try": "2.2.0" + } + }, + "p-locate": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", + "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", + "dev": true, + "requires": { + "p-limit": "2.2.1" + } + }, + "p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "dev": true + }, "parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -1626,12 +2046,24 @@ "resolved": "https://registry.npmjs.org/passport-strategy/-/passport-strategy-1.0.0.tgz", "integrity": "sha1-tVOaqPwiWj0a0XlHbd8ja0QPUuQ=" }, + "path-exists": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", + "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=", + "dev": true + }, "path-is-absolute": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", "dev": true }, + "path-key": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz", + "integrity": "sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A=", + "dev": true + }, "path-to-regexp": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", @@ -1679,7 +2111,7 @@ "github-from-package": "0.0.0", "minimist": "1.2.0", "mkdirp": "0.5.1", - "node-abi": "2.9.0", + "node-abi": "2.11.0", "noop-logger": "0.1.1", "npmlog": "4.1.2", "os-homedir": "1.0.2", @@ -1720,9 +2152,9 @@ "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=" }, "psl": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/psl/-/psl-1.2.0.tgz", - "integrity": "sha512-GEn74ZffufCmkDDLNcl3uuyF/aSD6exEyh1v/ZSdAomB82t6G9hzJVRx0jBmLDW+VfZqks3aScmMw9DszwUalA==" + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/psl/-/psl-1.3.0.tgz", + "integrity": "sha512-avHdspHO+9rQTLbv1RO+MPYeP/SzsCoxofjVnHanETfQhTJrmB0HlDoW+EiN/R+C0BZ+gERab9NY0lPN2TxNag==" }, "pump": { "version": "2.0.1", @@ -1836,7 +2268,7 @@ "safe-buffer": "5.2.0", "tough-cookie": "2.4.3", "tunnel-agent": "0.6.0", - "uuid": "3.3.2" + "uuid": "3.3.3" } }, "request-promise-core": { @@ -1857,6 +2289,18 @@ "tough-cookie": "2.4.3" } }, + "require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=", + "dev": true + }, + "require-main-filename": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", + "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==", + "dev": true + }, "ripemd160": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/ripemd160/-/ripemd160-2.0.2.tgz", @@ -1892,9 +2336,9 @@ } }, "semver": { - "version": "5.7.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz", - "integrity": "sha512-Ya52jSX2u7QKghxeoFGpLwCtGlt7j0oY9DYb5apt9nPlJ42ID+ulTXESnt/qAQcoSERyZ5sl3LDIOw0nAn/5DA==" + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==" }, "send": { "version": "0.16.2", @@ -1953,6 +2397,21 @@ "safe-buffer": "5.2.0" } }, + "shebang-command": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz", + "integrity": "sha1-RKrGW2lbAzmJaMOfNj/uXer98eo=", + "dev": true, + "requires": { + "shebang-regex": "1.0.0" + } + }, + "shebang-regex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-1.0.0.tgz", + "integrity": "sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM=", + "dev": true + }, "signal-exit": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", @@ -1996,6 +2455,12 @@ "socks": "2.2.3" } }, + "sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=", + "dev": true + }, "sqlstring": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.1.tgz", @@ -2060,18 +2525,24 @@ "ansi-regex": "2.1.1" } }, + "strip-eof": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", + "integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=", + "dev": true + }, "strip-json-comments": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=" }, "supports-color": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-3.1.2.tgz", - "integrity": "sha1-cqJiiU2dQIuVbKBf83su2KbiotU=", + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-6.0.0.tgz", + "integrity": "sha512-on9Kwidc1IUQo+bQdhi8+Tijpo0e1SS6RoGo2guUwn5vdaxw8RXOF9Vb2ws+ihWOmh4JnCJOvaziZWP1VABaLg==", "dev": true, "requires": { - "has-flag": "1.0.0" + "has-flag": "3.0.0" } }, "tar-fs": { @@ -2110,6 +2581,18 @@ "xtend": "4.0.2" } }, + "tiny-secp256k1": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/tiny-secp256k1/-/tiny-secp256k1-1.1.3.tgz", + "integrity": "sha512-ZpobrhOtHP98VYEN51IYQH1YcrbFpnxFhI6ceWa3OEbJn7eHvSd8YFjGPxbedGCy7PNYU1v/+BRsdvyr5uRd4g==", + "requires": { + "bindings": "1.5.0", + "bn.js": "4.11.8", + "create-hmac": "1.1.7", + "elliptic": "6.5.0", + "nan": "2.14.0" + } + }, "to-buffer": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/to-buffer/-/to-buffer-1.1.1.tgz", @@ -2120,7 +2603,7 @@ "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz", "integrity": "sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ==", "requires": { - "psl": "1.2.0", + "psl": "1.3.0", "punycode": "1.4.1" }, "dependencies": { @@ -2195,9 +2678,9 @@ "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" }, "uuid": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", - "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.3.tgz", + "integrity": "sha512-pW0No1RGHgzlpHJO1nsVrHKpOEIxkGg1xB+v0ZmdNH5OAeAwzAVrCnI2/6Mtx+Uys6iaylxa+D3g4j63IKKjSQ==" }, "validator": { "version": "10.8.0", @@ -2238,6 +2721,21 @@ "yaeti": "0.0.6" } }, + "which": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", + "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", + "dev": true, + "requires": { + "isexe": "2.0.0" + } + }, + "which-module": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz", + "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=", + "dev": true + }, "which-pm-runs": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/which-pm-runs/-/which-pm-runs-1.0.0.tgz", @@ -2259,6 +2757,16 @@ "bs58check": "2.1.2" } }, + "wrap-ansi": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz", + "integrity": "sha1-2Pw9KE3QV5T+hJc8rs3Rz4JP3YU=", + "dev": true, + "requires": { + "string-width": "1.0.2", + "strip-ansi": "3.0.1" + } + }, "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", @@ -2274,6 +2782,12 @@ "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" }, + "y18n": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz", + "integrity": "sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==", + "dev": true + }, "yaeti": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/yaeti/-/yaeti-0.0.6.tgz", @@ -2284,6 +2798,155 @@ "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=" }, + "yargs": { + "version": "13.2.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-13.2.2.tgz", + "integrity": "sha512-WyEoxgyTD3w5XRpAQNYUB9ycVH/PQrToaTXdYXRdOXvEy1l19br+VJsc0vcO8PTGg5ro/l/GY7F/JMEBmI0BxA==", + "dev": true, + "requires": { + "cliui": "4.1.0", + "find-up": "3.0.0", + "get-caller-file": "2.0.5", + "os-locale": "3.1.0", + "require-directory": "2.1.1", + "require-main-filename": "2.0.0", + "set-blocking": "2.0.0", + "string-width": "3.1.0", + "which-module": "2.0.0", + "y18n": "4.0.0", + "yargs-parser": "13.0.0" + }, + "dependencies": { + "ansi-regex": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", + "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "dev": true + }, + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", + "dev": true + }, + "string-width": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", + "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "dev": true, + "requires": { + "emoji-regex": "7.0.3", + "is-fullwidth-code-point": "2.0.0", + "strip-ansi": "5.2.0" + } + }, + "strip-ansi": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "dev": true, + "requires": { + "ansi-regex": "4.1.0" + } + } + } + }, + "yargs-parser": { + "version": "13.0.0", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-13.0.0.tgz", + "integrity": "sha512-w2LXjoL8oRdRQN+hOyppuXs+V/fVAYtpcrRxZuF7Kt/Oc+Jr2uAcVntaUTNT6w5ihoWfFDpNY8CPx1QskxZ/pw==", + "dev": true, + "requires": { + "camelcase": "5.3.1", + "decamelize": "1.2.0" + } + }, + "yargs-unparser": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/yargs-unparser/-/yargs-unparser-1.5.0.tgz", + "integrity": "sha512-HK25qidFTCVuj/D1VfNiEndpLIeJN78aqgR23nL3y4N0U/91cOAzqfHlF8n2BvoNDcZmJKin3ddNSvOxSr8flw==", + "dev": true, + "requires": { + "flat": "4.1.0", + "lodash": "4.17.14", + "yargs": "12.0.5" + }, + "dependencies": { + "ansi-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", + "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=", + "dev": true + }, + "get-caller-file": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-1.0.3.tgz", + "integrity": "sha512-3t6rVToeoZfYSGd8YoLFR2DJkiQrIiUrGcjvFX2mDw3bn6k2OtwHN0TNCLbBO+w8qTvimhDkv+LSscbJY1vE6w==", + "dev": true + }, + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", + "dev": true + }, + "require-main-filename": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-1.0.1.tgz", + "integrity": "sha1-l/cXtp1IeE9fUmpsWqj/3aBVpNE=", + "dev": true + }, + "string-width": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", + "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", + "dev": true, + "requires": { + "is-fullwidth-code-point": "2.0.0", + "strip-ansi": "4.0.0" + } + }, + "strip-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", + "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", + "dev": true, + "requires": { + "ansi-regex": "3.0.0" + } + }, + "yargs": { + "version": "12.0.5", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-12.0.5.tgz", + "integrity": "sha512-Lhz8TLaYnxq/2ObqHDql8dX8CJi97oHxrjUcYtzKbbykPtVW9WB+poxI+NM2UIzsMgNCZTIf0AQwsjK5yMAqZw==", + "dev": true, + "requires": { + "cliui": "4.1.0", + "decamelize": "1.2.0", + "find-up": "3.0.0", + "get-caller-file": "1.0.3", + "os-locale": "3.1.0", + "require-directory": "2.1.1", + "require-main-filename": "1.0.1", + "set-blocking": "2.0.0", + "string-width": "2.1.1", + "which-module": "2.0.0", + "y18n": "4.0.0", + "yargs-parser": "11.1.1" + } + }, + "yargs-parser": { + "version": "11.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-11.1.1.tgz", + "integrity": "sha512-C6kB/WJDiaxONLJQnF8ccx9SEeoTTLek8RVbaOIsrAUS8VrBEXfmeSnCZxygc+XC2sNMBIwOOnfcxiynjHsVSQ==", + "dev": true, + "requires": { + "camelcase": "5.3.1", + "decamelize": "1.2.0" + } + } + } + }, "zeromq": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/zeromq/-/zeromq-4.2.0.tgz", diff --git a/package.json b/package.json index c0952b3..4337281 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "samourai-dojo", - "version": "1.2.0", + "version": "1.3.0", "description": "Backend server for Samourai Wallet", "main": "accounts/index.js", "scripts": { @@ -17,7 +17,7 @@ "async-sema": "2.1.2", "bip39": "2.4.0", "bitcoind-rpc-client": "0.3.1", - "bitcoinjs-lib": "3.2.0", + "bitcoinjs-lib": "5.1.4", "bitcoinjs-message": "1.0.1", "body-parser": "1.18.3", "express": "4.16.3", @@ -37,6 +37,6 @@ "zeromq": "4.2.0" }, "devDependencies": { - "mocha": "^3.5.0" + "mocha": "^6.2.0" } } diff --git a/pushtx/index-orchestrator.js b/pushtx/index-orchestrator.js index 3123de5..0151b09 100644 --- a/pushtx/index-orchestrator.js +++ b/pushtx/index-orchestrator.js @@ -46,4 +46,7 @@ const orchestrator = new Orchestrator() orchestrator.start() -})() +})().catch(err => { + console.error(err) + process.exit(1) +}) diff --git a/pushtx/index.js b/pushtx/index.js index 865bcaa..729830b 100644 --- a/pushtx/index.js +++ b/pushtx/index.js @@ -45,8 +45,7 @@ // Initialize the http server const port = keys.ports.pushtx - const httpsOptions = keys.https.pushtx - const httpServer = new HttpServer(port, httpsOptions) + const httpServer = new HttpServer(port) // Initialize the PushTx rest api const pushtxRestApi = new PushTxRestApi(httpServer) @@ -54,4 +53,7 @@ // Start the http server httpServer.start() -})() +})().catch(err => { + console.error(err) + process.exit(1) +}) diff --git a/static/admin/lib/api-wrapper.js b/static/admin/lib/api-wrapper.js index e69eca7..8327724 100644 --- a/static/admin/lib/api-wrapper.js +++ b/static/admin/lib/api-wrapper.js @@ -133,7 +133,23 @@ var lib_api = { return this.sendGetUriEncoded(uri, {}); }, + /** + * Rescans a range of blocks + */ + getBlocksRescan: function(fromHeight, toHeight) { + let prefix = conf['prefixes']['support']; + let uri = this.baseUri + '/tracker/' + prefix + '/rescan'; + //let uri = 'http://127.0.0.1:8082/' + prefix + '/rescan'; + return this.sendGetUriEncoded( + uri, + { + 'fromHeight': fromHeight, + 'toHeight': toHeight + } + ); + }, + /** * HTTP requests methods */ diff --git a/static/admin/tool/index.html b/static/admin/tool/index.html index e38673f..350439b 100644 --- a/static/admin/tool/index.html +++ b/static/admin/tool/index.html @@ -71,7 +71,10 @@ UNSPENT + diff --git a/static/admin/tool/index.js b/static/admin/tool/index.js index 39f5b75..66f445f 100644 --- a/static/admin/tool/index.js +++ b/static/admin/tool/index.js @@ -48,6 +48,7 @@ function initTabs() { '#link-xpub', '#link-info-address', '#link-rescan-address', + '#link-rescan-blocks', '#link-multiaddr', '#link-unspent', '#link-tx' @@ -110,6 +111,12 @@ function preparePage() { placeholder = 'ENTER A BITCOIN ADDRESS'; } else if (activeTab == '#link-rescan-address') { placeholder = 'ENTER A BITCOIN ADDRESS'; + } else if (activeTab == '#link-rescan-blocks') { + $("#cell-args").removeClass('fullwidth'); + $("#cell-args").addClass('halfwidth'); + $("#cell-args2").show(); + placeholder = 'RESCAN BLOCKS FROM HEIGHT...'; + placeholder2 = '...TO HEIGHT (OPTIONAL)'; } else if (activeTab == '#link-multiaddr') { placeholder = 'ENTER /MULTIADDR URL ARGUMENTS (e.g.: active=xpub0123456789&new=address2|address3&pubkey=pubkey4)'; } else if (activeTab == '#link-unspent') { @@ -164,6 +171,10 @@ function processAction(activeTab, args, args2, args3) { return lib_api.getAddressInfo(args); } else if (activeTab == '#link-rescan-address') { return lib_api.getAddressRescan(args); + } else if (activeTab == '#link-rescan-blocks') { + const fromHeight = parseInt(args); + const toHeight = (args2) ? parseInt(args2) : fromHeight; + return lib_api.getBlocksRescan(fromHeight, toHeight); } else if (activeTab == '#link-tx') { return lib_api.getTransaction(args); } diff --git a/tracker/blockchain-processor.js b/tracker/blockchain-processor.js index a26a0cd..27caccd 100644 --- a/tracker/blockchain-processor.js +++ b/tracker/blockchain-processor.js @@ -320,6 +320,39 @@ class BlockchainProcessor extends AbstractProcessor { await db.deleteBlocksAfterHeight(height) } + /** + * Rescan a range of blocks + * @param {integer} fromHeight - height of first block + * @param {integer} toHeight - height of last block + * @returns {Promise} + */ + async rescanBlocks(fromHeight, toHeight) { + // Get highest block processed by the tracker + const highest = await db.getHighestBlock() + const dbMaxHeight = highest.blockHeight + + if (toHeight == null) + toHeight = fromHeight + + toHeight = Math.min(toHeight, dbMaxHeight) + const blockRange = _.range(fromHeight, toHeight + 1) + + Logger.info(`Blocks Rescan : starting a rescan for ${blockRange.length} blocks`) + + // Process the blocks + return util.seriesCall(blockRange, async height => { + try { + Logger.info(`Rescanning block ${height}`) + const hash = await this.client.getblockhash(height) + const header = await this.client.getblockheader(hash) + return this.processBlock(header) + } catch(e) { + Logger.error(e, 'BlockchainProcessor.rescan()') + throw e + } + }, 'Tracker rescan', true) + } + /** * Process a block * @param {object} header - block header diff --git a/tracker/index.js b/tracker/index.js index 54d2285..47d091c 100644 --- a/tracker/index.js +++ b/tracker/index.js @@ -11,7 +11,9 @@ const keys = require('../keys')[network.key] const db = require('../lib/db/mysql-db-wrapper') const Logger = require('../lib/logger') + const HttpServer = require('../lib/http-server/http-server') const Tracker = require('./tracker') + const TrackerRestApi = require('./tracker-rest-api') Logger.info('Process ID: ' + process.pid) @@ -33,8 +35,23 @@ db.connect(dbConfig) - // Start the tracker + // Initialize the tracker const tracker = new Tracker() + + // Initialize the http server + const port = keys.ports.trackerApi + const httpServer = new HttpServer(port) + + // Initialize the rest api endpoints + const trackerRestApi = new TrackerRestApi(httpServer, tracker) + + // Start the http server + httpServer.start() + + // Start the tracker tracker.start() -})() +})().catch(err => { + console.error(err) + process.exit(1) +}) diff --git a/tracker/tracker-rest-api.js b/tracker/tracker-rest-api.js new file mode 100644 index 0000000..85d7c5f --- /dev/null +++ b/tracker/tracker-rest-api.js @@ -0,0 +1,79 @@ +/*! + * tracker/tracker-rest-api.js + * Copyright (c) 2016-2019, Samourai Wallet (CC BY-NC-ND 4.0 License). + */ +'use strict' + +const qs = require('querystring') +const validator = require('validator') +const bodyParser = require('body-parser') +const Logger = require('../lib/logger') +const errors = require('../lib/errors') +const authMgr = require('../lib/auth/authorizations-manager') +const HttpServer = require('../lib/http-server/http-server') +const network = require('../lib/bitcoin/network') +const keys = require('../keys')[network.key] + + +/** + * Tracker API endpoints + */ +class TrackerRestApi { + + /** + * Constructor + * @param {pushtx.HttpServer} httpServer - HTTP server + * @param {tracker.Tracker} tracker - tracker + */ + constructor(httpServer, tracker) { + this.httpServer = httpServer + this.tracker = tracker + + const urlencodedParser = bodyParser.urlencoded({ extended: true }) + + // Establish routes. Proxy server strips /pushtx + this.httpServer.app.get( + `/${keys.prefixes.support}/rescan`, + authMgr.checkHasAdminProfile.bind(authMgr), + this.getBlocksRescan.bind(this), + HttpServer.sendAuthError + ) + } + + /** + * Rescan a range of blocks + */ + async getBlocksRescan(req, res) { + // Check request arguments + if (!req.query) + return HttpServer.sendError(res, errors.body.INVDATA) + + if (!req.query.fromHeight || !validator.isInt(req.query.fromHeight)) + return HttpServer.sendError(res, errors.body.INVDATA) + + if (req.query.toHeight && !validator.isInt(req.query.toHeight)) + return HttpServer.sendError(res, errors.body.INVDATA) + + // Retrieve the request arguments + const fromHeight = parseInt(req.query.fromHeight) + const toHeight = req.query.toHeight ? parseInt(req.query.toHeight) : fromHeight + + if (req.query.toHeight && (toHeight < fromHeight)) + return HttpServer.sendError(res, errors.body.INVDATA) + + try { + await this.tracker.blockchainProcessor.rescanBlocks(fromHeight, toHeight) + const ret = { + status: 'Rescan complete', + fromHeight: fromHeight, + toHeight: toHeight + } + HttpServer.sendRawData(res, JSON.stringify(ret, null, 2)) + } catch(e) { + return HttpServer.sendError(res, e) + } + } + +} + +module.exports = TrackerRestApi