From 758f25416d6757509af58b57f9a3283a8d4aee75 Mon Sep 17 00:00:00 2001 From: kenshin-samourai Date: Thu, 16 Jul 2020 17:01:04 +0200 Subject: [PATCH 01/36] prepare next iteration --- docker/my-dojo/.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/my-dojo/.env b/docker/my-dojo/.env index 5352e51..56f5196 100644 --- a/docker/my-dojo/.env +++ b/docker/my-dojo/.env @@ -10,7 +10,7 @@ COMPOSE_CONVERT_WINDOWS_PATHS=1 -DOJO_VERSION_TAG=1.7.0 +DOJO_VERSION_TAG=1.8.0 DOJO_DB_VERSION_TAG=1.2.0 DOJO_BITCOIND_VERSION_TAG=1.6.0 DOJO_NODEJS_VERSION_TAG=1.7.0 From f64325624d63e71c5e41e3ec5b55fbc2a997ce56 Mon Sep 17 00:00:00 2001 From: kenshin-samourai Date: Thu, 16 Jul 2020 17:06:05 +0200 Subject: [PATCH 02/36] bump version of nodejs app and container --- docker/my-dojo/.env | 2 +- package-lock.json | 2 +- package.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/my-dojo/.env b/docker/my-dojo/.env index 56f5196..3284196 100644 --- a/docker/my-dojo/.env +++ b/docker/my-dojo/.env @@ -13,7 +13,7 @@ COMPOSE_CONVERT_WINDOWS_PATHS=1 DOJO_VERSION_TAG=1.8.0 DOJO_DB_VERSION_TAG=1.2.0 DOJO_BITCOIND_VERSION_TAG=1.6.0 -DOJO_NODEJS_VERSION_TAG=1.7.0 +DOJO_NODEJS_VERSION_TAG=1.8.0 DOJO_NGINX_VERSION_TAG=1.5.0 DOJO_TOR_VERSION_TAG=1.4.0 DOJO_EXPLORER_VERSION_TAG=1.3.0 diff --git a/package-lock.json b/package-lock.json index 2a84bca..302d4e2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "samourai-dojo", - "version": "1.7.0", + "version": "1.8.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 9ea48f6..9b22210 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "samourai-dojo", - "version": "1.7.0", + "version": "1.8.0", "description": "Backend server for Samourai Wallet", "main": "accounts/index.js", "scripts": { From 800ea31e9c44c6e32d102493541e1f0f1e20b09c Mon Sep 17 00:00:00 2001 From: kenshin-samourai Date: Thu, 16 Jul 2020 17:43:14 +0200 Subject: [PATCH 03/36] add new /wallet endpoint and deprecate /multiaddr et /unspent --- accounts/multiaddr-rest-api.js | 1 + accounts/unspent-rest-api.js | 1 + accounts/wallet-rest-api..js | 136 +++++++++++++++++++++++++++++++++ lib/wallet/wallet-service.js | 88 ++++++++++++++++++++- 4 files changed, 225 insertions(+), 1 deletion(-) create mode 100644 accounts/wallet-rest-api..js diff --git a/accounts/multiaddr-rest-api.js b/accounts/multiaddr-rest-api.js index 3399e23..3bc7aaf 100644 --- a/accounts/multiaddr-rest-api.js +++ b/accounts/multiaddr-rest-api.js @@ -17,6 +17,7 @@ const debugApi = !!(process.argv.indexOf('api-debug') > -1) /** * Multiaddr API endpoints + * @deprecated */ class MultiaddrRestApi { diff --git a/accounts/unspent-rest-api.js b/accounts/unspent-rest-api.js index 617fb44..baf1779 100644 --- a/accounts/unspent-rest-api.js +++ b/accounts/unspent-rest-api.js @@ -17,6 +17,7 @@ const debugApi = !!(process.argv.indexOf('api-debug') > -1) /** * Unspent API endpoints + * @deprecated */ class UnspentRestApi { diff --git a/accounts/wallet-rest-api..js b/accounts/wallet-rest-api..js new file mode 100644 index 0000000..96ab5e1 --- /dev/null +++ b/accounts/wallet-rest-api..js @@ -0,0 +1,136 @@ +/*! + * accounts/wallet-rest-api.js + * Copyright © 2019 – Katana Cryptographic Ltd. All Rights Reserved. + */ +'use strict' + +const bodyParser = require('body-parser') +const Logger = require('../lib/logger') +const errors = require('../lib/errors') +const walletService = require('../lib/wallet/wallet-service') +const authMgr = require('../lib/auth/authorizations-manager') +const HttpServer = require('../lib/http-server/http-server') +const apiHelper = require('./api-helper') + +const debugApi = !!(process.argv.indexOf('api-debug') > -1) + + +/** + * Wallet API endpoints + */ +class WalletRestApi { + + /** + * Constructor + * @param {pushtx.HttpServer} httpServer - HTTP server + */ + constructor(httpServer) { + this.httpServer = httpServer + + // Establish routes + const urlencodedParser = bodyParser.urlencoded({ extended: true }) + + this.httpServer.app.get( + '/wallet', + authMgr.checkAuthentication.bind(authMgr), + apiHelper.validateEntitiesParams.bind(apiHelper), + this.getWallet.bind(this), + HttpServer.sendAuthError + ) + + this.httpServer.app.post( + '/wallet', + urlencodedParser, + authMgr.checkAuthentication.bind(authMgr), + apiHelper.validateEntitiesParams.bind(apiHelper), + this.postWallet.bind(this), + HttpServer.sendAuthError + ) + } + + /** + * Handle wallet GET request + * @param {object} req - http request object + * @param {object} res - http response object + */ + async getWallet(req, res) { + try { + // Check request params + if (!apiHelper.checkEntitiesParams(req.query)) + return HttpServer.sendError(res, errors.multiaddr.NOACT) + + // Parse params + const entities = apiHelper.parseEntitiesParams(req.query) + + const result = await walletService.getFullWalletInfo( + entities.active, + entities.legacy, + entities.bip49, + entities.bip84, + entities.pubkey + ) + + const ret = JSON.stringify(result, null, 2) + HttpServer.sendRawData(res, ret) + + } catch(e) { + HttpServer.sendError(res, e) + + } finally { + if (debugApi) { + const strParams = + `${req.query.active ? req.query.active : ''} \ + ${req.query.new ? req.query.new : ''} \ + ${req.query.pubkey ? req.query.pubkey : ''} \ + ${req.query.bip49 ? req.query.bip49 : ''} \ + ${req.query.bip84 ? req.query.bip84 : ''}` + + Logger.info(`API : Completed GET /wallet ${strParams}`) + } + } + } + + /** + * Handle wallet POST request + * @param {object} req - http request object + * @param {object} res - http response object + */ + async postWallet(req, res) { + try { + // Check request params + if (!apiHelper.checkEntitiesParams(req.body)) + return HttpServer.sendError(res, errors.multiaddr.NOACT) + + // Parse params + const entities = apiHelper.parseEntitiesParams(req.body) + + const result = await walletService.getFullWalletInfo( + entities.active, + entities.legacy, + entities.bip49, + entities.bip84, + entities.pubkey + ) + + HttpServer.sendOkDataOnly(res, result) + + } catch(e) { + HttpServer.sendError(res, e) + + } finally { + if (debugApi) { + const strParams = + `${req.body.active ? req.body.active : ''} \ + ${req.body.new ? req.body.new : ''} \ + ${req.body.pubkey ? req.body.pubkey : ''} \ + ${req.body.bip49 ? req.body.bip49 : ''} \ + ${req.body.bip84 ? req.body.bip84 : ''}` + + Logger.info(`API : Completed POST /wallet ${strParams}`) + } + } + } + +} + +module.exports = WalletRestApi diff --git a/lib/wallet/wallet-service.js b/lib/wallet/wallet-service.js index 78f75be..64ca4bc 100644 --- a/lib/wallet/wallet-service.js +++ b/lib/wallet/wallet-service.js @@ -25,8 +25,73 @@ class WalletService { */ constructor() {} + /** + * Get full wallet information + * @param {object} active - mapping of active entities + * @param {object} legacy - mapping of new legacy addresses + * @param {object} bip49 - mapping of new bip49 addresses + * @param {object} bip84 - mapping of new bip84 addresses + * @param {object} pubkeys - mapping of new pubkeys/addresses + * @returns {Promise} + */ + async getFullWalletInfo(active, legacy, bip49, bip84, pubkeys) { + // Check parameters + const validParams = this._checkEntities(active, legacy, bip49, bip84, pubkeys) + + if (!validParams) { + const info = new WalletInfo() + const ret = this._formatGetFullWalletInfoResult(info) + return Promise.resolve(ret) + } + + // Merge all entities into active mapping + active = this._mergeEntities(active, legacy, bip49, bip84, pubkeys) + + // Initialize a WalletInfo object + const walletInfo = new WalletInfo(active) + + try { + // Add the new xpubs + await util.seriesCall(legacy.xpubs, this._newBIP44) + await util.seriesCall(bip49.xpubs, this._newBIP49) + await util.seriesCall(bip84.xpubs, this._newBIP84) + // Load hd accounts info + await walletInfo.ensureHdAccounts() + await walletInfo.loadHdAccountsInfo() + // Add the new addresses + await db.addAddresses(legacy.addrs) + await db.addAddresses(bip49.addrs) + await db.addAddresses(bip84.addrs) + await db.addAddresses(pubkeys.addrs) + // Ensure addresses exist + await walletInfo.ensureAddresses() + // Force import of addresses associated to paynyms + // if dojo relies on a local index + if (keys.indexer.active != 'third_party_explorer') + await this._forceEnsureAddressesForActivePubkeys(active) + // Filter the addresses + await walletInfo.filterAddresses() + // Load the utxos + await walletInfo.loadUtxos() + // Load the addresses + await walletInfo.loadAddressesInfo() + // Load the most recent transactions + await walletInfo.loadTransactions(0, null, true) + // Postprocessing + await walletInfo.postProcessAddresses() + await walletInfo.postProcessHdAccounts() + // Format the result + return this._formatGetFullWalletInfoResult(walletInfo) + + } catch(e) { + Logger.error(e, 'WalletService.getWalletInfo()') + return Promise.reject({status:'error', error:'internal server error'}) + } + } + /** * Get wallet information + * @deprecated * @param {object} active - mapping of active entities * @param {object} legacy - mapping of new legacy addresses * @param {object} bip49 - mapping of new bip49 addresses @@ -86,8 +151,28 @@ class WalletService { } } + /** + * Prepares the result to be returned by getFullWalletInfo() + * @param {WalletInfo} info + * @returns {object} + */ + _formatGetFullWalletInfoResult(info) { + let ret = info.toPojo() + + delete ret['n_tx'] + + ret.addresses = ret.addresses.map(x => { + delete x['derivation'] + delete x['created'] + return x + }) + + return ret + } + /** * Prepares the result to be returned by getWalletInfo() + * @deprecated * @param {WalletInfo} info * @returns {object} */ @@ -108,6 +193,7 @@ class WalletService { /** * Get wallet unspent outputs + * @deprecated * @param {object} active - mapping of active entities * @param {object} legacy - mapping of new legacy addresses * @param {object} bip49 - mapping of new bip49 addresses @@ -167,7 +253,7 @@ class WalletService { } /** - * Get a subset of wallet transaction + * Get a subset of wallet transactions * @param {object} entities - mapping of active entities * @param {integer} page - page of transactions to be returned * @param {integer} count - number of transactions returned per page From 76032a6fcbb9667f23e2f1107c3a226b83a44bf5 Mon Sep 17 00:00:00 2001 From: kenshin-samourai Date: Thu, 16 Jul 2020 17:43:41 +0200 Subject: [PATCH 04/36] update doc for /multiaddr, /unspent and /wallet endpoints --- doc/GET_multiaddr.md | 2 + doc/GET_unspent.md | 2 + doc/GET_wallet.md | 158 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 162 insertions(+) create mode 100644 doc/GET_wallet.md diff --git a/doc/GET_multiaddr.md b/doc/GET_multiaddr.md index 49b1956..c22a800 100644 --- a/doc/GET_multiaddr.md +++ b/doc/GET_multiaddr.md @@ -1,5 +1,7 @@ # Get Multiaddr +Note: Starting with Dojo 1.8.0, this API endpoint is deprecated. See the new [/wallet endpoint](./GET_wallet.md) + Request details about a collection of HD accounts and/or loose addresses and/or pubkeys (derived in 3 formats P2PKH, P2WPKH/P2SH, P2WPKH Bech32). diff --git a/doc/GET_unspent.md b/doc/GET_unspent.md index b4fcd9f..26d70e2 100644 --- a/doc/GET_unspent.md +++ b/doc/GET_unspent.md @@ -1,5 +1,7 @@ # Get Unspent +Note: Starting with Dojo 1.8.0, this API endpoint is deprecated. See the new [/wallet endpoint](./GET_wallet.md) + Request a list of unspent transaction outputs from a collection of HD accounts and/or loose addresses and/or pubkeys (derived in 3 formats P2PKH, P2WPKH/P2SH, P2WPKH Bech32). diff --git a/doc/GET_wallet.md b/doc/GET_wallet.md new file mode 100644 index 0000000..1530435 --- /dev/null +++ b/doc/GET_wallet.md @@ -0,0 +1,158 @@ +# Get Wallet + +Request details about a collection of HD accounts and/or loose addresses and/or pubkeys (derived in 3 formats P2PKH, P2WPKH/P2SH, P2WPKH Bech32) including a list of unspent transaction outputs. + +This endpoint merges the deprecated /multiaddr and /unspent endpoints + + +## Behavior of the active parameter + +If accounts passed to `?active` do not exist, they will be created with a relayed call to the [POST /xpub](./POST_xpub.md) mechanics if new or will be imported from external data sources. + +If loose addresses passed to `?active` do not exist, they will be imported from external data sources. + +If addresses derived from pubkeys passed to `?active` do not exist, they will be imported from external data sources. + + +## Declaration of new entities + +Instruct the server that [BIP44](https://github.com/bitcoin/bips/blob/master/bip-0044.mediawiki) entities are new with `?new=xpub1|addr2|addr3` in the query parameters, and the server will skip importing for those entities. + +SegWit support via [BIP49](https://github.com/bitcoin/bips/blob/master/bip-0049.mediawiki) is activated for new ypubs and new P2WPKH/P2SH loose addresses with `?bip49=xpub3|xpub4`. + +SegWit support via [BIP84](https://github.com/bitcoin/bips/blob/master/bip-0084.mediawiki) is activated for new zpubs and new P2WPKH Bech32 loose addresses with `?bip84=xpub3|xpub4`. + +Support of [BIP47](https://github.com/bitcoin/bips/blob/master/bip-0047.mediawiki) with addresses derived in 3 formats (P2PKH, P2WPKH/P2SH, P2WPKH Bech32) is activated for new pubkeys with `?pubkey=pubkey1|pubkey2`. + + +Note that loose addresses that are also part of one of the HD accounts requested will be ignored. Their balances and transactions are listed as part of the HD account result. + +The `POST` version of `/wallet` is identical, except the parameters are in the POST body. + + +``` +GET /wallet?active=...[&new=...][&bip49=...][&bip84=...][&pubkey=...] +``` + +## Parameters +* **active** - `string` - A pipe-separated list of extended public keys and/or loose addresses and/or pubkeys (`xpub1|address1|address2|pubkey1|...`) +* **new** - `string` - A pipe-separated list of **new** extended public keys to be derived via [BIP44](https://github.com/bitcoin/bips/blob/master/bip-0044.mediawiki) and/or new P2PKH loose addresses +* **bip49** - `string` - A pipe-separated list of **new** extended public keys to be derived via [BIP49](https://github.com/bitcoin/bips/blob/master/bip-0049.mediawiki) and/or new P2WPKH/P2SH loose addresses +* **bip84** - `string` - A pipe-separated list of **new** extended public keys to be derived via [BIP84](https://github.com/bitcoin/bips/blob/master/bip-0084.mediawiki) and/or new P2WPKH Bech32 loose addresses +* **pubkey** - `string` - A pipe-separated list of **new** public keys to be derived as P2PKH, P2WPKH/P2SH, P2WPKH Bech32 addresses +* **at** - `string` (optional) - Access Token (json web token). Required if authentication is activated. Alternatively, the access token can be passed through the `Authorization` HTTP header (with the `Bearer` scheme). + +### Examples + +``` +GET /wallet?active=xpub0123456789&new=address2|address3&pubkey=pubkey4 +GET /wallet?active=xpub0123456789|address1|address2 +GET /wallet?bip49=xpub0123456789 +GET /wallet?bip84=xpub0123456789 +GET /wallet?pubkey=0312345678901 +``` + +#### Success +Status code 200 with JSON response: +```json +{ + "wallet": { + "final_balance": 100000000 + }, + "info": { + "latest_block": { + "height": 100000, + "hash": "abcdef", + "time": 1000000000 + } + }, + "addresses": [ + { + "address": "xpubABCDEF -or- 1xAddress", + "pubkey": "04Pubkey -or- inexistant attribute" + "final_balance": 100000000, + "account_index": 0, + "change_index": 0, + "n_tx": 0 + } + ], + "txs": [ + { + "block_height": 100000, + "hash": "abcdef", + "version": 1, + "locktime": 0, + "result": -10000, + "balance": 90000, + "time": 1400000000, + "inputs": [ + { + "vin": 1, + "prev_out": { + "txid": "abcdef", + "vout": 2, + "value": 20000, + "xpub": { + "m": "xpubABCDEF", + "path": "M/0/3" + }, + "addr": "1xAddress", + "pubkey": "04Pubkey" + }, + "sequence": 4294967295 + } + ], + "out": [ + { + "n": 2, + "value": 10000, + "addr": "1xAddress", + "pubkey": "03Pubkey" + "xpub": { + "m": "xpubABCDEF", + "path": "M/1/5" + } + } + ] + } + ], + "unspent_outputs": [ + { + "tx_hash": "abcdef", + "tx_output_n": 2, + "tx_version": 1, + "tx_locktime": 0, + "value": 10000, + "script": "abcdef", + "addr": "1xAddress", + "pubkey": "03Pubkey -or- inexistant attribute" + "confirmations": 10000, + "xpub": { + "m": "xpubABCDEF", + "path": "M/1/5" + } + } + ] +} +``` + +**Notes** +* The transaction `inputs` and `out` arrays are for known addresses only and do not reflect the full input and output list of the transaction on the blockchain +* `result.addresses[i].n_tx` used by BIP47 logic to detemine unused index +* `result.txs[i].block_height` should not be present for unconfirmed transactions +* `result.txs[i].result` is the change in value for the "wallet" as defined by all entries on the `active` query parameter +* `result.txs[i].inputs[j].prev_out.addr` should be present for BIP47-related addresses but may be `null` if the previous output address is unknown +* `result.txs[i].out[j].addr` should be present for BIP47-related addresses + + +#### Failure +Status code 400 with JSON response: +```json +{ + "status": "error", + "error": "" +} +``` + +## Notes +Wallet response is consumed by the wallet in the [APIFactory](https://code.samourai.io/wallet/samourai-wallet-android/-/blob/master/app/src/main/java/com/samourai/wallet/api/APIFactory.java) From 8cfb520d327b12232e9df732b4b7d37e79ea1829 Mon Sep 17 00:00:00 2001 From: kenshin-samourai Date: Thu, 16 Jul 2020 19:03:56 +0200 Subject: [PATCH 05/36] replace multiaddr & unspent tabs by wallet tab in DMT --- static/admin/lib/api-wrapper.js | 14 +++----------- static/admin/tool/index.html | 7 ++----- static/admin/tool/index.js | 15 +++++---------- 3 files changed, 10 insertions(+), 26 deletions(-) diff --git a/static/admin/lib/api-wrapper.js b/static/admin/lib/api-wrapper.js index 90ee2f9..e89133f 100644 --- a/static/admin/lib/api-wrapper.js +++ b/static/admin/lib/api-wrapper.js @@ -119,18 +119,10 @@ var lib_api = { }, /** - * Multiaddr + * Wallet */ - getMultiaddr: function(arguments) { - let uri = this.baseUri + '/multiaddr'; - return this.sendGetUriEncoded(uri, arguments); - }, - - /** - * Unspent - */ - getUnspent: function(arguments) { - let uri = this.baseUri + '/unspent'; + getWallet: function(arguments) { + let uri = this.baseUri + '/wallet'; return this.sendGetUriEncoded(uri, arguments); }, diff --git a/static/admin/tool/index.html b/static/admin/tool/index.html index 6d94410..64ff4aa 100644 --- a/static/admin/tool/index.html +++ b/static/admin/tool/index.html @@ -64,11 +64,8 @@ - -