diff --git a/lib/remote-importer/btccom-wrapper.js b/lib/remote-importer/btccom-wrapper.js index cc86d38..767f4b5 100644 --- a/lib/remote-importer/btccom-wrapper.js +++ b/lib/remote-importer/btccom-wrapper.js @@ -66,11 +66,11 @@ class BtcComWrapper extends Wrapper { * { address: , txids: , ntx: } */ async getAddress(address, filterAddr) { - // Extracts the scripthash from the bech32 address - // (btc.com api manages scripthashes, not bech32 addresses) - const scripthash = addrHelper.getScriptHashFromBech32(address) + const reqAddr = addrHelper.isBech32(address) + ? addrHelper.getScriptHashFromBech32(address) + : address - const uri = `/address/${scripthash}` + const uri = `/address/${reqAddr}` const result = await this._get(uri) const ret = { @@ -92,7 +92,7 @@ class BtcComWrapper extends Wrapper { const listPages = Array.from(aPages, (val, idx) => idx + 1) const results = await util.seriesCall(listPages, idx => { - return this._getTxsForAddress(scripthash, idx) + return this._getTxsForAddress(reqAddr, idx) }) for (let txids of results) @@ -103,15 +103,72 @@ class BtcComWrapper extends Wrapper { /** * Retrieve information for a given list of addresses - * @param {string} addresses - array of bitcoin addresses + * @param {string[]} addresses - array of bitcoin addresses * @param {boolean} filterAddr - True if an upper bound should be used * for #transactions associated to the address, False otherwise * @returns {Promise} returns an array of objects * { address: , txids: , ntx: } */ async getAddresses(addresses, filterAddr) { - // Not implemented for this api - throw "Not implemented" + const ret = [] + const reqAddresses = [] + const xlatedBech32Addr = {} + + for (let a of addresses) { + if (addrHelper.isBech32(a)) { + const scriptHash = addrHelper.getScriptHashFromBech32(a) + reqAddresses.push(scriptHash) + xlatedBech32Addr[scriptHash] = a + } else { + reqAddresses.push(a) + } + } + + // Send a batch request for all the addresses + const strReqAddresses = reqAddresses.join(',') + const uri = `/address/${strReqAddresses}` + const results = await this._get(uri) + + const foundAddresses = Array.isArray(results.data) + ? results.data + : [results.data] + + for (let a of foundAddresses) { + if (a && a.tx_count > 0) { + // Translate bech32 address + const address = xlatedBech32Addr.hasOwnProperty(a.address) + ? xlatedBech32Addr[a.address] + : a.address + + if (a.tx_count <= 2) { + // Less than 3 transactions for this address + // all good + const retAddr = { + address: address, + ntx: a.tx_count, + txids: [] + } + + retAddr.txids = (a.tx_count == 1) + ? [a.first_tx] + : [a.first_tx, a.last_tx] + + ret.push(retAddr) + + } else { + // More than 2 transactions for this address + // We need more requests to the API + if (filterAddr && a.tx_count > keys.addrFilterThreshold) { + Logger.info(` import of ${address} rejected (too many transactions - ${a.tx_count})`) + } else { + const retAddr = await this.getAddress(address) + ret.push(retAddr) + } + } + } + } + + return ret } } diff --git a/lib/remote-importer/sources-mainnet.js b/lib/remote-importer/sources-mainnet.js index 64bb932..6bc1121 100644 --- a/lib/remote-importer/sources-mainnet.js +++ b/lib/remote-importer/sources-mainnet.js @@ -4,9 +4,7 @@ */ 'use strict' -const addrHelper = require('../bitcoin/addresses-helper') const network = require('../bitcoin/network') -const util = require('../util') const Logger = require('../logger') const keys = require('../../keys')[network.key] const Sources = require('./sources') @@ -24,8 +22,6 @@ class SourcesMainnet extends Sources { */ constructor() { super() - // Initializes external source - this.source = null this._initSource() } @@ -45,68 +41,6 @@ class SourcesMainnet extends Sources { } } - /** - * Retrieve information for a given address - * @param {string} address - bitcoin address - * @param {boolean} filterAddr - True if an upper bound should be used - * for #transactions associated to the address, False otherwise - * @returns {Promise} returns an object - * { address: , txids: , ntx: } - */ - async getAddress(address, filterAddr) { - const ret = { - address, - txids: [], - ntx: 0 - } - - try { - const result = await this.source.getAddress(address, filterAddr) - - if (result.ntx) - ret.ntx = result.ntx - else if (result.txids) - ret.ntx = result.txids.length - - if (result.txids) - ret.txids = result.txids - - } catch(e) { - //Logger.error(e, `SourcesMainnet.getAddress() : ${address} from ${this.source.base}`) - Logger.error(null, `SourcesMainnet.getAddress() : ${address} from ${this.source.base}`) - } finally { - return ret - } - } - - /** - * Retrieve information for a list of addresses - * @param {string[]} addresses - array of bitcoin address - * @param {boolean} filterAddr - True if an upper bound should be used - * for #transactions associated to the address, False otherwise - * @returns {Promise} returns an object - * { address: , txids: , ntx: } - */ - async getAddresses(addresses, filterAddr) { - const ret = [] - - try { - const results = await this.source.getAddresses(addresses, filterAddr) - - for (let r of results) { - // Filter addresses with too many txs - if (!filterAddr || (r.ntx <= keys.addrFilterThreshold)) - ret.push(r) - } - - } catch(e) { - //Logger.error(e, `SourcesMainnet.getAddresses() : ${addresses} from ${this.source.base}`) - Logger.error(null, `SourcesMainnet.getAddresses() : ${addresses} from ${this.source.base}`) - } finally { - return ret - } - } - } module.exports = SourcesMainnet diff --git a/lib/remote-importer/sources-testnet.js b/lib/remote-importer/sources-testnet.js index 2dfbd01..588f9e0 100644 --- a/lib/remote-importer/sources-testnet.js +++ b/lib/remote-importer/sources-testnet.js @@ -4,19 +4,17 @@ */ 'use strict' -const addrHelper = require('../bitcoin/addresses-helper') const network = require('../bitcoin/network') const util = require('../util') const Logger = require('../logger') const keys = require('../../keys')[network.key] const Sources = require('./sources') const BitcoindWrapper = require('./bitcoind-wrapper') -const InsightWrapper = require('./insight-wrapper') const BtcComWrapper = require('./btccom-wrapper') /** - * Remote data sources for testnet polled round-robin to spread load + * Remote data sources for testnet */ class SourcesTestnet extends Sources { @@ -25,126 +23,22 @@ class SourcesTestnet extends Sources { */ constructor() { super() - this.sources = [] - this.index = 0 - this.sourceBech32 = null - this.isBitcoindActive = false - // Initializes external sources - this._initSources() + this._initSource() } /** - * Initialize the external data sources + * Initialize the external data source */ - _initSources() { + _initSource() { if (keys.explorers.bitcoind == 'active') { // If local bitcoind option is activated // we'll use the local node as our unique source - this.sourceBech32 = new BitcoindWrapper() - this.sources.push(this.sourceBech32) - this.isBitcoindActive = true + this.source = new BitcoindWrapper() + Logger.info('Activated Bitcoind as the data source for imports') } else { - // Otherwise, we use a set of insight servers + btc.com for bech32 addresses - this.sourceBech32 = new BtcComWrapper(keys.explorers.btccom) - for (let url of keys.explorers.insight) - this.sources.push(new InsightWrapper(url)) - this.isBitcoindActive = false - } - } - - /** - * Get the next source index - * @returns {integer} returns the next source index - */ - nextIndex() { - this.index++ - if (this.index >= this.sources.length) - this.index = 0 - return this.index - } - - /** - * Retrieve information for a given address - * @param {string} address - bitcoin address - * @param {boolean} filterAddr - True if an upper bound should be used - * for #transactions associated to the address, False otherwise - * @returns {Promise} returns an object - * { address: , txids: , ntx: } - */ - async getAddress(address, filterAddr) { - let source = '' - - const isBech32 = addrHelper.isBech32(address) - - const ret = { - address, - txids: [], - ntx: 0 - } - - try { - source = isBech32 ? this.sourceBech32 : this.sources[this.nextIndex()] - const result = await source.getAddress(address, filterAddr) - - if (result.ntx) - ret.ntx = result.ntx - else if (result.txids) - ret.ntx = result.txids.length - - if (result.txids) - ret.txids = result.txids - - return ret - - } catch(e) { - Logger.error(e, `SourcesTestnet.getAddress() : ${address} from ${source.base}`) - if (!isBech32 && this.sources.length > 1) { - // Try again with another source - return this.getAddress(address, filterAddr) - } else { - return ret - } - } - } - - /** - * Retrieve information for a list of addresses - * @param {string[]} addresses - array of bitcoin address - * @param {boolean} filterAddr - True if an upper bound should be used - * for #transactions associated to the address, False otherwise - * @returns {Promise} returns an object - * { address: , txids: , ntx: } - */ - async getAddresses(addresses, filterAddr) { - const ret = [] - - try { - if (this.isBitcoindActive) { - const source = this.sources[0] - const results = await source.getAddresses(addresses, filterAddr) - for (let r of results) { - // Filter addresses with too many txs - if (!filterAddr || (r.ntx <= keys.addrFilterThreshold)) - ret.push(r) - } - } else { - const lists = util.splitList(addresses, this.sources.length) - await util.seriesCall(lists, async list => { - const results = await Promise.all(list.map(a => { - return this.getAddress(a, filterAddr) - })) - - for (let r of results) { - // Filter addresses with too many txs - if (!filterAddr || (r.ntx <= keys.addrFilterThreshold)) - ret.push(r) - } - }) - } - } catch (e) { - Logger.error(e, `SourcesTestnet.getAddresses() : Addr list = ${addresses}`) - } finally { - return ret + // Otherwise, we'll use the rest api provided by OXT + this.source = new BtcComWrapper(keys.explorers.btccom) + Logger.info('Activated BTC.COM API as the data source for imports') } } diff --git a/lib/remote-importer/sources.js b/lib/remote-importer/sources.js index f070c8a..9f2d7d6 100644 --- a/lib/remote-importer/sources.js +++ b/lib/remote-importer/sources.js @@ -4,16 +4,22 @@ */ 'use strict' +const network = require('../bitcoin/network') +const Logger = require('../logger') +const keys = require('../../keys')[network.key] + /** - * Abstract class defining a list of blockchain explorer providing a remote API + * Base class defining data source for imports/rescans of HD accounts and addresses */ class Sources { /** * Constructor */ - constructor() {} + constructor() { + this.source = null + } /** * Retrieve information for a given address @@ -23,7 +29,30 @@ class Sources { * @returns {Promise} returns an object * { address: , txids: , ntx: } */ - async getAddress(address, filterAddr) {} + async getAddress(address, filterAddr) { + const ret = { + address, + txids: [], + ntx: 0 + } + + try { + const result = await this.source.getAddress(address, filterAddr) + + if (result.ntx) + ret.ntx = result.ntx + else if (result.txids) + ret.ntx = result.txids.length + + if (result.txids) + ret.txids = result.txids + + } catch(e) { + Logger.error(null, `Sources.getAddress() : ${address} from ${this.source.base}`) + } finally { + return ret + } + } /** * Retrieve information for a list of addresses @@ -33,7 +62,24 @@ class Sources { * @returns {Promise} returns an object * { address: , txids: , ntx: } */ - async getAddresses(addresses, filterAddr) {} + async getAddresses(addresses, filterAddr) { + const ret = [] + + try { + const results = await this.source.getAddresses(addresses, filterAddr) + + for (let r of results) { + // Filter addresses with too many txs + if (!filterAddr || (r.ntx <= keys.addrFilterThreshold)) + ret.push(r) + } + + } catch(e) { + Logger.error(e, `Sources.getAddresses() : ${addresses} from ${this.source.base}`) + } finally { + return ret + } + } }