rework RemoteImporter

lib/remote-importer/btccom-wrapper.js

@@ -66,11 +66,11 @@ class BtcComWrapper extends Wrapper {
    *  { address: <bitcoin_address>, txids: <txids>, ntx: <total_nb_txs>}
    */
   async getAddress(address, filterAddr) {
-    // Extracts the scripthash from the bech32 address
-    // (btc.com api manages scripthashes, not bech32 addresses)
-    const scripthash = addrHelper.getScriptHashFromBech32(address)
+    const reqAddr = addrHelper.isBech32(address)
+      ? addrHelper.getScriptHashFromBech32(address)
+      : address
 
-    const uri = `/address/${scripthash}`
+    const uri = `/address/${reqAddr}`
     const result = await this._get(uri)
 
     const ret = {
@@ -92,7 +92,7 @@ class BtcComWrapper extends Wrapper {
     const listPages = Array.from(aPages, (val, idx) => idx + 1)
 
     const results = await util.seriesCall(listPages, idx => {
-      return this._getTxsForAddress(scripthash, idx)
+      return this._getTxsForAddress(reqAddr, idx)
     })
 
     for (let txids of results)
@@ -103,15 +103,72 @@ class BtcComWrapper extends Wrapper {
 
   /**
    * Retrieve information for a given list of addresses
-   * @param {string} addresses - array of bitcoin addresses
+   * @param {string[]} addresses - array of bitcoin addresses
    * @param {boolean} filterAddr - True if an upper bound should be used
    *     for #transactions associated to the address, False otherwise
    * @returns {Promise} returns an array of objects
    *  { address: <bitcoin_address>, txids: <txids>, ntx: <total_nb_txs>}
    */
   async getAddresses(addresses, filterAddr) {
-    // Not implemented for this api
-    throw "Not implemented"
+    const ret = []
+    const reqAddresses = []
+    const xlatedBech32Addr = {}
+
+    for (let a of addresses) {
+      if (addrHelper.isBech32(a)) {
+        const scriptHash = addrHelper.getScriptHashFromBech32(a)
+        reqAddresses.push(scriptHash)
+        xlatedBech32Addr[scriptHash] = a
+      } else {
+        reqAddresses.push(a)
+      }
+    }
+
+    // Send a batch request for all the addresses
+    const strReqAddresses = reqAddresses.join(',')
+    const uri = `/address/${strReqAddresses}`
+    const results = await this._get(uri)
+
+    const foundAddresses = Array.isArray(results.data) 
+      ? results.data 
+      : [results.data]
+
+    for (let a of foundAddresses) {
+      if (a && a.tx_count > 0) {
+        // Translate bech32 address
+        const address = xlatedBech32Addr.hasOwnProperty(a.address)
+            ? xlatedBech32Addr[a.address]
+            : a.address
+
+        if (a.tx_count <= 2) {
+          // Less than 3 transactions for this address
+          // all good
+          const retAddr = {
+            address: address,
+            ntx: a.tx_count,
+            txids: []
+          }
+
+          retAddr.txids = (a.tx_count == 1)
+            ? [a.first_tx] 
+            : [a.first_tx, a.last_tx]
+          
+          ret.push(retAddr)
+
+        } else {
+          // More than 2 transactions for this address
+          // We need more requests to the API
+          if (filterAddr && a.tx_count > keys.addrFilterThreshold) {
+            Logger.info(` import of ${address} rejected (too many transactions - ${a.tx_count})`)
+          } else {
+            const retAddr = await this.getAddress(address)
+            ret.push(retAddr)
+          }
+        }
+      }
+    }
+
+    return ret
   }
 
 }

lib/remote-importer/sources-mainnet.js

@@ -4,9 +4,7 @@
  */
 'use strict'
 
-const addrHelper = require('../bitcoin/addresses-helper')
 const network = require('../bitcoin/network')
-const util = require('../util')
 const Logger = require('../logger')
 const keys = require('../../keys')[network.key]
 const Sources = require('./sources')
@@ -24,8 +22,6 @@ class SourcesMainnet extends Sources {
    */
   constructor() {
     super()
-    // Initializes external source
-    this.source = null
     this._initSource()
   }
 
@@ -45,68 +41,6 @@ class SourcesMainnet extends Sources {
     }
   }
 
-  /**
-   * Retrieve information for a given address
-   * @param {string} address - bitcoin address
-   * @param {boolean} filterAddr - True if an upper bound should be used
-   *     for #transactions associated to the address, False otherwise
-   * @returns {Promise} returns an object
-   *  { address: <bitcoin_address>, txids: <txids>, ntx: <total_nb_txs>}
-   */
-  async getAddress(address, filterAddr) {
-    const ret = {
-      address,
-      txids: [],
-      ntx: 0
-    }
-
-    try {
-      const result = await this.source.getAddress(address, filterAddr)
-      
-      if (result.ntx)
-        ret.ntx = result.ntx
-      else if (result.txids)
-        ret.ntx = result.txids.length
-
-      if (result.txids)
-        ret.txids = result.txids
-
-    } catch(e) {
-      //Logger.error(e, `SourcesMainnet.getAddress() : ${address} from ${this.source.base}`)
-      Logger.error(null, `SourcesMainnet.getAddress() : ${address} from ${this.source.base}`)
-    } finally {
-      return ret
-    }
-  }
-
-  /**
-   * Retrieve information for a list of addresses
-   * @param {string[]} addresses - array of bitcoin address
-   * @param {boolean} filterAddr - True if an upper bound should be used
-   *    for #transactions associated to the address, False otherwise
-   * @returns {Promise} returns an object
-   *    { address: <bitcoin_address>, txids: <txids>, ntx: <total_nb_txs>}
-   */
-  async getAddresses(addresses, filterAddr) {
-    const ret = []
-
-    try {
-      const results = await this.source.getAddresses(addresses, filterAddr)
-
-      for (let r of results) {
-        // Filter addresses with too many txs
-        if (!filterAddr || (r.ntx <= keys.addrFilterThreshold))
-          ret.push(r)
-      }
-
-    } catch(e) {
-      //Logger.error(e, `SourcesMainnet.getAddresses() : ${addresses} from ${this.source.base}`)
-      Logger.error(null, `SourcesMainnet.getAddresses() : ${addresses} from ${this.source.base}`)
-    } finally {
-      return ret
-    }
-  }
-
 }
 
 module.exports = SourcesMainnet

lib/remote-importer/sources-testnet.js

@@ -4,19 +4,17 @@
  */
 'use strict'
 
-const addrHelper = require('../bitcoin/addresses-helper')
 const network = require('../bitcoin/network')
 const util = require('../util')
 const Logger = require('../logger')
 const keys = require('../../keys')[network.key]
 const Sources = require('./sources')
 const BitcoindWrapper = require('./bitcoind-wrapper')
-const InsightWrapper = require('./insight-wrapper')
 const BtcComWrapper = require('./btccom-wrapper')
 
 
 /**
- * Remote data sources for testnet polled round-robin to spread load
+ * Remote data sources for testnet
  */
 class SourcesTestnet extends Sources {
 
@@ -25,126 +23,22 @@ class SourcesTestnet extends Sources {
    */
   constructor() {
     super()
-    this.sources = []
-    this.index = 0
-    this.sourceBech32 = null
-    this.isBitcoindActive = false
-    // Initializes external sources
-    this._initSources()
+    this._initSource()
   }
 
   /**
-   * Initialize the external data sources
+   * Initialize the external data source
    */
-  _initSources() {
+  _initSource() {
     if (keys.explorers.bitcoind == 'active') {
       // If local bitcoind option is activated
       // we'll use the local node as our unique source
-      this.sourceBech32 = new BitcoindWrapper()
-      this.sources.push(this.sourceBech32)
-      this.isBitcoindActive = true
+      this.source = new BitcoindWrapper()
+      Logger.info('Activated Bitcoind as the data source for imports')
     } else {
-      // Otherwise, we use a set of insight servers + btc.com for bech32 addresses
-      this.sourceBech32 = new BtcComWrapper(keys.explorers.btccom)
-      for (let url of keys.explorers.insight)
-        this.sources.push(new InsightWrapper(url))
-      this.isBitcoindActive = false
-    }
-  }
-
-  /**
-   * Get the next source index
-   * @returns {integer} returns the next source index
-   */
-  nextIndex() {
-    this.index++
-    if (this.index >= this.sources.length)
-      this.index = 0
-    return this.index
-  }
-
-  /**
-   * Retrieve information for a given address
-   * @param {string} address - bitcoin address
-   * @param {boolean} filterAddr - True if an upper bound should be used
-   *     for #transactions associated to the address, False otherwise
-   * @returns {Promise} returns an object
-   *  { address: <bitcoin_address>, txids: <txids>, ntx: <total_nb_txs>}
-   */
-  async getAddress(address, filterAddr) {
-    let source = ''
-
-    const isBech32 = addrHelper.isBech32(address)
-    
-    const ret = {
-      address,
-      txids: [],
-      ntx: 0
-    }
-
-    try {
-      source = isBech32 ? this.sourceBech32 : this.sources[this.nextIndex()]
-      const result = await source.getAddress(address, filterAddr)
-      
-      if (result.ntx)
-        ret.ntx = result.ntx
-      else if (result.txids)
-        ret.ntx = result.txids.length
-
-      if (result.txids)
-        ret.txids = result.txids
-      
-      return ret
-
-    } catch(e) {
-      Logger.error(e, `SourcesTestnet.getAddress() : ${address} from ${source.base}`)
-      if (!isBech32 && this.sources.length > 1) {
-        // Try again with another source
-        return this.getAddress(address, filterAddr)
-      } else {
-        return ret
-      }
-    }
-  }
-
-  /**
-   * Retrieve information for a list of addresses
-   * @param {string[]} addresses - array of bitcoin address
-   * @param {boolean} filterAddr - True if an upper bound should be used
-   *    for #transactions associated to the address, False otherwise
-   * @returns {Promise} returns an object
-   *    { address: <bitcoin_address>, txids: <txids>, ntx: <total_nb_txs>}
-   */
-  async getAddresses(addresses, filterAddr) {
-    const ret = []
-
-    try {
-      if (this.isBitcoindActive) {
-        const source = this.sources[0]
-        const results = await source.getAddresses(addresses, filterAddr)
-        for (let r of results) {
-          // Filter addresses with too many txs
-          if (!filterAddr || (r.ntx <= keys.addrFilterThreshold))
-            ret.push(r)
-        }
-      } else {
-        const lists = util.splitList(addresses, this.sources.length)
-        await util.seriesCall(lists, async list => {
-          const results = await Promise.all(list.map(a => {
-            return this.getAddress(a, filterAddr)
-          }))
-        
-          for (let r of results) {
-            // Filter addresses with too many txs
-            if (!filterAddr || (r.ntx <= keys.addrFilterThreshold))
-              ret.push(r)
-          }
-        })
-      }
-    } catch (e) {
-      Logger.error(e, `SourcesTestnet.getAddresses() : Addr list = ${addresses}`)
-    } finally {
-      return ret
+      // Otherwise, we'll use the rest api provided by OXT
+      this.source = new BtcComWrapper(keys.explorers.btccom)
+      Logger.info('Activated BTC.COM API as the data source for imports')
     }
   }
 

lib/remote-importer/sources.js

@@ -4,16 +4,22 @@
  */
 'use strict'
 
+const network = require('../bitcoin/network')
+const Logger = require('../logger')
+const keys = require('../../keys')[network.key]
+
 
 /**
- * Abstract class defining a list of blockchain explorer providing a remote API
+ * Base class defining data source for imports/rescans of HD accounts and addresses
  */
 class Sources {
 
   /**
    * Constructor
    */
-  constructor() {}
+  constructor() {
+    this.source = null
+  }
 
   /**
    * Retrieve information for a given address
@@ -23,7 +29,30 @@ class Sources {
    * @returns {Promise} returns an object
    *  { address: <bitcoin_address>, txids: <txids>, ntx: <total_nb_txs>}
    */
-  async getAddress(address, filterAddr) {}
+  async getAddress(address, filterAddr) {
+    const ret = {
+      address,
+      txids: [],
+      ntx: 0
+    }
+
+    try {
+      const result = await this.source.getAddress(address, filterAddr)
+      
+      if (result.ntx)
+        ret.ntx = result.ntx
+      else if (result.txids)
+        ret.ntx = result.txids.length
+
+      if (result.txids)
+        ret.txids = result.txids
+
+    } catch(e) {
+      Logger.error(null, `Sources.getAddress() : ${address} from ${this.source.base}`)
+    } finally {
+      return ret
+    }
+  }
 
   /**
    * Retrieve information for a list of addresses
@@ -33,7 +62,24 @@ class Sources {
    * @returns {Promise} returns an object
    *    { address: <bitcoin_address>, txids: <txids>, ntx: <total_nb_txs>}
    */
-  async getAddresses(addresses, filterAddr) {}
+  async getAddresses(addresses, filterAddr) {
+    const ret = []
+
+    try {
+      const results = await this.source.getAddresses(addresses, filterAddr)
+
+      for (let r of results) {
+        // Filter addresses with too many txs
+        if (!filterAddr || (r.ntx <= keys.addrFilterThreshold))
+          ret.push(r)
+      }
+
+    } catch(e) {
+      Logger.error(e, `Sources.getAddresses() : ${addresses} from ${this.source.base}`)
+    } finally {
+      return ret
+    }
+  }
 
 }