Merge pull request #85 from Samourai-Wallet/feat_esplora_testnet

add esplora as a data source for testnet imports and rescans

docker/my-dojo/.env

@@ -42,6 +42,7 @@ NODE_GAP_INTERNAL=100
 NODE_ADDR_FILTER_THRESHOLD=1000
 NODE_URL_OXT_API=https://api.oxt.me
 NODE_URL_BTCCOM_API=https://tchain.api.btc.com/v3
+NODE_URL_ESPLORA_API=https://blockstream.info/testnet
 NODE_ADDR_DERIVATION_MIN_CHILD=2
 NODE_ADDR_DERIVATION_MAX_CHILD=2
 NODE_ADDR_DERIVATION_THRESHOLD=10

docker/my-dojo/node/keys.index.js

@@ -165,7 +165,9 @@ module.exports = {
       // OXT (mainnet)
       oxt: process.env.NODE_URL_OXT_API,
       // BTC.COM (testnet)
-      btccom: process.env.NODE_URL_BTCCOM_API
+      btccom: process.env.NODE_URL_BTCCOM_API,
+      // Esplora (testnet)
+      esplora: process.env.NODE_URL_ESPLORA_API,
     },
     /*
      * Max number of transactions per address

keys/index-example.js

@@ -276,7 +276,8 @@ module.exports = {
       insight: [
         'https://testnet-api.example.com'
       ],
-      btccom: 'https://tchain.api.btc.com/v3'
+      btccom: 'https://tchain.api.btc.com/v3',
+      esplora: 'https://blockstream.info/testnet'
     },
     addrFilterThreshold: 1000,
     addrDerivationPool: {

lib/remote-importer/esplora-wrapper.js

@@ -0,0 +1,131 @@
+/*!
+ * lib/remote-importer\esplora-wrapper.js
+ * Copyright © 2019 – Katana Cryptographic Ltd. All Rights Reserved.
+ */
+'use strict'
+
+const rp = require('request-promise-native')
+const addrHelper = require('../bitcoin/addresses-helper')
+const util = require('../util')
+const Logger = require('../logger')
+const network = require('../bitcoin/network')
+const keys = require('../../keys')[network.key]
+const Wrapper = require('./wrapper')
+
+
+/**
+ * Wrapper for the esplora block explorer APIs
+ */
+class EsploraWrapper extends Wrapper {
+
+  /**
+   * Constructor
+   */
+  constructor(url) {
+    super(url, keys.explorers.socks5Proxy)
+  }
+
+  /**
+   * Send a GET request to the API
+   * @param {string} route
+   * @returns {Promise}
+   */
+  async _get(route) {
+    const params = {
+      url: `${this.base}${route}`,
+      method: 'GET',
+      json: true,
+      timeout: 15000
+    }
+    
+    // Sets socks proxy agent if required
+    if (keys.explorers.socks5Proxy != null)
+      params['agent'] = this.socksProxyAgent
+
+    return rp(params)
+  }
+
+  /**
+   * Get a page of transactions related to a given address
+   * @param {string} address - bitcoin address
+   * @param {string} lastSeenTxid - last seen txid
+   *  (see https://github.com/Blockstream/esplora/blob/master/API.md)
+   * @returns {Promise}
+   */
+  async _getTxsForAddress(address, lastSeenTxid) {
+    let uri = `/api/address/${address}/txs`
+    if (lastSeenTxid)
+      uri = uri + `/chain/${lastSeenTxid}`
+
+    const results = await this._get(uri)
+    return results.map(tx => tx.txid)
+  }
+
+  /**
+   * Retrieve information for a given address
+   * @param {string} address - bitcoin address
+   * @param {boolean} filterAddr - True if an upper bound should be used
+   *     for #transactions associated to the address, False otherwise
+   * @returns {Promise} returns an object
+   *  { address: <bitcoin_address>, txids: <txids>, ntx: <total_nb_txs>}
+   */
+  async getAddress(address, filterAddr) {
+    const ret = {
+      address: address,
+      ntx: 0,
+      txids: []
+    }
+
+    let lastSeenTxid = null
+
+    while (true) {
+      const txids = await this._getTxsForAddress(address, lastSeenTxid)
+
+      if (txids.length == 0)
+        // we have all the transactions
+        return ret
+
+      ret.txids = ret.txids.concat(txids)
+      ret.ntx += ret.txids.length
+
+      if (txids.length < EsploraWrapper.NB_TXS_PER_PAGE) {
+        // we have all the transactions
+        return ret
+      } else if (filterAddr && ret.ntx > keys.addrFilterThreshold) {
+        // we have too many transactions
+        Logger.info(` import of ${ret.address} rejected (too many transactions - ${ret.ntx})`)
+        ret.txids = []
+        ret.ntx = 0
+        return ret
+      } else {
+        // we need a new iteration
+        lastSeenTxid = txids[txids.length-1]
+      }
+    }
+  }
+
+  /**
+   * Retrieve information for a given list of addresses
+   * @param {string[]} addresses - array of bitcoin addresses
+   * @param {boolean} filterAddr - True if an upper bound should be used
+   *     for #transactions associated to the address, False otherwise
+   * @returns {Promise} returns an array of objects
+   *  { address: <bitcoin_address>, txids: <txids>, ntx: <total_nb_txs>}
+   */
+  async getAddresses(addresses, filterAddr) {
+    const ret = []
+
+    for (let a of addresses) {
+      const retAddr = await this.getAddress(a, filterAddr)
+      ret.push(retAddr)
+    }
+
+    return ret
+  }
+
+}
+
+// Esplora returns a max of 25 txs per page
+EsploraWrapper.NB_TXS_PER_PAGE = 25
+
+module.exports = EsploraWrapper

lib/remote-importer/sources-testnet.js

@@ -10,7 +10,7 @@ const Logger = require('../logger')
 const keys = require('../../keys')[network.key]
 const Sources = require('./sources')
 const BitcoindWrapper = require('./bitcoind-wrapper')
-const BtcComWrapper = require('./btccom-wrapper')
+const EsploraWrapper = require('./esplora-wrapper')
 
 
 /**
@@ -36,9 +36,9 @@ class SourcesTestnet extends Sources {
       this.source = new BitcoindWrapper()
       Logger.info('Activated Bitcoind as the data source for imports')
     } else {
-      // Otherwise, we'll use the rest api provided by OXT
-      this.source = new BtcComWrapper(keys.explorers.btccom)
-      Logger.info('Activated BTC.COM API as the data source for imports')
+      // Otherwise, we'll use the rest api provided by Esplora
+      this.source = new EsploraWrapper(keys.explorers.esplora)
+      Logger.info('Activated Esplora API as the data source for imports')
     }
   }