Merge branch 'feat_dojo_rest_indexer' into 'develop'

add support of rest api provided by addrindexrs

See merge request dojo/samourai-dojo!211

docker/my-dojo/indexer/Dockerfile

@@ -1,7 +1,7 @@
 FROM    rust:1.42.0-slim-buster
 
 ENV     INDEXER_HOME        /home/indexer
-ENV     INDEXER_VERSION     0.4.0
+ENV     INDEXER_VERSION     0.5.0
 ENV     INDEXER_URL         https://code.samourai.io/dojo/addrindexrs.git
 
 ARG     INDEXER_LINUX_GID
@@ -42,5 +42,6 @@ RUN     cd "$INDEXER_HOME/addrindexrs" && \
         cargo install --locked --path .
 
 EXPOSE  50001
+EXPOSE  8080
 
 STOPSIGNAL SIGINT

docker/my-dojo/indexer/restart.sh

@@ -7,6 +7,7 @@ indexer_options=(
   --jsonrpc-import
   --db-dir="/home/indexer/db"
   --indexer-rpc-addr="$NET_DOJO_INDEXER_IPV4:50001"
+  --indexer-http-addr="$NET_DOJO_INDEXER_IPV4:8080"
   --daemon-rpc-addr="$BITCOIND_IP:$BITCOIND_RPC_PORT"
   --cookie="$BITCOIND_RPC_USER:$BITCOIND_RPC_PASSWORD"
   --txid-limit="$INDEXER_TXID_LIMIT"

docker/my-dojo/overrides/indexer.install.yaml

@@ -18,6 +18,7 @@ services:
     command: "/wait-for-it.sh tor:9050 --timeout=360 --strict -- /restart.sh"
     expose:
       - "50001"
+      - "8080"
     volumes:
       - data-indexer:/home/indexer
     logging:

lib/remote-importer/local-rest-indexer-wrapper.js

@@ -0,0 +1,157 @@
+/*!
+ * lib/remote-importer/local-rest-indexer-wrapper.js
+ * Copyright © 2019 – Katana Cryptographic Ltd. All Rights Reserved.
+ */
+'use strict'
+
+const axios = require('axios')
+const bitcoin = require('bitcoinjs-lib')
+const Logger = require('../logger')
+const util = require('../util')
+const network = require('../bitcoin/network')
+const activeNet = network.network
+const keys = require('../../keys')[network.key]
+const Wrapper = require('./wrapper')
+
+
+/**
+ * Wrapper for a local indexer
+ * providing a REST API
+ */
+class LocalRestIndexerWrapper extends Wrapper {
+
+  /**
+   * Constructor
+   */
+  constructor(url) {
+    super(url, null)
+  }
+
+  /**
+   * Send a GET request to the API
+   * @param {string} route
+   * @returns {Promise}
+   */
+  async _get(route) {
+    const params = {
+      url: `${this.base}${route}`,
+      method: 'GET',
+      responseType: 'json',
+      timeout: 15000,
+      headers: {
+        'User-Agent': 'Dojo'
+      }
+    }
+
+    const result = await axios(params)
+    return result.data
+  }
+
+  /**
+   * Translate a bitcoin address into a script hash
+   * (@see https://electrumx.readthedocs.io/en/latest/protocol-basics.html#script-hashes)
+   * @param {string} address - bitcoin address
+   * @returns {string} returns the script hash associated to the address
+   */
+  _getScriptHash(address) {
+    const bScriptPubKey = bitcoin.address.toOutputScript(address, activeNet)
+    const bScriptHash = bitcoin.crypto.sha256(bScriptPubKey)
+    return bScriptHash.reverse().toString('hex')
+  }
+
+  /**
+   * Retrieve information for a given address
+   * @param {string} address - bitcoin address
+   * @param {boolean} filterAddr - True if an upper bound should be used
+   *     for #transactions associated to the address, False otherwise
+   * @returns {Promise} returns an object
+   *  { address: <bitcoin_address>, txids: <txids>, ntx: <total_nb_txs>}
+   */
+  async getAddress(address, filterAddr) {
+    const ret = {
+      address: address,
+      ntx: 0,
+      txids: []
+    }
+
+    const scriptHash = this._getScriptHash(address)
+    const uri = `/blockchain/scripthash/${scriptHash}/history`
+    const results = await this._get(uri)
+
+    for (let r of results) {
+      ret.txids.push(r.tx_hash)
+      ret.ntx++
+    }
+
+    if (filterAddr && ret.ntx > keys.addrFilterThreshold) {
+      Logger.info(`Importer : Import of ${address} rejected (too many transactions - ${ret.ntx})`)
+      return {
+        address: address,
+        ntx: 0,
+        txids: []
+      }
+    }
+
+    return ret
+  }
+
+  /**
+   * Retrieve information for a given list of addresses
+   * @param {string} addresses - array of bitcoin addresses
+   * @param {boolean} filterAddr - True if an upper bound should be used
+   *     for #transactions associated to the address, False otherwise
+   * @returns {Promise} returns an array of objects
+   *  { address: <bitcoin_address>, txids: <txids>, ntx: <total_nb_txs>}
+   */
+  async getAddresses(addresses, filterAddr) {
+    const ret = {}
+    const scriptHash2Address = {}
+    const scriptHashes = []
+
+    for (let a of addresses) {
+      const scriptHash = this._getScriptHash(a)
+      scriptHashes.push(scriptHash)
+      scriptHash2Address[scriptHash] = a
+    }
+
+    const sScriptHashes = scriptHashes.join(',')
+    const uri = `/blockchain/scripthashes/history?scripthashes=${sScriptHashes}`
+    const results = await this._get(uri)
+
+    for (let r of results) {
+      const a = scriptHash2Address[r.script_hash]
+      ret[a] = {
+        address: a,
+        ntx: r.txids.length,
+        txids: r.txids
+      }
+    }
+
+    const aRet = Object.values(ret)
+
+    for (let i in aRet) {
+      if (filterAddr && aRet[i].ntx > keys.addrFilterThreshold) {
+        Logger.info(`Importer : Import of ${aRet[i].address} rejected (too many transactions - ${aRet[i].ntx})`)
+        aRet.splice(i, 1)
+      }
+    }
+
+    return aRet
+  }
+
+  /**
+   * Retrieve the height of the chaintip for the remote source
+   * @returns {Promise} returns an object
+   *    {chainTipHeight: <chaintip_height>}
+   */
+  async getChainTipHeight() {
+    let chainTipHeight = null
+    const result = await this._get(`/blocks/tip`)
+    if (result != null && result['height'] != null)
+      chainTipHeight = parseInt(result['height'])
+    return {'chainTipHeight': chainTipHeight}
+  }
+
+}
+
+module.exports = LocalRestIndexerWrapper

lib/remote-importer/sources-mainnet.js

@@ -10,6 +10,7 @@ const keys = require('../../keys')[network.key]
 const Sources = require('./sources')
 const BitcoindWrapper = require('./bitcoind-wrapper')
 const LocalIndexerWrapper = require('./local-indexer-wrapper')
+const LocalRestIndexerWrapper = require('./local-rest-indexer-wrapper')
 const OxtWrapper = require('./oxt-wrapper')
 
 
@@ -40,6 +41,12 @@ class SourcesMainnet extends Sources {
       // we'll use the local indexer as our unique source
       this.source = new LocalIndexerWrapper()
       Logger.info('Importer : Activated local indexer as the data source for imports')
+    } else if (keys.indexer.active == 'local_rest_indexer') {
+      // If local rest indexer option is activated
+      // we'll use the local indexer as our unique source
+      const uri = `http://${keys.indexer.localIndexer.host}:${keys.indexer.localIndexer.port}`
+      this.source = new LocalRestIndexerWrapper(uri)
+      Logger.info('Importer : Activated local indexer (REST API) as the data source for imports')
     } else {
       // Otherwise, we'll use the rest api provided by OXT
       this.source = new OxtWrapper(keys.indexer.oxt)

lib/remote-importer/sources-testnet.js

@@ -11,6 +11,7 @@ const keys = require('../../keys')[network.key]
 const Sources = require('./sources')
 const BitcoindWrapper = require('./bitcoind-wrapper')
 const LocalIndexerWrapper = require('./local-indexer-wrapper')
+const LocalRestIndexerWrapper = require('./local-rest-indexer-wrapper')
 const EsploraWrapper = require('./esplora-wrapper')
 
 
@@ -41,6 +42,12 @@ class SourcesTestnet extends Sources {
       // we'll use the local indexer as our unique source
       this.source = new LocalIndexerWrapper()
       Logger.info('Importer : Activated local indexer as the data source for imports')
+    } else if (keys.indexer.active == 'local_rest_indexer') {
+      // If local rest indexer option is activated
+      // we'll use the local indexer as our unique source
+      const uri = `http://${keys.indexer.localIndexer.host}:${keys.indexer.localIndexer.port}`
+      this.source = new LocalRestIndexerWrapper(uri)
+      Logger.info('Importer : Activated local indexer (REST API) as the data source for imports')
     } else {
       // Otherwise, we'll use the rest api provided by Esplora
       this.source = new EsploraWrapper(keys.indexer.esplora)

static/admin/dmt/xpubs-tools/xpubs-tools.js

@@ -31,7 +31,9 @@ const screenXpubsToolsScript = {
 
   preparePage: function() {
     // Disable custom lookahead if data source is a third party explorer
-    const disableLookahead = sessionStorage.getItem('indexerType') == 'third_party_explorer'
+    const isTPE = sessionStorage.getItem('indexerType') == 'third_party_explorer'
+    const isLRI = sessionStorage.getItem('indexerType') == 'local_rest_indexer'
+    const disableLookahead = isTPE || isLRI
     $('#rescan-lookahead').prop('disabled', disableLookahead)
 
     this.hideRescanForm()