Merge branch 'feat_dojo_optimizations'

lib/bitcoin/addresses-helper.js

@@ -7,7 +7,9 @@
 const bitcoin = require('bitcoinjs-lib')
 const btcMessage = require('bitcoinjs-message')
 const activeNet = require('./network').network
-const { p2pkh, p2sh, p2wpkh } = bitcoin.payments
+const { p2pkh, p2sh, p2wpkh, p2wsh } = bitcoin.payments
+const { OPS } = bitcoin.script
+
 
 /**
  * A singleton providing Addresses helper functions
@@ -106,6 +108,86 @@ class AddressesHelper {
     }
   }
 
+  /**
+   * Check if an output script is a P2PKH script
+   * @param {Buffer} scriptpubkey - scriptpubkey
+   * @returns {boolean} return true if output is a P2PKH script, otherwise return false
+   */
+  isP2pkhScript(scriptpubkey) {
+    return scriptpubkey.length == 25
+      && scriptpubkey[0] == OPS.OP_DUP
+      && scriptpubkey[1] == OPS.OP_HASH160
+      && scriptpubkey[2] == 0x14
+      && scriptpubkey[23] == OPS.OP_EQUALVERIFY
+      && scriptpubkey[24] == OPS.OP_CHECKSIG
+  }
+
+  /**
+   * Check if an output script is a P2SH script
+   * @param {Buffer} scriptpubkey - scriptpubkey
+   * @returns {boolean} return true if output is a P2SH script, otherwise return false
+   */
+  isP2shScript(scriptpubkey) {
+    return scriptpubkey.length == 23
+      && scriptpubkey[0] == OPS.OP_HASH160
+      && scriptpubkey[1] == 0x14
+      && scriptpubkey[22] == OPS.OP_EQUAL
+  }
+
+  /**
+   * Check if an output script is a P2WPKH script
+   * @param {Buffer} scriptpubkey - scriptpubkey
+   * @returns {boolean} return true if output is a P2WPKH script, otherwise return false
+   */
+  isP2wpkhScript(scriptpubkey) {
+    return scriptpubkey.length == 22
+      && scriptpubkey[0] == OPS.OP_0
+      && scriptpubkey[1] == 0x14
+  }
+
+  /**
+   * Check if an output script is a P2WSH script
+   * @param {Buffer} scriptpubkey - scriptpubkey
+   * @returns {boolean} return true if output is a P2WSH script, otherwise return false
+   */
+  isP2wshScript(scriptpubkey) {
+    return scriptpubkey.length == 34
+      && scriptpubkey[0] == OPS.OP_0
+      && scriptpubkey[1] == 0x20
+  }
+
+  /**
+   * Return the bitcoin address corresponding to an output script
+   * @param {Buffer} scriptpubkey - scriptpubkey
+   * @returns {string} bitcoin address
+   */
+  outputScript2Address(scriptpubkey) {
+    if (this.isP2pkhScript(scriptpubkey))
+      return p2pkh({
+        output: scriptpubkey,
+        network: activeNet,
+      }).address
+
+    if (this.isP2shScript(scriptpubkey))
+      return p2sh({
+        output: scriptpubkey,
+        network: activeNet,
+      }).address
+
+    if (this.isP2wpkhScript(scriptpubkey))
+      return p2wpkh({
+        output: scriptpubkey,
+        network: activeNet,
+      }).address
+
+    if (this.isP2wshScript(scriptpubkey))
+      return p2wsh({
+        output: scriptpubkey,
+        network: activeNet,
+      }).address
+
+    throw 'unknown address format'
+  }
 }
 
 module.exports = new AddressesHelper()

lib/remote-importer/bitcoind-wrapper.js

@@ -8,6 +8,7 @@ const bitcoin = require('bitcoinjs-lib')
 const RpcClient = require('../bitcoind-rpc/rpc-client')
 const rpcLatestBlock = require('../bitcoind-rpc/latest-block')
 const Logger = require('../logger')
+const addrHelper = require('../bitcoin/addresses-helper')
 const network = require('../bitcoin/network')
 const activeNet = network.network
 const keys = require('../../keys')[network.key]
@@ -44,7 +45,7 @@ class BitcoindWrapper extends Wrapper {
    */
   _xlatScriptPubKey(scriptPubKey) {
     const bScriptPubKey = Buffer.from(scriptPubKey, 'hex')
-    return bitcoin.address.fromOutputScript(bScriptPubKey, activeNet)
+    return addrHelper.outputScript2Address(bScriptPubKey)
   }
 
   /**

lib/util.js

@@ -88,6 +88,15 @@ class Util {
     })
   }
 
+  /**
+   * Execute parallel asynchronous calls to a function
+   * over a list of objects
+   */
+  static parallelCall(list, fn) {
+    const operations = list.map(item => { return fn(item) })
+    return Promise.all(operations)
+  }
+
   /**
    * Delay the call to a function
    */

pushtx/pushtx-processor.js

@@ -10,6 +10,7 @@ const Logger = require('../lib/logger')
 const errors = require('../lib/errors')
 const db = require('../lib/db/mysql-db-wrapper')
 const RpcClient = require('../lib/bitcoind-rpc/rpc-client')
+const addrHelper = require('../lib/bitcoin/addresses-helper')
 const network = require('../lib/bitcoin/network')
 const activeNet = network.network
 const keys = require('../keys')[network.key]
@@ -68,7 +69,7 @@ class PushTxProcessor {
       if (vout >= tx.outs.length)
         throw errors.txout.VOUT
       const output = tx.outs[vout]
-      const address = bitcoin.address.fromOutputScript(output.script, activeNet)
+      const address = addrHelper.outputScript2Address(output.script)
       const nbTxs = await db.getAddressNbTransactions(address)
       if (nbTxs == null || nbTxs > 0)
         faultyOutputs.push(vout)

tracker/block.js

@@ -80,7 +80,7 @@ class Block extends TransactionsBundle {
     const txids = this.transactions.map(t => t.getId())
     ntx = txids.length
     const txidLists = util.splitList(txids, 100)
-    await util.seriesCall(txidLists, list => db.confirmTransactions(list, blockId))
+    await util.parallelCall(txidLists, list => db.confirmTransactions(list, blockId))
 
     // Logs and result returned
     const dt = ((Date.now()-t0)/1000).toFixed(1)

tracker/mempool-processor.js

@@ -218,7 +218,7 @@ class MempoolProcessor extends AbstractProcessor {
     const unconfirmedTxs = await db.getUnconfirmedTransactions()
 
     if (unconfirmedTxs.length > 0) {
-      await util.seriesCall(unconfirmedTxs, tx => {
+      await util.parallelCall(unconfirmedTxs, tx => {
         try {
           return this.client.getrawtransaction(tx.txnTxid, true)
             .then(async rtx => {

tracker/transaction.js

@@ -8,6 +8,7 @@ const _ = require('lodash')
 const bitcoin = require('bitcoinjs-lib')
 const util = require('../lib/util')
 const Logger = require('../lib/logger')
+const addrHelper = require('../lib/bitcoin/addresses-helper')
 const hdaHelper = require('../lib/bitcoin/hd-accounts-helper')
 const db = require('../lib/db/mysql-db-wrapper')
 const network = require('../lib/bitcoin/network')
@@ -159,7 +160,7 @@ class Transaction {
 
     for (let output of this.tx.outs) {
       try {
-        const address = bitcoin.address.fromOutputScript(output.script, activeNet)
+        const address = addrHelper.outputScript2Address(output.script)
         if (!indexedOutputs[address])
           indexedOutputs[address] = []
 
@@ -255,7 +256,7 @@ class Transaction {
     const xpubList = _.keys(hdAccounts)
 
     if (xpubList.length > 0) {
-      await util.seriesCall(xpubList, async xpub => {
+      await util.parallelCall(xpubList, async xpub => {
         const usedNewAddresses = await this._deriveNewAddresses(
           xpub,
           hdAccounts[xpub],

tracker/transactions-bundle.js

@@ -9,6 +9,7 @@ const LRU = require('lru-cache')
 const bitcoin = require('bitcoinjs-lib')
 const util = require('../lib/util')
 const db = require('../lib/db/mysql-db-wrapper')
+const addrHelper = require('../lib/bitcoin/addresses-helper')
 const network = require('../lib/bitcoin/network')
 const keys = require('../keys')[network.key]
 const activeNet = network.network
@@ -69,11 +70,7 @@ class TransactionsBundle {
     // Process transactions by slices of 5000 transactions
     const MAX_NB_TXS = 5000
     const lists = util.splitList(this.transactions, MAX_NB_TXS)
-
+    const results = await util.seriesCall(lists, txs => this._prefilterTransactions(txs))
-    const results = await util.seriesCall(lists, list => {
-      return this._prefilterTransactions(list)
-    })
-
     return _.flatten(results)
   }
 
@@ -118,7 +115,7 @@ class TransactionsBundle {
       for (const j in tx.outs) {
         try {
           const script = tx.outs[j].script
-          const address = bitcoin.address.fromOutputScript(script, activeNet)
+          const address = addrHelper.outputScript2Address(script)
           outputs.push(address)
           // Index the output
           if (!indexedOutputs[address])
@@ -174,7 +171,9 @@ class TransactionsBundle {
     }
 
     // Prefilter
-    const inRes = await db.getOutputSpends(inputs)
+    const lists = util.splitList(inputs, 1000)
+    const results = await util.parallelCall(lists, list => db.getOutputSpends(list))
+    const inRes = _.flatten(results)
 
     for (const i in inRes) {
       const key = inRes[i].txnTxid + '-' + inRes[i].outIndex