Merge branch 'feat_dojo_optimizations'

lib/bitcoin/addresses-helper.js

@@ -7,7 +7,9 @@
 const bitcoin = require('bitcoinjs-lib')
 const btcMessage = require('bitcoinjs-message')
 const activeNet = require('./network').network
-const { p2pkh, p2sh, p2wpkh } = bitcoin.payments
+const { p2pkh, p2sh, p2wpkh, p2wsh } = bitcoin.payments
+const { OPS } = bitcoin.script
+
 
 /**
  * A singleton providing Addresses helper functions
@@ -106,6 +108,86 @@ class AddressesHelper {
     }
   }
 
+  /**
+   * Check if an output script is a P2PKH script
+   * @param {Buffer} scriptpubkey - scriptpubkey
+   * @returns {boolean} return true if output is a P2PKH script, otherwise return false
+   */
+  isP2pkhScript(scriptpubkey) {
+    return scriptpubkey.length == 25
+      && scriptpubkey[0] == OPS.OP_DUP
+      && scriptpubkey[1] == OPS.OP_HASH160
+      && scriptpubkey[2] == 0x14
+      && scriptpubkey[23] == OPS.OP_EQUALVERIFY
+      && scriptpubkey[24] == OPS.OP_CHECKSIG
+  }
+
+  /**
+   * Check if an output script is a P2SH script
+   * @param {Buffer} scriptpubkey - scriptpubkey
+   * @returns {boolean} return true if output is a P2SH script, otherwise return false
+   */
+  isP2shScript(scriptpubkey) {
+    return scriptpubkey.length == 23
+      && scriptpubkey[0] == OPS.OP_HASH160
+      && scriptpubkey[1] == 0x14
+      && scriptpubkey[22] == OPS.OP_EQUAL
+  }
+
+  /**
+   * Check if an output script is a P2WPKH script
+   * @param {Buffer} scriptpubkey - scriptpubkey
+   * @returns {boolean} return true if output is a P2WPKH script, otherwise return false
+   */
+  isP2wpkhScript(scriptpubkey) {
+    return scriptpubkey.length == 22
+      && scriptpubkey[0] == OPS.OP_0
+      && scriptpubkey[1] == 0x14
+  }
+
+  /**
+   * Check if an output script is a P2WSH script
+   * @param {Buffer} scriptpubkey - scriptpubkey
+   * @returns {boolean} return true if output is a P2WSH script, otherwise return false
+   */
+  isP2wshScript(scriptpubkey) {
+    return scriptpubkey.length == 34
+      && scriptpubkey[0] == OPS.OP_0
+      && scriptpubkey[1] == 0x20
+  }
+
+  /**
+   * Return the bitcoin address corresponding to an output script
+   * @param {Buffer} scriptpubkey - scriptpubkey
+   * @returns {string} bitcoin address
+   */
+  outputScript2Address(scriptpubkey) {
+    if (this.isP2pkhScript(scriptpubkey))
+      return p2pkh({
+        output: scriptpubkey,
+        network: activeNet,
+      }).address
+
+    if (this.isP2shScript(scriptpubkey))
+      return p2sh({
+        output: scriptpubkey,
+        network: activeNet,
+      }).address
+
+    if (this.isP2wpkhScript(scriptpubkey))
+      return p2wpkh({
+        output: scriptpubkey,
+        network: activeNet,
+      }).address
+
+    if (this.isP2wshScript(scriptpubkey))
+      return p2wsh({
+        output: scriptpubkey,
+        network: activeNet,
+      }).address
+
+    throw 'unknown address format'
+  }
 }
 
 module.exports = new AddressesHelper()

lib/remote-importer/bitcoind-wrapper.js

@@ -8,6 +8,7 @@ const bitcoin = require('bitcoinjs-lib')
 const RpcClient = require('../bitcoind-rpc/rpc-client')
 const rpcLatestBlock = require('../bitcoind-rpc/latest-block')
 const Logger = require('../logger')
+const addrHelper = require('../bitcoin/addresses-helper')
 const network = require('../bitcoin/network')
 const activeNet = network.network
 const keys = require('../../keys')[network.key]
@@ -44,7 +45,7 @@ class BitcoindWrapper extends Wrapper {
    */
   _xlatScriptPubKey(scriptPubKey) {
     const bScriptPubKey = Buffer.from(scriptPubKey, 'hex')
-    return bitcoin.address.fromOutputScript(bScriptPubKey, activeNet)
+    return addrHelper.outputScript2Address(bScriptPubKey)
   }
 
   /**

lib/util.js

@@ -32,10 +32,10 @@ class Util {
    *
    * @param {object} parents - map of {[key]: [incoming edge keys]}
    * @param {object} children - a map of {[key]: [outgoing edge keys]}
-   * @returns {object} 
+   * @returns {object}
    *    if graph has edges then
    *      return error (graph has at least one cycle)
-   *    else 
+   *    else
    *    return L (a topologically sorted order)
    */
   static topologicalOrdering(parents, children) {
@@ -81,13 +81,22 @@ class Util {
         }).then(result => {
           results.push(result)
         })
-      }, 
+      },
       Promise.resolve()
     ).then(function() {
       return results
     })
   }
 
+  /**
+   * Execute parallel asynchronous calls to a function
+   * over a list of objects
+   */
+  static parallelCall(list, fn) {
+    const operations = list.map(item => { return fn(item) })
+    return Promise.all(operations)
+  }
+
   /**
    * Delay the call to a function
    */
@@ -159,16 +168,16 @@ class Util {
 
   /**
    * Median of an array of values
-   */ 
+   */
   static median(arr, sorted) {
     if (arr.length == 0) return NaN
     if (arr.length == 1) return arr[0]
 
     if (!sorted)
       arr.sort(Util.cmpAsc)
-    
+
     const midpoint = Math.floor(arr.length / 2)
-    
+
     if (arr.length % 2) {
       // Odd-length array
       return arr[midpoint]
@@ -252,10 +261,10 @@ class Util {
 
     // "Floor-x"
     const fx = Math.floor(x) - 1
-    
+
     // "Mod-x"
     const mx = x % 1
-    
+
     if (fx + 1 >= N) {
       return arr[fx]
     } else {
@@ -340,7 +349,7 @@ class Util {
 
     const parts = [Util.pad10(h), Util.pad10(m), Util.pad10(s)]
 
-    if (d > 0) 
+    if (d > 0)
       parts.splice(0, 0, Util.pad100(d))
 
     const str = parts.join(':')

pushtx/pushtx-processor.js

@@ -10,6 +10,7 @@ const Logger = require('../lib/logger')
 const errors = require('../lib/errors')
 const db = require('../lib/db/mysql-db-wrapper')
 const RpcClient = require('../lib/bitcoind-rpc/rpc-client')
+const addrHelper = require('../lib/bitcoin/addresses-helper')
 const network = require('../lib/bitcoin/network')
 const activeNet = network.network
 const keys = require('../keys')[network.key]
@@ -24,7 +25,7 @@ if (network.key == 'bitcoin') {
 
 
 /**
- * A singleton providing a wrapper 
+ * A singleton providing a wrapper
  * for pushing transactions with the local bitcoind
  */
 class PushTxProcessor {
@@ -52,7 +53,7 @@ class PushTxProcessor {
    * Enforce a strict verification mode on a list of outputs
    * @param {string} rawtx - raw bitcoin transaction in hex format
    * @param {array} vouts - output indices (integer)
-   * @returns {array} returns the indices of the faulty outputs 
+   * @returns {array} returns the indices of the faulty outputs
    */
   async enforceStrictModeVouts(rawtx, vouts) {
     const faultyOutputs = []
@@ -63,12 +64,12 @@ class PushTxProcessor {
     } catch(e) {
       throw errors.tx.PARSE
     }
-    // Check in db if addresses are known and have been used 
+    // Check in db if addresses are known and have been used
     for (let vout of vouts) {
       if (vout >= tx.outs.length)
         throw errors.txout.VOUT
       const output = tx.outs[vout]
-      const address = bitcoin.address.fromOutputScript(output.script, activeNet)
+      const address = addrHelper.outputScript2Address(output.script)
       const nbTxs = await db.getAddressNbTransactions(address)
       if (nbTxs == null || nbTxs > 0)
         faultyOutputs.push(vout)

tracker/block.js

@@ -37,7 +37,7 @@ class Block extends TransactionsBundle {
 
     let block
     const txsForBroadcast = []
-    
+
     try {
       block = bitcoin.Block.fromHex(this.hex)
       this.transactions = block.transactions
@@ -46,10 +46,10 @@ class Block extends TransactionsBundle {
       Logger.error(null, this.header)
       return Promise.reject(e)
     }
-    
+
     const t0 = Date.now()
     let ntx = 0
-    
+
     // Filter transactions
     const filteredTxs = await this.prefilterTransactions()
 
@@ -78,9 +78,9 @@ class Block extends TransactionsBundle {
 
     // Confirms the transactions
     const txids = this.transactions.map(t => t.getId())
-    ntx = txids.length    
+    ntx = txids.length
     const txidLists = util.splitList(txids, 100)
-    await util.seriesCall(txidLists, list => db.confirmTransactions(list, blockId))
+    await util.parallelCall(txidLists, list => db.confirmTransactions(list, blockId))
 
     // Logs and result returned
     const dt = ((Date.now()-t0)/1000).toFixed(1)

tracker/mempool-processor.js

@@ -69,7 +69,7 @@ class MempoolProcessor extends AbstractProcessor {
 
   /**
    * Stop processing
-   */ 
+   */
   async stop() {
     clearInterval(this.checkUnconfirmedId)
     clearInterval(this.processMempoolId)
@@ -218,11 +218,11 @@ class MempoolProcessor extends AbstractProcessor {
     const unconfirmedTxs = await db.getUnconfirmedTransactions()
 
     if (unconfirmedTxs.length > 0) {
-      await util.seriesCall(unconfirmedTxs, tx => {
+      await util.parallelCall(unconfirmedTxs, tx => {
         try {
           return this.client.getrawtransaction(tx.txnTxid, true)
             .then(async rtx => {
-              if (!rtx.blockhash) return null              
+              if (!rtx.blockhash) return null
               // Transaction is confirmed
               const block = await db.getBlockByHash(rtx.blockhash)
               if (block && block.blockID) {

tracker/transaction.js

@@ -8,6 +8,7 @@ const _ = require('lodash')
 const bitcoin = require('bitcoinjs-lib')
 const util = require('../lib/util')
 const Logger = require('../lib/logger')
+const addrHelper = require('../lib/bitcoin/addresses-helper')
 const hdaHelper = require('../lib/bitcoin/hd-accounts-helper')
 const db = require('../lib/db/mysql-db-wrapper')
 const network = require('../lib/bitcoin/network')
@@ -28,9 +29,9 @@ class Transaction {
    */
   constructor(tx) {
     this.tx = tx
-    this.txid = this.tx.getId()   
+    this.txid = this.tx.getId()
     // Id of transaction stored in db
-    this.storedTxnID = null 
+    this.storedTxnID = null
     // Should this transaction be broadcast out to connected clients?
     this.doBroadcast = false
   }
@@ -80,7 +81,7 @@ class Transaction {
     // Store database ids of double spend transactions
     const doubleSpentTxnIDs = []
     // Store inputs of interest
-    const inputs = [] 
+    const inputs = []
 
     // Extracts inputs information
     let index = 0
@@ -153,13 +154,13 @@ class Transaction {
   async _processOutputs() {
     // Store outputs, keyed by address. Values are arrays of outputs
     const indexedOutputs = {}
-    
+
     // Extracts outputs information
     let index = 0
 
     for (let output of this.tx.outs) {
       try {
-        const address = bitcoin.address.fromOutputScript(output.script, activeNet)
+        const address = addrHelper.outputScript2Address(output.script)
         if (!indexedOutputs[address])
           indexedOutputs[address] = []
 
@@ -174,7 +175,7 @@ class Transaction {
 
     // Array of addresses receiving tx outputs
     const addresses = _.keys(indexedOutputs)
-    
+
     // Store a list of known addresses that received funds
     let fundedAddresses = []
 
@@ -203,7 +204,7 @@ class Transaction {
 
     for (let a of fundedAddresses) {
       outputs.push({
-        txnID: this.storedTxnID, 
+        txnID: this.storedTxnID,
         addrID: a.addrID,
         outIndex: a.outIndex,
         outAmount: a.outAmount,
@@ -253,9 +254,9 @@ class Transaction {
     // Store a list of known addresses that received funds
     const fundedAddresses = []
     const xpubList = _.keys(hdAccounts)
-    
+
     if (xpubList.length > 0) {
-      await util.seriesCall(xpubList, async xpub => {
+      await util.parallelCall(xpubList, async xpub => {
         const usedNewAddresses = await this._deriveNewAddresses(
           xpub,
           hdAccounts[xpub],
@@ -281,7 +282,7 @@ class Transaction {
         }
       })
     }
-    
+
     return fundedAddresses
   }
 
@@ -290,7 +291,7 @@ class Transaction {
    * Check if tx addresses are at or beyond the next unused
    * index for the HD chain. Derive additional addresses
    * to replace the gap limit and add those addresses to
-   * the database. Make sure to account for tx sending to 
+   * the database. Make sure to account for tx sending to
    * newly-derived addresses.
    *
    * @param {string} xpub
@@ -389,7 +390,7 @@ class Transaction {
     await db.addAddressesToHDAccount(xpub, newAddresses)
     return _.keys(usedNewAddresses)
   }
-   
+
 
   /**
    * Store the transaction in database

tracker/transactions-bundle.js

@@ -9,6 +9,7 @@ const LRU = require('lru-cache')
 const bitcoin = require('bitcoinjs-lib')
 const util = require('../lib/util')
 const db = require('../lib/db/mysql-db-wrapper')
+const addrHelper = require('../lib/bitcoin/addresses-helper')
 const network = require('../lib/bitcoin/network')
 const keys = require('../keys')[network.key]
 const activeNet = network.network
@@ -69,11 +70,7 @@ class TransactionsBundle {
     // Process transactions by slices of 5000 transactions
     const MAX_NB_TXS = 5000
     const lists = util.splitList(this.transactions, MAX_NB_TXS)
-
-    const results = await util.seriesCall(lists, list => {
-      return this._prefilterTransactions(list)
-    })
-
+    const results = await util.seriesCall(lists, txs => this._prefilterTransactions(txs))
     return _.flatten(results)
   }
 
@@ -110,15 +107,15 @@ class TransactionsBundle {
       const txid = tx.getId()
 
       indexedTxs[txid] = i
-      
+
       // If we already checked this tx
       if (TransactionsBundle.cache.has(txid))
-        continue 
+        continue
 
       for (const j in tx.outs) {
         try {
           const script = tx.outs[j].script
-          const address = bitcoin.address.fromOutputScript(script, activeNet)
+          const address = addrHelper.outputScript2Address(script)
           outputs.push(address)
           // Index the output
           if (!indexedOutputs[address])
@@ -174,7 +171,9 @@ class TransactionsBundle {
     }
 
     // Prefilter
-    const inRes = await db.getOutputSpends(inputs)
+    const lists = util.splitList(inputs, 1000)
+    const results = await util.parallelCall(lists, list => db.getOutputSpends(list))
+    const inRes = _.flatten(results)
 
     for (const i in inRes) {
       const key = inRes[i].txnTxid + '-' + inRes[i].outIndex