From ce22eec4c630bd34ff20d5e6930f2f43db4615e2 Mon Sep 17 00:00:00 2001 From: kenshin-samourai Date: Wed, 17 Mar 2021 12:10:26 +0100 Subject: [PATCH] Merge branch 'feat_dojo_optimizations' --- lib/bitcoin/addresses-helper.js | 84 ++++++++++++++++++++++++- lib/remote-importer/bitcoind-wrapper.js | 3 +- lib/util.js | 27 +++++--- pushtx/pushtx-processor.js | 9 +-- tracker/block.js | 10 +-- tracker/mempool-processor.js | 6 +- tracker/transaction.js | 25 ++++---- tracker/transactions-bundle.js | 17 +++-- 8 files changed, 137 insertions(+), 44 deletions(-) diff --git a/lib/bitcoin/addresses-helper.js b/lib/bitcoin/addresses-helper.js index 05039dd..e5d6495 100644 --- a/lib/bitcoin/addresses-helper.js +++ b/lib/bitcoin/addresses-helper.js @@ -7,7 +7,9 @@ const bitcoin = require('bitcoinjs-lib') const btcMessage = require('bitcoinjs-message') const activeNet = require('./network').network -const { p2pkh, p2sh, p2wpkh } = bitcoin.payments +const { p2pkh, p2sh, p2wpkh, p2wsh } = bitcoin.payments +const { OPS } = bitcoin.script + /** * A singleton providing Addresses helper functions @@ -106,6 +108,86 @@ class AddressesHelper { } } + /** + * Check if an output script is a P2PKH script + * @param {Buffer} scriptpubkey - scriptpubkey + * @returns {boolean} return true if output is a P2PKH script, otherwise return false + */ + isP2pkhScript(scriptpubkey) { + return scriptpubkey.length == 25 + && scriptpubkey[0] == OPS.OP_DUP + && scriptpubkey[1] == OPS.OP_HASH160 + && scriptpubkey[2] == 0x14 + && scriptpubkey[23] == OPS.OP_EQUALVERIFY + && scriptpubkey[24] == OPS.OP_CHECKSIG + } + + /** + * Check if an output script is a P2SH script + * @param {Buffer} scriptpubkey - scriptpubkey + * @returns {boolean} return true if output is a P2SH script, otherwise return false + */ + isP2shScript(scriptpubkey) { + return scriptpubkey.length == 23 + && scriptpubkey[0] == OPS.OP_HASH160 + && scriptpubkey[1] == 0x14 + && scriptpubkey[22] == OPS.OP_EQUAL + } + + /** + * Check if an output script is a P2WPKH script + * @param {Buffer} scriptpubkey - scriptpubkey + * @returns {boolean} return true if output is a P2WPKH script, otherwise return false + */ + isP2wpkhScript(scriptpubkey) { + return scriptpubkey.length == 22 + && scriptpubkey[0] == OPS.OP_0 + && scriptpubkey[1] == 0x14 + } + + /** + * Check if an output script is a P2WSH script + * @param {Buffer} scriptpubkey - scriptpubkey + * @returns {boolean} return true if output is a P2WSH script, otherwise return false + */ + isP2wshScript(scriptpubkey) { + return scriptpubkey.length == 34 + && scriptpubkey[0] == OPS.OP_0 + && scriptpubkey[1] == 0x20 + } + + /** + * Return the bitcoin address corresponding to an output script + * @param {Buffer} scriptpubkey - scriptpubkey + * @returns {string} bitcoin address + */ + outputScript2Address(scriptpubkey) { + if (this.isP2pkhScript(scriptpubkey)) + return p2pkh({ + output: scriptpubkey, + network: activeNet, + }).address + + if (this.isP2shScript(scriptpubkey)) + return p2sh({ + output: scriptpubkey, + network: activeNet, + }).address + + if (this.isP2wpkhScript(scriptpubkey)) + return p2wpkh({ + output: scriptpubkey, + network: activeNet, + }).address + + if (this.isP2wshScript(scriptpubkey)) + return p2wsh({ + output: scriptpubkey, + network: activeNet, + }).address + + throw 'unknown address format' + } } module.exports = new AddressesHelper() diff --git a/lib/remote-importer/bitcoind-wrapper.js b/lib/remote-importer/bitcoind-wrapper.js index c66e52d..2450dca 100644 --- a/lib/remote-importer/bitcoind-wrapper.js +++ b/lib/remote-importer/bitcoind-wrapper.js @@ -8,6 +8,7 @@ const bitcoin = require('bitcoinjs-lib') const RpcClient = require('../bitcoind-rpc/rpc-client') const rpcLatestBlock = require('../bitcoind-rpc/latest-block') const Logger = require('../logger') +const addrHelper = require('../bitcoin/addresses-helper') const network = require('../bitcoin/network') const activeNet = network.network const keys = require('../../keys')[network.key] @@ -44,7 +45,7 @@ class BitcoindWrapper extends Wrapper { */ _xlatScriptPubKey(scriptPubKey) { const bScriptPubKey = Buffer.from(scriptPubKey, 'hex') - return bitcoin.address.fromOutputScript(bScriptPubKey, activeNet) + return addrHelper.outputScript2Address(bScriptPubKey) } /** diff --git a/lib/util.js b/lib/util.js index eb81edb..5300fa5 100644 --- a/lib/util.js +++ b/lib/util.js @@ -32,10 +32,10 @@ class Util { * * @param {object} parents - map of {[key]: [incoming edge keys]} * @param {object} children - a map of {[key]: [outgoing edge keys]} - * @returns {object} + * @returns {object} * if graph has edges then * return error (graph has at least one cycle) - * else + * else * return L (a topologically sorted order) */ static topologicalOrdering(parents, children) { @@ -81,13 +81,22 @@ class Util { }).then(result => { results.push(result) }) - }, + }, Promise.resolve() ).then(function() { return results }) } + /** + * Execute parallel asynchronous calls to a function + * over a list of objects + */ + static parallelCall(list, fn) { + const operations = list.map(item => { return fn(item) }) + return Promise.all(operations) + } + /** * Delay the call to a function */ @@ -159,16 +168,16 @@ class Util { /** * Median of an array of values - */ + */ static median(arr, sorted) { if (arr.length == 0) return NaN if (arr.length == 1) return arr[0] if (!sorted) arr.sort(Util.cmpAsc) - + const midpoint = Math.floor(arr.length / 2) - + if (arr.length % 2) { // Odd-length array return arr[midpoint] @@ -252,10 +261,10 @@ class Util { // "Floor-x" const fx = Math.floor(x) - 1 - + // "Mod-x" const mx = x % 1 - + if (fx + 1 >= N) { return arr[fx] } else { @@ -340,7 +349,7 @@ class Util { const parts = [Util.pad10(h), Util.pad10(m), Util.pad10(s)] - if (d > 0) + if (d > 0) parts.splice(0, 0, Util.pad100(d)) const str = parts.join(':') diff --git a/pushtx/pushtx-processor.js b/pushtx/pushtx-processor.js index 0ba6d4c..07a6593 100644 --- a/pushtx/pushtx-processor.js +++ b/pushtx/pushtx-processor.js @@ -10,6 +10,7 @@ const Logger = require('../lib/logger') const errors = require('../lib/errors') const db = require('../lib/db/mysql-db-wrapper') const RpcClient = require('../lib/bitcoind-rpc/rpc-client') +const addrHelper = require('../lib/bitcoin/addresses-helper') const network = require('../lib/bitcoin/network') const activeNet = network.network const keys = require('../keys')[network.key] @@ -24,7 +25,7 @@ if (network.key == 'bitcoin') { /** - * A singleton providing a wrapper + * A singleton providing a wrapper * for pushing transactions with the local bitcoind */ class PushTxProcessor { @@ -52,7 +53,7 @@ class PushTxProcessor { * Enforce a strict verification mode on a list of outputs * @param {string} rawtx - raw bitcoin transaction in hex format * @param {array} vouts - output indices (integer) - * @returns {array} returns the indices of the faulty outputs + * @returns {array} returns the indices of the faulty outputs */ async enforceStrictModeVouts(rawtx, vouts) { const faultyOutputs = [] @@ -63,12 +64,12 @@ class PushTxProcessor { } catch(e) { throw errors.tx.PARSE } - // Check in db if addresses are known and have been used + // Check in db if addresses are known and have been used for (let vout of vouts) { if (vout >= tx.outs.length) throw errors.txout.VOUT const output = tx.outs[vout] - const address = bitcoin.address.fromOutputScript(output.script, activeNet) + const address = addrHelper.outputScript2Address(output.script) const nbTxs = await db.getAddressNbTransactions(address) if (nbTxs == null || nbTxs > 0) faultyOutputs.push(vout) diff --git a/tracker/block.js b/tracker/block.js index b8cfe91..20c6354 100644 --- a/tracker/block.js +++ b/tracker/block.js @@ -37,7 +37,7 @@ class Block extends TransactionsBundle { let block const txsForBroadcast = [] - + try { block = bitcoin.Block.fromHex(this.hex) this.transactions = block.transactions @@ -46,10 +46,10 @@ class Block extends TransactionsBundle { Logger.error(null, this.header) return Promise.reject(e) } - + const t0 = Date.now() let ntx = 0 - + // Filter transactions const filteredTxs = await this.prefilterTransactions() @@ -78,9 +78,9 @@ class Block extends TransactionsBundle { // Confirms the transactions const txids = this.transactions.map(t => t.getId()) - ntx = txids.length + ntx = txids.length const txidLists = util.splitList(txids, 100) - await util.seriesCall(txidLists, list => db.confirmTransactions(list, blockId)) + await util.parallelCall(txidLists, list => db.confirmTransactions(list, blockId)) // Logs and result returned const dt = ((Date.now()-t0)/1000).toFixed(1) diff --git a/tracker/mempool-processor.js b/tracker/mempool-processor.js index a9f4563..e3f7b00 100644 --- a/tracker/mempool-processor.js +++ b/tracker/mempool-processor.js @@ -69,7 +69,7 @@ class MempoolProcessor extends AbstractProcessor { /** * Stop processing - */ + */ async stop() { clearInterval(this.checkUnconfirmedId) clearInterval(this.processMempoolId) @@ -218,11 +218,11 @@ class MempoolProcessor extends AbstractProcessor { const unconfirmedTxs = await db.getUnconfirmedTransactions() if (unconfirmedTxs.length > 0) { - await util.seriesCall(unconfirmedTxs, tx => { + await util.parallelCall(unconfirmedTxs, tx => { try { return this.client.getrawtransaction(tx.txnTxid, true) .then(async rtx => { - if (!rtx.blockhash) return null + if (!rtx.blockhash) return null // Transaction is confirmed const block = await db.getBlockByHash(rtx.blockhash) if (block && block.blockID) { diff --git a/tracker/transaction.js b/tracker/transaction.js index f81d0b8..ee51450 100644 --- a/tracker/transaction.js +++ b/tracker/transaction.js @@ -8,6 +8,7 @@ const _ = require('lodash') const bitcoin = require('bitcoinjs-lib') const util = require('../lib/util') const Logger = require('../lib/logger') +const addrHelper = require('../lib/bitcoin/addresses-helper') const hdaHelper = require('../lib/bitcoin/hd-accounts-helper') const db = require('../lib/db/mysql-db-wrapper') const network = require('../lib/bitcoin/network') @@ -28,9 +29,9 @@ class Transaction { */ constructor(tx) { this.tx = tx - this.txid = this.tx.getId() + this.txid = this.tx.getId() // Id of transaction stored in db - this.storedTxnID = null + this.storedTxnID = null // Should this transaction be broadcast out to connected clients? this.doBroadcast = false } @@ -80,7 +81,7 @@ class Transaction { // Store database ids of double spend transactions const doubleSpentTxnIDs = [] // Store inputs of interest - const inputs = [] + const inputs = [] // Extracts inputs information let index = 0 @@ -153,13 +154,13 @@ class Transaction { async _processOutputs() { // Store outputs, keyed by address. Values are arrays of outputs const indexedOutputs = {} - + // Extracts outputs information let index = 0 for (let output of this.tx.outs) { try { - const address = bitcoin.address.fromOutputScript(output.script, activeNet) + const address = addrHelper.outputScript2Address(output.script) if (!indexedOutputs[address]) indexedOutputs[address] = [] @@ -174,7 +175,7 @@ class Transaction { // Array of addresses receiving tx outputs const addresses = _.keys(indexedOutputs) - + // Store a list of known addresses that received funds let fundedAddresses = [] @@ -203,7 +204,7 @@ class Transaction { for (let a of fundedAddresses) { outputs.push({ - txnID: this.storedTxnID, + txnID: this.storedTxnID, addrID: a.addrID, outIndex: a.outIndex, outAmount: a.outAmount, @@ -253,9 +254,9 @@ class Transaction { // Store a list of known addresses that received funds const fundedAddresses = [] const xpubList = _.keys(hdAccounts) - + if (xpubList.length > 0) { - await util.seriesCall(xpubList, async xpub => { + await util.parallelCall(xpubList, async xpub => { const usedNewAddresses = await this._deriveNewAddresses( xpub, hdAccounts[xpub], @@ -281,7 +282,7 @@ class Transaction { } }) } - + return fundedAddresses } @@ -290,7 +291,7 @@ class Transaction { * Check if tx addresses are at or beyond the next unused * index for the HD chain. Derive additional addresses * to replace the gap limit and add those addresses to - * the database. Make sure to account for tx sending to + * the database. Make sure to account for tx sending to * newly-derived addresses. * * @param {string} xpub @@ -389,7 +390,7 @@ class Transaction { await db.addAddressesToHDAccount(xpub, newAddresses) return _.keys(usedNewAddresses) } - + /** * Store the transaction in database diff --git a/tracker/transactions-bundle.js b/tracker/transactions-bundle.js index 081eca7..3eea200 100644 --- a/tracker/transactions-bundle.js +++ b/tracker/transactions-bundle.js @@ -9,6 +9,7 @@ const LRU = require('lru-cache') const bitcoin = require('bitcoinjs-lib') const util = require('../lib/util') const db = require('../lib/db/mysql-db-wrapper') +const addrHelper = require('../lib/bitcoin/addresses-helper') const network = require('../lib/bitcoin/network') const keys = require('../keys')[network.key] const activeNet = network.network @@ -69,11 +70,7 @@ class TransactionsBundle { // Process transactions by slices of 5000 transactions const MAX_NB_TXS = 5000 const lists = util.splitList(this.transactions, MAX_NB_TXS) - - const results = await util.seriesCall(lists, list => { - return this._prefilterTransactions(list) - }) - + const results = await util.seriesCall(lists, txs => this._prefilterTransactions(txs)) return _.flatten(results) } @@ -110,15 +107,15 @@ class TransactionsBundle { const txid = tx.getId() indexedTxs[txid] = i - + // If we already checked this tx if (TransactionsBundle.cache.has(txid)) - continue + continue for (const j in tx.outs) { try { const script = tx.outs[j].script - const address = bitcoin.address.fromOutputScript(script, activeNet) + const address = addrHelper.outputScript2Address(script) outputs.push(address) // Index the output if (!indexedOutputs[address]) @@ -174,7 +171,9 @@ class TransactionsBundle { } // Prefilter - const inRes = await db.getOutputSpends(inputs) + const lists = util.splitList(inputs, 1000) + const results = await util.parallelCall(lists, list => db.getOutputSpends(list)) + const inRes = _.flatten(results) for (const i in inRes) { const key = inRes[i].txnTxid + '-' + inRes[i].outIndex