optimize imports and rescans with sql batch requests

lib/remote-importer/remote-importer.js

@@ -115,7 +115,10 @@ class RemoteImporter {
       if (txMaps.txMap[txid])
         aTxs.push(txMaps.txMap[txid])
 
-    return util.seriesCall(aTxs, tx => this.addTransaction(tx, addrIdMap))
+    // Store the transactions by batches of 200 transactions
+    const txsChunks = util.splitList(aTxs, 200)
+    for (let txsChunk of txsChunks)
+      await this.addTransactions(txsChunk, addrIdMap)
   }
 
   /**
@@ -260,8 +263,8 @@ class RemoteImporter {
 
       Logger.info(` Got ${scanTx.length} transactions`)
 
-      // Retrieve the transactions by batches of 100 transactions
+      // Retrieve the transactions by batches of 200 transactions
-      const txsChunks = util.splitList(scanTx, 100)
+      const txsChunks = util.splitList(scanTx, 200)
       try {
         for (let txsChunk of txsChunks) {
           const txs = await rpcTxns.getTransactions(txsChunk, false)
@@ -369,70 +372,86 @@ class RemoteImporter {
   }
 
   /**
-   * Add a transaction to the database.
+   * Add a collection of transactions to the database.
-   * @param {object} tx - transaction object
+   * @param {object[]} txs - array of transaction objects
-   * @params {Promise}
+   * @params {object} addrIdMap - map address => addrId
+   * @returns {Promise}
    */
-  async addTransaction(tx, addrIdMap) {
+  async addTransactions(txs, addrIdMap) {
-    const outputs = []
-
     try {
-      // Store the transaction into the database
+      // Store the transactions into the database
-      await db.addTransaction(tx)
+      await db.addTransactions(txs)
-
+
-      // Confirm the transaction
+      // Confirm the transactions if needed
-      if (tx.block) {
+      const blocksHashes = new Set()
-        const block = await db.getBlockByHash(tx.block.hash)
+      for (let tx of txs)
-        if (block)
+        if (tx.block)
-          await db.confirmTransactions([tx.txid], block.blockID)
+          blocksHashes.add(tx.block.hash)
+
+      const blocks = await db.getBlocksByHashes(Array.from(blocksHashes))
+
+      for (let block of blocks) {
+        // Filter the transactions by blockHash
+        const filteredTxs = txs.filter(tx => (tx.block && tx.block.hash == block.blockHash))
+        if (filteredTxs.length > 0) {
+          const txids = filteredTxs.map(tx => tx.txid)
+          // Asynchronous confirmations 
+          db.confirmTransactions(txids, block.blockID)
+        }
       }
 
-      // Retrieve the database id for the transaction
+      // Retrieve the database ids for the transactions
-      let txnID = await db.ensureTransactionId(tx.txid)
+      const txids = txs.map(tx => tx.txid)
-
+      const mapTxsIds = await db.getTransactionsIds(txids)
-      // Process the outputs
+
-      for (let output of tx.outputs) {
+      // Store the outputs in db
-        if (addrIdMap[output.address]) {
+      const outputs = []
-          outputs.push({
+      for (let tx of txs) {
-            txnID,
+        for (let output of tx.outputs) {
-            addrID: addrIdMap[output.address],
+          if (addrIdMap[output.address]) {
-            outIndex: output.n,
+            outputs.push({
-            outAmount: output.value,
+              txnID: mapTxsIds[tx.txid],
-            outScript: output.scriptpubkey,
+              addrID: addrIdMap[output.address],
-          })
+              outIndex: output.n,
+              outAmount: output.value,
+              outScript: output.scriptpubkey,
+            })
+          }
         }
       }
-
       await db.addOutputs(outputs)
-
+      
-      // Process the inputs
+      // Store the inputs in db
-      // Get any outputs spent by the inputs of this transaction, add those
-      // database outIDs to the corresponding transaction inputs, and store.
-      const res = await db.getOutputIds(tx.inputs.map(input => input.outpoint))
-
-      const spent = {}
       const inputs = []
+      const spent = {}
+
+      // Get any outputs spent by the inputs of this transaction, 
+      // add those database outIDs to the corresponding inputs, and store.
+      let outpoints = []
+      for (let tx of txs)
+        outpoints = outpoints.concat(tx.inputs.map(input => input.outpoint))
 
+      const res = await db.getOutputIds(outpoints)
       for (let r of res)
         spent[`${r.txnTxid}-${r.outIndex}`] = r.outID
 
-      for (let input of tx.inputs) {
+      for (let tx of txs) {
-        let key = `${input.outpoint.txid}-${input.outpoint.vout}`
+        for (let input of tx.inputs) {
-        if (spent[key]) {
+          const key = `${input.outpoint.txid}-${input.outpoint.vout}`
-          inputs.push({
+          if (spent[key]) {
-            outID: spent[key],
+            inputs.push({
-            txnID,
+              outID: spent[key],
-            inIndex: input.n,
+              txnID: mapTxsIds[tx.txid],
-            inSequence: input.seq
+              inIndex: input.n,
-          })
+              inSequence: input.seq
+            })
+          }
         }
       }
-
       await db.addInputs(inputs)
 
     } catch(e) {
-      Logger.error(e, `RemoteImporter.addTransaction() : xpub ${tx.txid}`)
+      Logger.error(e, `RemoteImporter.addTransactions() :`)
-      Logger.error(null, JSON.stringify(tx,null,2))
     }
   }