From c52bcf3e2d6d2677ac44662175621872cad03259 Mon Sep 17 00:00:00 2001
From: kenshin-samourai <kenshin_samourai@tutanota.com>
Date: Mon, 26 Apr 2021 19:05:11 +0200
Subject: [PATCH] process addresses in chunks

---
 lib/remote-importer/remote-importer.js | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/lib/remote-importer/remote-importer.js b/lib/remote-importer/remote-importer.js
index bd17949..7e6a504 100644
--- a/lib/remote-importer/remote-importer.js
+++ b/lib/remote-importer/remote-importer.js
@@ -114,7 +114,13 @@ class RemoteImporter {
    * @returns {Promise}
    */
   async _importTransactions(addresses, txns) {
-    const addrIdMap = await db.getAddressesIds(addresses)
+    let addrIdMap = {}
+    const addrChunks = util.splitList(addresses, 1000)
+    const addrIdMaps = await util.parallelCall(addrChunks, addrChunk => {
+      return db.getAddressesIds(addrChunk)
+    })
+    for (let map of addrIdMaps)
+      addrIdMap = Object.assign(addrIdMap, map)
 
     // The transactions array must be topologically ordered, such that
     // entries earlier in the array MUST NOT depend upon any entry later
@@ -129,8 +135,9 @@ class RemoteImporter {
 
     // Store the transactions by batches of 200 transactions
     const txsChunks = util.splitList(aTxs, 200)
-    for (let txsChunk of txsChunks)
-      await this.addTransactions(txsChunk, addrIdMap)
+    await util.seriesCall(txsChunks, txsChunk => {
+      return this.addTransactions(txsChunk, addrIdMap)
+    })
   }
 
   /**
@@ -188,6 +195,8 @@ class RemoteImporter {
         addresses = addresses.concat(result.addresses)
       }
 
+      const aAddresses = addresses.map(a => a.address)
+
       this.importing[xpub] = {
         'status': this.STATUS_IMPORT,
         'txs': txns.length
@@ -195,10 +204,12 @@ class RemoteImporter {
 
       // Store the hdaccount and the addresses into the database
       await db.ensureHDAccountId(xpub, type)
-      await db.addAddressesToHDAccount(xpub, addresses)
+      const addrChunks = util.splitList(addresses, 1000)
+      await util.parallelCall(addrChunks, addrChunk => {
+        return db.addAddressesToHDAccount(xpub, addrChunk)
+      })
 
       // Store the transaction into the database
-      const aAddresses = addresses.map(a => a.address)
       await this._importTransactions(aAddresses, txns)
 
     } catch(e) {