Merge pull request #718 from bitpay/feat/2-step-opt

Feat/2 step opt

lib/blockchainexplorers/insight.js

@@ -41,6 +41,16 @@ Insight.prototype._doRequest = function(args, cb) {
       'User-Agent': this.userAgent,
     }
   };
+
+  if (log.level == 'verbose') {
+    var s  = JSON.stringify(args);
+
+    if ( s.length > 100 ) 
+      s= s.substr(0,100) + '...';
+    
+    log.debug('', 'Insight Q: %s', s);
+  }
+
   requestList(_.defaults(args, opts), cb);
 };
 
@@ -59,9 +69,10 @@ Insight.prototype.getUtxos = function(addresses, cb) {
     json: {
       addrs: _.uniq([].concat(addresses)).join(',')
     },
-    timeout: 120000,
   };
 
+  log.info('','Querying utxos: %s addrs', addresses.length);
+
   this._doRequest(args, function(err, res, unspent) {
     if (err || res.statusCode !== 200) return cb(_parseErr(err, res));
     return cb(null, unspent);
@@ -122,6 +133,9 @@ Insight.prototype.getTransactions = function(addresses, from, to, cb) {
     timeout: 120000,
   };
 
+
+  log.info('','Querying addresses: %s addrs', addresses.length);
+
   this._doRequest(args, function(err, res, txs) {
     if (err || res.statusCode !== 200) return cb(_parseErr(err, res));
 

lib/blockchainexplorers/request-list.js

@@ -31,11 +31,23 @@ var requestList = function(args, cb) {
   async.whilst(
     function() {
       nextUrl = urls.shift();
+      if (!nextUrl && success === 'false') 
+        log.warn('no more servers to test for the request');
       return nextUrl && !success;
     },
     function(a_cb) {
       args.uri = nextUrl;
+
+      var time = 0;
+      var interval = setInterval(function() {
+        time += 10;
+        log.debug('', 'Delayed insight query: %s, time: %d s', args.uri, time);
+      }, 10000);
+
       request(args, function(err, res, body) {
+        clearInterval(interval);
+        sucess = false;
+
         if (err) {
           log.warn('REQUEST FAIL: ' + nextUrl + ' ERROR: ' + err);
         }

lib/blockchainmonitor.js

@@ -152,7 +152,6 @@ BlockchainMonitor.prototype._handleThirdPartyBroadcasts = function(data, process
 
 BlockchainMonitor.prototype._handleIncomingPayments = function(coin, network, data) {
   var self = this;
-
   if (!data || !data.vout) return;
 
   var outs = _.compact(_.map(data.vout, function(v) {
@@ -197,7 +196,7 @@ BlockchainMonitor.prototype._handleIncomingPayments = function(coin, network, da
           walletId: walletId,
         });
         self.storage.softResetTxHistoryCache(walletId, function() {
-          self._updateActiveAddresses(address, function() {
+          self._updateAddressesWithBalance(address, function() {
             self._storeAndBroadcastNotification(notification, next);
           });
         });
@@ -208,14 +207,29 @@ BlockchainMonitor.prototype._handleIncomingPayments = function(coin, network, da
   });
 };
 
-BlockchainMonitor.prototype._updateActiveAddresses = function(address, cb) {
+BlockchainMonitor.prototype._updateAddressesWithBalance = function(address, cb) {
+
   var self = this;
 
-  self.storage.storeActiveAddresses(address.walletId, address.address, function(err) {
+  self.storage.fetchAddressesWithBalance(address.walletId, function(err, result) {
     if (err) {
       log.warn('Could not update wallet cache', err);
+      return cb(err);
     }
-    return cb(err);
+    var addresses = _.map(result,'address');
+
+    if (_.indexOf(addresses, address.address) >= 0) {
+      return cb();
+    }
+
+    addresses.push(address.address);
+    log.info('Activating address '+address);
+    self.storage.storeAddressesWithBalance(address.walletId, addresses, function(err) {
+      if (err) {
+        log.warn('Could not update wallet cache', err);
+      }
+      return cb(err);
+    });
   });
 };
 

lib/common/defaults.js

@@ -60,6 +60,12 @@ Defaults.FEE_LEVELS_FALLBACK = 2;
 // Minimum nb of addresses a wallet must have to start using 2-step balance optimization
 Defaults.TWO_STEP_BALANCE_THRESHOLD = 100;
 
+// Age Limit for addresses to be considered 'active' always
+Defaults.TWO_STEP_CREATION_HOURS = 24;
+
+// Time to prevent re-quering inactive addresses (MIN)
+Defaults.TWO_STEP_INACTIVE_CLEAN_DURATION_MIN = 60;
+
 Defaults.FIAT_RATE_PROVIDER = 'BitPay';
 Defaults.FIAT_RATE_FETCH_INTERVAL = 10; // In minutes
 Defaults.FIAT_RATE_MAX_LOOK_BACK_TIME = 120; // In minutes

lib/errors/errordefinitions.js

@@ -33,7 +33,7 @@ var errors = {
   UPGRADE_NEEDED: 'Client app needs to be upgraded',
   WALLET_ALREADY_EXISTS: 'Wallet already exists',
   WALLET_FULL: 'Wallet full',
-  WALLET_LOCKED: 'Wallet is locked',
+  WALLET_BUSY: 'Wallet is busy, try later',
   WALLET_NOT_COMPLETE: 'Wallet is not complete',
   WALLET_NOT_FOUND: 'Wallet not found',
 };

lib/expressapp.js

@@ -17,7 +17,7 @@ var Stats = require('./stats');
 
 log.disableColor();
 log.debug = log.verbose;
-log.level = 'info';
+log.level = 'verbose';
 
 var ExpressApp = function() {
   this.app = express();
@@ -75,14 +75,10 @@ ExpressApp.prototype.start = function(opts, cb) {
   } else {
     var morgan = require('morgan');
     morgan.token('walletId', function getId(req) {
-      return req.walletId
+      return req.walletId ?  '<' + req.walletId + '>' :  '<>';
     });
 
-    morgan.token('copayerId', function getId(req) {
-      return req.copayerId
-    });
-
-    var logFormat = ':remote-addr :date[iso] ":method :url" :status :res[content-length] :response-time ":user-agent" :walletId :copayerId';
+    var logFormat = ':walletId :remote-addr :date[iso] ":method :url" :status :res[content-length] :response-time ":user-agent"  ';
     var logOpts = {
       skip: function(req, res) {
         if (res.statusCode != 200) return false;
@@ -141,6 +137,7 @@ ExpressApp.prototype.start = function(opts, cb) {
   };
 
   function getServer(req, res) {
+    log.heading = '<>';
     var opts = {
       clientVersion: req.header('x-client-version'),
     };
@@ -183,6 +180,8 @@ ExpressApp.prototype.start = function(opts, cb) {
       req.walletId = server.walletId;
       req.copayerId = server.copayerId;
 
+      log.heading = '<' + req.walletId + '>';
+
       return cb(server);
     });
   };

lib/lock.js

@@ -31,7 +31,7 @@ Lock.prototype.runLocked = function(token, cb, task) {
   $.shouldBeDefined(token);
 
   this.lock.locked(token, 5 * 1000, 5 * 60 * 1000, function(err, release) {
-    if (err) return cb(Errors.WALLET_LOCKED);
+    if (err) return cb(Errors.WALLET_BUSY);
     var _cb = function() {
       cb.apply(null, arguments);
       release();

lib/model/wallet.js

@@ -2,6 +2,7 @@
 
 var _ = require('lodash');
 var util = require('util');
+var log = require('npmlog');
 var $ = require('preconditions').singleton();
 var Uuid = require('uuid');
 
@@ -156,6 +157,7 @@ Wallet.prototype.createAddress = function(isChange) {
   var self = this;
 
   var path = this.addressManager.getNewAddressPath(isChange);
+  log.verbose('Deriving addr:' + path);
   var address = Address.derive(self.id, this.addressType, this.publicKeyRing, path, this.m, this.coin, this.network, isChange);
   return address;
 };

lib/server.js

@@ -463,7 +463,7 @@ WalletService.prototype.getWalletFromIdentifier = function(opts, cb) {
       bc.getTransaction(opts.identifier, function(err, tx) {
         if (err || !tx) return nextCoinNetwork(false);
         var outputs = _.first(self._normalizeTxHistory(tx)).outputs;
-        var toAddresses = _.pluck(outputs, 'address');
+        var toAddresses = _.map(outputs, 'address');
         async.detect(toAddresses, function(addressStr, nextAddress) {
           self.storage.fetchAddressByCoin(coinNetwork.coin, addressStr, function(err, address) {
             if (err || !address) return nextAddress(false);
@@ -873,7 +873,7 @@ WalletService.prototype.savePreferences = function(opts, cb) {
     },
   }];
 
-  opts = _.pick(opts, _.pluck(preferences, 'name'));
+  opts = _.pick(opts, _.map(preferences, 'name'));
   try {
     _.each(preferences, function(preference) {
       var value = opts[preference.name];
@@ -1131,7 +1131,7 @@ WalletService.prototype._getUtxosForCurrentWallet = function(opts, cb) {
       });
     },
     function(next) {
-      addressStrs = _.pluck(allAddresses, 'address');
+      addressStrs = _.map(allAddresses, 'address');
       if (!opts.coin) return next();
 
       coin = opts.coin;
@@ -1141,7 +1141,6 @@ WalletService.prototype._getUtxosForCurrentWallet = function(opts, cb) {
       next();
     },
     function(next) {
-
       self._getUtxos(coin, addressStrs, function(err, utxos) {
         if (err) return next(err);
 
@@ -1155,7 +1154,7 @@ WalletService.prototype._getUtxosForCurrentWallet = function(opts, cb) {
       self.getPendingTxs({}, function(err, txps) {
         if (err) return next(err);
 
-        var lockedInputs = _.map(_.flatten(_.pluck(txps, 'inputs')), utxoKey);
+        var lockedInputs = _.map(_.flatten(_.map(txps, 'inputs')), utxoKey);
         _.each(lockedInputs, function(input) {
           if (utxoIndex[input]) {
             utxoIndex[input].locked = true;
@@ -1175,7 +1174,7 @@ WalletService.prototype._getUtxosForCurrentWallet = function(opts, cb) {
         limit: 100
       }, function(err, txs) {
         if (err) return next(err);
-        var spentInputs = _.map(_.flatten(_.pluck(txs, 'inputs')), utxoKey);
+        var spentInputs = _.map(_.flatten(_.map(txs, 'inputs')), utxoKey);
         _.each(spentInputs, function(input) {
           if (utxoIndex[input]) {
             utxoIndex[input].spent = true;
@@ -1242,97 +1241,109 @@ WalletService.prototype._totalizeUtxos = function(utxos) {
 };
 
 
-WalletService.prototype._getBalanceFromAddresses = function(opts, cb) {
+WalletService.prototype._getBalanceFromAddresses = function(opts, cb, i) {
   var self = this;
 
   var opts = opts || {};
 
-  self._getUtxosForCurrentWallet({
-    coin: opts.coin,
-    addresses: opts.addresses
-  }, function(err, utxos) {
-    if (err) return cb(err);
-
-    var balance = self._totalizeUtxos(utxos);
+  // This lock is to prevent server starvation on big wallets
+  self._runLocked(cb, function(cb) {
+    self._getUtxosForCurrentWallet({
+      coin: opts.coin,
+      addresses: opts.addresses
+    }, function(err, utxos) {
+      if (err) return cb(err);
 
-    // Compute balance by address
-    var byAddress = {};
-    _.each(_.indexBy(_.sortBy(utxos, 'address'), 'address'), function(value, key) {
-      byAddress[key] = {
-        address: key,
-        path: value.path,
-        amount: 0,
-      };
-    });
+      var balance = self._totalizeUtxos(utxos);
 
-    _.each(utxos, function(utxo) {
-      byAddress[utxo.address].amount += utxo.satoshis;
-    });
+      // Compute balance by address
+      var byAddress = {};
+      _.each(_.indexBy(_.sortBy(utxos, 'address'), 'address'), function(value, key) {
+        byAddress[key] = {
+          address: key,
+          path: value.path,
+          amount: 0,
+        };
+      });
 
-    balance.byAddress = _.values(byAddress);
+      _.each(utxos, function(utxo) {
+        byAddress[utxo.address].amount += utxo.satoshis;
+      });
 
-    return cb(null, balance);
+      balance.byAddress = _.values(byAddress);
+      return cb(null, balance);
+    });
   });
 };
 
 WalletService.prototype._getBalanceOneStep = function(opts, cb) {
   var self = this;
 
-  self.storage.fetchAddresses(self.walletId, function(err, addresses) {
-    if (err) return cb(err);
-    self._getBalanceFromAddresses({
-      coin: opts.coin,
-      addresses: addresses
-    }, function(err, balance) {
+    self.storage.fetchAddresses(self.walletId, function(err, addresses) {
       if (err) return cb(err);
+      self._getBalanceFromAddresses({
+        coin: opts.coin,
+        addresses: addresses
+      }, function(err, balance) {
+        if (err) return cb(err);
 
-      // Update cache
-      async.series([
-
-        function(next) {
-          self.storage.cleanActiveAddresses(self.walletId, next);
-        },
-        function(next) {
-          var active = _.pluck(balance.byAddress, 'address')
-          self.storage.storeActiveAddresses(self.walletId, active, next);
-        },
-      ], function(err) {
-        if (err) {
-          log.warn('Could not update wallet cache', err);
-        }
-        return cb(null, balance);
+        // Update cache
+        var withBalance = _.map(balance.byAddress, 'address')
+        self.storage.storeAddressesWithBalance(self.walletId, withBalance, function(err) {
+          if (err) {
+            log.warn('Could not update wallet cache', err);
+          }
+          return cb(null, balance);
+        });
       });
     });
-  });
 };
 
 
 WalletService.prototype._getActiveAddresses = function(cb) {
   var self = this;
 
-  self.storage.fetchActiveAddresses(self.walletId, function(err, active) {
+  self.storage.fetchAddressesWithBalance(self.walletId, function(err, addressesWB) {
     if (err) {
       log.warn('Could not fetch active addresses from cache', err);
       return cb();
     }
+    if (!_.isArray(addressesWB)) 
+      addressesWB = [];
 
-    if (!_.isArray(active)) return cb();
+    var now = Math.floor(Date.now() / 1000);
+    var fromTs = now - Defaults.TWO_STEP_CREATION_HOURS * 3600;
 
-    self.storage.fetchAddresses(self.walletId, function(err, allAddresses) {
+    self.storage.fetchNewAddresses(self.walletId, fromTs,  function(err, recent) {
       if (err) return cb(err);
 
-      var now = Math.floor(Date.now() / 1000);
-      var recent = _.pluck(_.filter(allAddresses, function(address) {
-        return address.createdOn > (now - 24 * 3600);
-      }), 'address');
+      var result = _.uniq(_.union(addressesWB,  recent), 'address');
+      return cb(null, result);
+    });
+  });
+};
 
-      var result = _.union(active, recent);
+WalletService.prototype._checkAndUpdateAddressCount = function(twoStepCache, cb) {
+  var self = this;
 
-      var index = _.indexBy(allAddresses, 'address');
-      result = _.compact(_.map(result, function(r) {
-        return index[r];
-      }));
-      return cb(null, result);
+  if (twoStepCache.addressCount > Defaults.TWO_STEP_BALANCE_THRESHOLD) {
+    log.info('Not counting addresses for '+ self.walletId);
+    return cb(null, true);
+  }
+
+  self.storage.countAddresses(self.walletId, function(err, addressCount) {
+    if (err) return cb(err);
+
+    if (addressCount < Defaults.TWO_STEP_BALANCE_THRESHOLD) 
+      return cb(null, false);
+
+    twoStepCache.addressCount = addressCount;
+
+    // updates cache
+    self.storage.storeTwoStepCache(self.walletId, twoStepCache, function(err) {
+      if (err) return cb(err);
+
+      return cb(null, true);
     });
   });
 };
@@ -1344,7 +1355,8 @@ WalletService.prototype._getActiveAddresses = function(cb) {
  * @param {Boolean} opts.twoStep[=false] - Optional - Use 2 step balance computation for improved performance
  * @returns {Object} balance - Total amount & locked amount.
  */
-WalletService.prototype.getBalance = function(opts, cb) {
+
+WalletService.prototype.getBalance = function(opts, cb, i) {
   var self = this;
 
   opts = opts || {};
@@ -1354,40 +1366,69 @@ WalletService.prototype.getBalance = function(opts, cb) {
       return cb(new ClientError('Invalid coin'));
   }
 
-  if (!opts.twoStep)
+  if (!opts.twoStep) {
     return self._getBalanceOneStep(opts, cb);
+  }
+
 
-  self.storage.countAddresses(self.walletId, function(err, nbAddresses) {
+  self.storage.getTwoStepCache(self.walletId, function(err, twoStepCache) {
     if (err) return cb(err);
-    if (nbAddresses < Defaults.TWO_STEP_BALANCE_THRESHOLD) {
-      return self._getBalanceOneStep(opts, cb);
-    }
-    self._getActiveAddresses(function(err, activeAddresses) {
+    twoStepCache = twoStepCache || {};
+
+    self._checkAndUpdateAddressCount(twoStepCache, function(err, needsTwoStep ) {
       if (err) return cb(err);
-      if (!_.isArray(activeAddresses)) {
+
+      if (!needsTwoStep) {
         return self._getBalanceOneStep(opts, cb);
-      } else {
-        log.debug('Requesting partial balance for ' + activeAddresses.length + ' out of ' + nbAddresses + ' addresses');
-        self._getBalanceFromAddresses({
-          coin: opts.coin,
-          addresses: activeAddresses
-        }, function(err, partialBalance) {
-          if (err) return cb(err);
-          cb(null, partialBalance);
-          setTimeout(function() {
-            self._getBalanceOneStep(opts, function(err, fullBalance) {
-              if (err) return;
-              if (!_.isEqual(partialBalance, fullBalance)) {
-                log.info('Balance in active addresses differs from final balance');
-                self._notify('BalanceUpdated', fullBalance, {
-                  isGlobal: true
-                });
-              }
-            });
-          }, 1);
-          return;
-        });
       }
+
+      self._getActiveAddresses(function(err, activeAddresses) {
+        if (err) return cb(err);
+        if (!_.isArray(activeAddresses)) {
+          return self._getBalanceOneStep(opts, cb);
+        } else {
+          log.debug('Requesting partial balance for ' + activeAddresses.length + ' addresses');
+
+          self._getBalanceFromAddresses({
+            coin: opts.coin,
+            addresses: activeAddresses
+          }, function(err, partialBalance) {
+            if (err) return cb(err);
+            cb(null, partialBalance);
+
+            var now = Math.floor(Date.now() / 1000);
+
+            if (twoStepCache.lastEmpty > now - Defaults.TWO_STEP_INACTIVE_CLEAN_DURATION_MIN * 60 )  {
+              log.debug('Not running the FULL balance query due to TWO_STEP_INACTIVE_CLEAN_DURATION_MIN ');
+              return;
+            }
+
+
+            setTimeout(function() {
+              log.debug('Running full balance query');
+              self._getBalanceOneStep(opts, function(err, fullBalance) {
+                if (err) return;
+                if (!_.isEqual(partialBalance, fullBalance)) {
+                  log.info('Balance in active addresses differs from final balance');
+                  self._notify('BalanceUpdated', fullBalance, {
+                    isGlobal: true
+                  });
+                } else {
+                  // updates cache
+                  twoStepCache.lastEmpty = now;
+
+                  // updates cache
+                  return self.storage.storeTwoStepCache(self.walletId, twoStepCache, function(err) {
+                    return;
+      });
+
+                }
+              });
+            }, 1);
+            return;
+          }, i);
+        }
+      });
     });
   });
 };
@@ -1424,7 +1465,7 @@ WalletService.prototype.getSendMaxInfo = function(opts, cb) {
       if (!_.any(feeLevels, {
         name: opts.feeLevel
       }))
-        return cb(new ClientError('Invalid fee level. Valid values are ' + _.pluck(feeLevels, 'name').join(', ')));
+        return cb(new ClientError('Invalid fee level. Valid values are ' + _.map(feeLevels, 'name').join(', ')));
     }
 
     if (_.isNumber(opts.feePerKb)) {
@@ -1567,7 +1608,7 @@ WalletService.prototype.getFeeLevels = function(opts, cb) {
   var feeLevels = Defaults.FEE_LEVELS[opts.coin];
 
   function samplePoints() {
-    var definedPoints = _.uniq(_.pluck(feeLevels, 'nbBlocks'));
+    var definedPoints = _.uniq(_.map(feeLevels, 'nbBlocks'));
     return _.uniq(_.flatten(_.map(definedPoints, function(p) {
       return _.range(p, p + Defaults.FEE_LEVELS_FALLBACK + 1);
     })));
@@ -1986,7 +2027,7 @@ WalletService.prototype._validateAndSanitizeTxOpts = function(wallet, opts, cb)
         if (!_.any(feeLevels, {
           name: opts.feeLevel
         }))
-          return next(new ClientError('Invalid fee level. Valid values are ' + _.pluck(feeLevels, 'name').join(', ')));
+          return next(new ClientError('Invalid fee level. Valid values are ' + _.map(feeLevels, 'name').join(', ')));
       }
 
       if (_.isNumber(opts.feePerKb)) {
@@ -2642,7 +2683,7 @@ WalletService.prototype.rejectTx = function(opts, cb) {
         },
         function(next) {
           if (txp.status == 'rejected') {
-            var rejectedBy = _.pluck(_.filter(txp.actions, {
+            var rejectedBy = _.map(_.filter(txp.actions, {
               type: 'reject'
             }), 'copayerId');
 
@@ -3005,7 +3046,7 @@ WalletService.prototype.getTxHistory = function(opts, cb) {
       function(next) {
         if (txs) return next();
 
-        var addressStrs = _.pluck(addresses, 'address');
+        var addressStrs = _.map(addresses, 'address');
         var bc = self._getBlockchainExplorer(wallet.coin, wallet.network);
         if (!bc) return next(new Error('Could not get blockchain explorer instance'));
         bc.getTransactions(addressStrs, from, to, function(err, rawTxs, total) {
@@ -3169,6 +3210,7 @@ WalletService.prototype.scan = function(opts, cb) {
     var gap = Defaults.SCAN_ADDRESS_GAP;
 
     async.whilst(function() {
+      log.debug('Scanning addr gap:'+ inactiveCounter);
       return inactiveCounter < gap;
     }, function(next) {
       var address = derivator.derive();
@@ -3225,7 +3267,9 @@ WalletService.prototype.scan = function(opts, cb) {
               if (err) return cb(err);
               wallet.scanStatus = error ? 'error' : 'success';
               self.storage.storeWallet(wallet, function() {
-                return cb(error);
+                self.storage.storeTwoStepCache(self.walletId, {}, function(err) {
+                  return cb(error);
+                });
               });
             })
           });

lib/storage.js

@@ -70,6 +70,10 @@ Storage.prototype._createIndexes = function() {
   this.db.collection(collections.ADDRESSES).createIndex({
     address: 1,
   });
+  this.db.collection(collections.ADDRESSES).createIndex({
+    walletId: 1,
+    address: 1,
+  });
   this.db.collection(collections.EMAIL_QUEUE).createIndex({
     id: 1,
   });
@@ -468,6 +472,28 @@ Storage.prototype.fetchAddresses = function(walletId, cb) {
   });
 };
 
+
+Storage.prototype.fetchNewAddresses = function(walletId, fromTs, cb) {
+  var self = this;
+
+  this.db.collection(collections.ADDRESSES).find({
+    walletId: walletId,
+    createdOn: {
+      $gte: fromTs,
+    },
+  }).sort({
+    createdOn: 1
+  }).toArray(function(err, result) {
+    if (err) return cb(err);
+    if (!result) return cb();
+    var addresses = _.map(result, function(address) {
+      return Model.Address.fromObj(address);
+    });
+    return cb(null, addresses);
+  });
+};
+
+
 Storage.prototype.countAddresses = function(walletId, cb) {
   this.db.collection(collections.ADDRESSES).find({
     walletId: walletId,
@@ -598,52 +624,91 @@ Storage.prototype.fetchEmailByNotification = function(notificationId, cb) {
   });
 };
 
-Storage.prototype.cleanActiveAddresses = function(walletId, cb) {
+Storage.prototype.storeTwoStepCache = function(walletId, cacheStatus, cb) {
   var self = this;
+  self.db.collection(collections.CACHE).update( { 
+    walletId: walletId,
+    type: 'twoStep',
+    key: null,
+  }, {
+    "$set":
+    { 
+      addressCount: cacheStatus.addressCount,
+      lastEmpty: cacheStatus.lastEmpty, 
+    }
+  }, {
+    w: 1,
+    upsert: true,
+  }, cb);
+};
 
-  async.series([
+Storage.prototype.getTwoStepCache = function(walletId, cb) {
+  var self = this;
 
-    function(next) {
-      self.db.collection(collections.CACHE).remove({
-        walletId: walletId,
-        type: 'activeAddresses',
-      }, {
-        w: 1
-      }, next);
-    },
-    function(next) {
-      self.db.collection(collections.CACHE).insert({
-        walletId: walletId,
-        type: 'activeAddresses',
-        key: null
-      }, {
-        w: 1
-      }, next);
-    },
-  ], cb);
+
+  self.db.collection(collections.CACHE).findOne({
+    walletId: walletId,
+    type: 'twoStep',
+    key: null
+  }, function(err, result) {
+    if (err) return cb(err);
+    if (!result) return cb();
+    return cb(null, result);
+  });
 };
 
-Storage.prototype.storeActiveAddresses = function(walletId, addresses, cb) {
+
+Storage.prototype.storeAddressesWithBalance = function(walletId, addresses, cb) {
   var self = this;
 
-  if (_.isEmpty(addresses)) return cb();
-  async.each(addresses, function(address, next) {
-    var record = {
-      walletId: walletId,
-      type: 'activeAddresses',
-      key: address,
-    };
-    self.db.collection(collections.CACHE).update({
-      walletId: record.walletId,
-      type: record.type,
-      key: record.key,
-    }, record, {
-      w: 1,
-      upsert: true,
-    }, next);
+  if (_.isEmpty(addresses))  
+    addresses = [];
+
+  self.db.collection(collections.CACHE).update({
+    walletId: walletId,
+    type: 'addressesWithBalance',
+    key: null,
+  }, {
+    "$set":
+    { 
+    addresses: addresses,
+    }
+  }, {
+    w: 1,
+    upsert: true,
   }, cb);
 };
 
+Storage.prototype.fetchAddressesWithBalance = function(walletId, cb) {
+  var self = this;
+ 
+
+  self.db.collection(collections.CACHE).findOne({
+    walletId: walletId,
+    type: 'addressesWithBalance',
+    key: null, 
+  }, function(err, result) {
+    if (err) return cb(err);
+    if (_.isEmpty(result)) return cb(null, []);
+
+
+    self.db.collection(collections.ADDRESSES).find({
+      walletId: walletId,
+      address: { $in: result.addresses },
+    }).toArray(function(err, result2) {
+      if (err) return cb(err);
+      if (!result2) return cb(null, []);
+
+      var addresses = _.map(result2, function(address) {
+        return Model.Address.fromObj(address);
+      });
+      return cb(null, addresses);
+    });
+
+  });
+};
+
+
 // --------         ---------------------------  Total
 //           > Time >
 //                       ^to     <=  ^from
@@ -694,7 +759,7 @@ Storage.prototype.getTxHistoryCache = function(walletId, from, to, cb) {
         return cb();
       }
 
-      var txs = _.pluck(result, 'tx');
+      var txs = _.map(result, 'tx');
       return cb(null, txs);
     });
   })
@@ -796,23 +861,6 @@ Storage.prototype.storeTxHistoryCache = function(walletId, totalItems, firstPosi
 };
 
 
-
-
-
-Storage.prototype.fetchActiveAddresses = function(walletId, cb) {
-  var self = this;
-
-  self.db.collection(collections.CACHE).find({
-    walletId: walletId,
-    type: 'activeAddresses',
-  }).toArray(function(err, result) {
-    if (err) return cb(err);
-    if (_.isEmpty(result)) return cb();
-
-    return cb(null, _.compact(_.pluck(result, 'key')));
-  });
-};
-
 Storage.prototype.storeFiatRate = function(providerName, rates, cb) {
   var self = this;
 

package-lock.json

@@ -2709,9 +2709,9 @@
       }
     },
     "safe": {
-      "version": "0.4.5",
-      "resolved": "https://registry.npmjs.org/safe/-/safe-0.4.5.tgz",
-      "integrity": "sha1-MWEo64HpZFEe3OKAd55aP/N/LVg=",
+      "version": "0.3.9",
+      "resolved": "https://registry.npmjs.org/safe/-/safe-0.3.9.tgz",
+      "integrity": "sha1-F4FZvuRXkawhYoruLau3SlLV0HI=",
       "dev": true
     },
     "safe-buffer": {
@@ -3167,15 +3167,14 @@
       }
     },
     "tingodb": {
-      "version": "0.3.5",
-      "resolved": "https://registry.npmjs.org/tingodb/-/tingodb-0.3.5.tgz",
-      "integrity": "sha1-xs0G2WxzuCp4GyjDCC/EoZBaRWg=",
+      "version": "0.5.1",
+      "resolved": "https://registry.npmjs.org/tingodb/-/tingodb-0.5.1.tgz",
+      "integrity": "sha1-U8rlLLTUMQgImMwZ/F0EMzoDAck=",
       "dev": true,
       "requires": {
-        "async": "0.9.2",
         "bson": "0.2.22",
-        "lodash": "2.4.2",
-        "safe": "0.4.5"
+        "lodash": "4.11.2",
+        "safe": "0.3.9"
       },
       "dependencies": {
         "bson": {
@@ -3189,9 +3188,9 @@
           }
         },
         "lodash": {
-          "version": "2.4.2",
-          "resolved": "https://registry.npmjs.org/lodash/-/lodash-2.4.2.tgz",
-          "integrity": "sha1-+t2DS5aDBz2hebPq5tnA0VBT9z4=",
+          "version": "4.11.2",
+          "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.11.2.tgz",
+          "integrity": "sha1-1rQzixEKWOIdrlzrz9u/0rxM2zs=",
           "dev": true
         },
         "nan": {

package.json

@@ -61,7 +61,7 @@
     "proxyquire": "^1.7.2",
     "sinon": "1.10.3",
     "supertest": "*",
-    "tingodb": "^0.3.4"
+    "tingodb": "^0.5.1"
   },
   "scripts": {
     "start": "./start.sh",

test/integration/bcmonitor.js

@@ -13,7 +13,6 @@ log.level = 'info';
 var WalletService = require('../../lib/server');
 var BlockchainMonitor = require('../../lib/blockchainmonitor');
 
-var TestData = require('../testdata');
 var helpers = require('./helpers');
 var storage, blockchainExplorer;
 
@@ -90,6 +89,112 @@ describe('Blockchain monitor', function() {
     });
   });
 
+  it('should update addressWithBalance cache on 1 incoming tx', function(done) {
+    server.createAddress({}, function(err, address) {
+      should.not.exist(err);
+      var incoming = {
+        txid: '123',
+        vout: [{}],
+      };
+      server.storage.fetchAddressesWithBalance(wallet.id, function(err,ret) {
+        should.not.exist(err);
+        _.isEmpty(ret).should.equal(true);
+        incoming.vout[0][address.address] = 1500;
+        socket.handlers['tx'](incoming);
+        setTimeout(function() {
+          server.storage.fetchAddressesWithBalance(wallet.id, function(err,ret) {
+            should.not.exist(err);
+            ret.length.should.equal(1);
+            ret[0].address.should.equal(address.address);
+            done();
+          });
+        }, 100);
+      });
+    });
+  });
+
+  it('should update addressWithBalance cache on 2 incoming tx, same address', function(done) {
+    server.createAddress({}, function(err, address) {
+      should.not.exist(err);
+
+      server.storage.fetchAddressesWithBalance(wallet.id, function(err,ret) {
+        should.not.exist(err);
+        _.isEmpty(ret).should.equal(true);
+
+        var incoming = {
+          txid: '123',
+          vout: [{}],
+        };
+        incoming.vout[0][address.address] = 1500;
+
+        socket.handlers['tx'](incoming);
+        setTimeout(function() {
+
+
+          var incoming2 = {
+            txid: '456',
+            vout: [{}],
+          };
+          incoming2.vout[0][address.address] = 2500;
+          socket.handlers['tx'](incoming2);
+
+          setTimeout(function() {
+            server.storage.fetchAddressesWithBalance(wallet.id, function(err,ret) {
+              should.not.exist(err);
+              ret.length.should.equal(1);
+              ret[0].address.should.equal(address.address);
+              done();
+            });
+          }, 100);
+        }, 100);
+      });
+    });
+  });
+
+
+  it('should update addressWithBalance cache on 2 incoming tx, different address', function(done) {
+    server.createAddress({}, function(err, address) {
+      should.not.exist(err);
+      server.createAddress({}, function(err, address2) {
+        should.not.exist(err);
+        server.storage.fetchAddressesWithBalance(wallet.id, function(err,ret) {
+          should.not.exist(err);
+          _.isEmpty(ret).should.equal(true);
+
+          var incoming = {
+            txid: '123',
+            vout: [{}],
+          };
+          incoming.vout[0][address.address] = 1500;
+          socket.handlers['tx'](incoming);
+
+          setTimeout(function() {
+
+            var incoming2 = {
+              txid: '456',
+              vout: [{}],
+            };
+            incoming2.vout[0][address2.address] = 500;
+            socket.handlers['tx'](incoming2);
+
+            setTimeout(function() {
+              server.storage.fetchAddressesWithBalance(wallet.id, function(err,ret) {
+                should.not.exist(err);
+                ret.length.should.equal(2);
+                ret[0].address.should.equal(address.address);
+                done();
+              });
+            }, 100);
+          }, 100);
+        });
+      });
+    });
+  });
+
+
+
+
+
   it('should not notify copayers of incoming txs more than once', function(done) {
     server.createAddress({}, function(err, address) {
       should.not.exist(err);

test/integration/server.js

@@ -2027,12 +2027,13 @@ describe('Wallet service', function() {
           });
         },
         function(next) {
-          clock.tick(7 * 24 * 3600 * 1000);
+          clock.tick(7 * Defaults.TWO_STEP_CREATION_HOURS * 3600 * 1000);
           helpers.createAddresses(server, wallet, 2, 0, function(addrs) {
             newAddrs = addrs;
             server._getActiveAddresses(function(err, active) {
               should.not.exist(err);
-              should.not.exist(active);
+              active.length.should.equal(2);
+
               helpers.stubUtxos(server, wallet, [1, 2], {
                 addresses: [oldAddrs[0], newAddrs[0]],
               }, function() {
@@ -2047,7 +2048,9 @@ describe('Wallet service', function() {
           }, function(err, balance) {
             should.not.exist(err);
             should.exist(balance);
-            balance.totalAmount.should.equal(helpers.toSatoshi(3));
+
+            // Only should see newAddr[0]
+            balance.totalAmount.should.equal(helpers.toSatoshi(2));
             next();
           });
         },
@@ -2058,6 +2061,7 @@ describe('Wallet service', function() {
           server._getActiveAddresses(function(err, active) {
             should.not.exist(err);
             should.exist(active);
+            // 1 old (with balance) + 2 news
             active.length.should.equal(3);
             next();
           });
@@ -2099,6 +2103,76 @@ describe('Wallet service', function() {
       });
     });
 
+    it('should not count addresses if wallet have already passed the threshold', function(done) {
+      var oldAddrs, newAddrs, spy;
+      async.series([
+
+        function(next) {
+          spy = sinon.spy(server.storage, 'countAddresses');
+          helpers.createAddresses(server, wallet, 2, 0, function(addrs) {
+            oldAddrs = addrs;
+            next();
+          });
+        },
+        function(next) {
+          clock.tick(7 * Defaults.TWO_STEP_CREATION_HOURS * 3600 * 1000);
+          helpers.createAddresses(server, wallet, 2, 0, function(addrs) {
+            newAddrs = addrs;
+            server._getActiveAddresses(function(err, active) {
+              should.not.exist(err);
+              // the new 2
+              active.length.should.equal(2);
+              helpers.stubUtxos(server, wallet, [1, 2], {
+                addresses: [oldAddrs[0], newAddrs[0]],
+              }, function() {
+                next();
+              });
+            });
+          });
+        },
+        function(next) {
+          server.getBalance({
+            twoStep: true
+          }, function(err, balance) {
+            should.not.exist(err);
+            should.exist(balance);
+            balance.totalAmount.should.equal(helpers.toSatoshi(2));
+            next();
+          });
+        },
+        function(next) {
+          setTimeout(next, 100);
+        },
+        function(next) {
+          spy.calledOnce.should.equal(true);
+          next();
+        },
+        function(next) {
+          server.getBalance({
+            twoStep: true
+          }, function(err, balance) {
+            should.not.exist(err);
+            should.exist(balance);
+            balance.totalAmount.should.equal(helpers.toSatoshi(3));
+            next();
+          });
+        },
+        function(next) {
+          setTimeout(next, 100);
+        },
+        // should NOT count addresses again! (still only one call)
+        function(next) {
+          spy.calledOnce.should.equal(true);
+          next();
+        },
+ 
+      ], function(err) {
+        should.not.exist(err);
+        done();
+      });
+    });
+
+
     it('should not trigger notification when only balance of prioritary addresses is updated', function(done) {
       var oldAddrs, newAddrs;
 
@@ -2111,7 +2185,7 @@ describe('Wallet service', function() {
           });
         },
         function(next) {
-          clock.tick(7 * 24 * 3600 * 1000);
+          clock.tick(7 * Defaults.TWO_STEP_CREATION_HOURS * 3600 * 1000);
           helpers.createAddresses(server, wallet, 2, 0, function(addrs) {
             newAddrs = addrs;
             helpers.stubUtxos(server, wallet, [1, 2], {
@@ -2121,6 +2195,17 @@ describe('Wallet service', function() {
             });
           });
         },
+        function(next) {
+          setTimeout(next, 100);
+        },
+        function(next) {
+          server.getNotifications({}, function(err, notifications) {
+            should.not.exist(err);
+            var last = _.last(notifications);
+            last.type.should.not.equal('BalanceUpdated');
+            next();
+          });
+        },
         function(next) {
           server.getBalance({
             twoStep: true
@@ -2169,6 +2254,251 @@ describe('Wallet service', function() {
       });
     });
 
+    it('should not do 2 steps if called 2 times given the first one there found no funds on non-active addresses', function(done) {
+      var oldAddrs, newAddrs, spy;
+
+      async.series([
+        function(next) {
+          spy = sinon.spy(server, '_getBalanceOneStep');
+          helpers.createAddresses(server, wallet, 2, 0, function(addrs) {
+            oldAddrs = addrs;
+            next();
+          });
+        },
+        function(next) {
+          clock.tick(7 * Defaults.TWO_STEP_CREATION_HOURS * 3600 * 1000);
+          helpers.createAddresses(server, wallet, 2, 0, function(addrs) {
+            newAddrs = addrs;
+            helpers.stubUtxos(server, wallet, [1, 2], {
+              addresses: newAddrs,
+            }, function() {
+              next();
+            });
+          });
+        },
+        function(next) {
+          server.getBalance({
+            twoStep: true
+          }, function(err, balance) {
+            should.not.exist(err);
+            should.exist(balance);
+            balance.totalAmount.should.equal(helpers.toSatoshi(3));
+            next();
+          });
+        },
+        function(next) {
+          setTimeout(next, 100);
+        },
+        // Should _oneStep should be called once
+        function(next) {
+          spy.calledOnce.should.equal(true);
+          next();
+        },
+        function(next) {
+          helpers.stubUtxos(server, wallet, 0.5, {
+            addresses: newAddrs[0],
+            keepUtxos: true,
+          }, function() {
+            next();
+          });
+        },
+        function(next) {
+          server.getBalance({
+            twoStep: true
+          }, function(err, balance) {
+            should.not.exist(err);
+            should.exist(balance);
+            balance.totalAmount.should.equal(helpers.toSatoshi(3.5));
+            next();
+          });
+        },
+        function(next) {
+          setTimeout(next, 100);
+        },
+        // Should _oneStep should be called once
+        function(next) {
+          spy.calledOnce.should.equal(true);
+          next();
+        },
+        // Should not trigger notification either
+        function(next) {
+          server.getNotifications({}, function(err, notifications) {
+            should.not.exist(err);
+            var last = _.last(notifications);
+            last.type.should.not.equal('BalanceUpdated');
+            next();
+          });
+        },
+      ], function(err) {
+        should.not.exist(err);
+        done();
+      });
+    });
+
+  it('should do 2 steps if called 2 times given the first one there found no funds on non-active addresses, but times passes', function(done) {
+      var oldAddrs, newAddrs, spy;
+
+      async.series([
+        function(next) {
+          spy = sinon.spy(server, '_getBalanceOneStep');
+          helpers.createAddresses(server, wallet, 2, 0, function(addrs) {
+            oldAddrs = addrs;
+            next();
+          });
+        },
+        function(next) {
+          clock.tick(7 * Defaults.TWO_STEP_CREATION_HOURS * 3600 * 1000);
+          helpers.createAddresses(server, wallet, 2, 0, function(addrs) {
+            newAddrs = addrs;
+            helpers.stubUtxos(server, wallet, [1, 2], {
+              addresses: newAddrs,
+            }, function() {
+              next();
+            });
+          });
+        },
+        function(next) {
+          server.getBalance({
+            twoStep: true
+          }, function(err, balance) {
+            should.not.exist(err);
+            should.exist(balance);
+            balance.totalAmount.should.equal(helpers.toSatoshi(3));
+            next();
+          });
+        },
+        function(next) {
+          setTimeout(next, 100);
+        },
+        // Should _oneStep should be called once
+        function(next) {
+          spy.calledOnce.should.equal(true);
+          next();
+        },
+        function(next) {
+          helpers.stubUtxos(server, wallet, 0.5, {
+            addresses: newAddrs[0],
+            keepUtxos: true,
+          }, function() {
+            clock.tick(2 * Defaults.TWO_STEP_INACTIVE_CLEAN_DURATION_MIN * 60 * 1000);
+            next();
+          });
+        },
+        function(next) {
+          server.getBalance({
+            twoStep: true
+          }, function(err, balance) {
+            should.not.exist(err);
+            should.exist(balance);
+            balance.totalAmount.should.equal(helpers.toSatoshi(3.5));
+            next();
+          });
+        },
+        function(next) {
+          setTimeout(next, 100);
+        },
+        function(next) {
+          spy.calledTwice.should.equal(true);
+          next();
+        },
+      ], function(err) {
+        should.not.exist(err);
+        done();
+      });
+    });
+
+  it('should do 2 steps if called 2 times given the first one there found funds on non-active addresses', function(done) {
+      var oldAddrs, newAddrs, spy, notificationCount;
+
+      async.series([
+        function(next) {
+          spy = sinon.spy(server, '_getBalanceOneStep');
+          helpers.createAddresses(server, wallet, 2, 0, function(addrs) {
+            oldAddrs = addrs;
+            next();
+          });
+        },
+        function(next) {
+          clock.tick(7 * Defaults.TWO_STEP_CREATION_HOURS * 3600 * 1000);
+          helpers.createAddresses(server, wallet, 2, 0, function(addrs) {
+            newAddrs = addrs;
+            helpers.stubUtxos(server, wallet, [1, 2], {
+              addresses: [ oldAddrs[0], newAddrs[0] ],
+            }, function() {
+              next();
+            });
+          });
+        },
+        function(next) {
+          server.getBalance({
+            twoStep: true
+          }, function(err, balance) {
+            should.not.exist(err);
+            should.exist(balance);
+            balance.totalAmount.should.equal(helpers.toSatoshi(2));
+            next();
+          });
+        },
+        function(next) {
+          setTimeout(next, 100);
+        },
+        // Should not trigger notification either
+        function(next) {
+          server.getNotifications({}, function(err, notifications) {
+            notificationCount = notifications.length;
+
+            should.not.exist(err);
+            var last = _.last(notifications);
+            last.type.should.equal('BalanceUpdated');
+            next();
+          });
+        },
+        // Should _oneStep should be called once
+        function(next) {
+          spy.calledOnce.should.equal(true);
+          next();
+        },
+        function(next) {
+          helpers.stubUtxos(server, wallet, 0.5, {
+            addresses: newAddrs[0],
+            keepUtxos: true,
+          }, function() {
+            next();
+          });
+        },
+        function(next) {
+          server.getBalance({
+            twoStep: true
+          }, function(err, balance) {
+            should.not.exist(err);
+            should.exist(balance);
+            balance.totalAmount.should.equal(helpers.toSatoshi(3.5));
+            next();
+          });
+        },
+        function(next) {
+          setTimeout(next, 100);
+        },
+        // Should _oneStep should be called TWICE
+        function(next) {
+          spy.calledTwice.should.equal(true);
+          next();
+        },
+        // Should not trigger notification either
+        function(next) {
+          server.getNotifications({}, function(err, notifications) {
+            should.not.exist(err);
+            notifications.length.should.equal(notificationCount);
+            next();
+          });
+        },
+      ], function(err) {
+        should.not.exist(err);
+        done();
+      });
+    });
+
+
     it('should resolve balance of new addresses immediately', function(done) {
       var addresses;
 
@@ -2192,7 +2522,10 @@ describe('Wallet service', function() {
             should.exist(balance);
             balance.totalAmount.should.equal(helpers.toSatoshi(3));
             next();
-          });
+          }, 1);
+        },
+        function(next) {
+          setTimeout(next, 100);
         },
         function(next) {
           server.createAddress({}, function(err, addr) {
@@ -2212,7 +2545,7 @@ describe('Wallet service', function() {
             should.exist(balance);
             balance.totalAmount.should.equal(helpers.toSatoshi(3.5));
             next();
-          });
+          }, 2);
         },
         function(next) {
           setTimeout(next, 100);
@@ -2244,7 +2577,7 @@ describe('Wallet service', function() {
           });
         },
         function(next) {
-          clock.tick(7 * 24 * 3600 * 1000);
+          clock.tick(7 * Defaults.TWO_STEP_CREATION_HOURS * 3600 * 1000);
           helpers.createAddresses(server, wallet, 2, 0, function(addrs) {
             newAddrs = addrs;
             helpers.stubUtxos(server, wallet, [1, 2], {
@@ -2280,6 +2613,8 @@ describe('Wallet service', function() {
         done();
       });
     });
+
+
   });
 
   describe('#getFeeLevels', function() {