Merge pull request #545 from matiu/feat/history-cache

Feat/history cache

.travis.yml

@@ -1,7 +1,5 @@
 language: node_js
 node_js:
-  - '0.10'
-  - '0.12'
   - '4'
 install:
   - npm install

config.js

@@ -81,6 +81,5 @@ var config = {
   //  api_user: xxx,
   //  api_key: xxx,
   // });
-
 };
 module.exports = config;

lib/blockchainexplorers/insight.js

@@ -97,6 +97,7 @@ Insight.prototype.getTransaction = function(txid, cb) {
 
 Insight.prototype.getTransactions = function(addresses, from, to, cb) {
   var qs = [];
+  var total;
   if (_.isNumber(from)) qs.push('from=' + from);
   if (_.isNumber(to)) qs.push('to=' + to);
 
@@ -116,13 +117,18 @@ Insight.prototype.getTransactions = function(addresses, from, to, cb) {
   this._doRequest(args, function(err, res, txs) {
     if (err || res.statusCode !== 200) return cb(_parseErr(err, res));
 
-    if (_.isObject(txs) && txs.items)
-      txs = txs.items;
+    if (_.isObject(txs)) {
+      if (txs.totalItems)
+        total = txs.totalItems;
+
+      if (txs.items) 
+        txs = txs.items;
+    }
 
     // NOTE: Whenever Insight breaks communication with bitcoind, it returns invalid data but no error code.
     if (!_.isArray(txs) || (txs.length != _.compact(txs).length)) return cb(new Error('Could not retrieve transactions from blockchain. Request was:' + JSON.stringify(args)));
 
-    return cb(null, txs);
+    return cb(null, txs, total);
   });
 };
 

lib/blockchainmonitor.js

@@ -109,22 +109,25 @@ BlockchainMonitor.prototype._handleTxId = function(data, processIt) {
     log.info('Processing accepted txp [' + txp.id + '] for wallet ' + walletId + ' [' + txp.amount + 'sat ]');
 
     txp.setBroadcasted();
-    self.storage.storeTx(self.walletId, txp, function(err) {
-      if (err)
-        log.error('Could not save TX');
 
-      var args = {
-        txProposalId: txp.id,
-        txid: data.txid,
-        amount: txp.getTotalAmount(),
-      };
+    self.storage.softResetTxHistoryCache(walletId, function() {
+      self.storage.storeTx(self.walletId, txp, function(err) {
+        if (err)
+          log.error('Could not save TX');
 
-      var notification = Notification.create({
-        type: 'NewOutgoingTxByThirdParty',
-        data: args,
-        walletId: walletId,
+        var args = {
+          txProposalId: txp.id,
+          txid: data.txid,
+          amount: txp.getTotalAmount(),
+        };
+
+        var notification = Notification.create({
+          type: 'NewOutgoingTxByThirdParty',
+          data: args,
+          walletId: walletId,
+        });
+        self._storeAndBroadcastNotification(notification);
       });
-      self._storeAndBroadcastNotification(notification);
     });
   });
 };
@@ -166,8 +169,10 @@ BlockchainMonitor.prototype._handleTxOuts = function(data) {
         },
         walletId: walletId,
       });
-      self._updateActiveAddresses(address, function() {
-        self._storeAndBroadcastNotification(notification, next);
+      self.storage.softResetTxHistoryCache(walletId, function() {
+        self._updateActiveAddresses(address, function() {
+          self._storeAndBroadcastNotification(notification, next);
+        });
       });
     });
   }, function(err) {
@@ -202,8 +207,11 @@ BlockchainMonitor.prototype._handleNewBlock = function(network, hash) {
       hash: hash,
     },
   });
-  self._storeAndBroadcastNotification(notification, function(err) {
-    return;
+
+  self.storage.softResetAllTxHistoryCache(function() {
+    self._storeAndBroadcastNotification(notification, function(err) {
+      return;
+    });
   });
 };
 

lib/common/defaults.js

@@ -73,4 +73,14 @@ Defaults.UTXO_SELECTION_MAX_FEE_VS_SINGLE_UTXO_FEE_FACTOR = 5;
 // Minimum allowed amount for tx outputs (including change) in SAT
 Defaults.MIN_OUTPUT_AMOUNT = 5000;
 
+// Number of confirmations from which tx in history will be cached 
+// (ie we consider them inmutables)
+Defaults.CONFIRMATIONS_TO_START_CACHING = 6 * 6; // ~ 6hrs
+
+// Number of addresses from which tx history is enabled in a wallet
+Defaults.HISTORY_CACHE_ADDRESS_THRESOLD = 100;
+
+// Cache time for blockchain height (in seconds)
+Defaults.BLOCKHEIGHT_CACHE_TIME = 10 * 60;
+
 module.exports = Defaults;

lib/server.js

@@ -108,11 +108,11 @@ WalletService.initialize = function(opts, cb) {
   };
 
   function initMessageBroker(cb) {
-    if (opts.messageBroker) {
-      messageBroker = opts.messageBroker;
-    } else {
-      messageBroker = new MessageBroker(opts.messageBrokerOpts);
+    messageBroker = opts.messageBroker || new MessageBroker(opts.messageBrokerOpts);
+    if (messageBroker) {
+      messageBroker.onMessage(WalletService.handleIncomingNotification);
     }
+
     return cb();
   };
 
@@ -153,6 +153,14 @@ WalletService.initialize = function(opts, cb) {
   });
 };
 
+WalletService.handleIncomingNotification = function(notification, cb) {
+  cb = cb || function() {};
+
+  if (!notification || notification.type != 'NewBlock') return cb();
+  WalletService._clearBlockchainHeightCache();
+  return cb();
+};
+
 
 WalletService.shutDown = function(cb) {
   if (!initialized) return cb();
@@ -2196,7 +2204,9 @@ WalletService.prototype._processBroadcast = function(txp, opts, cb) {
       self._notify('NewOutgoingTx', args);
     }
 
-    return cb(err, txp);
+    self.storage.softResetTxHistoryCache(self.walletId, function() {
+      return cb(err, txp);
+    });
   });
 };
 
@@ -2421,6 +2431,7 @@ WalletService.prototype._normalizeTxHistory = function(txs) {
     return {
       txid: tx.txid,
       confirmations: tx.confirmations,
+      blockheight: tx.blockheight,
       fees: parseInt((tx.fees * 1e8).toFixed(0)),
       time: t,
       inputs: inputs,
@@ -2429,6 +2440,37 @@ WalletService.prototype._normalizeTxHistory = function(txs) {
   });
 };
 
+WalletService._cachedBlockheight;
+WalletService._clearBlockchainHeightCache = function() {
+  WalletService._cachedBlockheight.current = null;
+};
+
+WalletService.prototype._getBlockchainHeight = function(network, cb) {
+  var self = this;
+
+  var now = Date.now();
+  if (!WalletService._cachedBlockheight) WalletService._cachedBlockheight = {};
+  var cache = WalletService._cachedBlockheight;
+
+  function fetchFromBlockchain(cb) {
+    var bc = self._getBlockchainExplorer(network);
+    bc.getBlockchainHeight(function(err, height) {
+      if (!err && height > 0) {
+        cache.current = height;
+        cache.last = height;
+        cache.updatedOn = now;
+      }
+      return cb(null, cache.last);
+    });
+  };
+
+  if (!cache.current || (now - cache.updatedOn) > Defaults.BLOCKHEIGHT_CACHE_TIME * 1000) {
+    return fetchFromBlockchain(cb);
+  }
+
+  return cb(null, cache.current);
+};
+
 /**
  * Retrieves all transactions (incoming & outgoing)
  * Times are in UNIX EPOCH
@@ -2442,6 +2484,7 @@ WalletService.prototype._normalizeTxHistory = function(txs) {
 WalletService.prototype.getTxHistory = function(opts, cb) {
   var self = this;
 
+
   opts = opts || {};
   opts.limit = (_.isUndefined(opts.limit) ? Defaults.HISTORY_LIMIT : opts.limit);
   if (opts.limit > Defaults.HISTORY_LIMIT)
@@ -2574,27 +2617,90 @@ WalletService.prototype.getTxHistory = function(opts, cb) {
     });
   };
 
+  function getNormalizedTxs(addresses, from, to, cb) {
+    var txs, fromCache, totalItems;
+    var useCache = addresses.length >= Defaults.HISTORY_CACHE_ADDRESS_THRESOLD;
+    var network = Bitcore.Address(addresses[0].address).toObject().network;
+
+    async.series([
+
+      function(next) {
+        if (!useCache) return next();
+
+        self.storage.getTxHistoryCache(self.walletId, from, to, function(err, res) {
+          if (err) return next(err);
+          if (!res || !res[0]) return next();
+
+          txs = res;
+          fromCache = true;
+
+          return next()
+        });
+      },
+      function(next) {
+        if (txs) return next();
+
+        var addressStrs = _.pluck(addresses, 'address');
+        var bc = self._getBlockchainExplorer(network);
+        bc.getTransactions(addressStrs, from, to, function(err, rawTxs, total) {
+          if (err) return next(err);
+          txs = self._normalizeTxHistory(rawTxs);
+          totalItems = total;
+          return next();
+        });
+      },
+      function(next) {
+        if (!useCache || fromCache) return next();
+
+        var txsToCache = _.filter(txs, function(i) {
+          return i.confirmations >= Defaults.CONFIRMATIONS_TO_START_CACHING;
+        }).reverse();
+
+        if (!txsToCache.length) return next();
+
+        var fwdIndex = totalItems - to;
+        if (fwdIndex < 0) fwdIndex = 0;
+        self.storage.storeTxHistoryCache(self.walletId, totalItems, fwdIndex, txsToCache, next);
+      },
+      function(next) {
+        if (!txs) return next();
+
+        // Fix tx confirmations
+        self._getBlockchainHeight(network, function(err, height) {
+          if (err || !height) return next(err);
+          _.each(txs, function(tx) {
+            if (tx.blockheight >= 0) {
+              tx.confirmations = height - tx.blockheight + 1;
+            }
+          });
+          next();
+        });
+      },
+    ], function(err) {
+      if (err) return cb(err);
+
+      return cb(null, {
+        items: txs,
+        fromCache: fromCache
+      });
+    });
+  };
+
   // Get addresses for this wallet
   self.storage.fetchAddresses(self.walletId, function(err, addresses) {
     if (err) return cb(err);
     if (addresses.length == 0) return cb(null, []);
 
-    var addressStrs = _.pluck(addresses, 'address');
-    var networkName = Bitcore.Address(addressStrs[0]).toObject().network;
+    var from = opts.skip || 0;
+    var to = from + opts.limit;
 
-    var bc = self._getBlockchainExplorer(networkName);
     async.parallel([
 
       function(next) {
-        self.storage.fetchTxs(self.walletId, {}, next);
+        getNormalizedTxs(addresses, from, to, next);
       },
       function(next) {
-        var from = opts.skip || 0;
-        var to = from + opts.limit;
-        bc.getTransactions(addressStrs, from, to, function(err, txs) {
-          if (err) return cb(err);
-          next(null, self._normalizeTxHistory(txs));
-        });
+        self.storage.fetchTxs(self.walletId, {}, next);
       },
       function(next) {
         self.storage.fetchTxNotes(self.walletId, {}, next);
@@ -2602,13 +2708,12 @@ WalletService.prototype.getTxHistory = function(opts, cb) {
     ], function(err, res) {
       if (err) return cb(err);
 
-      var proposals = res[0];
-      var txs = res[1];
-      var notes = res[2];
+      var finalTxs = decorate(res[0].items, addresses, res[1], res[2]);
 
-      txs = decorate(txs, addresses, proposals, notes);
+      if (res[0].fromCache)
+        log.debug("History from cache for:", self.walletId, from, to);
 
-      return cb(null, txs);
+      return cb(null, finalTxs, !!res[0].fromCache);
     });
   });
 };
@@ -2659,40 +2764,43 @@ WalletService.prototype.scan = function(opts, cb) {
       if (!wallet.isComplete()) return cb(Errors.WALLET_NOT_COMPLETE);
 
       wallet.scanStatus = 'running';
-      self.storage.storeWallet(wallet, function(err) {
-        if (err) return cb(err);
 
-        var derivators = [];
-        _.each([false, true], function(isChange) {
-          derivators.push({
-            derive: _.bind(wallet.createAddress, wallet, isChange),
-            rewind: _.bind(wallet.addressManager.rewindIndex, wallet.addressManager, isChange),
-          });
-          if (opts.includeCopayerBranches) {
-            _.each(wallet.copayers, function(copayer) {
-              if (copayer.addressManager) {
-                derivators.push({
-                  derive: _.bind(copayer.createAddress, copayer, wallet, isChange),
-                  rewind: _.bind(copayer.addressManager.rewindIndex, copayer.addressManager, isChange),
-                });
-              }
-            });
-          }
-        });
+      self.storage.clearTxHistoryCache(self.walletId, function() {
+        self.storage.storeWallet(wallet, function(err) {
+          if (err) return cb(err);
 
-        async.eachSeries(derivators, function(derivator, next) {
-          scanBranch(derivator, function(err, addresses) {
-            if (err) return next(err);
-            self.storage.storeAddressAndWallet(wallet, addresses, next);
+          var derivators = [];
+          _.each([false, true], function(isChange) {
+            derivators.push({
+              derive: _.bind(wallet.createAddress, wallet, isChange),
+              rewind: _.bind(wallet.addressManager.rewindIndex, wallet.addressManager, isChange),
+            });
+            if (opts.includeCopayerBranches) {
+              _.each(wallet.copayers, function(copayer) {
+                if (copayer.addressManager) {
+                  derivators.push({
+                    derive: _.bind(copayer.createAddress, copayer, wallet, isChange),
+                    rewind: _.bind(copayer.addressManager.rewindIndex, copayer.addressManager, isChange),
+                  });
+                }
+              });
+            }
           });
-        }, function(error) {
-          self.storage.fetchWallet(wallet.id, function(err, wallet) {
-            if (err) return cb(err);
-            wallet.scanStatus = error ? 'error' : 'success';
-            self.storage.storeWallet(wallet, function() {
-              return cb(error);
+
+          async.eachSeries(derivators, function(derivator, next) {
+            scanBranch(derivator, function(err, addresses) {
+              if (err) return next(err);
+              self.storage.storeAddressAndWallet(wallet, addresses, next);
             });
-          })
+          }, function(error) {
+            self.storage.fetchWallet(wallet.id, function(err, wallet) {
+              if (err) return cb(err);
+              wallet.scanStatus = error ? 'error' : 'success';
+              self.storage.storeWallet(wallet, function() {
+                return cb(error);
+              });
+            })
+          });
         });
       });
     });

lib/storage.js

@@ -609,6 +609,161 @@ Storage.prototype.storeActiveAddresses = function(walletId, addresses, cb) {
   }, cb);
 };
 
+// --------         ---------------------------  Total
+//           > Time >                  
+//                       ^to     <=  ^from
+//                       ^fwdIndex  =>  ^end
+Storage.prototype.getTxHistoryCache = function(walletId, from, to, cb) {
+  var self = this;
+  $.checkArgument(from >= 0);
+  $.checkArgument(from <= to);
+
+  self.db.collection(collections.CACHE).findOne({
+    walletId: walletId,
+    type: 'historyCacheStatus',
+    key: null
+  }, function(err, result) {
+    if (err) return cb(err);
+    if (!result) return cb();
+    if (!result.isUpdated) return cb();
+
+    // Reverse indexes
+    var fwdIndex = result.totalItems - to;
+
+    if (fwdIndex < 0) {
+      fwdIndex = 0;
+    }
+
+    var end = result.totalItems - from;
+
+    // nothing to return
+    if (end <= 0) return cb(null, []);
+
+    // Cache is OK.
+    self.db.collection(collections.CACHE).find({
+      walletId: walletId,
+      type: 'historyCache',
+      key: {
+        $gte: fwdIndex,
+        $lt: end
+      },
+    }).sort({
+      key: -1,
+    }).toArray(function(err, result) {
+      if (err) return cb(err);
+
+      if (!result) return cb();
+
+      if (result.length < end - fwdIndex) {
+        // some items are not yet defined.
+        return cb();
+      }
+
+      var txs = _.pluck(result, 'tx');
+      return cb(null, txs);
+    });
+  })
+};
+
+Storage.prototype.softResetAllTxHistoryCache = function(cb) {
+  this.db.collection(collections.CACHE).update({
+    type: 'historyCacheStatus',
+  }, {
+    isUpdated: false,
+  }, {
+    multi: true,
+  }, cb);
+};
+
+Storage.prototype.softResetTxHistoryCache = function(walletId, cb) {
+  this.db.collection(collections.CACHE).update({
+    walletId: walletId,
+    type: 'historyCacheStatus',
+    key: null
+  }, {
+    isUpdated: false,
+  }, {
+    w: 1,
+    upsert: true,
+  }, cb);
+};
+
+Storage.prototype.clearTxHistoryCache = function(walletId, cb) {
+  var self = this;
+  self.db.collection(collections.CACHE).remove({
+    walletId: walletId,
+    type: 'historyCache',
+  }, {
+    multi: 1
+  }, function(err) {
+    if (err) return cb(err);
+    self.db.collection(collections.CACHE).remove({
+      walletId: walletId,
+      type: 'historyCacheStatus',
+      key: null
+    }, {
+      w: 1
+    }, cb);
+  });
+};
+
+// items should be in CHRONOLOGICAL order
+Storage.prototype.storeTxHistoryCache = function(walletId, totalItems, firstPosition, items, cb) {
+  $.shouldBeNumber(firstPosition);
+  $.checkArgument(firstPosition >= 0);
+  $.shouldBeNumber(totalItems);
+  $.checkArgument(totalItems >= 0);
+
+  var self = this;
+
+  _.each(items, function(item, i) {
+    item.position = firstPosition + i;
+  });
+  var cacheIsComplete = (firstPosition == 0);
+
+  // TODO: check txid uniqness?
+  async.each(items, function(item, next) {
+    var pos = item.position;
+    delete item.position;
+    self.db.collection(collections.CACHE).update({
+      walletId: walletId,
+      type: 'historyCache',
+      key: pos,
+    }, {
+      walletId: walletId,
+      type: 'historyCache',
+      key: pos,
+      tx: item,
+    }, {
+      w: 1,
+      upsert: true,
+    }, next);
+  }, function(err) {
+    if (err) return cb(err);
+
+    self.db.collection(collections.CACHE).update({
+      walletId: walletId,
+      type: 'historyCacheStatus',
+      key: null
+    }, {
+      walletId: walletId,
+      type: 'historyCacheStatus',
+      key: null,
+      totalItems: totalItems,
+      updatedOn: Date.now(),
+      isComplete: cacheIsComplete,
+      isUpdated: true,
+    }, {
+      w: 1,
+      upsert: true,
+    }, cb);
+  });
+};
+
+
+
+
+
 Storage.prototype.fetchActiveAddresses = function(walletId, cb) {
   var self = this;
 

test/integration/helpers.js

@@ -323,6 +323,7 @@ helpers.stubBroadcast = function(thirdPartyBroadcast) {
 };
 
 helpers.stubHistory = function(txs) {
+  var totalItems = txs.length;
   blockchainExplorer.getTransactions = function(addresses, from, to, cb) {
     var MAX_BATCH_SIZE = 100;
     var nbTxs = txs.length;
@@ -343,7 +344,7 @@ helpers.stubHistory = function(txs) {
     if (to > nbTxs) to = nbTxs;
 
     var page = txs.slice(from, to);
-    return cb(null, page);
+    return cb(null, page, totalItems);
   };
 };
 
@@ -428,4 +429,55 @@ helpers.createAndPublishTx = function(server, txOpts, signingKey, cb) {
   });
 };
 
+
+helpers.historyCacheTest = function(items) {
+  var template = {
+    txid: "fad88682ccd2ff34cac6f7355fe9ecd8addd9ef167e3788455972010e0d9d0de",
+    vin: [{
+      txid: "0279ef7b21630f859deb723e28beac9e7011660bd1346c2da40321d2f7e34f04",
+      vout: 0,
+      n: 0,
+      addr: "2NAVFnsHqy5JvqDJydbHPx393LFqFFBQ89V",
+      valueSat: 45753,
+      value: 0.00045753,
+    }],
+    vout: [{
+      value: "0.00011454",
+      n: 0,
+      scriptPubKey: {
+        addresses: [
+          "2N7GT7XaN637eBFMmeczton2aZz5rfRdZso"
+        ]
+      }
+    }, {
+      value: "0.00020000",
+      n: 1,
+      scriptPubKey: {
+        addresses: [
+          "mq4D3Va5mYHohMEHrgHNGzCjKhBKvuEhPE"
+        ]
+      }
+    }],
+    confirmations: 1,
+    blockheight: 423499,
+    time: 1424472242,
+    blocktime: 1424472242,
+    valueOut: 0.00031454,
+    valueIn: 0.00045753,
+    fees: 0.00014299
+  };
+
+  var ret = [];
+  _.each(_.range(0, items), function(i) {
+    var t = _.clone(template);
+    t.txid = 'txid:' + i;
+    t.confirmations = items - i - 1;
+    t.blockheight = i;
+    t.time = t.blocktime = i;
+    ret.unshift(t);
+  });
+
+  return ret;
+};
+
 module.exports = helpers;

test/integration/server.js

@@ -3727,6 +3727,7 @@ describe('Wallet service', function() {
     });
     it('should include the note in tx history listing', function(done) {
       helpers.createAddresses(server, wallet, 1, 1, function(mainAddresses, changeAddress) {
+        blockchainExplorer.getBlockchainHeight = sinon.stub().callsArgWith(0, null, 1000);
         server._normalizeTxHistory = sinon.stub().returnsArg(0);
         var txs = [{
           txid: '123',
@@ -5494,6 +5495,7 @@ describe('Wallet service', function() {
   describe('#getTxHistory', function() {
     var server, wallet, mainAddresses, changeAddresses;
     beforeEach(function(done) {
+      blockchainExplorer.getBlockchainHeight = sinon.stub().callsArgWith(0, null, 1000);
       helpers.createAndJoinWallet(1, 1, function(s, w) {
         server = s;
         wallet = w;
@@ -5519,6 +5521,7 @@ describe('Wallet service', function() {
         done();
       });
     });
+
     it('should get tx history for incoming txs', function(done) {
       server._normalizeTxHistory = sinon.stub().returnsArg(0);
       var txs = [{
@@ -5813,6 +5816,341 @@ describe('Wallet service', function() {
     });
   });
 
+  describe('#getTxHistory cache', function() {
+    var server, wallet, mainAddresses, changeAddresses;
+    var _threshold = Defaults.HISTORY_CACHE_ADDRESS_THRESOLD;
+    beforeEach(function(done) {
+      Defaults.HISTORY_CACHE_ADDRESS_THRESOLD = 1;
+      helpers.createAndJoinWallet(1, 1, function(s, w) {
+        server = s;
+        wallet = w;
+        helpers.createAddresses(server, wallet, 1, 1, function(main, change) {
+          mainAddresses = main;
+          changeAddresses = change;
+          done();
+        });
+      });
+    });
+    afterEach(function() {
+      Defaults.HISTORY_CACHE_ADDRESS_THRESOLD = _threshold;
+    });
+
+    it('should store partial cache tx history from insight', function(done) {
+      var skip = 31;
+      var limit = 10;
+      var totalItems = 200;
+      var currentHeight = 1000;
+
+      var h = helpers.historyCacheTest(totalItems);
+      helpers.stubHistory(h);
+      blockchainExplorer.getBlockchainHeight = sinon.stub().callsArgWith(0, null, currentHeight);
+      var storeTxHistoryCacheSpy = sinon.spy(server.storage, 'storeTxHistoryCache');
+
+
+      server.getTxHistory({
+        skip: skip,
+        limit: limit,
+      }, function(err, txs) {
+
+        // FROM the END, we are getting items
+        // End-1, end-2, end-3.
+
+        should.not.exist(err);
+        should.exist(txs);
+        txs.length.should.equal(limit);
+        var calls = storeTxHistoryCacheSpy.getCalls();
+        calls.length.should.equal(1);
+
+        calls[0].args[1].should.equal(totalItems); // total
+        calls[0].args[2].should.equal(totalItems - skip - limit); // position
+        calls[0].args[3].length.should.equal(5); // 5 txs have confirmations>= 100
+
+        // should be reversed!
+        calls[0].args[3][0].confirmations.should.equal(currentHeight - (totalItems - (skip + limit)) + 1);
+        calls[0].args[3][0].txid.should.equal(h[skip + limit - 1].txid);
+        server.storage.storeTxHistoryCache.restore();
+        done();
+      });
+    });
+
+
+    it('should not cache tx history when requesting txs with low # of confirmations', function(done) {
+      var h = helpers.historyCacheTest(200);
+      helpers.stubHistory(h);
+      blockchainExplorer.getBlockchainHeight = sinon.stub().callsArgWith(0, null, 1000);
+      var storeTxHistoryCacheSpy = sinon.spy(server.storage, 'storeTxHistoryCache');
+      server.getTxHistory({
+        skip: 0,
+        limit: 10,
+      }, function(err, txs) {
+        should.not.exist(err);
+        should.exist(txs);
+        var calls = storeTxHistoryCacheSpy.getCalls();
+        calls.length.should.equal(0);
+        server.storage.storeTxHistoryCache.restore();
+        done();
+      });
+    });
+
+
+    it('should store cache all tx history from insight', function(done) {
+      var skip = 195;
+      var limit = 5;
+      var totalItems = 200;
+      var currentHeight = 1000;
+
+      var h = helpers.historyCacheTest(totalItems);
+      helpers.stubHistory(h);
+      blockchainExplorer.getBlockchainHeight = sinon.stub().callsArgWith(0, null, currentHeight);
+      var storeTxHistoryCacheSpy = sinon.spy(server.storage, 'storeTxHistoryCache');
+
+      server.getTxHistory({
+        skip: skip,
+        limit: limit,
+      }, function(err, txs) {
+
+        should.not.exist(err);
+        should.exist(txs);
+        txs.length.should.equal(limit);
+        var calls = storeTxHistoryCacheSpy.getCalls();
+        calls.length.should.equal(1);
+
+        calls[0].args[1].should.equal(totalItems); // total
+        calls[0].args[2].should.equal(totalItems - skip - limit); // position
+        calls[0].args[3].length.should.equal(5);
+
+        // should be reversed!
+        calls[0].args[3][0].confirmations.should.equal(currentHeight + 1);
+        calls[0].args[3][0].txid.should.equal(h[totalItems - 1].txid);
+        server.storage.storeTxHistoryCache.restore();
+        done();
+      });
+    });
+
+    it('should get real # of confirmations based on current block height', function(done) {
+      var _confirmations = Defaults.CONFIRMATIONS_TO_START_CACHING;
+      Defaults.CONFIRMATIONS_TO_START_CACHING = 6;
+      WalletService._cachedBlockheight = null;
+
+      var h = helpers.historyCacheTest(20);
+      _.each(h, function(x, i) {
+        x.blockheight = 100 - i;
+      });
+      helpers.stubHistory(h);
+      var storeTxHistoryCacheSpy = sinon.spy(server.storage, 'storeTxHistoryCache');
+
+      blockchainExplorer.getBlockchainHeight = sinon.stub().callsArgWith(0, null, 100);
+
+      // Cache txs
+      server.getTxHistory({
+        skip: 0,
+        limit: 30,
+      }, function(err, txs) {
+        should.not.exist(err);
+        should.exist(txs);
+        var calls = storeTxHistoryCacheSpy.getCalls();
+        calls.length.should.equal(1);
+
+        server.getTxHistory({
+          skip: 0,
+          limit: 30,
+        }, function(err, txs) {
+          should.not.exist(err);
+          txs.length.should.equal(20);
+          _.first(txs).confirmations.should.equal(1);
+          _.last(txs).confirmations.should.equal(20);
+
+          blockchainExplorer.getBlockchainHeight = sinon.stub().callsArgWith(0, null, 200);
+          server._notify('NewBlock', {
+            hash: 'dummy hash',
+          }, {
+            isGlobal: true
+          }, function(err) {
+            should.not.exist(err);
+            setTimeout(function() {
+              server.getTxHistory({
+                skip: 0,
+                limit: 30,
+              }, function(err, txs) {
+                should.not.exist(err);
+                _.first(txs).confirmations.should.equal(101);
+                _.last(txs).confirmations.should.equal(120);
+
+                server.storage.storeTxHistoryCache.restore();
+                Defaults.CONFIRMATIONS_TO_START_CACHING = _confirmations;
+                done();
+              });
+            }, 100);
+          });
+        });
+      });
+    });
+
+    it('should get cached # of confirmations if current height unknown', function(done) {
+      var _confirmations = Defaults.CONFIRMATIONS_TO_START_CACHING;
+      Defaults.CONFIRMATIONS_TO_START_CACHING = 6;
+      WalletService._cachedBlockheight = null;
+
+      var h = helpers.historyCacheTest(20);
+      helpers.stubHistory(h);
+      var storeTxHistoryCacheSpy = sinon.spy(server.storage, 'storeTxHistoryCache');
+
+      blockchainExplorer.getBlockchainHeight = sinon.stub().callsArgWith(0, null, null);
+
+      // Cache txs
+      server.getTxHistory({
+        skip: 0,
+        limit: 30,
+      }, function(err, txs) {
+        should.not.exist(err);
+        should.exist(txs);
+        txs.length.should.equal(20);
+        var calls = storeTxHistoryCacheSpy.getCalls();
+        calls.length.should.equal(1);
+
+        server.getTxHistory({
+          skip: 0,
+          limit: 30,
+        }, function(err, txs) {
+          should.not.exist(err);
+          txs.length.should.equal(20);
+          _.first(txs).confirmations.should.equal(0);
+          _.last(txs).confirmations.should.equal(19);
+
+          server.storage.storeTxHistoryCache.restore();
+          Defaults.CONFIRMATIONS_TO_START_CACHING = _confirmations;
+          done();
+        });
+      });
+    });
+
+    describe('Downloading history', function() {
+      var h;
+      beforeEach(function(done) {
+        blockchainExplorer.getBlockchainHeight = sinon.stub().callsArgWith(0, null, 1000);
+        h = helpers.historyCacheTest(200);
+        helpers.stubHistory(h);
+        server.storage.clearTxHistoryCache(server.walletId, function() {
+          done();
+        });
+      });
+
+      it('from 0 to 200, two times, in order', function(done) {
+        async.eachSeries(_.range(0, 200, 5), function(i, next) {
+          server.getTxHistory({
+            skip: i,
+            limit: 5,
+          }, function(err, txs, fromCache) {
+            should.not.exist(err);
+            should.exist(txs);
+            txs.length.should.equal(5);
+            var s = h.slice(i, i + 5);
+            _.pluck(txs, 'txid').should.deep.equal(_.pluck(s, 'txid'));
+            fromCache.should.equal(false);
+            next();
+          });
+        }, function() {
+          async.eachSeries(_.range(0, 200, 5), function(i, next) {
+            server.getTxHistory({
+              skip: i,
+              limit: 5,
+            }, function(err, txs, fromCache) {
+              should.not.exist(err);
+              should.exist(txs);
+              txs.length.should.equal(5);
+              var s = h.slice(i, i + 5);
+              _.pluck(txs, 'txid').should.deep.equal(_.pluck(s, 'txid'));
+              fromCache.should.equal(i >= Defaults.CONFIRMATIONS_TO_START_CACHING && i < 200);
+              next();
+            });
+          }, done);
+        });
+      });
+
+      it('from 0 to 200, two times, random', function(done) {
+        var indexes = _.range(0, 200, 5);
+        async.eachSeries(_.shuffle(indexes), function(i, next) {
+          server.getTxHistory({
+            skip: i,
+            limit: 5,
+          }, function(err, txs, fromCache) {
+            should.not.exist(err);
+            should.exist(txs);
+            txs.length.should.equal(5);
+            var s = h.slice(i, i + 5);
+            _.pluck(txs, 'txid').should.deep.equal(_.pluck(s, 'txid'));
+            fromCache.should.equal(false);
+            next();
+          });
+        }, function() {
+          async.eachSeries(_.range(0, 190, 7), function(i, next) {
+            server.getTxHistory({
+              skip: i,
+              limit: 7,
+            }, function(err, txs, fromCache) {
+              should.not.exist(err);
+              should.exist(txs);
+              txs.length.should.equal(7);
+              var s = h.slice(i, i + 7);
+              _.pluck(txs, 'txid').should.deep.equal(_.pluck(s, 'txid'));
+              fromCache.should.equal(i >= Defaults.CONFIRMATIONS_TO_START_CACHING);
+              next();
+            });
+          }, done);
+        });
+      });
+
+
+      it('from 0 to 200, two times, random, with resets', function(done) {
+        var indexes = _.range(0, 200, 5);
+        async.eachSeries(_.shuffle(indexes), function(i, next) {
+          server.getTxHistory({
+            skip: i,
+            limit: 5,
+          }, function(err, txs, fromCache) {
+            should.not.exist(err);
+            should.exist(txs);
+            txs.length.should.equal(5);
+            var s = h.slice(i, i + 5);
+            _.pluck(txs, 'txid').should.deep.equal(_.pluck(s, 'txid'));
+            fromCache.should.equal(false);
+            next();
+          });
+        }, function() {
+          async.eachSeries(_.range(0, 200, 5), function(i, next) {
+
+            function resetCache(cb) {
+              if (!(i % 25)) {
+                storage.softResetTxHistoryCache(server.walletId, function() {
+                  return cb(true);
+                });
+              } else {
+                return cb(false);
+              }
+            }
+
+            resetCache(function(reset) {
+              server.getTxHistory({
+                skip: i,
+                limit: 5,
+              }, function(err, txs, fromCache) {
+                should.not.exist(err);
+                should.exist(txs);
+                txs.length.should.equal(5);
+                var s = h.slice(i, i + 5);
+                _.pluck(txs, 'txid').should.deep.equal(_.pluck(s, 'txid'));
+                fromCache.should.equal(i >= Defaults.CONFIRMATIONS_TO_START_CACHING && !reset);
+                next();
+              });
+            });
+          }, done);
+        });
+      });
+
+
+    });
+  });
+
   describe('#scan', function() {
     var server, wallet;
 

test/testdata.js

@@ -251,6 +251,7 @@ var history = [
   fees: 0.00014299
 }];
 
+
 module.exports.keyPair = keyPair;
 module.exports.copayers = copayers;
 module.exports.history = history;