store history cache

9 years ago · 35f44b4636
7 changed files with 349 additions and 20 deletions
--- a/config.js
+++ b/config.js
@ -81,6 +81,6 @@ var config = {
  //  api_user: xxx,
  //  api_key: xxx,
  // });
-
+  confirmationsToStartCaching: 100,
 };
 module.exports = config;
--- a/lib/blockchainexplorers/insight.js
+++ b/lib/blockchainexplorers/insight.js
@ -97,6 +97,7 @@ Insight.prototype.getTransaction = function(txid, cb) {

 Insight.prototype.getTransactions = function(addresses, from, to, cb) {
  var qs = [];
+  var total;
  if (_.isNumber(from)) qs.push('from=' + from);
  if (_.isNumber(to)) qs.push('to=' + to);

@ -116,13 +117,18 @@ Insight.prototype.getTransactions = function(addresses, from, to, cb) {
  this._doRequest(args, function(err, res, txs) {
    if (err || res.statusCode !== 200) return cb(_parseErr(err, res));

-    if (_.isObject(txs) && txs.items)
-      txs = txs.items;
+    if (_.isObject(txs)) {
+      if (txs.totalItems)
+        total = txs.totalItems;
+
+      if (txs.items) 
+        txs = txs.items;
+    }

    // NOTE: Whenever Insight breaks communication with bitcoind, it returns invalid data but no error code.
    if (!_.isArray(txs) || (txs.length != _.compact(txs).length)) return cb(new Error('Could not retrieve transactions from blockchain. Request was:' + JSON.stringify(args)));

-    return cb(null, txs);
+    return cb(null, txs, total);
  });
 };

--- a/lib/server.js
+++ b/lib/server.js
@ -57,6 +57,7 @@ function WalletService() {
  this.messageBroker = messageBroker;
  this.fiatRateService = fiatRateService;
  this.notifyTicker = 0;
+  this.confirmationsToStartCaching = config.confirmationsToStartCaching || 100;
 };

 function checkRequired(obj, args, cb) {
@ -2785,9 +2786,16 @@ WalletService.prototype.getTxHistory = function(opts, cb) {
      function(next) {
        var from = opts.skip || 0;
        var to = from + opts.limit;
-        bc.getTransactions(addressStrs, from, to, function(err, txs) {
+        bc.getTransactions(addressStrs, from, to, function(err, txs, total) {
          if (err) return cb(err);
-          next(null, self._normalizeTxHistory(txs));
+          var txsNormalized = self._normalizeTxHistory(txs);
+          var txsToCache = _.filter(txsNormalized, function(i){
+            return i.confirmations >= self.confirmationsToStartCaching;
+          });
+          var index = total - from;
+          self.storage.storeTxHistoryCache(self.walletId, total, index, txsToCache, function(err) {
+            next(err, txsNormalized);
+          })
        });
      },
      function(next) {
--- a/lib/storage.js
+++ b/lib/storage.js
@ -609,27 +609,129 @@ Storage.prototype.storeActiveAddresses = function(walletId, addresses, cb) {
  }, cb);
 };

+// --------         ---------------------------  Total
+//           > Time >                  
+//                       ^to     <=  ^from
+//                       ^start  =>  ^end

-Storage.prototype.storeTxHistoryCache = function(walletId, firstPosition, items, cb) {
+Storage.prototype.getTxHistoryCache = function(walletId, from, to, cb) {
  var self = this;

  self.db.collection(collections.CACHE).findOne({
-    walletId: record.walletId,
-    type: 'historyCache'
+    walletId: walletId,
+    type: 'historyCacheStatus',
+    key: null
+  }, function(err, result) {
+    if (err) return cb(err);
+    if (!result) return cb();
+
+    // Reverse indexes
+    var start = result.totalItems - from;
+    var end = start + to - from;
+
+
+    console.log('[storage.js.632] from,to:', from, to); //TODO
+    console.log('[storage.js.632] start,end:', start, end); //TODO
+
+    // Cache is OK.
+    self.db.collection(collections.CACHE).findOne({
+      walletId: walletId,
+      type: 'historyCache',
+      key: null
+    }, function(err, result) {
+      console.log('[storage.js.641:result:]', result); //TODO
+      if (err) return cb(err);
+      if (!_.any(result.history, function(i) {
+        return !!i;
+      })) return cb(); // some items are not yet defined.
+
+      var ret = result.history.slice(start, end);
+      return cb(null, ret);
+    });
+  })
+};
+
+Storage.prototype.softResetTxHistoryCache = function(walletId, cb) {
+  this.db.collection(collections.CACHE).remove({
+    walletId: walletId,
+    type: 'historyCacheStatus',
+    key: null
+  }, {
+    w: 1
+  }, cb);
+};
+
+
+Storage.prototype.clearTxHistoryCache = function(walletId, cb) {
+  var self = this;
+  self.db.collection(collections.CACHE).remove({
+    walletId: walletId,
+    type: 'historyCache',
+    key: null
+  }, {
+    w: 1
+  }, function(err) {
+    self.db.collection(collections.CACHE).remove({
+      walletId: walletId,
+      type: 'historyCacheStatus',
+      key: null
+    }, {
+      w: 1
+    }, cb);
+  });
+};
+
+Storage.prototype.storeTxHistoryCache = function(walletId, totalItems, firstPosition, itemsLastFirst, cb) {
+  $.shouldBeNumber(firstPosition);
+  $.checkArgument(firstPosition >= 0);
+  $.shouldBeNumber(totalItems);
+  $.checkArgument(totalItems >= 0);
+
+  var self = this;
+
+  self.db.collection(collections.CACHE).findOne({
+    walletId: walletId,
+    type: 'historyCache',
+    key: null
  }, function(err, result) {
    if (err) return cb(err);
    result = result || [];

-    result 
-  self.db.collection(collections.CACHE).update({
-      walletId: record.walletId,
-      type: record.type,
-      key: record.key,
-    }, record, {
+    //create a sparce array, from the reversed input
+    _.each(itemsLastFirst.reverse(), function(i) {
+      result[firstPosition++] = i;
+    });
+
+    var cacheIsReady = !!result[0];
+    var now = Date.now();
+
+    self.db.collection(collections.CACHE).update({
+      walletId: walletId,
+      type: 'historyCacheStatus',
+      key: null
+    }, {
+      totalItems: totalItems,
+      updatedOn: now,
+    }, {
      w: 1,
      upsert: true,
-    }, next);
-  }, cb);
+    }, function(err) {
+      if (err) return cb(err);
+
+      self.db.collection(collections.CACHE).update({
+        walletId: walletId,
+        type: 'historyCache',
+        key: null
+      }, {
+        history: result
+      }, {
+        w: 1,
+        upsert: true,
+      }, cb);
+
+    });
+
+  });
 };


--- a/test/integration/helpers.js
+++ b/test/integration/helpers.js
@ -322,7 +322,8 @@ helpers.stubBroadcast = function(thirdPartyBroadcast) {
  blockchainExplorer.getTransaction = sinon.stub().callsArgWith(1, null, null);
 };

-helpers.stubHistory = function(txs) {
+helpers.stubHistory = function(txs, totalItems) {
+  totalItems = totalItems || 100;
  blockchainExplorer.getTransactions = function(addresses, from, to, cb) {
    var MAX_BATCH_SIZE = 100;
    var nbTxs = txs.length;
@ -343,7 +344,7 @@ helpers.stubHistory = function(txs) {
    if (to > nbTxs) to = nbTxs;

    var page = txs.slice(from, to);
-    return cb(null, page);
+    return cb(null, page, totalItems);
  };
 };

--- a/test/integration/server.js
+++ b/test/integration/server.js
@ -4092,7 +4092,7 @@ describe('Wallet service', function() {
            amount: 200,
          }],
        }];
-        helpers.stubHistory(txs);
+        helpers.stubHistory(txs, 100);
        server.editTxNote({
          txid: '123',
          body: 'just some note'
@ -5923,6 +5923,59 @@ describe('Wallet service', function() {
        done();
      });
    });
+
+    it('should cache tx history from insight', function(done) {
+      helpers.stubHistory(TestData.historyToCache);
+      var spy = sinon.spy(server.storage, 'storeTxHistoryCache');
+      var toCache = _.filter(TestData.historyToCache, function(i) {
+        return i.confirmations >= server.confirmationsToStartCaching;
+      });
+      var skip  = 1;
+
+      server.getTxHistory({
+        skip: skip,
+      }, function(err, txs) {
+
+        // FROM the END, we are getting items
+        // End-1, end-2, end-3.
+
+        should.not.exist(err);
+        should.exist(txs);
+        txs.length.should.equal(TestData.historyToCache.length - skip);
+        var calls = spy.getCalls();
+        calls.length.should.equal(1);
+        calls[0].args[1].should.equal(100); // total
+        calls[0].args[2].should.equal(100 - skip);  // position
+        calls[0].args[3].length.should.equal(toCache.length - skip);
+
+        // should be reversed!
+        calls[0].args[3][0].txid.should.equal(toCache[toCache.length-1].txid);
+        server.storage.storeTxHistoryCache.restore();
+        done();
+      });
+    });
+
+
+    it('should not cache tx history from insight', function(done) {
+      helpers.stubHistory(TestData.history);
+      var spy = sinon.spy(server.storage, 'storeTxHistoryCache');
+      var total = _.filter(TestData.history, function(i) {
+        return i.confirmations >= server.confirmationsToStartCaching;
+      });
+      server.getTxHistory({}, function(err, txs) {
+        should.not.exist(err);
+        should.exist(txs);
+        txs.length.should.equal(TestData.history.length);
+        var calls = spy.getCalls();
+        calls.length.should.equal(1);
+        calls[0].args[1].should.equal(100);
+        calls[0].args[2].should.equal(100);
+        calls[0].args[3].length.should.deep.equal(total.length);
+        server.storage.storeTxHistoryCache.restore();
+        done();
+      });
+    });
+
    it('should get tx history for incoming txs', function(done) {
      server._normalizeTxHistory = sinon.stub().returnsArg(0);
      var txs = [{
--- a/test/testdata.js
+++ b/test/testdata.js
@ -251,6 +251,165 @@ var history = [
  fees: 0.00014299
 }];

+var historyToCache = [
+  {
+  txid: "0279ef7b21630f859deb723e28beac9e7011660bd1346c2da40321d2f7e34f04",
+  vin: [{
+    txid: "c8e221141e8bb60977896561b77fa59d6dacfcc10db82bf6f5f923048b11c70d",
+    vout: 0,
+    n: 0,
+    addr: "2N6Zutg26LEC4iYVxi7SHhopVLP1iZPU1rZ",
+    valueSat: 485645,
+    value: 0.00485645,
+  }, {
+    txid: "6e599eea3e2898b91087eb87e041c5d8dec5362447a8efba185ed593f6dc64c0",
+    vout: 1,
+    n: 1,
+    addr: "2MyqmcWjmVxW8i39wdk1CVPdEqKyFSY9H1S",
+    valueSat: 885590,
+    value: 0.0088559,
+  }],
+  vout: [{
+    value: "0.00045753",
+    n: 0,
+    scriptPubKey: {
+      addresses: [
+        "2NAVFnsHqy5JvqDJydbHPx393LFqFFBQ89V"
+      ]
+    },
+  }, {
+    value: "0.01300000",
+    n: 1,
+    scriptPubKey: {
+      addresses: [
+        "mq4D3Va5mYHohMEHrgHNGzCjKhBKvuEhPE"
+      ]
+    }
+  }],
+  confirmations: 120,
+  firstSeenTs: 1424471000,
+  valueOut: 0.01345753,
+  valueIn: 0.01371235,
+  fees: 0.00025482
+},
+  {
+  txid: "0279ef7b21630f959deb723e28beac9e7011660bd1346c2da40321d2f7e34f04",
+  vin: [{
+    txid: "c8e221141e8bb60977896561b77fa59d6dacfcc10db82bf6f5f923048b11c70d",
+    vout: 0,
+    n: 0,
+    addr: "2N6Zutg26LEC4iYVxi7SHhopVLP1iZPU1rZ",
+    valueSat: 485645,
+    value: 0.00485645,
+  }, {
+    txid: "6e599eea3e2898b91087eb87e041c5d8dec5362447a8efba185ed593f6dc64c0",
+    vout: 1,
+    n: 1,
+    addr: "2MyqmcWjmVxW8i39wdk1CVPdEqKyFSY9H1S",
+    valueSat: 885590,
+    value: 0.0088559,
+  }],
+  vout: [{
+    value: "0.00045753",
+    n: 0,
+    scriptPubKey: {
+      addresses: [
+        "2NAVFnsHqy5JvqDJydbHPx393LFqFFBQ89V"
+      ]
+    },
+  }, {
+    value: "0.01300000",
+    n: 1,
+    scriptPubKey: {
+      addresses: [
+        "mq4D3Va5mYHohMEHrgHNGzCjKhBKvuEhPE"
+      ]
+    }
+  }],
+  confirmations: 120,
+  firstSeenTs: 1424471000,
+  valueOut: 0.01345753,
+  valueIn: 0.01371235,
+  fees: 0.00025482
+},
+
+  {
+  txid: "0279ef7b21630f859deb723e28beac9e7011660bd1346c2da40321d2f7e34f04",
+  vin: [{
+    txid: "c8e221141e8bb60977896561b77fa59d6dacfcc10db82bf6f5f923048b11c70d",
+    vout: 0,
+    n: 0,
+    addr: "2N6Zutg26LEC4iYVxi7SHhopVLP1iZPU1rZ",
+    valueSat: 485645,
+    value: 0.00485645,
+  }, {
+    txid: "6e599eea3e2898b91087eb87e041c5d8dec5362447a8efba185ed593f6dc64c0",
+    vout: 1,
+    n: 1,
+    addr: "2MyqmcWjmVxW8i39wdk1CVPdEqKyFSY9H1S",
+    valueSat: 885590,
+    value: 0.0088559,
+  }],
+  vout: [{
+    value: "0.00045753",
+    n: 0,
+    scriptPubKey: {
+      addresses: [
+        "2NAVFnsHqy5JvqDJydbHPx393LFqFFBQ89V"
+      ]
+    },
+  }, {
+    value: "0.01300000",
+    n: 1,
+    scriptPubKey: {
+      addresses: [
+        "mq4D3Va5mYHohMEHrgHNGzCjKhBKvuEhPE"
+      ]
+    }
+  }],
+  confirmations: 2,
+  firstSeenTs: 1424471041,
+  blocktime: 1424471051,
+  valueOut: 0.01345753,
+  valueIn: 0.01371235,
+  fees: 0.00025482
+}, {
+  txid: "fad88682ccd2ff34cac6f7355fe9ecd8addd9ef167e3788455972010e0d9d0de",
+  vin: [{
+    txid: "0279ef7b21630f859deb723e28beac9e7011660bd1346c2da40321d2f7e34f04",
+    vout: 0,
+    n: 0,
+    addr: "2NAVFnsHqy5JvqDJydbHPx393LFqFFBQ89V",
+    valueSat: 45753,
+    value: 0.00045753,
+  }],
+  vout: [{
+    value: "0.00011454",
+    n: 0,
+    scriptPubKey: {
+      addresses: [
+        "2N7GT7XaN637eBFMmeczton2aZz5rfRdZso"
+      ]
+    }
+  }, {
+    value: "0.00020000",
+    n: 1,
+    scriptPubKey: {
+      addresses: [
+        "mq4D3Va5mYHohMEHrgHNGzCjKhBKvuEhPE"
+      ]
+    }
+  }],
+  confirmations: 101,
+  time: 1424472242,
+  blocktime: 1424472242,
+  valueOut: 0.00031454,
+  valueIn: 0.00045753,
+  fees: 0.00014299
+}];
+
+
 module.exports.keyPair = keyPair;
 module.exports.copayers = copayers;
 module.exports.history = history;
+module.exports.historyToCache = historyToCache;