Merge pull request #297 from isocolsky/feat/ignore_unconfirmed

Add option to exclude unconfirmed UTXOs at TXP creation

README.md

@@ -91,8 +91,13 @@ Returns:
 
 Returns:
  * totalAmount: Wallet's total balance
- * lockedAmount: Current balance of outstanding transaction proposals, that cannot be used on new transactions. 
+ * lockedAmount: Current balance of outstanding transaction proposals, that cannot be used on new transactions.
+ * availableAmount: Funds available for new proposals.
+ * totalConfirmedAmount: Same as totalAmount for confirmed UTXOs only.
+ * lockedConfirmedAmount: Same as lockedAmount for confirmed UTXOs only.
+ * availableConfirmedAmount: Same as availableAmount for confirmed UTXOs only.
  * byAddress array ['address', 'path', 'amount']: A list of addresses holding funds.
+ * totalKbToSendMax: An estimation of the number of KiB required to include all available UTXOs in a tx (including unconfirmed).
  
 ## POST Endpoints
 `/v1/wallets/`: Create a new Wallet
@@ -122,11 +127,14 @@ Returns:
 
 `/v1/txproposals/`: Add a new transaction proposal
 Required Arguments:
- * toAddress:  RCPT Bitcoin address 
+ * toAddress: RCPT Bitcoin address.
  * amount: amount (in satoshis) of the mount proposed to be transfered
  * proposalsSignature: Signature of the proposal by the creator peer, using prososalSigningKey.
- * (opt) message: Encrypted private message to peers
+ * (opt) message: Encrypted private message to peers.
- 
+ * (opt) payProUrl: Paypro URL for peers to verify TX
+ * (opt) feePerKb: Use an alternative fee per KB for this TX.
+ * (opt) excludeUnconfirmedUtxos: Do not use UTXOs of unconfirmed transactions as inputs for this TX.
+
 Returns:
  * TX Proposal object. (see [fields on the source code](https://github.com/bitpay/bitcore-wallet-service/blob/master/lib/model/txproposal.js)). `.id` is probably needed in this case.
 

lib/model/txproposal.js

@@ -68,6 +68,7 @@ TxProposal.create = function(opts) {
   x.actions = [];
   x.fee = null;
   x.feePerKb = opts.feePerKb;
+  x.excludeUnconfirmedUtxos = opts.excludeUnconfirmedUtxos;
 
   if (_.isFunction(TxProposal._create[x.type])) {
     TxProposal._create[x.type](x, opts);
@@ -110,6 +111,7 @@ TxProposal.fromObj = function(obj) {
   x.fee = obj.fee;
   x.network = obj.network;
   x.feePerKb = obj.feePerKb;
+  x.excludeUnconfirmedUtxos = obj.excludeUnconfirmedUtxos;
 
   return x;
 };

lib/server.js

@@ -626,6 +626,9 @@ WalletService.prototype._getBlockchainExplorer = function(network) {
 WalletService.prototype.getUtxos = function(cb) {
   var self = this;
 
+  function utxoKey(utxo) {
+    return utxo.txid + '|' + utxo.vout
+  };
 
   // Get addresses for this wallet
   self.storage.fetchAddresses(self.walletId, function(err, addresses) {
@@ -651,24 +654,13 @@ WalletService.prototype.getUtxos = function(cb) {
       self.getPendingTxs({}, function(err, txps) {
         if (err) return cb(err);
 
-        var utxoKey = function(utxo) {
+        var lockedInputs = _.map(_.flatten(_.pluck(txps, 'inputs')), utxoKey);
-          return utxo.txid + '|' + utxo.vout
-        };
-
-        var inputs = _.chain(txps)
-          .pluck('inputs')
-          .flatten()
-          .map(utxoKey)
-          .value();
 
-        var dictionary = _.reduce(utxos, function(memo, utxo) {
+        var utxoIndex = _.indexBy(utxos, utxoKey);
-          memo[utxoKey(utxo)] = utxo;
-          return memo;
-        }, {});
 
-        _.each(inputs, function(input) {
+        _.each(lockedInputs, function(input) {
-          if (dictionary[input]) {
+          if (utxoIndex[input]) {
-            dictionary[input].locked = true;
+            utxoIndex[input].locked = true;
           }
         });
 
@@ -687,16 +679,15 @@ WalletService.prototype.getUtxos = function(cb) {
 };
 
 WalletService.prototype._totalizeUtxos = function(utxos) {
-  var balance = {};
+  var balance = {
-  balance.totalAmount = Utils.strip(_.reduce(utxos, function(sum, utxo) {
+    totalAmount: _.sum(utxos, 'satoshis'),
-    return sum + utxo.satoshis;
+    lockedAmount: _.sum(_.filter(utxos, 'locked'), 'satoshis'),
-  }, 0));
+    totalConfirmedAmount: _.sum(_.filter(utxos, 'confirmations'), 'satoshis'),
-
+    lockedConfirmedAmount: _.sum(_.filter(_.filter(utxos, 'locked'), 'confirmations'), 'satoshis'),
-  balance.lockedAmount = Utils.strip(_.reduce(_.filter(utxos, {
+  };
-    locked: true
+  balance.availableAmount = balance.totalAmount - balance.lockedAmount;
-  }), function(sum, utxo) {
+  balance.availableConfirmedAmount = balance.totalConfirmedAmount - balance.lockedConfirmedAmount;
-    return sum + utxo.satoshis;
+
-  }, 0));
   return balance;
 };
 
@@ -704,9 +695,7 @@ WalletService.prototype._totalizeUtxos = function(utxos) {
 WalletService.prototype._computeKbToSendMax = function(utxos, amount, cb) {
   var self = this;
 
-  var unlockedUtxos = _.filter(utxos, {
+  var unlockedUtxos = _.reject(utxos, 'locked');
-    locked: false
-  });
   if (_.isEmpty(unlockedUtxos)) return cb(null, 0);
 
   self.getWallet({}, function(err, wallet) {
@@ -755,7 +744,7 @@ WalletService.prototype.getBalance = function(opts, cb) {
 
     balance.byAddress = _.values(byAddress);
 
-    self._computeKbToSendMax(utxos, balance.totalAmount - balance.lockedAmount, function(err, sizeInKb) {
+    self._computeKbToSendMax(utxos, balance.availableAmount, function(err, sizeInKb) {
       if (err) {
         log.error('Could not compute fees needed to transfer max amount', err);
       }
@@ -780,7 +769,7 @@ WalletService.prototype._sampleFeeLevels = function(network, points, cb) {
       if (feePerKB < 0) {
         log.warn('Could not compute fee estimation (nbBlocks=' + p + ')');
       }
-      return next(null, [p, feePerKB * 1e8]);
+      return next(null, [p, Utils.strip(feePerKB * 1e8)]);
     });
   }, function(err, results) {
     if (err) return cb(err);
@@ -869,16 +858,28 @@ WalletService.prototype._selectTxInputs = function(txp, cb) {
   self.getUtxos(function(err, utxos) {
     if (err) return cb(err);
 
+    var totalAmount;
+    var availableAmount;
+
     var balance = self._totalizeUtxos(utxos);
+    if (txp.excludeUnconfirmedUtxos) {
+      totalAmount = balance.totalConfirmedAmount;
+      availableAmount = balance.availableConfirmedAmount;
+    } else {
+      totalAmount = balance.totalAmount;
+      availableAmount = balance.availableAmount;
+    }
 
-    if (balance.totalAmount < txp.getTotalAmount())
+    if (totalAmount < txp.getTotalAmount())
       return cb(new ClientError('INSUFFICIENTFUNDS', 'Insufficient funds'));
-    if ((balance.totalAmount - balance.lockedAmount) < txp.amount)
+    if (availableAmount < txp.amount)
       return cb(new ClientError('LOCKEDFUNDS', 'Funds are locked by pending transaction proposals'));
 
-    utxos = _.reject(utxos, {
+    // Prepare UTXOs list
-      locked: true
+    utxos = _.reject(utxos, 'locked');
-    });
+    if (txp.excludeUnconfirmedUtxos) {
+      utxos = _.filter(utxos, 'confirmations');
+    }
 
     var i = 0;
     var total = 0;
@@ -908,7 +909,7 @@ WalletService.prototype._selectTxInputs = function(txp, cb) {
           return cb(ex);
         }
       }
-    };
+    }
 
     if (bitcoreError instanceof Bitcore.errors.Transaction.FeeError) {
       return cb(new ClientError('INSUFFICIENTFUNDS', 'Insufficient funds for fee'));
@@ -963,6 +964,7 @@ WalletService.prototype._canCreateTx = function(copayerId, cb) {
  * @param {string} opts.proposalSignature - S(toAddress|amount|message|payProUrl). Used by other copayers to verify the proposal.
  * @param {string} opts.feePerKb - Optional: Use an alternative fee per KB for this TX
  * @param {string} opts.payProUrl - Optional: Paypro URL for peers to verify TX
+ * @param {string} opts.excludeUnconfirmedUtxos - Optional: Do not use UTXOs of unconfirmed transactions as inputs
  * @returns {TxProposal} Transaction proposal.
  */
 WalletService.prototype.createTx = function(opts, cb) {
@@ -1066,6 +1068,7 @@ WalletService.prototype.createTx = function(opts, cb) {
           payProUrl: opts.payProUrl,
           requiredSignatures: wallet.m,
           requiredRejections: Math.min(wallet.m, wallet.n - wallet.m + 1),
+          excludeUnconfirmedUtxos: !!opts.excludeUnconfirmedUtxos,
         });
 
         self._selectTxInputs(txp, function(err) {
@@ -1501,10 +1504,7 @@ WalletService.prototype.getTxHistory = function(opts, cb) {
       var filter = {};
       if (_.isBoolean(isMine)) filter.isMine = isMine;
       if (_.isBoolean(isChange)) filter.isChange = isChange;
-      return _.reduce(_.where(items, filter),
+      return _.sum(_.filter(items, filter), 'amount');
-        function(memo, item) {
-          return memo + item.amount;
-        }, 0);
     };
 
     function classify(items) {

test/integration/server.js

@@ -135,29 +135,28 @@ helpers.toSatoshi = function(btc) {
 };
 
 helpers.stubUtxos = function(server, wallet, amounts, cb) {
-  var amounts = [].concat(amounts);
+  async.mapSeries(_.range(0, amounts.length > 2 ? 2 : 1), function(i, next) {
-
+    server.createAddress({}, next);
-  async.mapSeries(_.range(1, Math.ceil(amounts.length / 2) + 1), function(i, next) {
-    server.createAddress({}, function(err, address) {
-      next(err, address);
-    });
   }, function(err, addresses) {
-    if (err) throw new Error('Could not generate addresses');
+    should.not.exist(err);
-
+    addresses.should.not.be.empty;
-    var utxos = _.map(amounts, function(amount, i) {
+    var utxos = _.map([].concat(amounts), function(amount, i) {
       var address = addresses[i % addresses.length];
-      var obj = {
+      var confirmations;
+      if (_.isString(amount) && _.startsWith(amount, 'u')) {
+        amount = parseFloat(amount.substring(1));
+        confirmations = 0;
+      } else {
+        confirmations = Math.floor(Math.random() * 100 + 1);
+      }
+      return {
         txid: helpers.randomTXID(),
         vout: Math.floor(Math.random() * 10 + 1),
         satoshis: helpers.toSatoshi(amount).toString(),
         scriptPubKey: address.getScriptPubKey(wallet.m).toBuffer().toString('hex'),
         address: address.address,
-        confirmations: Math.floor(Math.random() * 100 + 1),
+        confirmations: confirmations,
-      };
-      obj.toObject = function() {
-        return obj;
       };
-      return obj;
     });
     blockchainExplorer.getUnspentUtxos = sinon.stub().callsArgWith(1, null, utxos);
 
@@ -289,11 +288,11 @@ helpers.createAddresses = function(server, wallet, main, change, cb) {
 
 var storage, blockchainExplorer;
 
-var useMongo = false;
+var useMongoDb = !!process.env.USE_MONGO_DB;
 
 function initStorage(cb) {
   function getDb(cb) {
-    if (useMongo) {
+    if (useMongoDb) {
       var mongodb = require('mongodb');
       mongodb.MongoClient.connect('mongodb://localhost:27017/bws_test', function(err, db) {
         if (err) throw err;
@@ -1364,13 +1363,19 @@ describe('Wallet service', function() {
     });
 
     it('should get balance', function(done) {
-      helpers.stubUtxos(server, wallet, [1, 2, 3], function() {
+      helpers.stubUtxos(server, wallet, [1, 'u2', 3], function() {
         server.getBalance({}, function(err, balance) {
           should.not.exist(err);
           should.exist(balance);
           balance.totalAmount.should.equal(helpers.toSatoshi(6));
           balance.lockedAmount.should.equal(0);
+          balance.availableAmount.should.equal(helpers.toSatoshi(6));
           balance.totalKbToSendMax.should.equal(1);
+
+          balance.totalConfirmedAmount.should.equal(helpers.toSatoshi(4));
+          balance.lockedConfirmedAmount.should.equal(0);
+          balance.availableConfirmedAmount.should.equal(helpers.toSatoshi(4));
+
           should.exist(balance.byAddress);
           balance.byAddress.length.should.equal(2);
           balance.byAddress[0].amount.should.equal(helpers.toSatoshi(4));
@@ -1390,6 +1395,7 @@ describe('Wallet service', function() {
         should.exist(balance);
         balance.totalAmount.should.equal(0);
         balance.lockedAmount.should.equal(0);
+        balance.availableAmount.should.equal(0);
         balance.totalKbToSendMax.should.equal(0);
         should.exist(balance.byAddress);
         balance.byAddress.length.should.equal(0);
@@ -1405,6 +1411,7 @@ describe('Wallet service', function() {
           should.exist(balance);
           balance.totalAmount.should.equal(0);
           balance.lockedAmount.should.equal(0);
+          balance.availableAmount.should.equal(0);
           balance.totalKbToSendMax.should.equal(0);
           should.exist(balance.byAddress);
           balance.byAddress.length.should.equal(0);
@@ -1610,6 +1617,7 @@ describe('Wallet service', function() {
               balance.totalAmount.should.equal(helpers.toSatoshi(300));
               balance.lockedAmount.should.equal(tx.inputs[0].satoshis);
               balance.lockedAmount.should.be.below(balance.totalAmount);
+              balance.availableAmount.should.equal(balance.totalAmount - balance.lockedAmount);
               server.storage.fetchAddresses(wallet.id, function(err, addresses) {
                 should.not.exist(err);
                 var change = _.filter(addresses, {
@@ -1636,19 +1644,7 @@ describe('Wallet service', function() {
     });
 
     it('should create a tx using confirmed utxos first', function(done) {
-      server.createAddress({}, function(err, address) {
+      helpers.stubUtxos(server, wallet, [1.3, 'u0.5', 'u0.1', 1.2], function(utxos) {
-        var utxos = _.map([1.3, 0.5, 0.1, 1.2], function(amount, i) {
-          return {
-            txid: helpers.randomTXID(),
-            vout: Math.floor((Math.random() * 10) + 1),
-            satoshis: helpers.toSatoshi(amount).toString(),
-            scriptPubKey: address.getScriptPubKey(wallet.m).toBuffer().toString('hex'),
-            address: address.address,
-            confirmations: amount < 1 ? 0 : 1,
-          };
-        });
-        blockchainExplorer.getUnspentUtxos = sinon.stub().callsArgWith(1, null, utxos);
-
         var txOpts = helpers.createSimpleProposalOpts('18PzpUFkFZE8zKWUPvfykkTxmB9oMR8qP7', 1.5, 'some message', TestData.copayers[0].privKey_1H_0);
         server.createTx(txOpts, function(err, tx) {
           should.not.exist(err);
@@ -1661,19 +1657,7 @@ describe('Wallet service', function() {
     });
 
     it('should use unconfirmed utxos only when no more confirmed utxos are available', function(done) {
-      server.createAddress({}, function(err, address) {
+      helpers.stubUtxos(server, wallet, [1.3, 'u0.5', 'u0.1', 1.2], function(utxos) {
-        var utxos = _.map([1.3, 0.5, 0.1, 1.2], function(amount, i) {
-          return {
-            txid: helpers.randomTXID(),
-            vout: Math.floor((Math.random() * 10) + 1),
-            satoshis: helpers.toSatoshi(amount).toString(),
-            scriptPubKey: address.getScriptPubKey(wallet.m).toBuffer().toString('hex'),
-            address: address.address,
-            confirmations: amount < 1 ? 0 : 1,
-          };
-        });
-        blockchainExplorer.getUnspentUtxos = sinon.stub().callsArgWith(1, null, utxos);
-
         var txOpts = helpers.createSimpleProposalOpts('18PzpUFkFZE8zKWUPvfykkTxmB9oMR8qP7', 2.55, 'some message', TestData.copayers[0].privKey_1H_0);
         server.createTx(txOpts, function(err, tx) {
           should.not.exist(err);
@@ -1687,6 +1671,50 @@ describe('Wallet service', function() {
       });
     });
 
+    it('should exclude unconfirmed utxos if specified', function(done) {
+      helpers.stubUtxos(server, wallet, [1.3, 'u2', 'u0.1', 1.2], function(utxos) {
+        var txOpts = helpers.createSimpleProposalOpts('18PzpUFkFZE8zKWUPvfykkTxmB9oMR8qP7', 3, 'some message', TestData.copayers[0].privKey_1H_0);
+        txOpts.excludeUnconfirmedUtxos = true;
+        server.createTx(txOpts, function(err, tx) {
+          should.exist(err);
+          err.code.should.equal('INSUFFICIENTFUNDS');
+          err.message.should.equal('Insufficient funds');
+          var txOpts = helpers.createSimpleProposalOpts('18PzpUFkFZE8zKWUPvfykkTxmB9oMR8qP7', 2.5, 'some message', TestData.copayers[0].privKey_1H_0);
+          txOpts.excludeUnconfirmedUtxos = true;
+          server.createTx(txOpts, function(err, tx) {
+            should.exist(err);
+            err.code.should.equal('INSUFFICIENTFUNDS');
+            err.message.should.equal('Insufficient funds for fee');
+            done();
+          });
+        });
+      });
+    });
+
+    it('should use non-locked confirmed utxos when specified', function(done) {
+      helpers.stubUtxos(server, wallet, [1.3, 'u2', 'u0.1', 1.2], function(utxos) {
+        var txOpts = helpers.createSimpleProposalOpts('18PzpUFkFZE8zKWUPvfykkTxmB9oMR8qP7', 1.4, 'some message', TestData.copayers[0].privKey_1H_0);
+        txOpts.excludeUnconfirmedUtxos = true;
+        server.createTx(txOpts, function(err, tx) {
+          should.not.exist(err);
+          should.exist(tx);
+          tx.inputs.length.should.equal(2);
+          server.getBalance({}, function(err, balance) {
+            should.not.exist(err);
+            balance.lockedConfirmedAmount.should.equal(helpers.toSatoshi(2.5));
+            balance.availableConfirmedAmount.should.equal(0);
+            var txOpts = helpers.createSimpleProposalOpts('18PzpUFkFZE8zKWUPvfykkTxmB9oMR8qP7', 0.01, 'some message', TestData.copayers[0].privKey_1H_0);
+            txOpts.excludeUnconfirmedUtxos = true;
+            server.createTx(txOpts, function(err, tx) {
+              should.exist(err);
+              err.code.should.equal('LOCKEDFUNDS');
+              done();
+            });
+          });
+        });
+      });
+    });
+
     it('should fail gracefully if unable to reach the blockchain', function(done) {
       blockchainExplorer.getUnspentUtxos = sinon.stub().callsArgWith(1, 'dummy error');
       server.createAddress({}, function(err, address) {
@@ -1956,11 +1984,10 @@ describe('Wallet service', function() {
               server.getBalance({}, function(err, balance) {
                 should.not.exist(err);
                 balance.totalAmount.should.equal(helpers.toSatoshi(30.6));
-                var amountInputs = _.reduce(_.pluck(txs[0].inputs, 'satoshis'), function(memo, satoshis) {
+                var amountInputs = _.sum(txs[0].inputs, 'satoshis');
-                  return memo + satoshis;
-                }, 0);
                 balance.lockedAmount.should.equal(amountInputs);
                 balance.lockedAmount.should.be.below(balance.totalAmount);
+                balance.availableAmount.should.equal(balance.totalAmount - balance.lockedAmount);
                 done();
               });
             });
@@ -2055,6 +2082,7 @@ describe('Wallet service', function() {
           should.not.exist(err);
           balance.totalAmount.should.equal(helpers.toSatoshi(9));
           balance.lockedAmount.should.equal(0);
+          balance.availableAmount.should.equal(helpers.toSatoshi(9));
           balance.totalKbToSendMax.should.equal(3);
           var max = (balance.totalAmount - balance.lockedAmount) - (balance.totalKbToSendMax * 10000);
           var txOpts = helpers.createSimpleProposalOpts('18PzpUFkFZE8zKWUPvfykkTxmB9oMR8qP7', max / 1e8, null, TestData.copayers[0].privKey_1H_0);
@@ -2081,6 +2109,7 @@ describe('Wallet service', function() {
             should.not.exist(err);
             balance.totalAmount.should.equal(helpers.toSatoshi(9));
             balance.lockedAmount.should.equal(helpers.toSatoshi(4));
+            balance.availableAmount.should.equal(helpers.toSatoshi(5));
             balance.totalKbToSendMax.should.equal(2);
             var max = (balance.totalAmount - balance.lockedAmount) - (balance.totalKbToSendMax * 2000);
             var txOpts = helpers.createSimpleProposalOpts('18PzpUFkFZE8zKWUPvfykkTxmB9oMR8qP7', max / 1e8, null, TestData.copayers[0].privKey_1H_0, 2000);