Merge pull request #477 from isocolsky/utxo-selection

Improve UTXO selection

config.js

@@ -43,6 +43,7 @@ var config = {
     testnet: {
       provider: 'insight',
       url: 'https://test-insight.bitpay.com:443',
+      // url: 'http://localhost:3001',
       // Multiple servers (in priority order)
       // url: ['http://a.b.c', 'https://test-insight.bitpay.com:443'],
     },

lib/common/defaults.js

@@ -49,4 +49,18 @@ Defaults.FIAT_RATE_MAX_LOOK_BACK_TIME = 120; // In minutes
 
 Defaults.HISTORY_LIMIT = 100;
 
+// The maximum amount of an UTXO to be considered too big to be used in the tx before exploring smaller
+// alternatives (proportinal to tx amount).
+Defaults.UTXO_SELECTION_MAX_SINGLE_UTXO_FACTOR = 2;
+
+// The minimum amount an UTXO need to contribute proportional to tx amount.
+Defaults.UTXO_SELECTION_MIN_TX_AMOUNT_VS_UTXO_FACTOR = 0.1;
+
+// The maximum threshold to consider fees non-significant in relation to tx amount.
+Defaults.UTXO_SELECTION_MAX_FEE_VS_TX_AMOUNT_FACTOR = 0.05;
+
+// The maximum amount to pay for using small inputs instead of one big input
+// when fees are significant (proportional to how much we would pay for using that big input only).
+Defaults.UTXO_SELECTION_MAX_FEE_VS_SINGLE_UTXO_FEE_FACTOR = 5;
+
 module.exports = Defaults;

lib/common/utils.js

@@ -23,7 +23,7 @@ Utils.checkRequired = function(obj, args) {
  * @return {number}
  */
 Utils.strip = function(number) {
-  return (parseFloat(number.toPrecision(12)));
+  return parseFloat(number.toPrecision(12));
 }
 
 /* TODO: It would be nice to be compatible with bitcoind signmessage. How
@@ -66,6 +66,11 @@ Utils.formatAmount = function(satoshis, unit, opts) {
       maxDecimals: 0,
       minDecimals: 0,
     },
+    sat: {
+      toSatoshis: 1,
+      maxDecimals: 0,
+      minDecimals: 0,
+    }
   };
 
   $.shouldBeNumber(satoshis);
@@ -88,10 +93,33 @@ Utils.formatAmount = function(satoshis, unit, opts) {
 
   opts = opts || {};
 
-  var u = UNITS[unit];
+  var u = _.assign(UNITS[unit], opts);
   var amount = (satoshis / u.toSatoshis).toFixed(u.maxDecimals);
   return addSeparators(amount, opts.thousandsSeparator || ',', opts.decimalSeparator || '.', u.minDecimals);
 };
 
+Utils.formatAmountInBtc = function(amount) {
+  return Utils.formatAmount(amount, 'btc', {
+    minDecimals: 8,
+    maxDecimals: 8,
+  }) + 'btc';
+};
+
+Utils.formatUtxos = function(utxos) {
+  if (_.isEmpty(utxos)) return 'none';
+  return _.map([].concat(utxos), function(i) {
+    var amount = Utils.formatAmountInBtc(i.satoshis);
+    var confirmations = i.confirmations ? i.confirmations + 'c' : 'u';
+    return amount + '/' + confirmations;
+  }).join(', ');
+};
+
+Utils.formatRatio = function(ratio) {
+  return (ratio * 100.).toFixed(4) + '%';
+};
+
+Utils.formatSize = function(size) {
+  return (size / 1000.).toFixed(4) + 'kB';
+};
 
 module.exports = Utils;

lib/model/txproposal.js

@@ -33,7 +33,6 @@ TxProposal.create = function(opts) {
   x.message = opts.message;
   x.payProUrl = opts.payProUrl;
   x.changeAddress = opts.changeAddress;
-  x.setInputs(opts.inputs);
   x.outputs = _.map(opts.outputs, function(output) {
     return _.pick(output, ['amount', 'toAddress', 'message']);
   });
@@ -44,7 +43,6 @@ TxProposal.create = function(opts) {
   x.requiredRejections = Math.min(x.walletM, x.walletN - x.walletM + 1),
   x.status = 'temporary';
   x.actions = [];
-  x.fee = null;
   x.feePerKb = opts.feePerKb;
   x.excludeUnconfirmedUtxos = opts.excludeUnconfirmedUtxos;
 
@@ -59,6 +57,9 @@ TxProposal.create = function(opts) {
   } catch (ex) {}
   $.checkState(_.contains(_.values(Constants.NETWORKS), x.network));
 
+  x.setInputs(opts.inputs);
+  x.fee = opts.fee;
+
   return x;
 };
 
@@ -137,6 +138,7 @@ TxProposal.prototype._buildTx = function() {
   switch (self.addressType) {
     case Constants.SCRIPT_TYPES.P2SH:
       _.each(self.inputs, function(i) {
+        $.checkState(i.publicKeys, 'Inputs should include public keys');
         t.from(i, i.publicKeys, self.requiredSignatures);
       });
       break;
@@ -158,7 +160,13 @@ TxProposal.prototype._buildTx = function() {
   });
 
   t.fee(self.fee);
+
+  var totalInputs = _.sum(self.inputs, 'satoshis');
+  var totalOutputs = _.sum(self.outputs, 'satoshis');
+
+  if (totalInputs - totalOutputs - self.fee > 0) {
     t.change(self.changeAddress.address);
+  }
 
   // Shuffle outputs for improved privacy
   if (t.outputs.length > 1) {
@@ -173,8 +181,8 @@ TxProposal.prototype._buildTx = function() {
     });
   }
 
-  // Validate inputs vs outputs independently of Bitcore
-  var totalInputs = _.sum(self.inputs, 'satoshis');
+  // Validate actual inputs vs outputs independently of Bitcore
+  var totalInputs = _.sum(t.inputs, 'satoshis');
   var totalOutputs = _.sum(t.outputs, 'satoshis');
 
   $.checkState(totalInputs - totalOutputs <= Defaults.MAX_TX_FEE);
@@ -219,13 +227,22 @@ TxProposal.prototype.getRawTx = function() {
   return t.uncheckedSerialize();
 };
 
+TxProposal.prototype.getEstimatedSizeForSingleInput = function() {
+  switch (this.addressType) {
+    case Constants.SCRIPT_TYPES.P2PKH:
+      return 147;
+    default:
+    case Constants.SCRIPT_TYPES.P2SH:
+      return this.requiredSignatures * 72 + this.walletN * 36 + 44;
+  }
+};
+
 TxProposal.prototype.getEstimatedSize = function() {
   // Note: found empirically based on all multisig P2SH inputs and within m & n allowed limits.
-  var safetyMargin = 0.05;
-  var walletM = this.requiredSignatures;
+  var safetyMargin = 0.02;
 
   var overhead = 4 + 4 + 9 + 9;
-  var inputSize = walletM * 72 + this.walletN * 36 + 44;
+  var inputSize = this.getEstimatedSizeForSingleInput();
   var outputSize = 34;
   var nbInputs = this.inputs.length;
   var nbOutputs = (_.isArray(this.outputs) ? Math.max(1, this.outputs.length) : 1) + 1;

lib/model/txproposal_legacy.js

@@ -273,16 +273,19 @@ TxProposal.prototype.getRawTx = function() {
   return t.uncheckedSerialize();
 };
 
+TxProposal.prototype.getEstimatedSizeForSingleInput = function() {
+  return this.requiredSignatures * 72 + this.walletN * 36 + 44;
+};
+
 TxProposal.prototype.getEstimatedSize = function() {
   // Note: found empirically based on all multisig P2SH inputs and within m & n allowed limits.
   var safetyMargin = 0.05;
-  var walletM = this.requiredSignatures;
 
   var overhead = 4 + 4 + 9 + 9;
-  var inputSize = walletM * 72 + this.walletN * 36 + 44;
+  var inputSize = this.getEstimatedSizeForSingleInput();
   var outputSize = 34;
   var nbInputs = this.inputs.length;
-  var nbOutputs = (_.isArray(this.outputs) ? this.outputs.length : 1) + 1;
+  var nbOutputs = (_.isArray(this.outputs) ? Math.max(1, this.outputs.length) : 1) + 1;
 
   var size = overhead + inputSize * nbInputs + outputSize * nbOutputs;
 

lib/server.js

@@ -1230,7 +1230,11 @@ WalletService.prototype.getFeeLevels = function(opts, cb) {
   });
 };
 
-WalletService.prototype._checkTxAndEstimateFee = function(txp) {
+WalletService.prototype._estimateFee = function(txp) {
+  txp.estimateFee();
+};
+
+WalletService.prototype._checkTx = function(txp) {
   var bitcoreError;
 
   var serializationOpts = {
@@ -1241,7 +1245,8 @@ WalletService.prototype._checkTxAndEstimateFee = function(txp) {
     serializationOpts.disableLargeFees = true;
   }
 
-  txp.estimateFee();
+  if (txp.getEstimatedSize() / 1000 > Defaults.MAX_TX_SIZE_IN_KB)
+    return Errors.TX_MAX_SIZE_EXCEEDED;
 
   try {
     var bitcoreTx = txp.getBitcoreTx();
@@ -1264,41 +1269,165 @@ WalletService.prototype._checkTxAndEstimateFee = function(txp) {
 
 WalletService.prototype._selectTxInputs = function(txp, utxosToExclude, cb) {
   var self = this;
-  //todo: check inputs are ours and has enough value
-  if (txp.inputs && txp.inputs.length > 0) {
-    return cb(self._checkTxAndEstimateFee(txp));
-  }
-
-  function sortUtxos(utxos) {
-    var list = _.map(utxos, function(utxo) {
-      var order;
-      if (utxo.confirmations == 0) {
-        order = 0;
-      } else if (utxo.confirmations < 6) {
-        order = -1;
-      } else {
-        order = -2;
+
+  //todo: check inputs are ours and have enough value
+  if (txp.inputs && !_.isEmpty(txp.inputs)) {
+    if (!_.isNumber(txp.fee))
+      self._estimateFee(txp);
+    return cb(self._checkTx(txp));
   }
-      return {
-        order: order,
-        utxo: utxo
-      };
-    });
-    return _.pluck(_.sortBy(list, 'order'), 'utxo');
-  };
 
-  self._getUtxosForCurrentWallet(null, function(err, utxos) {
-    if (err) return cb(err);
+  var txpAmount = txp.getTotalAmount();
+  var baseTxpSize = txp.getEstimatedSize();
+  var baseTxpFee = baseTxpSize * txp.feePerKb / 1000.;
+  var sizePerInput = txp.getEstimatedSizeForSingleInput();
+  var feePerInput = sizePerInput * txp.feePerKb / 1000.;
 
+  function sanitizeUtxos(utxos) {
     var excludeIndex = _.reduce(utxosToExclude, function(res, val) {
       res[val] = val;
       return res;
     }, {});
 
-    utxos = _.reject(utxos, function(utxo) {
-      return excludeIndex[utxo.txid + ":" + utxo.vout];
+    return _.filter(utxos, function(utxo) {
+      if (utxo.locked) return false;
+      if (utxo.satoshis <= feePerInput) return false;
+      if (txp.excludeUnconfirmedUtxos && !utxo.confirmations) return false;
+      if (excludeIndex[utxo.txid + ":" + utxo.vout]) return false;
+      return true;
+    });
+  };
+
+  function partitionUtxos(utxos) {
+    return _.groupBy(utxos, function(utxo) {
+      if (utxo.confirmations == 0) return '0'
+      if (utxo.confirmations < 6) return '<6';
+      return '6+';
+    });
+  };
+
+  function select(utxos, cb) {
+    var totalValueInUtxos = _.sum(utxos, 'satoshis');
+    var netValueInUtxos = totalValueInUtxos - baseTxpFee - (utxos.length * feePerInput);
+
+    if (totalValueInUtxos < txpAmount) {
+      log.debug('Total value in all utxos (' + Utils.formatAmountInBtc(totalValueInUtxos) + ') is insufficient to cover for txp amount (' + Utils.formatAmountInBtc(txpAmount) + ')');
+      return cb(Errors.INSUFFICIENT_FUNDS);
+    }
+    if (netValueInUtxos < txpAmount) {
+      log.debug('Value after fees in all utxos (' + Utils.formatAmountInBtc(netValueInUtxos) + ') is insufficient to cover for txp amount (' + Utils.formatAmountInBtc(txpAmount) + ')');
+      return cb(Errors.INSUFFICIENT_FUNDS_FOR_FEE);
+    }
+
+    var bigInputThreshold = txpAmount * Defaults.UTXO_SELECTION_MAX_SINGLE_UTXO_FACTOR + (baseTxpFee + feePerInput);
+    log.debug('Big input threshold ' + Utils.formatAmountInBtc(bigInputThreshold));
+
+    var partitions = _.partition(utxos, function(utxo) {
+      return utxo.satoshis > bigInputThreshold;
+    });
+
+    var bigInputs = _.sortBy(partitions[0], 'satoshis');
+    var smallInputs = _.sortBy(partitions[1], function(utxo) {
+      return -utxo.satoshis;
+    });
+
+    log.debug('Considering ' + bigInputs.length + ' big inputs (' + Utils.formatUtxos(bigInputs) + ')');
+    log.debug('Considering ' + smallInputs.length + ' small inputs (' + Utils.formatUtxos(smallInputs) + ')');
+
+    var total = 0;
+    var netTotal = -baseTxpFee;
+    var selected = [];
+    var fee;
+    var error;
+
+    _.each(smallInputs, function(input, i) {
+      log.debug('Input #' + i + ': ' + Utils.formatUtxos(input));
+
+      var netInputAmount = input.satoshis - feePerInput;
+
+      log.debug('The input contributes ' + Utils.formatAmountInBtc(netInputAmount));
+
+      selected.push(input);
+
+      total += input.satoshis;
+      netTotal += netInputAmount;
+
+      var txpSize = baseTxpSize + selected.length * sizePerInput;
+      fee = Math.round(baseTxpFee + selected.length * feePerInput);
+
+      log.debug('Tx size: ' + Utils.formatSize(txpSize) + ', Tx fee: ' + Utils.formatAmountInBtc(fee));
+
+      var feeVsAmountRatio = fee / txpAmount;
+      var amountVsUtxoRatio = netInputAmount / txpAmount;
+
+      log.debug('Fee/Tx amount: ' + Utils.formatRatio(feeVsAmountRatio) + ' (max: ' + Utils.formatRatio(Defaults.UTXO_SELECTION_MAX_FEE_VS_TX_AMOUNT_FACTOR) + ')');
+      log.debug('Tx amount/Input amount:' + Utils.formatRatio(amountVsUtxoRatio) + ' (min: ' + Utils.formatRatio(Defaults.UTXO_SELECTION_MIN_TX_AMOUNT_VS_UTXO_FACTOR) + ')');
+
+      if (txpSize / 1000. > Defaults.MAX_TX_SIZE_IN_KB) {
+        log.debug('Breaking because tx size (' + Utils.formatSize(txpSize) + ') is too big (max: ' + Utils.formatSize(Defaults.MAX_TX_SIZE_IN_KB * 1000.) + ')');
+        error = Errors.TX_MAX_SIZE_EXCEEDED;
+        return false;
+      }
+
+      if (!_.isEmpty(bigInputs)) {
+        if (amountVsUtxoRatio < Defaults.UTXO_SELECTION_MIN_TX_AMOUNT_VS_UTXO_FACTOR) {
+          log.debug('Breaking because utxo is too small compared to tx amount');
+          return false;
+        }
+
+        if (feeVsAmountRatio > Defaults.UTXO_SELECTION_MAX_FEE_VS_TX_AMOUNT_FACTOR) {
+          var feeVsSingleInputFeeRatio = fee / (baseTxpFee + feePerInput);
+          log.debug('Fee/Single-input fee: ' + Utils.formatRatio(feeVsSingleInputFeeRatio) + ' (max: ' + Utils.formatRatio(Defaults.UTXO_SELECTION_MAX_FEE_VS_SINGLE_UTXO_FEE_FACTOR) + ')' + ' loses wrt single-input tx: ' + Utils.formatAmountInBtc((selected.length - 1) * feePerInput));
+          if (feeVsSingleInputFeeRatio > Defaults.UTXO_SELECTION_MAX_FEE_VS_SINGLE_UTXO_FEE_FACTOR) {
+            log.debug('Breaking because fee is too significant compared to tx amount and it is too expensive compared to using single input');
+            return false;
+          }
+        }
+      }
+
+      log.debug('Cumuled total so far: ' + Utils.formatAmountInBtc(total) + ', Net total so far: ' + Utils.formatAmountInBtc(netTotal));
+
+      if (netTotal >= txpAmount) {
+        var changeAmount = Math.round(total - txpAmount - fee);
+        log.debug('Tx change: ', Utils.formatAmountInBtc(changeAmount));
+
+        if (changeAmount > 0 && changeAmount <= Bitcore.Transaction.DUST_AMOUNT) {
+          log.debug('Change below dust amount (' + Utils.formatAmountInBtc(Bitcore.Transaction.DUST_AMOUNT) + ')');
+          // Remove dust change by incrementing fee
+          fee += changeAmount;
+        }
+
+        return false;
+      }
     });
 
+    if (netTotal < txpAmount) {
+      log.debug('Could not reach Txp total (' + Utils.formatAmountInBtc(txpAmount) + '), still missing: ' + Utils.formatAmountInBtc(txpAmount - netTotal));
+
+      selected = [];
+      if (!_.isEmpty(bigInputs)) {
+        var input = _.first(bigInputs);
+        log.debug('Using big input: ', Utils.formatUtxos(input));
+        total = input.satoshis;
+        fee = Math.round(baseTxpFee + feePerInput);
+        netTotal = total - fee;
+        selected = [input];
+      }
+    }
+
+    if (_.isEmpty(selected)) {
+      log.debug('Could not find enough funds within this utxo subset');
+      return cb(error || Errors.INSUFFICIENT_FUNDS_FOR_FEE);
+    }
+
+    return cb(null, selected, fee);
+  };
+
+  log.debug('Selecting inputs for a ' + Utils.formatAmountInBtc(txp.getTotalAmount()) + ' txp');
+
+  self._getUtxosForCurrentWallet(null, function(err, utxos) {
+    if (err) return cb(err);
+
     var totalAmount;
     var availableAmount;
 
@@ -1314,36 +1443,72 @@ WalletService.prototype._selectTxInputs = function(txp, utxosToExclude, cb) {
     if (totalAmount < txp.getTotalAmount()) return cb(Errors.INSUFFICIENT_FUNDS);
     if (availableAmount < txp.getTotalAmount()) return cb(Errors.LOCKED_FUNDS);
 
-    // Prepare UTXOs list
-    utxos = _.reject(utxos, 'locked');
-    if (txp.excludeUnconfirmedUtxos) {
-      utxos = _.filter(utxos, 'confirmations');
-    }
+    utxos = sanitizeUtxos(utxos);
+
+    log.debug('Considering ' + utxos.length + ' utxos (' + Utils.formatUtxos(utxos) + ')');
 
+    var groups = [6, 1];
+    if (!txp.excludeUnconfirmedUtxos) groups.push(0);
+
+    var inputs = [];
+    var fee;
+    var selectionError;
     var i = 0;
-    var total = 0;
-    var selected = [];
-    var inputs = sortUtxos(utxos);
+    var lastGroupLength;
+    async.whilst(function() {
+      return i < groups.length && _.isEmpty(inputs);
+    }, function(next) {
+      var group = groups[i++];
 
-    var bitcoreTx, bitcoreError;
+      var candidateUtxos = _.filter(utxos, function(utxo) {
+        return utxo.confirmations >= group;
+      });
 
-    function select() {
-      if (i >= inputs.length) return cb(bitcoreError || new Error('Could not select tx inputs'));
+      log.debug('Group >= ' + group);
 
-      var input = inputs[i++];
-      selected.push(input);
-      total += input.satoshis;
-      if (total >= txp.getTotalAmount()) {
-        txp.setInputs(selected);
-        bitcoreError = self._checkTxAndEstimateFee(txp);
-        if (!bitcoreError) return cb();
-        if (txp.getEstimatedSize() / 1000 > Defaults.MAX_TX_SIZE_IN_KB)
-          return cb(Errors.TX_MAX_SIZE_EXCEEDED);
+      // If this group does not have any new elements, skip it
+      if (lastGroupLength === candidateUtxos.length) {
+        log.debug('This group is identical to the one already explored');
+        return next();
       }
-      setTimeout(select, 0);
-    };
 
-    select();
+      log.debug('Candidate utxos: ' + Utils.formatUtxos(candidateUtxos));
+
+      lastGroupLength = candidateUtxos.length;
+
+      select(candidateUtxos, function(err, selectedInputs, selectedFee) {
+        if (err) {
+          log.debug('No inputs selected on this group: ', err);
+          selectionError = err;
+          return next();
+        }
+
+        selectionError = null;
+        inputs = selectedInputs;
+        fee = selectedFee;
+
+        log.debug('Selected inputs from this group: ' + Utils.formatUtxos(inputs));
+        log.debug('Fee for this selection: ' + Utils.formatAmountInBtc(fee));
+
+        return next();
+      });
+    }, function(err) {
+      if (err) return cb(err);
+      if (selectionError || _.isEmpty(inputs)) return cb(selectionError || new Error('Could not select tx inputs'));
+
+      txp.setInputs(_.shuffle(inputs));
+      txp.fee = fee;
+
+      var err = self._checkTx(txp);
+
+      if (!err) {
+        log.debug('Successfully built transaction. Total fees: ' + Utils.formatAmountInBtc(txp.fee) + ', total change: ' + Utils.formatAmountInBtc(_.sum(txp.inputs, 'satoshis') - txp.fee));
+      } else {
+        log.warn('Error building transaction', err);
+      }
+
+      return cb(err);
+    });
   });
 };
 
@@ -1559,21 +1724,28 @@ WalletService.prototype.createTxLegacy = function(opts, cb) {
  * @param {number} opts.outputs[].amount - Amount to transfer in satoshi.
  * @param {string} opts.outputs[].message - A message to attach to this output.
  * @param {string} opts.message - A message to attach to this transaction.
- * @param {Array} opts.inputs - Optional. Inputs for this TX
- * @param {string} opts.feePerKb - The fee per kB to use for this TX.
+ * @param {number} opts.feePerKb - The fee per kB to use for this TX.
  * @param {string} opts.payProUrl - Optional. Paypro URL for peers to verify TX
  * @param {string} opts.excludeUnconfirmedUtxos[=false] - Optional. Do not use UTXOs of unconfirmed transactions as inputs
  * @param {string} opts.validateOutputs[=true] - Optional. Perform validation on outputs.
+ * @param {Array} opts.inputs - Optional. Inputs for this TX
+ * @param {number} opts.fee - Optional. The fee to use for this TX (used only when opts.inputs is specified).
  * @returns {TxProposal} Transaction proposal.
  */
 WalletService.prototype.createTx = function(opts, cb) {
   var self = this;
 
-  if (!Utils.checkRequired(opts, ['outputs', 'feePerKb']))
+  if (!Utils.checkRequired(opts, ['outputs']))
+    return cb(new ClientError('Required argument missing'));
+
+  // feePerKb is required unless inputs & fee are specified
+  if (!_.isNumber(opts.feePerKb) && !(opts.inputs && _.isNumber(opts.fee)))
     return cb(new ClientError('Required argument missing'));
 
+  if (_.isNumber(opts.feePerKb)) {
     if (opts.feePerKb < Defaults.MIN_FEE_PER_KB || opts.feePerKb > Defaults.MAX_FEE_PER_KB)
       return cb(new ClientError('Invalid fee per KB'));
+  }
 
   self._runLocked(cb, function(cb) {
     self.getWallet({}, function(err, wallet) {
@@ -1594,7 +1766,6 @@ WalletService.prototype.createTx = function(opts, cb) {
         var txOpts = {
           walletId: self.walletId,
           creatorId: self.copayerId,
-          inputs: opts.inputs,
           outputs: opts.outputs,
           message: opts.message,
           changeAddress: wallet.createAddress(true),
@@ -1606,6 +1777,8 @@ WalletService.prototype.createTx = function(opts, cb) {
           validateOutputs: !opts.validateOutputs,
           addressType: wallet.addressType,
           customData: opts.customData,
+          inputs: opts.inputs,
+          fee: opts.inputs && !_.isNumber(opts.feePerKb) ? opts.fee : null,
         };
 
         var txp = Model.TxProposal.create(txOpts);

test/integration/helpers.js

@@ -212,6 +212,40 @@ helpers.toSatoshi = function(btc) {
   }
 };
 
+helpers._parseAmount = function(str) {
+  var result = {
+    amount: +0,
+    confirmations: _.random(6, 100),
+  };
+
+  if (_.isNumber(str)) str = str.toString();
+
+  var re = /^((?:\d+c)|u)?\s*([\d\.]+)\s*(btc|bit|sat)?$/;
+  var match = str.match(re);
+
+  if (!match) throw new Error('Could not parse amount ' + str);
+
+  if (match[1]) {
+    if (match[1] == 'u') result.confirmations = 0;
+    if (_.endsWith(match[1], 'c')) result.confirmations = +match[1].slice(0, -1);
+  }
+
+  switch (match[3]) {
+    default:
+    case 'btc':
+      result.amount = Utils.strip(+match[2] * 1e8);
+      break;
+    case 'bit':
+      result.amount = Utils.strip(+match[2] * 1e2);
+      break
+    case 'sat':
+      result.amount = Utils.strip(+match[2]);
+      break;
+  };
+
+  return result;
+};
+
 helpers.stubUtxos = function(server, wallet, amounts, opts, cb) {
   if (_.isFunction(opts)) {
     cb = opts;
@@ -233,14 +267,9 @@ helpers.stubUtxos = function(server, wallet, amounts, opts, cb) {
       addresses.should.not.be.empty;
 
       var utxos = _.compact(_.map([].concat(amounts), function(amount, i) {
-        var confirmations;
-        if (_.isString(amount) && _.startsWith(amount, 'u')) {
-          amount = parseFloat(amount.substring(1));
-          confirmations = 0;
-        } else {
-          confirmations = Math.floor(Math.random() * 100 + 1);
-        }
-        if (amount <= 0) return null;
+        var parsed = helpers._parseAmount(amount);
+
+        if (parsed.amount <= 0) return null;
 
         var address = addresses[i % addresses.length];
 
@@ -257,11 +286,12 @@ helpers.stubUtxos = function(server, wallet, amounts, opts, cb) {
 
         return {
           txid: helpers.randomTXID(),
-          vout: Math.floor(Math.random() * 10 + 1),
-          satoshis: helpers.toSatoshi(amount),
+          vout: _.random(0, 10),
+          satoshis: parsed.amount,
           scriptPubKey: scriptPubKey.toBuffer().toString('hex'),
           address: address.address,
-          confirmations: confirmations
+          confirmations: parsed.confirmations,
+          publicKeys: address.publicKeys,
         };
       }));
 

test/integration/server.js

@@ -239,6 +239,8 @@ describe('Wallet service', function() {
   });
 
   describe('#joinWallet', function() {
+    describe('New clients', function() {
+
       var server, walletId;
       beforeEach(function(done) {
         server = new WalletService();
@@ -526,7 +528,7 @@ describe('Wallet service', function() {
       });
     });
 
-  describe('#joinWallet new/legacy clients', function() {
+    describe('Interaction new/legacy clients', function() {
       var server;
       beforeEach(function() {
         server = new WalletService();
@@ -556,6 +558,7 @@ describe('Wallet service', function() {
           });
         });
       });
+
       it('should fail to join new wallet from legacy client', function(done) {
         var walletOpts = {
           name: 'my wallet',
@@ -581,6 +584,7 @@ describe('Wallet service', function() {
         });
       });
     });
+  });
 
   describe('Address derivation strategy', function() {
     var server;
@@ -2128,6 +2132,7 @@ describe('Wallet service', function() {
       });
 
       it('should use unconfirmed utxos only when no more confirmed utxos are available', function(done) {
+        // log.level = 'debug';
         helpers.stubUtxos(server, wallet, [1.3, 'u0.5', 'u0.1', 1.2], function(utxos) {
           var txOpts = helpers.createSimpleProposalOpts('18PzpUFkFZE8zKWUPvfykkTxmB9oMR8qP7', 2.55, TestData.copayers[0].privKey_1H_0, {
             message: 'some message'
@@ -2350,10 +2355,11 @@ describe('Wallet service', function() {
       });
 
       it('should fail to create a tx exceeding max size in kb', function(done) {
-        helpers.stubUtxos(server, wallet, _.range(1, 10, 0), function() {
-          var txOpts = helpers.createSimpleProposalOpts('18PzpUFkFZE8zKWUPvfykkTxmB9oMR8qP7', 9, TestData.copayers[0].privKey_1H_0);
+        // log.level = 'debug';
         var _oldDefault = Defaults.MAX_TX_SIZE_IN_KB;
         Defaults.MAX_TX_SIZE_IN_KB = 1;
+        helpers.stubUtxos(server, wallet, _.range(1, 10, 0), function() {
+          var txOpts = helpers.createSimpleProposalOpts('18PzpUFkFZE8zKWUPvfykkTxmB9oMR8qP7', 8, TestData.copayers[0].privKey_1H_0);
           server.createTxLegacy(txOpts, function(err, tx) {
             should.exist(err);
             err.code.should.equal('TX_MAX_SIZE_EXCEEDED');
@@ -2375,19 +2381,18 @@ describe('Wallet service', function() {
         });
       });
 
-      it('should fail to create tx that would return change for dust amount', function(done) {
+      it('should modify fee if tx would return change for dust amount', function(done) {
         helpers.stubUtxos(server, wallet, [1], function() {
-          var fee = 4095 / 1e8; // The exact fee of the resulting tx
-          var change = 100 / 1e8; // Below dust
-          var amount = 1 - fee - change;
+          var fee = 4095; // The exact fee of the resulting tx (based exclusively on feePerKB && size)
+          var change = 100; // Below dust
+          var amount = (1e8 - fee - change) / 1e8;
 
           var txOpts = helpers.createSimpleProposalOpts('18PzpUFkFZE8zKWUPvfykkTxmB9oMR8qP7', amount, TestData.copayers[0].privKey_1H_0, {
             feePerKb: 10000
           });
           server.createTxLegacy(txOpts, function(err, tx) {
-            should.exist(err);
-            err.code.should.equal('DUST_AMOUNT');
-            err.message.should.equal('Amount below dust threshold');
+            should.not.exist(err);
+            tx.fee.should.equal(fee + change);
             done();
           });
         });
@@ -3085,10 +3090,32 @@ describe('Wallet service', function() {
         });
       });
 
+      it('should be able to specify inputs & absolute fee', function(done) {
+        helpers.stubUtxos(server, wallet, [1, 2], function(utxos) {
+          var txOpts = {
+            outputs: [{
+              toAddress: '18PzpUFkFZE8zKWUPvfykkTxmB9oMR8qP7',
+              amount: 0.8e8,
+            }],
+            inputs: utxos,
+            fee: 1000e2,
+          };
+          server.createTx(txOpts, function(err, tx) {
+            should.not.exist(err);
+            should.exist(tx);
+            tx.amount.should.equal(helpers.toSatoshi(0.8));
+            should.not.exist(tx.feePerKb);
+            tx.fee.should.equal(1000e2);
+            var t = tx.getBitcoreTx();
+            t.getFee().should.equal(1000e2);
+            t.getChangeOutput().satoshis.should.equal(3e8 - 0.8e8 - 1000e2);
+            done();
+          });
+        });
       });
     });
 
-  describe('#createTx backoff time', function(done) {
+    describe('Backoff time', function(done) {
       var server, wallet, txid;
 
       beforeEach(function(done) {
@@ -3165,6 +3192,397 @@ describe('Wallet service', function() {
       });
     });
 
+    describe('UTXO Selection', function() {
+      var server, wallet;
+      beforeEach(function(done) {
+        // log.level = 'debug';
+        helpers.createAndJoinWallet(2, 3, function(s, w) {
+          server = s;
+          wallet = w;
+          done();
+        });
+      });
+      afterEach(function() {
+        log.level = 'info';
+      });
+
+      it('should select a single utxo if within thresholds relative to tx amount', function(done) {
+        helpers.stubUtxos(server, wallet, [1, '350bit', '100bit', '100bit', '100bit'], function() {
+          var txOpts = {
+            outputs: [{
+              toAddress: '18PzpUFkFZE8zKWUPvfykkTxmB9oMR8qP7',
+              amount: 200e2,
+            }],
+            feePerKb: 10e2,
+          };
+          server.createTx(txOpts, function(err, txp) {
+            should.not.exist(err);
+            should.exist(txp);
+            txp.inputs.length.should.equal(1);
+            txp.inputs[0].satoshis.should.equal(35000);
+
+            done();
+          });
+        });
+      });
+      it('should return inputs in random order', function(done) {
+        // NOTE: this test has a chance of failing of 1 in 1'073'741'824 :P
+        helpers.stubUtxos(server, wallet, _.range(1, 31), function(utxos) {
+          var txOpts = {
+            outputs: [{
+              toAddress: '18PzpUFkFZE8zKWUPvfykkTxmB9oMR8qP7',
+              amount: _.sum(utxos, 'satoshis') - 0.5e8,
+            }],
+            feePerKb: 100e2,
+          };
+          server.createTx(txOpts, function(err, txp) {
+            should.not.exist(err);
+            should.exist(txp);
+            var amounts = _.pluck(txp.inputs, 'satoshis');
+            amounts.length.should.equal(30);
+            _.all(amounts, function(amount, i) {
+              if (i == 0) return true;
+              return amount < amounts[i - 1];
+            }).should.be.false;
+            done();
+          });
+        });
+      });
+      it('should select a confirmed utxos if within thresholds relative to tx amount', function(done) {
+        helpers.stubUtxos(server, wallet, [1, 'u 350bit', '100bit', '100bit', '100bit'], function() {
+          var txOpts = {
+            outputs: [{
+              toAddress: '18PzpUFkFZE8zKWUPvfykkTxmB9oMR8qP7',
+              amount: 200e2,
+            }],
+            feePerKb: 10e2,
+          };
+          server.createTx(txOpts, function(err, txp) {
+            should.not.exist(err);
+            should.exist(txp);
+            txp.inputs.length.should.equal(3);
+            txp.inputs[0].satoshis.should.equal(10000);
+
+            done();
+          });
+        });
+      });
+
+      it('should select smaller utxos if within fee constraints', function(done) {
+        helpers.stubUtxos(server, wallet, [1, '800bit', '800bit', '800bit'], function() {
+          var txOpts = {
+            outputs: [{
+              toAddress: '18PzpUFkFZE8zKWUPvfykkTxmB9oMR8qP7',
+              amount: 2000e2,
+            }],
+            feePerKb: 10e2,
+          };
+          server.createTx(txOpts, function(err, txp) {
+            should.not.exist(err);
+            should.exist(txp);
+            txp.inputs.length.should.equal(3);
+            _.all(txp.inputs, function(input) {
+              return input == 100e2;
+            });
+            done();
+          });
+        });
+      });
+      it('should select smallest big utxo if small utxos are insufficient', function(done) {
+        helpers.stubUtxos(server, wallet, [3, 1, 2, '100bit', '100bit', '100bit'], function() {
+          var txOpts = {
+            outputs: [{
+              toAddress: '18PzpUFkFZE8zKWUPvfykkTxmB9oMR8qP7',
+              amount: 300e2,
+            }],
+            feePerKb: 10e2,
+          };
+          server.createTx(txOpts, function(err, txp) {
+            should.not.exist(err);
+            should.exist(txp);
+            txp.inputs.length.should.equal(1);
+            txp.inputs[0].satoshis.should.equal(1e8);
+            done();
+          });
+        });
+      });
+      it('should account for fee when selecting smallest big utxo', function(done) {
+        // log.level = 'debug';
+        var _old = Defaults.UTXO_SELECTION_MAX_SINGLE_UTXO_FACTOR;
+        Defaults.UTXO_SELECTION_MAX_SINGLE_UTXO_FACTOR = 2;
+        // The 605 bits input cannot be selected even if it is > 2 * tx amount
+        // because it cannot cover for fee on its own.
+        helpers.stubUtxos(server, wallet, [1, '605bit', '100bit', '100bit', '100bit'], function() {
+          var txOpts = {
+            outputs: [{
+              toAddress: '18PzpUFkFZE8zKWUPvfykkTxmB9oMR8qP7',
+              amount: 300e2,
+            }],
+            feePerKb: 1200e2,
+          };
+          server.createTx(txOpts, function(err, txp) {
+            should.not.exist(err);
+            should.exist(txp);
+            txp.inputs.length.should.equal(1);
+            txp.inputs[0].satoshis.should.equal(1e8);
+            Defaults.UTXO_SELECTION_MAX_SINGLE_UTXO_FACTOR = _old;
+            done();
+          });
+        });
+      });
+      it('should select smallest big utxo if small utxos exceed maximum fee', function(done) {
+        helpers.stubUtxos(server, wallet, [3, 1, 2].concat(_.times(20, function() {
+          return '1000bit';
+        })), function() {
+          var txOpts = {
+            outputs: [{
+              toAddress: '18PzpUFkFZE8zKWUPvfykkTxmB9oMR8qP7',
+              amount: 12000e2,
+            }],
+            feePerKb: 20e2,
+          };
+          server.createTx(txOpts, function(err, txp) {
+            should.not.exist(err);
+            should.exist(txp);
+            txp.inputs.length.should.equal(1);
+            txp.inputs[0].satoshis.should.equal(1e8);
+
+            done();
+          });
+        });
+      });
+      it('should select smallest big utxo if small utxos are below accepted ratio of txp amount', function(done) {
+        helpers.stubUtxos(server, wallet, [9, 1, 1, 0.5, 0.2, 0.2, 0.2], function() {
+          var txOpts = {
+            outputs: [{
+              toAddress: '18PzpUFkFZE8zKWUPvfykkTxmB9oMR8qP7',
+              amount: 3e8,
+            }],
+            feePerKb: 10e2,
+          };
+          server.createTx(txOpts, function(err, txp) {
+            should.not.exist(err);
+            should.exist(txp);
+            txp.inputs.length.should.equal(1);
+            txp.inputs[0].satoshis.should.equal(9e8);
+            done();
+          });
+        });
+      });
+      it('should not fail with tx exceeded max size if there is at least 1 big input', function(done) {
+        var _old1 = Defaults.UTXO_SELECTION_MIN_TX_AMOUNT_VS_UTXO_FACTOR;
+        var _old2 = Defaults.MAX_TX_SIZE_IN_KB;
+        Defaults.UTXO_SELECTION_MIN_TX_AMOUNT_VS_UTXO_FACTOR = 0.0001;
+        Defaults.MAX_TX_SIZE_IN_KB = 3;
+
+        helpers.stubUtxos(server, wallet, [100].concat(_.range(1, 20, 0)), function() {
+          var txOpts = {
+            outputs: [{
+              toAddress: '18PzpUFkFZE8zKWUPvfykkTxmB9oMR8qP7',
+              amount: 15e8,
+            }],
+            feePerKb: 120e2,
+          };
+          server.createTx(txOpts, function(err, txp) {
+            should.not.exist(err);
+            should.exist(txp);
+            txp.inputs.length.should.equal(1);
+            txp.inputs[0].satoshis.should.equal(100e8);
+            Defaults.UTXO_SELECTION_MIN_TX_AMOUNT_VS_UTXO_FACTOR = _old1;
+            Defaults.MAX_TX_SIZE_IN_KB = _old2;
+            done();
+          });
+        });
+      });
+      it('should ignore utxos not contributing enough to cover increase in fee', function(done) {
+        helpers.stubUtxos(server, wallet, ['100bit', '100bit', '100bit'], function() {
+          var txOpts = {
+            outputs: [{
+              toAddress: '18PzpUFkFZE8zKWUPvfykkTxmB9oMR8qP7',
+              amount: 200e2,
+            }],
+            feePerKb: 80e2,
+          };
+          server.createTx(txOpts, function(err, txp) {
+            should.not.exist(err);
+            should.exist(txp);
+            txp.inputs.length.should.equal(3);
+            txOpts.feePerKb = 120e2;
+            server.createTx(txOpts, function(err, txp) {
+              should.exist(err);
+              should.not.exist(txp);
+              done();
+            });
+          });
+        });
+      });
+      it('should fail to select utxos if not enough to cover tx amount', function(done) {
+        helpers.stubUtxos(server, wallet, ['100bit', '100bit', '100bit'], function() {
+          var txOpts = {
+            outputs: [{
+              toAddress: '18PzpUFkFZE8zKWUPvfykkTxmB9oMR8qP7',
+              amount: 400e2,
+            }],
+            feePerKb: 10e2,
+          };
+          server.createTx(txOpts, function(err, txp) {
+            should.exist(err);
+            should.not.exist(txp);
+            err.code.should.equal('INSUFFICIENT_FUNDS');
+            done();
+          });
+        });
+      });
+      it('should fail to select utxos if not enough to cover fees', function(done) {
+        helpers.stubUtxos(server, wallet, ['100bit', '100bit', '100bit'], function() {
+          var txOpts = {
+            outputs: [{
+              toAddress: '18PzpUFkFZE8zKWUPvfykkTxmB9oMR8qP7',
+              amount: 299e2,
+            }],
+            feePerKb: 10e2,
+          };
+          server.createTx(txOpts, function(err, txp) {
+            should.exist(err);
+            should.not.exist(txp);
+            err.code.should.equal('INSUFFICIENT_FUNDS_FOR_FEE');
+            done();
+          });
+        });
+      });
+      it('should prefer a higher fee (breaking all limits) if inputs have 6+ confirmations', function(done) {
+        helpers.stubUtxos(server, wallet, ['2c 2000bit'].concat(_.times(20, function() {
+          return '100bit';
+        })), function() {
+          var txOpts = {
+            outputs: [{
+              toAddress: '18PzpUFkFZE8zKWUPvfykkTxmB9oMR8qP7',
+              amount: 1500e2,
+            }],
+            feePerKb: 10e2,
+          };
+          server.createTx(txOpts, function(err, txp) {
+            should.not.exist(err);
+            should.exist(txp);
+            _.all(txp.inputs, function(input) {
+              return input == 100e2;
+            });
+            done();
+          });
+        });
+      });
+      it('should select unconfirmed utxos if not enough confirmed utxos', function(done) {
+        // log.level = 'debug';
+        helpers.stubUtxos(server, wallet, ['u 1btc', '0.5btc'], function() {
+          var txOpts = {
+            outputs: [{
+              toAddress: '18PzpUFkFZE8zKWUPvfykkTxmB9oMR8qP7',
+              amount: 0.8e8,
+            }],
+            feePerKb: 100e2,
+          };
+          server.createTx(txOpts, function(err, txp) {
+            should.not.exist(err);
+            should.exist(txp);
+            txp.inputs.length.should.equal(1);
+            txp.inputs[0].satoshis.should.equal(1e8);
+            done();
+          });
+        });
+      });
+      it('should ignore utxos too small to pay for fee', function(done) {
+        helpers.stubUtxos(server, wallet, ['1c200bit', '200bit'].concat(_.times(20, function() {
+          return '1bit';
+        })), function() {
+          var txOpts = {
+            outputs: [{
+              toAddress: '18PzpUFkFZE8zKWUPvfykkTxmB9oMR8qP7',
+              amount: 200e2,
+            }],
+            feePerKb: 90e2,
+          };
+          server.createTx(txOpts, function(err, txp) {
+            should.not.exist(err);
+            should.exist(txp);
+            txp.inputs.length.should.equal(2);
+            done();
+          });
+        });
+      });
+      it('should use small utxos if fee is low', function(done) {
+        helpers.stubUtxos(server, wallet, [].concat(_.times(10, function() {
+          return '30bit';
+        })), function() {
+          var txOpts = {
+            outputs: [{
+              toAddress: '18PzpUFkFZE8zKWUPvfykkTxmB9oMR8qP7',
+              amount: 200e2,
+            }],
+            feePerKb: 10e2,
+          };
+          server.createTx(txOpts, function(err, txp) {
+            should.not.exist(err);
+            should.exist(txp);
+            txp.inputs.length.should.equal(8);
+            done();
+          });
+        });
+      });
+      it('should correct fee if resulting change would be below dust', function(done) {
+        helpers.stubUtxos(server, wallet, ['200bit', '500sat'], function() {
+          var txOpts = {
+            outputs: [{
+              toAddress: '18PzpUFkFZE8zKWUPvfykkTxmB9oMR8qP7',
+              amount: 200e2,
+            }],
+            feePerKb: 400,
+          };
+          server.createTx(txOpts, function(err, txp) {
+            should.not.exist(err);
+            (_.sum(txp.inputs, 'satoshis') - txp.outputs[0].amount - txp.fee).should.equal(0);
+            var changeOutput = txp.getBitcoreTx().getChangeOutput();
+            should.not.exist(changeOutput);
+            done();
+          });
+        });
+      });
+      it('should ignore small utxos if fee is higher', function(done) {
+        helpers.stubUtxos(server, wallet, [].concat(_.times(10, function() {
+          return '30bit';
+        })), function() {
+          var txOpts = {
+            outputs: [{
+              toAddress: '18PzpUFkFZE8zKWUPvfykkTxmB9oMR8qP7',
+              amount: 200e2,
+            }],
+            feePerKb: 50e2,
+          };
+          server.createTx(txOpts, function(err, txp) {
+            should.exist(err);
+            err.code.should.equal('INSUFFICIENT_FUNDS_FOR_FEE');
+            done();
+          });
+        });
+      });
+      it('should always select inputs as long as there are sufficient funds', function(done) {
+        helpers.stubUtxos(server, wallet, [80, '50bit', '50bit', '50bit', '50bit', '50bit'], function() {
+          var txOpts = {
+            outputs: [{
+              toAddress: '18PzpUFkFZE8zKWUPvfykkTxmB9oMR8qP7',
+              amount: 101e2,
+            }],
+            feePerKb: 100e2,
+          };
+          server.createTx(txOpts, function(err, txp) {
+            should.not.exist(err);
+            should.exist(txp);
+            done();
+          });
+        });
+      });
+    });
+  });
+
   describe('#rejectTx', function() {
     var server, wallet, txid;
 
@@ -5023,7 +5441,6 @@ describe('Wallet service', function() {
         done();
       });
     });
-
   });
 
   describe('#scan', function() {
@@ -5686,5 +6103,4 @@ describe('Wallet service', function() {
       });
     });
   });
-
 });

test/models/txproposal.js

@@ -56,7 +56,7 @@ describe('TxProposal', function() {
   describe('#getEstimatedSize', function() {
     it('should return estimated size in bytes', function() {
       var x = TxProposal.fromObj(aTXP());
-      x.getEstimatedSize().should.equal(407);
+      x.getEstimatedSize().should.equal(396);
     });
   });