Add coin chooser to try and minimize loss of privacy.

gui/qt/main_window.py

@@ -43,7 +43,7 @@ from electrum.util import PrintError, NotEnoughFunds, StoreDict
 from electrum import Transaction
 from electrum import mnemonic
 from electrum import util, bitcoin, commands, Wallet
-from electrum import SimpleConfig, Wallet, WalletStorage
+from electrum import SimpleConfig, COIN_CHOOSERS, WalletStorage
 from electrum import Imported_Wallet
 from electrum import paymentrequest
 
@@ -2593,6 +2593,21 @@ class ElectrumWindow(QMainWindow, PrintError):
         nz.valueChanged.connect(on_nz)
         gui_widgets.append((nz_label, nz))
 
+        choosers = sorted(COIN_CHOOSERS.keys())
+        chooser_name = self.wallet.coin_chooser_name(self.config)
+        msg = _('Choose coin (UTXO) selection method.  The following are available:\n\n')
+        msg += '\n\n'.join(key + ": " + klass.__doc__
+                         for key, klass in COIN_CHOOSERS.items())
+        chooser_label = HelpLabel(_('Coin selection') + ':', msg)
+        chooser_combo = QComboBox()
+        chooser_combo.addItems(choosers)
+        chooser_combo.setCurrentIndex(choosers.index(chooser_name))
+        def on_chooser(x):
+            chooser_name = choosers[chooser_combo.currentIndex()]
+            self.config.set_key('coin_chooser', chooser_name)
+        chooser_combo.currentIndexChanged.connect(on_chooser)
+        tx_widgets.append((chooser_label, chooser_combo))
+
         msg = _('Fee per kilobyte of transaction.') + '\n' \
               + _('If you enable dynamic fees, this parameter will be used as upper bound.')
         fee_label = HelpLabel(_('Transaction fee per kb') + ':', msg)

lib/__init__.py

@@ -1,7 +1,7 @@
 from version import ELECTRUM_VERSION
 from util import format_satoshis, print_msg, print_error, set_verbosity
-from wallet import Synchronizer, WalletStorage
+from wallet import Synchronizer, WalletStorage, Wallet, Imported_Wallet
-from wallet import Wallet, Imported_Wallet
+from coinchooser import COIN_CHOOSERS
 from network import Network, DEFAULT_SERVERS, DEFAULT_PORTS, pick_random_server
 from interface import Connection, Interface
 from simple_config import SimpleConfig, get_config, set_config

lib/coinchooser.py

@@ -17,14 +17,19 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 
 from collections import defaultdict, namedtuple
+from random import shuffle
 
-from util import NotEnoughFunds, PrintError, profiler
+from bitcoin import COIN
 from transaction import Transaction
+from util import NotEnoughFunds, PrintError, profiler
 
 Bucket = namedtuple('Bucket', ['desc', 'size', 'value', 'coins'])
 
 class CoinChooserBase(PrintError):
 
+    def keys(self, coins):
+        raise NotImplementedError
+
     def bucketize_coins(self, coins):
         keys = self.keys(coins)
         buckets = defaultdict(list)
@@ -39,55 +44,66 @@ class CoinChooserBase(PrintError):
 
         return map(make_Bucket, buckets.keys(), buckets.values())
 
+    def penalty_func(self, tx):
+        def penalty(candidate):
+            return 0
+        return penalty
+
+    def add_change(self, tx, change_addrs, fee_estimator, dust_threshold):
+        # How much is left if we add 1 change output?
+        change_amount = tx.get_fee() - fee_estimator(1)
+
+        # If change is above dust threshold after accounting for the
+        # size of the change output, add it to the transaction.
+        if change_amount > dust_threshold:
+            tx.outputs.append(('address', change_addrs[0], change_amount))
+            self.print_error('change', change_amount)
+        elif change_amount:
+            self.print_error('not keeping dust', change_amount)
+
     def make_tx(self, coins, outputs, change_addrs, fee_estimator,
                 dust_threshold):
         '''Select unspent coins to spend to pay outputs.  If the change is
         greater than dust_threshold (after adding the change output to
         the transaction) it is kept, otherwise none is sent and it is
         added to the transaction fee.'''
-        output_total = sum(map(lambda x: x[2], outputs))
 
+        # Copy the ouputs so when adding change we don't modify "outputs"
+        tx = Transaction.from_io([], outputs[:])
         # Size of the transaction with no inputs and no change
-        tx = Transaction.from_io([], outputs)
         base_size = tx.estimated_size()
         # Returns fee given input size
         fee = lambda input_size: fee_estimator(base_size + input_size)
 
         # Collect the coins into buckets, choose a subset of the buckets
         buckets = self.bucketize_coins(coins)
-        buckets = self.choose_buckets(buckets, output_total, fee)
+        buckets = self.choose_buckets(buckets, tx.output_value(), fee,
+                                      self.penalty_func(tx))
 
         tx.inputs = [coin for b in buckets for coin in b.coins]
-        input_total = sum(bucket.value for bucket in buckets)
         tx_size = base_size + sum(bucket.size for bucket in buckets)
 
-        # If change is above dust threshold after accounting for the
+        # This takes a count of change outputs and returns a tx fee;
-        # size of the change output, add it to the transaction.
+        # each pay-to-bitcoin-address output serializes as 34 bytes
-        # Pay to bitcoin address serializes as 34 bytes
+        fee = lambda count: fee_estimator(tx_size + count * 34)
-        change_size = 34
+        self.add_change(tx, change_addrs, fee, dust_threshold)
-        fee = fee_estimator(tx_size + change_size)
-        change_amount = input_total - (output_total + fee)
-        if change_amount > dust_threshold:
-            tx.outputs.append(('address', change_addrs[0], change_amount))
-            self.print_error('change', change_amount)
-        elif change_amount:
-            self.print_error('not keeping dust', change_amount)
 
         self.print_error("using %d inputs" % len(tx.inputs))
         self.print_error("using buckets:", [bucket.desc for bucket in buckets])
 
         return tx
 
-class CoinChooser(CoinChooserBase):
+class CoinChooserClassic(CoinChooserBase):
-    '''The original electrum algorithm.  Chooses coins starting with the
+    '''
-    oldest that are sufficient to cover the spent amount, and then
+    The classic electrum algorithm.  Chooses coins starting with
-    removes any not needed starting with the smallest in value.'''
+    the oldest that are sufficient to cover the spent amount, and
+    then removes any unneeded starting with the smallest in value.'''
 
     def keys(self, coins):
         return [coin['prevout_hash'] + ':' + str(coin['prevout_n'])
                 for coin in coins]
 
-    def choose_buckets(self, buckets, spent_amount, fee):
+    def choose_buckets(self, buckets, spent_amount, fee, penalty_func):
         '''Spend the oldest buckets first.'''
         # Unconfirmed coins are young, not old
         adj_height = lambda height: 99999999 if height == 0 else height
@@ -113,3 +129,89 @@ class CoinChooser(CoinChooserBase):
                 dropped.append(bucket)
 
         return [bucket for bucket in selected if bucket not in dropped]
+
+class CoinChooserRandom(CoinChooserBase):
+
+    def bucket_candidates(self, buckets, sufficient_funds):
+        '''Returns a list of bucket sets.'''
+        candidates = set()
+
+        # Add all singletons
+        for n, bucket in enumerate(buckets):
+            if sufficient_funds([bucket]):
+                candidates.add((n, ))
+
+        # And now some random ones
+        attempts = min(100, (len(buckets) - 1) * 10 + 1)
+        permutation = range(len(buckets))
+        for i in range(attempts):
+            # Get a random permutation of the buckets, and
+            # incrementally combine buckets until sufficient
+            shuffle(permutation)
+            bkts = []
+            for count, index in enumerate(permutation):
+                bkts.append(buckets[index])
+                if sufficient_funds(bkts):
+                    candidates.add(tuple(sorted(permutation[:count + 1])))
+                    break
+            else:
+                raise NotEnoughFunds()
+
+        return [[buckets[n] for n in candidate] for candidate in candidates]
+
+    def choose_buckets(self, buckets, spent_amount, fee, penalty_func):
+
+        def sufficient(buckets):
+            '''Given a set of buckets, return True if it has enough
+            value to pay for the transaction'''
+            total_input = sum(bucket.value for bucket in buckets)
+            total_size = sum(bucket.size for bucket in buckets)
+            return total_input >= spent_amount + fee(total_size)
+
+        candidates = self.bucket_candidates(buckets, sufficient)
+        penalties = [penalty_func(cand) for cand in candidates]
+        winner = candidates[penalties.index(min(penalties))]
+        self.print_error("Bucket sets:", len(buckets))
+        self.print_error("Winning penalty:", min(penalties))
+        return winner
+
+class CoinChooserPrivacy(CoinChooserRandom):
+    '''
+    Attempts to better preserve user privacy.  First, if any coin is
+    spent from a user address, all coins are.  Compared to spending
+    from other addresses to make up an amount, this reduces
+    information leakage about sender holdings.  It also helps to
+    reduce blockchain UTXO bloat, and reduce future privacy loss
+    that would come from reusing that address' remaining UTXOs.
+    Second, it penalizes change that is quite different to the sent
+    amount.  Third, it penalizes change that is too big.'''
+
+    def keys(self, coins):
+        return [coin['address'] for coin in coins]
+
+    def penalty_func(self, buckets, tx):
+        '''Returns a penalty for a candidate set of buckets.'''
+        raise NotImplementedError
+
+    def penalty_func(self, tx):
+        min_change = min(o[2] for o in tx.outputs) * 0.75
+        max_change = max(o[2] for o in tx.outputs) * 1.33
+        spent_amount = sum(o[2] for o in tx.outputs)
+
+        def penalty(buckets):
+            badness = len(buckets) - 1
+            total_input = sum(bucket.value for bucket in buckets)
+            change = float(total_input - spent_amount)
+            # Penalize change not roughly in output range
+            if change < min_change:
+                badness += (min_change - change) / (min_change + 10000)
+            elif change > max_change:
+                badness += (change - max_change) / (max_change + 10000)
+                # Penalize large change; 5 BTC excess ~= using 1 more input
+                badness += change / (COIN * 5)
+            return badness
+
+        return penalty
+
+COIN_CHOOSERS = {'Classic': CoinChooserClassic,
+                 'Privacy': CoinChooserPrivacy}

lib/wallet.py

@@ -35,7 +35,7 @@ from version import *
 from transaction import Transaction
 from plugins import run_hook
 import bitcoin
-from coinchooser import CoinChooser
+from coinchooser import COIN_CHOOSERS
 from synchronizer import Synchronizer
 from verifier import SPV
 from mnemonic import Mnemonic
@@ -45,7 +45,6 @@ import paymentrequest
 # internal ID for imported account
 IMPORTED_ACCOUNT = '/x'
 
-
 class WalletStorage(PrintError):
 
     def __init__(self, path):
@@ -156,7 +155,6 @@ class Abstract_Wallet(PrintError):
         self.network = None
         self.electrum_version = ELECTRUM_VERSION
         self.gap_limit_for_change = 6 # constant
-        self.coin_chooser = CoinChooser()
         # saved fields
         self.seed_version          = storage.get('seed_version', NEW_SEED_VERSION)
         self.use_change            = storage.get('use_change',True)
@@ -195,6 +193,7 @@ class Abstract_Wallet(PrintError):
         self.lock = threading.Lock()
         self.transaction_lock = threading.Lock()
         self.tx_event = threading.Event()
+        self.tx_cache = (None, None, None, None, None)
 
         self.check_history()
 
@@ -205,6 +204,7 @@ class Abstract_Wallet(PrintError):
     def diagnostic_name(self):
         return self.basename()
 
+
     @profiler
     def load_transactions(self):
         self.txi = self.storage.get('txi', {})
@@ -896,6 +896,16 @@ class Abstract_Wallet(PrintError):
         # this method can be overloaded
         return tx.get_fee()
 
+    def coin_chooser_name(self, config):
+        kind = config.get('coin_chooser')
+        if not kind in COIN_CHOOSERS:
+            kind = 'Classic'
+        return kind
+
+    def coin_chooser(self, config):
+        klass = COIN_CHOOSERS[self.coin_chooser_name(config)]
+        return klass()
+
     def make_unsigned_transaction(self, coins, outputs, config, fixed_fee=None, change_addr=None):
         # check outputs
         for type, data, value in outputs:
@@ -934,13 +944,24 @@ class Abstract_Wallet(PrintError):
         # Change <= dust threshold is added to the tx fee
         dust_threshold = 182 * 3 * MIN_RELAY_TX_FEE / 1000
 
+        # Check cache to see if we just calculated this.  If prior
+        # calculated a fee and this fixes it to the same, return same
+        # answer, to avoid random coin selection changing the answer
+        if self.tx_cache[:4] == (outputs, coins, change_addrs, dust_threshold):
+            tx = self.tx_cache[4]
+            if tx.get_fee() == fee_estimator(tx.estimated_size()):
+                return tx
+
         # Let the coin chooser select the coins to spend
-        tx = self.coin_chooser.make_tx(coins, outputs, change_addrs,
+        coin_chooser = self.coin_chooser(config)
-                                       fee_estimator, dust_threshold)
+        tx = coin_chooser.make_tx(coins, outputs, change_addrs,
+                                  fee_estimator, dust_threshold)
 
         # Sort the inputs and outputs deterministically
         tx.BIP_LI01_sort()
 
+        self.tx_cache = (outputs, coins, change_addrs, dust_threshold, tx)
+
         run_hook('make_unsigned_transaction', self, tx)
         return tx