From 75b3ecee491ea1962a0be043eaceec9e8348a676 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sun, 29 Nov 2015 23:19:13 +0900 Subject: [PATCH] Add coin chooser to try and minimize loss of privacy. --- gui/qt/main_window.py | 17 ++++- lib/__init__.py | 4 +- lib/coinchooser.py | 144 ++++++++++++++++++++++++++++++++++++------ lib/wallet.py | 31 +++++++-- 4 files changed, 167 insertions(+), 29 deletions(-) diff --git a/gui/qt/main_window.py b/gui/qt/main_window.py index d6f911600..bda09737a 100644 --- a/gui/qt/main_window.py +++ b/gui/qt/main_window.py @@ -43,7 +43,7 @@ from electrum.util import PrintError, NotEnoughFunds, StoreDict from electrum import Transaction from electrum import mnemonic from electrum import util, bitcoin, commands, Wallet -from electrum import SimpleConfig, Wallet, WalletStorage +from electrum import SimpleConfig, COIN_CHOOSERS, WalletStorage from electrum import Imported_Wallet from electrum import paymentrequest @@ -2593,6 +2593,21 @@ class ElectrumWindow(QMainWindow, PrintError): nz.valueChanged.connect(on_nz) gui_widgets.append((nz_label, nz)) + choosers = sorted(COIN_CHOOSERS.keys()) + chooser_name = self.wallet.coin_chooser_name(self.config) + msg = _('Choose coin (UTXO) selection method. The following are available:\n\n') + msg += '\n\n'.join(key + ": " + klass.__doc__ + for key, klass in COIN_CHOOSERS.items()) + chooser_label = HelpLabel(_('Coin selection') + ':', msg) + chooser_combo = QComboBox() + chooser_combo.addItems(choosers) + chooser_combo.setCurrentIndex(choosers.index(chooser_name)) + def on_chooser(x): + chooser_name = choosers[chooser_combo.currentIndex()] + self.config.set_key('coin_chooser', chooser_name) + chooser_combo.currentIndexChanged.connect(on_chooser) + tx_widgets.append((chooser_label, chooser_combo)) + msg = _('Fee per kilobyte of transaction.') + '\n' \ + _('If you enable dynamic fees, this parameter will be used as upper bound.') fee_label = HelpLabel(_('Transaction fee per kb') + ':', msg) diff --git a/lib/__init__.py b/lib/__init__.py index f174a041a..e2abad7ab 100644 --- a/lib/__init__.py +++ b/lib/__init__.py @@ -1,7 +1,7 @@ from version import ELECTRUM_VERSION from util import format_satoshis, print_msg, print_error, set_verbosity -from wallet import Synchronizer, WalletStorage -from wallet import Wallet, Imported_Wallet +from wallet import Synchronizer, WalletStorage, Wallet, Imported_Wallet +from coinchooser import COIN_CHOOSERS from network import Network, DEFAULT_SERVERS, DEFAULT_PORTS, pick_random_server from interface import Connection, Interface from simple_config import SimpleConfig, get_config, set_config diff --git a/lib/coinchooser.py b/lib/coinchooser.py index d42268248..32a5ab44a 100644 --- a/lib/coinchooser.py +++ b/lib/coinchooser.py @@ -17,14 +17,19 @@ # along with this program. If not, see . from collections import defaultdict, namedtuple +from random import shuffle -from util import NotEnoughFunds, PrintError, profiler +from bitcoin import COIN from transaction import Transaction +from util import NotEnoughFunds, PrintError, profiler Bucket = namedtuple('Bucket', ['desc', 'size', 'value', 'coins']) class CoinChooserBase(PrintError): + def keys(self, coins): + raise NotImplementedError + def bucketize_coins(self, coins): keys = self.keys(coins) buckets = defaultdict(list) @@ -39,55 +44,66 @@ class CoinChooserBase(PrintError): return map(make_Bucket, buckets.keys(), buckets.values()) + def penalty_func(self, tx): + def penalty(candidate): + return 0 + return penalty + + def add_change(self, tx, change_addrs, fee_estimator, dust_threshold): + # How much is left if we add 1 change output? + change_amount = tx.get_fee() - fee_estimator(1) + + # If change is above dust threshold after accounting for the + # size of the change output, add it to the transaction. + if change_amount > dust_threshold: + tx.outputs.append(('address', change_addrs[0], change_amount)) + self.print_error('change', change_amount) + elif change_amount: + self.print_error('not keeping dust', change_amount) + def make_tx(self, coins, outputs, change_addrs, fee_estimator, dust_threshold): '''Select unspent coins to spend to pay outputs. If the change is greater than dust_threshold (after adding the change output to the transaction) it is kept, otherwise none is sent and it is added to the transaction fee.''' - output_total = sum(map(lambda x: x[2], outputs)) + # Copy the ouputs so when adding change we don't modify "outputs" + tx = Transaction.from_io([], outputs[:]) # Size of the transaction with no inputs and no change - tx = Transaction.from_io([], outputs) base_size = tx.estimated_size() # Returns fee given input size fee = lambda input_size: fee_estimator(base_size + input_size) # Collect the coins into buckets, choose a subset of the buckets buckets = self.bucketize_coins(coins) - buckets = self.choose_buckets(buckets, output_total, fee) + buckets = self.choose_buckets(buckets, tx.output_value(), fee, + self.penalty_func(tx)) tx.inputs = [coin for b in buckets for coin in b.coins] - input_total = sum(bucket.value for bucket in buckets) tx_size = base_size + sum(bucket.size for bucket in buckets) - # If change is above dust threshold after accounting for the - # size of the change output, add it to the transaction. - # Pay to bitcoin address serializes as 34 bytes - change_size = 34 - fee = fee_estimator(tx_size + change_size) - change_amount = input_total - (output_total + fee) - if change_amount > dust_threshold: - tx.outputs.append(('address', change_addrs[0], change_amount)) - self.print_error('change', change_amount) - elif change_amount: - self.print_error('not keeping dust', change_amount) + # This takes a count of change outputs and returns a tx fee; + # each pay-to-bitcoin-address output serializes as 34 bytes + fee = lambda count: fee_estimator(tx_size + count * 34) + self.add_change(tx, change_addrs, fee, dust_threshold) self.print_error("using %d inputs" % len(tx.inputs)) self.print_error("using buckets:", [bucket.desc for bucket in buckets]) return tx -class CoinChooser(CoinChooserBase): - '''The original electrum algorithm. Chooses coins starting with the - oldest that are sufficient to cover the spent amount, and then - removes any not needed starting with the smallest in value.''' +class CoinChooserClassic(CoinChooserBase): + ''' + The classic electrum algorithm. Chooses coins starting with + the oldest that are sufficient to cover the spent amount, and + then removes any unneeded starting with the smallest in value.''' def keys(self, coins): return [coin['prevout_hash'] + ':' + str(coin['prevout_n']) for coin in coins] - def choose_buckets(self, buckets, spent_amount, fee): + def choose_buckets(self, buckets, spent_amount, fee, penalty_func): '''Spend the oldest buckets first.''' # Unconfirmed coins are young, not old adj_height = lambda height: 99999999 if height == 0 else height @@ -113,3 +129,89 @@ class CoinChooser(CoinChooserBase): dropped.append(bucket) return [bucket for bucket in selected if bucket not in dropped] + +class CoinChooserRandom(CoinChooserBase): + + def bucket_candidates(self, buckets, sufficient_funds): + '''Returns a list of bucket sets.''' + candidates = set() + + # Add all singletons + for n, bucket in enumerate(buckets): + if sufficient_funds([bucket]): + candidates.add((n, )) + + # And now some random ones + attempts = min(100, (len(buckets) - 1) * 10 + 1) + permutation = range(len(buckets)) + for i in range(attempts): + # Get a random permutation of the buckets, and + # incrementally combine buckets until sufficient + shuffle(permutation) + bkts = [] + for count, index in enumerate(permutation): + bkts.append(buckets[index]) + if sufficient_funds(bkts): + candidates.add(tuple(sorted(permutation[:count + 1]))) + break + else: + raise NotEnoughFunds() + + return [[buckets[n] for n in candidate] for candidate in candidates] + + def choose_buckets(self, buckets, spent_amount, fee, penalty_func): + + def sufficient(buckets): + '''Given a set of buckets, return True if it has enough + value to pay for the transaction''' + total_input = sum(bucket.value for bucket in buckets) + total_size = sum(bucket.size for bucket in buckets) + return total_input >= spent_amount + fee(total_size) + + candidates = self.bucket_candidates(buckets, sufficient) + penalties = [penalty_func(cand) for cand in candidates] + winner = candidates[penalties.index(min(penalties))] + self.print_error("Bucket sets:", len(buckets)) + self.print_error("Winning penalty:", min(penalties)) + return winner + +class CoinChooserPrivacy(CoinChooserRandom): + ''' + Attempts to better preserve user privacy. First, if any coin is + spent from a user address, all coins are. Compared to spending + from other addresses to make up an amount, this reduces + information leakage about sender holdings. It also helps to + reduce blockchain UTXO bloat, and reduce future privacy loss + that would come from reusing that address' remaining UTXOs. + Second, it penalizes change that is quite different to the sent + amount. Third, it penalizes change that is too big.''' + + def keys(self, coins): + return [coin['address'] for coin in coins] + + def penalty_func(self, buckets, tx): + '''Returns a penalty for a candidate set of buckets.''' + raise NotImplementedError + + def penalty_func(self, tx): + min_change = min(o[2] for o in tx.outputs) * 0.75 + max_change = max(o[2] for o in tx.outputs) * 1.33 + spent_amount = sum(o[2] for o in tx.outputs) + + def penalty(buckets): + badness = len(buckets) - 1 + total_input = sum(bucket.value for bucket in buckets) + change = float(total_input - spent_amount) + # Penalize change not roughly in output range + if change < min_change: + badness += (min_change - change) / (min_change + 10000) + elif change > max_change: + badness += (change - max_change) / (max_change + 10000) + # Penalize large change; 5 BTC excess ~= using 1 more input + badness += change / (COIN * 5) + return badness + + return penalty + +COIN_CHOOSERS = {'Classic': CoinChooserClassic, + 'Privacy': CoinChooserPrivacy} diff --git a/lib/wallet.py b/lib/wallet.py index ed60220e6..4bda5fabc 100644 --- a/lib/wallet.py +++ b/lib/wallet.py @@ -35,7 +35,7 @@ from version import * from transaction import Transaction from plugins import run_hook import bitcoin -from coinchooser import CoinChooser +from coinchooser import COIN_CHOOSERS from synchronizer import Synchronizer from verifier import SPV from mnemonic import Mnemonic @@ -45,7 +45,6 @@ import paymentrequest # internal ID for imported account IMPORTED_ACCOUNT = '/x' - class WalletStorage(PrintError): def __init__(self, path): @@ -156,7 +155,6 @@ class Abstract_Wallet(PrintError): self.network = None self.electrum_version = ELECTRUM_VERSION self.gap_limit_for_change = 6 # constant - self.coin_chooser = CoinChooser() # saved fields self.seed_version = storage.get('seed_version', NEW_SEED_VERSION) self.use_change = storage.get('use_change',True) @@ -195,6 +193,7 @@ class Abstract_Wallet(PrintError): self.lock = threading.Lock() self.transaction_lock = threading.Lock() self.tx_event = threading.Event() + self.tx_cache = (None, None, None, None, None) self.check_history() @@ -205,6 +204,7 @@ class Abstract_Wallet(PrintError): def diagnostic_name(self): return self.basename() + @profiler def load_transactions(self): self.txi = self.storage.get('txi', {}) @@ -896,6 +896,16 @@ class Abstract_Wallet(PrintError): # this method can be overloaded return tx.get_fee() + def coin_chooser_name(self, config): + kind = config.get('coin_chooser') + if not kind in COIN_CHOOSERS: + kind = 'Classic' + return kind + + def coin_chooser(self, config): + klass = COIN_CHOOSERS[self.coin_chooser_name(config)] + return klass() + def make_unsigned_transaction(self, coins, outputs, config, fixed_fee=None, change_addr=None): # check outputs for type, data, value in outputs: @@ -934,13 +944,24 @@ class Abstract_Wallet(PrintError): # Change <= dust threshold is added to the tx fee dust_threshold = 182 * 3 * MIN_RELAY_TX_FEE / 1000 + # Check cache to see if we just calculated this. If prior + # calculated a fee and this fixes it to the same, return same + # answer, to avoid random coin selection changing the answer + if self.tx_cache[:4] == (outputs, coins, change_addrs, dust_threshold): + tx = self.tx_cache[4] + if tx.get_fee() == fee_estimator(tx.estimated_size()): + return tx + # Let the coin chooser select the coins to spend - tx = self.coin_chooser.make_tx(coins, outputs, change_addrs, - fee_estimator, dust_threshold) + coin_chooser = self.coin_chooser(config) + tx = coin_chooser.make_tx(coins, outputs, change_addrs, + fee_estimator, dust_threshold) # Sort the inputs and outputs deterministically tx.BIP_LI01_sort() + self.tx_cache = (outputs, coins, change_addrs, dust_threshold, tx) + run_hook('make_unsigned_transaction', self, tx) return tx