Track tx size directly; calculate fees from that

This has several advantages.  Fee calculation is now very fast,
as we don't need to keep reserializing the tx.  Another is that
we can reason about the fees after adding a change output without
having to add it, recalculate the tx fee, and remove it again.

lib/coinchooser.py

@@ -33,60 +33,42 @@ class CoinChooser(PrintError):
         amount = sum(map(lambda x: x[2], outputs))
         total = 0
         tx = Transaction.from_io([], outputs)
-        fee = fee_estimator(tx)
+
+        # Size of the transaction with no inputs and no change
+        base_size = tx.estimated_size()
+        # Pay to bitcoin address serializes as 34 bytes
+        change_size = 34
+        # Size of each serialized coin
+        for coin in coins:
+            coin['size'] = Transaction.estimated_input_size(coin)
+
+        size = base_size
         # add inputs, sorted by age
         for item in coins:
             v = item.get('value')
             total += v
+            size += item['size']
             tx.add_input(item)
-            # no need to estimate fee until we have reached desired amount
+            if total >= amount + fee_estimator(size):
-            if total < amount + fee:
-                continue
-            fee = fee_estimator(tx)
-            if total >= amount + fee:
                 break
         else:
             raise NotEnoughFunds()
 
         # remove unneeded inputs.
-        removed = False
-        for item in sorted(tx.inputs, key=itemgetter('value')):
-            v = item.get('value')
-            if total - v >= amount + fee:
-                tx.inputs.remove(item)
-                total -= v
-                removed = True
-                continue
-            else:
-                break
-        if removed:
-            fee = fee_estimator(tx)
         for item in sorted(tx.inputs, key=itemgetter('value')):
             v = item.get('value')
-                if total - v >= amount + fee:
+            if total - v >= amount + fee_estimator(size - item['size']):
                 tx.inputs.remove(item)
                 total -= v
-                    fee = fee_estimator(tx)
+                size -= item['size']
-                    continue
-                break
         self.print_error("using %d inputs" % len(tx.inputs))
 
-        # if change is above dust threshold, add a change output.
+        # If change is above dust threshold after accounting for the
-        change_addr = change_addrs[0]
+        # size of the change output, add it to the transaction.
-        change_amount = total - (amount + fee)
+        change_amount = total - (amount + fee_estimator(size + change_size))
-
-        # See if change would still be greater than dust after adding
-        # the change to the transaction
-        if change_amount > dust_threshold:
-            tx.outputs.append(('address', change_addr, change_amount))
-            fee = fee_estimator(tx)
-            # remove change output
-            tx.outputs.pop()
-            change_amount = total - (amount + fee)
-
-        # If change is still above dust threshold, keep the change.
         if change_amount > dust_threshold:
-            tx.outputs.append(('address', change_addr, change_amount))
+            tx.outputs.append(('address', change_addrs[0], change_amount))
+            size += change_size
             self.print_error('change', change_amount)
         elif change_amount:
             self.print_error('not keeping dust', change_amount)

lib/transaction.py

@@ -595,6 +595,7 @@ class Transaction:
             raise
         return script
 
+    @classmethod
     def input_script(self, txin, i, for_sig):
         # for_sig:
         #   -1   : do not sign, estimate length
@@ -641,6 +642,18 @@ class Transaction:
 
         return script
 
+    @classmethod
+    def serialize_input(self, txin, i, for_sig):
+        # Prev hash and index
+        s = txin['prevout_hash'].decode('hex')[::-1].encode('hex')
+        s = int_to_hex(txin['prevout_n'], 4)
+        # Script length, script, sequence
+        script = self.input_script(txin, i, for_sig)
+        s += var_int(len(script) / 2)
+        s += script
+        s += "ffffffff"
+        return s
+
     def BIP_LI01_sort(self):
         # See https://github.com/kristovatlas/rfc/blob/master/bips/bip-li01.mediawiki
         self.inputs.sort(key = lambda i: (i['prevout_hash'], i['prevout_n']))
@@ -652,12 +665,7 @@ class Transaction:
         s  = int_to_hex(1,4)                                         # version
         s += var_int( len(inputs) )                                  # number of inputs
         for i, txin in enumerate(inputs):
-            s += txin['prevout_hash'].decode('hex')[::-1].encode('hex')   # prev hash
+            s += self.serialize_input(txin, i, for_sig)
-            s += int_to_hex(txin['prevout_n'], 4)                         # prev index
-            script = self.input_script(txin, i, for_sig)
-            s += var_int( len(script)/2 )                            # script length
-            s += script
-            s += "ffffffff"                                          # sequence
         s += var_int( len(outputs) )                                 # number of outputs
         for output in outputs:
             output_type, addr, amount = output
@@ -690,7 +698,7 @@ class Transaction:
         return self.input_value() - self.output_value()
 
     @classmethod
-    def estimated_fee_for_size(self, fee_per_kb, size):
+    def fee_for_size(self, fee_per_kb, size):
         '''Given a fee per kB in satoshis, and a tx size in bytes,
         returns the transaction fee.'''
         fee = int(fee_per_kb * size / 1000.)
@@ -699,11 +707,18 @@ class Transaction:
         return fee
 
     @profiler
+    def estimated_size(self):
+        '''Return an estimated tx size in bytes.'''
+        return len(self.serialize(-1)) / 2  # ASCII hex string
+
+    @classmethod
+    def estimated_input_size(self, txin):
+        '''Return an estimated of serialized input size in bytes.'''
+        return len(self.serialize_input(txin, -1, -1)) / 2
+
     def estimated_fee(self, fee_per_kb):
         '''Return an estimated fee given a fee per kB in satoshis.'''
-        # Remember self.serialize returns an ASCII hex string
+        return self.fee_for_size(fee_per_kb, self.estimated_size())
-        size = len(self.serialize(-1)) / 2
-        return self.estimated_fee_for_size(fee_per_kb, size)
 
     def signature_count(self):
         r = 0

lib/wallet.py

@@ -24,6 +24,7 @@ import random
 import time
 import json
 import copy
+from functools import partial
 
 from util import PrintError, profiler
 
@@ -929,12 +930,10 @@ class Abstract_Wallet(PrintError):
 
         # Fee estimator
         if fixed_fee is None:
-            fee_per_kb = self.fee_per_kb(config)
+            fee_estimator = partial(Transaction.fee_for_size,
-            def fee_estimator(tx):
+                                    self.fee_per_kb(config))
-                return tx.estimated_fee(fee_per_kb)
         else:
-            def fee_estimator(tx):
+            fee_estimator = lambda size: fixed_fee
-                return fixed_fee
 
         # Change <= dust threshold is added to the tx fee
         dust_threshold = 182 * 3 * MIN_RELAY_TX_FEE / 1000