new protocol: the server sends serialized tx, deserialize it in the client

lib/deserialize.py

@@ -0,0 +1,319 @@
+# this code comes from ABE. it can probably be simplified
+#
+#
+
+from bitcoin import public_key_to_bc_address, hash_160_to_bc_address, hash_encode
+#import socket
+import time
+import struct
+
+#
+# Workalike python implementation of Bitcoin's CDataStream class.
+#
+import struct
+import StringIO
+import mmap
+
+class SerializationError(Exception):
+  """ Thrown when there's a problem deserializing or serializing """
+
+class BCDataStream(object):
+  def __init__(self):
+    self.input = None
+    self.read_cursor = 0
+
+  def clear(self):
+    self.input = None
+    self.read_cursor = 0
+
+  def write(self, bytes):  # Initialize with string of bytes
+    if self.input is None:
+      self.input = bytes
+    else:
+      self.input += bytes
+
+  def map_file(self, file, start):  # Initialize with bytes from file
+    self.input = mmap.mmap(file.fileno(), 0, access=mmap.ACCESS_READ)
+    self.read_cursor = start
+  def seek_file(self, position):
+    self.read_cursor = position
+  def close_file(self):
+    self.input.close()
+
+  def read_string(self):
+    # Strings are encoded depending on length:
+    # 0 to 252 :  1-byte-length followed by bytes (if any)
+    # 253 to 65,535 : byte'253' 2-byte-length followed by bytes
+    # 65,536 to 4,294,967,295 : byte '254' 4-byte-length followed by bytes
+    # ... and the Bitcoin client is coded to understand:
+    # greater than 4,294,967,295 : byte '255' 8-byte-length followed by bytes of string
+    # ... but I don't think it actually handles any strings that big.
+    if self.input is None:
+      raise SerializationError("call write(bytes) before trying to deserialize")
+
+    try:
+      length = self.read_compact_size()
+    except IndexError:
+      raise SerializationError("attempt to read past end of buffer")
+
+    return self.read_bytes(length)
+
+  def write_string(self, string):
+    # Length-encoded as with read-string
+    self.write_compact_size(len(string))
+    self.write(string)
+
+  def read_bytes(self, length):
+    try:
+      result = self.input[self.read_cursor:self.read_cursor+length]
+      self.read_cursor += length
+      return result
+    except IndexError:
+      raise SerializationError("attempt to read past end of buffer")
+
+    return ''
+
+  def read_boolean(self): return self.read_bytes(1)[0] != chr(0)
+  def read_int16(self): return self._read_num('<h')
+  def read_uint16(self): return self._read_num('<H')
+  def read_int32(self): return self._read_num('<i')
+  def read_uint32(self): return self._read_num('<I')
+  def read_int64(self): return self._read_num('<q')
+  def read_uint64(self): return self._read_num('<Q')
+
+  def write_boolean(self, val): return self.write(chr(1) if val else chr(0))
+  def write_int16(self, val): return self._write_num('<h', val)
+  def write_uint16(self, val): return self._write_num('<H', val)
+  def write_int32(self, val): return self._write_num('<i', val)
+  def write_uint32(self, val): return self._write_num('<I', val)
+  def write_int64(self, val): return self._write_num('<q', val)
+  def write_uint64(self, val): return self._write_num('<Q', val)
+
+  def read_compact_size(self):
+    size = ord(self.input[self.read_cursor])
+    self.read_cursor += 1
+    if size == 253:
+      size = self._read_num('<H')
+    elif size == 254:
+      size = self._read_num('<I')
+    elif size == 255:
+      size = self._read_num('<Q')
+    return size
+
+  def write_compact_size(self, size):
+    if size < 0:
+      raise SerializationError("attempt to write size < 0")
+    elif size < 253:
+       self.write(chr(size))
+    elif size < 2**16:
+      self.write('\xfd')
+      self._write_num('<H', size)
+    elif size < 2**32:
+      self.write('\xfe')
+      self._write_num('<I', size)
+    elif size < 2**64:
+      self.write('\xff')
+      self._write_num('<Q', size)
+
+  def _read_num(self, format):
+    (i,) = struct.unpack_from(format, self.input, self.read_cursor)
+    self.read_cursor += struct.calcsize(format)
+    return i
+
+  def _write_num(self, format, num):
+    s = struct.pack(format, num)
+    self.write(s)
+
+#
+# enum-like type
+# From the Python Cookbook, downloaded from http://code.activestate.com/recipes/67107/
+#
+import types, string, exceptions
+
+class EnumException(exceptions.Exception):
+    pass
+
+class Enumeration:
+    def __init__(self, name, enumList):
+        self.__doc__ = name
+        lookup = { }
+        reverseLookup = { }
+        i = 0
+        uniqueNames = [ ]
+        uniqueValues = [ ]
+        for x in enumList:
+            if type(x) == types.TupleType:
+                x, i = x
+            if type(x) != types.StringType:
+                raise EnumException, "enum name is not a string: " + x
+            if type(i) != types.IntType:
+                raise EnumException, "enum value is not an integer: " + i
+            if x in uniqueNames:
+                raise EnumException, "enum name is not unique: " + x
+            if i in uniqueValues:
+                raise EnumException, "enum value is not unique for " + x
+            uniqueNames.append(x)
+            uniqueValues.append(i)
+            lookup[x] = i
+            reverseLookup[i] = x
+            i = i + 1
+        self.lookup = lookup
+        self.reverseLookup = reverseLookup
+    def __getattr__(self, attr):
+        if not self.lookup.has_key(attr):
+            raise AttributeError
+        return self.lookup[attr]
+    def whatis(self, value):
+        return self.reverseLookup[value]
+
+
+# This function comes from bitcointools, bct-LICENSE.txt.
+def long_hex(bytes):
+    return bytes.encode('hex_codec')
+
+# This function comes from bitcointools, bct-LICENSE.txt.
+def short_hex(bytes):
+    t = bytes.encode('hex_codec')
+    if len(t) < 11:
+        return t
+    return t[0:4]+"..."+t[-4:]
+
+
+
+def parse_TxIn(vds):
+  d = {}
+  d['prevout_hash'] = hash_encode(vds.read_bytes(32))
+  d['prevout_n'] = vds.read_uint32()
+  scriptSig = vds.read_bytes(vds.read_compact_size())
+  d['sequence'] = vds.read_uint32()
+  d['address'] = extract_public_key(scriptSig)
+  #d['script'] = decode_script(scriptSig)
+  return d
+
+
+def parse_TxOut(vds, i):
+  d = {}
+  d['value'] = vds.read_int64()
+  scriptPubKey = vds.read_bytes(vds.read_compact_size())
+  d['address'] = extract_public_key(scriptPubKey)
+  #d['script'] = decode_script(scriptPubKey)
+  d['raw_output_script'] = scriptPubKey.encode('hex')
+  d['index'] = i
+  return d
+
+
+def parse_Transaction(vds):
+  d = {}
+  start = vds.read_cursor
+  d['version'] = vds.read_int32()
+  n_vin = vds.read_compact_size()
+  d['inputs'] = []
+  for i in xrange(n_vin):
+    d['inputs'].append(parse_TxIn(vds))
+  n_vout = vds.read_compact_size()
+  d['outputs'] = []
+  for i in xrange(n_vout):
+    d['outputs'].append(parse_TxOut(vds, i))
+  d['lockTime'] = vds.read_uint32()
+  print d
+  return d
+
+
+
+
+opcodes = Enumeration("Opcodes", [
+    ("OP_0", 0), ("OP_PUSHDATA1",76), "OP_PUSHDATA2", "OP_PUSHDATA4", "OP_1NEGATE", "OP_RESERVED",
+    "OP_1", "OP_2", "OP_3", "OP_4", "OP_5", "OP_6", "OP_7",
+    "OP_8", "OP_9", "OP_10", "OP_11", "OP_12", "OP_13", "OP_14", "OP_15", "OP_16",
+    "OP_NOP", "OP_VER", "OP_IF", "OP_NOTIF", "OP_VERIF", "OP_VERNOTIF", "OP_ELSE", "OP_ENDIF", "OP_VERIFY",
+    "OP_RETURN", "OP_TOALTSTACK", "OP_FROMALTSTACK", "OP_2DROP", "OP_2DUP", "OP_3DUP", "OP_2OVER", "OP_2ROT", "OP_2SWAP",
+    "OP_IFDUP", "OP_DEPTH", "OP_DROP", "OP_DUP", "OP_NIP", "OP_OVER", "OP_PICK", "OP_ROLL", "OP_ROT",
+    "OP_SWAP", "OP_TUCK", "OP_CAT", "OP_SUBSTR", "OP_LEFT", "OP_RIGHT", "OP_SIZE", "OP_INVERT", "OP_AND",
+    "OP_OR", "OP_XOR", "OP_EQUAL", "OP_EQUALVERIFY", "OP_RESERVED1", "OP_RESERVED2", "OP_1ADD", "OP_1SUB", "OP_2MUL",
+    "OP_2DIV", "OP_NEGATE", "OP_ABS", "OP_NOT", "OP_0NOTEQUAL", "OP_ADD", "OP_SUB", "OP_MUL", "OP_DIV",
+    "OP_MOD", "OP_LSHIFT", "OP_RSHIFT", "OP_BOOLAND", "OP_BOOLOR",
+    "OP_NUMEQUAL", "OP_NUMEQUALVERIFY", "OP_NUMNOTEQUAL", "OP_LESSTHAN",
+    "OP_GREATERTHAN", "OP_LESSTHANOREQUAL", "OP_GREATERTHANOREQUAL", "OP_MIN", "OP_MAX",
+    "OP_WITHIN", "OP_RIPEMD160", "OP_SHA1", "OP_SHA256", "OP_HASH160",
+    "OP_HASH256", "OP_CODESEPARATOR", "OP_CHECKSIG", "OP_CHECKSIGVERIFY", "OP_CHECKMULTISIG",
+    "OP_CHECKMULTISIGVERIFY",
+    ("OP_SINGLEBYTE_END", 0xF0),
+    ("OP_DOUBLEBYTE_BEGIN", 0xF000),
+    "OP_PUBKEY", "OP_PUBKEYHASH",
+    ("OP_INVALIDOPCODE", 0xFFFF),
+])
+
+def script_GetOp(bytes):
+  i = 0
+  while i < len(bytes):
+    vch = None
+    opcode = ord(bytes[i])
+    i += 1
+    if opcode >= opcodes.OP_SINGLEBYTE_END:
+      opcode <<= 8
+      opcode |= ord(bytes[i])
+      i += 1
+
+    if opcode <= opcodes.OP_PUSHDATA4:
+      nSize = opcode
+      if opcode == opcodes.OP_PUSHDATA1:
+        nSize = ord(bytes[i])
+        i += 1
+      elif opcode == opcodes.OP_PUSHDATA2:
+        (nSize,) = struct.unpack_from('<H', bytes, i)
+        i += 2
+      elif opcode == opcodes.OP_PUSHDATA4:
+        (nSize,) = struct.unpack_from('<I', bytes, i)
+        i += 4
+      vch = bytes[i:i+nSize]
+      i += nSize
+
+    yield (opcode, vch, i)
+
+def script_GetOpName(opcode):
+  return (opcodes.whatis(opcode)).replace("OP_", "")
+
+def decode_script(bytes):
+  result = ''
+  for (opcode, vch, i) in script_GetOp(bytes):
+    if len(result) > 0: result += " "
+    if opcode <= opcodes.OP_PUSHDATA4:
+      result += "%d:"%(opcode,)
+      result += short_hex(vch)
+    else:
+      result += script_GetOpName(opcode)
+  return result
+
+def match_decoded(decoded, to_match):
+  if len(decoded) != len(to_match):
+    return False;
+  for i in range(len(decoded)):
+    if to_match[i] == opcodes.OP_PUSHDATA4 and decoded[i][0] <= opcodes.OP_PUSHDATA4:
+      continue  # Opcodes below OP_PUSHDATA4 all just push data onto stack, and are equivalent.
+    if to_match[i] != decoded[i][0]:
+      return False
+  return True
+
+def extract_public_key(bytes):
+  decoded = [ x for x in script_GetOp(bytes) ]
+
+  # non-generated TxIn transactions push a signature
+  # (seventy-something bytes) and then their public key
+  # (65 bytes) onto the stack:
+  match = [ opcodes.OP_PUSHDATA4, opcodes.OP_PUSHDATA4 ]
+  if match_decoded(decoded, match):
+    return public_key_to_bc_address(decoded[1][1])
+
+  # The Genesis Block, self-payments, and pay-by-IP-address payments look like:
+  # 65 BYTES:... CHECKSIG
+  match = [ opcodes.OP_PUSHDATA4, opcodes.OP_CHECKSIG ]
+  if match_decoded(decoded, match):
+    return public_key_to_bc_address(decoded[0][1])
+
+  # Pay-by-Bitcoin-address TxOuts look like:
+  # DUP HASH160 20 BYTES:... EQUALVERIFY CHECKSIG
+  match = [ opcodes.OP_DUP, opcodes.OP_HASH160, opcodes.OP_PUSHDATA4, opcodes.OP_EQUALVERIFY, opcodes.OP_CHECKSIG ]
+  if match_decoded(decoded, match):
+    return hash_160_to_bc_address(decoded[2][1])
+
+  return "(None)"

lib/gui_qt.py

@@ -330,7 +330,7 @@ class ElectrumWindow(QMainWindow):
         menu.exec_(self.contacts_list.viewport().mapToGlobal(position))
 
     def tx_details(self, tx_hash):
-        tx = self.wallet.tx_history.get(tx_hash)
+        tx = self.wallet.transactions.get(tx_hash)
 
         if tx['height']:
             conf = self.wallet.verifier.get_confirmations(tx_hash)
@@ -371,7 +371,7 @@ class ElectrumWindow(QMainWindow):
             return
         self.is_edit=True
         tx_hash = str(item.toolTip(0))
-        tx = self.wallet.tx_history.get(tx_hash)
+        tx = self.wallet.transactions.get(tx_hash)
         s = self.wallet.labels.get(tx_hash)
         text = unicode( item.text(2) )
         if text: 
@@ -447,11 +447,12 @@ class ElectrumWindow(QMainWindow):
                 conf = 0
                 time_str = 'pending'
                 icon = QIcon(":icons/unconfirmed.png")
-            v = tx['value']
+            v = self.wallet.get_tx_value(tx_hash)
             balance += v 
             label = self.wallet.labels.get(tx_hash)
             is_default_label = (label == '') or (label is None)
-            if is_default_label: label = tx['default_label']
+            if is_default_label:
+                label = self.wallet.get_default_label(tx_hash)
 
             item = QTreeWidgetItem( [ '', time_str, label, format_satoshis(v,True,self.wallet.num_zeros), format_satoshis(balance,False,self.wallet.num_zeros)] )
             item.setFont(2, QFont(MONOSPACE_FONT))
@@ -847,8 +848,9 @@ class ElectrumWindow(QMainWindow):
             label = self.wallet.labels.get(address,'')
             n = 0 
             h = self.wallet.history.get(address,[])
-            for item in h:
-                if not item['is_input'] : n=n+1
+            for tx_hash, tx_height in h:
+                tx = self.wallet.transactions.get(tx_hash)
+                if tx: n += 1
 
             tx = "%d "%n
             if n==0:
@@ -910,7 +912,7 @@ class ElectrumWindow(QMainWindow):
             if address in alias_targets: continue
             label = self.wallet.labels.get(address,'')
             n = 0 
-            for item in self.wallet.tx_history.values():
+            for item in self.wallet.transactions.values():
                 if address in item['outputs'] : n=n+1
             tx = "%d"%n
             item = QTreeWidgetItem( [ address, label, tx] )
@@ -1595,7 +1597,6 @@ class ElectrumGui:
             waiting_dialog(waiting)
             if wallet.is_found():
                 # history and addressbook
-                wallet.update_tx_history()
                 wallet.fill_addressbook()
                 print "Recovery successful"
                 wallet.save()

lib/verifier.py

@@ -34,8 +34,10 @@ class WalletVerifier(threading.Thread):
         self.interface = interface
         self.transactions    = []                                 # monitored transactions
         self.interface.register_channel('verifier')
-        self.verified_tx     = config.get('verified_tx',{})
+
+        self.verified_tx     = config.get('verified_tx',{})       # height of verified tx
         self.merkle_roots    = config.get('merkle_roots',{})      # hashed by me
+        
         self.targets         = config.get('targets',{})           # compute targets
         self.lock = threading.Lock()
         self.pending_headers = [] # headers that have not been verified
@@ -51,11 +53,11 @@ class WalletVerifier(threading.Thread):
             else:
                 return 0
 
-    def add(self, tx):
+    def add(self, tx_hash):
         """ add a transaction to the list of monitored transactions. """
         with self.lock:
-            if tx not in self.transactions:
-                self.transactions.append(tx)
+            if tx_hash not in self.transactions:
+                self.transactions.append(tx_hash)
 
     def run(self):
         requested_merkle = []
@@ -82,14 +84,14 @@ class WalletVerifier(threading.Thread):
                     all_chunks = True
                     print_error("downloaded all chunks")
 
-            # request missing tx merkle
-            for tx in self.transactions:
-                if tx not in self.verified_tx:
-                    if tx not in requested_merkle:
-                        requested_merkle.append(tx)
-                        self.request_merkle(tx)
-                        #break
-
+            # request missing tx
+            if all_chunks:
+                for tx_hash in self.transactions:
+                    if tx_hash not in self.verified_tx:
+                        if self.merkle_roots.get(tx_hash) is None and tx_hash not in requested_merkle:
+                            print_error('requesting merkle', tx_hash)
+                            self.interface.send([ ('blockchain.transaction.get_merkle',[tx_hash]) ], 'verifier')
+                            requested_merkle.append(tx_hash)
 
             # process pending headers
             if self.pending_headers and all_chunks:
@@ -141,25 +143,20 @@ class WalletVerifier(threading.Thread):
                 self.pending_headers.sort(key=lambda x: x.get('block_height'))
                 # print "pending headers", map(lambda x: x.get('block_height'), self.pending_headers)
 
-
-
             self.interface.trigger_callback('updated')
 
 
-    def request_merkle(self, tx_hash):
-        self.interface.send([ ('blockchain.transaction.get_merkle',[tx_hash]) ], 'verifier')
-
 
     def verify_merkle(self, tx_hash, result):
         tx_height = result.get('block_height')
         self.merkle_roots[tx_hash] = self.hash_merkle_root(result['merkle'], tx_hash, result.get('pos'))
         header = self.read_header(tx_height)
-        if header:
-            assert header.get('merkle_root') == self.merkle_roots[tx_hash]
-            self.verified_tx[tx_hash] = tx_height
-            print_error("verified %s"%tx_hash)
-            self.config.set_key('verified_tx', self.verified_tx, True)
-
+        if not header: return
+        assert header.get('merkle_root') == self.merkle_roots[tx_hash]
+        # we passed all the tests
+        self.verified_tx[tx_hash] = tx_height
+        print_error("verified %s"%tx_hash)
+        self.config.set_key('verified_tx', self.verified_tx, True)
 
     def verify_chunk(self, index, hexdata):
         data = hexdata.decode('hex')

lib/version.py

@@ -1,3 +1,3 @@
-ELECTRUM_VERSION = "1.2"
+ELECTRUM_VERSION = "1.3"
 SEED_VERSION = 4  # bump this everytime the seed generation is modified
 TRANSLATION_ID = 32150 # version of the wiki page 

lib/wallet.py

@@ -64,7 +64,6 @@ class Wallet:
         self.addresses             = config.get('addresses', [])          # receiving addresses visible for user
         self.change_addresses      = config.get('change_addresses', [])   # addresses used as change
         self.seed                  = config.get('seed', '')               # encrypted
-        self.history               = config.get('history',{})
         self.labels                = config.get('labels',{})              # labels for addresses and transactions
         self.aliases               = config.get('aliases', {})            # aliases for addresses
         self.authorities           = config.get('authorities', {})        # trusted addresses
@@ -73,10 +72,13 @@ class Wallet:
         self.receipts              = config.get('receipts',{})            # signed URIs
         self.addressbook           = config.get('contacts', [])           # outgoing addresses, for payments
         self.imported_keys         = config.get('imported_keys',{})
+        self.history               = config.get('history',{})             # address -> list(txid, height, timestamp)
+        self.transactions          = config.get('transactions',{})        # txid -> deserialised
 
         # not saved
+        self.prevout_values = {}
+        self.spent_outputs = []
         self.receipt = None          # next receipt
-        self.tx_history = {}
         self.banner = ''
 
         # spv
@@ -91,14 +93,18 @@ class Wallet:
         self.lock = threading.Lock()
         self.tx_event = threading.Event()
 
-        self.update_tx_history()
         if self.seed_version != SEED_VERSION:
             raise ValueError("This wallet seed is deprecated. Please run upgrade.py for a diagnostic.")
 
+        for tx_hash in self.transactions.keys():
+            self.update_tx_outputs(tx_hash)
+
+
     def init_up_to_date(self):
         self.up_to_date_event.clear()
         self.up_to_date = False
 
+
     def import_key(self, keypair, password):
         address, key = keypair.split(':')
         if not self.is_valid(address):
@@ -348,8 +354,8 @@ class Wallet:
         return (len(self.change_addresses) > 1 ) or ( len(self.addresses) > self.gap_limit )
 
     def fill_addressbook(self):
-        for tx in self.tx_history.values():
-            if tx['value']<0:
+        for tx_hash, tx in self.transactions.items():
+            if self.get_tx_value(tx_hash)<0:
                 for i in tx['outputs']:
                     if not self.is_mine(i) and i not in self.addressbook:
                         self.addressbook.append(i)
@@ -363,13 +369,51 @@ class Wallet:
         return flags
         
 
+    def get_tx_value(self, tx_hash, addresses = None):
+        # return the balance for that tx
+        if addresses is None: addresses = self.all_addresses()
+        v = 0
+        d = self.transactions.get(tx_hash)
+        if not d: return 0
+
+        for item in d.get('inputs'):
+            addr = item.get('address')
+            if addr in addresses:
+                key = item['prevout_hash']  + ':%d'%item['prevout_n']
+                value = self.prevout_values[ key ]
+                v -= value
+
+        for item in d.get('outputs'):
+            addr = item.get('address')
+            if addr in addresses: 
+                value = item.get('value')
+                v += value 
+
+        return v
+
+
+    
+    def update_tx_outputs(self, tx_hash):
+        tx = self.transactions.get(tx_hash)
+        for item in tx.get('outputs'):
+            value = item.get('value')
+            key = tx_hash+ ':%d'%item.get('index')
+            with self.lock:
+                self.prevout_values[key] = value 
+
+        for item in tx.get('inputs'):
+            if self.is_mine(item.get('address')):
+                key = item['prevout_hash'] + ':%d'%item['prevout_n']
+                self.spent_outputs.append(key)
+
+
     def get_addr_balance(self, addr):
         assert self.is_mine(addr)
         h = self.history.get(addr,[])
         c = u = 0
-        for item in h:
-            v = item['value']
-            if item['height']:
+        for tx_hash, tx_height in h:
+            v = self.get_tx_value(tx_hash, [addr])
+            if tx_height:
                 c += v
             else:
                 u += v
@@ -399,28 +443,35 @@ class Wallet:
             if i in domain: domain.remove(i)
 
         for addr in domain:
-            h = self.history.get(addr)
-            if h is None: continue
-            for item in h:
-                if item.get('raw_output_script'):
-                    coins.append( (addr,item))
-
-        coins = sorted( coins, key = lambda x: x[1]['timestamp'] )
+            h = self.history.get(addr, [])
+            for tx_hash, tx_height, in h:
+                tx = self.transactions.get(tx_hash)
+                for output in tx.get('outputs'):
+                    if output.get('address') != addr: continue
+                    key = tx_hash + ":%d" % output.get('index')
+                    if key in self.spent_outputs: continue
+                    output['tx_hash'] = tx_hash
+                    coins.append(output)
+
+        #coins = sorted( coins, key = lambda x: x[1]['timestamp'] )
 
         for addr in self.prioritized_addresses:
-            h = self.history.get(addr)
-            if h is None: continue
-            for item in h:
-                if item.get('raw_output_script'):
-                    prioritized_coins.append( (addr,item))
+            h = self.history.get(addr, [])
+            for tx_hash, tx_height, in h:
+                for output in tx.get('outputs'):
+                    if output.get('address') != addr: continue
+                    key = tx_hash + ":%d" % output.get('index')
+                    if key in self.spent_outputs: continue
+                    output['tx_hash'] = tx_hash
+                    prioritized_coins.append(output)
 
-        prioritized_coins = sorted( prioritized_coins, key = lambda x: x[1]['timestamp'] )
+        #prioritized_coins = sorted( prioritized_coins, key = lambda x: x[1]['timestamp'] )
 
         inputs = []
         coins = prioritized_coins + coins
 
-        for c in coins: 
-            addr, item = c
+        for item in coins: 
+            addr = item.get('address')
             v = item.get('value')
             total += v
             inputs.append((addr, v, item['tx_hash'], item['index'], item['raw_output_script'], None, None) )
@@ -474,35 +525,46 @@ class Wallet:
         else:
             return s
 
+
     def get_status(self, address):
         with self.lock:
             h = self.history.get(address)
-        if not h:
-            status = None
-        else:
-            lastpoint = h[-1]
-            status = lastpoint['block_hash']
-            if status == 'mempool': 
-                status = status + ':%d'% len(h)
-        return status
+        if not h: return None
+        status = ''
+        for tx_hash, height in h:
+            status += tx_hash + ':%d:' % height
+        return hashlib.sha256( status ).digest().encode('hex')
+
 
 
-    def receive_history_callback(self, addr, data): 
+    def receive_tx_callback(self, tx_hash, d):
         #print "updating history for", addr
         with self.lock:
-            self.history[addr] = data
-            self.update_tx_history()
+            self.transactions[tx_hash] = d
+
+        if self.verifier: self.verifier.add(tx_hash)
+        self.update_tx_outputs(tx_hash)
+        self.save()
+
+
+    def receive_history_callback(self, addr, hist):
+        #print "updating history for", addr
+        with self.lock:
+            self.history[addr] = hist
             self.save()
 
+
+
     def get_tx_history(self):
         with self.lock:
-            lines = self.tx_history.values()
+            lines = self.transactions.values()
+
         lines = sorted(lines, key=operator.itemgetter("timestamp"))
         return lines
 
     def get_transactions_at_height(self, height):
         with self.lock:
-            values = self.tx_history.values()[:]
+            values = self.transactions.values()[:]
 
         out = []
         for tx in values:
@@ -510,42 +572,27 @@ class Wallet:
                 out.append(tx['tx_hash'])
         return out
 
-    def update_tx_history(self):
-        self.tx_history= {}
-        for addr in self.all_addresses():
-            h = self.history.get(addr)
-            if h is None: continue
-            for tx in h:
-                tx_hash = tx['tx_hash']
-                line = self.tx_history.get(tx_hash)
-                if not line:
-                    self.tx_history[tx_hash] = copy.copy(tx)
-                    line = self.tx_history.get(tx_hash)
-                else:
-                    line['value'] += tx['value']
-                if line['height'] == 0:
-                    line['timestamp'] = 1e12
-                else:
-                    if self.verifier: self.verifier.add(tx_hash)
-                    
-        self.update_tx_labels()
 
-    def update_tx_labels(self):
-        for tx in self.tx_history.values():
+    def get_default_label(self, tx_hash):
+        tx = self.transactions.get(tx_hash)
+        if tx:
             default_label = ''
-            if tx['value']<0:
-                for o_addr in tx['outputs']:
+            if self.get_tx_value(tx_hash)<0:
+                for o in tx['outputs']:
+                    o_addr = o.get('address')
                     if not self.is_mine(o_addr):
                         try:
                             default_label = self.labels[o_addr]
                         except KeyError:
                             default_label = o_addr
             else:
-                for o_addr in tx['outputs']:
+                for o in tx['outputs']:
+                    o_addr = o.get('address')
                     if self.is_mine(o_addr) and not self.is_change(o_addr):
                         break
                 else:
-                    for o_addr in tx['outputs']:
+                    for o in tx['outputs']:
+                        o_addr = o.get('address')
                         if self.is_mine(o_addr):
                             break
                     else:
@@ -558,7 +605,8 @@ class Wallet:
                     except KeyError:
                         default_label = o_addr
 
-            tx['default_label'] = default_label
+        return default_label
+
 
     def mktx(self, to_address, amount, label, password, fee=None, change_addr=None, from_addr= None):
         if not self.is_valid(to_address):
@@ -811,6 +859,7 @@ class Wallet:
             'frozen_addresses': self.frozen_addresses,
             'prioritized_addresses': self.prioritized_addresses,
             'gap_limit': self.gap_limit,
+            'transactions': self.transactions,
         }
         for k, v in s.items():
             self.config.set_key(k,v)
@@ -818,7 +867,9 @@ class Wallet:
 
     def set_verifier(self, verifier):
         self.verifier = verifier
-        self.update_tx_history()
+        for tx_hash in self.transactions.keys(): 
+            self.verifier.add(tx_hash)
+
 
 
 
@@ -862,6 +913,7 @@ class WalletSynchronizer(threading.Thread):
 
 
     def run(self):
+        requested_tx = []
 
         # wait until we are connected, in case the user is not connected
         while not self.interface.is_connected:
@@ -897,9 +949,28 @@ class WalletSynchronizer(threading.Thread):
                             
             elif method == 'blockchain.address.get_history':
                 addr = params[0]
-                self.wallet.receive_history_callback(addr, result)
+                hist = []
+                # in the new protocol, we will receive a list of (tx_hash, height)
+                for tx in result: hist.append( (tx['tx_hash'], tx['height']) )
+                # store it
+                self.wallet.receive_history_callback(addr, hist)
+                # request transactions that we don't have 
+                for tx_hash, tx_height in hist:
+                    if self.wallet.transactions.get(tx_hash) is None and tx_hash not in requested_tx:
+                        self.interface.send([ ('blockchain.transaction.get',[tx_hash, tx_height]) ], 'synchronizer')
+                        requested_tx.append(tx_hash)
+
+            elif method == 'blockchain.transaction.get':
+                tx_hash = params[0]
+                tx_height = params[1]
+                header = self.wallet.verifier.read_header(tx_height)
+                timestamp = header.get('timestamp')
+                tx = result
+                self.receive_tx(tx_hash, tx_height, timestamp, tx)
+                requested_tx.remove(tx_hash)
                 self.was_updated = True
 
+
             elif method == 'blockchain.transaction.broadcast':
                 self.wallet.tx_result = result
                 self.wallet.tx_event.set()
@@ -916,3 +987,20 @@ class WalletSynchronizer(threading.Thread):
                 self.was_updated = False
 
 
+    def receive_tx(self, tx_hash, tx_height, timestamp, raw_tx):
+
+        assert tx_hash == hash_encode(Hash(raw_tx.decode('hex')))
+
+        import deserialize, BCDataStream
+
+        # deserialize
+        vds = BCDataStream.BCDataStream()
+        vds.write(raw_tx.decode('hex'))
+        d = deserialize.parse_Transaction(vds)
+        d['height'] = tx_height
+        d['tx_hash'] = tx_hash
+        d['timestamp'] = timestamp
+        d['default_label'] = tx_hash
+        print d
+        self.wallet.receive_tx_callback(tx_hash, d)
+

scripts/validate_tx

@@ -29,6 +29,10 @@ def hash_header(res):
 
 
 def verify_tx(tx_hash):
+    rawtx = i.synchronous_get([ ('blockchain.transaction.get',[tx_hash]) ])[0]
+    print rawtx
+    return
+    
     res = i.synchronous_get([ ('blockchain.transaction.get_merkle',[tx_hash]) ])[0]
     merkle_root = hash_merkle_root(res['merkle'], tx_hash, res['pos'])
     tx_height = res.get('block_height')