diff --git a/lib/coins.py b/lib/coins.py index 989bec1..f9922d1 100644 --- a/lib/coins.py +++ b/lib/coins.py @@ -21,7 +21,7 @@ import sys from lib.hash import Base58, hash160, ripemd160, double_sha256, hash_to_str from lib.script import ScriptPubKey -from lib.tx import Deserializer +from lib.tx import Deserializer, DeserializerSegWit from lib.util import cachedproperty, subclasses @@ -204,11 +204,14 @@ class Coin(object): @classmethod def read_block(cls, block, height): - '''Return a tuple (header, tx_hashes, txs) given a raw block at - the given height.''' + '''Returns a pair (header, tx_list) given a raw block and height. + + tx_list is a list of (deserialized_tx, tx_hash) pairs. + ''' + deserializer = cls.deserializer() hlen = cls.header_len(height) header, rest = block[:hlen], block[hlen:] - return (header, ) + Deserializer(rest).read_block() + return (header, deserializer(rest).read_block()) @classmethod def decimal_value(cls, value): @@ -234,6 +237,10 @@ class Coin(object): 'nonce': nonce, } + @classmethod + def deserializer(cls): + return Deserializer + class Bitcoin(Coin): NAME = "Bitcoin" @@ -271,6 +278,19 @@ class BitcoinTestnet(Bitcoin): IRC_PREFIX = "ET_" +class BitcoinTestnetSegWit(BitcoinTestnet): + '''Bitcoin Testnet for Core bitcoind >= 0.13.1. + + Unfortunately 0.13.1 broke backwards compatibility of the RPC + interface's TX serialization, SegWit transactions serialize + differently than with earlier versions. If you are using such a + bitcoind on testnet, you must use this class as your "COIN". + ''' + @classmethod + def deserializer(cls): + return DeserializerSegWit + + class Litecoin(Coin): NAME = "Litecoin" SHORTNAME = "LTC" diff --git a/lib/tx.py b/lib/tx.py index 2fb86cd..565595e 100644 --- a/lib/tx.py +++ b/lib/tx.py @@ -72,28 +72,25 @@ class Deserializer(object): self.cursor = 0 def read_tx(self): + '''Return a (Deserialized TX, TX_HASH) pair. + + The hash needs to be reversed for human display; for efficiency + we process it in the natural serialized order. + ''' + start = self.cursor return Tx( self._read_le_int32(), # version self._read_inputs(), # inputs self._read_outputs(), # outputs self._read_le_uint32() # locktime - ) + ), double_sha256(self.binary[start:self.cursor]) def read_block(self): - tx_hashes = [] - txs = [] - binary = self.binary - hash = double_sha256 + '''Returns a list of (deserialized_tx, tx_hash) pairs.''' read_tx = self.read_tx - append_hash = tx_hashes.append - for n in range(self._read_varint()): - start = self.cursor - txs.append(read_tx()) - # Note this hash needs to be reversed for human display - # For efficiency we store it in the natural serialized order - append_hash(hash(binary[start:self.cursor])) - assert self.cursor == len(binary) - return tx_hashes, txs + txs = [read_tx() for n in range(self._read_varint())] + assert self.cursor == len(self.binary) + return txs def _read_inputs(self): read_input = self._read_input @@ -161,3 +158,62 @@ class Deserializer(object): result, = unpack_from('= self.cache_MB or hist_MB >= self.cache_MB // 5: self.flush(utxo_MB >= self.cache_MB * 4 // 5) - def fs_advance_block(self, header, tx_hashes, txs): + def fs_advance_block(self, header, txs): '''Update unflushed FS state for a new block.''' prior_tx_count = self.tx_counts[-1] if self.tx_counts else 0 # Cache the new header, tx hashes and cumulative tx count self.headers.append(header) - self.tx_hashes.append(tx_hashes) + self.tx_hashes.append(b''.join(tx_hash for tx, tx_hash in txs)) self.tx_counts.append(prior_tx_count + len(txs)) def advance_block(self, block, touched): - header, tx_hashes, txs = self.coin.read_block(block, self.height + 1) + header, txs = self.coin.read_block(block, self.height + 1) if self.tip != self.coin.header_prevhash(header): raise ChainReorg - self.fs_advance_block(header, tx_hashes, txs) + self.fs_advance_block(header, txs) self.tip = self.coin.header_hash(header) self.height += 1 - undo_info = self.advance_txs(tx_hashes, txs, touched) + undo_info = self.advance_txs(txs, touched) if self.daemon.cached_height() - self.height <= self.env.reorg_limit: self.write_undo_info(self.height, b''.join(undo_info)) - def advance_txs(self, tx_hashes, txs, touched): + def advance_txs(self, txs, touched): undo_info = [] # Use local vars for speed in the loops @@ -492,7 +494,7 @@ class BlockProcessor(server.db.DB): spend_utxo = self.spend_utxo undo_info_append = undo_info.append - for tx, tx_hash in zip(txs, tx_hashes): + for tx, tx_hash in txs: hashXs = set() add_hashX = hashXs.add tx_numb = s_pack('= 0 self.height -= 1 @@ -553,7 +555,7 @@ class BlockProcessor(server.db.DB): touched.discard(None) self.backup_flush(touched) - def backup_txs(self, tx_hashes, txs, touched): + def backup_txs(self, txs, touched): # Prevout values, in order down the block (coinbase first if present) # undo_info is in reverse block order undo_info = self.read_undo_info(self.height) @@ -569,10 +571,7 @@ class BlockProcessor(server.db.DB): script_hashX = self.coin.hashX_from_script undo_entry_len = 12 + self.coin.HASHX_LEN - rtxs = reversed(txs) - rtx_hashes = reversed(tx_hashes) - - for tx_hash, tx in zip(rtx_hashes, rtxs): + for tx, tx_hash in reversed(txs): for idx, txout in enumerate(tx.outputs): # Spend the TX outputs. Be careful with unspendable # outputs - we didn't save those in the first place. diff --git a/server/db.py b/server/db.py index 0c653e3..9b1b2d2 100644 --- a/server/db.py +++ b/server/db.py @@ -10,7 +10,6 @@ import array import ast -import itertools import os from struct import pack, unpack from bisect import bisect_left, bisect_right @@ -234,7 +233,7 @@ class DB(util.LoggedClass): assert len(self.tx_hashes) == blocks_done assert len(self.tx_counts) == new_height + 1 - hashes = b''.join(itertools.chain(*block_tx_hashes)) + hashes = b''.join(block_tx_hashes) assert len(hashes) % 32 == 0 assert len(hashes) // 32 == txs_done diff --git a/server/mempool.py b/server/mempool.py index 09b9fd7..e61bf0b 100644 --- a/server/mempool.py +++ b/server/mempool.py @@ -13,7 +13,6 @@ import time from collections import defaultdict from lib.hash import hash_to_str, hex_str_to_hash -from lib.tx import Deserializer import lib.util as util from server.daemon import DaemonError @@ -200,6 +199,7 @@ class MemPool(util.LoggedClass): not depend on the result remaining the same are fine. ''' script_hashX = self.coin.hashX_from_script + deserializer = self.coin.deserializer() db_utxo_lookup = self.db.db_utxo_lookup txs = self.txs @@ -207,7 +207,7 @@ class MemPool(util.LoggedClass): for tx_hash, raw_tx in raw_tx_map.items(): if not tx_hash in txs: continue - tx = Deserializer(raw_tx).read_tx() + tx, _tx_hash = deserializer(raw_tx).read_tx() # Convert the tx outputs into (hashX, value) pairs txout_pairs = [(script_hashX(txout.pk_script), txout.value) @@ -271,6 +271,7 @@ class MemPool(util.LoggedClass): if not hashX in self.hashXs: return [] + deserializer = self.coin.deserializer() hex_hashes = self.hashXs[hashX] raw_txs = await self.daemon.getrawtransactions(hex_hashes) result = [] @@ -281,7 +282,7 @@ class MemPool(util.LoggedClass): txin_pairs, txout_pairs = item tx_fee = (sum(v for hashX, v in txin_pairs) - sum(v for hashX, v in txout_pairs)) - tx = Deserializer(raw_tx).read_tx() + tx, tx_hash = deserializer(raw_tx).read_tx() unconfirmed = any(txin.prev_hash in self.txs for txin in tx.inputs) result.append((hex_hash, tx_fee, unconfirmed)) return result diff --git a/server/protocol.py b/server/protocol.py index 1b0516f..60b7a45 100644 --- a/server/protocol.py +++ b/server/protocol.py @@ -14,7 +14,6 @@ import traceback from lib.hash import sha256, double_sha256, hash_to_str, hex_str_to_hash from lib.jsonrpc import JSONRPC -from lib.tx import Deserializer from server.daemon import DaemonError from server.version import VERSION @@ -427,7 +426,8 @@ class ElectrumX(Session): if not raw_tx: return None raw_tx = bytes.fromhex(raw_tx) - tx = Deserializer(raw_tx).read_tx() + deserializer = self.coin.deserializer() + tx, tx_hash = deserializer(raw_tx).read_tx() if index >= len(tx.outputs): return None return self.coin.address_from_script(tx.outputs[index].pk_script)