Merge branch 'develop'

README.rst

@@ -115,7 +115,6 @@ Roadmap Pre-1.0
 ===============
 
 - minor code cleanups.
-- support bitcoin testnet with Satoshi bitcoind 0.13.1
 - implement simple protocol to discover peers without resorting to IRC.
   This may slip to post 1.0
 
@@ -142,6 +141,24 @@ version prior to the release of 1.0.
 ChangeLog
 =========
 
+Version 0.10.2
+--------------
+
+* Note the **NETWORK** environment variable was renamed **NET** to
+  bring it into line with lib/coins.py.
+* The genesis hash is now compared with the genesis hash expected by
+  **COIN** and **NET**.  This sanity check was not done previously, so
+  you could easily be syncing to a network daemon different to what
+  you thought.
+* SegWit-compatible testnet support for bitcoin core versions 0.13.1
+  or higher.  Resolves issue `#92#`.  Testnet worked with prior
+  versions of ElectrumX as long as you used an older bitcoind too,
+  such as 0.13.0 or Bitcoin Unlimited.
+
+  **Note**: for testnet, you need to set *NET** to *testnet-segwit* if
+  using recent RPC incompatible core bitcoinds, or *testnet* if using
+  older RPC compatible bitcoinds.
+
 Version 0.10.1
 --------------
 
@@ -167,6 +184,12 @@ variables to use roughly the same amount of memory.
 For now this code should be considered experimental; if you want
 stability please stick with the 0.9 series.
 
+Version 0.9.23
+--------------
+
+* Backport of the fix for issue `#94#` - stale references to old
+  sessions.  This would effectively memory and network handles.
+
 Version 0.9.22
 --------------
 
@@ -334,6 +357,7 @@ Version 0.9.0
 .. _#75: https://github.com/kyuupichan/electrumx/issues/75
 .. _#88: https://github.com/kyuupichan/electrumx/issues/88
 .. _#89: https://github.com/kyuupichan/electrumx/issues/89
+.. _#92: https://github.com/kyuupichan/electrumx/issues/92
 .. _#93: https://github.com/kyuupichan/electrumx/issues/93
 .. _#94: https://github.com/kyuupichan/electrumx/issues/94
 .. _docs/HOWTO.rst: https://github.com/kyuupichan/electrumx/blob/master/docs/HOWTO.rst

docs/ENVIRONMENT.rst

@@ -31,7 +31,7 @@ These environment variables are always required:
 
   The leading `http://` is optional, as is the trailing slash.  The
   `:port` part is also optional and will default to the standard RPC
-  port for **COIN** and **NETWORK** if omitted.
+  port for **COIN** and **NET** if omitted.
 
 
 For the `run` script
@@ -58,7 +58,7 @@ These environment variables are optional:
   Must be a *NAME* from one of the **Coin** classes in
   `lib/coins.py`_.  Defaults to `Bitcoin`.
 
-* **NETWORK**
+* **NET**
 
   Must be a *NET* from one of the **Coin** classes in `lib/coins.py`_.
   Defaults to `mainnet`.
@@ -77,7 +77,7 @@ These environment variables are optional:
   The maximum number of blocks to be able to handle in a chain
   reorganisation.  ElectrumX retains some fairly compact undo
   information for this many blocks in levelDB.  The default is a
-  function of **COIN** and **NETWORK**; for Bitcoin mainnet it is 200.
+  function of **COIN** and **NET**; for Bitcoin mainnet it is 200.
 
 * **HOST**
 
@@ -98,7 +98,7 @@ These environment variables are optional:
 
   ElectrumX will listen on this port for local RPC connections.
   ElectrumX listens for RPC connections unless this is explicitly set
-  to blank.  The default is appropriate for **COIN** and **NETWORK**
+  to blank.  The default is appropriate for **COIN** and **NET**
   (e.g., 8000 for Bitcoin mainnet) if not set.
 
 * **DONATION_ADDRESS**
@@ -223,7 +223,7 @@ connectivity on IRC:
 
   The nick to use when connecting to IRC.  The default is a hash of
   **REPORT_HOST**.  Either way a prefix will be prepended depending on
-  **COIN** and **NETWORK**.
+  **COIN** and **NET**.
 
 * **REPORT_HOST**
 

lib/coins.py

@@ -21,8 +21,8 @@ import sys
 
 from lib.hash import Base58, hash160, ripemd160, double_sha256, hash_to_str
 from lib.script import ScriptPubKey
-from lib.tx import Deserializer
-from lib.util import cachedproperty, subclasses
+from lib.tx import Deserializer, DeserializerSegWit
+import lib.util as util
 
 
 class CoinError(Exception):
@@ -46,7 +46,7 @@ class Coin(object):
         '''Return a coin class given name and network.
 
         Raise an exception if unrecognised.'''
-        for coin in subclasses(Coin):
+        for coin in util.subclasses(Coin):
             if (coin.NAME.lower() == name.lower()
                     and coin.NET.lower() == net.lower()):
                 return coin
@@ -70,6 +70,20 @@ class Coin(object):
     def daemon_urls(cls, urls):
         return [cls.sanitize_url(url) for url in urls.split(',')]
 
+    @classmethod
+    def genesis_block(cls, block):
+        '''Check the Genesis block is the right one for this coin.
+
+        Return the block less its unspendable coinbase.
+        '''
+        header = block[:cls.header_len(0)]
+        header_hex_hash = hash_to_str(cls.header_hash(header))
+        if header_hex_hash != cls.GENESIS_HASH:
+            raise CoinError('genesis block has hash {} expected {}'
+                            .format(header_hex_hash, cls.GENESIS_HASH))
+
+        return header + bytes(1)
+
     @classmethod
     def hashX_from_script(cls, script):
         '''Returns a hashX from a script.'''
@@ -78,7 +92,7 @@ class Coin(object):
             return None
         return sha256(script).digest()[:cls.HASHX_LEN]
 
-    @cachedproperty
+    @util.cachedproperty
     def address_handlers(cls):
         return ScriptPubKey.PayToHandlers(
             address = cls.P2PKH_address_from_hash160,
@@ -204,11 +218,14 @@ class Coin(object):
 
     @classmethod
     def read_block(cls, block, height):
-        '''Return a tuple (header, tx_hashes, txs) given a raw block at
-        the given height.'''
+        '''Returns a pair (header, tx_list) given a raw block and height.
+
+        tx_list is a list of (deserialized_tx, tx_hash) pairs.
+        '''
+        deserializer = cls.deserializer()
         hlen = cls.header_len(height)
         header, rest = block[:hlen], block[hlen:]
-        return (header, ) + Deserializer(rest).read_block()
+        return (header, deserializer(rest).read_block())
 
     @classmethod
     def decimal_value(cls, value):
@@ -234,6 +251,10 @@ class Coin(object):
             'nonce': nonce,
         }
 
+    @classmethod
+    def deserializer(cls):
+        return Deserializer
+
 
 class Bitcoin(Coin):
     NAME = "Bitcoin"
@@ -244,8 +265,8 @@ class Bitcoin(Coin):
     P2PKH_VERBYTE = 0x00
     P2SH_VERBYTE = 0x05
     WIF_BYTE = 0x80
-    GENESIS_HASH=(b'000000000019d6689c085ae165831e93'
-                  b'4ff763ae46a2a6c172b3f1b60a8ce26f')
+    GENESIS_HASH=('000000000019d6689c085ae165831e93'
+                  '4ff763ae46a2a6c172b3f1b60a8ce26f')
     TX_COUNT = 156335304
     TX_COUNT_HEIGHT = 429972
     TX_PER_BLOCK = 1800
@@ -262,13 +283,29 @@ class BitcoinTestnet(Bitcoin):
     P2PKH_VERBYTE = 0x6f
     P2SH_VERBYTE = 0xc4
     WIF_BYTE = 0xef
-    GENESIS_HASH=(b'000000000933ea01ad0ee984209779ba'
-                  b'aec3ced90fa3f408719526f8d77f4943')
+    GENESIS_HASH=('000000000933ea01ad0ee984209779ba'
+                  'aec3ced90fa3f408719526f8d77f4943')
     REORG_LIMIT = 2000
     TX_COUNT = 12242438
     TX_COUNT_HEIGHT = 1035428
     TX_PER_BLOCK = 21
     IRC_PREFIX = "ET_"
+    RPC_PORT = 18332
+
+
+class BitcoinTestnetSegWit(BitcoinTestnet):
+    '''Bitcoin Testnet for Core bitcoind >= 0.13.1.
+
+    Unfortunately 0.13.1 broke backwards compatibility of the RPC
+    interface's TX serialization, SegWit transactions serialize
+    differently than with earlier versions.  If you are using such a
+    bitcoind on testnet, you must use this class as your "COIN".
+    '''
+    NET = "testnet-segwit"
+
+    @classmethod
+    def deserializer(cls):
+        return DeserializerSegWit
 
 
 class Litecoin(Coin):
@@ -280,8 +317,8 @@ class Litecoin(Coin):
     P2PKH_VERBYTE = 0x30
     P2SH_VERBYTE = 0x05
     WIF_BYTE = 0xb0
-    GENESIS_HASH=(b'000000000019d6689c085ae165831e93'
-                  b'4ff763ae46a2a6c172b3f1b60a8ce26f')
+    GENESIS_HASH=('000000000019d6689c085ae165831e93'
+                  '4ff763ae46a2a6c172b3f1b60a8ce26f')
     TX_COUNT = 8908766
     TX_COUNT_HEIGHT = 1105256
     TX_PER_BLOCK = 10
@@ -355,8 +392,8 @@ class Dash(Coin):
     NET = "mainnet"
     XPUB_VERBYTES = bytes.fromhex("02fe52cc")
     XPRV_VERBYTES = bytes.fromhex("02fe52f8")
-    GENESIS_HASH = (b'00000ffd590b1485b3caadc19b22e637'
-                    b'9c733355108f107a430458cdf3407ab6')
+    GENESIS_HASH = ('00000ffd590b1485b3caadc19b22e637'
+                    '9c733355108f107a430458cdf3407ab6')
     P2PKH_VERBYTE = 0x4c
     P2SH_VERBYTE = 0x10
     WIF_BYTE = 0xcc
@@ -378,8 +415,8 @@ class DashTestnet(Dash):
     NET = "testnet"
     XPUB_VERBYTES = bytes.fromhex("3a805837")
     XPRV_VERBYTES = bytes.fromhex("3a8061a0")
-    GENESIS_HASH = (b'00000bafbc94add76cb75e2ec9289483'
-                    b'7288a481e5c005f6563d91623bf8bc2c')
+    GENESIS_HASH = ('00000bafbc94add76cb75e2ec9289483'
+                    '7288a481e5c005f6563d91623bf8bc2c')
     P2PKH_VERBYTE = 0x8c
     P2SH_VERBYTE = 0x13
     WIF_BYTE = 0xef

lib/tx.py

@@ -72,28 +72,25 @@ class Deserializer(object):
         self.cursor = 0
 
     def read_tx(self):
+        '''Return a (Deserialized TX, TX_HASH) pair.
+
+        The hash needs to be reversed for human display; for efficiency
+        we process it in the natural serialized order.
+        '''
+        start = self.cursor
         return Tx(
             self._read_le_int32(),  # version
             self._read_inputs(),    # inputs
             self._read_outputs(),   # outputs
             self._read_le_uint32()  # locktime
-        )
+        ), double_sha256(self.binary[start:self.cursor])
 
     def read_block(self):
-        tx_hashes = []
-        txs = []
-        binary = self.binary
-        hash = double_sha256
+        '''Returns a list of (deserialized_tx, tx_hash) pairs.'''
         read_tx = self.read_tx
-        append_hash = tx_hashes.append
-        for n in range(self._read_varint()):
-            start = self.cursor
-            txs.append(read_tx())
-            # Note this hash needs to be reversed for human display
-            # For efficiency we store it in the natural serialized order
-            append_hash(hash(binary[start:self.cursor]))
-        assert self.cursor == len(binary)
-        return tx_hashes, txs
+        txs = [read_tx() for n in range(self._read_varint())]
+        assert self.cursor == len(self.binary)
+        return txs
 
     def _read_inputs(self):
         read_input = self._read_input
@@ -161,3 +158,62 @@ class Deserializer(object):
         result, = unpack_from('<Q', self.binary, self.cursor)
         self.cursor += 8
         return result
+
+
+class TxSegWit(namedtuple("Tx", "version marker flag inputs outputs "
+                          "witness locktime")):
+    '''Class representing a SegWit transaction.'''
+
+    @cachedproperty
+    def is_coinbase(self):
+        return self.inputs[0].is_coinbase
+
+
+class DeserializerSegWit(Deserializer):
+
+    # https://bitcoincore.org/en/segwit_wallet_dev/#transaction-serialization
+
+    def _read_byte(self):
+        cursor = self.cursor
+        self.cursor += 1
+        return self.binary[cursor]
+
+    def _read_witness(self, fields):
+        read_witness_field = self._read_witness_field
+        return [read_witness_field() for i in range(fields)]
+
+    def _read_witness_field(self):
+        read_varbytes = self._read_varbytes
+        return [read_varbytes() for i in range(self._read_varint())]
+
+    def read_tx(self):
+        '''Return a (Deserialized TX, TX_HASH) pair.
+
+        The hash needs to be reversed for human display; for efficiency
+        we process it in the natural serialized order.
+        '''
+        marker = self.binary[self.cursor + 4]
+        if marker:
+            return super().read_tx()
+
+        # Ugh, this is nasty.
+        start = self.cursor
+        version = self._read_le_int32()
+        orig_ser = self.binary[start:self.cursor]
+
+        marker = self._read_byte()
+        flag = self._read_byte()
+
+        start = self.cursor
+        inputs = self._read_inputs()
+        outputs = self._read_outputs()
+        orig_ser += self.binary[start:self.cursor]
+
+        witness = self._read_witness(len(inputs))
+
+        start = self.cursor
+        locktime = self._read_le_uint32()
+        orig_ser += self.binary[start:self.cursor]
+
+        return TxSegWit(version, marker, flag, inputs,
+                        outputs, witness, locktime), double_sha256(orig_ser)

samples/systemd/electrumx.conf

@@ -42,7 +42,7 @@
 #MISC
 #
     #COIN = Bitcoin  # lib/coins.py
-    #NETWORK = mainnet  # lib/coins.py
+    #NET = mainnet  # lib/coins.py
     #DB_ENGINE = leveldb
         #leveldb, rocksdb, lmdb (You'll need to install appropriate python packages)
 

server/block_processor.py

@@ -125,9 +125,11 @@ class Prefetcher(LoggedClass):
 
                 assert count == len(blocks)
 
-                # Strip the unspendable genesis coinbase
+                # Special handling for genesis block
                 if first == 0:
-                    blocks[0] = blocks[0][:self.coin.header_len(0)] + bytes(1)
+                    blocks[0] = self.coin.genesis_block(blocks[0])
+                    self.logger.info('verified genesis block with hash {}'
+                                     .format(hex_hashes[0]))
 
                 # Update our recent average block size estimate
                 size = sum(len(block) for block in blocks)
@@ -444,7 +446,9 @@ class BlockProcessor(server.db.DB):
         utxo_cache_size = len(self.utxo_cache) * 205
         db_deletes_size = len(self.db_deletes) * 57
         hist_cache_size = len(self.history) * 180 + self.history_size * 4
-        tx_hash_size = (self.tx_count - self.fs_tx_count) * 74
+        # Roughly ntxs * 32 + nblocks * 42
+        tx_hash_size = ((self.tx_count - self.fs_tx_count) * 32
+                        + (self.height - self.fs_height) * 42)
         utxo_MB = (db_deletes_size + utxo_cache_size) // one_MB
         hist_MB = (hist_cache_size + tx_hash_size) // one_MB
 
@@ -458,28 +462,28 @@ class BlockProcessor(server.db.DB):
         if utxo_MB + hist_MB >= self.cache_MB or hist_MB >= self.cache_MB // 5:
             self.flush(utxo_MB >= self.cache_MB * 4 // 5)
 
-    def fs_advance_block(self, header, tx_hashes, txs):
+    def fs_advance_block(self, header, txs):
         '''Update unflushed FS state for a new block.'''
         prior_tx_count = self.tx_counts[-1] if self.tx_counts else 0
 
         # Cache the new header, tx hashes and cumulative tx count
         self.headers.append(header)
-        self.tx_hashes.append(tx_hashes)
+        self.tx_hashes.append(b''.join(tx_hash for tx, tx_hash in txs))
         self.tx_counts.append(prior_tx_count + len(txs))
 
     def advance_block(self, block, touched):
-        header, tx_hashes, txs = self.coin.read_block(block, self.height + 1)
+        header, txs = self.coin.read_block(block, self.height + 1)
         if self.tip != self.coin.header_prevhash(header):
             raise ChainReorg
 
-        self.fs_advance_block(header, tx_hashes, txs)
+        self.fs_advance_block(header, txs)
         self.tip = self.coin.header_hash(header)
         self.height += 1
-        undo_info = self.advance_txs(tx_hashes, txs, touched)
+        undo_info = self.advance_txs(txs, touched)
         if self.daemon.cached_height() - self.height <= self.env.reorg_limit:
             self.write_undo_info(self.height, b''.join(undo_info))
 
-    def advance_txs(self, tx_hashes, txs, touched):
+    def advance_txs(self, txs, touched):
         undo_info = []
 
         # Use local vars for speed in the loops
@@ -492,7 +496,7 @@ class BlockProcessor(server.db.DB):
         spend_utxo = self.spend_utxo
         undo_info_append = undo_info.append
 
-        for tx, tx_hash in zip(txs, tx_hashes):
+        for tx, tx_hash in txs:
             hashXs = set()
             add_hashX = hashXs.add
             tx_numb = s_pack('<I', tx_num)
@@ -533,14 +537,14 @@ class BlockProcessor(server.db.DB):
         self.assert_flushed()
 
         for block in blocks:
-            header, tx_hashes, txs = self.coin.read_block(block, self.height)
+            header, txs = self.coin.read_block(block, self.height)
             header_hash = self.coin.header_hash(header)
             if header_hash != self.tip:
                 raise ChainError('backup block {} is not tip {} at height {:,d}'
                                  .format(hash_to_str(header_hash),
                                          hash_to_str(self.tip), self.height))
 
-            self.backup_txs(tx_hashes, txs, touched)
+            self.backup_txs(txs, touched)
             self.tip = self.coin.header_prevhash(header)
             assert self.height >= 0
             self.height -= 1
@@ -553,7 +557,7 @@ class BlockProcessor(server.db.DB):
         touched.discard(None)
         self.backup_flush(touched)
 
-    def backup_txs(self, tx_hashes, txs, touched):
+    def backup_txs(self, txs, touched):
         # Prevout values, in order down the block (coinbase first if present)
         # undo_info is in reverse block order
         undo_info = self.read_undo_info(self.height)
@@ -569,10 +573,7 @@ class BlockProcessor(server.db.DB):
         script_hashX = self.coin.hashX_from_script
         undo_entry_len = 12 + self.coin.HASHX_LEN
 
-        rtxs = reversed(txs)
-        rtx_hashes = reversed(tx_hashes)
-
-        for tx_hash, tx in zip(rtx_hashes, rtxs):
+        for tx, tx_hash in reversed(txs):
             for idx, txout in enumerate(tx.outputs):
                 # Spend the TX outputs.  Be careful with unspendable
                 # outputs - we didn't save those in the first place.

server/db.py

@@ -10,7 +10,6 @@
 
 import array
 import ast
-import itertools
 import os
 from struct import pack, unpack
 from bisect import bisect_left, bisect_right
@@ -143,7 +142,11 @@ class DB(util.LoggedClass):
                 raise self.DBError('your DB version is {} but this software '
                                    'only handles versions {}'
                                    .format(self.db_version, self.DB_VERSIONS))
-            if state['genesis'] != self.coin.GENESIS_HASH:
+            # backwards compat
+            genesis_hash = state['genesis']
+            if isinstance(genesis_hash, bytes):
+                genesis_hash = genesis_hash.decode()
+            if genesis_hash != self.coin.GENESIS_HASH:
                 raise self.DBError('DB genesis hash {} does not match coin {}'
                                    .format(state['genesis_hash'],
                                            self.coin.GENESIS_HASH))
@@ -234,7 +237,7 @@ class DB(util.LoggedClass):
 
         assert len(self.tx_hashes) == blocks_done
         assert len(self.tx_counts) == new_height + 1
-        hashes = b''.join(itertools.chain(*block_tx_hashes))
+        hashes = b''.join(block_tx_hashes)
         assert len(hashes) % 32 == 0
         assert len(hashes) // 32 == txs_done
 

server/env.py

@@ -22,9 +22,9 @@ class Env(LoggedClass):
 
     def __init__(self):
         super().__init__()
-        self.obsolete(['UTXO_MB', 'HIST_MB'])
+        self.obsolete(['UTXO_MB', 'HIST_MB', 'NETWORK'])
         coin_name = self.default('COIN', 'Bitcoin')
-        network = self.default('NETWORK', 'mainnet')
+        network = self.default('NET', 'mainnet')
         self.coin = Coin.lookup_coin_class(coin_name, network)
         self.db_dir = self.required('DB_DIRECTORY')
         self.cache_MB = self.integer('CACHE_MB', 1200)

server/mempool.py

@@ -13,7 +13,6 @@ import time
 from collections import defaultdict
 
 from lib.hash import hash_to_str, hex_str_to_hash
-from lib.tx import Deserializer
 import lib.util as util
 from server.daemon import DaemonError
 
@@ -200,6 +199,7 @@ class MemPool(util.LoggedClass):
         not depend on the result remaining the same are fine.
         '''
         script_hashX = self.coin.hashX_from_script
+        deserializer = self.coin.deserializer()
         db_utxo_lookup = self.db.db_utxo_lookup
         txs = self.txs
 
@@ -207,7 +207,7 @@ class MemPool(util.LoggedClass):
         for tx_hash, raw_tx in raw_tx_map.items():
             if not tx_hash in txs:
                 continue
-            tx = Deserializer(raw_tx).read_tx()
+            tx, _tx_hash = deserializer(raw_tx).read_tx()
 
             # Convert the tx outputs into (hashX, value) pairs
             txout_pairs = [(script_hashX(txout.pk_script), txout.value)
@@ -271,6 +271,7 @@ class MemPool(util.LoggedClass):
         if not hashX in self.hashXs:
             return []
 
+        deserializer = self.coin.deserializer()
         hex_hashes = self.hashXs[hashX]
         raw_txs = await self.daemon.getrawtransactions(hex_hashes)
         result = []
@@ -281,7 +282,7 @@ class MemPool(util.LoggedClass):
             txin_pairs, txout_pairs = item
             tx_fee = (sum(v for hashX, v in txin_pairs)
                       - sum(v for hashX, v in txout_pairs))
-            tx = Deserializer(raw_tx).read_tx()
+            tx, tx_hash = deserializer(raw_tx).read_tx()
             unconfirmed = any(txin.prev_hash in self.txs for txin in tx.inputs)
             result.append((hex_hash, tx_fee, unconfirmed))
         return result

server/protocol.py

@@ -14,7 +14,6 @@ import traceback
 
 from lib.hash import sha256, double_sha256, hash_to_str, hex_str_to_hash
 from lib.jsonrpc import JSONRPC
-from lib.tx import Deserializer
 from server.daemon import DaemonError
 from server.version import VERSION
 
@@ -427,7 +426,8 @@ class ElectrumX(Session):
             if not raw_tx:
                 return None
             raw_tx = bytes.fromhex(raw_tx)
-            tx = Deserializer(raw_tx).read_tx()
+            deserializer = self.coin.deserializer()
+            tx, tx_hash = deserializer(raw_tx).read_tx()
             if index >= len(tx.outputs):
                 return None
             return self.coin.address_from_script(tx.outputs[index].pk_script)

server/version.py

@@ -1 +1 @@
-VERSION = "ElectrumX 0.10.1"
+VERSION = "ElectrumX 0.10.2"