diff --git a/README.rst b/README.rst index d03930c..64d6d3b 100644 --- a/README.rst +++ b/README.rst @@ -115,7 +115,6 @@ Roadmap Pre-1.0 =============== - minor code cleanups. -- support bitcoin testnet with Satoshi bitcoind 0.13.1 - implement simple protocol to discover peers without resorting to IRC. This may slip to post 1.0 @@ -142,6 +141,24 @@ version prior to the release of 1.0. ChangeLog ========= +Version 0.10.2 +-------------- + +* Note the **NETWORK** environment variable was renamed **NET** to + bring it into line with lib/coins.py. +* The genesis hash is now compared with the genesis hash expected by + **COIN** and **NET**. This sanity check was not done previously, so + you could easily be syncing to a network daemon different to what + you thought. +* SegWit-compatible testnet support for bitcoin core versions 0.13.1 + or higher. Resolves issue `#92#`. Testnet worked with prior + versions of ElectrumX as long as you used an older bitcoind too, + such as 0.13.0 or Bitcoin Unlimited. + + **Note**: for testnet, you need to set *NET** to *testnet-segwit* if + using recent RPC incompatible core bitcoinds, or *testnet* if using + older RPC compatible bitcoinds. + Version 0.10.1 -------------- @@ -167,6 +184,12 @@ variables to use roughly the same amount of memory. For now this code should be considered experimental; if you want stability please stick with the 0.9 series. +Version 0.9.23 +-------------- + +* Backport of the fix for issue `#94#` - stale references to old + sessions. This would effectively memory and network handles. + Version 0.9.22 -------------- @@ -334,6 +357,7 @@ Version 0.9.0 .. _#75: https://github.com/kyuupichan/electrumx/issues/75 .. _#88: https://github.com/kyuupichan/electrumx/issues/88 .. _#89: https://github.com/kyuupichan/electrumx/issues/89 +.. _#92: https://github.com/kyuupichan/electrumx/issues/92 .. _#93: https://github.com/kyuupichan/electrumx/issues/93 .. _#94: https://github.com/kyuupichan/electrumx/issues/94 .. _docs/HOWTO.rst: https://github.com/kyuupichan/electrumx/blob/master/docs/HOWTO.rst diff --git a/docs/ENVIRONMENT.rst b/docs/ENVIRONMENT.rst index 8d9ff9c..5a46695 100644 --- a/docs/ENVIRONMENT.rst +++ b/docs/ENVIRONMENT.rst @@ -31,7 +31,7 @@ These environment variables are always required: The leading `http://` is optional, as is the trailing slash. The `:port` part is also optional and will default to the standard RPC - port for **COIN** and **NETWORK** if omitted. + port for **COIN** and **NET** if omitted. For the `run` script @@ -58,7 +58,7 @@ These environment variables are optional: Must be a *NAME* from one of the **Coin** classes in `lib/coins.py`_. Defaults to `Bitcoin`. -* **NETWORK** +* **NET** Must be a *NET* from one of the **Coin** classes in `lib/coins.py`_. Defaults to `mainnet`. @@ -77,7 +77,7 @@ These environment variables are optional: The maximum number of blocks to be able to handle in a chain reorganisation. ElectrumX retains some fairly compact undo information for this many blocks in levelDB. The default is a - function of **COIN** and **NETWORK**; for Bitcoin mainnet it is 200. + function of **COIN** and **NET**; for Bitcoin mainnet it is 200. * **HOST** @@ -98,7 +98,7 @@ These environment variables are optional: ElectrumX will listen on this port for local RPC connections. ElectrumX listens for RPC connections unless this is explicitly set - to blank. The default is appropriate for **COIN** and **NETWORK** + to blank. The default is appropriate for **COIN** and **NET** (e.g., 8000 for Bitcoin mainnet) if not set. * **DONATION_ADDRESS** @@ -223,7 +223,7 @@ connectivity on IRC: The nick to use when connecting to IRC. The default is a hash of **REPORT_HOST**. Either way a prefix will be prepended depending on - **COIN** and **NETWORK**. + **COIN** and **NET**. * **REPORT_HOST** diff --git a/lib/coins.py b/lib/coins.py index 989bec1..9ad774e 100644 --- a/lib/coins.py +++ b/lib/coins.py @@ -21,8 +21,8 @@ import sys from lib.hash import Base58, hash160, ripemd160, double_sha256, hash_to_str from lib.script import ScriptPubKey -from lib.tx import Deserializer -from lib.util import cachedproperty, subclasses +from lib.tx import Deserializer, DeserializerSegWit +import lib.util as util class CoinError(Exception): @@ -46,7 +46,7 @@ class Coin(object): '''Return a coin class given name and network. Raise an exception if unrecognised.''' - for coin in subclasses(Coin): + for coin in util.subclasses(Coin): if (coin.NAME.lower() == name.lower() and coin.NET.lower() == net.lower()): return coin @@ -70,6 +70,20 @@ class Coin(object): def daemon_urls(cls, urls): return [cls.sanitize_url(url) for url in urls.split(',')] + @classmethod + def genesis_block(cls, block): + '''Check the Genesis block is the right one for this coin. + + Return the block less its unspendable coinbase. + ''' + header = block[:cls.header_len(0)] + header_hex_hash = hash_to_str(cls.header_hash(header)) + if header_hex_hash != cls.GENESIS_HASH: + raise CoinError('genesis block has hash {} expected {}' + .format(header_hex_hash, cls.GENESIS_HASH)) + + return header + bytes(1) + @classmethod def hashX_from_script(cls, script): '''Returns a hashX from a script.''' @@ -78,7 +92,7 @@ class Coin(object): return None return sha256(script).digest()[:cls.HASHX_LEN] - @cachedproperty + @util.cachedproperty def address_handlers(cls): return ScriptPubKey.PayToHandlers( address = cls.P2PKH_address_from_hash160, @@ -204,11 +218,14 @@ class Coin(object): @classmethod def read_block(cls, block, height): - '''Return a tuple (header, tx_hashes, txs) given a raw block at - the given height.''' + '''Returns a pair (header, tx_list) given a raw block and height. + + tx_list is a list of (deserialized_tx, tx_hash) pairs. + ''' + deserializer = cls.deserializer() hlen = cls.header_len(height) header, rest = block[:hlen], block[hlen:] - return (header, ) + Deserializer(rest).read_block() + return (header, deserializer(rest).read_block()) @classmethod def decimal_value(cls, value): @@ -234,6 +251,10 @@ class Coin(object): 'nonce': nonce, } + @classmethod + def deserializer(cls): + return Deserializer + class Bitcoin(Coin): NAME = "Bitcoin" @@ -244,8 +265,8 @@ class Bitcoin(Coin): P2PKH_VERBYTE = 0x00 P2SH_VERBYTE = 0x05 WIF_BYTE = 0x80 - GENESIS_HASH=(b'000000000019d6689c085ae165831e93' - b'4ff763ae46a2a6c172b3f1b60a8ce26f') + GENESIS_HASH=('000000000019d6689c085ae165831e93' + '4ff763ae46a2a6c172b3f1b60a8ce26f') TX_COUNT = 156335304 TX_COUNT_HEIGHT = 429972 TX_PER_BLOCK = 1800 @@ -262,13 +283,29 @@ class BitcoinTestnet(Bitcoin): P2PKH_VERBYTE = 0x6f P2SH_VERBYTE = 0xc4 WIF_BYTE = 0xef - GENESIS_HASH=(b'000000000933ea01ad0ee984209779ba' - b'aec3ced90fa3f408719526f8d77f4943') + GENESIS_HASH=('000000000933ea01ad0ee984209779ba' + 'aec3ced90fa3f408719526f8d77f4943') REORG_LIMIT = 2000 TX_COUNT = 12242438 TX_COUNT_HEIGHT = 1035428 TX_PER_BLOCK = 21 IRC_PREFIX = "ET_" + RPC_PORT = 18332 + + +class BitcoinTestnetSegWit(BitcoinTestnet): + '''Bitcoin Testnet for Core bitcoind >= 0.13.1. + + Unfortunately 0.13.1 broke backwards compatibility of the RPC + interface's TX serialization, SegWit transactions serialize + differently than with earlier versions. If you are using such a + bitcoind on testnet, you must use this class as your "COIN". + ''' + NET = "testnet-segwit" + + @classmethod + def deserializer(cls): + return DeserializerSegWit class Litecoin(Coin): @@ -280,8 +317,8 @@ class Litecoin(Coin): P2PKH_VERBYTE = 0x30 P2SH_VERBYTE = 0x05 WIF_BYTE = 0xb0 - GENESIS_HASH=(b'000000000019d6689c085ae165831e93' - b'4ff763ae46a2a6c172b3f1b60a8ce26f') + GENESIS_HASH=('000000000019d6689c085ae165831e93' + '4ff763ae46a2a6c172b3f1b60a8ce26f') TX_COUNT = 8908766 TX_COUNT_HEIGHT = 1105256 TX_PER_BLOCK = 10 @@ -355,8 +392,8 @@ class Dash(Coin): NET = "mainnet" XPUB_VERBYTES = bytes.fromhex("02fe52cc") XPRV_VERBYTES = bytes.fromhex("02fe52f8") - GENESIS_HASH = (b'00000ffd590b1485b3caadc19b22e637' - b'9c733355108f107a430458cdf3407ab6') + GENESIS_HASH = ('00000ffd590b1485b3caadc19b22e637' + '9c733355108f107a430458cdf3407ab6') P2PKH_VERBYTE = 0x4c P2SH_VERBYTE = 0x10 WIF_BYTE = 0xcc @@ -378,8 +415,8 @@ class DashTestnet(Dash): NET = "testnet" XPUB_VERBYTES = bytes.fromhex("3a805837") XPRV_VERBYTES = bytes.fromhex("3a8061a0") - GENESIS_HASH = (b'00000bafbc94add76cb75e2ec9289483' - b'7288a481e5c005f6563d91623bf8bc2c') + GENESIS_HASH = ('00000bafbc94add76cb75e2ec9289483' + '7288a481e5c005f6563d91623bf8bc2c') P2PKH_VERBYTE = 0x8c P2SH_VERBYTE = 0x13 WIF_BYTE = 0xef diff --git a/lib/tx.py b/lib/tx.py index 2fb86cd..565595e 100644 --- a/lib/tx.py +++ b/lib/tx.py @@ -72,28 +72,25 @@ class Deserializer(object): self.cursor = 0 def read_tx(self): + '''Return a (Deserialized TX, TX_HASH) pair. + + The hash needs to be reversed for human display; for efficiency + we process it in the natural serialized order. + ''' + start = self.cursor return Tx( self._read_le_int32(), # version self._read_inputs(), # inputs self._read_outputs(), # outputs self._read_le_uint32() # locktime - ) + ), double_sha256(self.binary[start:self.cursor]) def read_block(self): - tx_hashes = [] - txs = [] - binary = self.binary - hash = double_sha256 + '''Returns a list of (deserialized_tx, tx_hash) pairs.''' read_tx = self.read_tx - append_hash = tx_hashes.append - for n in range(self._read_varint()): - start = self.cursor - txs.append(read_tx()) - # Note this hash needs to be reversed for human display - # For efficiency we store it in the natural serialized order - append_hash(hash(binary[start:self.cursor])) - assert self.cursor == len(binary) - return tx_hashes, txs + txs = [read_tx() for n in range(self._read_varint())] + assert self.cursor == len(self.binary) + return txs def _read_inputs(self): read_input = self._read_input @@ -161,3 +158,62 @@ class Deserializer(object): result, = unpack_from('= self.cache_MB or hist_MB >= self.cache_MB // 5: self.flush(utxo_MB >= self.cache_MB * 4 // 5) - def fs_advance_block(self, header, tx_hashes, txs): + def fs_advance_block(self, header, txs): '''Update unflushed FS state for a new block.''' prior_tx_count = self.tx_counts[-1] if self.tx_counts else 0 # Cache the new header, tx hashes and cumulative tx count self.headers.append(header) - self.tx_hashes.append(tx_hashes) + self.tx_hashes.append(b''.join(tx_hash for tx, tx_hash in txs)) self.tx_counts.append(prior_tx_count + len(txs)) def advance_block(self, block, touched): - header, tx_hashes, txs = self.coin.read_block(block, self.height + 1) + header, txs = self.coin.read_block(block, self.height + 1) if self.tip != self.coin.header_prevhash(header): raise ChainReorg - self.fs_advance_block(header, tx_hashes, txs) + self.fs_advance_block(header, txs) self.tip = self.coin.header_hash(header) self.height += 1 - undo_info = self.advance_txs(tx_hashes, txs, touched) + undo_info = self.advance_txs(txs, touched) if self.daemon.cached_height() - self.height <= self.env.reorg_limit: self.write_undo_info(self.height, b''.join(undo_info)) - def advance_txs(self, tx_hashes, txs, touched): + def advance_txs(self, txs, touched): undo_info = [] # Use local vars for speed in the loops @@ -492,7 +496,7 @@ class BlockProcessor(server.db.DB): spend_utxo = self.spend_utxo undo_info_append = undo_info.append - for tx, tx_hash in zip(txs, tx_hashes): + for tx, tx_hash in txs: hashXs = set() add_hashX = hashXs.add tx_numb = s_pack('= 0 self.height -= 1 @@ -553,7 +557,7 @@ class BlockProcessor(server.db.DB): touched.discard(None) self.backup_flush(touched) - def backup_txs(self, tx_hashes, txs, touched): + def backup_txs(self, txs, touched): # Prevout values, in order down the block (coinbase first if present) # undo_info is in reverse block order undo_info = self.read_undo_info(self.height) @@ -569,10 +573,7 @@ class BlockProcessor(server.db.DB): script_hashX = self.coin.hashX_from_script undo_entry_len = 12 + self.coin.HASHX_LEN - rtxs = reversed(txs) - rtx_hashes = reversed(tx_hashes) - - for tx_hash, tx in zip(rtx_hashes, rtxs): + for tx, tx_hash in reversed(txs): for idx, txout in enumerate(tx.outputs): # Spend the TX outputs. Be careful with unspendable # outputs - we didn't save those in the first place. diff --git a/server/db.py b/server/db.py index 0c653e3..c124594 100644 --- a/server/db.py +++ b/server/db.py @@ -10,7 +10,6 @@ import array import ast -import itertools import os from struct import pack, unpack from bisect import bisect_left, bisect_right @@ -143,7 +142,11 @@ class DB(util.LoggedClass): raise self.DBError('your DB version is {} but this software ' 'only handles versions {}' .format(self.db_version, self.DB_VERSIONS)) - if state['genesis'] != self.coin.GENESIS_HASH: + # backwards compat + genesis_hash = state['genesis'] + if isinstance(genesis_hash, bytes): + genesis_hash = genesis_hash.decode() + if genesis_hash != self.coin.GENESIS_HASH: raise self.DBError('DB genesis hash {} does not match coin {}' .format(state['genesis_hash'], self.coin.GENESIS_HASH)) @@ -234,7 +237,7 @@ class DB(util.LoggedClass): assert len(self.tx_hashes) == blocks_done assert len(self.tx_counts) == new_height + 1 - hashes = b''.join(itertools.chain(*block_tx_hashes)) + hashes = b''.join(block_tx_hashes) assert len(hashes) % 32 == 0 assert len(hashes) // 32 == txs_done diff --git a/server/env.py b/server/env.py index 4ce5c14..647078d 100644 --- a/server/env.py +++ b/server/env.py @@ -22,9 +22,9 @@ class Env(LoggedClass): def __init__(self): super().__init__() - self.obsolete(['UTXO_MB', 'HIST_MB']) + self.obsolete(['UTXO_MB', 'HIST_MB', 'NETWORK']) coin_name = self.default('COIN', 'Bitcoin') - network = self.default('NETWORK', 'mainnet') + network = self.default('NET', 'mainnet') self.coin = Coin.lookup_coin_class(coin_name, network) self.db_dir = self.required('DB_DIRECTORY') self.cache_MB = self.integer('CACHE_MB', 1200) diff --git a/server/mempool.py b/server/mempool.py index 09b9fd7..e61bf0b 100644 --- a/server/mempool.py +++ b/server/mempool.py @@ -13,7 +13,6 @@ import time from collections import defaultdict from lib.hash import hash_to_str, hex_str_to_hash -from lib.tx import Deserializer import lib.util as util from server.daemon import DaemonError @@ -200,6 +199,7 @@ class MemPool(util.LoggedClass): not depend on the result remaining the same are fine. ''' script_hashX = self.coin.hashX_from_script + deserializer = self.coin.deserializer() db_utxo_lookup = self.db.db_utxo_lookup txs = self.txs @@ -207,7 +207,7 @@ class MemPool(util.LoggedClass): for tx_hash, raw_tx in raw_tx_map.items(): if not tx_hash in txs: continue - tx = Deserializer(raw_tx).read_tx() + tx, _tx_hash = deserializer(raw_tx).read_tx() # Convert the tx outputs into (hashX, value) pairs txout_pairs = [(script_hashX(txout.pk_script), txout.value) @@ -271,6 +271,7 @@ class MemPool(util.LoggedClass): if not hashX in self.hashXs: return [] + deserializer = self.coin.deserializer() hex_hashes = self.hashXs[hashX] raw_txs = await self.daemon.getrawtransactions(hex_hashes) result = [] @@ -281,7 +282,7 @@ class MemPool(util.LoggedClass): txin_pairs, txout_pairs = item tx_fee = (sum(v for hashX, v in txin_pairs) - sum(v for hashX, v in txout_pairs)) - tx = Deserializer(raw_tx).read_tx() + tx, tx_hash = deserializer(raw_tx).read_tx() unconfirmed = any(txin.prev_hash in self.txs for txin in tx.inputs) result.append((hex_hash, tx_fee, unconfirmed)) return result diff --git a/server/protocol.py b/server/protocol.py index 1b0516f..60b7a45 100644 --- a/server/protocol.py +++ b/server/protocol.py @@ -14,7 +14,6 @@ import traceback from lib.hash import sha256, double_sha256, hash_to_str, hex_str_to_hash from lib.jsonrpc import JSONRPC -from lib.tx import Deserializer from server.daemon import DaemonError from server.version import VERSION @@ -427,7 +426,8 @@ class ElectrumX(Session): if not raw_tx: return None raw_tx = bytes.fromhex(raw_tx) - tx = Deserializer(raw_tx).read_tx() + deserializer = self.coin.deserializer() + tx, tx_hash = deserializer(raw_tx).read_tx() if index >= len(tx.outputs): return None return self.coin.address_from_script(tx.outputs[index].pk_script) diff --git a/server/version.py b/server/version.py index 51c5fd6..34099ad 100644 --- a/server/version.py +++ b/server/version.py @@ -1 +1 @@ -VERSION = "ElectrumX 0.10.1" +VERSION = "ElectrumX 0.10.2"