From 184fc615bda9e2072a84a0fe8da599ff505dfd39 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Wed, 1 Feb 2017 19:25:45 +0900 Subject: [PATCH 1/4] Improve clarity about current daemon URL --- server/controller.py | 3 ++- server/daemon.py | 32 +++++++++++++++++++++++--------- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/server/controller.py b/server/controller.py index a1f23a6..f814e6f 100644 --- a/server/controller.py +++ b/server/controller.py @@ -450,6 +450,7 @@ class Controller(util.LoggedClass): def getinfo(self): '''A one-line summary of server state.''' return { + 'daemon': self.daemon.logged_url(), 'daemon_height': self.daemon.cached_height(), 'db_height': self.bp.db_height, 'closing': len([s for s in self.sessions if s.is_closing()]), @@ -599,7 +600,7 @@ class Controller(util.LoggedClass): self.daemon.set_urls(self.env.coin.daemon_urls(daemon_url)) except Exception as e: raise RPCError('an error occured: {}'.format(e)) - return 'set daemon URL to {}'.format(daemon_url) + return 'now using daemon at {}'.format(self.daemon.logged_url()) def rpc_stop(self): '''Shut down the server cleanly.''' diff --git a/server/daemon.py b/server/daemon.py index 44a810c..e5d1fda 100644 --- a/server/daemon.py +++ b/server/daemon.py @@ -42,10 +42,27 @@ class Daemon(util.LoggedClass): '''Set the URLS to the given list, and switch to the first one.''' if not urls: raise DaemonError('no daemon URLs provided') - for url in urls: - self.logger.info('daemon at {}'.format(self.logged_url(url))) self.urls = urls self.url_index = 0 + for n, url in enumerate(urls): + self.logger.info('daemon #{:d} at {}{}' + .format(n + 1, self.logged_url(url), + '' if n else ' (current)')) + + def url(self): + '''Returns the current daemon URL.''' + return self.urls[self.url_index] + + def failover(self): + '''Call to fail-over to the next daemon URL. + + Returns False if there is only one, otherwise True. + ''' + if len(self.urls) > 1: + self.url_index = (self.url_index + 1) % len(self.urls) + self.logger.info('failing over to {}'.format(self.logged_url())) + return True + return False async def _send(self, payload, processor): '''Send a payload to be converted to JSON. @@ -72,8 +89,7 @@ class Daemon(util.LoggedClass): while True: try: async with self.workqueue_semaphore: - url = self.urls[self.url_index] - async with aiohttp.post(url, data=data) as resp: + async with aiohttp.post(self.url(), data=data) as resp: # If bitcoind can't find a tx, for some reason # it returns 500 but fills out the JSON. # Should still return 200 IMO. @@ -99,17 +115,15 @@ class Daemon(util.LoggedClass): except Exception: self.log_error(traceback.format_exc()) - if secs >= max_secs and len(self.urls) > 1: - self.url_index = (self.url_index + 1) % len(self.urls) - logged_url = self.logged_url(self.urls[self.url_index]) - self.logger.info('failing over to {}'.format(logged_url)) + if secs >= max_secs and self.failover(): secs = 1 else: await asyncio.sleep(secs) secs = min(max_secs, secs * 2) - def logged_url(self, url): + def logged_url(self, url=None): '''The host and port part, for logging.''' + url = url or self.url() return url[url.rindex('@') + 1:] async def _send_single(self, method, params=None): From a019fde85305985561de0c0c95fef5426eba851b Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sat, 4 Feb 2017 10:33:45 +0900 Subject: [PATCH 2/4] Show uptime in RPC getinfo result --- lib/util.py | 16 ++++++++++++---- server/controller.py | 9 ++------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/lib/util.py b/lib/util.py index 1fdf1f7..e5c8674 100644 --- a/lib/util.py +++ b/lib/util.py @@ -57,12 +57,20 @@ class cachedproperty(object): return value -def formatted_time(t): +def formatted_time(t, sep=' '): '''Return a number of seconds as a string in days, hours, mins and - secs.''' + maybe secs.''' t = int(t) - return '{:d}d {:02d}h {:02d}m {:02d}s'.format( - t // 86400, (t % 86400) // 3600, (t % 3600) // 60, t % 60) + fmts = (('{:d}d', 86400), ('{:02d}h', 3600), ('{:02d}m', 60)) + parts = [] + for fmt, n in fmts: + val = t // n + if parts or val: + parts.append(fmt.format(val)) + t %= n + if len(parts) < 3: + parts.append('{:02d}s'.format(t)) + return sep.join(parts) def deep_getsizeof(obj): diff --git a/server/controller.py b/server/controller.py index f814e6f..db161c8 100644 --- a/server/controller.py +++ b/server/controller.py @@ -464,6 +464,7 @@ class Controller(util.LoggedClass): 'sessions': self.session_count(), 'subs': self.sub_count(), 'txs_sent': self.txs_sent, + 'uptime': util.formatted_time(time.time() - self.start_time), } def sub_count(self): @@ -515,12 +516,6 @@ class Controller(util.LoggedClass): '''A generator returning lines for a list of sessions. data is the return value of rpc_sessions().''' - - def time_fmt(t): - t = int(t) - return ('{:3d}:{:02d}:{:02d}' - .format(t // 3600, (t % 3600) // 60, t % 60)) - fmt = ('{:<6} {:<5} {:>17} {:>5} {:>5} ' '{:>7} {:>7} {:>7} {:>7} {:>7} {:>9} {:>21}') yield fmt.format('ID', 'Flags', 'Client', 'Reqs', 'Txs', 'Subs', @@ -535,7 +530,7 @@ class Controller(util.LoggedClass): '{:,d}'.format(recv_size // 1024), '{:,d}'.format(send_count), '{:,d}'.format(send_size // 1024), - time_fmt(time), peer) + util.formatted_time(time, sep=''), peer) def session_data(self, for_log): '''Returned to the RPC 'sessions' call.''' From 1856cbe6c066702b69d59adb49a6493c8fce25ef Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sun, 5 Feb 2017 09:41:27 +0900 Subject: [PATCH 3/4] Coin fixes/cleanup (erasmospunk) 2 out of 3 from #118 --- lib/coins.py | 75 ++++++++++++++++++++++----------------- server/block_processor.py | 36 +++++++++++-------- 2 files changed, 64 insertions(+), 47 deletions(-) diff --git a/lib/coins.py b/lib/coins.py index c9fd1d6..bb065cf 100644 --- a/lib/coins.py +++ b/lib/coins.py @@ -32,17 +32,18 @@ class CoinError(Exception): class Coin(object): '''Base class of coin hierarchy.''' - REORG_LIMIT=200 + REORG_LIMIT = 200 # Not sure if these are coin-specific RPC_URL_REGEX = re.compile('.+@[^:]+(:[0-9]+)?') VALUE_PER_COIN = 100000000 - CHUNK_SIZE=2016 + CHUNK_SIZE = 2016 IRC_SERVER = "irc.freenode.net" IRC_PORT = 6667 HASHX_LEN = 11 # Peer discovery - PEER_DEFAULT_PORTS = {'t':'50001', 's':'50002'} + PEER_DEFAULT_PORTS = {'t': '50001', 's': '50002'} PEERS = [] + TX_COUNT_HEIGHT = 0 @classmethod def lookup_coin_class(cls, name, net): @@ -98,11 +99,11 @@ class Coin(object): @util.cachedproperty def address_handlers(cls): return ScriptPubKey.PayToHandlers( - address = cls.P2PKH_address_from_hash160, - script_hash = cls.P2SH_address_from_hash160, - pubkey = cls.P2PKH_address_from_pubkey, - unspendable = lambda : None, - strange = lambda script: None, + address=cls.P2PKH_address_from_hash160, + script_hash=cls.P2SH_address_from_hash160, + pubkey=cls.P2PKH_address_from_pubkey, + unspendable=lambda: None, + strange=lambda script: None, ) @classmethod @@ -269,8 +270,8 @@ class Bitcoin(Coin): P2PKH_VERBYTE = 0x00 P2SH_VERBYTE = 0x05 WIF_BYTE = 0x80 - GENESIS_HASH=('000000000019d6689c085ae165831e93' - '4ff763ae46a2a6c172b3f1b60a8ce26f') + GENESIS_HASH = ('000000000019d6689c085ae165831e93' + '4ff763ae46a2a6c172b3f1b60a8ce26f') TX_COUNT = 156335304 TX_COUNT_HEIGHT = 429972 TX_PER_BLOCK = 1800 @@ -302,15 +303,15 @@ class BitcoinTestnet(Bitcoin): P2PKH_VERBYTE = 0x6f P2SH_VERBYTE = 0xc4 WIF_BYTE = 0xef - GENESIS_HASH=('000000000933ea01ad0ee984209779ba' - 'aec3ced90fa3f408719526f8d77f4943') + GENESIS_HASH = ('000000000933ea01ad0ee984209779ba' + 'aec3ced90fa3f408719526f8d77f4943') REORG_LIMIT = 2000 TX_COUNT = 12242438 TX_COUNT_HEIGHT = 1035428 TX_PER_BLOCK = 21 IRC_PREFIX = "ET_" RPC_PORT = 18332 - PEER_DEFAULT_PORTS = {'t':'51001', 's':'51002'} + PEER_DEFAULT_PORTS = {'t': '51001', 's': '51002'} PEERS = [ 'electrum.akinbo.org s t', 'he36kyperp3kbuxu.onion s t', @@ -319,6 +320,7 @@ class BitcoinTestnet(Bitcoin): 'testnet.not.fyi s t', ] + class BitcoinTestnetSegWit(BitcoinTestnet): '''Bitcoin Testnet for Core bitcoind >= 0.13.1. @@ -343,8 +345,8 @@ class Litecoin(Coin): P2PKH_VERBYTE = 0x30 P2SH_VERBYTE = 0x05 WIF_BYTE = 0xb0 - GENESIS_HASH=('12a765e31ffd4059bada1e25190f6e98' - 'c99d9714d334efa41a195a7e7e04bfe2') + GENESIS_HASH = ('12a765e31ffd4059bada1e25190f6e98' + 'c99d9714d334efa41a195a7e7e04bfe2') TX_COUNT = 8908766 TX_COUNT_HEIGHT = 1105256 TX_PER_BLOCK = 10 @@ -361,7 +363,8 @@ class LitecoinTestnet(Litecoin): P2PKH_VERBYTE = 0x6f P2SH_VERBYTE = 0xc4 WIF_BYTE = 0xef - # Some details missing... + GENESIS_HASH = ('f5ae71e26c74beacc88382716aced69c' + 'ddf3dffff24f384e1808905e0188f68f') # Source: namecoin.org @@ -374,9 +377,11 @@ class Namecoin(Coin): P2PKH_VERBYTE = 0x34 P2SH_VERBYTE = 0x0d WIF_BYTE = 0xe4 + GENESIS_HASH = ('000000000062b72c5e2ceb45fbc8587e' + '807c155b0da735e6483dfba2f0a9c770') -class NamecoinTestnet(Coin): +class NamecoinTestnet(Namecoin): NAME = "Namecoin" SHORTNAME = "XNM" NET = "testnet" @@ -385,6 +390,7 @@ class NamecoinTestnet(Coin): P2PKH_VERBYTE = 0x6f P2SH_VERBYTE = 0xc4 WIF_BYTE = 0xef + # TODO add GENESIS_HASH # For DOGE there is disagreement across sites like bip32.org and @@ -398,9 +404,11 @@ class Dogecoin(Coin): P2PKH_VERBYTE = 0x1e P2SH_VERBYTE = 0x16 WIF_BYTE = 0x9e + GENESIS_HASH = ('1a91e3dace36e2be3bf030a65679fe82' + '1aa1d6ef92e7c9902eb318182c355691') -class DogecoinTestnet(Coin): +class DogecoinTestnet(Dogecoin): NAME = "Dogecoin" SHORTNAME = "XDT" NET = "testnet" @@ -409,6 +417,8 @@ class DogecoinTestnet(Coin): P2PKH_VERBYTE = 0x71 P2SH_VERBYTE = 0xc4 WIF_BYTE = 0xf1 + GENESIS_HASH = ('bb0a78264637406b6360aad926284d54' + '4d7049f45189db5664f3c4d07350559e') # Source: https://github.com/dashpay/dash @@ -436,6 +446,7 @@ class Dash(Coin): import x11_hash return x11_hash.getPoWHash(header) + class DashTestnet(Dash): SHORTNAME = "tDASH" NET = "testnet" @@ -462,8 +473,8 @@ class Argentum(Coin): P2PKH_VERBYTE = 0x17 P2SH_VERBYTE = 0x05 WIF_BYTE = 0x97 - GENESIS_HASH=('88c667bc63167685e4e4da058fffdfe8' - 'e007e5abffd6855de52ad59df7bb0bb2') + GENESIS_HASH = ('88c667bc63167685e4e4da058fffdfe8' + 'e007e5abffd6855de52ad59df7bb0bb2') TX_COUNT = 2263089 TX_COUNT_HEIGHT = 2050260 TX_PER_BLOCK = 2000 @@ -473,14 +484,14 @@ class Argentum(Coin): class ArgentumTestnet(Argentum): - SHORTNAME = "XRG" - NET = "testnet" - XPUB_VERBYTES = bytes.fromhex("043587cf") - XPRV_VERBYTES = bytes.fromhex("04358394") - P2PKH_VERBYTE = 0x6f - P2SH_VERBYTE = 0xc4 - WIF_BYTE = 0xef - REORG_LIMIT = 2000 + SHORTNAME = "XRG" + NET = "testnet" + XPUB_VERBYTES = bytes.fromhex("043587cf") + XPRV_VERBYTES = bytes.fromhex("04358394") + P2PKH_VERBYTE = 0x6f + P2SH_VERBYTE = 0xc4 + WIF_BYTE = 0xef + REORG_LIMIT = 2000 class DigiByte(Coin): @@ -492,8 +503,8 @@ class DigiByte(Coin): P2PKH_VERBYTE = 0x1E P2SH_VERBYTE = 0x05 WIF_BYTE = 0x80 - GENESIS_HASH=('7497ea1b465eb39f1c8f507bc877078f' - 'e016d6fcb6dfad3a64c98dcc6e1e8496') + GENESIS_HASH = ('7497ea1b465eb39f1c8f507bc877078f' + 'e016d6fcb6dfad3a64c98dcc6e1e8496') TX_COUNT = 1046018 TX_COUNT_HEIGHT = 1435000 TX_PER_BLOCK = 1000 @@ -509,8 +520,8 @@ class DigiByteTestnet(DigiByte): P2PKH_VERBYTE = 0x6f P2SH_VERBYTE = 0xc4 WIF_BYTE = 0xef - GENESIS_HASH=('b5dca8039e300198e5fe7cd23bdd1728' - 'e2a444af34c447dbd0916fa3430a68c2') + GENESIS_HASH = ('b5dca8039e300198e5fe7cd23bdd1728' + 'e2a444af34c447dbd0916fa3430a68c2') IRC_PREFIX = "DET_" IRC_CHANNEL = "#electrum-dgb" RPC_PORT = 15022 diff --git a/server/block_processor.py b/server/block_processor.py index 74ae878..9d7d6a6 100644 --- a/server/block_processor.py +++ b/server/block_processor.py @@ -383,26 +383,32 @@ class BlockProcessor(server.db.DB): # Catch-up stats if self.utxo_db.for_sync: - daemon_height = self.daemon.cached_height() tx_per_sec = int(self.tx_count / self.wall_time) this_tx_per_sec = 1 + int(tx_diff / (self.last_flush - last_flush)) - if self.height > self.coin.TX_COUNT_HEIGHT: - tx_est = (daemon_height - self.height) * self.coin.TX_PER_BLOCK - else: - tx_est = ((daemon_height - self.coin.TX_COUNT_HEIGHT) - * self.coin.TX_PER_BLOCK - + (self.coin.TX_COUNT - self.tx_count)) - - # Damp the enthusiasm - realism = 2.0 - 0.9 * self.height / self.coin.TX_COUNT_HEIGHT - tx_est *= max(realism, 1.0) - self.logger.info('tx/sec since genesis: {:,d}, ' 'since last flush: {:,d}' .format(tx_per_sec, this_tx_per_sec)) - self.logger.info('sync time: {} ETA: {}' - .format(formatted_time(self.wall_time), - formatted_time(tx_est / this_tx_per_sec))) + if self.coin.TX_COUNT_HEIGHT > 0: + daemon_height = self.daemon.cached_height() + if self.height > self.coin.TX_COUNT_HEIGHT: + tx_est = (daemon_height - self.height) * self.coin.TX_PER_BLOCK + else: + tx_est = ((daemon_height - self.coin.TX_COUNT_HEIGHT) + * self.coin.TX_PER_BLOCK + + (self.coin.TX_COUNT - self.tx_count)) + + # Damp the enthusiasm + realism = 2.0 - 0.9 * self.height / self.coin.TX_COUNT_HEIGHT + tx_est *= max(realism, 1.0) + + + self.logger.info('sync time: {} ETA: {}' + .format(formatted_time(self.wall_time), + formatted_time(tx_est / this_tx_per_sec))) + else: + self.logger.info('sync time: {}' + .format(formatted_time(self.wall_time))) + def fs_flush(self): '''Flush the things stored on the filesystem.''' From ba45e001fa9ab5456b929bf073afbdba8f0b34b2 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sun, 5 Feb 2017 09:49:46 +0900 Subject: [PATCH 4/4] Prepare 0.10.17 --- README.rst | 64 ++++++++++++++++++++++++++--------------------- server/version.py | 2 +- 2 files changed, 37 insertions(+), 29 deletions(-) diff --git a/README.rst b/README.rst index 2efe57a..eaca758 100644 --- a/README.rst +++ b/README.rst @@ -16,6 +16,34 @@ Getting Started See `docs/HOWTO.rst`_. +Features +======== + +- Efficient, lightweight reimplementation of electrum-server +- Efficient synchronization of bitcoin mainnet from Genesis. Recent + hardware should synchronize in well under 24 hours. The fastest + time to height 448k (mid January 2017) reported is under 4h 30m. On + the same hardware JElectrum would take around 4 days and + electrum-server probably around 1 month. +- The full Electrum protocol is implemented. The only exception is + the blockchain.address.get_proof RPC call, which is not used by + Electrum GUI clients, and can only be invoked from the command line. +- Various configurable means of controlling resource consumption and + handling denial of service attacks. These include maximum + connection counts, subscription limits per-connection and across all + connections, maximum response size, per-session bandwidth limits, + and session timeouts. +- Minimal resource usage once caught up and serving clients; tracking the + transaction mempool appears to be the most expensive part. +- Fully asynchronous processing of new blocks, mempool updates, and + client requests. Busy clients should not noticeably impede other + clients' requests and notifications, nor the processing of incoming + blocks and mempool updates. +- Daemon failover. More than one daemon can be specified, and + ElectrumX will failover round-robin style if the current one fails + for any reason. +- Coin abstraction makes compatible altcoin and testnet support easy. + Motivation ========== @@ -45,34 +73,6 @@ that could easily be reused for those alts that are reasonably compatible with Bitcoin. Such an abstraction is also useful for testnets. -Features -======== - -- The full Electrum protocol is implemented. The only exception is - the blockchain.address.get_proof RPC call, which is not used by - Electrum GUI clients, and can only be invoked from the command line. -- Efficient synchronization from Genesis. Recent hardware should - synchronize in well under 24 hours, possibly much faster for recent - CPUs or if you have an SSD. The fastest time to height 439k (mid - November 2016) reported is under 5 hours. For comparison, JElectrum - would take around 4 days, and electrum-server probably around 1 - month, on the same hardware. -- Various configurable means of controlling resource consumption and - handling denial of service attacks. These include maximum - connection counts, subscription limits per-connection and across all - connections, maximum response size, per-session bandwidth limits, - and session timeouts. -- Minimal resource usage once caught up and serving clients; tracking the - transaction mempool appears to be the most expensive part. -- Fully asynchronous processing of new blocks, mempool updates, and - client requests. Busy clients should not noticeably impede other - clients' requests and notifications, nor the processing of incoming - blocks and mempool updates. -- Daemon failover. More than one daemon can be specified, and - ElectrumX will failover round-robin style if the current one fails - for any reason. -- Coin abstraction makes compatible altcoin and testnet support easy. - Implementation ============== @@ -135,6 +135,14 @@ version prior to the release of 1.0. ChangeLog ========= +Version 0.10.17 +--------------- + +Minor upgrade + +* added current daemon URL and uptime to getinfo RPC call +* altcoin cleanups / fixes (erasmospunk) + Version 0.10.16 --------------- diff --git a/server/version.py b/server/version.py index 2b71e77..96f3e09 100644 --- a/server/version.py +++ b/server/version.py @@ -1 +1 @@ -VERSION = "ElectrumX 0.10.16" +VERSION = "ElectrumX 0.10.17"