Merge branch 'develop'

README.rst

@@ -16,6 +16,34 @@ Getting Started
 
 See `docs/HOWTO.rst`_.
 
+Features
+========
+
+- Efficient, lightweight reimplementation of electrum-server
+- Efficient synchronization of bitcoin mainnet from Genesis.  Recent
+  hardware should synchronize in well under 24 hours.  The fastest
+  time to height 448k (mid January 2017) reported is under 4h 30m.  On
+  the same hardware JElectrum would take around 4 days and
+  electrum-server probably around 1 month.
+- The full Electrum protocol is implemented.  The only exception is
+  the blockchain.address.get_proof RPC call, which is not used by
+  Electrum GUI clients, and can only be invoked from the command line.
+- Various configurable means of controlling resource consumption and
+  handling denial of service attacks.  These include maximum
+  connection counts, subscription limits per-connection and across all
+  connections, maximum response size, per-session bandwidth limits,
+  and session timeouts.
+- Minimal resource usage once caught up and serving clients; tracking the
+  transaction mempool appears to be the most expensive part.
+- Fully asynchronous processing of new blocks, mempool updates, and
+  client requests.  Busy clients should not noticeably impede other
+  clients' requests and notifications, nor the processing of incoming
+  blocks and mempool updates.
+- Daemon failover.  More than one daemon can be specified, and
+  ElectrumX will failover round-robin style if the current one fails
+  for any reason.
+- Coin abstraction makes compatible altcoin and testnet support easy.
+
 Motivation
 ==========
 
@@ -45,34 +73,6 @@ that could easily be reused for those alts that are reasonably
 compatible with Bitcoin.  Such an abstraction is also useful for
 testnets.
 
-Features
-========
-
-- The full Electrum protocol is implemented.  The only exception is
-  the blockchain.address.get_proof RPC call, which is not used by
-  Electrum GUI clients, and can only be invoked from the command line.
-- Efficient synchronization from Genesis.  Recent hardware should
-  synchronize in well under 24 hours, possibly much faster for recent
-  CPUs or if you have an SSD.  The fastest time to height 439k (mid
-  November 2016) reported is under 5 hours.  For comparison, JElectrum
-  would take around 4 days, and electrum-server probably around 1
-  month, on the same hardware.
-- Various configurable means of controlling resource consumption and
-  handling denial of service attacks.  These include maximum
-  connection counts, subscription limits per-connection and across all
-  connections, maximum response size, per-session bandwidth limits,
-  and session timeouts.
-- Minimal resource usage once caught up and serving clients; tracking the
-  transaction mempool appears to be the most expensive part.
-- Fully asynchronous processing of new blocks, mempool updates, and
-  client requests.  Busy clients should not noticeably impede other
-  clients' requests and notifications, nor the processing of incoming
-  blocks and mempool updates.
-- Daemon failover.  More than one daemon can be specified, and
-  ElectrumX will failover round-robin style if the current one fails
-  for any reason.
-- Coin abstraction makes compatible altcoin and testnet support easy.
-
 Implementation
 ==============
 
@@ -135,6 +135,14 @@ version prior to the release of 1.0.
 ChangeLog
 =========
 
+Version 0.10.17
+---------------
+
+Minor upgrade
+
+* added current daemon URL and uptime to getinfo RPC call
+* altcoin cleanups / fixes (erasmospunk)
+
 Version 0.10.16
 ---------------
 

lib/coins.py

@@ -43,6 +43,7 @@ class Coin(object):
     # Peer discovery
     PEER_DEFAULT_PORTS = {'t': '50001', 's': '50002'}
     PEERS = []
+    TX_COUNT_HEIGHT = 0
 
     @classmethod
     def lookup_coin_class(cls, name, net):
@@ -319,6 +320,7 @@ class BitcoinTestnet(Bitcoin):
         'testnet.not.fyi s t',
     ]
 
+
 class BitcoinTestnetSegWit(BitcoinTestnet):
     '''Bitcoin Testnet for Core bitcoind >= 0.13.1.
 
@@ -361,7 +363,8 @@ class LitecoinTestnet(Litecoin):
     P2PKH_VERBYTE = 0x6f
     P2SH_VERBYTE = 0xc4
     WIF_BYTE = 0xef
-    # Some details missing...
+    GENESIS_HASH = ('f5ae71e26c74beacc88382716aced69c'
+                    'ddf3dffff24f384e1808905e0188f68f')
 
 
 # Source: namecoin.org
@@ -374,9 +377,11 @@ class Namecoin(Coin):
     P2PKH_VERBYTE = 0x34
     P2SH_VERBYTE = 0x0d
     WIF_BYTE = 0xe4
+    GENESIS_HASH = ('000000000062b72c5e2ceb45fbc8587e'
+                    '807c155b0da735e6483dfba2f0a9c770')
 
 
-class NamecoinTestnet(Coin):
+class NamecoinTestnet(Namecoin):
     NAME = "Namecoin"
     SHORTNAME = "XNM"
     NET = "testnet"
@@ -385,6 +390,7 @@ class NamecoinTestnet(Coin):
     P2PKH_VERBYTE = 0x6f
     P2SH_VERBYTE = 0xc4
     WIF_BYTE = 0xef
+    # TODO add GENESIS_HASH
 
 
 # For DOGE there is disagreement across sites like bip32.org and
@@ -398,9 +404,11 @@ class Dogecoin(Coin):
     P2PKH_VERBYTE = 0x1e
     P2SH_VERBYTE = 0x16
     WIF_BYTE = 0x9e
+    GENESIS_HASH = ('1a91e3dace36e2be3bf030a65679fe82'
+                    '1aa1d6ef92e7c9902eb318182c355691')
 
 
-class DogecoinTestnet(Coin):
+class DogecoinTestnet(Dogecoin):
     NAME = "Dogecoin"
     SHORTNAME = "XDT"
     NET = "testnet"
@@ -409,6 +417,8 @@ class DogecoinTestnet(Coin):
     P2PKH_VERBYTE = 0x71
     P2SH_VERBYTE = 0xc4
     WIF_BYTE = 0xf1
+    GENESIS_HASH = ('bb0a78264637406b6360aad926284d54'
+                    '4d7049f45189db5664f3c4d07350559e')
 
 
 # Source: https://github.com/dashpay/dash
@@ -436,6 +446,7 @@ class Dash(Coin):
         import x11_hash
         return x11_hash.getPoWHash(header)
 
+
 class DashTestnet(Dash):
     SHORTNAME = "tDASH"
     NET = "testnet"

lib/util.py

@@ -57,12 +57,20 @@ class cachedproperty(object):
         return value
 
 
-def formatted_time(t):
+def formatted_time(t, sep=' '):
     '''Return a number of seconds as a string in days, hours, mins and
-    secs.'''
+    maybe secs.'''
     t = int(t)
-    return '{:d}d {:02d}h {:02d}m {:02d}s'.format(
+    fmts = (('{:d}d', 86400), ('{:02d}h', 3600), ('{:02d}m', 60))
-        t // 86400, (t % 86400) // 3600, (t % 3600) // 60, t % 60)
+    parts = []
+    for fmt, n in fmts:
+        val = t // n
+        if parts or val:
+            parts.append(fmt.format(val))
+        t %= n
+    if len(parts) < 3:
+        parts.append('{:02d}s'.format(t))
+    return sep.join(parts)
 
 
 def deep_getsizeof(obj):

server/block_processor.py

@@ -383,9 +383,13 @@ class BlockProcessor(server.db.DB):
 
         # Catch-up stats
         if self.utxo_db.for_sync:
-            daemon_height = self.daemon.cached_height()
             tx_per_sec = int(self.tx_count / self.wall_time)
             this_tx_per_sec = 1 + int(tx_diff / (self.last_flush - last_flush))
+            self.logger.info('tx/sec since genesis: {:,d}, '
+                             'since last flush: {:,d}'
+                             .format(tx_per_sec, this_tx_per_sec))
+            if self.coin.TX_COUNT_HEIGHT > 0:
+                daemon_height = self.daemon.cached_height()
                 if self.height > self.coin.TX_COUNT_HEIGHT:
                     tx_est = (daemon_height - self.height) * self.coin.TX_PER_BLOCK
                 else:
@@ -397,12 +401,14 @@ class BlockProcessor(server.db.DB):
                 realism = 2.0 - 0.9 * self.height / self.coin.TX_COUNT_HEIGHT
                 tx_est *= max(realism, 1.0)
 
-            self.logger.info('tx/sec since genesis: {:,d}, '
+
-                             'since last flush: {:,d}'
-                             .format(tx_per_sec, this_tx_per_sec))
                 self.logger.info('sync time: {}  ETA: {}'
                                  .format(formatted_time(self.wall_time),
                                          formatted_time(tx_est / this_tx_per_sec)))
+            else:
+                self.logger.info('sync time: {}'
+                                 .format(formatted_time(self.wall_time)))
+
 
     def fs_flush(self):
         '''Flush the things stored on the filesystem.'''

server/controller.py

@@ -450,6 +450,7 @@ class Controller(util.LoggedClass):
     def getinfo(self):
         '''A one-line summary of server state.'''
         return {
+            'daemon': self.daemon.logged_url(),
             'daemon_height': self.daemon.cached_height(),
             'db_height': self.bp.db_height,
             'closing': len([s for s in self.sessions if s.is_closing()]),
@@ -463,6 +464,7 @@ class Controller(util.LoggedClass):
             'sessions': self.session_count(),
             'subs': self.sub_count(),
             'txs_sent': self.txs_sent,
+            'uptime': util.formatted_time(time.time() - self.start_time),
         }
 
     def sub_count(self):
@@ -514,12 +516,6 @@ class Controller(util.LoggedClass):
         '''A generator returning lines for a list of sessions.
 
         data is the return value of rpc_sessions().'''
-
-        def time_fmt(t):
-            t = int(t)
-            return ('{:3d}:{:02d}:{:02d}'
-                    .format(t // 3600, (t % 3600) // 60, t % 60))
-
         fmt = ('{:<6} {:<5} {:>17} {:>5} {:>5} '
                '{:>7} {:>7} {:>7} {:>7} {:>7} {:>9} {:>21}')
         yield fmt.format('ID', 'Flags', 'Client', 'Reqs', 'Txs', 'Subs',
@@ -534,7 +530,7 @@ class Controller(util.LoggedClass):
                              '{:,d}'.format(recv_size // 1024),
                              '{:,d}'.format(send_count),
                              '{:,d}'.format(send_size // 1024),
-                             time_fmt(time), peer)
+                             util.formatted_time(time, sep=''), peer)
 
     def session_data(self, for_log):
         '''Returned to the RPC 'sessions' call.'''
@@ -599,7 +595,7 @@ class Controller(util.LoggedClass):
             self.daemon.set_urls(self.env.coin.daemon_urls(daemon_url))
         except Exception as e:
             raise RPCError('an error occured: {}'.format(e))
-        return 'set daemon URL to {}'.format(daemon_url)
+        return 'now using daemon at {}'.format(self.daemon.logged_url())
 
     def rpc_stop(self):
         '''Shut down the server cleanly.'''

server/daemon.py

@@ -42,10 +42,27 @@ class Daemon(util.LoggedClass):
         '''Set the URLS to the given list, and switch to the first one.'''
         if not urls:
             raise DaemonError('no daemon URLs provided')
-        for url in urls:
-            self.logger.info('daemon at {}'.format(self.logged_url(url)))
         self.urls = urls
         self.url_index = 0
+        for n, url in enumerate(urls):
+            self.logger.info('daemon #{:d} at {}{}'
+                             .format(n + 1, self.logged_url(url),
+                                     '' if n else ' (current)'))
+
+    def url(self):
+        '''Returns the current daemon URL.'''
+        return self.urls[self.url_index]
+
+    def failover(self):
+        '''Call to fail-over to the next daemon URL.
+
+        Returns False if there is only one, otherwise True.
+        '''
+        if len(self.urls) > 1:
+            self.url_index = (self.url_index + 1) % len(self.urls)
+            self.logger.info('failing over to {}'.format(self.logged_url()))
+            return True
+        return False
 
     async def _send(self, payload, processor):
         '''Send a payload to be converted to JSON.
@@ -72,8 +89,7 @@ class Daemon(util.LoggedClass):
         while True:
             try:
                 async with self.workqueue_semaphore:
-                    url = self.urls[self.url_index]
+                    async with aiohttp.post(self.url(), data=data) as resp:
-                    async with aiohttp.post(url, data=data) as resp:
                         # If bitcoind can't find a tx, for some reason
                         # it returns 500 but fills out the JSON.
                         # Should still return 200 IMO.
@@ -99,17 +115,15 @@ class Daemon(util.LoggedClass):
             except Exception:
                 self.log_error(traceback.format_exc())
 
-            if secs >= max_secs and len(self.urls) > 1:
+            if secs >= max_secs and self.failover():
-                self.url_index = (self.url_index + 1) % len(self.urls)
-                logged_url = self.logged_url(self.urls[self.url_index])
-                self.logger.info('failing over to {}'.format(logged_url))
                 secs = 1
             else:
                 await asyncio.sleep(secs)
                 secs = min(max_secs, secs * 2)
 
-    def logged_url(self, url):
+    def logged_url(self, url=None):
         '''The host and port part, for logging.'''
+        url = url or self.url()
         return url[url.rindex('@') + 1:]
 
     async def _send_single(self, method, params=None):

server/version.py

@@ -1 +1 @@
-VERSION = "ElectrumX 0.10.16"
+VERSION = "ElectrumX 0.10.17"