From 5bcbdea16a7681a4470a1c0cae3856e0d4e45b8b Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Mon, 20 Feb 2017 05:37:53 +0900 Subject: [PATCH 1/6] Only set last_connect if the connection was good --- server/peers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/peers.py b/server/peers.py index 4d35628..1eab5b0 100644 --- a/server/peers.py +++ b/server/peers.py @@ -189,7 +189,6 @@ class PeerSession(JSONSession): def close_if_done(self): if not self.has_pending_requests(): is_good = not self.failed - self.peer.last_connect = time.time() self.peer_mgr.set_connection_status(self.peer, is_good) if is_good: if self.peer.is_tor: @@ -512,6 +511,7 @@ class PeerManager(util.LoggedClass): '''Called when a connection succeeded or failed.''' if good: peer.try_count = 0 + peer.last_connect = time.time() peer.source = 'peer' # Remove matching IP addresses for match in peer.matches(self.peers): From 4c8b5a643991080d19a94a64887337721ad6d8fe Mon Sep 17 00:00:00 2001 From: fr3aker Date: Thu, 23 Feb 2017 23:42:03 +0100 Subject: [PATCH 2/6] fix tor coin peers always scheduled for check (#139) fixes #138 --- server/peers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/peers.py b/server/peers.py index 1eab5b0..730b085 100644 --- a/server/peers.py +++ b/server/peers.py @@ -439,7 +439,7 @@ class PeerManager(util.LoggedClass): def retry_peer(peer): # Try some Tor at startup to determine the proxy so we can # serve the right banner file - if self.last_tor_retry_time == 0 and self.is_coin_onion_peer(peer): + if self.tor_proxy.port is None and self.is_coin_onion_peer(peer): return True # Retry a peer whose ports might have updated if peer.other_port_pairs: From 04e7dbe99ed64c130f21975e626743322832123b Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sun, 26 Feb 2017 09:05:40 +0900 Subject: [PATCH 3/6] Only serve chunks in forward direction This disconnects bad clients that are on the wrong network for some reason, before they consume enormous bandwidth. Closes #132 --- server/controller.py | 9 +-------- server/session.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/server/controller.py b/server/controller.py index ab05da7..e04d13f 100644 --- a/server/controller.py +++ b/server/controller.py @@ -81,7 +81,7 @@ class Controller(util.LoggedClass): ('blockchain', 'address.get_balance address.get_history address.get_mempool ' 'address.get_proof address.listunspent ' - 'block.get_header block.get_chunk estimatefee relayfee ' + 'block.get_header estimatefee relayfee ' 'transaction.get transaction.get_merkle utxo.get_address'), ('server', 'donation_address'), ] @@ -794,13 +794,6 @@ class Controller(util.LoggedClass): 'height': utxo.height, 'value': utxo.value} for utxo in sorted(await self.get_utxos(hashX))] - def block_get_chunk(self, index): - '''Return a chunk of block headers. - - index: the chunk index''' - index = self.non_negative_integer(index) - return self.get_chunk(index) - def block_get_header(self, height): '''The deserialized header at a given height. diff --git a/server/session.py b/server/session.py index a9a4adc..cd43071 100644 --- a/server/session.py +++ b/server/session.py @@ -112,8 +112,10 @@ class ElectrumX(SessionBase): self.max_subs = self.env.max_session_subs self.hashX_subs = {} self.mempool_statuses = {} + self.chunk_indices = [] self.electrumx_handlers = { 'blockchain.address.subscribe': self.address_subscribe, + 'blockchain.block.get_chunk': self.block_get_chunk, 'blockchain.headers.subscribe': self.headers_subscribe, 'blockchain.numblocks.subscribe': self.numblocks_subscribe, 'blockchain.script_hash.subscribe': self.script_hash_subscribe, @@ -261,6 +263,21 @@ class ElectrumX(SessionBase): '''Returns a dictionary of server features.''' return self.controller.peer_mgr.myself.features + def block_get_chunk(self, index): + '''Return a chunk of block headers as a hexadecimal string. + + index: the chunk index''' + index = self.controller.non_negative_integer(index) + self.chunk_indices.append(index) + self.chunk_indices = self.chunk_indices[-5:] + # -2 allows backing up a single chunk but no more. + if index <= max(self.chunk_indices[:-2], default=-1): + msg = ('chunk indices not advancing (wrong network?): {}' + .format(self.chunk_indices)) + # INVALID_REQUEST triggers a disconnect + raise RPCError(mrg, JSONRPC.INVALID_REQUEST) + return self.controller.get_chunk(index) + def is_tor(self): '''Try to detect if the connection is to a tor hidden service we are running.''' From d36d2224894a9f487b9a276f80ff8b37312607a6 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sun, 26 Feb 2017 10:09:37 +0900 Subject: [PATCH 4/6] Maintain more than one self-identity Rather than just retaining our clearnet identity from the environment, maintain any tor one as well. This is a little cleaner and avoids confusion over port assignment when converting to real names. Fixes #136 --- lib/peer.py | 4 ++-- server/peers.py | 26 ++++++++++++++------------ server/session.py | 2 +- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/lib/peer.py b/lib/peer.py index 2009c29..fff3dd1 100644 --- a/lib/peer.py +++ b/lib/peer.py @@ -243,7 +243,7 @@ class Peer(object): details = self.real_name().split()[1:] return (self.ip_addr or self.host, self.host, details) - def real_name(self, host_override=None): + def real_name(self): '''Real name of this peer as used on IRC.''' def port_text(letter, port): if port == self.DEFAULT_PORTS.get(letter): @@ -251,7 +251,7 @@ class Peer(object): else: return letter + str(port) - parts = [host_override or self.host, 'v' + self.protocol_max] + parts = [self.host, 'v' + self.protocol_max] if self.pruning: parts.append('p{:d}'.format(self.pruning)) for letter, port in (('s', self.ssl_port), ('t', self.tcp_port)): diff --git a/server/peers.py b/server/peers.py index 730b085..c9e83d8 100644 --- a/server/peers.py +++ b/server/peers.py @@ -29,9 +29,8 @@ STALE_SECS = 86400 WAKEUP_SECS = 300 -def peer_from_env(env): - '''Return ourself as a peer from the environment settings.''' - main_identity = env.identities[0] +def peers_from_env(env): + '''Return a list of peers from the environment settings.''' hosts = {identity.host: {'tcp_port': identity.tcp_port, 'ssl_port': identity.ssl_port} for identity in env.identities} @@ -44,7 +43,7 @@ def peer_from_env(env): 'genesis_hash': env.coin.GENESIS_HASH, } - return Peer(main_identity.host, features, 'env') + return [Peer(ident.host, features, 'env') for ident in env.identities] class PeerSession(JSONSession): @@ -117,7 +116,7 @@ class PeerSession(JSONSession): return # Announce ourself if not present - my = self.peer_mgr.myself + my = self.peer_mgr.my_clearnet_peer() for peer in my.matches(peers): if peer.tcp_port == my.tcp_port and peer.ssl_port == my.ssl_port: return @@ -214,7 +213,7 @@ class PeerManager(util.LoggedClass): self.controller = controller self.loop = controller.loop self.irc = IRC(env, self) - self.myself = peer_from_env(env) + self.myselves = peers_from_env(env) # value is max outgoing connections at a time self.semaphore = asyncio.BoundedSemaphore(value=8) self.retry_event = asyncio.Event() @@ -229,6 +228,10 @@ class PeerManager(util.LoggedClass): loop=self.loop) self.import_peers() + def my_clearnet_peer(self): + '''Returns the clearnet peer representing this server.''' + return [peer for peer in self.myselves if not peer.is_tor][0] + def info(self): '''The number of peers.''' self.set_peer_statuses() @@ -322,9 +325,8 @@ class PeerManager(util.LoggedClass): onion_peers = [] # Always report ourselves if valid (even if not public) - peers = set() - if self.myself.last_connect > cutoff: - peers.add(self.myself) + peers = set(myself for myself in self.myselves + if myself.last_connect > cutoff) # Bucket the clearnet peers and select one from each buckets = defaultdict(list) @@ -370,7 +372,7 @@ class PeerManager(util.LoggedClass): def import_peers(self): '''Import hard-coded peers from a file or the coin defaults.''' - self.add_peers([self.myself]) + self.add_peers(self.myselves) coin_peers = self.env.coin.PEERS self.onion_peers = [Peer.from_real_name(rn, 'coins.py') for rn in coin_peers if '.onion ' in rn] @@ -386,8 +388,8 @@ class PeerManager(util.LoggedClass): def connect_to_irc(self): '''Connect to IRC if not disabled.''' if self.env.irc and self.env.coin.IRC_PREFIX: - pairs = [(self.myself.real_name(ident.host), ident.nick_suffix) - for ident in self.env.identities] + pairs = [(peer.real_name(), ident.nick_suffix) for peer, ident + in zip(self.myselves, self.env.identities)] self.ensure_future(self.irc.start(pairs)) else: self.logger.info('IRC is disabled') diff --git a/server/session.py b/server/session.py index cd43071..120c701 100644 --- a/server/session.py +++ b/server/session.py @@ -261,7 +261,7 @@ class ElectrumX(SessionBase): def server_features(self): '''Returns a dictionary of server features.''' - return self.controller.peer_mgr.myself.features + return self.controller.peer_mgr.my_clearnet_peer().features def block_get_chunk(self, index): '''Return a chunk of block headers as a hexadecimal string. From 0145098165fae2199c72d77e1b715ff2ca564023 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sun, 26 Feb 2017 10:42:32 +0900 Subject: [PATCH 5/6] Add debug output Idea is to help understand #135 --- server/peers.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/server/peers.py b/server/peers.py index c9e83d8..a085813 100644 --- a/server/peers.py +++ b/server/peers.py @@ -189,13 +189,14 @@ class PeerSession(JSONSession): if not self.has_pending_requests(): is_good = not self.failed self.peer_mgr.set_connection_status(self.peer, is_good) - if is_good: - if self.peer.is_tor: - self.log_info('verified via {} over Tor'.format(self.kind)) - else: - self.log_info('verified via {} at {}' - .format(self.kind, - self.peer_addr(anon=False))) + if self.peer.is_tor: + how = 'via {} over Tor'.format(self.kind) + else: + how = 'via {} at {}'.format(self.kind, + self.peer_addr(anon=False)) + status = 'verified' if is_good else 'failed to verify' + elapsed = time.time() - self.peer.last_try + self.log_info('{} {} in {:.1f}s'.format(status, how, elapsed)) self.close_connection() @@ -463,16 +464,21 @@ class PeerManager(util.LoggedClass): self.last_tor_retry_time = now for peer in peers: - peer.last_try = time.time() peer.try_count += 1 pairs = peer.connection_port_pairs() if peer.bad or not pairs: self.maybe_forget_peer(peer) else: + start = time.time() await self.semaphore.acquire() + elapsed = time.time() - start + if elapsed > 5: + self.log_warning('waited {:.1f}s for connection semaphore' + .format(elapsed)) self.retry_peer(peer, pairs) def retry_peer(self, peer, port_pairs): + peer.last_try = time.time() kind, port = port_pairs[0] # Python 3.5.3: use PROTOCOL_TLS sslc = ssl.SSLContext(ssl.PROTOCOL_SSLv23) if kind == 'SSL' else None @@ -496,8 +502,10 @@ class PeerManager(util.LoggedClass): exception = future.exception() if exception: kind, port = port_pairs[0] - self.logger.info('failed connecting to {} at {} port {:d}: {}' - .format(peer, kind, port, exception)) + self.logger.info('failed connecting to {} at {} port {:d} ' + 'in {:.1f}s: {}' + .format(peer, kind, port, + time.time() - peer.last_try, exception)) port_pairs = port_pairs[1:] if port_pairs: self.retry_peer(peer, port_pairs) From b0039a25a71a6da0e1f9b5d5dc9407105163d356 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sun, 26 Feb 2017 10:49:27 +0900 Subject: [PATCH 6/6] Prepare 0.99.1 --- README.rst | 15 +++++++++++++++ server/version.py | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 47650b0..74650f3 100644 --- a/README.rst +++ b/README.rst @@ -132,6 +132,17 @@ version for the release of 1.0. ChangeLog ========= +Version 0.99.1 +-------------- + +* Add more verbose logging in attempt to understand issue `#135`_ +* REPORT_TCP_PORT_TOR and REPORT_SSL_PORT_TOR were ignored when constructing + IRC real names. Fixes `#136`_ +* Only serve chunk requests in forward direction; disconnect clients iterating + backwards. Minimizes bandwidth consumption caused by misbehaving Electrum + clients. Closes `#132`_ +* Tor coin peers would always be scheduled for check, fixes `#138`_ (fr3aker) + Version 0.99 ------------ @@ -367,6 +378,10 @@ stability please stick with the 0.9 series. .. _#124: https://github.com/kyuupichan/electrumx/issues/124 .. _#126: https://github.com/kyuupichan/electrumx/issues/126 .. _#129: https://github.com/kyuupichan/electrumx/issues/129 +.. _#132: https://github.com/kyuupichan/electrumx/issues/132 +.. _#135: https://github.com/kyuupichan/electrumx/issues/135 +.. _#136: https://github.com/kyuupichan/electrumx/issues/136 +.. _#138: https://github.com/kyuupichan/electrumx/issues/138 .. _docs/HOWTO.rst: https://github.com/kyuupichan/electrumx/blob/master/docs/HOWTO.rst .. _docs/ENVIRONMENT.rst: https://github.com/kyuupichan/electrumx/blob/master/docs/ENVIRONMENT.rst .. _docs/PEER_DISCOVERY.rst: https://github.com/kyuupichan/electrumx/blob/master/docs/PEER_DISCOVERY.rst diff --git a/server/version.py b/server/version.py index 2a6011e..4d9138d 100644 --- a/server/version.py +++ b/server/version.py @@ -1,5 +1,5 @@ # Server name and protocol versions -VERSION = 'ElectrumX 0.99' +VERSION = 'ElectrumX 0.99.1' PROTOCOL_MIN = '1.0' PROTOCOL_MAX = '1.0'