Blacklist logic: (#723)

* Blacklist logic:
* do not return blacklisted peers to clients
* blacklist is retrieved from BLACKLIST_URL
* blacklist is refreshed every 10 minutes
* for BitcoinSegwit, BLACKLIST_URL defaults to https://electrum.org/blacklist.json

* make sybil detection independent from peer discovery

docs/environment.rst

@@ -345,6 +345,11 @@ some of this.
   will autodetect any proxy running on the usual ports 9050 (Tor),
   9150 (Tor browser bundle) and 1080 (socks).
 
+.. envvar:: BLACKLIST_URL
+
+  URL to retrieve a list of blacklisted peers.
+
+
 
 Server Advertising
 ==================

electrumx/server/env.py

@@ -75,6 +75,10 @@ class Env(EnvBase):
         self.bandwidth_limit = self.integer('BANDWIDTH_LIMIT', 2000000)
         self.session_timeout = self.integer('SESSION_TIMEOUT', 600)
         self.drop_client = self.custom("DROP_CLIENT", None, re.compile)
+        self.blacklist_url = self.default('BLACKLIST_URL', None)
+        # temporary default
+        if self.blacklist_url is None and self.coin.NAME == 'BitcoinSegwit':
+            self.blacklist_url = 'https://electrum.org/blacklist.json'
 
         # Identities
         clearnet_identity = self.clearnet_identity()

electrumx/server/peers.py

@@ -12,6 +12,8 @@ import random
 import socket
 import ssl
 import time
+import json
+import aiohttp
 from collections import defaultdict, Counter
 
 from aiorpcx import (Connector, RPCSession, SOCKSProxy,
@@ -75,6 +77,9 @@ class PeerManager(object):
         self.permit_onion_peer_time = time.time()
         self.proxy = None
         self.group = TaskGroup()
+        # refreshed
+        self.blacklist = set()
+        self.sybils = set()
 
     def _my_clearnet_peer(self):
         '''Returns the clearnet peer representing this server, if any.'''
@@ -129,6 +134,29 @@ class PeerManager(object):
                                   for real_name in self.env.coin.PEERS)
         await self._note_peers(imported_peers, limit=None)
 
+    def _is_allowed(self, peer):
+        if peer.host in self.blacklist:
+            return False
+        if '*.' + '.'.join(peer.host.split('.')[-2:]) in self.blacklist:
+            return False
+        return True
+
+    async def _refresh_blacklist(self):
+        session = aiohttp.ClientSession()
+        url = self.env.blacklist_url
+        if url is None:
+            return
+        while True:
+            try:
+                async with session.get(url) as response:
+                    r = await response.text()
+                    self.blacklist = set(json.loads(r))
+                    self.logger.info('blacklist retrieved from "%s": %d'
+                                     % (url, len(self.blacklist)))
+            except Exception as e:
+                self.logger.info('could not retrieve blacklist, "%s"' % url)
+            await sleep(600)
+
     async def _detect_proxy(self):
         '''Detect a proxy if we don't have one and some time has passed since
         the last attempt.
@@ -308,13 +336,13 @@ class PeerManager(object):
         # Process reported peers if remote peer is good
         peers = peers_task.result()
         if await self._note_peers(peers, check_matches=not peer.is_tor):
+            self.sybils.add(peer.host)
+
         features = self._features_to_register(peer, peers)
         if features:
             self.logger.info(f'registering ourself with {peer}')
             # We only care to wait for the response
             await session.send_request('server.add_peer', [features])
-        else:
-            raise BadPeerError('potential sybil detected')
 
     async def _send_headers_subscribe(self, session, peer, ptuple):
         message = 'blockchain.headers.subscribe'
@@ -401,6 +429,7 @@ class PeerManager(object):
         forever = Event()
         async with self.group as group:
             await group.spawn(forever.wait())
+            await group.spawn(self._refresh_blacklist())
             await group.spawn(self._detect_proxy())
             await group.spawn(self._import_peers())
             # Consume tasks as they complete, logging unexpected failures
@@ -464,7 +493,7 @@ class PeerManager(object):
 
         return permit
 
-    def on_peers_subscribe(self, is_tor):
+    def on_peers_subscribe(self, is_tor, is_peer):
         '''Returns the server peers as a list of (ip, host, details) tuples.
 
         We return all peers we've connected to in the last day.
@@ -477,6 +506,12 @@ class PeerManager(object):
                   not peer.bad and peer.is_public]
         onion_peers = []
 
+        if not is_peer:
+            recent = filter(self._is_allowed, recent)
+            recent = [peer for peer in self.peers
+                      if self._is_allowed(peer)
+                      and peer.host in self.sybils]
+
         # Always report ourselves if valid (even if not public)
         peers = set(myself for myself in self.myselves
                     if myself.last_good > cutoff)

electrumx/server/session.py

@@ -714,6 +714,7 @@ class ElectrumX(SessionBase):
         self.sv_seen = False
         self.mempool_statuses = {}
         self.set_request_handlers(self.PROTOCOL_MIN)
+        self.is_peer = False
 
     @classmethod
     def protocol_min_max_strings(cls):
@@ -812,11 +813,12 @@ class ElectrumX(SessionBase):
 
     async def add_peer(self, features):
         '''Add a peer (but only if the peer resolves to the source).'''
+        self.is_peer = True
         return await self.peer_mgr.on_add_peer(features, self.peer_address())
 
     async def peers_subscribe(self):
         '''Return the server peers as a list of (ip, host, details) tuples.'''
-        return self.peer_mgr.on_peers_subscribe(self.is_tor())
+        return self.peer_mgr.on_peers_subscribe(self.is_tor(), self.is_peer)
 
     async def address_status(self, hashX):
         '''Returns an address status.