diff --git a/RELEASE-NOTES b/RELEASE-NOTES index ffe8355..016f7a6 100644 --- a/RELEASE-NOTES +++ b/RELEASE-NOTES @@ -1,7 +1,14 @@ +version 0.7.18 +-------------- + +- better IRC support for tor (valesi) +- issues: suppressed some uninteresting socket logging to fix #52 +- mempool: fixed small memory leak + version 0.7.17 -------------- -- upped read buffer limit to 1000000 bytes. +- upped read buffer limit to 1,000,000 bytes. version 0.7.16 -------------- @@ -12,7 +19,7 @@ version 0.7.15 -------------- The following meta variables in your banner file are now replaced in -addition to $VERSION described in the notes to 0.17.11. If you type +addition to $VERSION described in the notes to 0.7.11. If you type getnetworkinfo in your daemon's debug console you will see what they are based on: @@ -60,7 +67,7 @@ version 0.7.11 - increased MAX_SEND default value to 1 million bytes so as to be able to serve large historical transactions of up to ~500K in size. The - MAX_SEND floor remains at 350,000 bytes so you can reduct it if you + MAX_SEND floor remains at 350,000 bytes so you can reduce it if you wish. To serve any historical transaction for bitcoin youd should set this to around 2,000,100 bytes (one byte becomes 2 ASCII hex chars) - issue #46: fix reorgs for coinbase-only blocks. We would not distinguish diff --git a/docs/ENV-NOTES b/docs/ENV-NOTES index f51004e..26048a6 100644 --- a/docs/ENV-NOTES +++ b/docs/ENV-NOTES @@ -86,12 +86,21 @@ BANDWIDTH_LIMIT - per-session periodic bandwith usage limit in bytes. If you want IRC connectivity to advertise your node: -IRC - set to anything non-empty -IRC_NICK - the nick to use when connecting to IRC. The default is a - hash of REPORT_HOST. Either way 'E_' will be prepended. -REPORT_HOST - the host to advertise. Defaults to HOST. -REPORT_SSL_PORT - the SSL port to advertise. Defaults to SSL_PORT. -REPORT_TCP_PORT - the TCP port to advertise. Defaults to TCP_PORT. +IRC - set to anything non-empty +IRC_NICK - the nick to use when connecting to IRC. The default is a + hash of REPORT_HOST. Either way 'E_' will be prepended. +REPORT_HOST - the host to advertise. Defaults to HOST. +REPORT_TCP_PORT - the TCP port to advertise. Defaults to TCP_PORT. + '0' disables publishing the port. +REPORT_SSL_PORT - the SSL port to advertise. Defaults to SSL_PORT. + '0' disables publishing the port. +REPORT_HOST_TOR - Tor .onion address to advertise. Appends '_tor" to nick. +REPORT_TCP_PORT_TOR - the TCP port to advertise for Tor. Defaults to + REPORT_TCP_PORT, unless it is '0', then use TCP_PORT. + '0' disables publishing the port. +REPORT_SSL_PORT_TOR - the SSL port to advertise for Tor. Defaults to + REPORT_SSL_PORT, unless it is '0', then use SSL_PORT. + '0' disables publishing the port. If synchronizing from the Genesis block your performance might change by tweaking the following cache variables. Cache size is only checked @@ -124,4 +133,4 @@ FORCE_REORG - if set to a positive integer, it will simulate a reorg of the blockchain for that number of blocks on startup. Although it should fail gracefully if set to a value greater than REORG_LIMIT, I do not recommend it as I have - not tried it and there is a chance your DB might corrupt. \ No newline at end of file + not tried it and there is a chance your DB might corrupt. diff --git a/electrumx_server.py b/electrumx_server.py index 940828e..7129a8a 100755 --- a/electrumx_server.py +++ b/electrumx_server.py @@ -21,6 +21,7 @@ from server.protocol import ServerManager SUPPRESS_MESSAGES = [ 'Fatal read error on socket transport', + 'Fatal write error on socket transport', ] def main_loop(): diff --git a/lib/jsonrpc.py b/lib/jsonrpc.py index 1cdc5fa..6329a83 100644 --- a/lib/jsonrpc.py +++ b/lib/jsonrpc.py @@ -311,7 +311,7 @@ class JSONRPC(asyncio.Protocol, LoggedClass): async def json_notification(self, message): try: method, params = self.method_and_params(message) - except RCPError: + except self.RPCError: pass else: await self.handle_notification(method, params) @@ -336,7 +336,7 @@ class JSONRPC(asyncio.Protocol, LoggedClass): def raise_unknown_method(self, method): '''Respond to a request with an unknown method.''' - raise self.RPCError('unknown method: "{}"'.format(method), + raise self.RPCError("unknown method: '{}'".format(method), self.METHOD_NOT_FOUND) # --- derived classes are intended to override these functions diff --git a/server/env.py b/server/env.py index 58b8b9d..2b875a4 100644 --- a/server/env.py +++ b/server/env.py @@ -51,11 +51,20 @@ class Env(LoggedClass): self.max_session_subs = self.integer('MAX_SESSION_SUBS', 50000) self.bandwidth_limit = self.integer('BANDWIDTH_LIMIT', 2000000) # IRC + self.irc = self.default('IRC', False) + self.irc_nick = self.default('IRC_NICK', None) self.report_tcp_port = self.integer('REPORT_TCP_PORT', self.tcp_port) self.report_ssl_port = self.integer('REPORT_SSL_PORT', self.ssl_port) self.report_host = self.default('REPORT_HOST', self.host) - self.irc_nick = self.default('IRC_NICK', None) - self.irc = self.default('IRC', False) + self.report_tcp_port_tor = self.integer('REPORT_TCP_PORT_TOR', + self.report_tcp_port + if self.report_tcp_port else + self.tcp_port) + self.report_ssl_port_tor = self.integer('REPORT_SSL_PORT_TOR', + self.report_ssl_port + if self.report_ssl_port else + self.ssl_port) + self.report_host_tor = self.default('REPORT_HOST_TOR', None) # Debugging self.force_reorg = self.integer('FORCE_REORG', 0) diff --git a/server/irc.py b/server/irc.py index 7930d07..694c81a 100644 --- a/server/irc.py +++ b/server/irc.py @@ -20,14 +20,6 @@ from lib.hash import double_sha256 from lib.util import LoggedClass -def port_text(letter, port, default): - if not port: - return '' - if port == default: - return letter - return letter + str(port) - - class IRC(LoggedClass): Peer = namedtuple('Peer', 'ip_addr host ports') @@ -37,22 +29,29 @@ class IRC(LoggedClass): def __init__(self, env): super().__init__() - tcp_text = port_text('t', env.report_tcp_port, 50001) - ssl_text = port_text('s', env.report_ssl_port, 50002) - # If this isn't something the client expects you won't appear - # in the client's network dialog box self.env = env - version = '1.0' - self.real_name = '{} v{} {} {}'.format(env.report_host, version, - tcp_text, ssl_text) + + # If this isn't something a peer or client expects + # then you won't appear in the client's network dialog box + irc_address = (env.coin.IRC_SERVER, env.coin.IRC_PORT) + self.channel = env.coin.IRC_CHANNEL self.prefix = env.coin.IRC_PREFIX + + self.clients = [] self.nick = '{}{}'.format(self.prefix, env.irc_nick if env.irc_nick else double_sha256(env.report_host.encode()) [:5].hex()) - self.channel = env.coin.IRC_CHANNEL - self.irc_server = env.coin.IRC_SERVER - self.irc_port = env.coin.IRC_PORT + self.clients.append( IrcClient(irc_address, self.nick, + env.report_host, + env.report_tcp_port, + env.report_ssl_port) ) + if env.report_host_tor: + self.clients.append( IrcClient(irc_address, self.nick + '_tor', + env.report_host_tor, + env.report_tcp_port_tor, + env.report_ssl_port_tor) ) + self.peer_regexp = re.compile('({}[^!]*)!'.format(self.prefix)) self.peers = {} @@ -72,20 +71,23 @@ class IRC(LoggedClass): async def join(self): import irc.client as irc_client - self.logger.info('joining IRC with nick "{}" and real name "{}"' - .format(self.nick, self.real_name)) - reactor = irc_client.Reactor() for event in ['welcome', 'join', 'quit', 'kick', 'whoreply', 'namreply', 'disconnect']: reactor.add_global_handler(event, getattr(self, 'on_' + event)) - connection = reactor.server() + # Note: Multiple nicks in same channel will trigger duplicate events + for client in self.clients: + client.connection = reactor.server() + while True: try: - connection.connect(self.irc_server, self.irc_port, - self.nick, ircname=self.real_name) - connection.set_keepalive(60) + for client in self.clients: + self.logger.info('Joining IRC in {} as "{}" with ' + 'real name "{}"' + .format(self.channel, client.nick, + client.realname)) + client.connect() while True: reactor.process_once() await asyncio.sleep(2) @@ -155,3 +157,36 @@ class IRC(LoggedClass): self.peers[nick] = peer except IndexError: pass + + +class IrcClient(LoggedClass): + + VERSION = '1.0' + DEFAULT_PORTS = {'t': 50001, 's': 50002} + + def __init__(self, irc_address, nick, host, tcp_port, ssl_port): + super().__init__() + self.irc_host, self.irc_port = irc_address + self.nick = nick + self.realname = self.create_realname(host, tcp_port, ssl_port) + self.connection = None + + def connect(self, keepalive=60): + '''Connect this client to its IRC server''' + self.connection.connect(self.irc_host, self.irc_port, self.nick, + ircname=self.realname) + self.connection.set_keepalive(keepalive) + + @classmethod + def create_realname(cls, host, tcp_port, ssl_port): + def port_text(letter, port): + if not port: + return '' + if port == cls.DEFAULT_PORTS.get(letter): + return ' ' + letter + else: + return ' ' + letter + str(port) + + tcp = port_text('t', tcp_port) + ssl = port_text('s', ssl_port) + return '{} v{}{}{}'.format(host, cls.VERSION, tcp, ssl) diff --git a/server/protocol.py b/server/protocol.py index bae092d..d6c06eb 100644 --- a/server/protocol.py +++ b/server/protocol.py @@ -77,6 +77,16 @@ class MemPool(util.LoggedClass): touched = set() missing_utxos = [] + def drop_tx(hex_hash): + txin_pairs, txout_pairs, _u = self.txs.pop(hex_hash) + hash168s = set(hash168 for hash168, value in txin_pairs) + hash168s.update(hash168 for hash168, value in txout_pairs) + for hash168 in hash168s: + self.hash168s[hash168].remove(hex_hash) + if not self.hash168s[hash168]: + del self.hash168s[hash168] + touched.update(hash168s) + initial = self.count < 0 if initial: self.logger.info('beginning import of {:,d} mempool txs' @@ -85,14 +95,7 @@ class MemPool(util.LoggedClass): # Remove gone items gone = set(self.txs).difference(hex_hashes) for hex_hash in gone: - txin_pairs, txout_pairs, unconfirmed = self.txs.pop(hex_hash) - hash168s = set(hash168 for hash168, value in txin_pairs) - hash168s.update(hash168 for hash168, value in txout_pairs) - for hash168 in hash168s: - self.hash168s[hash168].remove(hex_hash) - if not self.hash168s[hash168]: - del self.hash168s[hash168] - touched.update(hash168s) + drop_tx(hex_hash) # Get the raw transactions for the new hashes. Ignore the # ones the daemon no longer has (it will return None). Put @@ -119,7 +122,7 @@ class MemPool(util.LoggedClass): if n % 20 == 0: await asyncio.sleep(0) txout_pairs = [txout_pair(txout) for txout in tx.outputs] - self.txs[hex_hash] = (None, txout_pairs, None) + self.txs[hex_hash] = ([], txout_pairs, None) def txin_info(txin): hex_hash = hash_to_str(txin.prev_hash) @@ -153,7 +156,7 @@ class MemPool(util.LoggedClass): # it's harmless - next time the mempool is refreshed # they'll either be cleaned up or the UTXOs will no # longer be missing. - del self.txs[hex_hash] + drop_tx(hex_hash) continue self.txs[hex_hash] = (txin_pairs, txout_pairs, any(unconfs)) diff --git a/server/version.py b/server/version.py index 3a2a92c..71fdbbb 100644 --- a/server/version.py +++ b/server/version.py @@ -1 +1 @@ -VERSION = "ElectrumX 0.7.17" +VERSION = "ElectrumX 0.7.18" diff --git a/tests/test_storage.py b/tests/test_storage.py index 2c3d429..504f3f5 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -22,8 +22,8 @@ def db(tmpdir, request): cwd = os.getcwd() os.chdir(str(tmpdir)) db = open_db("db", request.param, False) - os.chdir(cwd) yield db + os.chdir(cwd) # Make sure all the locks and handles are closed del db gc.collect() @@ -67,3 +67,10 @@ def test_iterator_reverse(db): (b"abc" + str.encode(str(i)), str.encode(str(i))) for i in reversed(range(5)) ] + + +def test_close(db): + db.put(b"a", b"b") + db.close() + db = open_db("db", db.__class__.__name__, False) + assert db.get(b"a") == b"b" \ No newline at end of file