diff --git a/README.rst b/README.rst index 0afadbf..d03930c 100644 --- a/README.rst +++ b/README.rst @@ -142,13 +142,19 @@ version prior to the release of 1.0. ChangeLog ========= +Version 0.10.1 +-------------- + +* Includes what should be a fix for issue `#94#` - stale references to + old sessions. This would effectively memory and network handles. + Version 0.10.0 -------------- -Major rewrite of DB layer as per issue `#72`_. UTXOs and history are -now indexed by the hash of the pay to script, making the index -independent of the address scheme. The history and UTXO DBs are also -now separate. +* Major rewrite of DB layer as per issue `#72`_. UTXOs and history + are now indexed by the hash of the pay to script, making the index + independent of the address scheme. +* The history and UTXO DBs are also now separate. Together these changes reduce the size of the DB by approximately 15% and the time taken to sync from genesis by about 20%. @@ -329,4 +335,5 @@ Version 0.9.0 .. _#88: https://github.com/kyuupichan/electrumx/issues/88 .. _#89: https://github.com/kyuupichan/electrumx/issues/89 .. _#93: https://github.com/kyuupichan/electrumx/issues/93 +.. _#94: https://github.com/kyuupichan/electrumx/issues/94 .. _docs/HOWTO.rst: https://github.com/kyuupichan/electrumx/blob/master/docs/HOWTO.rst diff --git a/samples/daemontools/env/HIST_MB b/samples/daemontools/env/HIST_MB deleted file mode 100644 index 697cb3a..0000000 --- a/samples/daemontools/env/HIST_MB +++ /dev/null @@ -1 +0,0 @@ -300 diff --git a/samples/daemontools/env/UTXO_MB b/samples/daemontools/env/UTXO_MB deleted file mode 100644 index 83b33d2..0000000 --- a/samples/daemontools/env/UTXO_MB +++ /dev/null @@ -1 +0,0 @@ -1000 diff --git a/samples/systemd/electrumx.conf b/samples/systemd/electrumx.conf index 54d1f0a..0c9413f 100644 --- a/samples/systemd/electrumx.conf +++ b/samples/systemd/electrumx.conf @@ -65,31 +65,3 @@ #Maximum number of address subscriptions across all sessions #MAX_SESSION_SUBS = 50000 #Maximum number of address subscriptions permitted to a single session. - - -#If synchronizing from the Genesis block your performance might change -#by tweaking the following cache variables. Cache size is only checked -#roughly every minute, so the caches can grow beyond the specified -#size. Also the Python process is often quite a bit fatter than the -#combined cache size, because of Python overhead and also because -#leveldb consumes a lot of memory during UTXO flushing. So I recommend -#you set the sum of these to nothing over half your available physical -#RAM: - - #HIST_MB = 300 - #amount of history cache, in MB, to retain before flushing to - #disk. Default is 300; probably no benefit being much larger - #as history is append-only and not searched. - - #UTXO_MB = 1000 - #amount of UTXO and history cache, in MB, to retain before - #flushing to disk. Default is 1000. This may be too large - #for small boxes or too small for machines with lots of RAM. - #Larger caches generally perform better as there is - #significant searching of the UTXO cache during indexing. - #However, I don't see much benefit in my tests pushing this - #too high, and in fact performance begins to fall. My - #machine has 24GB RAM; the slow down is probably because of - #leveldb caching and Python GC effects. However this may be - #very dependent on hardware and you may have different - #results. diff --git a/server/controller.py b/server/controller.py index 3febb59..ce8bef4 100644 --- a/server/controller.py +++ b/server/controller.py @@ -53,8 +53,9 @@ class Controller(util.LoggedClass): self.irc = IRC(env) self.env = env self.servers = {} + # Map of session to the key of its list in self.groups self.sessions = {} - self.groups = defaultdict(set) + self.groups = defaultdict(list) self.txs_sent = 0 self.next_log_sessions = 0 self.state = self.CATCHING_UP @@ -108,9 +109,10 @@ class Controller(util.LoggedClass): def session_priority(self, session): if isinstance(session, LocalRPC): return 0 - group_bandwidth = sum(s.bandwidth_used for s in self.sessions[session]) + gid = self.sessions[session] + group_bandwidth = sum(s.bandwidth_used for s in self.groups[gid]) return 1 + (bisect_left(self.bands, session.bandwidth_used) - + bisect_left(self.bands, group_bandwidth) + 1) // 2 + + bisect_left(self.bands, group_bandwidth)) // 2 def is_deprioritized(self, session): return self.session_priority(session) > self.BANDS @@ -333,9 +335,9 @@ class Controller(util.LoggedClass): if now > self.next_stale_check: self.next_stale_check = now + 300 self.clear_stale_sessions() - group = self.groups[int(session.start - self.start) // 900] - group.add(session) - self.sessions[session] = group + gid = int(session.start - self.start) // 900 + self.groups[gid].append(session) + self.sessions[session] = gid session.log_info('{} {}, {:,d} total' .format(session.kind, session.peername(), len(self.sessions))) @@ -350,8 +352,9 @@ class Controller(util.LoggedClass): def remove_session(self, session): '''Remove a session from our sessions list if there.''' if session in self.sessions: - group = self.sessions.pop(session) - group.remove(session) + gid = self.sessions.pop(session) + assert gid in self.groups + self.groups[gid].remove(session) self.subscription_count -= session.sub_count() def close_session(self, session): @@ -385,13 +388,16 @@ class Controller(util.LoggedClass): self.logger.info('closing stale connections {}'.format(stale)) # Consolidate small groups - keys = [k for k, v in self.groups.items() if len(v) <= 4 - and sum(session.bandwidth_used for session in v) < 10000] - if len(keys) > 1: - group = set.union(*(self.groups[key] for key in keys)) - for key in keys: - del self.groups[key] - self.groups[max(keys)] = group + gids = [gid for gid, l in self.groups.items() if len(l) <= 4 + and sum(session.bandwidth_used for session in l) < 10000] + if len(gids) > 1: + sessions = sum([self.groups[gid] for gid in gids], []) + new_gid = max(gids) + for gid in gids: + del self.groups[gid] + for session in sessions: + self.sessions[session] = new_gid + self.groups[new_gid] = sessions def new_subscription(self): if self.subscription_count >= self.max_subs: @@ -457,9 +463,9 @@ class Controller(util.LoggedClass): def group_data(self): '''Returned to the RPC 'groups' call.''' result = [] - for group_id in sorted(self.groups.keys()): - sessions = self.groups[group_id] - result.append([group_id, + for gid in sorted(self.groups.keys()): + sessions = self.groups[gid] + result.append([gid, len(sessions), sum(s.bandwidth_used for s in sessions), sum(s.requests_remaining() for s in sessions), diff --git a/server/version.py b/server/version.py index ccde3f4..51c5fd6 100644 --- a/server/version.py +++ b/server/version.py @@ -1 +1 @@ -VERSION = "ElectrumX 0.10.0" +VERSION = "ElectrumX 0.10.1"