From 7b36a1431cfd5bdd20ba5b99524881aa2f405e7a Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sun, 9 Oct 2016 13:00:46 +0900 Subject: [PATCH 1/2] Add instrumentation --- server/db.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/server/db.py b/server/db.py index 35c7e47..9eacc54 100644 --- a/server/db.py +++ b/server/db.py @@ -191,13 +191,16 @@ class DB(object): with self.db.write_batch(transaction=True) as batch: # Flush the state, then the cache, then the history self.put_state() - for key, value in self.write_cache.items(): + for n, (key, value) in enumerate(self.write_cache.items()): if value is None: batch.delete(key) deletes += 1 else: batch.put(key, value) writes += 1 + if n % 1000 == 0: + pct = n // len(self.write_cache) * 100 + self.logger.info('{:d} {:d}% done...'.format(n, pct)) self.flush_history() @@ -220,7 +223,7 @@ class DB(object): # Drop any None entry self.history.pop(None, None) - for hash160, hist in self.history.items(): + for m, (hash160, hist) in enumerate(self.history.items()): prefix = b'H' + hash160 for key, v in self.db.iterator(reverse=True, prefix=prefix, fill_cache=False): @@ -245,6 +248,10 @@ class DB(object): .format(addr, idx)) self.db.put(key, v[n:n + HIST_ENTRY_LEN]) + if m % 1000 == 0: + pct = m // len(self.history) * 100 + self.logger.info('{:d} {:d}% done...'.format(m, pct)) + self.history = defaultdict(list) def get_hash160(self, tx_hash, idx, delete=True): From 329f41164049da35f70a69e321df059e68c4e90e Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sun, 9 Oct 2016 13:27:02 +0900 Subject: [PATCH 2/2] Fix percentages --- server/db.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/server/db.py b/server/db.py index 9eacc54..40a9e02 100644 --- a/server/db.py +++ b/server/db.py @@ -198,9 +198,9 @@ class DB(object): else: batch.put(key, value) writes += 1 - if n % 1000 == 0: - pct = n // len(self.write_cache) * 100 - self.logger.info('{:d} {:d}% done...'.format(n, pct)) + if n % 20000 == 0: + pct = n * 100 // len(self.write_cache) + self.logger.info('U {:d} {:d}% done...'.format(n, pct)) self.flush_history() @@ -248,9 +248,9 @@ class DB(object): .format(addr, idx)) self.db.put(key, v[n:n + HIST_ENTRY_LEN]) - if m % 1000 == 0: - pct = m // len(self.history) * 100 - self.logger.info('{:d} {:d}% done...'.format(m, pct)) + if m % 20000 == 0: + pct = m * 100 // len(self.history) + self.logger.info('H {:d} {:d}% done...'.format(m, pct)) self.history = defaultdict(list)