diff --git a/server/block_processor.py b/server/block_processor.py index 6aa3a9b..43755ed 100644 --- a/server/block_processor.py +++ b/server/block_processor.py @@ -259,7 +259,7 @@ class BlockProcessor(server.db.DB): .format(VERSION, self.height)) self.first_sync = False self.flush_state(self.db) - self.reopen_db(False) + self.open_db(for_sync=False) self.caught_up_event.set() async def handle_chain_reorg(self, touched, count=None): diff --git a/server/db.py b/server/db.py index 5dad58b..b5ec12d 100644 --- a/server/db.py +++ b/server/db.py @@ -51,7 +51,7 @@ class DB(LoggedClass): .format(self.env.reorg_limit)) self.db = None - self.reopen_db(True) + self.open_db(for_sync=False) create = self.db_height == -1 self.headers_file = self.open_file('headers', create) @@ -69,30 +69,36 @@ class DB(LoggedClass): assert self.db_tx_count == 0 self.clean_db() - def reopen_db(self, first_sync): + def open_db(self, for_sync): '''Open the database. If the database is already open, it is - closed (implicitly via GC) and re-opened. + closed and re-opened. + If for_sync is True, it is opened for sync (high number of open + file, etc.) Re-open to set the maximum number of open files appropriately. ''' + def log_reason(message, is_for_sync): + reason = 'sync' if is_for_sync else 'serving' + self.logger.info('{} for {}'.format(message, reason)) + if self.db: - self.logger.info('closing DB to re-open') + if self.db.for_sync == for_sync: + return + log_reason('closing DB to re-open', for_sync) self.db.close() - max_open_files = 1024 if first_sync else 256 - # Open DB and metadata files. Record some of its state. db_name = '{}-{}'.format(self.coin.NAME, self.coin.NET) - self.db = open_db(db_name, self.env.db_engine, max_open_files) + self.db = open_db(db_name, self.env.db_engine, for_sync) if self.db.is_new: self.logger.info('created new {} database {}' .format(self.env.db_engine, db_name)) else: - self.logger.info('successfully opened {} database {} for sync: {}' - .format(self.env.db_engine, db_name, first_sync)) - self.read_state() + log_reason('opened {} database {}' + .format(self.env.db_engine, db_name), self.db.for_sync) - if self.first_sync == first_sync: + self.read_state() + if self.first_sync == self.db.for_sync: self.logger.info('software version: {}'.format(VERSION)) self.logger.info('DB version: {:d}'.format(self.db_version)) self.logger.info('coin: {}'.format(self.coin.NAME)) @@ -104,7 +110,7 @@ class DB(LoggedClass): self.logger.info('sync time so far: {}' .format(formatted_time(self.wall_time))) else: - self.reopen_db(self.first_sync) + self.open_db(self.first_sync) def read_state(self): if self.db.is_new: diff --git a/server/storage.py b/server/storage.py index b7d28fc..edf62d0 100644 --- a/server/storage.py +++ b/server/storage.py @@ -1,4 +1,4 @@ -# Copyright (c) 2016, the ElectrumX authors +# Copyright (c) 2016-2017, the ElectrumX authors # # All rights reserved. # @@ -31,7 +31,8 @@ class Storage(object): def __init__(self, name, for_sync): self.is_new = not os.path.exists(name) - self.open(name, create=self.is_new, for_sync=for_sync) + self.for_sync = for_sync or self.is_new + self.open(name, create=self.is_new) @classmethod def import_module(cls): @@ -79,8 +80,8 @@ class LevelDB(Storage): import plyvel cls.module = plyvel - def open(self, name, create, for_sync): - mof = 1024 if for_sync else 256 + def open(self, name, create): + mof = 1024 if self.for_sync else 256 self.db = self.module.DB(name, create_if_missing=create, max_open_files=mof, compression=None) self.close = self.db.close @@ -99,8 +100,8 @@ class RocksDB(Storage): import rocksdb cls.module = rocksdb - def open(self, name, create, for_sync): - mof = 1024 if for_sync else 256 + def open(self, name, create): + mof = 1024 if self.for_sync else 256 compression = "no" compression = getattr(self.module.CompressionType, compression + "_compression") @@ -172,7 +173,7 @@ class LMDB(Storage): import lmdb cls.module = lmdb - def open(self, name, create, for_sync): + def open(self, name, create): # I don't see anything equivalent to max_open_files for for_sync self.env = LMDB.module.Environment('.', subdir=True, create=create, max_dbs=32, map_size=5 * 10 ** 10)