diff --git a/server/block_processor.py b/server/block_processor.py index 63e2b1a..157d226 100644 --- a/server/block_processor.py +++ b/server/block_processor.py @@ -337,7 +337,7 @@ class BlockProcessor(server.db.DB): self.wall_time += now - self.last_flush self.last_flush = now self.last_flush_tx_count = self.tx_count - self.utxo_write_state(batch) + self.write_utxo_state(batch) def assert_flushed(self): '''Asserts state is fully flushed.''' diff --git a/server/db.py b/server/db.py index b802fcf..ef6963f 100644 --- a/server/db.py +++ b/server/db.py @@ -142,54 +142,6 @@ class DB(util.LoggedClass): self.logger.info('sync time so far: {}' .format(util.formatted_time(self.wall_time))) - def read_utxo_state(self): - state = self.utxo_db.get(b'state') - if not state: - self.db_height = -1 - self.db_tx_count = 0 - self.db_tip = b'\0' * 32 - self.db_version = max(self.DB_VERSIONS) - self.utxo_flush_count = 0 - self.wall_time = 0 - self.first_sync = True - else: - state = ast.literal_eval(state.decode()) - if not isinstance(state, dict): - raise self.DBError('failed reading state from DB') - self.db_version = state['db_version'] - if self.db_version not in self.DB_VERSIONS: - raise self.DBError('your DB version is {} but this software ' - 'only handles versions {}' - .format(self.db_version, self.DB_VERSIONS)) - # backwards compat - genesis_hash = state['genesis'] - if isinstance(genesis_hash, bytes): - genesis_hash = genesis_hash.decode() - if genesis_hash != self.coin.GENESIS_HASH: - raise self.DBError('DB genesis hash {} does not match coin {}' - .format(state['genesis_hash'], - self.coin.GENESIS_HASH)) - self.db_height = state['height'] - self.db_tx_count = state['tx_count'] - self.db_tip = state['tip'] - self.utxo_flush_count = state['utxo_flush_count'] - self.wall_time = state['wall_time'] - self.first_sync = state['first_sync'] - - def utxo_write_state(self, batch): - '''Write (UTXO) state to the batch.''' - state = { - 'genesis': self.coin.GENESIS_HASH, - 'height': self.db_height, - 'tx_count': self.db_tx_count, - 'tip': self.db_tip, - 'utxo_flush_count': self.utxo_flush_count, - 'wall_time': self.wall_time, - 'first_sync': self.first_sync, - 'db_version': self.db_version, - } - batch.put(b'state', repr(state).encode()) - def clean_db(self): '''Clean out stale DB items. @@ -299,6 +251,93 @@ class DB(util.LoggedClass): assert isinstance(limit, int) and limit >= 0 return limit + # -- Undo information + + def min_undo_height(self, max_height): + '''Returns a height from which we should store undo info.''' + return max_height - self.env.reorg_limit + 1 + + def undo_key(self, height): + '''DB key for undo information at the given height.''' + return b'U' + pack('>I', height) + + def read_undo_info(self, height): + '''Read undo information from a file for the current height.''' + return self.utxo_db.get(self.undo_key(height)) + + def flush_undo_infos(self, batch_put, undo_infos): + '''undo_infos is a list of (undo_info, height) pairs.''' + for undo_info, height in undo_infos: + batch_put(self.undo_key(height), b''.join(undo_info)) + + def clear_excess_undo_info(self): + '''Clear excess undo info. Only most recent N are kept.''' + prefix = b'U' + min_height = self.min_undo_height(self.db_height) + keys = [] + for key, hist in self.utxo_db.iterator(prefix=prefix): + height, = unpack('>I', key[-4:]) + if height >= min_height: + break + keys.append(key) + + if keys: + with self.utxo_db.write_batch() as batch: + for key in keys: + batch.delete(key) + self.logger.info('deleted {:,d} stale undo entries' + .format(len(keys))) + + # -- UTXO database + + def read_utxo_state(self): + state = self.utxo_db.get(b'state') + if not state: + self.db_height = -1 + self.db_tx_count = 0 + self.db_tip = b'\0' * 32 + self.db_version = max(self.DB_VERSIONS) + self.utxo_flush_count = 0 + self.wall_time = 0 + self.first_sync = True + else: + state = ast.literal_eval(state.decode()) + if not isinstance(state, dict): + raise self.DBError('failed reading state from DB') + self.db_version = state['db_version'] + if self.db_version not in self.DB_VERSIONS: + raise self.DBError('your DB version is {} but this software ' + 'only handles versions {}' + .format(self.db_version, self.DB_VERSIONS)) + # backwards compat + genesis_hash = state['genesis'] + if isinstance(genesis_hash, bytes): + genesis_hash = genesis_hash.decode() + if genesis_hash != self.coin.GENESIS_HASH: + raise self.DBError('DB genesis hash {} does not match coin {}' + .format(state['genesis_hash'], + self.coin.GENESIS_HASH)) + self.db_height = state['height'] + self.db_tx_count = state['tx_count'] + self.db_tip = state['tip'] + self.utxo_flush_count = state['utxo_flush_count'] + self.wall_time = state['wall_time'] + self.first_sync = state['first_sync'] + + def write_utxo_state(self, batch): + '''Write (UTXO) state to the batch.''' + state = { + 'genesis': self.coin.GENESIS_HASH, + 'height': self.db_height, + 'tx_count': self.db_tx_count, + 'tip': self.db_tip, + 'utxo_flush_count': self.utxo_flush_count, + 'wall_time': self.wall_time, + 'first_sync': self.first_sync, + 'db_version': self.db_version, + } + batch.put(b'state', repr(state).encode()) + def get_balance(self, hashX): '''Returns the confirmed balance of an address.''' return sum(utxo.value for utxo in self.get_utxos(hashX, limit=None)) @@ -322,24 +361,6 @@ class DB(util.LoggedClass): tx_hash, height = self.fs_tx_hash(tx_num) yield UTXO(tx_num, tx_pos, tx_hash, height, value) - def db_hashX(self, tx_hash, idx_packed): - '''Return (hashX, tx_num_packed) for the given TXO. - - Both are None if not found.''' - # Key: b'h' + compressed_tx_hash + tx_idx + tx_num - # Value: hashX - prefix = b'h' + tx_hash[:4] + idx_packed - - # Find which entry, if any, the TX_HASH matches. - for db_key, hashX in self.utxo_db.iterator(prefix=prefix): - tx_num_packed = db_key[-4:] - tx_num, = unpack('I', height) - - def read_undo_info(self, height): - '''Read undo information from a file for the current height.''' - return self.utxo_db.get(self.undo_key(height)) + def _db_hashX(self, tx_hash, idx_packed): + '''Return (hashX, tx_num_packed) for the given TXO. - def flush_undo_infos(self, batch_put, undo_infos): - '''undo_infos is a list of (undo_info, height) pairs.''' - for undo_info, height in undo_infos: - batch_put(self.undo_key(height), b''.join(undo_info)) + Both are None if not found.''' + # Key: b'h' + compressed_tx_hash + tx_idx + tx_num + # Value: hashX + prefix = b'h' + tx_hash[:4] + idx_packed - def clear_excess_undo_info(self): - '''Clear excess undo info. Only most recent N are kept.''' - prefix = b'U' - min_height = self.min_undo_height(self.db_height) - keys = [] - for key, hist in self.utxo_db.iterator(prefix=prefix): - height, = unpack('>I', key[-4:]) - if height >= min_height: - break - keys.append(key) + # Find which entry, if any, the TX_HASH matches. + for db_key, hashX in self.utxo_db.iterator(prefix=prefix): + tx_num_packed = db_key[-4:] + tx_num, = unpack('