diff --git a/docs/RELEASE-NOTES b/docs/RELEASE-NOTES index ecdc22d..a5e120f 100644 --- a/docs/RELEASE-NOTES +++ b/docs/RELEASE-NOTES @@ -1,3 +1,16 @@ +Version 0.3 +----------- + +- Database format has changed; old DBs are incompatible. They will + not work and will probably die miserably as I'm not yet versioning + them for helpful warnings (coming soon). +- The change in on-disk format makes UTXO flushes noticeably more + efficient. My gut feeling is it probably benefits HDDs more than + SSDs, but I have no numbers to back that up other than that my HDD + synced about 90 minutes (10%) faster. Until the treacle hits at + blocks 300k+ there will probably be little noticeable difference in + sync time. + Version 0.2.3 ------------- diff --git a/query.py b/query.py index 8fe7c94..a419c8d 100755 --- a/query.py +++ b/query.py @@ -16,7 +16,7 @@ Not currently documented; might become easier to use in future. import sys from server.env import Env -from server.DB import DB +from server.db import DB from lib.hash import hash_to_str diff --git a/server/block_processor.py b/server/block_processor.py index 288f558..33d3936 100644 --- a/server/block_processor.py +++ b/server/block_processor.py @@ -12,7 +12,7 @@ import array import asyncio import itertools import os -import struct +from struct import pack, unpack import time from bisect import bisect_left from collections import defaultdict @@ -28,11 +28,10 @@ from server.storage import open_db # Limits single address history to ~ 65536 * HIST_ENTRIES_PER_KEY entries HIST_ENTRIES_PER_KEY = 1024 HIST_VALUE_BYTES = HIST_ENTRIES_PER_KEY * 4 -ADDR_TX_HASH_LEN = 4 -UTXO_TX_HASH_LEN = 4 NO_HASH_168 = bytes([255]) * 21 NO_CACHE_ENTRY = NO_HASH_168 + bytes(12) + def formatted_time(t): '''Return a number of seconds as a string in days, hours, mins and secs.''' @@ -143,10 +142,6 @@ class Prefetcher(LoggedClass): return blocks, size -class MissingUTXOError(Exception): - '''Raised if a mempool tx input UTXO couldn't be found.''' - - class ChainReorg(Exception): '''Raised on a blockchain reorganisation.''' @@ -214,7 +209,7 @@ class MemPool(LoggedClass): # The mempool is unordered, so process all outputs first so # that looking for inputs has full info. script_hash168 = self.bp.coin.hash168_from_script - utxo_lookup = self.bp.utxo_lookup + db_utxo_lookup = self.bp.db_utxo_lookup def txout_pair(txout): return (script_hash168(txout.pk_script), txout.value) @@ -231,14 +226,8 @@ class MemPool(LoggedClass): mempool_entry = self.txs.get(hex_hash) if mempool_entry: return mempool_entry[1][txin.prev_idx], True - entry = utxo_lookup(txin.prev_hash, txin.prev_idx) - if entry == NO_CACHE_ENTRY: - # This happens when the daemon is a block ahead of us - # and has mempool txs spending new txs in that block - missing_utxos.append(txin) - raise MissingUTXOError - value, = struct.unpack('H', key[-2:]) @@ -513,7 +499,6 @@ class BlockProcessor(server.db.DB): def remove_stale_undo_items(self, batch): prefix = b'U' - unpack = struct.unpack cutoff = self.db_height - self.reorg_limit keys = [] for key, hist in self.db.iterator(prefix=prefix): @@ -614,7 +599,7 @@ class BlockProcessor(server.db.DB): def flush_history(self, batch): flush_start = time.time() - flush_id = struct.pack('>H', self.flush_count) + flush_id = pack('>H', self.flush_count) for hash168, hist in self.history.items(): key = b'H' + hash168 + flush_id @@ -736,7 +721,7 @@ class BlockProcessor(server.db.DB): def undo_key(self, height): '''DB key for undo information at the given height.''' - return b'U' + struct.pack('>I', height) + return b'U' + pack('>I', height) def write_undo_info(self, height, undo_info): '''Write out undo information for the current height.''' @@ -792,11 +777,11 @@ class BlockProcessor(server.db.DB): history = self.history tx_num = self.tx_count script_hash168 = self.coin.hash168_from_script - pack = struct.pack + s_pack = pack for tx, tx_hash in zip(txs, tx_hashes): hash168s = set() - tx_numb = pack('