Neil Booth
8 years ago
3 changed files with 247 additions and 282 deletions
@ -1,257 +0,0 @@ |
|||||
# Copyright (c) 2016, Neil Booth |
|
||||
# |
|
||||
# All rights reserved. |
|
||||
# |
|
||||
# See the file "LICENCE" for information about the copyright |
|
||||
# and warranty status of this software. |
|
||||
|
|
||||
'''UTXO and file cache. |
|
||||
|
|
||||
During initial sync these cache data and only flush occasionally. |
|
||||
Once synced flushes are performed after processing each block. |
|
||||
''' |
|
||||
|
|
||||
|
|
||||
import struct |
|
||||
|
|
||||
from lib.util import LoggedClass |
|
||||
from lib.hash import hash_to_str |
|
||||
|
|
||||
|
|
||||
# History can hold approx. 65536 * HIST_ENTRIES_PER_KEY entries |
|
||||
HIST_ENTRIES_PER_KEY = 1024 |
|
||||
HIST_VALUE_BYTES = HIST_ENTRIES_PER_KEY * 4 |
|
||||
ADDR_TX_HASH_LEN = 4 |
|
||||
UTXO_TX_HASH_LEN = 4 |
|
||||
NO_HASH_168 = bytes([255]) * 21 |
|
||||
NO_CACHE_ENTRY = NO_HASH_168 + bytes(12) |
|
||||
|
|
||||
|
|
||||
class UTXOCache(LoggedClass): |
|
||||
'''An in-memory UTXO cache, representing all changes to UTXO state |
|
||||
since the last DB flush. |
|
||||
|
|
||||
We want to store millions, perhaps 10s of millions of these in |
|
||||
memory for optimal performance during initial sync, because then |
|
||||
it is possible to spend UTXOs without ever going to the database |
|
||||
(other than as an entry in the address history, and there is only |
|
||||
one such entry per TX not per UTXO). So store them in a Python |
|
||||
dictionary with binary keys and values. |
|
||||
|
|
||||
Key: TX_HASH + TX_IDX (32 + 2 = 34 bytes) |
|
||||
Value: HASH168 + TX_NUM + VALUE (21 + 4 + 8 = 33 bytes) |
|
||||
|
|
||||
That's 67 bytes of raw data. Python dictionary overhead means |
|
||||
each entry actually uses about 187 bytes of memory. So almost |
|
||||
11.5 million UTXOs can fit in 2GB of RAM. There are approximately |
|
||||
42 million UTXOs on bitcoin mainnet at height 433,000. |
|
||||
|
|
||||
Semantics: |
|
||||
|
|
||||
add: Add it to the cache dictionary. |
|
||||
spend: Remove it if in the cache dictionary. |
|
||||
Otherwise it's been flushed to the DB. Each UTXO |
|
||||
is responsible for two entries in the DB stored using |
|
||||
compressed keys. Mark both for deletion in the next |
|
||||
flush of the in-memory UTXO cache. |
|
||||
|
|
||||
A UTXO is stored in the DB in 2 "tables": |
|
||||
|
|
||||
1. The output value and tx number. Must be keyed with a |
|
||||
hash168 prefix so the unspent outputs and balance of an |
|
||||
arbitrary address can be looked up with a simple key |
|
||||
traversal. |
|
||||
Key: b'u' + hash168 + compressed_tx_hash + tx_idx |
|
||||
Value: a (tx_num, value) pair |
|
||||
|
|
||||
2. Given a prevout, we need to be able to look up the UTXO key |
|
||||
to remove it. As is keyed by hash168 and that is not part |
|
||||
of the prevout, we need a hash168 lookup. |
|
||||
Key: b'h' + compressed tx_hash + tx_idx |
|
||||
Value: (hash168, tx_num) pair |
|
||||
|
|
||||
The compressed TX hash is just the first few bytes of the hash of |
|
||||
the TX the UTXO is in (and needn't be the same number of bytes in |
|
||||
each table). As this is not unique there will be collisions; |
|
||||
tx_num is stored to resolve them. The collision rate is around |
|
||||
0.02% for the hash168 table, and almost zero for the UTXO table |
|
||||
(there are around 100 collisions in the whole bitcoin blockchain). |
|
||||
|
|
||||
''' |
|
||||
|
|
||||
def __init__(self, get_tx_hash, db, coin): |
|
||||
super().__init__() |
|
||||
self.get_tx_hash = get_tx_hash |
|
||||
self.coin = coin |
|
||||
self.cache = {} |
|
||||
self.put = self.cache.__setitem__ |
|
||||
self.db = db |
|
||||
self.db_cache = {} |
|
||||
# Statistics |
|
||||
self.cache_spends = 0 |
|
||||
self.db_deletes = 0 |
|
||||
|
|
||||
def lookup(self, prev_hash, prev_idx): |
|
||||
'''Given a prevout, return a pair (hash168, value). |
|
||||
|
|
||||
If the UTXO is not found, returns (None, None).''' |
|
||||
# Fast track is it being in the cache |
|
||||
idx_packed = struct.pack('<H', prev_idx) |
|
||||
value = self.cache.get(prev_hash + idx_packed, None) |
|
||||
if value: |
|
||||
return value |
|
||||
return self.db_lookup(prev_hash, idx_packed, False) |
|
||||
|
|
||||
def db_lookup(self, tx_hash, idx_packed, delete=True): |
|
||||
'''Return a UTXO from the DB. Remove it if delete is True. |
|
||||
|
|
||||
Return NO_CACHE_ENTRY if it is not in the DB.''' |
|
||||
hash168 = self.hash168(tx_hash, idx_packed, delete) |
|
||||
if not hash168: |
|
||||
return NO_CACHE_ENTRY |
|
||||
|
|
||||
# Read the UTXO through the cache from the disk. We have to |
|
||||
# go through the cache because compressed keys can collide. |
|
||||
key = b'u' + hash168 + tx_hash[:UTXO_TX_HASH_LEN] + idx_packed |
|
||||
data = self.cache_get(key) |
|
||||
if data is None: |
|
||||
# Uh-oh, this should not happen... |
|
||||
self.logger.error('found no UTXO for {} / {:d} key {}' |
|
||||
.format(hash_to_str(tx_hash), |
|
||||
struct.unpack('<H', idx_packed), |
|
||||
bytes(key).hex())) |
|
||||
return NO_CACHE_ENTRY |
|
||||
|
|
||||
if len(data) == 12: |
|
||||
if delete: |
|
||||
self.db_deletes += 1 |
|
||||
self.cache_delete(key) |
|
||||
return hash168 + data |
|
||||
|
|
||||
# Resolve the compressed key collison. These should be |
|
||||
# extremely rare. |
|
||||
assert len(data) % 12 == 0 |
|
||||
for n in range(0, len(data), 12): |
|
||||
(tx_num, ) = struct.unpack('<I', data[n:n+4]) |
|
||||
this_tx_hash, height = self.get_tx_hash(tx_num) |
|
||||
if tx_hash == this_tx_hash: |
|
||||
result = hash168 + data[n:n+12] |
|
||||
if delete: |
|
||||
self.db_deletes += 1 |
|
||||
self.cache_write(key, data[:n] + data[n+12:]) |
|
||||
return result |
|
||||
|
|
||||
raise Exception('could not resolve UTXO key collision') |
|
||||
|
|
||||
def spend(self, prev_hash, prev_idx): |
|
||||
'''Spend a UTXO and return the cache's value. |
|
||||
|
|
||||
If the UTXO is not in the cache it must be on disk. |
|
||||
''' |
|
||||
# Fast track is it being in the cache |
|
||||
idx_packed = struct.pack('<H', prev_idx) |
|
||||
value = self.cache.pop(prev_hash + idx_packed, None) |
|
||||
if value: |
|
||||
self.cache_spends += 1 |
|
||||
return value |
|
||||
|
|
||||
return self.db_lookup(prev_hash, idx_packed) |
|
||||
|
|
||||
def hash168(self, tx_hash, idx_packed, delete=True): |
|
||||
'''Return the hash168 paid to by the given TXO. |
|
||||
|
|
||||
Look it up in the DB and removes it if delete is True. Return |
|
||||
None if not found. |
|
||||
''' |
|
||||
key = b'h' + tx_hash[:ADDR_TX_HASH_LEN] + idx_packed |
|
||||
data = self.cache_get(key) |
|
||||
if data is None: |
|
||||
# Assuming the DB is not corrupt, if delete is True this |
|
||||
# indicates a successful spend of a non-standard script |
|
||||
# as we don't currently record those |
|
||||
return None |
|
||||
|
|
||||
if len(data) == 25: |
|
||||
if delete: |
|
||||
self.cache_delete(key) |
|
||||
return data[:21] |
|
||||
|
|
||||
assert len(data) % 25 == 0 |
|
||||
|
|
||||
# Resolve the compressed key collision using the TX number |
|
||||
for n in range(0, len(data), 25): |
|
||||
(tx_num, ) = struct.unpack('<I', data[n+21:n+25]) |
|
||||
my_hash, height = self.get_tx_hash(tx_num) |
|
||||
if my_hash == tx_hash: |
|
||||
if delete: |
|
||||
self.cache_write(key, data[:n] + data[n+25:]) |
|
||||
return data[n:n+21] |
|
||||
|
|
||||
raise Exception('could not resolve hash168 collision') |
|
||||
|
|
||||
def cache_write(self, key, value): |
|
||||
'''Cache write of a (key, value) pair to the DB.''' |
|
||||
assert(bool(value)) |
|
||||
self.db_cache[key] = value |
|
||||
|
|
||||
def cache_delete(self, key): |
|
||||
'''Cache deletion of a key from the DB.''' |
|
||||
self.db_cache[key] = None |
|
||||
|
|
||||
def cache_get(self, key): |
|
||||
'''Fetch a value from the DB through our write cache.''' |
|
||||
value = self.db_cache.get(key) |
|
||||
if value: |
|
||||
return value |
|
||||
return self.db.get(key) |
|
||||
|
|
||||
def flush(self, batch): |
|
||||
'''Flush the cached DB writes and UTXO set to the batch.''' |
|
||||
# Care is needed because the writes generated by flushing the |
|
||||
# UTXO state may have keys in common with our write cache or |
|
||||
# may be in the DB already. |
|
||||
hcolls = ucolls = 0 |
|
||||
new_utxos = len(self.cache) |
|
||||
|
|
||||
for cache_key, cache_value in self.cache.items(): |
|
||||
# Frist write to the hash168 lookup table |
|
||||
key = b'h' + cache_key[:ADDR_TX_HASH_LEN] + cache_key[-2:] |
|
||||
value = cache_value[:25] |
|
||||
prior_value = self.cache_get(key) |
|
||||
if prior_value: # Should rarely happen |
|
||||
hcolls += 1 |
|
||||
value += prior_value |
|
||||
self.cache_write(key, value) |
|
||||
|
|
||||
# Next write the UTXO table |
|
||||
key = (b'u' + cache_value[:21] + cache_key[:UTXO_TX_HASH_LEN] |
|
||||
+ cache_key[-2:]) |
|
||||
value = cache_value[-12:] |
|
||||
prior_value = self.cache_get(key) |
|
||||
if prior_value: # Should almost never happen |
|
||||
ucolls += 1 |
|
||||
value += prior_value |
|
||||
self.cache_write(key, value) |
|
||||
|
|
||||
# GC-ing this now can only help the levelDB write. |
|
||||
self.cache = {} |
|
||||
self.put = self.cache.__setitem__ |
|
||||
|
|
||||
# Now we can update to the batch. |
|
||||
for key, value in self.db_cache.items(): |
|
||||
if value: |
|
||||
batch.put(key, value) |
|
||||
else: |
|
||||
batch.delete(key) |
|
||||
|
|
||||
self.db_cache = {} |
|
||||
|
|
||||
adds = new_utxos + self.cache_spends |
|
||||
|
|
||||
self.logger.info('UTXO cache adds: {:,d} spends: {:,d} ' |
|
||||
.format(adds, self.cache_spends)) |
|
||||
self.logger.info('UTXO DB adds: {:,d} spends: {:,d}. ' |
|
||||
'Collisions: hash168: {:,d} UTXO: {:,d}' |
|
||||
.format(new_utxos, self.db_deletes, |
|
||||
hcolls, ucolls)) |
|
||||
self.cache_spends = self.db_deletes = 0 |
|
||||
Loading…
Reference in new issue