Neil Booth
8 years ago
26 changed files with 1864 additions and 942 deletions
@ -0,0 +1,11 @@ |
|||
Version 0.02 |
|||
------------ |
|||
|
|||
- fix bug where tx counts were incorrectly saved |
|||
- large clean-up and refactoring of code, breakout into new files |
|||
- several efficiency improvements |
|||
- initial implementation of chain reorg handling |
|||
- work on RPC and TCP server functionality. Code committed but not |
|||
functional, so currently disabled |
|||
- note that some of the enivronment variables have been renamed, |
|||
see samples/scripts/NOTES for the list |
|||
@ -0,0 +1,62 @@ |
|||
#!/usr/bin/env python3 |
|||
|
|||
# See the file "LICENSE" for information about the copyright |
|||
# and warranty status of this software. |
|||
|
|||
import argparse |
|||
import asyncio |
|||
import json |
|||
from functools import partial |
|||
from os import environ |
|||
|
|||
|
|||
class RPCClient(asyncio.Protocol): |
|||
|
|||
def __init__(self, loop): |
|||
self.loop = loop |
|||
|
|||
def connection_made(self, transport): |
|||
self.transport = transport |
|||
|
|||
def connection_lost(self, exc): |
|||
self.loop.stop() |
|||
|
|||
def send(self, payload): |
|||
data = json.dumps(payload) + '\n' |
|||
self.transport.write(data.encode()) |
|||
|
|||
def data_received(self, data): |
|||
payload = json.loads(data.decode()) |
|||
self.transport.close() |
|||
print(json.dumps(payload, indent=4, sort_keys=True)) |
|||
|
|||
|
|||
def main(): |
|||
'''Send the RPC command to the server and print the result.''' |
|||
parser = argparse.ArgumentParser('Send electrumx an RPC command' ) |
|||
parser.add_argument('-p', '--port', metavar='port_num', type=int, |
|||
help='RPC port number') |
|||
parser.add_argument('command', nargs='*', default=[], |
|||
help='command to send') |
|||
args = parser.parse_args() |
|||
|
|||
if args.port is None: |
|||
args.port = int(environ.get('ELECTRUMX_RPC_PORT', 8000)) |
|||
|
|||
payload = {'method': args.command[0], 'params': args.command[1:]} |
|||
|
|||
loop = asyncio.get_event_loop() |
|||
proto_factory = partial(RPCClient, loop) |
|||
coro = loop.create_connection(proto_factory, 'localhost', args.port) |
|||
try: |
|||
transport, protocol = loop.run_until_complete(coro) |
|||
protocol.send(payload) |
|||
loop.run_forever() |
|||
except OSError: |
|||
print('error connecting - is ElectrumX running?') |
|||
finally: |
|||
loop.close() |
|||
|
|||
|
|||
if __name__ == '__main__': |
|||
main() |
|||
@ -0,0 +1,11 @@ |
|||
[Unit] |
|||
Description=Electrumx |
|||
After=network.target |
|||
|
|||
[Service] |
|||
EnvironmentFile=/etc/electrumx.conf |
|||
ExecStart=/home/electrumx/electrumx/server_main.py |
|||
User=electrumx |
|||
|
|||
[Install] |
|||
WantedBy=multi-user.target |
|||
@ -0,0 +1,714 @@ |
|||
# See the file "LICENSE" for information about the copyright |
|||
# and warranty status of this software. |
|||
|
|||
import array |
|||
import ast |
|||
import asyncio |
|||
import struct |
|||
import time |
|||
from bisect import bisect_left |
|||
from collections import defaultdict, namedtuple |
|||
from functools import partial |
|||
|
|||
import plyvel |
|||
|
|||
from server.cache import FSCache, UTXOCache, NO_CACHE_ENTRY |
|||
from server.daemon import DaemonError |
|||
from lib.hash import hash_to_str |
|||
from lib.script import ScriptPubKey |
|||
from lib.util import chunks, LoggedClass |
|||
|
|||
|
|||
def formatted_time(t): |
|||
'''Return a number of seconds as a string in days, hours, mins and |
|||
secs.''' |
|||
t = int(t) |
|||
return '{:d}d {:02d}h {:02d}m {:02d}s'.format( |
|||
t // 86400, (t % 86400) // 3600, (t % 3600) // 60, t % 60) |
|||
|
|||
|
|||
UTXO = namedtuple("UTXO", "tx_num tx_pos tx_hash height value") |
|||
|
|||
|
|||
class ChainError(Exception): |
|||
pass |
|||
|
|||
|
|||
class Prefetcher(LoggedClass): |
|||
'''Prefetches blocks (in the forward direction only).''' |
|||
|
|||
def __init__(self, daemon, height): |
|||
super().__init__() |
|||
self.daemon = daemon |
|||
self.semaphore = asyncio.Semaphore() |
|||
self.queue = asyncio.Queue() |
|||
self.queue_size = 0 |
|||
# Target cache size. Has little effect on sync time. |
|||
self.target_cache_size = 10 * 1024 * 1024 |
|||
self.fetched_height = height |
|||
self.recent_sizes = [0] |
|||
|
|||
async def get_blocks(self): |
|||
'''Returns a list of prefetched blocks.''' |
|||
blocks, total_size = await self.queue.get() |
|||
self.queue_size -= total_size |
|||
return blocks |
|||
|
|||
async def clear(self, height): |
|||
'''Clear prefetched blocks and restart from the given height. |
|||
|
|||
Used in blockchain reorganisations. This coroutine can be |
|||
called asynchronously to the _prefetch coroutine so we must |
|||
synchronize. |
|||
''' |
|||
with await self.semaphore: |
|||
while not self.queue.empty(): |
|||
self.queue.get_nowait() |
|||
self.queue_size = 0 |
|||
self.fetched_height = height |
|||
|
|||
async def start(self): |
|||
'''Loop forever polling for more blocks.''' |
|||
self.logger.info('prefetching blocks...') |
|||
while True: |
|||
while self.queue_size < self.target_cache_size: |
|||
try: |
|||
with await self.semaphore: |
|||
await self._prefetch() |
|||
except DaemonError as e: |
|||
self.logger.info('ignoring daemon errors: {}'.format(e)) |
|||
await asyncio.sleep(2) |
|||
|
|||
def _prefill_count(self, room): |
|||
ave_size = sum(self.recent_sizes) // len(self.recent_sizes) |
|||
count = room // ave_size if ave_size else 0 |
|||
return max(count, 10) |
|||
|
|||
async def _prefetch(self): |
|||
'''Prefetch blocks if there are any to prefetch.''' |
|||
daemon_height = await self.daemon.height() |
|||
max_count = min(daemon_height - self.fetched_height, 4000) |
|||
count = min(max_count, self._prefill_count(self.target_cache_size)) |
|||
first = self.fetched_height + 1 |
|||
hex_hashes = await self.daemon.block_hex_hashes(first, count) |
|||
if not hex_hashes: |
|||
return |
|||
|
|||
blocks = await self.daemon.raw_blocks(hex_hashes) |
|||
sizes = [len(block) for block in blocks] |
|||
total_size = sum(sizes) |
|||
self.queue.put_nowait((blocks, total_size)) |
|||
self.queue_size += total_size |
|||
self.fetched_height += len(blocks) |
|||
|
|||
# Keep 50 most recent block sizes for fetch count estimation |
|||
self.recent_sizes.extend(sizes) |
|||
excess = len(self.recent_sizes) - 50 |
|||
if excess > 0: |
|||
self.recent_sizes = self.recent_sizes[excess:] |
|||
|
|||
|
|||
class BlockProcessor(LoggedClass): |
|||
'''Process blocks and update the DB state to match. |
|||
|
|||
Employ a prefetcher to prefetch blocks in batches for processing. |
|||
Coordinate backing up in case of chain reorganisations. |
|||
''' |
|||
|
|||
def __init__(self, env, daemon): |
|||
super().__init__() |
|||
|
|||
self.daemon = daemon |
|||
|
|||
# Meta |
|||
self.utxo_MB = env.utxo_MB |
|||
self.hist_MB = env.hist_MB |
|||
self.next_cache_check = 0 |
|||
self.coin = env.coin |
|||
self.caught_up = False |
|||
self.reorg_limit = env.reorg_limit |
|||
|
|||
# Chain state (initialize to genesis in case of new DB) |
|||
self.db_height = -1 |
|||
self.db_tx_count = 0 |
|||
self.db_tip = b'\0' * 32 |
|||
self.flush_count = 0 |
|||
self.utxo_flush_count = 0 |
|||
self.wall_time = 0 |
|||
|
|||
# Open DB and metadata files. Record some of its state. |
|||
self.db = self.open_db(self.coin) |
|||
self.tx_count = self.db_tx_count |
|||
self.height = self.db_height |
|||
self.tip = self.db_tip |
|||
|
|||
# Caches to be flushed later. Headers and tx_hashes have one |
|||
# entry per block |
|||
self.history = defaultdict(partial(array.array, 'I')) |
|||
self.history_size = 0 |
|||
self.backup_hash168s = set() |
|||
self.utxo_cache = UTXOCache(self, self.db, self.coin) |
|||
self.fs_cache = FSCache(self.coin, self.height, self.tx_count) |
|||
self.prefetcher = Prefetcher(daemon, self.height) |
|||
|
|||
self.last_flush = time.time() |
|||
self.last_flush_tx_count = self.tx_count |
|||
|
|||
# Redirected member func |
|||
self.get_tx_hash = self.fs_cache.get_tx_hash |
|||
|
|||
# Log state |
|||
self.logger.info('{}/{} height: {:,d} tx count: {:,d} ' |
|||
'flush count: {:,d} utxo flush count: {:,d} ' |
|||
'sync time: {}' |
|||
.format(self.coin.NAME, self.coin.NET, self.height, |
|||
self.tx_count, self.flush_count, |
|||
self.utxo_flush_count, |
|||
formatted_time(self.wall_time))) |
|||
self.logger.info('reorg limit of {:,d} blocks' |
|||
.format(self.reorg_limit)) |
|||
self.logger.info('flushing UTXO cache at {:,d} MB' |
|||
.format(self.utxo_MB)) |
|||
self.logger.info('flushing history cache at {:,d} MB' |
|||
.format(self.hist_MB)) |
|||
|
|||
self.clean_db() |
|||
|
|||
def coros(self, force_backup=False): |
|||
if force_backup: |
|||
return [self.force_chain_reorg(True), self.prefetcher.start()] |
|||
else: |
|||
return [self.start(), self.prefetcher.start()] |
|||
|
|||
async def start(self): |
|||
'''External entry point for block processing. |
|||
|
|||
A simple wrapper that safely flushes the DB on clean |
|||
shutdown. |
|||
''' |
|||
try: |
|||
await self.advance_blocks() |
|||
finally: |
|||
self.flush(True) |
|||
|
|||
async def advance_blocks(self): |
|||
'''Loop forever processing blocks in the forward direction.''' |
|||
while True: |
|||
blocks = await self.prefetcher.get_blocks() |
|||
for block in blocks: |
|||
if not self.advance_block(block): |
|||
await self.handle_chain_reorg() |
|||
self.caught_up = False |
|||
break |
|||
await asyncio.sleep(0) # Yield |
|||
|
|||
if self.height != self.daemon.cached_height(): |
|||
continue |
|||
|
|||
if not self.caught_up: |
|||
self.caught_up = True |
|||
self.logger.info('caught up to height {:,d}' |
|||
.format(self.height)) |
|||
|
|||
# Flush everything when in caught-up state as queries |
|||
# are performed on DB not in-memory |
|||
self.flush(True) |
|||
|
|||
async def force_chain_reorg(self, to_genesis): |
|||
try: |
|||
await self.handle_chain_reorg(to_genesis) |
|||
finally: |
|||
self.flush(True) |
|||
|
|||
async def handle_chain_reorg(self, to_genesis=False): |
|||
# First get all state on disk |
|||
self.logger.info('chain reorg detected') |
|||
self.flush(True) |
|||
self.logger.info('finding common height...') |
|||
hashes = await self.reorg_hashes(to_genesis) |
|||
# Reverse and convert to hex strings. |
|||
hashes = [hash_to_str(hash) for hash in reversed(hashes)] |
|||
for hex_hashes in chunks(hashes, 50): |
|||
blocks = await self.daemon.raw_blocks(hex_hashes) |
|||
self.backup_blocks(blocks) |
|||
self.logger.info('backed up to height {:,d}'.format(self.height)) |
|||
await self.prefetcher.clear(self.height) |
|||
self.logger.info('prefetcher reset') |
|||
|
|||
async def reorg_hashes(self, to_genesis): |
|||
'''Return the list of hashes to back up beacuse of a reorg. |
|||
|
|||
The hashes are returned in order of increasing height.''' |
|||
def match_pos(hashes1, hashes2): |
|||
for n, (hash1, hash2) in enumerate(zip(hashes1, hashes2)): |
|||
if hash1 == hash2: |
|||
return n |
|||
return -1 |
|||
|
|||
start = self.height - 1 |
|||
count = 1 |
|||
while start > 0: |
|||
self.logger.info('start: {:,d} count: {:,d}'.format(start, count)) |
|||
hashes = self.fs_cache.block_hashes(start, count) |
|||
hex_hashes = [hash_to_str(hash) for hash in hashes] |
|||
d_hex_hashes = await self.daemon.block_hex_hashes(start, count) |
|||
n = match_pos(hex_hashes, d_hex_hashes) |
|||
if n >= 0 and not to_genesis: |
|||
start += n + 1 |
|||
break |
|||
count = min(count * 2, start) |
|||
start -= count |
|||
|
|||
# Hashes differ from height 'start' |
|||
count = (self.height - start) + 1 |
|||
|
|||
self.logger.info('chain was reorganised for {:,d} blocks from ' |
|||
'height {:,d} to height {:,d}' |
|||
.format(count, start, start + count - 1)) |
|||
|
|||
return self.fs_cache.block_hashes(start, count) |
|||
|
|||
def open_db(self, coin): |
|||
db_name = '{}-{}'.format(coin.NAME, coin.NET) |
|||
try: |
|||
db = plyvel.DB(db_name, create_if_missing=False, |
|||
error_if_exists=False, compression=None) |
|||
except: |
|||
db = plyvel.DB(db_name, create_if_missing=True, |
|||
error_if_exists=True, compression=None) |
|||
self.logger.info('created new database {}'.format(db_name)) |
|||
else: |
|||
self.logger.info('successfully opened database {}'.format(db_name)) |
|||
self.read_state(db) |
|||
|
|||
return db |
|||
|
|||
def read_state(self, db): |
|||
state = db.get(b'state') |
|||
state = ast.literal_eval(state.decode()) |
|||
if state['genesis'] != self.coin.GENESIS_HASH: |
|||
raise ChainError('DB genesis hash {} does not match coin {}' |
|||
.format(state['genesis_hash'], |
|||
self.coin.GENESIS_HASH)) |
|||
self.db_height = state['height'] |
|||
self.db_tx_count = state['tx_count'] |
|||
self.db_tip = state['tip'] |
|||
self.flush_count = state['flush_count'] |
|||
self.utxo_flush_count = state['utxo_flush_count'] |
|||
self.wall_time = state['wall_time'] |
|||
|
|||
def clean_db(self): |
|||
'''Clean out stale DB items. |
|||
|
|||
Stale DB items are excess history flushed since the most |
|||
recent UTXO flush (only happens on unclean shutdown), and aged |
|||
undo information. |
|||
''' |
|||
if self.flush_count < self.utxo_flush_count: |
|||
raise ChainError('DB corrupt: flush_count < utxo_flush_count') |
|||
with self.db.write_batch(transaction=True) as batch: |
|||
if self.flush_count > self.utxo_flush_count: |
|||
self.logger.info('DB shut down uncleanly. Scanning for ' |
|||
'excess history flushes...') |
|||
self.remove_excess_history(batch) |
|||
self.utxo_flush_count = self.flush_count |
|||
self.remove_stale_undo_items(batch) |
|||
self.flush_state(batch) |
|||
|
|||
def remove_excess_history(self, batch): |
|||
prefix = b'H' |
|||
unpack = struct.unpack |
|||
keys = [] |
|||
for key, hist in self.db.iterator(prefix=prefix): |
|||
flush_id, = unpack('>H', key[-2:]) |
|||
if flush_id > self.utxo_flush_count: |
|||
keys.append(key) |
|||
|
|||
self.logger.info('deleting {:,d} history entries' |
|||
.format(len(keys))) |
|||
for key in keys: |
|||
batch.delete(key) |
|||
|
|||
def remove_stale_undo_items(self, batch): |
|||
prefix = b'U' |
|||
unpack = struct.unpack |
|||
cutoff = self.db_height - self.reorg_limit |
|||
keys = [] |
|||
for key, hist in self.db.iterator(prefix=prefix): |
|||
height, = unpack('>I', key[-4:]) |
|||
if height > cutoff: |
|||
break |
|||
keys.append(key) |
|||
|
|||
self.logger.info('deleting {:,d} stale undo entries' |
|||
.format(len(keys))) |
|||
for key in keys: |
|||
batch.delete(key) |
|||
|
|||
def flush_state(self, batch): |
|||
'''Flush chain state to the batch.''' |
|||
now = time.time() |
|||
self.wall_time += now - self.last_flush |
|||
self.last_flush = now |
|||
self.last_flush_tx_count = self.tx_count |
|||
state = { |
|||
'genesis': self.coin.GENESIS_HASH, |
|||
'height': self.db_height, |
|||
'tx_count': self.db_tx_count, |
|||
'tip': self.db_tip, |
|||
'flush_count': self.flush_count, |
|||
'utxo_flush_count': self.utxo_flush_count, |
|||
'wall_time': self.wall_time, |
|||
} |
|||
batch.put(b'state', repr(state).encode()) |
|||
|
|||
def flush_utxos(self, batch): |
|||
self.logger.info('flushing UTXOs: {:,d} txs and {:,d} blocks' |
|||
.format(self.tx_count - self.db_tx_count, |
|||
self.height - self.db_height)) |
|||
self.utxo_cache.flush(batch) |
|||
self.utxo_flush_count = self.flush_count |
|||
self.db_tx_count = self.tx_count |
|||
self.db_height = self.height |
|||
self.db_tip = self.tip |
|||
|
|||
def assert_flushed(self): |
|||
'''Asserts state is fully flushed.''' |
|||
assert self.tx_count == self.db_tx_count |
|||
assert not self.history |
|||
assert not self.utxo_cache.cache |
|||
assert not self.utxo_cache.db_cache |
|||
assert not self.backup_hash168s |
|||
|
|||
def flush(self, flush_utxos=False): |
|||
'''Flush out cached state. |
|||
|
|||
History is always flushed. UTXOs are flushed if flush_utxos.''' |
|||
if self.height == self.db_height: |
|||
self.logger.info('nothing to flush') |
|||
self.assert_flushed() |
|||
return |
|||
|
|||
flush_start = time.time() |
|||
last_flush = self.last_flush |
|||
tx_diff = self.tx_count - self.last_flush_tx_count |
|||
|
|||
# Write out the files to the FS before flushing to the DB. If |
|||
# the DB transaction fails, the files being too long doesn't |
|||
# matter. But if writing the files fails we do not want to |
|||
# have updated the DB. |
|||
if self.height > self.db_height: |
|||
self.fs_cache.flush(self.height, self.tx_count) |
|||
|
|||
with self.db.write_batch(transaction=True) as batch: |
|||
# History first - fast and frees memory. Flush state last |
|||
# as it reads the wall time. |
|||
if self.height > self.db_height: |
|||
self.flush_history(batch) |
|||
else: |
|||
self.backup_history(batch) |
|||
if flush_utxos: |
|||
self.flush_utxos(batch) |
|||
self.flush_state(batch) |
|||
self.logger.info('committing transaction...') |
|||
|
|||
# Update and put the wall time again - otherwise we drop the |
|||
# time it took to commit the batch |
|||
self.flush_state(self.db) |
|||
|
|||
flush_time = int(self.last_flush - flush_start) |
|||
self.logger.info('flush #{:,d} to height {:,d} txs: {:,d} took {:,d}s' |
|||
.format(self.flush_count, self.height, self.tx_count, |
|||
flush_time)) |
|||
|
|||
# Catch-up stats |
|||
if not self.caught_up and tx_diff > 0: |
|||
daemon_height = self.daemon.cached_height() |
|||
txs_per_sec = int(self.tx_count / self.wall_time) |
|||
this_txs_per_sec = 1 + int(tx_diff / (self.last_flush - last_flush)) |
|||
if self.height > self.coin.TX_COUNT_HEIGHT: |
|||
tx_est = (daemon_height - self.height) * self.coin.TX_PER_BLOCK |
|||
else: |
|||
tx_est = ((daemon_height - self.coin.TX_COUNT_HEIGHT) |
|||
* self.coin.TX_PER_BLOCK |
|||
+ (self.coin.TX_COUNT - self.tx_count)) |
|||
|
|||
self.logger.info('tx/sec since genesis: {:,d}, ' |
|||
'since last flush: {:,d}' |
|||
.format(txs_per_sec, this_txs_per_sec)) |
|||
self.logger.info('sync time: {} ETA: {}' |
|||
.format(formatted_time(self.wall_time), |
|||
formatted_time(tx_est / this_txs_per_sec))) |
|||
|
|||
def flush_history(self, batch): |
|||
self.logger.info('flushing history') |
|||
assert not self.backup_hash168s |
|||
|
|||
self.flush_count += 1 |
|||
flush_id = struct.pack('>H', self.flush_count) |
|||
|
|||
for hash168, hist in self.history.items(): |
|||
key = b'H' + hash168 + flush_id |
|||
batch.put(key, hist.tobytes()) |
|||
|
|||
self.logger.info('{:,d} history entries in {:,d} addrs' |
|||
.format(self.history_size, len(self.history))) |
|||
|
|||
self.history = defaultdict(partial(array.array, 'I')) |
|||
self.history_size = 0 |
|||
|
|||
def backup_history(self, batch): |
|||
self.logger.info('backing up history to height {:,d} tx_count {:,d}' |
|||
.format(self.height, self.tx_count)) |
|||
|
|||
# Drop any NO_CACHE entry |
|||
self.backup_hash168s.discard(NO_CACHE_ENTRY) |
|||
assert not self.history |
|||
|
|||
nremoves = 0 |
|||
for hash168 in sorted(self.backup_hash168s): |
|||
prefix = b'H' + hash168 |
|||
deletes = [] |
|||
puts = {} |
|||
for key, hist in self.db.iterator(reverse=True, prefix=prefix): |
|||
a = array.array('I') |
|||
a.frombytes(hist) |
|||
# Remove all history entries >= self.tx_count |
|||
idx = bisect_left(a, self.tx_count) |
|||
nremoves += len(a) - idx |
|||
if idx > 0: |
|||
puts[key] = a[:idx].tobytes() |
|||
break |
|||
deletes.append(key) |
|||
|
|||
for key in deletes: |
|||
batch.delete(key) |
|||
for key, value in puts.items(): |
|||
batch.put(key, value) |
|||
|
|||
self.logger.info('removed {:,d} history entries from {:,d} addresses' |
|||
.format(nremoves, len(self.backup_hash168s))) |
|||
self.backup_hash168s = set() |
|||
|
|||
def cache_sizes(self): |
|||
'''Returns the approximate size of the cache, in MB.''' |
|||
# Good average estimates based on traversal of subobjects and |
|||
# requesting size from Python (see deep_getsizeof). For |
|||
# whatever reason Python O/S mem usage is typically +30% or |
|||
# more, so we scale our already bloated object sizes. |
|||
one_MB = int(1048576 / 1.3) |
|||
utxo_cache_size = len(self.utxo_cache.cache) * 187 |
|||
db_cache_size = len(self.utxo_cache.db_cache) * 105 |
|||
hist_cache_size = len(self.history) * 180 + self.history_size * 4 |
|||
utxo_MB = (db_cache_size + utxo_cache_size) // one_MB |
|||
hist_MB = hist_cache_size // one_MB |
|||
|
|||
self.logger.info('cache stats at height {:,d} daemon height: {:,d}' |
|||
.format(self.height, self.daemon.cached_height())) |
|||
self.logger.info(' entries: UTXO: {:,d} DB: {:,d} ' |
|||
'hist addrs: {:,d} hist size {:,d}' |
|||
.format(len(self.utxo_cache.cache), |
|||
len(self.utxo_cache.db_cache), |
|||
len(self.history), |
|||
self.history_size)) |
|||
self.logger.info(' size: {:,d}MB (UTXOs {:,d}MB hist {:,d}MB)' |
|||
.format(utxo_MB + hist_MB, utxo_MB, hist_MB)) |
|||
return utxo_MB, hist_MB |
|||
|
|||
def undo_key(self, height): |
|||
'''DB key for undo information at the given height.''' |
|||
return b'U' + struct.pack('>I', height) |
|||
|
|||
def write_undo_info(self, height, undo_info): |
|||
'''Write out undo information for the current height.''' |
|||
self.db.put(self.undo_key(height), undo_info) |
|||
|
|||
def read_undo_info(self, height): |
|||
'''Read undo information from a file for the current height.''' |
|||
return self.db.get(self.undo_key(height)) |
|||
|
|||
def advance_block(self, block): |
|||
# We must update the fs_cache before calling advance_txs() as |
|||
# the UTXO cache uses the fs_cache via get_tx_hash() to |
|||
# resolve compressed key collisions |
|||
header, tx_hashes, txs = self.coin.read_block(block) |
|||
self.fs_cache.advance_block(header, tx_hashes, txs) |
|||
prev_hash, header_hash = self.coin.header_hashes(header) |
|||
if prev_hash != self.tip: |
|||
return False |
|||
|
|||
self.tip = header_hash |
|||
self.height += 1 |
|||
undo_info = self.advance_txs(tx_hashes, txs) |
|||
if self.daemon.cached_height() - self.height <= self.reorg_limit: |
|||
self.write_undo_info(self.height, b''.join(undo_info)) |
|||
|
|||
# Check if we're getting full and time to flush? |
|||
now = time.time() |
|||
if now > self.next_cache_check: |
|||
self.next_cache_check = now + 60 |
|||
utxo_MB, hist_MB = self.cache_sizes() |
|||
if utxo_MB >= self.utxo_MB or hist_MB >= self.hist_MB: |
|||
self.flush(utxo_MB >= self.utxo_MB) |
|||
|
|||
return True |
|||
|
|||
def advance_txs(self, tx_hashes, txs): |
|||
put_utxo = self.utxo_cache.put |
|||
spend_utxo = self.utxo_cache.spend |
|||
undo_info = [] |
|||
|
|||
# Use local vars for speed in the loops |
|||
history = self.history |
|||
tx_num = self.tx_count |
|||
coin = self.coin |
|||
parse_script = ScriptPubKey.from_script |
|||
pack = struct.pack |
|||
|
|||
for tx, tx_hash in zip(txs, tx_hashes): |
|||
hash168s = set() |
|||
tx_numb = pack('<I', tx_num) |
|||
|
|||
# Spend the inputs |
|||
if not tx.is_coinbase: |
|||
for txin in tx.inputs: |
|||
cache_value = spend_utxo(txin.prev_hash, txin.prev_idx) |
|||
undo_info.append(cache_value) |
|||
hash168s.add(cache_value[:21]) |
|||
|
|||
# Add the new UTXOs |
|||
for idx, txout in enumerate(tx.outputs): |
|||
# Get the hash168. Ignore scripts we can't grok. |
|||
hash168 = parse_script(txout.pk_script, coin).hash168 |
|||
if hash168: |
|||
hash168s.add(hash168) |
|||
put_utxo(tx_hash + pack('<H', idx), |
|||
hash168 + tx_numb + pack('<Q', txout.value)) |
|||
|
|||
# Drop any NO_CACHE entry |
|||
hash168s.discard(NO_CACHE_ENTRY) |
|||
for hash168 in hash168s: |
|||
history[hash168].append(tx_num) |
|||
self.history_size += len(hash168s) |
|||
tx_num += 1 |
|||
|
|||
self.tx_count = tx_num |
|||
|
|||
return undo_info |
|||
|
|||
def backup_blocks(self, blocks): |
|||
'''Backup the blocks and flush. |
|||
|
|||
The blocks should be in order of decreasing height. |
|||
A flush is performed once the blocks are backed up. |
|||
''' |
|||
self.logger.info('backing up {:,d} blocks'.format(len(blocks))) |
|||
self.assert_flushed() |
|||
|
|||
for block in blocks: |
|||
header, tx_hashes, txs = self.coin.read_block(block) |
|||
prev_hash, header_hash = self.coin.header_hashes(header) |
|||
if header_hash != self.tip: |
|||
raise ChainError('backup block {} is not tip {} at height {:,d}' |
|||
.format(hash_to_str(header_hash), |
|||
hash_to_str(self.tip), self.height)) |
|||
|
|||
self.backup_txs(tx_hashes, txs) |
|||
self.fs_cache.backup_block() |
|||
self.tip = prev_hash |
|||
self.height -= 1 |
|||
|
|||
self.logger.info('backed up to height {:,d}'.format(self.height)) |
|||
self.flush(True) |
|||
|
|||
def backup_txs(self, tx_hashes, txs): |
|||
# Prevout values, in order down the block (coinbase first if present) |
|||
# undo_info is in reverse block order |
|||
undo_info = self.read_undo_info(self.height) |
|||
n = len(undo_info) |
|||
|
|||
# Use local vars for speed in the loops |
|||
pack = struct.pack |
|||
put_utxo = self.utxo_cache.put |
|||
spend_utxo = self.utxo_cache.spend |
|||
hash168s = self.backup_hash168s |
|||
|
|||
rtxs = reversed(txs) |
|||
rtx_hashes = reversed(tx_hashes) |
|||
|
|||
for tx_hash, tx in zip(rtx_hashes, rtxs): |
|||
# Spend the outputs |
|||
for idx, txout in enumerate(tx.outputs): |
|||
cache_value = spend_utxo(tx_hash, idx) |
|||
hash168s.add(cache_value[:21]) |
|||
|
|||
# Restore the inputs |
|||
if not tx.is_coinbase: |
|||
for txin in reversed(tx.inputs): |
|||
n -= 33 |
|||
undo_item = undo_info[n:n+33] |
|||
put_utxo(txin.prev_hash + pack('<H', txin.prev_idx), |
|||
undo_item) |
|||
hash168s.add(undo_item[:21]) |
|||
|
|||
assert n == 0 |
|||
self.tx_count -= len(txs) |
|||
|
|||
@staticmethod |
|||
def resolve_limit(limit): |
|||
if limit is None: |
|||
return -1 |
|||
assert isinstance(limit, int) and limit >= 0 |
|||
return limit |
|||
|
|||
def get_history(self, hash168, limit=1000): |
|||
'''Generator that returns an unpruned, sorted list of (tx_hash, |
|||
height) tuples of transactions that touched the address, |
|||
earliest in the blockchain first. Includes both spending and |
|||
receiving transactions. By default yields at most 1000 entries. |
|||
Set limit to None to get them all. |
|||
''' |
|||
limit = self.resolve_limit(limit) |
|||
prefix = b'H' + hash168 |
|||
for key, hist in self.db.iterator(prefix=prefix): |
|||
a = array.array('I') |
|||
a.frombytes(hist) |
|||
for tx_num in a: |
|||
if limit == 0: |
|||
return |
|||
yield self.get_tx_hash(tx_num) |
|||
limit -= 1 |
|||
|
|||
def get_balance(self, hash168): |
|||
'''Returns the confirmed balance of an address.''' |
|||
return sum(utxo.value for utxo in self.get_utxos(hash168, limit=None)) |
|||
|
|||
def get_utxos(self, hash168, limit=1000): |
|||
'''Generator that yields all UTXOs for an address sorted in no |
|||
particular order. By default yields at most 1000 entries. |
|||
Set limit to None to get them all. |
|||
''' |
|||
limit = self.resolve_limit(limit) |
|||
unpack = struct.unpack |
|||
prefix = b'u' + hash168 |
|||
utxos = [] |
|||
for k, v in self.db.iterator(prefix=prefix): |
|||
(tx_pos, ) = unpack('<H', k[-2:]) |
|||
|
|||
for n in range(0, len(v), 12): |
|||
if limit == 0: |
|||
return |
|||
(tx_num, ) = unpack('<I', v[n:n+4]) |
|||
(value, ) = unpack('<Q', v[n+4:n+12]) |
|||
tx_hash, height = self.get_tx_hash(tx_num) |
|||
yield UTXO(tx_num, tx_pos, tx_hash, height, value) |
|||
limit -= 1 |
|||
|
|||
def get_utxos_sorted(self, hash168): |
|||
'''Returns all the UTXOs for an address sorted by height and |
|||
position in the block.''' |
|||
return sorted(self.get_utxos(hash168, limit=None)) |
|||
|
|||
def get_current_header(self): |
|||
'''Returns the current header as a dictionary.''' |
|||
return self.fs_cache.encode_header(self.height) |
|||
@ -0,0 +1,400 @@ |
|||
# See the file "LICENSE" for information about the copyright |
|||
# and warranty status of this software. |
|||
|
|||
import array |
|||
import itertools |
|||
import os |
|||
import struct |
|||
from bisect import bisect_right |
|||
|
|||
from lib.util import chunks, LoggedClass |
|||
from lib.hash import double_sha256, hash_to_str |
|||
|
|||
|
|||
# History can hold approx. 65536 * HIST_ENTRIES_PER_KEY entries |
|||
HIST_ENTRIES_PER_KEY = 1024 |
|||
HIST_VALUE_BYTES = HIST_ENTRIES_PER_KEY * 4 |
|||
ADDR_TX_HASH_LEN = 4 |
|||
UTXO_TX_HASH_LEN = 4 |
|||
NO_HASH_168 = bytes([255]) * 21 |
|||
NO_CACHE_ENTRY = NO_HASH_168 + bytes(12) |
|||
|
|||
|
|||
class UTXOCache(LoggedClass): |
|||
'''An in-memory UTXO cache, representing all changes to UTXO state |
|||
since the last DB flush. |
|||
|
|||
We want to store millions, perhaps 10s of millions of these in |
|||
memory for optimal performance during initial sync, because then |
|||
it is possible to spend UTXOs without ever going to the database |
|||
(other than as an entry in the address history, and there is only |
|||
one such entry per TX not per UTXO). So store them in a Python |
|||
dictionary with binary keys and values. |
|||
|
|||
Key: TX_HASH + TX_IDX (32 + 2 = 34 bytes) |
|||
Value: HASH168 + TX_NUM + VALUE (21 + 4 + 8 = 33 bytes) |
|||
|
|||
That's 67 bytes of raw data. Python dictionary overhead means |
|||
each entry actually uses about 187 bytes of memory. So almost |
|||
11.5 million UTXOs can fit in 2GB of RAM. There are approximately |
|||
42 million UTXOs on bitcoin mainnet at height 433,000. |
|||
|
|||
Semantics: |
|||
|
|||
add: Add it to the cache dictionary. |
|||
spend: Remove it if in the cache dictionary. |
|||
Otherwise it's been flushed to the DB. Each UTXO |
|||
is responsible for two entries in the DB stored using |
|||
compressed keys. Mark both for deletion in the next |
|||
flush of the in-memory UTXO cache. |
|||
|
|||
A UTXO is stored in the DB in 2 "tables": |
|||
|
|||
1. The output value and tx number. Must be keyed with a |
|||
hash168 prefix so the unspent outputs and balance of an |
|||
arbitrary address can be looked up with a simple key |
|||
traversal. |
|||
Key: b'u' + hash168 + compressed_tx_hash + tx_idx |
|||
Value: a (tx_num, value) pair |
|||
|
|||
2. Given a prevout, we need to be able to look up the UTXO key |
|||
to remove it. As is keyed by hash168 and that is not part |
|||
of the prevout, we need a hash168 lookup. |
|||
Key: b'h' + compressed tx_hash + tx_idx |
|||
Value: (hash168, tx_num) pair |
|||
|
|||
The compressed TX hash is just the first few bytes of the hash of |
|||
the TX the UTXO is in (and needn't be the same number of bytes in |
|||
each table). As this is not unique there will be collisions; |
|||
tx_num is stored to resolve them. The collision rate is around |
|||
0.02% for the hash168 table, and almost zero for the UTXO table |
|||
(there are around 100 collisions in the whole bitcoin blockchain). |
|||
|
|||
''' |
|||
|
|||
def __init__(self, parent, db, coin): |
|||
super().__init__() |
|||
self.parent = parent |
|||
self.coin = coin |
|||
self.cache = {} |
|||
self.put = self.cache.__setitem__ |
|||
self.db = db |
|||
self.db_cache = {} |
|||
# Statistics |
|||
self.cache_spends = 0 |
|||
self.db_deletes = 0 |
|||
|
|||
def spend(self, prev_hash, prev_idx): |
|||
'''Spend a UTXO and return the cache's value. |
|||
|
|||
If the UTXO is not in the cache it must be on disk. |
|||
''' |
|||
# Fast track is it's in the cache |
|||
pack = struct.pack |
|||
idx_packed = pack('<H', prev_idx) |
|||
value = self.cache.pop(prev_hash + idx_packed, None) |
|||
if value: |
|||
self.cache_spends += 1 |
|||
return value |
|||
|
|||
# Oh well. Find and remove it from the DB. |
|||
hash168 = self.hash168(prev_hash, idx_packed) |
|||
if not hash168: |
|||
return NO_CACHE_ENTRY |
|||
|
|||
self.db_deletes += 1 |
|||
|
|||
# Read the UTXO through the cache from the disk. We have to |
|||
# go through the cache because compressed keys can collide. |
|||
key = b'u' + hash168 + prev_hash[:UTXO_TX_HASH_LEN] + idx_packed |
|||
data = self.cache_get(key) |
|||
if data is None: |
|||
# Uh-oh, this should not happen... |
|||
self.logger.error('found no UTXO for {} / {:d} key {}' |
|||
.format(hash_to_str(prev_hash), prev_idx, |
|||
bytes(key).hex())) |
|||
return NO_CACHE_ENTRY |
|||
|
|||
if len(data) == 12: |
|||
self.cache_delete(key) |
|||
return hash168 + data |
|||
|
|||
# Resolve the compressed key collison. These should be |
|||
# extremely rare. |
|||
assert len(data) % 12 == 0 |
|||
for n in range(0, len(data), 12): |
|||
(tx_num, ) = struct.unpack('<I', data[n:n+4]) |
|||
tx_hash, height = self.parent.get_tx_hash(tx_num) |
|||
if prev_hash == tx_hash: |
|||
result = hash168 + data[n: n+12] |
|||
data = data[:n] + data[n+12:] |
|||
self.cache_write(key, data) |
|||
return result |
|||
|
|||
raise Exception('could not resolve UTXO key collision') |
|||
|
|||
def hash168(self, tx_hash, idx_packed): |
|||
'''Return the hash168 paid to by the given TXO. |
|||
|
|||
Refers to the database. Returns None if not found (which is |
|||
indicates a non-standard script). |
|||
''' |
|||
key = b'h' + tx_hash[:ADDR_TX_HASH_LEN] + idx_packed |
|||
data = self.cache_get(key) |
|||
if data is None: |
|||
# Assuming the DB is not corrupt, this indicates a |
|||
# successful spend of a non-standard script |
|||
return None |
|||
|
|||
if len(data) == 25: |
|||
self.cache_delete(key) |
|||
return data[:21] |
|||
|
|||
assert len(data) % 25 == 0 |
|||
|
|||
# Resolve the compressed key collision using the TX number |
|||
for n in range(0, len(data), 25): |
|||
(tx_num, ) = struct.unpack('<I', data[n+21:n+25]) |
|||
my_hash, height = self.parent.get_tx_hash(tx_num) |
|||
if my_hash == tx_hash: |
|||
self.cache_write(key, data[:n] + data[n+25:]) |
|||
return data[n:n+21] |
|||
|
|||
raise Exception('could not resolve hash168 collision') |
|||
|
|||
def cache_write(self, key, value): |
|||
'''Cache write of a (key, value) pair to the DB.''' |
|||
assert(bool(value)) |
|||
self.db_cache[key] = value |
|||
|
|||
def cache_delete(self, key): |
|||
'''Cache deletion of a key from the DB.''' |
|||
self.db_cache[key] = None |
|||
|
|||
def cache_get(self, key): |
|||
'''Fetch a value from the DB through our write cache.''' |
|||
value = self.db_cache.get(key) |
|||
if value: |
|||
return value |
|||
return self.db.get(key) |
|||
|
|||
def flush(self, batch): |
|||
'''Flush the cached DB writes and UTXO set to the batch.''' |
|||
# Care is needed because the writes generated by flushing the |
|||
# UTXO state may have keys in common with our write cache or |
|||
# may be in the DB already. |
|||
hcolls = ucolls = 0 |
|||
new_utxos = len(self.cache) |
|||
|
|||
for cache_key, cache_value in self.cache.items(): |
|||
# Frist write to the hash168 lookup table |
|||
key = b'h' + cache_key[:ADDR_TX_HASH_LEN] + cache_key[-2:] |
|||
value = cache_value[:25] |
|||
prior_value = self.cache_get(key) |
|||
if prior_value: # Should rarely happen |
|||
hcolls += 1 |
|||
value += prior_value |
|||
self.cache_write(key, value) |
|||
|
|||
# Next write the UTXO table |
|||
key = (b'u' + cache_value[:21] + cache_key[:UTXO_TX_HASH_LEN] |
|||
+ cache_key[-2:]) |
|||
value = cache_value[-12:] |
|||
prior_value = self.cache_get(key) |
|||
if prior_value: # Should almost never happen |
|||
ucolls += 1 |
|||
value += prior_value |
|||
self.cache_write(key, value) |
|||
|
|||
# GC-ing this now can only help the levelDB write. |
|||
self.cache = {} |
|||
self.put = self.cache.__setitem__ |
|||
|
|||
# Now we can update to the batch. |
|||
for key, value in self.db_cache.items(): |
|||
if value: |
|||
batch.put(key, value) |
|||
else: |
|||
batch.delete(key) |
|||
|
|||
self.db_cache = {} |
|||
|
|||
adds = new_utxos + self.cache_spends |
|||
|
|||
self.logger.info('UTXO cache adds: {:,d} spends: {:,d} ' |
|||
.format(adds, self.cache_spends)) |
|||
self.logger.info('UTXO DB adds: {:,d} spends: {:,d}. ' |
|||
'Collisions: hash168: {:,d} UTXO: {:,d}' |
|||
.format(new_utxos, self.db_deletes, |
|||
hcolls, ucolls)) |
|||
self.cache_spends = self.db_deletes = 0 |
|||
|
|||
|
|||
class FSCache(LoggedClass): |
|||
|
|||
def __init__(self, coin, height, tx_count): |
|||
super().__init__() |
|||
|
|||
self.coin = coin |
|||
self.tx_hash_file_size = 16 * 1024 * 1024 |
|||
assert self.tx_hash_file_size % 32 == 0 |
|||
|
|||
# On-disk values, updated by a flush |
|||
self.height = height |
|||
|
|||
# Unflushed items |
|||
self.headers = [] |
|||
self.tx_hashes = [] |
|||
|
|||
is_new = height == -1 |
|||
self.headers_file = self.open_file('headers', is_new) |
|||
self.txcount_file = self.open_file('txcount', is_new) |
|||
|
|||
# tx_counts[N] has the cumulative number of txs at the end of |
|||
# height N. So tx_counts[0] is 1 - the genesis coinbase |
|||
self.tx_counts = array.array('I') |
|||
self.txcount_file.seek(0) |
|||
self.tx_counts.fromfile(self.txcount_file, self.height + 1) |
|||
if self.tx_counts: |
|||
assert tx_count == self.tx_counts[-1] |
|||
else: |
|||
assert tx_count == 0 |
|||
|
|||
def open_file(self, filename, create=False): |
|||
'''Open the file name. Return its handle.''' |
|||
try: |
|||
return open(filename, 'rb+') |
|||
except FileNotFoundError: |
|||
if create: |
|||
return open(filename, 'wb+') |
|||
raise |
|||
|
|||
def advance_block(self, header, tx_hashes, txs): |
|||
'''Update the FS cache for a new block.''' |
|||
prior_tx_count = self.tx_counts[-1] if self.tx_counts else 0 |
|||
|
|||
# Cache the new header, tx hashes and cumulative tx count |
|||
self.headers.append(header) |
|||
self.tx_hashes.append(tx_hashes) |
|||
self.tx_counts.append(prior_tx_count + len(txs)) |
|||
|
|||
def backup_block(self): |
|||
'''Revert a block.''' |
|||
assert not self.headers |
|||
assert not self.tx_hashes |
|||
assert self.height >= 0 |
|||
# Just update in-memory. It doesn't matter if disk files are |
|||
# too long, they will be overwritten when advancing. |
|||
self.height -= 1 |
|||
self.tx_counts.pop() |
|||
|
|||
def flush(self, new_height, new_tx_count): |
|||
'''Flush the things stored on the filesystem. |
|||
The arguments are passed for sanity check assertions only.''' |
|||
self.logger.info('flushing to file system') |
|||
|
|||
blocks_done = len(self.headers) |
|||
prior_tx_count = self.tx_counts[self.height] if self.height >= 0 else 0 |
|||
cur_tx_count = self.tx_counts[-1] if self.tx_counts else 0 |
|||
txs_done = cur_tx_count - prior_tx_count |
|||
|
|||
assert self.height + blocks_done == new_height |
|||
assert len(self.tx_hashes) == blocks_done |
|||
assert len(self.tx_counts) == new_height + 1 |
|||
assert cur_tx_count == new_tx_count, \ |
|||
'cur: {:,d} new: {:,d}'.format(cur_tx_count, new_tx_count) |
|||
|
|||
# First the headers |
|||
headers = b''.join(self.headers) |
|||
header_len = self.coin.HEADER_LEN |
|||
self.headers_file.seek((self.height + 1) * header_len) |
|||
self.headers_file.write(headers) |
|||
self.headers_file.flush() |
|||
|
|||
# Then the tx counts |
|||
self.txcount_file.seek((self.height + 1) * self.tx_counts.itemsize) |
|||
self.txcount_file.write(self.tx_counts[self.height + 1:]) |
|||
self.txcount_file.flush() |
|||
|
|||
# Finally the hashes |
|||
hashes = memoryview(b''.join(itertools.chain(*self.tx_hashes))) |
|||
assert len(hashes) % 32 == 0 |
|||
assert len(hashes) // 32 == txs_done |
|||
cursor = 0 |
|||
file_pos = prior_tx_count * 32 |
|||
while cursor < len(hashes): |
|||
file_num, offset = divmod(file_pos, self.tx_hash_file_size) |
|||
size = min(len(hashes) - cursor, self.tx_hash_file_size - offset) |
|||
filename = 'hashes{:04d}'.format(file_num) |
|||
with self.open_file(filename, create=True) as f: |
|||
f.seek(offset) |
|||
f.write(hashes[cursor:cursor + size]) |
|||
cursor += size |
|||
file_pos += size |
|||
|
|||
os.sync() |
|||
|
|||
self.tx_hashes = [] |
|||
self.headers = [] |
|||
self.height += blocks_done |
|||
|
|||
def read_headers(self, height, count): |
|||
read_count = min(count, self.height + 1 - height) |
|||
|
|||
assert height >= 0 and read_count >= 0 |
|||
assert count <= read_count + len(self.headers) |
|||
|
|||
result = b'' |
|||
if read_count > 0: |
|||
header_len = self.coin.HEADER_LEN |
|||
self.headers_file.seek(height * header_len) |
|||
result = self.headers_file.read(read_count * header_len) |
|||
|
|||
count -= read_count |
|||
if count: |
|||
start = (height + read_count) - (self.height + 1) |
|||
result += b''.join(self.headers[start: start + count]) |
|||
|
|||
return result |
|||
|
|||
def get_tx_hash(self, tx_num): |
|||
'''Returns the tx_hash and height of a tx number.''' |
|||
height = bisect_right(self.tx_counts, tx_num) |
|||
|
|||
# Is this on disk or unflushed? |
|||
if height > self.height: |
|||
tx_hashes = self.tx_hashes[height - (self.height + 1)] |
|||
tx_hash = tx_hashes[tx_num - self.tx_counts[height - 1]] |
|||
else: |
|||
file_pos = tx_num * 32 |
|||
file_num, offset = divmod(file_pos, self.tx_hash_file_size) |
|||
filename = 'hashes{:04d}'.format(file_num) |
|||
with self.open_file(filename) as f: |
|||
f.seek(offset) |
|||
tx_hash = f.read(32) |
|||
|
|||
return tx_hash, height |
|||
|
|||
def block_hashes(self, height, count): |
|||
headers = self.read_headers(height, count) |
|||
hlen = self.coin.HEADER_LEN |
|||
return [double_sha256(header) for header in chunks(headers, hlen)] |
|||
|
|||
def encode_header(self, height): |
|||
if height < 0 or height > self.height + len(self.headers): |
|||
raise Exception('no header information for height {:,d}' |
|||
.format(height)) |
|||
header = self.read_headers(self.height, 1) |
|||
unpack = struct.unpack |
|||
version, = unpack('<I', header[:4]) |
|||
timestamp, bits, nonce = unpack('<III', header[68:80]) |
|||
|
|||
return { |
|||
'block_height': self.height, |
|||
'version': version, |
|||
'prev_block_hash': hash_to_str(header[4:36]), |
|||
'merkle_root': hash_to_str(header[36:68]), |
|||
'timestamp': timestamp, |
|||
'bits': bits, |
|||
'nonce': nonce, |
|||
} |
|||
@ -0,0 +1,165 @@ |
|||
# See the file "LICENSE" for information about the copyright |
|||
# and warranty status of this software. |
|||
|
|||
import asyncio |
|||
import signal |
|||
import traceback |
|||
from functools import partial |
|||
|
|||
from server.daemon import Daemon, DaemonError |
|||
from server.block_processor import BlockProcessor |
|||
from server.protocol import ElectrumX, LocalRPC |
|||
from lib.hash import (sha256, double_sha256, hash_to_str, |
|||
Base58, hex_str_to_hash) |
|||
from lib.util import LoggedClass |
|||
|
|||
|
|||
class Controller(LoggedClass): |
|||
|
|||
def __init__(self, loop, env): |
|||
'''Create up the controller. |
|||
|
|||
Creates DB, Daemon and BlockProcessor instances. |
|||
''' |
|||
super().__init__() |
|||
self.loop = loop |
|||
self.env = env |
|||
self.daemon = Daemon(env.daemon_url) |
|||
self.block_processor = BlockProcessor(env, self.daemon) |
|||
self.servers = [] |
|||
self.sessions = set() |
|||
self.addresses = {} |
|||
self.jobs = set() |
|||
self.peers = {} |
|||
|
|||
def start(self): |
|||
'''Prime the event loop with asynchronous servers and jobs.''' |
|||
env = self.env |
|||
loop = self.loop |
|||
|
|||
coros = self.block_processor.coros() |
|||
|
|||
if False: |
|||
self.start_servers() |
|||
coros.append(self.reap_jobs()) |
|||
|
|||
for coro in coros: |
|||
asyncio.ensure_future(coro) |
|||
|
|||
# Signal handlers |
|||
for signame in ('SIGINT', 'SIGTERM'): |
|||
loop.add_signal_handler(getattr(signal, signame), |
|||
partial(self.on_signal, signame)) |
|||
|
|||
def start_servers(self): |
|||
protocol = partial(LocalRPC, self) |
|||
if env.rpc_port is not None: |
|||
host = 'localhost' |
|||
rpc_server = loop.create_server(protocol, host, env.rpc_port) |
|||
self.servers.append(loop.run_until_complete(rpc_server)) |
|||
self.logger.info('RPC server listening on {}:{:d}' |
|||
.format(host, env.rpc_port)) |
|||
|
|||
protocol = partial(ElectrumX, self, self.daemon, env) |
|||
if env.tcp_port is not None: |
|||
tcp_server = loop.create_server(protocol, env.host, env.tcp_port) |
|||
self.servers.append(loop.run_until_complete(tcp_server)) |
|||
self.logger.info('TCP server listening on {}:{:d}' |
|||
.format(env.host, env.tcp_port)) |
|||
|
|||
if env.ssl_port is not None: |
|||
ssl_server = loop.create_server(protocol, env.host, env.ssl_port) |
|||
self.servers.append(loop.run_until_complete(ssl_server)) |
|||
self.logger.info('SSL server listening on {}:{:d}' |
|||
.format(env.host, env.ssl_port)) |
|||
|
|||
def stop(self): |
|||
'''Close the listening servers.''' |
|||
for server in self.servers: |
|||
server.close() |
|||
|
|||
def on_signal(self, signame): |
|||
'''Call on receipt of a signal to cleanly shutdown.''' |
|||
self.logger.warning('received {} signal, preparing to shut down' |
|||
.format(signame)) |
|||
for task in asyncio.Task.all_tasks(self.loop): |
|||
task.cancel() |
|||
|
|||
def add_session(self, session): |
|||
self.sessions.add(session) |
|||
|
|||
def remove_session(self, session): |
|||
self.sessions.remove(session) |
|||
|
|||
def add_job(self, coro): |
|||
'''Queue a job for asynchronous processing.''' |
|||
self.jobs.add(asyncio.ensure_future(coro)) |
|||
|
|||
async def reap_jobs(self): |
|||
while True: |
|||
jobs = set() |
|||
for job in self.jobs: |
|||
if job.done(): |
|||
try: |
|||
job.result() |
|||
except Exception as e: |
|||
traceback.print_exc() |
|||
else: |
|||
jobs.add(job) |
|||
self.logger.info('reaped {:d} jobs, {:d} jobs pending' |
|||
.format(len(self.jobs) - len(jobs), len(jobs))) |
|||
self.jobs = jobs |
|||
await asyncio.sleep(5) |
|||
|
|||
def address_status(self, hash168): |
|||
'''Returns status as 32 bytes.''' |
|||
status = self.addresses.get(hash168) |
|||
if status is None: |
|||
history = self.block_processor.get_history(hash168) |
|||
status = ''.join('{}:{:d}:'.format(hash_to_str(tx_hash), height) |
|||
for tx_hash, height in history) |
|||
if status: |
|||
status = sha256(status.encode()) |
|||
self.addresses[hash168] = status |
|||
|
|||
return status |
|||
|
|||
async def get_merkle(self, tx_hash, height): |
|||
'''tx_hash is a hex string.''' |
|||
block_hash = await self.daemon.send_single('getblockhash', (height,)) |
|||
block = await self.daemon.send_single('getblock', (block_hash, True)) |
|||
tx_hashes = block['tx'] |
|||
# This will throw if the tx_hash is bad |
|||
pos = tx_hashes.index(tx_hash) |
|||
|
|||
idx = pos |
|||
hashes = [hex_str_to_hash(txh) for txh in tx_hashes] |
|||
merkle_branch = [] |
|||
while len(hashes) > 1: |
|||
if len(hashes) & 1: |
|||
hashes.append(hashes[-1]) |
|||
idx = idx - 1 if (idx & 1) else idx + 1 |
|||
merkle_branch.append(hash_to_str(hashes[idx])) |
|||
idx //= 2 |
|||
hashes = [double_sha256(hashes[n] + hashes[n + 1]) |
|||
for n in range(0, len(hashes), 2)] |
|||
|
|||
return {"block_height": height, "merkle": merkle_branch, "pos": pos} |
|||
|
|||
def get_peers(self): |
|||
'''Returns a dictionary of IRC nick to (ip, host, ports) tuples, one |
|||
per peer.''' |
|||
return self.peers |
|||
|
|||
def height(self): |
|||
return self.block_processor.height |
|||
|
|||
def get_current_header(self): |
|||
return self.block_processor.get_current_header() |
|||
|
|||
def get_history(self, hash168): |
|||
history = self.block_processor.get_history(hash168, limit=None) |
|||
return [ |
|||
{'tx_hash': hash_to_str(tx_hash), 'height': height} |
|||
for tx_hash, height in history |
|||
] |
|||
@ -0,0 +1,98 @@ |
|||
# See the file "LICENSE" for information about the copyright |
|||
# and warranty status of this software. |
|||
|
|||
'''Classes for handling asynchronous connections to a blockchain |
|||
daemon.''' |
|||
|
|||
import asyncio |
|||
import json |
|||
|
|||
import aiohttp |
|||
|
|||
from lib.util import LoggedClass |
|||
|
|||
|
|||
class DaemonError(Exception): |
|||
'''Raised when the daemon returns an error in its results that |
|||
cannot be remedied by retrying.''' |
|||
|
|||
|
|||
class Daemon(LoggedClass): |
|||
'''Handles connections to a daemon at the given URL.''' |
|||
|
|||
def __init__(self, url): |
|||
super().__init__() |
|||
self.url = url |
|||
self._height = None |
|||
self.logger.info('connecting to daemon at URL {}'.format(url)) |
|||
|
|||
async def send_single(self, method, params=None): |
|||
payload = {'method': method} |
|||
if params: |
|||
payload['params'] = params |
|||
result, = await self.send((payload, )) |
|||
return result |
|||
|
|||
async def send_many(self, mp_pairs): |
|||
if mp_pairs: |
|||
payload = [{'method': method, 'params': params} |
|||
for method, params in mp_pairs] |
|||
return await self.send(payload) |
|||
return [] |
|||
|
|||
async def send_vector(self, method, params_list): |
|||
if params_list: |
|||
payload = [{'method': method, 'params': params} |
|||
for params in params_list] |
|||
return await self.send(payload) |
|||
return [] |
|||
|
|||
async def send(self, payload): |
|||
assert isinstance(payload, (tuple, list)) |
|||
data = json.dumps(payload) |
|||
while True: |
|||
try: |
|||
async with aiohttp.post(self.url, data=data) as resp: |
|||
result = await resp.json() |
|||
except asyncio.CancelledError: |
|||
raise |
|||
except Exception as e: |
|||
msg = 'aiohttp error: {}'.format(e) |
|||
secs = 3 |
|||
else: |
|||
errs = tuple(item['error'] for item in result) |
|||
if not any(errs): |
|||
return tuple(item['result'] for item in result) |
|||
if any(err.get('code') == -28 for err in errs): |
|||
msg = 'daemon still warming up.' |
|||
secs = 30 |
|||
else: |
|||
msg = '{}'.format(errs) |
|||
raise DaemonError(msg) |
|||
|
|||
self.logger.error('{}. Sleeping {:d}s and trying again...' |
|||
.format(msg, secs)) |
|||
await asyncio.sleep(secs) |
|||
|
|||
async def block_hex_hashes(self, first, count): |
|||
'''Return the hex hashes of count block starting at height first.''' |
|||
param_lists = [[height] for height in range(first, first + count)] |
|||
return await self.send_vector('getblockhash', param_lists) |
|||
|
|||
async def raw_blocks(self, hex_hashes): |
|||
'''Return the raw binary blocks with the given hex hashes.''' |
|||
param_lists = [(h, False) for h in hex_hashes] |
|||
blocks = await self.send_vector('getblock', param_lists) |
|||
# Convert hex string to bytes |
|||
return [bytes.fromhex(block) for block in blocks] |
|||
|
|||
async def height(self): |
|||
'''Query the daemon for its current height.''' |
|||
self._height = await self.send_single('getblockcount') |
|||
return self._height |
|||
|
|||
def cached_height(self): |
|||
'''Return the cached daemon height. |
|||
|
|||
If the daemon has not been queried yet this returns None.''' |
|||
return self._height |
|||
@ -1,679 +0,0 @@ |
|||
# See the file "LICENSE" for information about the copyright |
|||
# and warranty status of this software. |
|||
|
|||
import array |
|||
import ast |
|||
import itertools |
|||
import os |
|||
import struct |
|||
import time |
|||
from binascii import hexlify, unhexlify |
|||
from bisect import bisect_right |
|||
from collections import defaultdict, namedtuple |
|||
from functools import partial |
|||
import logging |
|||
|
|||
import plyvel |
|||
|
|||
from lib.coins import Bitcoin |
|||
from lib.script import ScriptPubKey |
|||
|
|||
# History can hold approx. 65536 * HIST_ENTRIES_PER_KEY entries |
|||
HIST_ENTRIES_PER_KEY = 1024 |
|||
HIST_VALUE_BYTES = HIST_ENTRIES_PER_KEY * 4 |
|||
ADDR_TX_HASH_LEN = 4 |
|||
UTXO_TX_HASH_LEN = 4 |
|||
UTXO = namedtuple("UTXO", "tx_num tx_pos tx_hash height value") |
|||
|
|||
|
|||
def formatted_time(t): |
|||
t = int(t) |
|||
return '{:d}d {:02d}h {:02d}m {:02d}s'.format( |
|||
t // 86400, (t % 86400) // 3600, (t % 3600) // 60, t % 60) |
|||
|
|||
|
|||
class UTXOCache(object): |
|||
'''An in-memory UTXO cache, representing all changes to UTXO state |
|||
since the last DB flush. |
|||
|
|||
We want to store millions, perhaps 10s of millions of these in |
|||
memory for optimal performance during initial sync, because then |
|||
it is possible to spend UTXOs without ever going to the database |
|||
(other than as an entry in the address history, and there is only |
|||
one such entry per TX not per UTXO). So store them in a Python |
|||
dictionary with binary keys and values. |
|||
|
|||
Key: TX_HASH + TX_IDX (32 + 2 = 34 bytes) |
|||
Value: HASH168 + TX_NUM + VALUE (21 + 4 + 8 = 33 bytes) |
|||
|
|||
That's 67 bytes of raw data. Python dictionary overhead means |
|||
each entry actually uses about 187 bytes of memory. So almost |
|||
11.5 million UTXOs can fit in 2GB of RAM. There are approximately |
|||
42 million UTXOs on bitcoin mainnet at height 433,000. |
|||
|
|||
Semantics: |
|||
|
|||
add: Add it to the cache dictionary. |
|||
spend: Remove it if in the cache dictionary. |
|||
Otherwise it's been flushed to the DB. Each UTXO |
|||
is responsible for two entries in the DB stored using |
|||
compressed keys. Mark both for deletion in the next |
|||
flush of the in-memory UTXO cache. |
|||
|
|||
A UTXO is stored in the DB in 2 "tables": |
|||
|
|||
1. The output value and tx number. Must be keyed with a |
|||
hash168 prefix so the unspent outputs and balance of an |
|||
arbitrary address can be looked up with a simple key |
|||
traversal. |
|||
Key: b'u' + hash168 + compressed_tx_hash + tx_idx |
|||
Value: a (tx_num, value) pair |
|||
|
|||
2. Given a prevout, we need to be able to look up the UTXO key |
|||
to remove it. As is keyed by hash168 and that is not part |
|||
of the prevout, we need a hash168 lookup. |
|||
Key: b'h' + compressed tx_hash + tx_idx |
|||
Value: (hash168, tx_num) pair |
|||
|
|||
The compressed TX hash is just the first few bytes of the hash of |
|||
the TX the UTXO is in (and needn't be the same number of bytes in |
|||
each table). As this is not unique there will be collisions; |
|||
tx_num is stored to resolve them. The collision rate is around |
|||
0.02% for the hash168 table, and almost zero for the UTXO table |
|||
(there are around 100 collisions in the whole bitcoin blockchain). |
|||
|
|||
''' |
|||
|
|||
def __init__(self, parent, db, coin): |
|||
self.logger = logging.getLogger('UTXO') |
|||
self.logger.setLevel(logging.INFO) |
|||
self.parent = parent |
|||
self.coin = coin |
|||
self.cache = {} |
|||
self.db = db |
|||
self.db_cache = {} |
|||
# Statistics |
|||
self.adds = 0 |
|||
self.cache_hits = 0 |
|||
self.db_deletes = 0 |
|||
|
|||
def add_many(self, tx_hash, tx_num, txouts): |
|||
'''Add a sequence of UTXOs to the cache, return the set of hash168s |
|||
seen. |
|||
|
|||
Pass the hash of the TX it appears in, its TX number, and the |
|||
TX outputs. |
|||
''' |
|||
parse_script = ScriptPubKey.from_script |
|||
pack = struct.pack |
|||
tx_numb = pack('<I', tx_num) |
|||
hash168s = set() |
|||
|
|||
self.adds += len(txouts) |
|||
for idx, txout in enumerate(txouts): |
|||
# Get the hash168. Ignore scripts we can't grok. |
|||
pk = parse_script(txout.pk_script, self.coin) |
|||
hash168 = pk.hash168 |
|||
if not hash168: |
|||
continue |
|||
|
|||
hash168s.add(hash168) |
|||
key = tx_hash + pack('<H', idx) |
|||
|
|||
# Well-known duplicate coinbases from heights 91722-91880 |
|||
# that destoyed 100 BTC forever: |
|||
# e3bf3d07d4b0375638d5f1db5255fe07ba2c4cb067cd81b84ee974b6585fb468 |
|||
# d5d27987d2a3dfc724e359870c6644b40e497bdc0589a033220fe15429d88599 |
|||
#if key in self.cache: |
|||
# self.logger.info('duplicate tx hash {}' |
|||
# .format(bytes(reversed(tx_hash)).hex())) |
|||
|
|||
# b''.join avoids this: https://bugs.python.org/issue13298 |
|||
self.cache[key] = b''.join( |
|||
(hash168, tx_numb, pack('<Q', txout.value))) |
|||
|
|||
return hash168s |
|||
|
|||
def spend(self, prevout): |
|||
'''Spend a UTXO and return the address spent. |
|||
|
|||
If the UTXO is not in the cache it must be on disk. |
|||
''' |
|||
# Fast track is it's in the cache |
|||
pack = struct.pack |
|||
key = b''.join((prevout.hash, pack('<H', prevout.n))) |
|||
value = self.cache.pop(key, None) |
|||
if value: |
|||
self.cache_hits += 1 |
|||
return value[:21] |
|||
|
|||
# Oh well. Find and remove it from the DB. |
|||
hash168 = self.hash168(prevout.hash, prevout.n) |
|||
if not hash168: |
|||
return None |
|||
|
|||
self.db_deletes += 1 |
|||
|
|||
# Read the UTXO through the cache from the disk. We have to |
|||
# go through the cache because compressed keys can collide. |
|||
key = (b'u' + hash168 + prevout.hash[:UTXO_TX_HASH_LEN] |
|||
+ pack('<H', prevout.n)) |
|||
data = self.cache_get(key) |
|||
if data is None: |
|||
# Uh-oh, this should not happen... |
|||
self.logger.error('found no UTXO for {} / {:d} key {}' |
|||
.format(bytes(reversed(prevout.hash)).hex(), |
|||
prevout.n, bytes(key).hex())) |
|||
return hash168 |
|||
|
|||
if len(data) == 12: |
|||
(tx_num, ) = struct.unpack('<I', data[:4]) |
|||
self.cache_delete(key) |
|||
return hash168 |
|||
|
|||
# Resolve the compressed key collison. These should be |
|||
# extremely rare. |
|||
assert len(data) % 12 == 0 |
|||
for n in range(0, len(data), 12): |
|||
(tx_num, ) = struct.unpack('<I', data[n:n+4]) |
|||
tx_hash, height = self.parent.get_tx_hash(tx_num) |
|||
if prevout.hash == tx_hash: |
|||
data = data[:n] + data[n + 12:] |
|||
self.cache_write(key, data) |
|||
return hash168 |
|||
|
|||
raise Exception('could not resolve UTXO key collision') |
|||
|
|||
def hash168(self, tx_hash, idx): |
|||
'''Return the hash168 paid to by the given TXO. |
|||
|
|||
Refers to the database. Returns None if not found (which is |
|||
indicates a non-standard script). |
|||
''' |
|||
key = b'h' + tx_hash[:ADDR_TX_HASH_LEN] + struct.pack('<H', idx) |
|||
data = self.cache_get(key) |
|||
if data is None: |
|||
# Assuming the DB is not corrupt, this indicates a |
|||
# successful spend of a non-standard script |
|||
# self.logger.info('ignoring spend of non-standard UTXO {} / {:d}' |
|||
# .format(bytes(reversed(tx_hash)).hex(), idx))) |
|||
return None |
|||
|
|||
if len(data) == 25: |
|||
self.cache_delete(key) |
|||
return data[:21] |
|||
|
|||
assert len(data) % 25 == 0 |
|||
|
|||
# Resolve the compressed key collision using the TX number |
|||
for n in range(0, len(data), 25): |
|||
(tx_num, ) = struct.unpack('<I', data[n+21:n+25]) |
|||
my_hash, height = self.parent.get_tx_hash(tx_num) |
|||
if my_hash == tx_hash: |
|||
self.cache_write(key, data[:n] + data[n+25:]) |
|||
return data[n:n+21] |
|||
|
|||
raise Exception('could not resolve hash168 collision') |
|||
|
|||
def cache_write(self, key, value): |
|||
'''Cache write of a (key, value) pair to the DB.''' |
|||
assert(bool(value)) |
|||
self.db_cache[key] = value |
|||
|
|||
def cache_delete(self, key): |
|||
'''Cache deletion of a key from the DB.''' |
|||
self.db_cache[key] = None |
|||
|
|||
def cache_get(self, key): |
|||
'''Fetch a value from the DB through our write cache.''' |
|||
value = self.db_cache.get(key) |
|||
if value: |
|||
return value |
|||
return self.db.get(key) |
|||
|
|||
def flush(self, batch): |
|||
'''Flush the cached DB writes and UTXO set to the batch.''' |
|||
# Care is needed because the writes generated by flushing the |
|||
# UTXO state may have keys in common with our write cache or |
|||
# may be in the DB already. |
|||
hcolls = ucolls = 0 |
|||
new_utxos = len(self.cache) |
|||
for cache_key, cache_value in self.cache.items(): |
|||
# Frist write to the hash168 lookup table |
|||
key = b'h' + cache_key[:ADDR_TX_HASH_LEN] + cache_key[-2:] |
|||
value = cache_value[:25] |
|||
prior_value = self.cache_get(key) |
|||
if prior_value: # Should rarely happen |
|||
hcolls += 1 |
|||
value += prior_value |
|||
self.cache_write(key, value) |
|||
|
|||
# Next write the UTXO table |
|||
key = (b'u' + cache_value[:21] + cache_key[:UTXO_TX_HASH_LEN] |
|||
+ cache_key[-2:]) |
|||
value = cache_value[-12:] |
|||
prior_value = self.cache_get(key) |
|||
if prior_value: # Should almost never happen |
|||
ucolls += 1 |
|||
value += prior_value |
|||
self.cache_write(key, value) |
|||
|
|||
# GC-ing this now can only help the levelDB write. |
|||
self.cache = {} |
|||
|
|||
# Now we can update to the batch. |
|||
for key, value in self.db_cache.items(): |
|||
if value: |
|||
batch.put(key, value) |
|||
else: |
|||
batch.delete(key) |
|||
|
|||
self.db_cache = {} |
|||
|
|||
self.logger.info('UTXO cache adds: {:,d} spends: {:,d} ' |
|||
.format(self.adds, self.cache_hits)) |
|||
self.logger.info('UTXO DB adds: {:,d} spends: {:,d}. ' |
|||
'Collisions: hash168: {:,d} UTXO: {:,d}' |
|||
.format(new_utxos, self.db_deletes, |
|||
hcolls, ucolls)) |
|||
self.adds = self.cache_hits = self.db_deletes = 0 |
|||
|
|||
|
|||
class DB(object): |
|||
|
|||
class Error(Exception): |
|||
pass |
|||
|
|||
def __init__(self, env): |
|||
self.logger = logging.getLogger('DB') |
|||
self.logger.setLevel(logging.INFO) |
|||
|
|||
# Meta |
|||
self.tx_hash_file_size = 16 * 1024 * 1024 |
|||
self.utxo_MB = env.utxo_MB |
|||
self.hist_MB = env.hist_MB |
|||
self.next_cache_check = 0 |
|||
self.last_flush = time.time() |
|||
self.coin = env.coin |
|||
|
|||
# Chain state (initialize to genesis in case of new DB) |
|||
self.db_height = -1 |
|||
self.db_tx_count = 0 |
|||
self.flush_count = 0 |
|||
self.utxo_flush_count = 0 |
|||
self.wall_time = 0 |
|||
self.tip = self.coin.GENESIS_HASH |
|||
|
|||
# Open DB and metadata files. Record some of its state. |
|||
self.db = self.open_db(self.coin) |
|||
self.tx_count = self.fs_tx_count = self.db_tx_count |
|||
self.height = self.fs_height = self.db_height |
|||
|
|||
# Caches to be flushed later. Headers and tx_hashes have one |
|||
# entry per block |
|||
self.headers = [] |
|||
self.tx_hashes = [] |
|||
self.history = defaultdict(partial(array.array, 'I')) |
|||
self.history_size = 0 |
|||
self.utxo_cache = UTXOCache(self, self.db, self.coin) |
|||
self.tx_counts = array.array('I') |
|||
self.txcount_file.seek(0) |
|||
self.tx_counts.fromfile(self.txcount_file, self.height + 1) |
|||
|
|||
# Log state |
|||
self.logger.info('{}/{} height: {:,d} tx count: {:,d} ' |
|||
'flush count: {:,d} utxo flush count: {:,d} ' |
|||
'sync time: {}' |
|||
.format(self.coin.NAME, self.coin.NET, self.height, |
|||
self.tx_count, self.flush_count, |
|||
self.utxo_flush_count, |
|||
formatted_time(self.wall_time))) |
|||
self.logger.info('flushing UTXO cache at {:,d} MB' |
|||
.format(self.utxo_MB)) |
|||
self.logger.info('flushing history cache at {:,d} MB' |
|||
.format(self.hist_MB)) |
|||
|
|||
|
|||
def open_db(self, coin): |
|||
db_name = '{}-{}'.format(coin.NAME, coin.NET) |
|||
is_new = False |
|||
try: |
|||
db = plyvel.DB(db_name, create_if_missing=False, |
|||
error_if_exists=False, compression=None) |
|||
except: |
|||
db = plyvel.DB(db_name, create_if_missing=True, |
|||
error_if_exists=True, compression=None) |
|||
is_new = True |
|||
|
|||
if is_new: |
|||
self.logger.info('created new database {}'.format(db_name)) |
|||
self.flush_state(db) |
|||
else: |
|||
self.logger.info('successfully opened database {}'.format(db_name)) |
|||
self.read_state(db) |
|||
self.delete_excess_history(db) |
|||
|
|||
self.headers_file = self.open_file('headers', is_new) |
|||
self.txcount_file = self.open_file('txcount', is_new) |
|||
|
|||
return db |
|||
|
|||
def read_state(self, db): |
|||
state = db.get(b'state') |
|||
state = ast.literal_eval(state.decode('ascii')) |
|||
if state['genesis'] != self.coin.GENESIS_HASH: |
|||
raise self.Error('DB genesis hash {} does not match coin {}' |
|||
.format(state['genesis_hash'], |
|||
self.coin.GENESIS_HASH)) |
|||
self.db_height = state['height'] |
|||
self.db_tx_count = state['tx_count'] |
|||
self.tip = state['tip'] |
|||
self.flush_count = state['flush_count'] |
|||
self.utxo_flush_count = state['utxo_flush_count'] |
|||
self.wall_time = state['wall_time'] |
|||
|
|||
def delete_excess_history(self, db): |
|||
'''Clear history flushed since the most recent UTXO flush.''' |
|||
utxo_flush_count = self.utxo_flush_count |
|||
diff = self.flush_count - utxo_flush_count |
|||
if diff == 0: |
|||
return |
|||
if diff < 0: |
|||
raise self.Error('DB corrupt: flush_count < utxo_flush_count') |
|||
|
|||
self.logger.info('DB not shut down cleanly. Scanning for most ' |
|||
'recent {:,d} history flushes'.format(diff)) |
|||
prefix = b'H' |
|||
unpack = struct.unpack |
|||
keys = [] |
|||
for key, hist in db.iterator(prefix=prefix): |
|||
flush_id, = unpack('>H', key[-2:]) |
|||
if flush_id > self.utxo_flush_count: |
|||
keys.append(key) |
|||
|
|||
self.logger.info('deleting {:,d} history entries'.format(len(keys))) |
|||
with db.write_batch(transaction=True) as batch: |
|||
for key in keys: |
|||
db.delete(key) |
|||
self.utxo_flush_count = self.flush_count |
|||
self.flush_state(batch) |
|||
self.logger.info('deletion complete') |
|||
|
|||
def flush_to_fs(self): |
|||
'''Flush the things stored on the filesystem.''' |
|||
# First the headers |
|||
headers = b''.join(self.headers) |
|||
header_len = self.coin.HEADER_LEN |
|||
self.headers_file.seek((self.fs_height + 1) * header_len) |
|||
self.headers_file.write(headers) |
|||
self.headers_file.flush() |
|||
self.headers = [] |
|||
|
|||
# Then the tx counts |
|||
self.txcount_file.seek((self.fs_height + 1) * self.tx_counts.itemsize) |
|||
self.txcount_file.write(self.tx_counts[self.fs_height + 1: |
|||
self.height + 1]) |
|||
self.txcount_file.flush() |
|||
|
|||
# Finally the hashes |
|||
hashes = memoryview(b''.join(itertools.chain(*self.tx_hashes))) |
|||
assert len(hashes) % 32 == 0 |
|||
assert self.tx_hash_file_size % 32 == 0 |
|||
cursor = 0 |
|||
file_pos = self.fs_tx_count * 32 |
|||
while cursor < len(hashes): |
|||
file_num, offset = divmod(file_pos, self.tx_hash_file_size) |
|||
size = min(len(hashes) - cursor, self.tx_hash_file_size - offset) |
|||
filename = 'hashes{:04d}'.format(file_num) |
|||
with self.open_file(filename, create=True) as f: |
|||
f.seek(offset) |
|||
f.write(hashes[cursor:cursor + size]) |
|||
cursor += size |
|||
file_pos += size |
|||
self.tx_hashes = [] |
|||
|
|||
self.fs_height = self.height |
|||
self.fs_tx_count = self.tx_count |
|||
os.sync() |
|||
|
|||
def flush_state(self, batch): |
|||
'''Flush chain state to the batch.''' |
|||
now = time.time() |
|||
self.wall_time += now - self.last_flush |
|||
self.last_flush = now |
|||
state = { |
|||
'genesis': self.coin.GENESIS_HASH, |
|||
'height': self.db_height, |
|||
'tx_count': self.db_tx_count, |
|||
'tip': self.tip, |
|||
'flush_count': self.flush_count, |
|||
'utxo_flush_count': self.utxo_flush_count, |
|||
'wall_time': self.wall_time, |
|||
} |
|||
batch.put(b'state', repr(state).encode('ascii')) |
|||
|
|||
def flush_utxos(self, batch): |
|||
self.logger.info('flushing UTXOs: {:,d} txs and {:,d} blocks' |
|||
.format(self.tx_count - self.db_tx_count, |
|||
self.height - self.db_height)) |
|||
self.utxo_cache.flush(batch) |
|||
self.utxo_flush_count = self.flush_count |
|||
self.db_tx_count = self.tx_count |
|||
self.db_height = self.height |
|||
|
|||
def flush(self, daemon_height, flush_utxos=False): |
|||
'''Flush out cached state. |
|||
|
|||
History is always flushed. UTXOs are flushed if flush_utxos.''' |
|||
flush_start = time.time() |
|||
last_flush = self.last_flush |
|||
tx_diff = self.tx_count - self.fs_tx_count |
|||
|
|||
# Write out the files to the FS before flushing to the DB. If |
|||
# the DB transaction fails, the files being too long doesn't |
|||
# matter. But if writing the files fails we do not want to |
|||
# have updated the DB. |
|||
self.logger.info('commencing history flush') |
|||
self.flush_to_fs() |
|||
|
|||
with self.db.write_batch(transaction=True) as batch: |
|||
# History first - fast and frees memory. Flush state last |
|||
# as it reads the wall time. |
|||
self.flush_history(batch) |
|||
if flush_utxos: |
|||
self.flush_utxos(batch) |
|||
self.flush_state(batch) |
|||
self.logger.info('committing transaction...') |
|||
|
|||
# Update and put the wall time again - otherwise we drop the |
|||
# time it took leveldb to commit the batch |
|||
self.flush_state(self.db) |
|||
|
|||
flush_time = int(self.last_flush - flush_start) |
|||
self.logger.info('flush #{:,d} to height {:,d} took {:,d}s' |
|||
.format(self.flush_count, self.height, flush_time)) |
|||
|
|||
# Log handy stats |
|||
txs_per_sec = int(self.tx_count / self.wall_time) |
|||
this_txs_per_sec = 1 + int(tx_diff / (self.last_flush - last_flush)) |
|||
if self.height > self.coin.TX_COUNT_HEIGHT: |
|||
tx_est = (daemon_height - self.height) * self.coin.TX_PER_BLOCK |
|||
else: |
|||
tx_est = ((daemon_height - self.coin.TX_COUNT_HEIGHT) |
|||
* self.coin.TX_PER_BLOCK |
|||
+ (self.coin.TX_COUNT - self.tx_count)) |
|||
|
|||
self.logger.info('txs: {:,d} tx/sec since genesis: {:,d}, ' |
|||
'since last flush: {:,d}' |
|||
.format(self.tx_count, txs_per_sec, this_txs_per_sec)) |
|||
self.logger.info('sync time: {} ETA: {}' |
|||
.format(formatted_time(self.wall_time), |
|||
formatted_time(tx_est / this_txs_per_sec))) |
|||
|
|||
def flush_history(self, batch): |
|||
# Drop any None entry |
|||
self.history.pop(None, None) |
|||
|
|||
self.flush_count += 1 |
|||
flush_id = struct.pack('>H', self.flush_count) |
|||
for hash168, hist in self.history.items(): |
|||
key = b'H' + hash168 + flush_id |
|||
batch.put(key, hist.tobytes()) |
|||
|
|||
self.logger.info('{:,d} history entries in {:,d} addrs' |
|||
.format(self.history_size, len(self.history))) |
|||
|
|||
self.history = defaultdict(partial(array.array, 'I')) |
|||
self.history_size = 0 |
|||
|
|||
def open_file(self, filename, create=False): |
|||
'''Open the file name. Return its handle.''' |
|||
try: |
|||
return open(filename, 'rb+') |
|||
except FileNotFoundError: |
|||
if create: |
|||
return open(filename, 'wb+') |
|||
raise |
|||
|
|||
def read_headers(self, height, count): |
|||
header_len = self.coin.HEADER_LEN |
|||
self.headers_file.seek(height * header_len) |
|||
return self.headers_file.read(count * header_len) |
|||
|
|||
def cache_sizes(self, daemon_height): |
|||
'''Returns the approximate size of the cache, in MB.''' |
|||
# Good average estimates based on traversal of subobjects and |
|||
# requesting size from Python (see deep_getsizeof). For |
|||
# whatever reason Python O/S mem usage is typically +30% or |
|||
# more, so we scale our already bloated object sizes. |
|||
one_MB = int(1048576 / 1.3) |
|||
utxo_cache_size = len(self.utxo_cache.cache) * 187 |
|||
db_cache_size = len(self.utxo_cache.db_cache) * 105 |
|||
hist_cache_size = len(self.history) * 180 + self.history_size * 4 |
|||
utxo_MB = (db_cache_size + utxo_cache_size) // one_MB |
|||
hist_MB = hist_cache_size // one_MB |
|||
|
|||
self.logger.info('cache stats at height {:,d} daemon height: {:,d}' |
|||
.format(self.height, daemon_height)) |
|||
self.logger.info(' entries: UTXO: {:,d} DB: {:,d} ' |
|||
'hist addrs: {:,d} hist size: {:,d}' |
|||
.format(len(self.utxo_cache.cache), |
|||
len(self.utxo_cache.db_cache), |
|||
len(self.history), |
|||
self.history_size)) |
|||
self.logger.info(' size: {:,d}MB (UTXOs {:,d}MB hist {:,d}MB)' |
|||
.format(utxo_MB + hist_MB, utxo_MB, hist_MB)) |
|||
return utxo_MB, hist_MB |
|||
|
|||
def process_block(self, block, daemon_height): |
|||
self.headers.append(block[:self.coin.HEADER_LEN]) |
|||
|
|||
tx_hashes, txs = self.coin.read_block(block) |
|||
self.height += 1 |
|||
|
|||
assert len(self.tx_counts) == self.height |
|||
|
|||
# These both need to be updated before calling process_tx(). |
|||
# It uses them for tx hash lookup |
|||
self.tx_hashes.append(tx_hashes) |
|||
self.tx_counts.append(self.tx_count + len(txs)) |
|||
|
|||
for tx_hash, tx in zip(tx_hashes, txs): |
|||
self.process_tx(tx_hash, tx) |
|||
|
|||
# Check if we're getting full and time to flush? |
|||
now = time.time() |
|||
if now > self.next_cache_check: |
|||
self.next_cache_check = now + 60 |
|||
utxo_MB, hist_MB = self.cache_sizes(daemon_height) |
|||
if utxo_MB >= self.utxo_MB or hist_MB >= self.hist_MB: |
|||
self.flush(daemon_height, utxo_MB >= self.utxo_MB) |
|||
|
|||
def process_tx(self, tx_hash, tx): |
|||
cache = self.utxo_cache |
|||
tx_num = self.tx_count |
|||
|
|||
# Add the outputs as new UTXOs; spend the inputs |
|||
hash168s = cache.add_many(tx_hash, tx_num, tx.outputs) |
|||
if not tx.is_coinbase: |
|||
for txin in tx.inputs: |
|||
hash168s.add(cache.spend(txin.prevout)) |
|||
|
|||
for hash168 in hash168s: |
|||
self.history[hash168].append(tx_num) |
|||
self.history_size += len(hash168s) |
|||
|
|||
self.tx_count += 1 |
|||
|
|||
def get_tx_hash(self, tx_num): |
|||
'''Returns the tx_hash and height of a tx number.''' |
|||
height = bisect_right(self.tx_counts, tx_num) |
|||
|
|||
# Is this on disk or unflushed? |
|||
if height > self.fs_height: |
|||
tx_hashes = self.tx_hashes[height - (self.fs_height + 1)] |
|||
tx_hash = tx_hashes[tx_num - self.tx_counts[height - 1]] |
|||
else: |
|||
file_pos = tx_num * 32 |
|||
file_num, offset = divmod(file_pos, self.tx_hash_file_size) |
|||
filename = 'hashes{:04d}'.format(file_num) |
|||
with self.open_file(filename) as f: |
|||
f.seek(offset) |
|||
tx_hash = f.read(32) |
|||
|
|||
return tx_hash, height |
|||
|
|||
@staticmethod |
|||
def resolve_limit(limit): |
|||
if limit is None: |
|||
return -1 |
|||
assert isinstance(limit, int) and limit >= 0 |
|||
return limit |
|||
|
|||
def get_history(self, hash168, limit=1000): |
|||
'''Generator that returns an unpruned, sorted list of (tx_hash, |
|||
height) tuples of transactions that touched the address, |
|||
earliest in the blockchain first. Includes both spending and |
|||
receiving transactions. By default yields at most 1000 entries. |
|||
Set limit to None to get them all. |
|||
''' |
|||
limit = self.resolve_limit(limit) |
|||
prefix = b'H' + hash168 |
|||
for key, hist in self.db.iterator(prefix=prefix): |
|||
a = array.array('I') |
|||
a.frombytes(hist) |
|||
for tx_num in a: |
|||
if limit == 0: |
|||
return |
|||
yield self.get_tx_hash(tx_num) |
|||
limit -= 1 |
|||
|
|||
def get_balance(self, hash168): |
|||
'''Returns the confirmed balance of an address.''' |
|||
return sum(utxo.value for utxo in self.get_utxos(hash168, limit=None)) |
|||
|
|||
def get_utxos(self, hash168, limit=1000): |
|||
'''Generator that yields all UTXOs for an address sorted in no |
|||
particular order. By default yields at most 1000 entries. |
|||
Set limit to None to get them all. |
|||
''' |
|||
limit = self.resolve_limit(limit) |
|||
unpack = struct.unpack |
|||
prefix = b'u' + hash168 |
|||
utxos = [] |
|||
for k, v in self.db.iterator(prefix=prefix): |
|||
(tx_pos, ) = unpack('<H', k[-2:]) |
|||
|
|||
for n in range(0, len(v), 12): |
|||
if limit == 0: |
|||
return |
|||
(tx_num, ) = unpack('<I', v[n:n+4]) |
|||
(value, ) = unpack('<Q', v[n+4:n+12]) |
|||
tx_hash, height = self.get_tx_hash(tx_num) |
|||
yield UTXO(tx_num, tx_pos, tx_hash, height, value) |
|||
limit -= 1 |
|||
|
|||
def get_utxos_sorted(self, hash168): |
|||
'''Returns all the UTXOs for an address sorted by height and |
|||
position in the block.''' |
|||
return sorted(self.get_utxos(hash168, limit=None)) |
|||
@ -0,0 +1,216 @@ |
|||
# See the file "LICENSE" for information about the copyright |
|||
# and warranty status of this software. |
|||
|
|||
import asyncio |
|||
import codecs |
|||
import json |
|||
import traceback |
|||
from functools import partial |
|||
|
|||
from lib.util import LoggedClass |
|||
from server.version import VERSION |
|||
|
|||
|
|||
class Error(Exception): |
|||
BAD_REQUEST = 1 |
|||
INTERNAL_ERROR = 2 |
|||
|
|||
|
|||
class JSONRPC(asyncio.Protocol, LoggedClass): |
|||
|
|||
def __init__(self, controller): |
|||
super().__init__() |
|||
self.controller = controller |
|||
self.parts = [] |
|||
|
|||
def connection_made(self, transport): |
|||
self.transport = transport |
|||
peername = transport.get_extra_info('peername') |
|||
self.logger.info('connection from {}'.format(peername)) |
|||
self.controller.add_session(self) |
|||
|
|||
def connection_lost(self, exc): |
|||
self.logger.info('disconnected') |
|||
self.controller.remove_session(self) |
|||
|
|||
def data_received(self, data): |
|||
while True: |
|||
npos = data.find(ord('\n')) |
|||
if npos == -1: |
|||
break |
|||
tail, data = data[:npos], data[npos + 1:] |
|||
parts = self.parts |
|||
self.parts = [] |
|||
parts.append(tail) |
|||
self.decode_message(b''.join(parts)) |
|||
|
|||
if data: |
|||
self.parts.append(data) |
|||
|
|||
def decode_message(self, message): |
|||
'''Message is a binary message.''' |
|||
try: |
|||
message = json.loads(message.decode()) |
|||
except Exception as e: |
|||
self.logger.info('caught exception decoding message'.format(e)) |
|||
return |
|||
|
|||
job = self.request_handler(message) |
|||
self.controller.add_job(job) |
|||
|
|||
async def request_handler(self, request): |
|||
'''Called asynchronously.''' |
|||
error = result = None |
|||
try: |
|||
result = await self.json_handler(request) |
|||
except Error as e: |
|||
error = {'code': e.args[0], 'message': e.args[1]} |
|||
except asyncio.CancelledError: |
|||
raise |
|||
except Exception as e: |
|||
# This should be considered a bug and fixed |
|||
traceback.print_exc() |
|||
error = {'code': Error.INTERNAL_ERROR, 'message': str(e)} |
|||
|
|||
payload = {'id': request.get('id'), 'error': error, 'result': result} |
|||
try: |
|||
data = json.dumps(payload) + '\n' |
|||
except TypeError: |
|||
msg = 'cannot JSON encode response to request {}'.format(request) |
|||
self.logger.error(msg) |
|||
error = {'code': Error.INTERNAL_ERROR, 'message': msg} |
|||
payload = {'id': request.get('id'), 'error': error, 'result': None} |
|||
data = json.dumps(payload) + '\n' |
|||
self.transport.write(data.encode()) |
|||
|
|||
async def json_handler(self, request): |
|||
method = request.get('method') |
|||
handler = None |
|||
if isinstance(method, str): |
|||
handler_name = 'handle_{}'.format(method.replace('.', '_')) |
|||
handler = getattr(self, handler_name, None) |
|||
if not handler: |
|||
self.logger.info('unknown method: {}'.format(method)) |
|||
raise Error(Error.BAD_REQUEST, 'unknown method: {}'.format(method)) |
|||
params = request.get('params', []) |
|||
if not isinstance(params, list): |
|||
raise Error(Error.BAD_REQUEST, 'params should be an array') |
|||
return await handler(params) |
|||
|
|||
|
|||
class ElectrumX(JSONRPC): |
|||
'''A TCP server that handles incoming Electrum connections.''' |
|||
|
|||
def __init__(self, controller, daemon, env): |
|||
super().__init__(controller) |
|||
self.daemon = daemon |
|||
self.env = env |
|||
self.addresses = set() |
|||
self.subscribe_headers = False |
|||
|
|||
def params_to_hash168(self, params): |
|||
if len(params) != 1: |
|||
raise Error(Error.BAD_REQUEST, |
|||
'params should contain a single address') |
|||
address = params[0] |
|||
try: |
|||
return self.env.coin.address_to_hash168(address) |
|||
except: |
|||
raise Error(Error.BAD_REQUEST, |
|||
'invalid address: {}'.format(address)) |
|||
|
|||
async def handle_blockchain_address_get_history(self, params): |
|||
hash168 = self.params_to_hash168(params) |
|||
return self.controller.get_history(hash168) |
|||
|
|||
async def handle_blockchain_address_subscribe(self, params): |
|||
hash168 = self.params_to_hash168(params) |
|||
status = self.controller.address_status(hash168) |
|||
return status.hex() if status else None |
|||
|
|||
async def handle_blockchain_estimatefee(self, params): |
|||
result = await self.daemon.send_single('estimatefee', params) |
|||
return result |
|||
|
|||
async def handle_blockchain_headers_subscribe(self, params): |
|||
self.subscribe_headers = True |
|||
return self.controller.get_current_header() |
|||
|
|||
async def handle_blockchain_relayfee(self, params): |
|||
'''The minimum fee a low-priority tx must pay in order to be accepted |
|||
to this daemon's memory pool. |
|||
''' |
|||
net_info = await self.daemon.send_single('getnetworkinfo') |
|||
return net_info['relayfee'] |
|||
|
|||
async def handle_blockchain_transaction_get(self, params): |
|||
if len(params) != 1: |
|||
raise Error(Error.BAD_REQUEST, |
|||
'params should contain a transaction hash') |
|||
tx_hash = params[0] |
|||
return await self.daemon.send_single('getrawtransaction', (tx_hash, 0)) |
|||
|
|||
async def handle_blockchain_transaction_get_merkle(self, params): |
|||
if len(params) != 2: |
|||
raise Error(Error.BAD_REQUEST, |
|||
'params should contain a transaction hash and height') |
|||
tx_hash, height = params |
|||
return await self.controller.get_merkle(tx_hash, height) |
|||
|
|||
async def handle_server_banner(self, params): |
|||
'''Return the server banner.''' |
|||
banner = 'Welcome to Electrum!' |
|||
if self.env.banner_file: |
|||
try: |
|||
with codecs.open(self.env.banner_file, 'r', 'utf-8') as f: |
|||
banner = f.read() |
|||
except Exception as e: |
|||
self.logger.error('reading banner file {}: {}' |
|||
.format(self.env.banner_file, e)) |
|||
return banner |
|||
|
|||
async def handle_server_donation_address(self, params): |
|||
'''Return the donation address as a string. |
|||
|
|||
If none is specified return the empty string. |
|||
''' |
|||
return self.env.donation_address |
|||
|
|||
async def handle_server_peers_subscribe(self, params): |
|||
'''Returns the peer (ip, host, ports) tuples. |
|||
|
|||
Despite the name electrum-server does not treat this as a |
|||
subscription. |
|||
''' |
|||
peers = self.controller.get_peers() |
|||
return tuple(peers.values()) |
|||
|
|||
async def handle_server_version(self, params): |
|||
'''Return the server version as a string.''' |
|||
return VERSION |
|||
|
|||
|
|||
class LocalRPC(JSONRPC): |
|||
'''A local TCP RPC server for querying status.''' |
|||
|
|||
async def handle_getinfo(self, params): |
|||
return { |
|||
'blocks': self.controller.height(), |
|||
'peers': len(self.controller.get_peers()), |
|||
'sessions': len(self.controller.sessions), |
|||
'watched': sum(len(s.addresses) for s in self.controller.sessions |
|||
if isinstance(s, ElectrumX)), |
|||
'cached': 0, |
|||
} |
|||
|
|||
async def handle_sessions(self, params): |
|||
return [] |
|||
|
|||
async def handle_numsessions(self, params): |
|||
return len(self.controller.sessions) |
|||
|
|||
async def handle_peers(self, params): |
|||
return tuple(self.controller.get_peers().keys()) |
|||
|
|||
async def handle_numpeers(self, params): |
|||
return len(self.controller.get_peers()) |
|||
@ -1,174 +0,0 @@ |
|||
# See the file "LICENSE" for information about the copyright |
|||
# and warranty status of this software. |
|||
|
|||
import asyncio |
|||
import json |
|||
import logging |
|||
import os |
|||
import signal |
|||
import time |
|||
from functools import partial |
|||
|
|||
import aiohttp |
|||
|
|||
from server.db import DB |
|||
|
|||
|
|||
class Server(object): |
|||
|
|||
def __init__(self, env): |
|||
self.env = env |
|||
self.db = DB(env) |
|||
self.block_cache = BlockCache(env, self.db) |
|||
self.tasks = [ |
|||
asyncio.ensure_future(self.block_cache.catch_up()), |
|||
asyncio.ensure_future(self.block_cache.process_cache()), |
|||
] |
|||
|
|||
loop = asyncio.get_event_loop() |
|||
for signame in ('SIGINT', 'SIGTERM'): |
|||
loop.add_signal_handler(getattr(signal, signame), |
|||
partial(self.on_signal, signame)) |
|||
|
|||
def on_signal(self, signame): |
|||
logging.warning('received {} signal, preparing to shut down' |
|||
.format(signame)) |
|||
for task in self.tasks: |
|||
task.cancel() |
|||
|
|||
def async_tasks(self): |
|||
return self.tasks |
|||
|
|||
|
|||
class BlockCache(object): |
|||
'''Requests blocks ahead of time from the daemon. Serves them |
|||
to the blockchain processor.''' |
|||
|
|||
def __init__(self, env, db): |
|||
self.logger = logging.getLogger('BlockCache') |
|||
self.logger.setLevel(logging.INFO) |
|||
|
|||
self.db = db |
|||
self.rpc_url = env.rpc_url |
|||
# Cache target size is in MB. Has little effect on sync time. |
|||
self.cache_limit = 10 |
|||
self.daemon_height = 0 |
|||
self.fetched_height = db.height |
|||
# Blocks stored in reverse order. Next block is at end of list. |
|||
self.blocks = [] |
|||
self.recent_sizes = [] |
|||
self.ave_size = 0 |
|||
|
|||
self.logger.info('using RPC URL {}'.format(self.rpc_url)) |
|||
|
|||
async def process_cache(self): |
|||
while True: |
|||
await asyncio.sleep(1) |
|||
while self.blocks: |
|||
self.db.process_block(self.blocks.pop(), self.daemon_height) |
|||
# Release asynchronous block fetching |
|||
await asyncio.sleep(0) |
|||
|
|||
async def catch_up(self): |
|||
self.logger.info('catching up, block cache limit {:d}MB...' |
|||
.format(self.cache_limit)) |
|||
|
|||
try: |
|||
while await self.maybe_prefill(): |
|||
await asyncio.sleep(1) |
|||
self.logger.info('caught up to height {:d}' |
|||
.format(self.daemon_height)) |
|||
finally: |
|||
self.db.flush(self.daemon_height, True) |
|||
|
|||
def cache_used(self): |
|||
return sum(len(block) for block in self.blocks) |
|||
|
|||
def prefill_count(self, room): |
|||
count = 0 |
|||
if self.ave_size: |
|||
count = room // self.ave_size |
|||
return max(count, 10) |
|||
|
|||
async def maybe_prefill(self): |
|||
'''Returns False to stop. True to sleep a while for asynchronous |
|||
processing.''' |
|||
cache_limit = self.cache_limit * 1024 * 1024 |
|||
while True: |
|||
cache_used = self.cache_used() |
|||
if cache_used > cache_limit: |
|||
return True |
|||
|
|||
# Keep going by getting a whole new cache_limit of blocks |
|||
self.daemon_height = await self.send_single('getblockcount') |
|||
max_count = min(self.daemon_height - self.fetched_height, 4000) |
|||
count = min(max_count, self.prefill_count(cache_limit)) |
|||
if not count: |
|||
return False # Done catching up |
|||
|
|||
first = self.fetched_height + 1 |
|||
param_lists = [[height] for height in range(first, first + count)] |
|||
hashes = await self.send_vector('getblockhash', param_lists) |
|||
|
|||
# Hashes is an array of hex strings |
|||
param_lists = [(h, False) for h in hashes] |
|||
blocks = await self.send_vector('getblock', param_lists) |
|||
self.fetched_height += count |
|||
|
|||
# Convert hex string to bytes and put in memoryview |
|||
blocks = [bytes.fromhex(block) for block in blocks] |
|||
# Reverse order and place at front of list |
|||
self.blocks = list(reversed(blocks)) + self.blocks |
|||
|
|||
# Keep 50 most recent block sizes for fetch count estimation |
|||
sizes = [len(block) for block in blocks] |
|||
self.recent_sizes.extend(sizes) |
|||
excess = len(self.recent_sizes) - 50 |
|||
if excess > 0: |
|||
self.recent_sizes = self.recent_sizes[excess:] |
|||
self.ave_size = sum(self.recent_sizes) // len(self.recent_sizes) |
|||
|
|||
async def send_single(self, method, params=None): |
|||
payload = {'method': method} |
|||
if params: |
|||
payload['params'] = params |
|||
result, = await self.send((payload, )) |
|||
return result |
|||
|
|||
async def send_many(self, mp_pairs): |
|||
payload = [{'method': method, 'params': params} |
|||
for method, params in mp_pairs] |
|||
return await self.send(payload) |
|||
|
|||
async def send_vector(self, method, params_list): |
|||
payload = [{'method': method, 'params': params} |
|||
for params in params_list] |
|||
return await self.send(payload) |
|||
|
|||
async def send(self, payload): |
|||
assert isinstance(payload, (tuple, list)) |
|||
data = json.dumps(payload) |
|||
while True: |
|||
try: |
|||
async with aiohttp.request('POST', self.rpc_url, |
|||
data = data) as resp: |
|||
result = await resp.json() |
|||
except asyncio.CancelledError: |
|||
raise |
|||
except Exception as e: |
|||
msg = 'aiohttp error: {}'.format(e) |
|||
secs = 3 |
|||
else: |
|||
errs = tuple(item['error'] for item in result) |
|||
if not any(errs): |
|||
return tuple(item['result'] for item in result) |
|||
if any(err.get('code') == -28 for err in errs): |
|||
msg = 'daemon still warming up.' |
|||
secs = 30 |
|||
else: |
|||
msg = 'daemon errors: {}'.format(errs) |
|||
secs = 3 |
|||
|
|||
self.logger.error('{}. Sleeping {:d}s and trying again...' |
|||
.format(msg, secs)) |
|||
await asyncio.sleep(secs) |
|||
@ -0,0 +1 @@ |
|||
VERSION = "ElectrumX 0.02" |
|||
Loading…
Reference in new issue