lukechilds
/
electrumx
mirror of https://github.com/lukechilds/electrumx.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Make flushes and reorgs async 

Apart from the flush on shutdown and the flush when caught up,
neither of which matter, this makes flushes asynchronous.

Also, block processing for reorgs is now asynchronous.

This also removes the FORCE_REORG debug envvar; I want to
put that into the RPC interface.

Closes #102
master
Neil Booth 8 years ago
parent
89209632ae
commit
15051124af
6 changed files with 81 additions and 91 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 16
      docs/ENVIRONMENT.rst
  2. 17
      lib/coins.py
  3. 135
      server/block_processor.py
  4. 2
      server/controller.py
  5. 2
      server/env.py
  6. 0
      server/session.py

16
docs/ENVIRONMENT.rst
View File

@ -284,20 +284,4 @@ your available physical RAM:
versions, a value of 90% of the sum of the old UTXO_MB and HIST_MB
variables is roughly equivalent.
Debugging
---------
The following are for debugging purposes:
* **FORCE_REORG**
If set to a positive integer, it will simulate a reorg of the
blockchain for that number of blocks on startup. You must have
synced before using this, otherwise there will be no undo
information.
Although it should fail gracefully if set to a value greater than
**REORG_LIMIT**, I do not recommend it as I have not tried it and
there is a chance your DB might corrupt.
.. _lib/coins.py: https://github.com/kyuupichan/electrumx/blob/master/lib/coins.py

17
lib/coins.py
View File

@ -76,7 +76,7 @@ class Coin(object):
Return the block less its unspendable coinbase.
'''
header = block[:cls.header_len(0)]
header = cls.block_header(block, 0)
header_hex_hash = hash_to_str(cls.header_hash(header))
if header_hex_hash != cls.GENESIS_HASH:
raise CoinError('genesis block has hash {} expected {}'
@ -217,15 +217,16 @@ class Coin(object):
return cls.header_offset(height + 1) - cls.header_offset(height)
@classmethod
def read_block(cls, block, height):
'''Returns a pair (header, tx_list) given a raw block and height.
def block_header(cls, block, height):
'''Returns the block header given a block and its height.'''
return block[:cls.header_len(height)]
tx_list is a list of (deserialized_tx, tx_hash) pairs.
'''
@classmethod
def block_txs(cls, block, height):
'''Returns a list of (deserialized_tx, tx_hash) pairs given a
block and its height.'''
deserializer = cls.deserializer()
hlen = cls.header_len(height)
header, rest = block[:hlen], block[hlen:]
return (header, deserializer(rest).read_block())
return deserializer(block[cls.header_len(height):]).read_block()
@classmethod
def decimal_value(cls, value):

135
server/block_processor.py
View File

@ -197,19 +197,35 @@ class BlockProcessor(server.db.DB):
self.logger.info('flushing DB cache at {:,d} MB'
.format(self.cache_MB))
async def executor(self, func, *args, **kwargs):
'''Run func taking args in the executor.'''
loop = asyncio.get_event_loop()
await loop.run_in_executor(None, partial(func, *args, **kwargs))
def caught_up(self):
'''Called when first caught up after starting.'''
if not self.caught_up_event.is_set():
self.first_sync = False
self.flush(True)
if self.utxo_db.for_sync:
self.logger.info('{} synced to height {:,d}'
.format(VERSION, self.height))
self.open_dbs()
self.caught_up_event.set()
async def main_loop(self, touched):
'''Main loop for block processing.'''
# Simulate a reorg if requested
if self.env.force_reorg > 0:
self.logger.info('DEBUG: simulating reorg of {:,d} blocks'
.format(self.env.force_reorg))
await self.handle_chain_reorg(set(), self.env.force_reorg)
while True:
blocks = await self.prefetcher.get_blocks()
if blocks:
await self.advance_blocks(blocks, touched)
start = time.time()
await self.check_and_advance_blocks(blocks, touched)
if not self.first_sync:
s = '' if len(blocks) == 1 else 's'
self.logger.info('processed {:,d} block{} in {:.1f}s'
.format(len(blocks), s,
time.time() - start))
elif blocks is None:
break # Shutdown
else:
@ -219,22 +235,50 @@ class BlockProcessor(server.db.DB):
self.flush(True)
self.logger.info('shutdown complete')
async def advance_blocks(self, blocks, touched):
async def check_and_advance_blocks(self, blocks, touched):
'''Process the list of blocks passed. Detects and handles reorgs.'''
def job():
for block in blocks:
self.advance_block(block, touched)
start = time.time()
loop = asyncio.get_event_loop()
try:
await loop.run_in_executor(None, job)
except ChainReorg:
first = self.height + 1
headers = [self.coin.block_header(block, first + n)
for n, block in enumerate(blocks)]
hprevs = [self.coin.header_prevhash(h) for h in headers]
chain = [self.tip] + [self.coin.header_hash(h) for h in headers[:-1]]
if hprevs == chain:
await self.executor(self.advance_blocks, blocks, headers, touched)
elif hprevs[0] != chain[0]:
await self.handle_chain_reorg(touched)
else:
# It is probably possible but extremely rare that what
# bitcoind returns doesn't form a chain because it
# reorg-ed the chain as it was processing the batched
# block hash requests. Should this happen it's simplest
# just to reset the prefetcher and try again.
self.logger.warning('daemon blocks do not form a chain; '
'resetting the prefetcher')
await self.prefetcher.clear(self.height)
def advance_blocks(self, blocks, headers, touched):
'''Synchronously advance the blocks.
It is already verified they correctly connect onto our tip.
'''
block_txs = self.coin.block_txs
daemon_height = self.daemon.cached_height()
for block in blocks:
self.height += 1
txs = block_txs(block, self.height)
self.tx_hashes.append(b''.join(tx_hash for tx, tx_hash in txs))
undo_info = self.advance_txs(txs, touched)
if daemon_height - self.height <= self.env.reorg_limit:
self.write_undo_info(self.height, b''.join(undo_info))
self.headers.extend(headers)
self.tip = self.coin.header_hash(headers[-1])
# If caught up, flush everything as client queries are
# performed on the DB.
if self.caught_up_event.is_set():
# Flush everything as queries are performed on the DB and
# not in-memory.
self.flush(True)
else:
touched.clear()
@ -242,38 +286,20 @@ class BlockProcessor(server.db.DB):
self.check_cache_size()
self.next_cache_check = time.time() + 30
if not self.first_sync:
s = '' if len(blocks) == 1 else 's'
self.logger.info('processed {:,d} block{} in {:.1f}s'
.format(len(blocks), s,
time.time() - start))
def caught_up(self):
'''Called when first caught up after starting.'''
if not self.caught_up_event.is_set():
self.first_sync = False
self.flush(True)
if self.utxo_db.for_sync:
self.logger.info('{} synced to height {:,d}'
.format(VERSION, self.height))
self.open_dbs()
self.caught_up_event.set()
async def handle_chain_reorg(self, touched, count=None):
'''Handle a chain reorganisation.
Count is the number of blocks to simulate a reorg, or None for
a real reorg.'''
self.logger.info('chain reorg detected')
self.flush(True)
await self.executor(self.flush, True)
hashes = await self.reorg_hashes(count)
# Reverse and convert to hex strings.
hashes = [hash_to_str(hash) for hash in reversed(hashes)]
for hex_hashes in chunks(hashes, 50):
blocks = await self.daemon.raw_blocks(hex_hashes)
self.backup_blocks(blocks, touched)
await self.executor(self.backup_blocks, blocks, touched)
await self.prefetcher.clear(self.height)
async def reorg_hashes(self, count):
@ -306,9 +332,10 @@ class BlockProcessor(server.db.DB):
else:
start = (self.height - count) + 1
self.logger.info('chain was reorganised: {:,d} blocks at '
'heights {:,d}-{:,d} were replaced'
.format(count, start, start + count - 1))
s = '' if count == 1 else 's'
self.logger.info('chain was reorganised replacing {:,d} block{} at '
'heights {:,d}-{:,d}'
.format(count, s, start, start + count - 1))
return self.fs_block_hashes(start, count)
@ -462,27 +489,6 @@ class BlockProcessor(server.db.DB):
if utxo_MB + hist_MB >= self.cache_MB or hist_MB >= self.cache_MB // 5:
self.flush(utxo_MB >= self.cache_MB * 4 // 5)
def fs_advance_block(self, header, txs):
'''Update unflushed FS state for a new block.'''
prior_tx_count = self.tx_counts[-1] if self.tx_counts else 0
# Cache the new header, tx hashes and cumulative tx count
self.headers.append(header)
self.tx_hashes.append(b''.join(tx_hash for tx, tx_hash in txs))
self.tx_counts.append(prior_tx_count + len(txs))
def advance_block(self, block, touched):
header, txs = self.coin.read_block(block, self.height + 1)
if self.tip != self.coin.header_prevhash(header):
raise ChainReorg
self.fs_advance_block(header, txs)
self.tip = self.coin.header_hash(header)
self.height += 1
undo_info = self.advance_txs(txs, touched)
if self.daemon.cached_height() - self.height <= self.env.reorg_limit:
self.write_undo_info(self.height, b''.join(undo_info))
def advance_txs(self, txs, touched):
undo_info = []
@ -524,6 +530,7 @@ class BlockProcessor(server.db.DB):
tx_num += 1
self.tx_count = tx_num
self.tx_counts.append(tx_num)
self.history_size = history_size
return undo_info
@ -537,7 +544,7 @@ class BlockProcessor(server.db.DB):
self.assert_flushed()
for block in blocks:
header, txs = self.coin.read_block(block, self.height)
txs = self.coin.block_txs(block, self.height)
header_hash = self.coin.header_hash(header)
if header_hash != self.tip:
raise ChainError('backup block {} is not tip {} at height {:,d}'

2
server/controller.py
View File

@ -20,7 +20,7 @@ from lib.jsonrpc import JSONRPC, RequestBase
import lib.util as util
from server.block_processor import BlockProcessor
from server.irc import IRC
from server.protocol import LocalRPC, ElectrumX
from server.session import LocalRPC, ElectrumX
from server.mempool import MemPool

2
server/env.py
View File

@ -67,8 +67,6 @@ class Env(LoggedClass):
if self.report_ssl_port else
self.ssl_port)
self.report_host_tor = self.default('REPORT_HOST_TOR', None)
# Debugging
self.force_reorg = self.integer('FORCE_REORG', 0)
def default(self, envvar, default):
return environ.get(envvar, default)

0
server/protocol.py → server/session.py
View File

Write Preview
Loading…
Cancel
Save