Neil Booth
7 years ago
5 changed files with 639 additions and 69 deletions
@ -0,0 +1,502 @@ |
|||||
|
import logging |
||||
|
import os |
||||
|
from collections import defaultdict |
||||
|
from functools import partial |
||||
|
from random import randrange, choice |
||||
|
|
||||
|
import pytest |
||||
|
from aiorpcx import Event, TaskGroup, sleep, spawn, ignore_after |
||||
|
|
||||
|
from electrumx.server.mempool import MemPool, MemPoolAPI |
||||
|
from electrumx.lib.coins import BitcoinCash |
||||
|
from electrumx.lib.hash import HASHX_LEN, hex_str_to_hash, hash_to_hex_str |
||||
|
from electrumx.lib.tx import Tx, TxInput, TxOutput |
||||
|
from electrumx.lib.util import make_logger |
||||
|
|
||||
|
|
||||
|
coin = BitcoinCash |
||||
|
tx_hash_fn = coin.DESERIALIZER.TX_HASH_FN |
||||
|
|
||||
|
|
||||
|
def random_tx(hash160s, utxos): |
||||
|
'''Create a random TX paying to some of the hash160s using some of the |
||||
|
UTXOS. Return the TX. UTXOs is updated for the effects of the TX. |
||||
|
''' |
||||
|
inputs = [] |
||||
|
n_inputs = min(randrange(1, 4), len(utxos)) |
||||
|
input_value = 0 |
||||
|
# Create inputs spending random UTXOs. total the inpu |
||||
|
for n in range(n_inputs): |
||||
|
prevout = choice(list(utxos)) |
||||
|
hashX, value = utxos.pop(prevout) |
||||
|
inputs.append(TxInput(prevout[0], prevout[1], b'', 4294967295)) |
||||
|
input_value += value |
||||
|
|
||||
|
fee = min(input_value, randrange(500)) |
||||
|
input_value -= fee |
||||
|
outputs = [] |
||||
|
n_outputs = randrange(1, 4) |
||||
|
for n in range(n_outputs): |
||||
|
value = randrange(input_value) |
||||
|
input_value -= value |
||||
|
pk_script = coin.hash160_to_P2PKH_script(choice(hash160s)) |
||||
|
outputs.append(TxOutput(value, pk_script)) |
||||
|
|
||||
|
tx = Tx(2, inputs, outputs, 0) |
||||
|
tx_bytes = tx.serialize() |
||||
|
tx_hash = tx_hash_fn(tx_bytes) |
||||
|
for n, output in enumerate(tx.outputs): |
||||
|
utxos[(tx_hash, n)] = (coin.hashX_from_script(output.pk_script), |
||||
|
output.value) |
||||
|
return tx, tx_hash, tx_bytes |
||||
|
|
||||
|
|
||||
|
class API(MemPoolAPI): |
||||
|
|
||||
|
def __init__(self): |
||||
|
self._height = 0 |
||||
|
self._cached_height = self._height |
||||
|
# Create a pool of hash160s. Map them to their script hashes |
||||
|
# Create a bunch of UTXOs paying to those script hashes |
||||
|
# Create a bunch of TXs that spend from the UTXO set and create |
||||
|
# new outpus, which are added to the UTXO set for later TXs to |
||||
|
# spend |
||||
|
self.db_utxos = {} |
||||
|
self.on_mempool_calls = [] |
||||
|
self.hashXs = [] |
||||
|
# Maps of mempool txs from tx_hash to raw and Tx object forms |
||||
|
self.raw_txs = {} |
||||
|
self.txs = {} |
||||
|
self.ordered_adds = [] |
||||
|
|
||||
|
def initialize(self, addr_count=100, db_utxo_count=100, mempool_size=50): |
||||
|
hash160s = [os.urandom(20) for n in range(addr_count)] |
||||
|
self.hashXs = [coin.hash160_to_P2PKH_hashX(hash160) |
||||
|
for hash160 in hash160s] |
||||
|
prevouts = [(os.urandom(32), randrange(0, 10)) |
||||
|
for n in range (db_utxo_count)] |
||||
|
random_value = partial(randrange, coin.VALUE_PER_COIN * 10) |
||||
|
self.db_utxos = {prevout: (choice(self.hashXs), random_value()) |
||||
|
for prevout in prevouts} |
||||
|
|
||||
|
unspent_utxos = self.db_utxos.copy() |
||||
|
for n in range(mempool_size): |
||||
|
tx, tx_hash, raw_tx = random_tx(hash160s, unspent_utxos) |
||||
|
self.raw_txs[tx_hash] = raw_tx |
||||
|
self.txs[tx_hash] = tx |
||||
|
self.ordered_adds.append(tx_hash) |
||||
|
|
||||
|
def mempool_utxos(self): |
||||
|
utxos = {} |
||||
|
for tx_hash, tx in self.txs.items(): |
||||
|
for n, output in enumerate(tx.outputs): |
||||
|
hashX = coin.hashX_from_script(output.pk_script) |
||||
|
utxos[(tx_hash, n)] = (hashX, output.value) |
||||
|
return utxos |
||||
|
|
||||
|
def mempool_spends(self): |
||||
|
return [(input.prev_hash, input.prev_idx) |
||||
|
for tx in self.txs.values() for input in tx.inputs] |
||||
|
|
||||
|
def balance_deltas(self): |
||||
|
# Return mempool balance deltas indexed by hashX |
||||
|
deltas = defaultdict(int) |
||||
|
utxos = self.mempool_utxos() |
||||
|
for tx_hash, tx in self.txs.items(): |
||||
|
for n, input in enumerate(tx.inputs): |
||||
|
prevout = (input.prev_hash, input.prev_idx) |
||||
|
if prevout in utxos: |
||||
|
utxos.pop(prevout) |
||||
|
else: |
||||
|
hashX, value = self.db_utxos[prevout] |
||||
|
deltas[hashX] -= value |
||||
|
for hashX, value in utxos.values(): |
||||
|
deltas[hashX] += value |
||||
|
return deltas |
||||
|
|
||||
|
def spends(self): |
||||
|
# Return spends indexed by hashX |
||||
|
spends = defaultdict(list) |
||||
|
utxos = self.mempool_utxos() |
||||
|
for tx_hash, tx in self.txs.items(): |
||||
|
for n, input in enumerate(tx.inputs): |
||||
|
prevout = (input.prev_hash, input.prev_idx) |
||||
|
if prevout in utxos: |
||||
|
hashX, value = utxos.pop(prevout) |
||||
|
else: |
||||
|
hashX, value = self.db_utxos[prevout] |
||||
|
spends[hashX].append(prevout) |
||||
|
return spends |
||||
|
|
||||
|
def summaries(self): |
||||
|
# Return lists of (tx_hash, fee, has_unconfirmed_inputs) by hashX |
||||
|
summaries = defaultdict(list) |
||||
|
utxos = self.mempool_utxos() |
||||
|
for tx_hash, tx in self.txs.items(): |
||||
|
fee = 0 |
||||
|
hashXs = set() |
||||
|
has_ui = False |
||||
|
for n, input in enumerate(tx.inputs): |
||||
|
has_ui = has_ui or (input.prev_hash in self.txs) |
||||
|
prevout = (input.prev_hash, input.prev_idx) |
||||
|
if prevout in utxos: |
||||
|
hashX, value = utxos[prevout] |
||||
|
else: |
||||
|
hashX, value = self.db_utxos[prevout] |
||||
|
hashXs.add(hashX) |
||||
|
fee += value |
||||
|
|
||||
|
for output in tx.outputs: |
||||
|
hashXs.add(coin.hashX_from_script(output.pk_script)) |
||||
|
fee -= output.value |
||||
|
|
||||
|
summary = (tx_hash, fee, has_ui) |
||||
|
for hashX in hashXs: |
||||
|
summaries[hashX].append(summary) |
||||
|
return summaries |
||||
|
|
||||
|
def touched(self, tx_hashes): |
||||
|
touched = set() |
||||
|
utxos = self.mempool_utxos() |
||||
|
for tx_hash in tx_hashes: |
||||
|
tx = self.txs[tx_hash] |
||||
|
for n, input in enumerate(tx.inputs): |
||||
|
prevout = (input.prev_hash, input.prev_idx) |
||||
|
if prevout in utxos: |
||||
|
hashX, value = utxos[prevout] |
||||
|
else: |
||||
|
hashX, value = self.db_utxos[prevout] |
||||
|
touched.add(hashX) |
||||
|
|
||||
|
for output in tx.outputs: |
||||
|
touched.add(coin.hashX_from_script(output.pk_script)) |
||||
|
return touched |
||||
|
|
||||
|
def UTXOs(self): |
||||
|
# Return lists of UTXO 5-tuples by hashX |
||||
|
utxos = defaultdict(list) |
||||
|
for tx_hash, tx in self.txs.items(): |
||||
|
for n, output in enumerate(tx.outputs): |
||||
|
hashX = coin.hashX_from_script(output.pk_script) |
||||
|
utxos[hashX].append((-1, n, tx_hash, 0, output.value)) |
||||
|
return utxos |
||||
|
|
||||
|
async def height(self): |
||||
|
await sleep(0) |
||||
|
self._cached_height = self._height |
||||
|
return self._height |
||||
|
|
||||
|
def cached_height(self): |
||||
|
return self._cached_height |
||||
|
|
||||
|
async def mempool_hashes(self): |
||||
|
'''Query bitcoind for the hashes of all transactions in its |
||||
|
mempool, returned as a list.''' |
||||
|
await sleep(0) |
||||
|
return [hash_to_hex_str(hash) for hash in self.txs] |
||||
|
|
||||
|
async def raw_transactions(self, hex_hashes): |
||||
|
'''Query bitcoind for the serialized raw transactions with the given |
||||
|
hashes. Missing transactions are returned as None. |
||||
|
|
||||
|
hex_hashes is an iterable of hexadecimal hash strings.''' |
||||
|
await sleep(0) |
||||
|
hashes = [hex_str_to_hash(hex_hash) for hex_hash in hex_hashes] |
||||
|
return [self.raw_txs.get(hash) for hash in hashes] |
||||
|
|
||||
|
async def lookup_utxos(self, prevouts): |
||||
|
'''Return a list of (hashX, value) pairs each prevout if unspent, |
||||
|
otherwise return None if spent or not found. |
||||
|
|
||||
|
prevouts - an iterable of (hash, index) pairs |
||||
|
''' |
||||
|
await sleep(0) |
||||
|
return [self.db_utxos.get(prevout) for prevout in prevouts] |
||||
|
|
||||
|
async def on_mempool(self, touched, height): |
||||
|
'''Called each time the mempool is synchronized. touched is a set of |
||||
|
hashXs touched since the previous call. height is the |
||||
|
daemon's height at the time the mempool was obtained.''' |
||||
|
self.on_mempool_calls.append((touched, height)) |
||||
|
await sleep(0) |
||||
|
|
||||
|
|
||||
|
class DropAPI(API): |
||||
|
|
||||
|
def __init__(self, drop_count): |
||||
|
super().__init__() |
||||
|
self.drop_count = drop_count |
||||
|
self.dropped = False |
||||
|
|
||||
|
async def raw_transactions(self, hex_hashes): |
||||
|
if not self.dropped: |
||||
|
self.dropped = True |
||||
|
for hash in self.ordered_adds[-self.drop_count:]: |
||||
|
del self.raw_txs[hash] |
||||
|
del self.txs[hash] |
||||
|
return await super().raw_transactions(hex_hashes) |
||||
|
|
||||
|
|
||||
|
def in_caplog(caplog, message): |
||||
|
return any(message in record.message for record in caplog.records) |
||||
|
|
||||
|
|
||||
|
@pytest.mark.asyncio |
||||
|
async def test_keep_synchronized(caplog): |
||||
|
api = API() |
||||
|
mempool = MemPool(coin, api) |
||||
|
event = Event() |
||||
|
with caplog.at_level(logging.INFO): |
||||
|
async with TaskGroup() as group: |
||||
|
await group.spawn(mempool.keep_synchronized, event) |
||||
|
await event.wait() |
||||
|
await group.cancel_remaining() |
||||
|
|
||||
|
assert in_caplog(caplog, 'beginning processing of daemon mempool') |
||||
|
assert in_caplog(caplog, 'compact fee histogram') |
||||
|
assert in_caplog(caplog, 'synced in ') |
||||
|
assert in_caplog(caplog, '0 txs touching 0 addresses') |
||||
|
assert not in_caplog(caplog, 'txs dropped') |
||||
|
|
||||
|
|
||||
|
@pytest.mark.asyncio |
||||
|
async def test_balance_delta(): |
||||
|
api = API() |
||||
|
api.initialize() |
||||
|
mempool = MemPool(coin, api) |
||||
|
event = Event() |
||||
|
async with TaskGroup() as group: |
||||
|
await group.spawn(mempool.keep_synchronized, event) |
||||
|
await event.wait() |
||||
|
await group.cancel_remaining() |
||||
|
|
||||
|
# Check the default dict is handled properly |
||||
|
prior_len = len(mempool.hashXs) |
||||
|
assert await mempool.balance_delta(os.urandom(HASHX_LEN)) == 0 |
||||
|
assert prior_len == len(mempool.hashXs) |
||||
|
|
||||
|
# Test all hashXs |
||||
|
deltas = api.balance_deltas() |
||||
|
for hashX in api.hashXs: |
||||
|
expected = deltas.get(hashX, 0) |
||||
|
assert await mempool.balance_delta(hashX) == expected |
||||
|
|
||||
|
|
||||
|
@pytest.mark.asyncio |
||||
|
async def test_compact_fee_histogram(): |
||||
|
api = API() |
||||
|
api.initialize() |
||||
|
mempool = MemPool(coin, api) |
||||
|
event = Event() |
||||
|
async with TaskGroup() as group: |
||||
|
await group.spawn(mempool.keep_synchronized, event) |
||||
|
await event.wait() |
||||
|
await group.cancel_remaining() |
||||
|
|
||||
|
histogram = await mempool.compact_fee_histogram() |
||||
|
assert histogram == [] |
||||
|
bin_size = 1000 |
||||
|
mempool._update_histogram(bin_size) |
||||
|
histogram = await mempool.compact_fee_histogram() |
||||
|
assert len(histogram) > 0 |
||||
|
rates, sizes = zip(*histogram) |
||||
|
assert all(rates[n] < rates[n - 1] for n in range(1, len(rates))) |
||||
|
assert all(size > bin_size * 0.95 for size in sizes) |
||||
|
|
||||
|
|
||||
|
@pytest.mark.asyncio |
||||
|
async def test_potential_spends(): |
||||
|
api = API() |
||||
|
api.initialize() |
||||
|
mempool = MemPool(coin, api) |
||||
|
event = Event() |
||||
|
async with TaskGroup() as group: |
||||
|
await group.spawn(mempool.keep_synchronized, event) |
||||
|
await event.wait() |
||||
|
await group.cancel_remaining() |
||||
|
|
||||
|
# Check the default dict is handled properly |
||||
|
prior_len = len(mempool.hashXs) |
||||
|
assert await mempool.potential_spends(os.urandom(HASHX_LEN)) == set() |
||||
|
assert prior_len == len(mempool.hashXs) |
||||
|
|
||||
|
# Test all hashXs |
||||
|
spends = api.spends() |
||||
|
for hashX in api.hashXs: |
||||
|
ps = await mempool.potential_spends(hashX) |
||||
|
assert all(spend in ps for spend in spends[hashX]) |
||||
|
|
||||
|
|
||||
|
async def _test_summaries(mempool, api): |
||||
|
# Test all hashXs |
||||
|
summaries = api.summaries() |
||||
|
for hashX in api.hashXs: |
||||
|
mempool_result = await mempool.transaction_summaries(hashX) |
||||
|
mempool_result = [(item.hash, item.fee, item.has_unconfirmed_inputs) |
||||
|
for item in mempool_result] |
||||
|
our_result = summaries.get(hashX, []) |
||||
|
assert set(our_result) == set(mempool_result) |
||||
|
|
||||
|
|
||||
|
@pytest.mark.asyncio |
||||
|
async def test_transaction_summaries(caplog): |
||||
|
api = API() |
||||
|
api.initialize() |
||||
|
mempool = MemPool(coin, api) |
||||
|
event = Event() |
||||
|
with caplog.at_level(logging.INFO): |
||||
|
async with TaskGroup() as group: |
||||
|
await group.spawn(mempool.keep_synchronized, event) |
||||
|
await event.wait() |
||||
|
await group.cancel_remaining() |
||||
|
|
||||
|
# Check the default dict is handled properly |
||||
|
prior_len = len(mempool.hashXs) |
||||
|
assert await mempool.transaction_summaries(os.urandom(HASHX_LEN)) == [] |
||||
|
assert prior_len == len(mempool.hashXs) |
||||
|
|
||||
|
await _test_summaries(mempool, api) |
||||
|
assert not in_caplog(caplog, 'txs dropped') |
||||
|
|
||||
|
|
||||
|
@pytest.mark.asyncio |
||||
|
async def test_unordered_UTXOs(): |
||||
|
api = API() |
||||
|
api.initialize() |
||||
|
mempool = MemPool(coin, api) |
||||
|
event = Event() |
||||
|
async with TaskGroup() as group: |
||||
|
await group.spawn(mempool.keep_synchronized, event) |
||||
|
await event.wait() |
||||
|
await group.cancel_remaining() |
||||
|
|
||||
|
# Check the default dict is handled properly |
||||
|
prior_len = len(mempool.hashXs) |
||||
|
assert await mempool.unordered_UTXOs(os.urandom(HASHX_LEN)) == [] |
||||
|
assert prior_len == len(mempool.hashXs) |
||||
|
|
||||
|
# Test all hashXs |
||||
|
utxos = api.UTXOs() |
||||
|
for hashX in api.hashXs: |
||||
|
mempool_result = await mempool.unordered_UTXOs(hashX) |
||||
|
our_result = utxos.get(hashX, []) |
||||
|
assert set(our_result) == set(mempool_result) |
||||
|
|
||||
|
|
||||
|
@pytest.mark.asyncio |
||||
|
async def test_mempool_removals(): |
||||
|
api = API() |
||||
|
api.initialize() |
||||
|
mempool = MemPool(coin, api, refresh_secs=0.01) |
||||
|
event = Event() |
||||
|
async with TaskGroup() as group: |
||||
|
await group.spawn(mempool.keep_synchronized, event) |
||||
|
await event.wait() |
||||
|
# Remove half the TXs from the mempool |
||||
|
start = len(api.ordered_adds) // 2 |
||||
|
for tx_hash in api.ordered_adds[start:]: |
||||
|
del api.txs[tx_hash] |
||||
|
del api.raw_txs[tx_hash] |
||||
|
await event.wait() |
||||
|
await _test_summaries(mempool, api) |
||||
|
# Removed hashXs should have key destroyed |
||||
|
assert all(mempool.hashXs.values()) |
||||
|
# Remove the rest |
||||
|
api.txs.clear() |
||||
|
api.raw_txs.clear() |
||||
|
await event.wait() |
||||
|
await _test_summaries(mempool, api) |
||||
|
assert not mempool.hashXs |
||||
|
assert not mempool.txs |
||||
|
await group.cancel_remaining() |
||||
|
|
||||
|
|
||||
|
@pytest.mark.asyncio |
||||
|
async def test_daemon_drops_txs(): |
||||
|
# Tests things work if the daemon drops some transactions between |
||||
|
# returning their hashes and the mempool requesting the raw txs |
||||
|
api = DropAPI(10) |
||||
|
api.initialize() |
||||
|
mempool = MemPool(coin, api, refresh_secs=0.01) |
||||
|
event = Event() |
||||
|
async with TaskGroup() as group: |
||||
|
await group.spawn(mempool.keep_synchronized, event) |
||||
|
await event.wait() |
||||
|
await _test_summaries(mempool, api) |
||||
|
await group.cancel_remaining() |
||||
|
|
||||
|
|
||||
|
@pytest.mark.asyncio |
||||
|
async def test_notifications(): |
||||
|
# Tests notifications over a cycle of: |
||||
|
# 1) A first batch of txs come in |
||||
|
# 2) A second batch of txs come in |
||||
|
# 3) A block comes in confirming the first batch only |
||||
|
api = API() |
||||
|
api.initialize() |
||||
|
mempool = MemPool(coin, api, refresh_secs=0.001, log_status_secs=0) |
||||
|
event = Event() |
||||
|
|
||||
|
n = len(api.ordered_adds) // 2 |
||||
|
raw_txs = api.raw_txs.copy() |
||||
|
txs = api.txs.copy() |
||||
|
first_hashes = api.ordered_adds[:n] |
||||
|
first_touched = api.touched(first_hashes) |
||||
|
second_hashes = api.ordered_adds[n:] |
||||
|
second_touched = api.touched(second_hashes) |
||||
|
|
||||
|
async with TaskGroup() as group: |
||||
|
# First batch enters the mempool |
||||
|
api.raw_txs = {hash: raw_txs[hash] for hash in first_hashes} |
||||
|
api.txs = {hash: txs[hash] for hash in first_hashes} |
||||
|
first_utxos = api.mempool_utxos() |
||||
|
first_spends = api.mempool_spends() |
||||
|
await group.spawn(mempool.keep_synchronized, event) |
||||
|
await event.wait() |
||||
|
assert len(api.on_mempool_calls) == 1 |
||||
|
touched, height = api.on_mempool_calls[0] |
||||
|
assert height == api._height == api._cached_height |
||||
|
assert touched == first_touched |
||||
|
# Second batch enters the mempool |
||||
|
api.raw_txs = raw_txs |
||||
|
api.txs = txs |
||||
|
await event.wait() |
||||
|
assert len(api.on_mempool_calls) == 2 |
||||
|
touched, height = api.on_mempool_calls[1] |
||||
|
assert height == api._height == api._cached_height |
||||
|
# Touched is incremental |
||||
|
assert touched == second_touched |
||||
|
# Block found; first half confirm |
||||
|
new_height = 2 |
||||
|
api._height = new_height |
||||
|
api.db_utxos.update(first_utxos) |
||||
|
for spend in first_spends: |
||||
|
del api.db_utxos[spend] |
||||
|
api.raw_txs = {hash: raw_txs[hash] for hash in second_hashes} |
||||
|
api.txs = {hash: txs[hash] for hash in second_hashes} |
||||
|
await event.wait() |
||||
|
assert len(api.on_mempool_calls) == 3 |
||||
|
touched, height = api.on_mempool_calls[2] |
||||
|
assert height == api._height == api._cached_height == new_height |
||||
|
assert touched == first_touched |
||||
|
await group.cancel_remaining() |
||||
|
|
||||
|
|
||||
|
@pytest.mark.asyncio |
||||
|
async def test_dropped_txs(caplog): |
||||
|
api = API() |
||||
|
api.initialize() |
||||
|
mempool = MemPool(coin, api) |
||||
|
event = Event() |
||||
|
# Remove a single TX_HASH that is used in another mempool tx |
||||
|
for prev_hash, prev_idx in api.mempool_spends(): |
||||
|
if prev_hash in api.txs: |
||||
|
del api.txs[prev_hash] |
||||
|
|
||||
|
with caplog.at_level(logging.INFO): |
||||
|
async with TaskGroup() as group: |
||||
|
await group.spawn(mempool.keep_synchronized, event) |
||||
|
await event.wait() |
||||
|
await group.cancel_remaining() |
||||
|
|
||||
|
assert in_caplog(caplog, 'txs dropped') |
||||
Loading…
Reference in new issue