lukechilds
/
electrumx
mirror of https://github.com/lukechilds/electrumx.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Save all UTXOs 

Change the DB version
master
Neil Booth 8 years ago
parent
4dac728984
commit
85786b87a2
4 changed files with 100 additions and 91 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 19
      lib/coins.py
  2. 84
      lib/script.py
  3. 36
      server/block_processor.py
  4. 2
      server/db.py

19
lib/coins.py
View File

@ -12,6 +12,7 @@ necessary for appropriate handling.
''' '''
from decimal import Decimal from decimal import Decimal
from functools import partial
import inspect import inspect
import struct import struct
import sys import sys
@ -34,6 +35,7 @@ class Coin(object):
DEFAULT_RPC_PORT = 8332 DEFAULT_RPC_PORT = 8332
VALUE_PER_COIN = 100000000 VALUE_PER_COIN = 100000000
CHUNK_SIZE=2016 CHUNK_SIZE=2016
STRANGE_VERBYTE = 0xff
@classmethod @classmethod
def lookup_coin_class(cls, name, net): def lookup_coin_class(cls, name, net):
@ -53,11 +55,14 @@ class Coin(object):
address = cls.P2PKH_hash168_from_hash160, address = cls.P2PKH_hash168_from_hash160,
script_hash = cls.P2SH_hash168_from_hash160, script_hash = cls.P2SH_hash168_from_hash160,
pubkey = cls.P2PKH_hash168_from_pubkey, pubkey = cls.P2PKH_hash168_from_pubkey,
unspendable = cls.hash168_from_unspendable,
strange = cls.hash168_from_strange,
) )
@classmethod @classmethod
def hash168_from_script(cls, script): def hash168_from_script(cls):
return ScriptPubKey.pay_to(script, cls.hash168_handlers) '''Returns a function that is passed a script to return a hash168.'''
return partial(ScriptPubKey.pay_to, cls.hash168_handlers)
@staticmethod @staticmethod
def lookup_xverbytes(verbytes): def lookup_xverbytes(verbytes):
@ -86,6 +91,16 @@ class Coin(object):
'''Return an address given a 21-byte hash.''' '''Return an address given a 21-byte hash.'''
return Base58.encode_check(hash168) return Base58.encode_check(hash168)
@classmethod
def hash168_from_unspendable(cls):
'''Return a hash168 for an unspendable script.'''
return None
@classmethod
def hash168_from_strange(cls, script):
'''Return a hash168 for a strange script.'''
return bytes([cls.STRANGE_VERBYTE]) + hash160(script)
@classmethod @classmethod
def P2PKH_hash168_from_hash160(cls, hash160): def P2PKH_hash168_from_hash160(cls, hash160):
'''Return a hash168 if hash160 is 160 bits otherwise None.''' '''Return a hash168 if hash160 is 160 bits otherwise None.'''

84
lib/script.py
View File

@ -56,6 +56,19 @@ assert OpCodes.OP_CHECKSIG == 0xac
assert OpCodes.OP_CHECKMULTISIG == 0xae assert OpCodes.OP_CHECKMULTISIG == 0xae
def _match_ops(ops, pattern):
if len(ops) != len(pattern):
return False
for op, pop in zip(ops, pattern):
if pop != op:
# -1 means 'data push', whose op is an (op, data) tuple
if pop == -1 and isinstance(op, tuple):
continue
return False
return True
class ScriptPubKey(object): class ScriptPubKey(object):
'''A class for handling a tx output script that gives conditions '''A class for handling a tx output script that gives conditions
necessary for spending. necessary for spending.
@ -66,10 +79,11 @@ class ScriptPubKey(object):
TO_P2SH_OPS = [OpCodes.OP_HASH160, -1, OpCodes.OP_EQUAL] TO_P2SH_OPS = [OpCodes.OP_HASH160, -1, OpCodes.OP_EQUAL]
TO_PUBKEY_OPS = [-1, OpCodes.OP_CHECKSIG] TO_PUBKEY_OPS = [-1, OpCodes.OP_CHECKSIG]
PayToHandlers = namedtuple('PayToHandlers', 'address script_hash pubkey') PayToHandlers = namedtuple('PayToHandlers', 'address script_hash pubkey '
'unspendable strange')
@classmethod @classmethod
def pay_to(cls, script, handlers): def pay_to(cls, handlers, script):
'''Parse a script, invoke the appropriate handler and '''Parse a script, invoke the appropriate handler and
return the result. return the result.
@ -77,21 +91,25 @@ class ScriptPubKey(object):
handlers.address(hash160) handlers.address(hash160)
handlers.script_hash(hash160) handlers.script_hash(hash160)
handlers.pubkey(pubkey) handlers.pubkey(pubkey)
or None is returned if the script is invalid or unregonised. handlers.unspendable()
handlers.strange(script)
''' '''
try: try:
ops, datas = Script.get_ops(script) ops = Script.get_ops(script)
except ScriptError: except ScriptError:
return None return handlers.unspendable()
if Script.match_ops(ops, cls.TO_ADDRESS_OPS): match = _match_ops
return handlers.address(datas[2])
if Script.match_ops(ops, cls.TO_P2SH_OPS):
return handlers.script_hash(datas[1])
if Script.match_ops(ops, cls.TO_PUBKEY_OPS):
return handlers.pubkey(datas[0])
return None if match(ops, cls.TO_ADDRESS_OPS):
return handlers.address(ops[2][-1])
if match(ops, cls.TO_P2SH_OPS):
return handlers.script_hash(ops[1][-1])
if match(ops, cls.TO_PUBKEY_OPS):
return handlers.pubkey(ops[0][-1])
if OpCodes.OP_RETURN in ops:
return handlers.unspendable()
return handlers.strange(script)
@classmethod @classmethod
def P2SH_script(cls, hash160): def P2SH_script(cls, hash160):
@ -141,54 +159,40 @@ class Script(object):
@classmethod @classmethod
def get_ops(cls, script): def get_ops(cls, script):
opcodes, datas = [], [] ops = []
# The unpacks or script[n] below throw on truncated scripts # The unpacks or script[n] below throw on truncated scripts
try: try:
n = 0 n = 0
while n < len(script): while n < len(script):
opcode, data = script[n], None op = script[n]
n += 1 n += 1
if opcode <= OpCodes.OP_PUSHDATA4: if op <= OpCodes.OP_PUSHDATA4:
# Raw bytes follow # Raw bytes follow
if opcode < OpCodes.OP_PUSHDATA1: if op < OpCodes.OP_PUSHDATA1:
dlen = opcode dlen = op
elif opcode == OpCodes.OP_PUSHDATA1: elif op == OpCodes.OP_PUSHDATA1:
dlen = script[n] dlen = script[n]
n += 1 n += 1
elif opcode == OpCodes.OP_PUSHDATA2: elif op == OpCodes.OP_PUSHDATA2:
(dlen,) = struct.unpack('<H', script[n: n + 2]) dlen, = struct.unpack('<H', script[n: n + 2])
n += 2 n += 2
else: else:
(dlen,) = struct.unpack('<I', script[n: n + 4]) dlen, = struct.unpack('<I', script[n: n + 4])
n += 4 n += 4
data = script[n:n + dlen] if n + dlen > len(script):
if len(data) != dlen: raise IndexError
raise ScriptError('truncated script') op = (op, script[n:n + dlen])
n += dlen n += dlen
opcodes.append(opcode) ops.append(op)
datas.append(data)
except: except:
# Truncated script; e.g. tx_hash # Truncated script; e.g. tx_hash
# ebc9fa1196a59e192352d76c0f6e73167046b9d37b8302b6bb6968dfd279b767 # ebc9fa1196a59e192352d76c0f6e73167046b9d37b8302b6bb6968dfd279b767
raise ScriptError('truncated script') raise ScriptError('truncated script')
return opcodes, datas return ops
@classmethod
def match_ops(cls, ops, pattern):
if len(ops) != len(pattern):
return False
for op, pop in zip(ops, pattern):
if pop != op:
# -1 Indicates data push expected
if pop == -1 and OpCodes.OP_0 <= op <= OpCodes.OP_PUSHDATA4:
continue
return False
return True
@classmethod @classmethod
def push_data(cls, data): def push_data(cls, data):

36
server/block_processor.py
View File

@ -28,8 +28,6 @@ from server.storage import open_db
# Limits single address history to ~ 65536 * HIST_ENTRIES_PER_KEY entries # Limits single address history to ~ 65536 * HIST_ENTRIES_PER_KEY entries
HIST_ENTRIES_PER_KEY = 1024 HIST_ENTRIES_PER_KEY = 1024
HIST_VALUE_BYTES = HIST_ENTRIES_PER_KEY * 4 HIST_VALUE_BYTES = HIST_ENTRIES_PER_KEY * 4
NO_HASH_168 = bytes([255]) * 21
NO_CACHE_ENTRY = NO_HASH_168 + bytes(12)
def formatted_time(t): def formatted_time(t):
@ -209,7 +207,7 @@ class MemPool(LoggedClass):
# The mempool is unordered, so process all outputs first so # The mempool is unordered, so process all outputs first so
# that looking for inputs has full info. # that looking for inputs has full info.
script_hash168 = self.bp.coin.hash168_from_script script_hash168 = self.bp.coin.hash168_from_script()
db_utxo_lookup = self.bp.db_utxo_lookup db_utxo_lookup = self.bp.db_utxo_lookup
def txout_pair(txout): def txout_pair(txout):
@ -658,8 +656,6 @@ class BlockProcessor(server.db.DB):
self.logger.info('backing up history to height {:,d} tx_count {:,d}' self.logger.info('backing up history to height {:,d} tx_count {:,d}'
.format(self.height, self.tx_count)) .format(self.height, self.tx_count))
# Drop any NO_CACHE entry
hash168s.discard(NO_CACHE_ENTRY)
assert not self.history assert not self.history
nremoves = 0 nremoves = 0
@ -765,7 +761,7 @@ class BlockProcessor(server.db.DB):
# Use local vars for speed in the loops # Use local vars for speed in the loops
history = self.history history = self.history
tx_num = self.tx_count tx_num = self.tx_count
script_hash168 = self.coin.hash168_from_script script_hash168 = self.coin.hash168_from_script()
s_pack = pack s_pack = pack
for tx, tx_hash in zip(txs, tx_hashes): for tx, tx_hash in zip(txs, tx_hashes):
@ -781,15 +777,13 @@ class BlockProcessor(server.db.DB):
# Add the new UTXOs # Add the new UTXOs
for idx, txout in enumerate(tx.outputs): for idx, txout in enumerate(tx.outputs):
# Get the hash168. Ignore scripts we can't grok. # Get the hash168. Ignore unspendable outputs
hash168 = script_hash168(txout.pk_script) hash168 = script_hash168(txout.pk_script)
if hash168: if hash168:
hash168s.add(hash168) hash168s.add(hash168)
put_utxo(tx_hash + s_pack('<H', idx), put_utxo(tx_hash + s_pack('<H', idx),
hash168 + tx_numb + s_pack('<Q', txout.value)) hash168 + tx_numb + s_pack('<Q', txout.value))
# Drop any NO_CACHE entry
hash168s.discard(NO_CACHE_ENTRY)
for hash168 in hash168s: for hash168 in hash168s:
history[hash168].append(tx_num) history[hash168].append(tx_num)
self.history_size += len(hash168s) self.history_size += len(hash168s)
@ -913,15 +907,15 @@ class BlockProcessor(server.db.DB):
the tx in which the UTXO was created. As this is not unique there the tx in which the UTXO was created. As this is not unique there
will are potential collisions when saving and looking up UTXOs; will are potential collisions when saving and looking up UTXOs;
hence why the second table has a list as its value. The collision hence why the second table has a list as its value. The collision
can be resolved with the tx_num. The collision rate is almost can be resolved with the tx_num. The collision rate is low (<0.1%).
zero (I believe there are around 100 collisions in the whole
bitcoin blockchain).
''' '''
def spend_utxo(self, tx_hash, tx_idx): def spend_utxo(self, tx_hash, tx_idx):
'''Spend a UTXO and return the 33-byte value. '''Spend a UTXO and return the 33-byte value.
If the UTXO is not in the cache it may be on disk. If the UTXO is not in the cache it must be on disk. We store
all UTXOs so not finding one indicates a logic error or DB
corruption.
''' '''
# Fast track is it being in the cache # Fast track is it being in the cache
idx_packed = pack('<H', tx_idx) idx_packed = pack('<H', tx_idx)
@ -935,10 +929,7 @@ class BlockProcessor(server.db.DB):
# The 4 is the COMPRESSED_TX_HASH_LEN # The 4 is the COMPRESSED_TX_HASH_LEN
db_key = b'h' + tx_hash[:4] + idx_packed db_key = b'h' + tx_hash[:4] + idx_packed
db_value = self.db_cache_get(db_key) db_value = self.db_cache_get(db_key)
if db_value is None: if db_value:
# Probably a strange UTXO
return NO_CACHE_ENTRY
# FIXME: this matches what we did previously but until we store # FIXME: this matches what we did previously but until we store
# all UTXOs isn't safe # all UTXOs isn't safe
if len(db_value) == 25: if len(db_value) == 25:
@ -950,7 +941,7 @@ class BlockProcessor(server.db.DB):
self.db_cache[db_key] = None self.db_cache[db_key] = None
self.db_cache[udb_key] = None self.db_cache[udb_key] = None
return db_value + utxo_value_packed return db_value + utxo_value_packed
# Fall through to below # Fall through to below loop for error
assert len(db_value) % 25 == 0 assert len(db_value) % 25 == 0
@ -969,12 +960,11 @@ class BlockProcessor(server.db.DB):
self.db_cache[udb_key] = None self.db_cache[udb_key] = None
return match + utxo_value_packed return match + utxo_value_packed
# Uh-oh, this should not happen... raise self.DBError('UTXO {} / {:,d} not found in "u" table'
raise self.DBError('UTXO {} / {:,d} not found, key {}' .format(hash_to_str(tx_hash), tx_idx))
.format(hash_to_str(tx_hash), tx_idx,
bytes(key).hex()))
return NO_CACHE_ENTRY raise ChainError('UTXO {} / {:,d} not found in "h" table'
.format(hash_to_str(tx_hash), tx_idx))
def db_cache_get(self, key): def db_cache_get(self, key):
'''Fetch a 'h' value from the DB through our write cache.''' '''Fetch a 'h' value from the DB through our write cache.'''

2
server/db.py
View File

@ -29,7 +29,7 @@ class DB(LoggedClass):
it was shutdown uncleanly. it was shutdown uncleanly.
''' '''
VERSIONS = [0] VERSIONS = [2]
class MissingUTXOError(Exception): class MissingUTXOError(Exception):
'''Raised if a mempool tx input UTXO couldn't be found.''' '''Raised if a mempool tx input UTXO couldn't be found.'''

Write Preview
Loading…
Cancel
Save