lukechilds
/
electrumx
mirror of https://github.com/lukechilds/electrumx.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Save all UTXOs 

Change the DB version
master
Neil Booth 8 years ago
parent
4dac728984
commit
85786b87a2
4 changed files with 100 additions and 91 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 19
      lib/coins.py
  2. 84
      lib/script.py
  3. 86
      server/block_processor.py
  4. 2
      server/db.py

19
lib/coins.py
View File

@ -12,6 +12,7 @@ necessary for appropriate handling.
'''
from decimal import Decimal
from functools import partial
import inspect
import struct
import sys
@ -34,6 +35,7 @@ class Coin(object):
DEFAULT_RPC_PORT = 8332
VALUE_PER_COIN = 100000000
CHUNK_SIZE=2016
STRANGE_VERBYTE = 0xff
@classmethod
def lookup_coin_class(cls, name, net):
@ -53,11 +55,14 @@ class Coin(object):
address = cls.P2PKH_hash168_from_hash160,
script_hash = cls.P2SH_hash168_from_hash160,
pubkey = cls.P2PKH_hash168_from_pubkey,
unspendable = cls.hash168_from_unspendable,
strange = cls.hash168_from_strange,
)
@classmethod
def hash168_from_script(cls, script):
return ScriptPubKey.pay_to(script, cls.hash168_handlers)
def hash168_from_script(cls):
'''Returns a function that is passed a script to return a hash168.'''
return partial(ScriptPubKey.pay_to, cls.hash168_handlers)
@staticmethod
def lookup_xverbytes(verbytes):
@ -86,6 +91,16 @@ class Coin(object):
'''Return an address given a 21-byte hash.'''
return Base58.encode_check(hash168)
@classmethod
def hash168_from_unspendable(cls):
'''Return a hash168 for an unspendable script.'''
return None
@classmethod
def hash168_from_strange(cls, script):
'''Return a hash168 for a strange script.'''
return bytes([cls.STRANGE_VERBYTE]) + hash160(script)
@classmethod
def P2PKH_hash168_from_hash160(cls, hash160):
'''Return a hash168 if hash160 is 160 bits otherwise None.'''

84
lib/script.py
View File

@ -56,6 +56,19 @@ assert OpCodes.OP_CHECKSIG == 0xac
assert OpCodes.OP_CHECKMULTISIG == 0xae
def _match_ops(ops, pattern):
if len(ops) != len(pattern):
return False
for op, pop in zip(ops, pattern):
if pop != op:
# -1 means 'data push', whose op is an (op, data) tuple
if pop == -1 and isinstance(op, tuple):
continue
return False
return True
class ScriptPubKey(object):
'''A class for handling a tx output script that gives conditions
necessary for spending.
@ -66,10 +79,11 @@ class ScriptPubKey(object):
TO_P2SH_OPS = [OpCodes.OP_HASH160, -1, OpCodes.OP_EQUAL]
TO_PUBKEY_OPS = [-1, OpCodes.OP_CHECKSIG]
PayToHandlers = namedtuple('PayToHandlers', 'address script_hash pubkey')
PayToHandlers = namedtuple('PayToHandlers', 'address script_hash pubkey '
'unspendable strange')
@classmethod
def pay_to(cls, script, handlers):
def pay_to(cls, handlers, script):
'''Parse a script, invoke the appropriate handler and
return the result.
@ -77,21 +91,25 @@ class ScriptPubKey(object):
handlers.address(hash160)
handlers.script_hash(hash160)
handlers.pubkey(pubkey)
or None is returned if the script is invalid or unregonised.
handlers.unspendable()
handlers.strange(script)
'''
try:
ops, datas = Script.get_ops(script)
ops = Script.get_ops(script)
except ScriptError:
return None
return handlers.unspendable()
if Script.match_ops(ops, cls.TO_ADDRESS_OPS):
return handlers.address(datas[2])
if Script.match_ops(ops, cls.TO_P2SH_OPS):
return handlers.script_hash(datas[1])
if Script.match_ops(ops, cls.TO_PUBKEY_OPS):
return handlers.pubkey(datas[0])
match = _match_ops
return None
if match(ops, cls.TO_ADDRESS_OPS):
return handlers.address(ops[2][-1])
if match(ops, cls.TO_P2SH_OPS):
return handlers.script_hash(ops[1][-1])
if match(ops, cls.TO_PUBKEY_OPS):
return handlers.pubkey(ops[0][-1])
if OpCodes.OP_RETURN in ops:
return handlers.unspendable()
return handlers.strange(script)
@classmethod
def P2SH_script(cls, hash160):
@ -141,54 +159,40 @@ class Script(object):
@classmethod
def get_ops(cls, script):
opcodes, datas = [], []
ops = []
# The unpacks or script[n] below throw on truncated scripts
try:
n = 0
while n < len(script):
opcode, data = script[n], None
op = script[n]
n += 1
if opcode <= OpCodes.OP_PUSHDATA4:
if op <= OpCodes.OP_PUSHDATA4:
# Raw bytes follow
if opcode < OpCodes.OP_PUSHDATA1:
dlen = opcode
elif opcode == OpCodes.OP_PUSHDATA1:
if op < OpCodes.OP_PUSHDATA1:
dlen = op
elif op == OpCodes.OP_PUSHDATA1:
dlen = script[n]
n += 1
elif opcode == OpCodes.OP_PUSHDATA2:
(dlen,) = struct.unpack('<H', script[n: n + 2])
elif op == OpCodes.OP_PUSHDATA2:
dlen, = struct.unpack('<H', script[n: n + 2])
n += 2
else:
(dlen,) = struct.unpack('<I', script[n: n + 4])
dlen, = struct.unpack('<I', script[n: n + 4])
n += 4
data = script[n:n + dlen]
if len(data) != dlen:
raise ScriptError('truncated script')
if n + dlen > len(script):
raise IndexError
op = (op, script[n:n + dlen])
n += dlen
opcodes.append(opcode)
datas.append(data)
ops.append(op)
except:
# Truncated script; e.g. tx_hash
# ebc9fa1196a59e192352d76c0f6e73167046b9d37b8302b6bb6968dfd279b767
raise ScriptError('truncated script')
return opcodes, datas
@classmethod
def match_ops(cls, ops, pattern):
if len(ops) != len(pattern):
return False
for op, pop in zip(ops, pattern):
if pop != op:
# -1 Indicates data push expected
if pop == -1 and OpCodes.OP_0 <= op <= OpCodes.OP_PUSHDATA4:
continue
return False
return True
return ops
@classmethod
def push_data(cls, data):

86
server/block_processor.py
View File

@ -28,8 +28,6 @@ from server.storage import open_db
# Limits single address history to ~ 65536 * HIST_ENTRIES_PER_KEY entries
HIST_ENTRIES_PER_KEY = 1024
HIST_VALUE_BYTES = HIST_ENTRIES_PER_KEY * 4
NO_HASH_168 = bytes([255]) * 21
NO_CACHE_ENTRY = NO_HASH_168 + bytes(12)
def formatted_time(t):
@ -209,7 +207,7 @@ class MemPool(LoggedClass):
# The mempool is unordered, so process all outputs first so
# that looking for inputs has full info.
script_hash168 = self.bp.coin.hash168_from_script
script_hash168 = self.bp.coin.hash168_from_script()
db_utxo_lookup = self.bp.db_utxo_lookup
def txout_pair(txout):
@ -658,8 +656,6 @@ class BlockProcessor(server.db.DB):
self.logger.info('backing up history to height {:,d} tx_count {:,d}'
.format(self.height, self.tx_count))
# Drop any NO_CACHE entry
hash168s.discard(NO_CACHE_ENTRY)
assert not self.history
nremoves = 0
@ -765,7 +761,7 @@ class BlockProcessor(server.db.DB):
# Use local vars for speed in the loops
history = self.history
tx_num = self.tx_count
script_hash168 = self.coin.hash168_from_script
script_hash168 = self.coin.hash168_from_script()
s_pack = pack
for tx, tx_hash in zip(txs, tx_hashes):
@ -781,15 +777,13 @@ class BlockProcessor(server.db.DB):
# Add the new UTXOs
for idx, txout in enumerate(tx.outputs):
# Get the hash168. Ignore scripts we can't grok.
# Get the hash168. Ignore unspendable outputs
hash168 = script_hash168(txout.pk_script)
if hash168:
hash168s.add(hash168)
put_utxo(tx_hash + s_pack('<H', idx),
hash168 + tx_numb + s_pack('<Q', txout.value))
# Drop any NO_CACHE entry
hash168s.discard(NO_CACHE_ENTRY)
for hash168 in hash168s:
history[hash168].append(tx_num)
self.history_size += len(hash168s)
@ -913,15 +907,15 @@ class BlockProcessor(server.db.DB):
the tx in which the UTXO was created. As this is not unique there
will are potential collisions when saving and looking up UTXOs;
hence why the second table has a list as its value. The collision
can be resolved with the tx_num. The collision rate is almost
zero (I believe there are around 100 collisions in the whole
bitcoin blockchain).
can be resolved with the tx_num. The collision rate is low (<0.1%).
'''
def spend_utxo(self, tx_hash, tx_idx):
'''Spend a UTXO and return the 33-byte value.
If the UTXO is not in the cache it may be on disk.
If the UTXO is not in the cache it must be on disk. We store
all UTXOs so not finding one indicates a logic error or DB
corruption.
'''
# Fast track is it being in the cache
idx_packed = pack('<H', tx_idx)
@ -935,46 +929,42 @@ class BlockProcessor(server.db.DB):
# The 4 is the COMPRESSED_TX_HASH_LEN
db_key = b'h' + tx_hash[:4] + idx_packed
db_value = self.db_cache_get(db_key)
if db_value is None:
# Probably a strange UTXO
return NO_CACHE_ENTRY
# FIXME: this matches what we did previously but until we store
# all UTXOs isn't safe
if len(db_value) == 25:
udb_key = b'u' + db_value + idx_packed
utxo_value_packed = self.db.get(udb_key)
if utxo_value_packed:
# Remove the UTXO from both tables
self.db_deletes += 1
self.db_cache[db_key] = None
self.db_cache[udb_key] = None
return db_value + utxo_value_packed
# Fall through to below
assert len(db_value) % 25 == 0
# Find which entry, if any, the TX_HASH matches.
for n in range(0, len(db_value), 25):
tx_num, = unpack('<I', db_value[n+21:n+25])
hash, height = self.get_tx_hash(tx_num)
if hash == tx_hash:
match = db_value[n:n+25]
udb_key = b'u' + match + idx_packed
if db_value:
# FIXME: this matches what we did previously but until we store
# all UTXOs isn't safe
if len(db_value) == 25:
udb_key = b'u' + db_value + idx_packed
utxo_value_packed = self.db.get(udb_key)
if utxo_value_packed:
# Remove the UTXO from both tables
self.db_deletes += 1
self.db_cache[db_key] = db_value[:n] + db_value[n + 25:]
self.db_cache[db_key] = None
self.db_cache[udb_key] = None
return match + utxo_value_packed
# Uh-oh, this should not happen...
raise self.DBError('UTXO {} / {:,d} not found, key {}'
.format(hash_to_str(tx_hash), tx_idx,
bytes(key).hex()))
return NO_CACHE_ENTRY
return db_value + utxo_value_packed
# Fall through to below loop for error
assert len(db_value) % 25 == 0
# Find which entry, if any, the TX_HASH matches.
for n in range(0, len(db_value), 25):
tx_num, = unpack('<I', db_value[n + 21:n + 25])
hash, height = self.get_tx_hash(tx_num)
if hash == tx_hash:
match = db_value[n:n+25]
udb_key = b'u' + match + idx_packed
utxo_value_packed = self.db.get(udb_key)
if utxo_value_packed:
# Remove the UTXO from both tables
self.db_deletes += 1
self.db_cache[db_key] = db_value[:n] + db_value[n+25:]
self.db_cache[udb_key] = None
return match + utxo_value_packed
raise self.DBError('UTXO {} / {:,d} not found in "u" table'
.format(hash_to_str(tx_hash), tx_idx))
raise ChainError('UTXO {} / {:,d} not found in "h" table'
.format(hash_to_str(tx_hash), tx_idx))
def db_cache_get(self, key):
'''Fetch a 'h' value from the DB through our write cache.'''

2
server/db.py
View File

@ -29,7 +29,7 @@ class DB(LoggedClass):
it was shutdown uncleanly.
'''
VERSIONS = [0]
VERSIONS = [2]
class MissingUTXOError(Exception):
'''Raised if a mempool tx input UTXO couldn't be found.'''

Write Preview
Loading…
Cancel
Save