lukechilds
/
electrumx
mirror of https://github.com/lukechilds/electrumx.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Use hash168 to distinguish address types in DB

master
Neil Booth 8 years ago
parent
4879422e92
commit
6644102441
5 changed files with 79 additions and 86 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 46
      lib/coins.py
  2. 17
      lib/script.py
  3. 18
      lib/util.py
  4. 8
      query.py
  5. 76
      server/db.py

46
lib/coins.py
View File

@ -16,7 +16,7 @@ class CoinError(Exception):
class Coin(object): class Coin(object):
'''Base class of coin hierarchy''' '''Base class of coin hierarchy.'''
# Not sure if these are coin-specific # Not sure if these are coin-specific
HEADER_LEN = 80 HEADER_LEN = 80
@ -52,59 +52,67 @@ class Coin(object):
raise CoinError("version bytes unrecognised") raise CoinError("version bytes unrecognised")
@classmethod @classmethod
def address_to_hash160(cls, addr): def address_to_hash168(cls, addr):
'''Returns a hash160 given an address''' '''Return a 21-byte hash given an address.
This is the hash160 prefixed by the address version byte.
'''
result = Base58.decode_check(addr) result = Base58.decode_check(addr)
if len(result) != 21: if len(result) != 21:
raise CoinError('invalid address: {}'.format(addr)) raise CoinError('invalid address: {}'.format(addr))
return result[1:] return result
@classmethod @classmethod
def P2PKH_address_from_hash160(cls, hash_bytes): def P2PKH_address_from_hash160(cls, hash_bytes):
'''Returns a P2PKH address given a public key''' '''Return a P2PKH address given a public key.'''
assert len(hash_bytes) == 20 assert len(hash_bytes) == 20
payload = bytes([cls.P2PKH_VERBYTE]) + hash_bytes payload = bytes([cls.P2PKH_VERBYTE]) + hash_bytes
return Base58.encode_check(payload) return Base58.encode_check(payload)
@classmethod @classmethod
def P2PKH_address_from_pubkey(cls, pubkey): def P2PKH_address_from_pubkey(cls, pubkey):
'''Returns a coin address given a public key''' '''Return a coin address given a public key.'''
return cls.P2PKH_address_from_hash160(hash160(pubkey)) return cls.P2PKH_address_from_hash160(hash160(pubkey))
@classmethod @classmethod
def P2SH_address_from_hash160(cls, pubkey_bytes): def P2SH_address_from_hash160(cls, pubkey_bytes):
'''Returns a coin address given a public key''' '''Return a coin address given a public key.'''
assert len(hash_bytes) == 20 assert len(hash_bytes) == 20
payload = bytes([cls.P2SH_VERBYTE]) + hash_bytes payload = bytes([cls.P2SH_VERBYTE]) + hash_bytes
return Base58.encode_check(payload) return Base58.encode_check(payload)
@classmethod @classmethod
def multisig_address(cls, m, pubkeys): def multisig_address(cls, m, pubkeys):
'''Returns the P2SH address for an M of N multisig transaction. Pass '''Return the P2SH address for an M of N multisig transaction.
the N pubkeys of which M are needed to sign it. If generating
an address for a wallet, it is the caller's responsibility to Pass the N pubkeys of which M are needed to sign it. If
sort them to ensure order does not matter for, e.g., wallet generating an address for a wallet, it is the caller's
recovery.''' responsibility to sort them to ensure order does not matter
for, e.g., wallet recovery.
'''
script = cls.pay_to_multisig_script(m, pubkeys) script = cls.pay_to_multisig_script(m, pubkeys)
payload = bytes([cls.P2SH_VERBYTE]) + hash160(pubkey_bytes) payload = bytes([cls.P2SH_VERBYTE]) + hash160(pubkey_bytes)
return Base58.encode_check(payload) return Base58.encode_check(payload)
@classmethod @classmethod
def pay_to_multisig_script(cls, m, pubkeys): def pay_to_multisig_script(cls, m, pubkeys):
'''Returns a P2SH multisig script for an M of N multisig '''Return a P2SH script for an M of N multisig transaction.'''
transaction.'''
return ScriptPubKey.multisig_script(m, pubkeys) return ScriptPubKey.multisig_script(m, pubkeys)
@classmethod @classmethod
def pay_to_pubkey_script(cls, pubkey): def pay_to_pubkey_script(cls, pubkey):
'''Returns a pubkey script that pays to pubkey. The input is the '''Return a pubkey script that pays to a pubkey.
raw pubkey bytes (length 33 or 65).'''
Pass the raw pubkey bytes (length 33 or 65).
'''
return ScriptPubKey.P2PK_script(pubkey) return ScriptPubKey.P2PK_script(pubkey)
@classmethod @classmethod
def pay_to_address_script(cls, address): def pay_to_address_script(cls, address):
'''Returns a pubkey script that pays to pubkey hash. Input is the '''Return a pubkey script that pays to a pubkey hash.
address (either P2PKH or P2SH) in base58 form.'''
Pass the address (either P2PKH or P2SH) in base58 form.
'''
raw = Base58.decode_check(address) raw = Base58.decode_check(address)
# Require version byte plus hash160. # Require version byte plus hash160.
@ -121,7 +129,7 @@ class Coin(object):
@classmethod @classmethod
def prvkey_WIF(privkey_bytes, compressed): def prvkey_WIF(privkey_bytes, compressed):
"The private key encoded in Wallet Import Format" "Return the private key encoded in Wallet Import Format."
payload = bytearray([cls.WIF_BYTE]) + privkey_bytes payload = bytearray([cls.WIF_BYTE]) + privkey_bytes
if compressed: if compressed:
payload.append(0x01) payload.append(0x01)

17
lib/script.py
View File

@ -131,20 +131,20 @@ class ScriptPubKey(object):
TO_P2SH_OPS = [OpCodes.OP_HASH160, -1, OpCodes.OP_EQUAL] TO_P2SH_OPS = [OpCodes.OP_HASH160, -1, OpCodes.OP_EQUAL]
TO_PUBKEY_OPS = [-1, OpCodes.OP_CHECKSIG] TO_PUBKEY_OPS = [-1, OpCodes.OP_CHECKSIG]
def __init__(self, script, coin, kind, hash160, pubkey=None): def __init__(self, script, coin, kind, hash168, pubkey=None):
self.script = script self.script = script
self.coin = coin self.coin = coin
self.kind = kind self.kind = kind
self.hash160 = hash160 self.hash168 = hash168
if pubkey: if pubkey:
self.pubkey = pubkey self.pubkey = pubkey
@cachedproperty @cachedproperty
def address(self): def address(self):
if self.kind == ScriptPubKey.TO_P2SH: if self.kind == ScriptPubKey.TO_P2SH:
return self.coin.P2SH_address_from_hash160(self.hash160) return self.coin.P2SH_address_from_hash160(self.hash168[1:])
if self.hash160: if self.hash160:
return self.coin.P2PKH_address_from_hash160(self.hash160) return self.coin.P2PKH_address_from_hash160(self.hash168[1:])
return '' return ''
@classmethod @classmethod
@ -163,14 +163,17 @@ class ScriptPubKey(object):
ops, datas = Script.get_ops(script) ops, datas = Script.get_ops(script)
if Script.match_ops(ops, cls.TO_ADDRESS_OPS): if Script.match_ops(ops, cls.TO_ADDRESS_OPS):
return cls(script, coin, cls.TO_ADDRESS, datas[2]) return cls(script, coin, cls.TO_ADDRESS,
bytes([coin.P2PKH_VERBYTE]) + datas[2])
if Script.match_ops(ops, cls.TO_P2SH_OPS): if Script.match_ops(ops, cls.TO_P2SH_OPS):
return cls(script, coin, cls.TO_P2SH, datas[1]) return cls(script, coin, cls.TO_P2SH,
bytes([coin.P2SH_VERBYTE]) + datas[1])
if Script.match_ops(ops, cls.TO_PUBKEY_OPS): if Script.match_ops(ops, cls.TO_PUBKEY_OPS):
pubkey = datas[0] pubkey = datas[0]
return cls(script, coin, cls.TO_PUBKEY, hash160(pubkey), pubkey) return cls(script, coin, cls.TO_PUBKEY,
bytes([coin.P2PKH_VERBYTE]) + hash160(pubkey), pubkey)
raise ScriptError('unknown script pubkey pattern') raise ScriptError('unknown script pubkey pattern')

18
lib/util.py
View File

@ -5,24 +5,6 @@
import sys import sys
class Log(object):
'''Logging base class'''
VERBOSE = True
def diagnostic_name(self):
return self.__class__.__name__
def log(self, *msgs):
if Log.VERBOSE:
print('[{}]: '.format(self.diagnostic_name()), *msgs,
file=sys.stdout, flush=True)
def log_error(self, *msg):
print('[{}]: ERROR: {}'.format(self.diagnostic_name()), *msgs,
file=sys.stderr, flush=True)
# Method decorator. To be used for calculations that will always # Method decorator. To be used for calculations that will always
# deliver the same result. The method cannot take any arguments # deliver the same result. The method cannot take any arguments
# and should be accessed as an attribute. # and should be accessed as an attribute.

8
query.py
View File

@ -23,21 +23,21 @@ def main():
limit = 10 limit = 10
for addr in sys.argv[argc:]: for addr in sys.argv[argc:]:
print('Address: ', addr) print('Address: ', addr)
hash160 = coin.address_to_hash160(addr) hash168 = coin.address_to_hash168(addr)
n = None n = None
for n, (tx_hash, height) in enumerate(db.get_history(hash160, limit)): for n, (tx_hash, height) in enumerate(db.get_history(hash168, limit)):
print('History #{:d}: hash: {} height: {:d}' print('History #{:d}: hash: {} height: {:d}'
.format(n + 1, bytes(reversed(tx_hash)).hex(), height)) .format(n + 1, bytes(reversed(tx_hash)).hex(), height))
if n is None: if n is None:
print('No history') print('No history')
n = None n = None
for n, utxo in enumerate(db.get_utxos(hash160, limit)): for n, utxo in enumerate(db.get_utxos(hash168, limit)):
print('UTXOs #{:d}: hash: {} pos: {:d} height: {:d} value: {:d}' print('UTXOs #{:d}: hash: {} pos: {:d} height: {:d} value: {:d}'
.format(n, bytes(reversed(utxo.tx_hash)).hex(), .format(n, bytes(reversed(utxo.tx_hash)).hex(),
utxo.tx_pos, utxo.height, utxo.value)) utxo.tx_pos, utxo.height, utxo.value))
if n is None: if n is None:
print('No UTXOs') print('No UTXOs')
balance = db.get_balance(hash160) balance = db.get_balance(hash168)
print('Balance: {} {}'.format(coin.decimal_value(balance), print('Balance: {} {}'.format(coin.decimal_value(balance),
coin.SHORTNAME)) coin.SHORTNAME))

76
server/db.py
View File

@ -256,56 +256,56 @@ class DB(object):
self.history.pop(None, None) self.history.pop(None, None)
flush_id = struct.pack('>H', self.flush_count) flush_id = struct.pack('>H', self.flush_count)
for hash160, hist in self.history.items(): for hash168, hist in self.history.items():
key = b'H' + hash160 + flush_id key = b'H' + hash168 + flush_id
batch.put(key, array.array('I', hist).tobytes()) batch.put(key, array.array('I', hist).tobytes())
self.history = defaultdict(list) self.history = defaultdict(list)
def get_hash160(self, tx_hash, idx, delete=True): def get_hash168(self, tx_hash, idx, delete=True):
key = b'h' + tx_hash[:ADDR_TX_HASH_LEN] + struct.pack('<H', idx) key = b'h' + tx_hash[:ADDR_TX_HASH_LEN] + struct.pack('<H', idx)
data = self.get(key) data = self.get(key)
if data is None: if data is None:
return None return None
if len(data) == 24: if len(data) == 25:
if delete: if delete:
self.delete(key) self.delete(key)
return data[:20] return data[:21]
assert len(data) % 24 == 0 assert len(data) % 25 == 0
self.hcolls += 1 self.hcolls += 1
if self.hcolls % 1000 == 0: if self.hcolls % 1000 == 0:
self.logger.info('{} total hash160 compressed key collisions' self.logger.info('{} total hash168 compressed key collisions'
.format(self.hcolls)) .format(self.hcolls))
for n in range(0, len(data), 24): for n in range(0, len(data), 25):
(tx_num, ) = struct.unpack('<I', data[n+20:n+24]) (tx_num, ) = struct.unpack('<I', data[n+21 : n+25])
my_hash, height = self.get_tx_hash(tx_num) my_hash, height = self.get_tx_hash(tx_num)
if my_hash == tx_hash: if my_hash == tx_hash:
if delete: if delete:
self.put(key, data[:n] + data[n + 24:]) self.put(key, data[:n] + data[n+25:])
return data[n:n+20] return data[n : n+21]
else:
raise Exception('could not resolve hash160 collision') raise Exception('could not resolve hash168 collision')
def spend_utxo(self, prevout): def spend_utxo(self, prevout):
hash160 = self.get_hash160(prevout.hash, prevout.n) hash168 = self.get_hash168(prevout.hash, prevout.n)
if hash160 is None: if hash168 is None:
# This indicates a successful spend of a non-standard script # This indicates a successful spend of a non-standard script
# self.logger.info('ignoring spend of non-standard UTXO {}/{:d} ' # self.logger.info('ignoring spend of non-standard UTXO {}/{:d} '
# 'at height {:d}' # 'at height {:d}'
# .format(bytes(reversed(prevout.hash)).hex(), # .format(bytes(reversed(prevout.hash)).hex(),
# prevout.n, self.height)) # prevout.n, self.height))
return None return None
key = (b'u' + hash168 + prevout.hash[:UTXO_TX_HASH_LEN]
key = (b'u' + hash160 + prevout.hash[:UTXO_TX_HASH_LEN]
+ struct.pack('<H', prevout.n)) + struct.pack('<H', prevout.n))
data = self.get(key) data = self.get(key)
if data is None: if data is None:
# Uh-oh, this should not happen. It may be recoverable...
self.logger.error('found no UTXO for {} / {:d} key {}' self.logger.error('found no UTXO for {} / {:d} key {}'
.format(bytes(reversed(prevout.hash)).hex(), .format(bytes(reversed(prevout.hash)).hex(),
prevout.n, bytes(key).hex())) prevout.n, bytes(key).hex()))
return hash160 return hash168
if len(data) == 12: if len(data) == 12:
(tx_num, ) = struct.unpack('<I', data[:4]) (tx_num, ) = struct.unpack('<I', data[:4])
@ -324,35 +324,35 @@ class DB(object):
data = data[:n] + data[n + 12:] data = data[:n] + data[n + 12:]
self.put(key, data) self.put(key, data)
return hash160 return hash168
def put_utxo(self, tx_hash, idx, txout): def put_utxo(self, tx_hash, idx, txout):
pk = ScriptPubKey.from_script(txout.pk_script, self.coin) pk = ScriptPubKey.from_script(txout.pk_script, self.coin)
if not pk.hash160: if not pk.hash168:
return None return None
pack = struct.pack pack = struct.pack
idxb = pack('<H', idx) idxb = pack('<H', idx)
txcb = pack('<I', self.tx_count) txcb = pack('<I', self.tx_count)
# First write the hash160 lookup # First write the hash168 lookup
key = b'h' + tx_hash[:ADDR_TX_HASH_LEN] + idxb key = b'h' + tx_hash[:ADDR_TX_HASH_LEN] + idxb
# b'' avoids this annoyance: https://bugs.python.org/issue13298 # b''.join avoids this: https://bugs.python.org/issue13298
value = b''.join([pk.hash160, txcb]) value = b''.join((pk.hash168, txcb))
prior_value = self.get(key) prior_value = self.get(key)
if prior_value: # Should almost never happen if prior_value: # Should almost never happen
value += prior_value value += prior_value
self.put(key, value) self.put(key, value)
# Next write the UTXO # Next write the UTXO
key = b'u' + pk.hash160 + tx_hash[:UTXO_TX_HASH_LEN] + idxb key = b'u' + pk.hash168 + tx_hash[:UTXO_TX_HASH_LEN] + idxb
value = txcb + pack('<Q', txout.value) value = txcb + pack('<Q', txout.value)
prior_value = self.get(key) prior_value = self.get(key)
if prior_value: # Should almost never happen if prior_value: # Should almost never happen
value += prior_value value += prior_value
self.put(key, value) self.put(key, value)
return pk.hash160 return pk.hash168
def open_file(self, filename, truncate=False, create=False): def open_file(self, filename, truncate=False, create=False):
try: try:
@ -420,16 +420,16 @@ class DB(object):
self.flush() self.flush()
def process_tx(self, tx_hash, tx): def process_tx(self, tx_hash, tx):
hash160s = set() hash168s = set()
if not tx.is_coinbase: if not tx.is_coinbase:
for txin in tx.inputs: for txin in tx.inputs:
hash160s.add(self.spend_utxo(txin.prevout)) hash168s.add(self.spend_utxo(txin.prevout))
for idx, txout in enumerate(tx.outputs): for idx, txout in enumerate(tx.outputs):
hash160s.add(self.put_utxo(tx_hash, idx, txout)) hash168s.add(self.put_utxo(tx_hash, idx, txout))
for hash160 in hash160s: for hash168 in hash168s:
self.history[hash160].append(self.tx_count) self.history[hash168].append(self.tx_count)
self.tx_count += 1 self.tx_count += 1
@ -458,7 +458,7 @@ class DB(object):
assert isinstance(limit, int) and limit >= 0 assert isinstance(limit, int) and limit >= 0
return limit return limit
def get_history(self, hash160, limit=1000): def get_history(self, hash168, limit=1000):
'''Generator that returns an unpruned, sorted list of (tx_hash, '''Generator that returns an unpruned, sorted list of (tx_hash,
height) tuples of transactions that touched the address, height) tuples of transactions that touched the address,
earliest in the blockchain first. Includes both spending and earliest in the blockchain first. Includes both spending and
@ -466,7 +466,7 @@ class DB(object):
Set limit to None to get them all. Set limit to None to get them all.
''' '''
limit = self.resolve_limit(limit) limit = self.resolve_limit(limit)
prefix = b'H' + hash160 prefix = b'H' + hash168
for key, hist in self.db.iterator(prefix=prefix): for key, hist in self.db.iterator(prefix=prefix):
a = array.array('I') a = array.array('I')
a.frombytes(hist) a.frombytes(hist)
@ -476,18 +476,18 @@ class DB(object):
yield self.get_tx_hash(tx_num) yield self.get_tx_hash(tx_num)
limit -= 1 limit -= 1
def get_balance(self, hash160): def get_balance(self, hash168):
'''Returns the confirmed balance of an address.''' '''Returns the confirmed balance of an address.'''
return sum(utxo.value for utxo in self.get_utxos(hash160, limit=None)) return sum(utxo.value for utxo in self.get_utxos(hash168, limit=None))
def get_utxos(self, hash160, limit=1000): def get_utxos(self, hash168, limit=1000):
'''Generator that yields all UTXOs for an address sorted in no '''Generator that yields all UTXOs for an address sorted in no
particular order. By default yields at most 1000 entries. particular order. By default yields at most 1000 entries.
Set limit to None to get them all. Set limit to None to get them all.
''' '''
limit = self.resolve_limit(limit) limit = self.resolve_limit(limit)
unpack = struct.unpack unpack = struct.unpack
prefix = b'u' + hash160 prefix = b'u' + hash168
utxos = [] utxos = []
for k, v in self.db.iterator(prefix=prefix): for k, v in self.db.iterator(prefix=prefix):
(tx_pos, ) = unpack('<H', k[-2:]) (tx_pos, ) = unpack('<H', k[-2:])
@ -501,7 +501,7 @@ class DB(object):
yield UTXO(tx_num, tx_pos, tx_hash, height, value) yield UTXO(tx_num, tx_pos, tx_hash, height, value)
limit -= 1 limit -= 1
def get_utxos_sorted(self, hash160): def get_utxos_sorted(self, hash168):
'''Returns all the UTXOs for an address sorted by height and '''Returns all the UTXOs for an address sorted by height and
position in the block.''' position in the block.'''
return sorted(self.get_utxos(hash160, limit=None)) return sorted(self.get_utxos(hash168, limit=None))

Write Preview
Loading…
Cancel
Save