Merge pull request #572 from erasmospunk/struct-refactoring

Struct refactoring

electrumx/lib/coins.py

@@ -70,6 +70,9 @@ class Coin(object):
     DESERIALIZER = lib_tx.Deserializer
     DAEMON = daemon.Daemon
     BLOCK_PROCESSOR = block_proc.BlockProcessor
+    HEADER_VALUES = ('version', 'prev_block_hash', 'merkle_root', 'timestamp',
+                     'bits', 'nonce')
+    HEADER_UNPACK = struct.Struct('< I 32s 32s I I I').unpack_from
     MEMPOOL_HISTOGRAM_REFRESH_SECS = 500
     XPUB_VERBYTES = bytes('????', 'utf-8')
     XPRV_VERBYTES = bytes('????', 'utf-8')
@@ -296,18 +299,13 @@ class Coin(object):
 
     @classmethod
     def electrum_header(cls, header, height):
-        version, = struct.unpack('<I', header[:4])
-        timestamp, bits, nonce = struct.unpack('<III', header[68:80])
-
-        return {
-            'block_height': height,
-            'version': version,
-            'prev_block_hash': hash_to_hex_str(header[4:36]),
-            'merkle_root': hash_to_hex_str(header[36:68]),
-            'timestamp': timestamp,
-            'bits': bits,
-            'nonce': nonce,
-        }
+        h = dict(zip(cls.HEADER_VALUES, cls.HEADER_UNPACK(header)))
+        # Add the height that is not present in the header itself
+        h['block_height'] = height
+        # Convert bytes to str
+        h['prev_block_hash'] = hash_to_hex_str(h['prev_block_hash'])
+        h['merkle_root'] = hash_to_hex_str(h['merkle_root'])
+        return h
 
 
 class AuxPowMixin(object):
@@ -330,21 +328,21 @@ class EquihashMixin(object):
     STATIC_BLOCK_HEADERS = False
     BASIC_HEADER_SIZE = 140  # Excluding Equihash solution
     DESERIALIZER = lib_tx.DeserializerEquihash
+    HEADER_VALUES = ('version', 'prev_block_hash', 'merkle_root', 'reserved',
+                     'timestamp', 'bits', 'nonce')
+    HEADER_UNPACK = struct.Struct('< I 32s 32s 32s I I 32s').unpack_from
 
     @classmethod
     def electrum_header(cls, header, height):
-        version, = struct.unpack('<I', header[:4])
-        timestamp, bits = struct.unpack('<II', header[100:108])
-
-        return {
-            'block_height': height,
-            'version': version,
-            'prev_block_hash': hash_to_hex_str(header[4:36]),
-            'merkle_root': hash_to_hex_str(header[36:68]),
-            'timestamp': timestamp,
-            'bits': bits,
-            'nonce': hash_to_hex_str(header[108:140]),
-        }
+        h = dict(zip(cls.HEADER_VALUES, cls.HEADER_UNPACK(header)))
+        # Add the height that is not present in the header itself
+        h['block_height'] = height
+        # Convert bytes to str
+        h['prev_block_hash'] = hash_to_hex_str(h['prev_block_hash'])
+        h['merkle_root'] = hash_to_hex_str(h['merkle_root'])
+        h['reserved'] = hash_to_hex_str(h['reserved'])
+        h['nonce'] = hash_to_hex_str(h['nonce'])
+        return h
 
     @classmethod
     def block_header(cls, block, height):
@@ -365,7 +363,7 @@ class ScryptMixin(object):
             import scrypt
             cls.HEADER_HASH = lambda x: scrypt.hash(x, x, 1024, 1, 1, 32)
 
-        version, = struct.unpack('<I', header[:4])
+        version, = util.unpack_le_uint32_from(header)
         if version > 6:
             return super().header_hash(header)
         else:
@@ -475,8 +473,7 @@ class BitcoinGold(EquihashMixin, BitcoinMixin, Coin):
     @classmethod
     def header_hash(cls, header):
         '''Given a header return hash'''
-        height, = struct.unpack('<I', header[68:72])
-
+        height, = util.unpack_le_uint32_from(header, 68)
         if height >= cls.FORK_HEIGHT:
             return double_sha256(header)
         else:
@@ -484,18 +481,9 @@ class BitcoinGold(EquihashMixin, BitcoinMixin, Coin):
 
     @classmethod
     def electrum_header(cls, header, height):
-        h = dict(
-            block_height=height,
-            version=struct.unpack('<I', header[:4])[0],
-            prev_block_hash=hash_to_hex_str(header[4:36]),
-            merkle_root=hash_to_hex_str(header[36:68]),
-            timestamp=struct.unpack('<I', header[100:104])[0],
-            reserved=hash_to_hex_str(header[72:100]),
-            bits=struct.unpack('<I', header[104:108])[0],
-            nonce=hash_to_hex_str(header[108:140]),
-            solution=hash_to_hex_str(header[140:])
-        )
-
+        h = super().electrum_header(header, height)
+        h['reserved'] = hash_to_hex_str(header[72:100])
+        h['solution'] = hash_to_hex_str(header[140:])
         return h
 
 
@@ -948,6 +936,9 @@ class FairCoin(Coin):
     GENESIS_HASH = ('beed44fa5e96150d95d56ebd5d262578'
                     '1825a9407a5215dd7eda723373a0a1d7')
     BASIC_HEADER_SIZE = 108
+    HEADER_VALUES = ('version', 'prev_block_hash', 'merkle_root',
+                     'payload_hash', 'timestamp', 'creatorId')
+    HEADER_UNPACK = struct.Struct('< I 32s 32s 32s I I').unpack_from
     TX_COUNT = 505
     TX_COUNT_HEIGHT = 470
     TX_PER_BLOCK = 1
@@ -968,17 +959,9 @@ class FairCoin(Coin):
 
     @classmethod
     def electrum_header(cls, header, height):
-        version, = struct.unpack('<I', header[:4])
-        timestamp, creatorId = struct.unpack('<II', header[100:108])
-        return {
-            'block_height': height,
-            'version': version,
-            'prev_block_hash': hash_to_hex_str(header[4:36]),
-            'merkle_root': hash_to_hex_str(header[36:68]),
-            'payload_hash': hash_to_hex_str(header[68:100]),
-            'timestamp': timestamp,
-            'creatorId': creatorId,
-        }
+        h = super().electrum_header(header, height)
+        h['payload_hash'] = hash_to_hex_str(h['payload_hash'])
+        return h
 
 
 class Zcash(EquihashMixin, Coin):
@@ -1031,21 +1014,10 @@ class SnowGem(EquihashMixin, Coin):
 
     @classmethod
     def electrum_header(cls, header, height):
-        version, = struct.unpack('<I', header[:4])
-        timestamp, bits = struct.unpack('<II', header[100:108])
-
-        return {
-            'block_height': height,
-            'version': version,
-            'prev_block_hash': hash_to_hex_str(header[4:36]),
-            'merkle_root': hash_to_hex_str(header[36:68]),
-            'hash_reserved': hash_to_hex_str(header[68:100]),
-            'timestamp': timestamp,
-            'bits': bits,
-            'nonce': hash_to_hex_str(header[108:140]),
-            'n_solution': base64.b64encode(lib_tx.Deserializer(
-                header, start=140)._read_varbytes()).decode('utf8')
-        }
+        h = super().electrum_header(header, height)
+        h['n_solution'] = base64.b64encode(lib_tx.Deserializer(
+            header, start=140)._read_varbytes()).decode('utf8')
+        return h
 
 
 class BitcoinZ(EquihashMixin, Coin):
@@ -1670,7 +1642,7 @@ class BitcoinAtom(Coin):
         header_to_be_hashed = header[:cls.BASIC_HEADER_SIZE]
         # New block header format has some extra flags in the end
         if len(header) == cls.HEADER_SIZE_POST_FORK:
-            flags, = struct.unpack('<I', header[-4:])
+            flags, = util.unpack_le_uint32_from(header, len(header) - 4)
             # Proof of work blocks have special serialization
             if flags & cls.BLOCK_PROOF_OF_STAKE != 0:
                 header_to_be_hashed += cls.BLOCK_PROOF_OF_STAKE_FLAGS
@@ -1704,7 +1676,13 @@ class Decred(Coin):
                            hash_fn=lib_tx.DeserializerDecred.blake256d)
     DECODE_CHECK = partial(Base58.decode_check,
                            hash_fn=lib_tx.DeserializerDecred.blake256d)
-    HEADER_UNPACK = struct.Struct('<i32s32s32sH6sHBBIIQIIII32sI').unpack_from
+    HEADER_VALUES = ('version', 'prev_block_hash', 'merkle_root', 'stake_root',
+                     'vote_bits', 'final_state', 'voters', 'fresh_stake',
+                     'revocations', 'pool_size', 'bits', 'sbits',
+                     'block_height', 'size', 'timestamp', 'nonce',
+                     'extra_data', 'stake_version')
+    HEADER_UNPACK = struct.Struct(
+        '< i 32s 32s 32s H 6s H B B I I Q I I I I 32s I').unpack_from
     TX_COUNT = 4629388
     TX_COUNT_HEIGHT = 260628
     TX_PER_BLOCK = 17
@@ -1726,20 +1704,10 @@ class Decred(Coin):
 
     @classmethod
     def electrum_header(cls, header, height):
-        labels = ('version', 'prev_block_hash', 'merkle_root', 'stake_root',
-                  'vote_bits', 'final_state', 'voters', 'fresh_stake',
-                  'revocations', 'pool_size', 'bits', 'sbits', 'block_height',
-                  'size', 'timestamp', 'nonce', 'extra_data', 'stake_version')
-        values = cls.HEADER_UNPACK(header)
-        h = dict(zip(labels, values))
-
-        # Convert some values
-        assert h['block_height'] == height
-        h['prev_block_hash'] = hash_to_hex_str(h['prev_block_hash'])
-        h['merkle_root'] = hash_to_hex_str(h['merkle_root'])
+        h = super().electrum_header(header, height)
         h['stake_root'] = hash_to_hex_str(h['stake_root'])
-        h['final_state'] = h['final_state'].hex()
-        h['extra_data'] = h['extra_data'].hex()
+        h['final_state'] = hash_to_hex_str(h['final_state'])
+        h['extra_data'] = hash_to_hex_str(h['extra_data'])
         return h
 
 
@@ -1799,7 +1767,7 @@ class Xuez(Coin):
         Need to download `xevan_hash` module
         Source code: https://github.com/xuez/xuez
         '''
-        version, = struct.unpack('<I', header[:4])
+        version, = util.unpack_le_uint32_from(header)
 
         import xevan_hash
 
@@ -1810,29 +1778,10 @@ class Xuez(Coin):
 
     @classmethod
     def electrum_header(cls, header, height):
-        version, = struct.unpack('<I', header[:4])
-        timestamp, bits, nonce = struct.unpack('<III', header[68:80])
-        if version == 1:
-            return {
-                'block_height': height,
-                'version': version,
-                'prev_block_hash': hash_to_hex_str(header[4:36]),
-                'merkle_root': hash_to_hex_str(header[36:68]),
-                'timestamp': timestamp,
-                'bits': bits,
-                'nonce': nonce,
-            }
-        else:
-            return {
-                'block_height': height,
-                'version': version,
-                'prev_block_hash': hash_to_hex_str(header[4:36]),
-                'merkle_root': hash_to_hex_str(header[36:68]),
-                'timestamp': timestamp,
-                'bits': bits,
-                'nonce': nonce,
-                'nAccumulatorCheckpoint': hash_to_hex_str(header[80:112]),
-            }
+        h = super().electrum_header(header, height)
+        if h['version'] > 1:
+            h['nAccumulatorCheckpoint'] = hash_to_hex_str(header[80:])
+        return h
 
 
 class Pac(Coin):
@@ -1998,35 +1947,7 @@ class Monoeci(Coin):
                 return x11_hash.getPoWHash(header)
 
 
-class MinexcoinMixin(object):
-    STATIC_BLOCK_HEADERS = True
-    BASIC_HEADER_SIZE = 209
-    DESERIALIZER = lib_tx.DeserializerEquihash
-
-    @classmethod
-    def electrum_header(cls, header, height):
-        version, = struct.unpack('<I', header[:4])
-        timestamp, bits = struct.unpack('<II', header[100:108])
-
-        return {
-            'block_height': height,
-            'version': version,
-            'prev_block_hash': hash_to_hex_str(header[4:36]),
-            'merkle_root': hash_to_hex_str(header[36:68]),
-            'timestamp': timestamp,
-            'bits': bits,
-            'nonce': hash_to_hex_str(header[108:140]),
-            'solution': hash_to_hex_str(header[140:209]),
-        }
-
-    @classmethod
-    def block_header(cls, block, height):
-        '''Return the block header bytes'''
-        deserializer = cls.DESERIALIZER(block)
-        return deserializer.read_header(height, 140)
-
-
-class Minexcoin(MinexcoinMixin, Coin):
+class Minexcoin(EquihashMixin, Coin):
     NAME = "Minexcoin"
     SHORTNAME = "MNX"
     NET = "mainnet"
@@ -2035,7 +1956,9 @@ class Minexcoin(MinexcoinMixin, Coin):
     WIF_BYTE = bytes.fromhex("80")
     GENESIS_HASH = ('490a36d9451a55ed197e34aca7414b35'
                     'd775baa4a8e896f1c577f65ce2d214cb')
-    DESERIALIZER = lib_tx.DeserializerEquihash
+    STATIC_BLOCK_HEADERS = True
+    BASIC_HEADER_SIZE = 209
+    HEADER_SIZE_NO_SOLUTION = 140
     TX_COUNT = 327963
     TX_COUNT_HEIGHT = 74495
     TX_PER_BLOCK = 5
@@ -2046,6 +1969,18 @@ class Minexcoin(MinexcoinMixin, Coin):
         'eu.minexpool.nl s t'
     ]
 
+    @classmethod
+    def electrum_header(cls, header, height):
+        h = super().electrum_header(header, height)
+        h['solution'] = hash_to_hex_str(header[cls.HEADER_SIZE_NO_SOLUTION:])
+        return h
+
+    @classmethod
+    def block_header(cls, block, height):
+        '''Return the block header bytes'''
+        deserializer = cls.DESERIALIZER(block)
+        return deserializer.read_header(height, cls.HEADER_SIZE_NO_SOLUTION)
+
 
 class Groestlcoin(Coin):
     NAME = "Groestlcoin"
@@ -2133,7 +2068,7 @@ class Pivx(Coin):
 
     @classmethod
     def header_hash(cls, header):
-        version, = struct.unpack('<I', header[:4])
+        version, = util.unpack_le_uint32_from(header)
         if version >= 4:
             return super().header_hash(header)
         else:

electrumx/lib/script.py

@@ -32,6 +32,8 @@ from collections import namedtuple
 
 from electrumx.lib.enum import Enumeration
 from electrumx.lib.hash import hash160
+from electrumx.lib.util import unpack_le_uint16_from, unpack_le_uint32_from, \
+    pack_le_uint16, pack_le_uint32
 
 
 class ScriptError(Exception):
@@ -196,10 +198,10 @@ class Script(object):
                         dlen = script[n]
                         n += 1
                     elif op == OpCodes.OP_PUSHDATA2:
-                        dlen, = struct.unpack('<H', script[n: n + 2])
+                        dlen, = unpack_le_uint16_from(script[n: n + 2])
                         n += 2
                     else:
-                        dlen, = struct.unpack('<I', script[n: n + 4])
+                        dlen, = unpack_le_uint32_from(script[n: n + 4])
                         n += 4
                     if n + dlen > len(script):
                         raise IndexError
@@ -225,8 +227,8 @@ class Script(object):
         if n < 256:
             return bytes([OpCodes.OP_PUSHDATA1, n]) + data
         if n < 65536:
-            return bytes([OpCodes.OP_PUSHDATA2]) + struct.pack('<H', n) + data
-        return bytes([OpCodes.OP_PUSHDATA4]) + struct.pack('<I', n) + data
+            return bytes([OpCodes.OP_PUSHDATA2]) + pack_le_uint16(n) + data
+        return bytes([OpCodes.OP_PUSHDATA4]) + pack_le_uint32(n) + data
 
     @classmethod
     def opcode_name(cls, opcode):

electrumx/lib/tx.py

@@ -31,8 +31,8 @@ from collections import namedtuple
 
 from electrumx.lib.hash import sha256, double_sha256, hash_to_hex_str
 from electrumx.lib.util import (
-    cachedproperty, unpack_int32_from, unpack_int64_from,
-    unpack_uint16_from, unpack_uint32_from, unpack_uint64_from,
+    cachedproperty, unpack_le_int32_from, unpack_le_int64_from,
+    unpack_le_uint16_from, unpack_le_uint32_from, unpack_le_uint64_from,
     pack_le_int32, pack_varint, pack_le_uint32, pack_le_uint32, pack_le_int64,
     pack_varbytes,
 )
@@ -185,27 +185,27 @@ class Deserializer(object):
         return self._read_le_uint64()
 
     def _read_le_int32(self):
-        result, = unpack_int32_from(self.binary, self.cursor)
+        result, = unpack_le_int32_from(self.binary, self.cursor)
         self.cursor += 4
         return result
 
     def _read_le_int64(self):
-        result, = unpack_int64_from(self.binary, self.cursor)
+        result, = unpack_le_int64_from(self.binary, self.cursor)
         self.cursor += 8
         return result
 
     def _read_le_uint16(self):
-        result, = unpack_uint16_from(self.binary, self.cursor)
+        result, = unpack_le_uint16_from(self.binary, self.cursor)
         self.cursor += 2
         return result
 
     def _read_le_uint32(self):
-        result, = unpack_uint32_from(self.binary, self.cursor)
+        result, = unpack_le_uint32_from(self.binary, self.cursor)
         self.cursor += 4
         return result
 
     def _read_le_uint64(self):
-        result, = unpack_uint64_from(self.binary, self.cursor)
+        result, = unpack_le_uint64_from(self.binary, self.cursor)
         self.cursor += 8
         return result
 

electrumx/lib/util.py

@@ -168,20 +168,6 @@ def int_to_bytes(value):
     return value.to_bytes((value.bit_length() + 7) // 8, 'big')
 
 
-def int_to_varint(value):
-    '''Converts an integer to a Bitcoin-like varint bytes'''
-    if value < 0:
-        raise ValueError("attempt to write size < 0")
-    elif value < 253:
-        return pack('<B', value)
-    elif value < 2**16:
-        return b'\xfd' + pack('<H', value)
-    elif value < 2**32:
-        return b'\xfe' + pack('<I', value)
-    elif value < 2**64:
-        return b'\xff' + pack('<Q', value)
-
-
 def increment_byte_string(bs):
     '''Return the lexicographically next byte string of the same length.
 
@@ -330,24 +316,30 @@ def protocol_version(client_req, min_tuple, max_tuple):
     return result, client_min
 
 
-structi = Struct('<i')
-structq = Struct('<q')
-structH = Struct('<H')
-structI = Struct('<I')
-structQ = Struct('<Q')
+struct_le_i = Struct('<i')
+struct_le_q = Struct('<q')
+struct_le_H = Struct('<H')
+struct_le_I = Struct('<I')
+struct_le_Q = Struct('<Q')
+struct_be_H = Struct('>H')
+struct_be_I = Struct('>I')
 structB = Struct('B')
 
-unpack_int32_from = structi.unpack_from
-unpack_int64_from = structq.unpack_from
-unpack_uint16_from = structH.unpack_from
-unpack_uint32_from = structI.unpack_from
-unpack_uint64_from = structQ.unpack_from
-
-pack_le_int32 = structi.pack
-pack_le_int64 = structq.pack
-pack_le_uint16 = structH.pack
-pack_le_uint32 = structI.pack
-pack_le_uint64 = structQ.pack
+unpack_le_int32_from = struct_le_i.unpack_from
+unpack_le_int64_from = struct_le_q.unpack_from
+unpack_le_uint16_from = struct_le_H.unpack_from
+unpack_le_uint32_from = struct_le_I.unpack_from
+unpack_le_uint64_from = struct_le_Q.unpack_from
+unpack_be_uint16_from = struct_be_H.unpack_from
+unpack_be_uint32_from = struct_be_I.unpack_from
+
+pack_le_int32 = struct_le_i.pack
+pack_le_int64 = struct_le_q.pack
+pack_le_uint16 = struct_le_H.pack
+pack_le_uint32 = struct_le_I.pack
+pack_le_uint64 = struct_le_Q.pack
+pack_be_uint16 = struct_be_H.pack
+pack_be_uint32 = struct_be_I.pack
 pack_byte = structB.pack
 
 hex_to_bytes = bytes.fromhex

electrumx/server/daemon.py

@@ -17,8 +17,8 @@ from time import strptime
 
 import aiohttp
 
-from electrumx.lib.util import int_to_varint, hex_to_bytes, class_logger, \
-    unpack_uint16_from
+from electrumx.lib.util import hex_to_bytes, class_logger,\
+    unpack_le_uint16_from, pack_varint
 from electrumx.lib.hash import hex_str_to_hash, hash_to_hex_str
 from electrumx.lib.tx import DeserializerDecred
 from aiorpcx import JSONRPC
@@ -356,7 +356,7 @@ class LegacyRPCDaemon(Daemon):
         raw_block = header
         num_txs = len(transactions)
         if num_txs > 0:
-            raw_block += int_to_varint(num_txs)
+            raw_block += pack_varint(num_txs)
             raw_block += b''.join(transactions)
         else:
             raw_block += b'\x00'
@@ -384,7 +384,7 @@ class DecredDaemon(Daemon):
             raw_blocks.append(raw_block)
             # Check if previous block is valid
             prev = self.prev_hex_hash(raw_block)
-            votebits = unpack_uint16_from(raw_block[100:102])[0]
+            votebits = unpack_le_uint16_from(raw_block[100:102])[0]
             valid_tx_tree[prev] = self.is_valid_tx_tree(votebits)
 
         processed_raw_blocks = []

electrumx/server/history.py

@@ -14,9 +14,9 @@ import bisect
 import time
 from collections import defaultdict
 from functools import partial
-from struct import pack, unpack
 
 import electrumx.lib.util as util
+from electrumx.lib.util import pack_be_uint16, unpack_be_uint16_from
 from electrumx.lib.hash import hash_to_hex_str, HASHX_LEN
 
 
@@ -81,7 +81,7 @@ class History(object):
 
         keys = []
         for key, hist in self.db.iterator(prefix=b''):
-            flush_id, = unpack('>H', key[-2:])
+            flush_id, = unpack_be_uint16_from(key[-2:])
             if flush_id > utxo_flush_count:
                 keys.append(key)
 
@@ -126,7 +126,7 @@ class History(object):
     def flush(self):
         start_time = time.time()
         self.flush_count += 1
-        flush_id = pack('>H', self.flush_count)
+        flush_id = pack_be_uint16(self.flush_count)
         unflushed = self.unflushed
 
         with self.db.write_batch() as batch:
@@ -250,7 +250,7 @@ class History(object):
         write_size = 0
         keys_to_delete.update(hist_map)
         for n, chunk in enumerate(util.chunks(full_hist, max_row_size)):
-            key = hashX + pack('>H', n)
+            key = hashX + pack_be_uint16(n)
             if hist_map.get(key) == chunk:
                 keys_to_delete.remove(key)
             else:
@@ -302,7 +302,7 @@ class History(object):
         # Loop over 2-byte prefixes
         cursor = self.comp_cursor
         while write_size < limit and cursor < 65536:
-            prefix = pack('>H', cursor)
+            prefix = pack_be_uint16(cursor)
             write_size += self._compact_prefix(prefix, write_items,
                                                keys_to_delete)
             cursor += 1

electrumx/wallet/bip32.py

@@ -15,7 +15,8 @@ import ecdsa.numbertheory as NT
 
 from electrumx.lib.coins import Coin
 from electrumx.lib.hash import Base58, hmac_sha512, hash160
-from electrumx.lib.util import cachedproperty, bytes_to_int, int_to_bytes
+from electrumx.lib.util import cachedproperty, bytes_to_int, int_to_bytes, \
+    pack_be_uint32, unpack_be_uint32_from
 
 
 class DerivationError(Exception):
@@ -65,7 +66,7 @@ class _KeyBase(object):
             raise ValueError('raw_serkey must have length 33')
 
         return (ver_bytes + bytes([self.depth])
-                + self.parent_fingerprint() + struct.pack('>I', self.n)
+                + self.parent_fingerprint() + pack_be_uint32(self.n)
                 + self.chain_code + raw_serkey)
 
     def fingerprint(self):
@@ -142,7 +143,7 @@ class PubKey(_KeyBase):
         if not 0 <= n < (1 << 31):
             raise ValueError('invalid BIP32 public key child number')
 
-        msg = self.pubkey_bytes + struct.pack('>I', n)
+        msg = self.pubkey_bytes + pack_be_uint32(n)
         L, R = self._hmac_sha512(msg)
 
         curve = self.CURVE
@@ -244,7 +245,7 @@ class PrivKey(_KeyBase):
         else:
             serkey = self.public_key.pubkey_bytes
 
-        msg = serkey + struct.pack('>I', n)
+        msg = serkey + pack_be_uint32(n)
         L, R = self._hmac_sha512(msg)
 
         curve = self.CURVE
@@ -282,7 +283,7 @@ def _from_extended_key(ekey):
     is_public, coin = Coin.lookup_xverbytes(ekey[:4])
     depth = ekey[4]
     fingerprint = ekey[5:9]   # Not used
-    n, = struct.unpack('>I', ekey[9:13])
+    n, = unpack_be_uint32_from(ekey[9:13])
     chain_code = ekey[13:45]
 
     if is_public:

tests/blocks/newyorkcoin_mainnet_3956926.json

@@ -7,7 +7,7 @@
     "0300b2fe16c049641ec63f9eb435b9d773c847bc0185c927d5b8d87cd1ad3095"
   ],
   "time": 1515308763,
-  "nonce": "1570457681",
+  "nonce": 1570457681,
   "bits": "1b172a4e",
   "previousblockhash": "10c91aefd6698f03a2820b8aa462738a5c1c21f4f975786a8eb1ba25e4b33af3",
   "block": "01000000f33ab3e425bab18e6a7875f9f4211c5c8a7362a48a0b82a2038f69d6ef1ac9109530add17cd8b8d527c98501bc47c873d7b935b49e3fc61e6449c016feb20003dbc6515a4e2a171b51489b5d0101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff5203be603c062f503253482f04dfc6515afabe6d6dce67985c41a5f1ba5d526614e1744010b5e6a34d33c50ece5d420d75da15454e100000000000000008b8000274020000000c2f30324d515156434c59582f00000000010010a5d4e80000001976a914f0a150ec5709fae1d1814227b69cd1f0baf528c588ac00000000"

tests/lib/test_util.py

@@ -71,25 +71,14 @@ def test_increment_byte_string():
     assert util.increment_byte_string(b'\x01\x01') == b'\x01\x02'
     assert util.increment_byte_string(b'\xff\xff') is None
 
+
 def test_bytes_to_int():
     assert util.bytes_to_int(b'\x07[\xcd\x15') == 123456789
 
+
 def test_int_to_bytes():
     assert util.int_to_bytes(456789) == b'\x06\xf8U'
 
-def test_int_to_varint():
-    with pytest.raises(ValueError):
-        util.int_to_varint(-1)
-    assert util.int_to_varint(0) == b'\0'
-    assert util.int_to_varint(5) == b'\5'
-    assert util.int_to_varint(252) == b'\xfc'
-    assert util.int_to_varint(253) == b'\xfd\xfd\0'
-    assert util.int_to_varint(65535) == b'\xfd\xff\xff'
-    assert util.int_to_varint(65536) == b'\xfe\0\0\1\0'
-    assert util.int_to_varint(2**32-1) == b'\xfe\xff\xff\xff\xff'
-    assert util.int_to_varint(2**32) == b'\xff\0\0\0\0\1\0\0\0'
-    assert util.int_to_varint(2**64-1) \
-        == b'\xff\xff\xff\xff\xff\xff\xff\xff\xff'
 
 def test_LogicalFile(tmpdir):
     prefix = os.path.join(tmpdir, 'log')
@@ -208,17 +197,17 @@ def test_protocol_version():
 
 def test_unpackers():
     b = bytes(range(256))
-    assert util.unpack_int32_from(b, 0) == (50462976,)
-    assert util.unpack_int32_from(b, 42) == (757869354,)
-    assert util.unpack_int64_from(b, 0) == (506097522914230528,)
-    assert util.unpack_int64_from(b, 42) == (3544384782113450794,)
-
-    assert util.unpack_uint16_from(b, 0) == (256,)
-    assert util.unpack_uint16_from(b, 42) == (11050,)
-    assert util.unpack_uint32_from(b, 0) == (50462976,)
-    assert util.unpack_uint32_from(b, 42) == (757869354,)
-    assert util.unpack_uint64_from(b, 0) == (506097522914230528,)
-    assert util.unpack_uint64_from(b, 42) == (3544384782113450794,)
+    assert util.unpack_le_int32_from(b, 0) == (50462976,)
+    assert util.unpack_le_int32_from(b, 42) == (757869354,)
+    assert util.unpack_le_int64_from(b, 0) == (506097522914230528,)
+    assert util.unpack_le_int64_from(b, 42) == (3544384782113450794,)
+
+    assert util.unpack_le_uint16_from(b, 0) == (256,)
+    assert util.unpack_le_uint16_from(b, 42) == (11050,)
+    assert util.unpack_le_uint32_from(b, 0) == (50462976,)
+    assert util.unpack_le_uint32_from(b, 42) == (757869354,)
+    assert util.unpack_le_uint64_from(b, 0) == (506097522914230528,)
+    assert util.unpack_le_uint64_from(b, 42) == (3544384782113450794,)
 
 def test_hex_transforms():
     h = "AABBCCDDEEFF"
@@ -234,6 +223,20 @@ def test_pack_varint():
         deser = tx.Deserializer(data)
         assert deser._read_varint() == n
 
+    import struct
+    with pytest.raises(struct.error):
+        util.pack_varint(-1)
+    assert util.pack_varint(0) == b'\0'
+    assert util.pack_varint(5) == b'\5'
+    assert util.pack_varint(252) == b'\xfc'
+    assert util.pack_varint(253) == b'\xfd\xfd\0'
+    assert util.pack_varint(65535) == b'\xfd\xff\xff'
+    assert util.pack_varint(65536) == b'\xfe\0\0\1\0'
+    assert util.pack_varint(2**32-1) == b'\xfe\xff\xff\xff\xff'
+    assert util.pack_varint(2**32) == b'\xff\0\0\0\0\1\0\0\0'
+    assert util.pack_varint(2**64-1) \
+           == b'\xff\xff\xff\xff\xff\xff\xff\xff\xff'
+
 def test_pack_varbytes():
     tests = [b'', b'1', b'2' * 253, b'3' * 254, b'4' * 256, b'5' * 65536]
 

tests/server/test_compaction.py

@@ -2,12 +2,11 @@
 
 import array
 import asyncio
-from collections import defaultdict
 from os import environ, urandom
-from struct import pack
 import random
 
 from electrumx.lib.hash import HASHX_LEN
+from electrumx.lib.util import pack_be_uint16
 from electrumx.server.env import Env
 from electrumx.server.db import DB
 
@@ -49,7 +48,7 @@ def check_hashX_compaction(history):
     hist_list = []
     hist_map = {}
     for flush_count, count in pairs:
-        key = hashX + pack('>H', flush_count)
+        key = hashX + pack_be_uint16(flush_count)
         hist = full_hist[cum * 4: (cum+count) * 4]
         hist_map[key] = hist
         hist_list.append(hist)
@@ -65,10 +64,10 @@ def check_hashX_compaction(history):
     assert len(keys_to_delete) == 3
     assert len(hist_map) == len(pairs)
     for n, item in enumerate(write_items):
-        assert item == (hashX + pack('>H', n),
+        assert item == (hashX + pack_be_uint16(n),
                         full_hist[n * row_size: (n + 1) * row_size])
     for flush_count, count in pairs:
-        assert hashX + pack('>H', flush_count) in keys_to_delete
+        assert hashX + pack_be_uint16(flush_count) in keys_to_delete
 
     # Check re-compaction is null
     hist_map = {key: value for key, value in write_items}
@@ -87,7 +86,7 @@ def check_hashX_compaction(history):
     write_size = history._compact_hashX(hashX, hist_map, hist_list,
                                         write_items, keys_to_delete)
     assert write_size == len(hist_list[-1])
-    assert write_items == [(hashX + pack('>H', 2), hist_list[-1])]
+    assert write_items == [(hashX + pack_be_uint16(2), hist_list[-1])]
     assert len(keys_to_delete) == 1
     assert write_items[0][0] in keys_to_delete
     assert len(hist_map) == len(pairs)

tests/test_blocks.py

@@ -32,6 +32,7 @@ import pytest
 
 from electrumx.lib.coins import Coin
 from electrumx.lib.hash import hex_str_to_hash
+from electrumx.lib.util import pack_be_uint32
 
 BLOCKS_DIR = os.path.join(
     os.path.dirname(os.path.realpath(__file__)), 'blocks')
@@ -60,6 +61,13 @@ def test_block(block_details):
 
     raw_block = unhexlify(block_info['block'])
     block = coin.block(raw_block, block_info['height'])
+    h = coin.electrum_header(block.header, block_info['height'])
+    assert block_info['merkleroot'] == h['merkle_root']
+    assert block_info['time'] == h['timestamp']
+    assert block_info['previousblockhash'] == h['prev_block_hash']
+    assert block_info['height'] == h['block_height']
+    assert block_info['nonce'] == h['nonce']
+    assert block_info['bits'] == pack_be_uint32(h['bits']).hex()
 
     assert coin.header_hash(block.header) == hex_str_to_hash(block_info['hash'])
     assert (coin.header_prevhash(block.header)