Browse Source
Still to do: running compression in background when the flush count reaches a certain levelmaster
Neil Booth
8 years ago
4 changed files with 390 additions and 4 deletions
@ -0,0 +1,76 @@ |
|||||
|
#!/usr/bin/env python3 |
||||
|
# |
||||
|
# Copyright (c) 2017, Neil Booth |
||||
|
# |
||||
|
# All rights reserved. |
||||
|
# |
||||
|
# See the file "LICENCE" for information about the copyright |
||||
|
# and warranty status of this software. |
||||
|
|
||||
|
'''Script to compact the history database. This should save space and |
||||
|
will reset the flush counter to a low number, avoiding overflow when |
||||
|
the flush count reaches 65,536. |
||||
|
|
||||
|
This needs to lock the database so ElectrumX must not be running - |
||||
|
shut it down cleanly first. |
||||
|
|
||||
|
It is recommended you run this script with the same environment as |
||||
|
ElectrumX. However it is intended to be runnable with just |
||||
|
DB_DIRECTORY and COIN set (COIN defaults as for ElectrumX). |
||||
|
|
||||
|
If you use daemon tools, you might run this script like so: |
||||
|
|
||||
|
envdir /path/to/the/environment/directory ./compact_history.py |
||||
|
|
||||
|
Depending on your hardware this script may take up to 6 hours to |
||||
|
complete; it logs progress regularly. |
||||
|
|
||||
|
Compaction can be interrupted and restarted harmlessly and will pick |
||||
|
up where it left off. However, if you restart ElectrumX without |
||||
|
running the compaction to completion, it will not benefit and |
||||
|
subsequent compactions will restart from the beginning. |
||||
|
''' |
||||
|
|
||||
|
import logging |
||||
|
import sys |
||||
|
import traceback |
||||
|
from os import environ |
||||
|
|
||||
|
from server.env import Env |
||||
|
from server.db import DB |
||||
|
|
||||
|
|
||||
|
def compact_history(): |
||||
|
if sys.version_info < (3, 5, 3): |
||||
|
raise RuntimeError('Python >= 3.5.3 is required to run ElectrumX') |
||||
|
|
||||
|
environ['DAEMON_URL'] = '' # Avoid Env erroring out |
||||
|
env = Env() |
||||
|
db = DB(env) |
||||
|
|
||||
|
assert not db.first_sync |
||||
|
# Continue where we left off, if interrupted |
||||
|
if db.comp_cursor == -1: |
||||
|
db.comp_cursor = 0 |
||||
|
|
||||
|
db.comp_flush_count = max(db.comp_flush_count, 1) |
||||
|
limit = 8 * 1000 * 1000 |
||||
|
|
||||
|
while db.comp_cursor != -1: |
||||
|
db._compact_history(limit) |
||||
|
|
||||
|
|
||||
|
def main(): |
||||
|
logging.basicConfig(level=logging.INFO) |
||||
|
logging.info('Starting history compaction...') |
||||
|
try: |
||||
|
compact_history() |
||||
|
except Exception: |
||||
|
traceback.print_exc() |
||||
|
logging.critical('History compaction terminated abnormally') |
||||
|
else: |
||||
|
logging.info('History compaction complete') |
||||
|
|
||||
|
|
||||
|
if __name__ == '__main__': |
||||
|
main() |
@ -0,0 +1,131 @@ |
|||||
|
# Test of compaction code in server/db.py |
||||
|
|
||||
|
import array |
||||
|
from collections import defaultdict |
||||
|
from os import environ, urandom |
||||
|
from struct import pack |
||||
|
import random |
||||
|
|
||||
|
from lib.hash import hash_to_str |
||||
|
from server.env import Env |
||||
|
from server.db import DB |
||||
|
|
||||
|
|
||||
|
def create_histories(db, hashX_count=100): |
||||
|
'''Creates a bunch of random transaction histories, and write them |
||||
|
to disk in a series of small flushes.''' |
||||
|
hashXs = [urandom(db.coin.HASHX_LEN) for n in range(hashX_count)] |
||||
|
mk_array = lambda : array.array('I') |
||||
|
histories = {hashX : mk_array() for hashX in hashXs} |
||||
|
this_history = defaultdict(mk_array) |
||||
|
tx_num = 0 |
||||
|
while hashXs: |
||||
|
hash_indexes = set(random.randrange(len(hashXs)) |
||||
|
for n in range(1 + random.randrange(4))) |
||||
|
for index in hash_indexes: |
||||
|
histories[hashXs[index]].append(tx_num) |
||||
|
this_history[hashXs[index]].append(tx_num) |
||||
|
|
||||
|
tx_num += 1 |
||||
|
# Occasionally flush and drop a random hashX if non-empty |
||||
|
if random.random() < 0.1: |
||||
|
db.flush_history(this_history) |
||||
|
this_history.clear() |
||||
|
index = random.randrange(0, len(hashXs)) |
||||
|
if histories[hashXs[index]]: |
||||
|
del hashXs[index] |
||||
|
|
||||
|
return histories |
||||
|
|
||||
|
|
||||
|
def check_hashX_compaction(db): |
||||
|
db.max_hist_row_entries = 40 |
||||
|
row_size = db.max_hist_row_entries * 4 |
||||
|
full_hist = array.array('I', range(100)).tobytes() |
||||
|
hashX = urandom(db.coin.HASHX_LEN) |
||||
|
pairs = ((1, 20), (26, 50), (56, 30)) |
||||
|
|
||||
|
cum = 0 |
||||
|
hist_list = [] |
||||
|
hist_map = {} |
||||
|
for flush_count, count in pairs: |
||||
|
key = hashX + pack('>H', flush_count) |
||||
|
hist = full_hist[cum * 4: (cum+count) * 4] |
||||
|
hist_map[key] = hist |
||||
|
hist_list.append(hist) |
||||
|
cum += count |
||||
|
|
||||
|
write_items = [] |
||||
|
keys_to_delete = set() |
||||
|
write_size = db._compact_hashX(hashX, hist_map, hist_list, |
||||
|
write_items, keys_to_delete) |
||||
|
# Check results for sanity |
||||
|
assert write_size == len(full_hist) |
||||
|
assert len(write_items) == 3 |
||||
|
assert len(keys_to_delete) == 3 |
||||
|
assert len(hist_map) == len(pairs) |
||||
|
for n, item in enumerate(write_items): |
||||
|
assert item == (hashX + pack('>H', n), |
||||
|
full_hist[n * row_size: (n + 1) * row_size]) |
||||
|
for flush_count, count in pairs: |
||||
|
assert hashX + pack('>H', flush_count) in keys_to_delete |
||||
|
|
||||
|
# Check re-compaction is null |
||||
|
hist_map = {key: value for key, value in write_items} |
||||
|
hist_list = [value for key, value in write_items] |
||||
|
write_items.clear() |
||||
|
keys_to_delete.clear() |
||||
|
write_size = db._compact_hashX(hashX, hist_map, hist_list, |
||||
|
write_items, keys_to_delete) |
||||
|
assert write_size == 0 |
||||
|
assert len(write_items) == 0 |
||||
|
assert len(keys_to_delete) == 0 |
||||
|
assert len(hist_map) == len(pairs) |
||||
|
|
||||
|
# Check re-compaction adding a single tx writes the one row |
||||
|
hist_list[-1] += array.array('I', [100]).tobytes() |
||||
|
write_size = db._compact_hashX(hashX, hist_map, hist_list, |
||||
|
write_items, keys_to_delete) |
||||
|
assert write_size == len(hist_list[-1]) |
||||
|
assert write_items == [(hashX + pack('>H', 2), hist_list[-1])] |
||||
|
assert len(keys_to_delete) == 1 |
||||
|
assert write_items[0][0] in keys_to_delete |
||||
|
assert len(hist_map) == len(pairs) |
||||
|
|
||||
|
|
||||
|
def check_written(db, histories): |
||||
|
for hashX, hist in histories.items(): |
||||
|
db_hist = array.array('I', db.get_history_txnums(hashX, limit=None)) |
||||
|
assert hist == db_hist |
||||
|
|
||||
|
def compact_history(db): |
||||
|
'''Synchronously compact the DB history.''' |
||||
|
db.first_sync = False |
||||
|
db.comp_cursor = 0 |
||||
|
|
||||
|
db.comp_flush_count = max(db.comp_flush_count, 1) |
||||
|
limit = 5 * 1000 |
||||
|
|
||||
|
write_size = 0 |
||||
|
while db.comp_cursor != -1: |
||||
|
write_size += db._compact_history(limit) |
||||
|
assert write_size != 0 |
||||
|
|
||||
|
def run_test(db_dir): |
||||
|
environ.clear() |
||||
|
environ['DB_DIRECTORY'] = db_dir |
||||
|
environ['DAEMON_URL'] = '' |
||||
|
env = Env() |
||||
|
db = DB(env) |
||||
|
# Test abstract compaction |
||||
|
check_hashX_compaction(db) |
||||
|
# Now test in with random data |
||||
|
histories = create_histories(db) |
||||
|
check_written(db, histories) |
||||
|
compact_history(db) |
||||
|
check_written(db, histories) |
||||
|
|
||||
|
def test_compaction(tmpdir): |
||||
|
db_dir = str(tmpdir) |
||||
|
print('Temp dir: {}'.format(db_dir)) |
||||
|
run_test(db_dir) |
Loading…
Reference in new issue