# Copyright (c) 2016, the ElectrumX authors # # All rights reserved. # # See the file "LICENCE" for information about the copyright # and warranty status of this software. '''Backend database abstraction. The abstraction needs to be improved to not heavily penalise LMDB. ''' import os from functools import partial from lib.util import subclasses, increment_byte_string def open_db(name, db_engine, for_sync): '''Returns a database handle.''' for db_class in subclasses(Storage): if db_class.__name__.lower() == db_engine.lower(): db_class.import_module() return db_class(name, for_sync) raise RuntimeError('unrecognised DB engine "{}"'.format(db_engine)) class Storage(object): '''Abstract base class of the DB backend abstraction.''' def __init__(self, name, for_sync): self.is_new = not os.path.exists(name) self.open(name, create=self.is_new, for_sync=for_sync) @classmethod def import_module(cls): '''Import the DB engine module.''' raise NotImplementedError def open(self, name, create): '''Open an existing database or create a new one.''' raise NotImplementedError def close(self): '''Close an existing database.''' raise NotImplementedError def get(self, key): raise NotImplementedError def put(self, key, value): raise NotImplementedError def write_batch(self): '''Return a context manager that provides `put` and `delete`. Changes should only be committed when the context manager closes without an exception. ''' raise NotImplementedError def iterator(self, prefix=b'', reverse=False): '''Return an iterator that yields (key, value) pairs from the database sorted by key. If `prefix` is set, only keys starting with `prefix` will be included. If `reverse` is True the items are returned in reverse order. ''' raise NotImplementedError class LevelDB(Storage): '''LevelDB database engine.''' @classmethod def import_module(cls): import plyvel cls.module = plyvel def open(self, name, create, for_sync): mof = 1024 if for_sync else 256 self.db = self.module.DB(name, create_if_missing=create, max_open_files=mof, compression=None) self.close = self.db.close self.get = self.db.get self.put = self.db.put self.iterator = self.db.iterator self.write_batch = partial(self.db.write_batch, transaction=True, sync=True) class RocksDB(Storage): '''RocksDB database engine.''' @classmethod def import_module(cls): import rocksdb cls.module = rocksdb def open(self, name, create, for_sync): mof = 1024 if for_sync else 256 compression = "no" compression = getattr(self.module.CompressionType, compression + "_compression") options = self.module.Options(create_if_missing=create, compression=compression, use_fsync=True, target_file_size_base=33554432, max_open_files=mof) self.db = self.module.DB(name, options) self.get = self.db.get self.put = self.db.put def close(self): # PyRocksDB doesn't provide a close method; hopefully this is enough self.db = self.get = self.put = None import gc gc.collect() class WriteBatch(object): def __init__(self, db): self.batch = RocksDB.module.WriteBatch() self.db = db def __enter__(self): return self.batch def __exit__(self, exc_type, exc_val, exc_tb): if not exc_val: self.db.write(self.batch) def write_batch(self): return RocksDB.WriteBatch(self.db) class Iterator(object): def __init__(self, db, prefix, reverse): self.it = db.iteritems() self.reverse = reverse self.prefix = prefix # Whether we are at the first item self.first = True def __iter__(self): prefix = self.prefix if self.reverse: prefix = increment_byte_string(prefix) self.it = reversed(self.it) self.it.seek(prefix) return self def __next__(self): k, v = self.it.__next__() if self.first and self.reverse and not k.startswith(self.prefix): k, v = self.it.__next__() self.first = False if not k.startswith(self.prefix): # We're already ahead of the prefix raise StopIteration return k, v def iterator(self, prefix=b'', reverse=False): return RocksDB.Iterator(self.db, prefix, reverse) class LMDB(Storage): '''RocksDB database engine.''' @classmethod def import_module(cls): import lmdb cls.module = lmdb def open(self, name, create, for_sync): # I don't see anything equivalent to max_open_files for for_sync self.env = LMDB.module.Environment('.', subdir=True, create=create, max_dbs=32, map_size=5 * 10 ** 10) self.db = self.env.open_db(create=create) def close(self): self.env.close() def get(self, key): with self.env.begin(db=self.db) as tx: return tx.get(key) def put(self, key, value): with self.env.begin(db=self.db, write=True) as tx: tx.put(key, value) def write_batch(self): return self.env.begin(db=self.db, write=True) def iterator(self, prefix=b'', reverse=False): return LMDB.Iterator(self.db, self.env, prefix, reverse) class Iterator: def __init__(self, db, env, prefix, reverse): self.transaction = env.begin(db=db) self.transaction.__enter__() self.db = db self.prefix = prefix self.reverse = reverse self._stop = False def __iter__(self): self.iterator = LMDB.module.Cursor(self.db, self.transaction) prefix = self.prefix if self.reverse: # Go to the first value after the prefix prefix = increment_byte_string(prefix) self.iterator.set_range(prefix) if not self.iterator.key().startswith(self.prefix) and self.reverse: # Go back to the first item starting with the prefix self.iterator.prev() return self def __next__(self): k, v = self.iterator.item() if not k.startswith(self.prefix) or self._stop: # We're already ahead of the prefix self.transaction.__exit__() raise StopIteration next = self.iterator.next \ if not self.reverse else self.iterator.prev # Stop after the next value if we're at the end of the DB self._stop = not next() return k, v