diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..4729f1d --- /dev/null +++ b/.travis.yml @@ -0,0 +1,19 @@ +sudo: required +dist: trusty +language: python +before_install: + - sudo add-apt-repository -y ppa:giskou/librocksdb + - sudo apt-get -qq update + - sudo apt-get install -yq libleveldb-dev librocksdb libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev +python: + - "3.5" + - "3.6-dev" + - "nightly" +# command to install dependencies +install: + - pip install aiohttp + - pip install lmdb + - pip install plyvel + - pip install pyrocksdb +# command to run tests +script: pytest \ No newline at end of file diff --git a/README.rst b/README.rst index c070f75..ea75618 100644 --- a/README.rst +++ b/README.rst @@ -1,3 +1,6 @@ +.. image:: https://travis-ci.org/kyuupichan/electrumx.svg?branch=master + :target: https://travis-ci.org/kyuupichan/electrumx + ElectrumX - Reimplementation of Electrum-server =============================================== :: diff --git a/docs/RELEASE-NOTES b/docs/RELEASE-NOTES index c1b7d90..bf1214c 100644 --- a/docs/RELEASE-NOTES +++ b/docs/RELEASE-NOTES @@ -1,3 +1,12 @@ +Version 0.2.1 +------------- + +- fix rocksdb and lmdb abstractions (bauerj) +- limit concurrent daemon requests +- improve script + coin abstractions +- faster tx and script parsing +- minor bug fixes + Version 0.2 ----------- diff --git a/lib/util.py b/lib/util.py index a537737..eb34f20 100644 --- a/lib/util.py +++ b/lib/util.py @@ -104,3 +104,20 @@ def int_to_bytes(value): value, mod = divmod(value, 256) mods.append(mod) return bytes(reversed(mods)) + + +def increment_byte_string(bs): + bs = bytearray(bs) + incremented = False + for i in reversed(range(len(bs))): + if bs[i] < 0xff: + # This is easy + bs[i] += 1 + incremented = True + break + # Otherwise we need to look at the previous character + bs[i] = 0 + if not incremented: + # This can only happen if all characters are 0xff + bs = bytes([1]) + bs + return bytes(bs) diff --git a/server/storage.py b/server/storage.py index d4557d4..35e0a7e 100644 --- a/server/storage.py +++ b/server/storage.py @@ -13,7 +13,7 @@ The abstraction needs to be improved to not heavily penalise LMDB. import os from functools import partial -from lib.util import subclasses +from lib.util import subclasses, increment_byte_string def open_db(name, db_engine): @@ -122,16 +122,24 @@ class RocksDB(Storage): class Iterator(object): def __init__(self, db, prefix, reverse): self.it = db.iteritems() - if reverse: - self.it = reversed(self.it) + self.reverse = reverse self.prefix = prefix + # Whether we are at the first item + self.first = True def __iter__(self): - self.it.seek(self.prefix) + prefix = self.prefix + if self.reverse: + prefix = increment_byte_string(prefix) + self.it = reversed(self.it) + self.it.seek(prefix) return self def __next__(self): k, v = self.it.__next__() + if self.first and self.reverse and not k.startswith(self.prefix): + k, v = self.it.__next__() + self.first = False if not k.startswith(self.prefix): # We're already ahead of the prefix raise StopIteration @@ -150,7 +158,7 @@ class LMDB(Storage): cls.module = lmdb def open(self, name, create): - self.env = cls.module.Environment('.', subdir=True, create=create, + self.env = LMDB.module.Environment('.', subdir=True, create=create, max_dbs=32, map_size=5 * 10 ** 10) self.db = self.env.open_db(create=create) @@ -174,17 +182,29 @@ class LMDB(Storage): self.transaction.__enter__() self.db = db self.prefix = prefix - self.reverse = reverse # FIXME + self.reverse = reverse + self._stop = False def __iter__(self): - self.iterator = LMDB.lmdb.Cursor(self.db, self.transaction) - self.iterator.set_range(self.prefix) + self.iterator = LMDB.module.Cursor(self.db, self.transaction) + prefix = self.prefix + if self.reverse: + # Go to the first value after the prefix + prefix = increment_byte_string(prefix) + self.iterator.set_range(prefix) + if not self.iterator.key().startswith(self.prefix) and self.reverse: + # Go back to the first item starting with the prefix + self.iterator.prev() return self def __next__(self): k, v = self.iterator.item() - if not k.startswith(self.prefix) or not self.iterator.next(): + if not k.startswith(self.prefix) or self._stop: # We're already ahead of the prefix self.transaction.__exit__() raise StopIteration + next = self.iterator.next \ + if not self.reverse else self.iterator.prev + # Stop after the next value if we're at the end of the DB + self._stop = not next() return k, v diff --git a/server/version.py b/server/version.py index 764e2d3..7dd5120 100644 --- a/server/version.py +++ b/server/version.py @@ -1 +1 @@ -VERSION = "ElectrumX 0.2" +VERSION = "ElectrumX 0.2.1" diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_storage.py b/tests/test_storage.py new file mode 100644 index 0000000..b765c27 --- /dev/null +++ b/tests/test_storage.py @@ -0,0 +1,69 @@ +import gc +import pytest +import os + +from server.storage import Storage, open_db +from lib.util import subclasses + +# Find out which db engines to test +# Those that are not installed will be skipped +db_engines = [] +for c in subclasses(Storage): + try: + c.import_module() + except ImportError: + db_engines.append(pytest.mark.skip(c.__name__)) + else: + db_engines.append(c.__name__) + + +@pytest.fixture(params=db_engines) +def db(tmpdir, request): + cwd = os.getcwd() + os.chdir(str(tmpdir)) + db = open_db("db", request.param) + os.chdir(cwd) + yield db + # Make sure all the locks and handles are closed + del db + gc.collect() + + +def test_put_get(db): + db.put(b"x", b"y") + assert db.get(b"x") == b"y" + + +def test_batch(db): + db.put(b"a", b"1") + with db.write_batch() as b: + b.put(b"a", b"2") + assert db.get(b"a") == b"1" + assert db.get(b"a") == b"2" + + +def test_iterator(db): + """ + The iterator should contain all key/value pairs starting with prefix ordered + by key. + """ + for i in range(5): + db.put(b"abc" + str.encode(str(i)), str.encode(str(i))) + db.put(b"abc", b"") + db.put(b"a", b"xyz") + db.put(b"abd", b"x") + assert list(db.iterator(prefix=b"abc")) == [(b"abc", b"")] + [ + (b"abc" + str.encode(str(i)), str.encode(str(i))) for + i in range(5) + ] + + +def test_iterator_reverse(db): + for i in range(5): + db.put(b"abc" + str.encode(str(i)), str.encode(str(i))) + db.put(b"a", b"xyz") + db.put(b"abd", b"x") + assert list(db.iterator(prefix=b"abc", reverse=True)) == [ + (b"abc" + str.encode(str(i)), str.encode(str(i))) for + i in reversed(range(5)) + ] \ No newline at end of file