Merge remote-tracking branch 'github/master' into release-0.2.1

Include's bauerj's storage.py improvements.

.travis.yml

@@ -0,0 +1,19 @@
+sudo: required
+dist: trusty
+language: python
+before_install:
+  - sudo add-apt-repository -y ppa:giskou/librocksdb
+  - sudo apt-get -qq update
+  - sudo apt-get install -yq libleveldb-dev librocksdb libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev
+python:
+  - "3.5"
+  - "3.6-dev"
+  - "nightly"
+# command to install dependencies
+install:
+  - pip install aiohttp
+  - pip install lmdb
+  - pip install plyvel
+  - pip install pyrocksdb
+# command to run tests
+script: pytest

README.rst

@@ -1,3 +1,6 @@
+.. image:: https://travis-ci.org/kyuupichan/electrumx.svg?branch=master
+    :target: https://travis-ci.org/kyuupichan/electrumx
+
 ElectrumX - Reimplementation of Electrum-server
 ===============================================
 ::

lib/util.py

@@ -104,3 +104,20 @@ def int_to_bytes(value):
         value, mod = divmod(value, 256)
         mods.append(mod)
     return bytes(reversed(mods))
+
+
+def increment_byte_string(bs):
+    bs = bytearray(bs)
+    incremented = False
+    for i in reversed(range(len(bs))):
+        if bs[i] < 0xff:
+            # This is easy
+            bs[i] += 1
+            incremented = True
+            break
+        # Otherwise we need to look at the previous character
+        bs[i] = 0
+    if not incremented:
+        # This can only happen if all characters are 0xff
+        bs = bytes([1]) + bs
+    return bytes(bs)

server/storage.py

@@ -13,7 +13,7 @@ The abstraction needs to be improved to not heavily penalise LMDB.
 import os
 from functools import partial
 
-from lib.util import subclasses
+from lib.util import subclasses, increment_byte_string
 
 
 def open_db(name, db_engine):
@@ -122,16 +122,24 @@ class RocksDB(Storage):
     class Iterator(object):
         def __init__(self, db, prefix, reverse):
             self.it = db.iteritems()
-            if reverse:
-                self.it = reversed(self.it)
+            self.reverse = reverse
             self.prefix = prefix
+            # Whether we are at the first item
+            self.first = True
 
         def __iter__(self):
-            self.it.seek(self.prefix)
+            prefix = self.prefix
+            if self.reverse:
+                prefix = increment_byte_string(prefix)
+                self.it = reversed(self.it)
+            self.it.seek(prefix)
             return self
 
         def __next__(self):
             k, v = self.it.__next__()
+            if self.first and self.reverse and not k.startswith(self.prefix):
+                k, v = self.it.__next__()
+            self.first = False
             if not k.startswith(self.prefix):
                 # We're already ahead of the prefix
                 raise StopIteration
@@ -150,7 +158,7 @@ class LMDB(Storage):
         cls.module = lmdb
 
     def open(self, name, create):
-        self.env = cls.module.Environment('.', subdir=True, create=create,
+        self.env = LMDB.module.Environment('.', subdir=True, create=create,
                                           max_dbs=32, map_size=5 * 10 ** 10)
         self.db = self.env.open_db(create=create)
 
@@ -174,17 +182,29 @@ class LMDB(Storage):
             self.transaction.__enter__()
             self.db = db
             self.prefix = prefix
-            self.reverse = reverse   # FIXME
+            self.reverse = reverse
+            self._stop = False
 
         def __iter__(self):
-            self.iterator = LMDB.lmdb.Cursor(self.db, self.transaction)
-            self.iterator.set_range(self.prefix)
+            self.iterator = LMDB.module.Cursor(self.db, self.transaction)
+            prefix = self.prefix
+            if self.reverse:
+                # Go to the first value after the prefix
+                prefix = increment_byte_string(prefix)
+            self.iterator.set_range(prefix)
+            if not self.iterator.key().startswith(self.prefix) and self.reverse:
+                # Go back to the first item starting with the prefix
+                self.iterator.prev()
             return self
 
         def __next__(self):
             k, v = self.iterator.item()
-            if not k.startswith(self.prefix) or not self.iterator.next():
+            if not k.startswith(self.prefix) or self._stop:
                 # We're already ahead of the prefix
                 self.transaction.__exit__()
                 raise StopIteration
+            next = self.iterator.next \
+                if not self.reverse else self.iterator.prev
+            # Stop after the next value if we're at the end of the DB
+            self._stop = not next()
             return k, v

tests/test_storage.py

@@ -0,0 +1,69 @@
+import gc
+import pytest
+import os
+
+from server.storage import Storage, open_db
+from lib.util import subclasses
+
+# Find out which db engines to test
+# Those that are not installed will be skipped
+db_engines = []
+for c in subclasses(Storage):
+    try:
+        c.import_module()
+    except ImportError:
+        db_engines.append(pytest.mark.skip(c.__name__))
+    else:
+        db_engines.append(c.__name__)
+
+
+@pytest.fixture(params=db_engines)
+def db(tmpdir, request):
+    cwd = os.getcwd()
+    os.chdir(str(tmpdir))
+    db = open_db("db", request.param)
+    os.chdir(cwd)
+    yield db
+    # Make sure all the locks and handles are closed
+    del db
+    gc.collect()
+
+
+def test_put_get(db):
+    db.put(b"x", b"y")
+    assert db.get(b"x") == b"y"
+
+
+def test_batch(db):
+    db.put(b"a", b"1")
+    with db.write_batch() as b:
+        b.put(b"a", b"2")
+        assert db.get(b"a") == b"1"
+    assert db.get(b"a") == b"2"
+
+
+def test_iterator(db):
+    """
+    The iterator should contain all key/value pairs starting with prefix ordered
+    by key.
+    """
+    for i in range(5):
+        db.put(b"abc" + str.encode(str(i)), str.encode(str(i)))
+    db.put(b"abc", b"")
+    db.put(b"a", b"xyz")
+    db.put(b"abd", b"x")
+    assert list(db.iterator(prefix=b"abc")) == [(b"abc", b"")] + [
+            (b"abc" + str.encode(str(i)), str.encode(str(i))) for
+            i in range(5)
+        ]
+
+
+def test_iterator_reverse(db):
+    for i in range(5):
+        db.put(b"abc" + str.encode(str(i)), str.encode(str(i)))
+    db.put(b"a", b"xyz")
+    db.put(b"abd", b"x")
+    assert list(db.iterator(prefix=b"abc", reverse=True)) == [
+            (b"abc" + str.encode(str(i)), str.encode(str(i))) for
+            i in reversed(range(5))
+        ]