From 091c2fc8f53293c1dffe4f17aaeaae2633ecf1d8 Mon Sep 17 00:00:00 2001
From: Christian Decker <decker.christian@gmail.com>
Date: Tue, 17 Jan 2017 23:34:28 +0100
Subject: [PATCH 1/4] log: Flushing logs on each new entry
This is to speed up the python testing framework and should not have a
big impact on performance.
---
daemon/log.c | 1 +
1 file changed, 1 insertion(+)
diff --git a/daemon/log.c b/daemon/log.c
index 3011b3c13..564776204 100644
--- a/daemon/log.c
+++ b/daemon/log.c
@@ -421,6 +421,7 @@ static void log_to_file(const char *prefix,
} else {
fprintf(logf, "%s \t%s\n", prefix, str);
}
+ fflush(logf);
}
static char *arg_log_to_file(const char *arg, struct log *log)
From 5f61b3a27285a57656b3e4ef8ee62674ee5880ba Mon Sep 17 00:00:00 2001
From: Christian Decker <decker.christian@gmail.com>
Date: Tue, 17 Jan 2017 23:26:00 +0100
Subject: [PATCH 2/4] pytest: Separating new lightningd and legacy lightningd
RPC client
We intend to ultimately no longer use the legacy `daemon/lightningd`
and instead use `lightningd/lightningd`, so I grouped the new RPC and
the legacy RPC and implemented stubs for the new daemon.
---
contrib/pylightning/lightning/__init__.py | 2 +-
contrib/pylightning/lightning/lightning.py | 70 ++++++++++++++++------
2 files changed, 54 insertions(+), 18 deletions(-)
diff --git a/contrib/pylightning/lightning/__init__.py b/contrib/pylightning/lightning/__init__.py
index 8192d42c8..ef4a36347 100644
--- a/contrib/pylightning/lightning/__init__.py
+++ b/contrib/pylightning/lightning/__init__.py
@@ -1 +1 @@
-from .lightning import LightningRpc
+from .lightning import LightningRpc, LegacyLightningRpc
diff --git a/contrib/pylightning/lightning/lightning.py b/contrib/pylightning/lightning/lightning.py
index 7f4bb0247..3682245fc 100644
--- a/contrib/pylightning/lightning/lightning.py
+++ b/contrib/pylightning/lightning/lightning.py
@@ -7,28 +7,12 @@ import socket
import sys
import threading
-class LightningRpc(object):
- """RPC client for the `lightningd` daemon.
-
- This RPC client connects to the `lightningd` daemon through a unix
- domain socket and passes calls through. Since some of the calls
- are blocking, the corresponding python methods include an `async`
- keyword argument. If `async` is set to true then the method
- returns a future immediately, instead of blocking indefinitely.
-
- This implementation is thread safe in that it locks the socket
- between calls, but it does not (yet) support concurrent calls.
- """
+class UnixDomainSocketRpc(object):
def __init__(self, socket_path, executor=None):
self.socket_path = socket_path
- self.socket = None
- self.buff = b''
self.decoder = json.JSONDecoder()
self.executor = executor
- def connect_rpc(self):
- pass
-
def _writeobj(self, sock, obj):
s = json.dumps(obj)
sock.sendall(bytearray(s, 'UTF-8'))
@@ -66,6 +50,58 @@ class LightningRpc(object):
raise ValueError("Malformed response, 'result' missing.")
return resp['result']
+
+class LightningRpc(UnixDomainSocketRpc):
+ """RPC client for the `lightningd` daemon.
+
+ This RPC client connects to the `lightningd` daemon through a unix
+ domain socket and passes calls through. Since some of the calls
+ are blocking, the corresponding python methods include an `async`
+ keyword argument. If `async` is set to true then the method
+ returns a future immediately, instead of blocking indefinitely.
+
+ This implementation is thread safe in that it locks the socket
+ between calls, but it does not (yet) support concurrent calls.
+ """
+ def connect(self, hostname, port, remote_id):
+ return self._call("connect", [hostname, port, remote_id])
+
+ def getpeers(self):
+ return self._call("getpeers", [])
+
+ def getpeer(self, peer_id):
+ """Get info about a specific peer.
+ """
+ peers = self.getpeers()['peers']
+ for p in peers:
+ if p['peerid'] == peer_id:
+ return p
+ return None
+
+ def stop(self):
+ return self._call("stop", [])
+
+ def getlog(self, level=None):
+ args = []
+ if level is not None:
+ args.append(level)
+ return self._call("getlog", args)
+
+ def getinfo(self):
+ return self._call("getinfo", [])
+
+ def dev_add_route(self, src, dst, base, var, delay, minblocks):
+ """Add route from {src} to {dst}, {base} rate in msatoshi, {var} rate in msatoshi, {delay} blocks delay and {minblocks} minimum timeout
+ """
+ return self._call("dev-add-route", [src, dst, base, var, delay, minblocks])
+
+ def getchannels(self):
+ return self._call("getchannels", [])
+
+ def getnodes(self):
+ return self._call("getnodes", [])
+
+class LegacyLightningRpc(UnixDomainSocketRpc):
def getchannels(self):
"""List all known channels.
"""
From 180c96776bfd818de3472f358cd513be8a0546bc Mon Sep 17 00:00:00 2001
From: Christian Decker <decker.christian@gmail.com>
Date: Tue, 17 Jan 2017 23:30:22 +0100
Subject: [PATCH 3/4] pytest: Split the python integration tests into new and
legacy
---
tests/test_lightningd.py | 40 +++++++++++++++++++++++++++++++---------
tests/utils.py | 15 +++++++++++++--
2 files changed, 44 insertions(+), 11 deletions(-)
diff --git a/tests/test_lightningd.py b/tests/test_lightningd.py
index 36bd0778e..a0ed79c68 100644
--- a/tests/test_lightningd.py
+++ b/tests/test_lightningd.py
@@ -1,7 +1,7 @@
from binascii import hexlify, unhexlify
from concurrent import futures
from hashlib import sha256
-from utils import BitcoinD, LightningD, LightningRpc, LightningNode
+from lightning import LightningRpc, LegacyLightningRpc
import logging
import os
@@ -9,6 +9,7 @@ import sys
import tempfile
import time
import unittest
+import utils
bitcoind = None
TEST_DIR = tempfile.mkdtemp(prefix='lightning-')
@@ -21,7 +22,7 @@ logging.info("Tests running in '%s'", TEST_DIR)
def setupBitcoind():
global bitcoind
- bitcoind = BitcoinD(rpcport=28332)
+ bitcoind = utils.BitcoinD(rpcport=28332)
bitcoind.start()
info = bitcoind.rpc.getinfo()
# Make sure we have segwit and some funds
@@ -59,7 +60,7 @@ class NodeFactory(object):
self.nodes = []
self.executor = executor
- def get_node(self):
+ def get_node(self, legacy=True):
node_id = self.next_id
self.next_id += 1
@@ -67,9 +68,15 @@ class NodeFactory(object):
TEST_DIR, self.func._testMethodName, "lightning-{}/".format(node_id))
socket_path = os.path.join(lightning_dir, "lightning-rpc").format(node_id)
- daemon = LightningD(lightning_dir, bitcoind.bitcoin_dir, port=16330+node_id)
- rpc = LightningRpc(socket_path, self.executor)
- node = LightningNode(daemon, rpc, bitcoind, self.executor)
+ port = 16330+node_id
+ if legacy:
+ daemon = utils.LegacyLightningD(lightning_dir, bitcoind.bitcoin_dir, port=port)
+ rpc = LegacyLightningRpc(socket_path, self.executor)
+ else:
+ daemon = utils.LightningD(lightning_dir, bitcoind.bitcoin_dir, port=port)
+ rpc = LightningRpc(socket_path, self.executor)
+
+ node = utils.LightningNode(daemon, rpc, bitcoind, self.executor)
self.nodes.append(node)
if VALGRIND:
node.daemon.cmd_line = [
@@ -80,7 +87,6 @@ class NodeFactory(object):
] + node.daemon.cmd_line
node.daemon.start()
- node.rpc.connect_rpc()
# Cache `getinfo`, we'll be using it a lot
node.info = node.rpc.getinfo()
return node
@@ -90,8 +96,7 @@ class NodeFactory(object):
n.daemon.stop()
-class LightningDTests(unittest.TestCase):
-
+class BaseLightningDTests(unittest.TestCase):
def setUp(self):
# Most of the executor threads will be waiting for IO, so
# let's have a few of them
@@ -103,6 +108,23 @@ class LightningDTests(unittest.TestCase):
self.executor.shutdown(wait=False)
# TODO(cdecker) Check that valgrind didn't find any errors
+
+class LightningDTests(BaseLightningDTests):
+ def test_connect(self):
+ l1 = self.node_factory.get_node(legacy=False)
+ l2 = self.node_factory.get_node(legacy=False)
+ ret = l1.rpc.connect('localhost', l2.info['port'], l2.info['id'])
+
+ assert ret['id'] == l2.info['id']
+
+ p1 = l1.rpc.getpeer(l2.info['id'])
+ p2 = l2.rpc.getpeer(l1.info['id'])
+
+ assert p1['condition'] == 'Exchanging gossip'
+ assert p2['condition'] == 'Exchanging gossip'
+
+class LegacyLightningDTests(BaseLightningDTests):
+
def test_connect(self):
l1 = self.node_factory.get_node()
l2 = self.node_factory.get_node()
diff --git a/tests/utils.py b/tests/utils.py
index c8bc89156..b0fdbd8b0 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -160,7 +160,7 @@ class LightningD(TailableProc):
self.lightning_dir = lightning_dir
self.port = port
self.cmd_line = [
- 'daemon/lightningd',
+ 'lightningd/lightningd',
'--bitcoin-datadir={}'.format(bitcoin_dir),
'--lightning-dir={}'.format(lightning_dir),
'--port={}'.format(port),
@@ -176,13 +176,24 @@ class LightningD(TailableProc):
def start(self):
TailableProc.start(self)
- self.wait_for_log("Hello world!")
+ self.wait_for_log("Creating IPv6 listener on port")
logging.info("LightningD started")
def stop(self):
TailableProc.stop(self)
logging.info("LightningD stopped")
+class LegacyLightningD(LightningD):
+ def __init__(self, *args, **kwargs):
+ LightningD.__init__(self, *args, **kwargs)
+ self.cmd_line[0] = 'daemon/lightningd'
+
+ def start(self):
+ TailableProc.start(self)
+ self.wait_for_log("Hello world!")
+ logging.info("LightningD started")
+
+
class LightningNode(object):
def __init__(self, daemon, rpc, btc, executor):
self.rpc = rpc
From 1948062250aecc510debf012856043372d7c0205 Mon Sep 17 00:00:00 2001
From: Christian Decker <decker.christian@gmail.com>
Date: Sat, 21 Jan 2017 14:10:55 +0100
Subject: [PATCH 4/4] pytest: Using python-bitcoinlib instead of
python-bitcoinrpc
The latter is unmaintained and has a number of problems.
---
tests/requirements.txt | 2 +-
tests/utils.py | 15 ++-------------
2 files changed, 3 insertions(+), 14 deletions(-)
diff --git a/tests/requirements.txt b/tests/requirements.txt
index 115c99979..5928dec47 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -1 +1 @@
-python-bitcoinrpc==1.0
+python-bitcoinlib==0.7.0
diff --git a/tests/utils.py b/tests/utils.py
index b0fdbd8b0..3cad8d277 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -1,4 +1,4 @@
-from bitcoinrpc.authproxy import AuthServiceProxy
+from bitcoin.rpc import RawProxy as BitcoinProxy
from lightning import LightningRpc
import logging
@@ -108,17 +108,6 @@ class TailableProc(object):
pos += 1
-class ThreadSafeAuthServiceProxy(AuthServiceProxy):
- """Thread-safe variant of the AuthServiceProxy.
- """
-
- lock = threading.RLock()
-
- def __call__(self, *args):
- with ThreadSafeAuthServiceProxy.lock:
- AuthServiceProxy.__call__(self, *args)
-
-
class BitcoinD(TailableProc):
def __init__(self, bitcoin_dir="/tmp/bitcoind-test", rpcport=18332):
@@ -145,7 +134,7 @@ class BitcoinD(TailableProc):
BITCOIND_CONFIG['rpcport'] = rpcport
write_config(os.path.join(bitcoin_dir, 'bitcoin.conf'), BITCOIND_CONFIG)
write_config(os.path.join(regtestdir, 'bitcoin.conf'), BITCOIND_CONFIG)
- self.rpc = ThreadSafeAuthServiceProxy(
+ self.rpc = BitcoinProxy(
"http://rpcuser:rpcpass@127.0.0.1:{}".format(self.rpcport))
def start(self):