From bf4892dfeadcc7199234cc0be4bb6746c18f4225 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Tue, 23 Jul 2019 15:26:28 +0200 Subject: [PATCH] pyln: Split pylightning into multiple pyln modules This is the first step to transition to a better organized python module structure. Sadly we can't reuse the `pylightning` module as a namespace module since having importable things in the top level of the namespace is not allowed in any of the namespace variants [1], hence we just switch over to the `pyln` namespace. The code the was under `lightning` will now be reachable under `pyln.client` and we add the `pyln.proto` module for all the things that are independent of talking to lightningd and can be used for protocol testing. [1] https://packaging.python.org/guides/packaging-namespace-packages/ Signed-off-by: Christian Decker --- .gitignore | 3 + contrib/pylightning/requirements.txt | 2 - contrib/pyln-client/README.md | 101 +++++ contrib/pyln-client/pyln/client/__init__.py | 10 + contrib/pyln-client/requirements.txt | 1 + contrib/pyln-client/setup.py | 24 ++ contrib/pyln-proto/README.md | 30 ++ contrib/pyln-proto/examples/connect.py | 23 + contrib/pyln-proto/examples/listen.py | 47 +++ contrib/pyln-proto/pyln/proto/__init__.py | 1 + contrib/pyln-proto/pyln/proto/wire.py | 395 ++++++++++++++++++ contrib/pyln-proto/requirements.txt | 2 + contrib/pyln-proto/setup.py | 24 ++ .../tests/test_wire.py | 0 14 files changed, 661 insertions(+), 2 deletions(-) create mode 100644 contrib/pyln-client/README.md create mode 100644 contrib/pyln-client/pyln/client/__init__.py create mode 100644 contrib/pyln-client/requirements.txt create mode 100644 contrib/pyln-client/setup.py create mode 100644 contrib/pyln-proto/README.md create mode 100644 contrib/pyln-proto/examples/connect.py create mode 100644 contrib/pyln-proto/examples/listen.py create mode 100644 contrib/pyln-proto/pyln/proto/__init__.py create mode 100644 contrib/pyln-proto/pyln/proto/wire.py create mode 100644 contrib/pyln-proto/requirements.txt create mode 100644 contrib/pyln-proto/setup.py rename contrib/{pylightning => pyln-proto}/tests/test_wire.py (100%) diff --git a/.gitignore b/.gitignore index 936d62a07..c8819dea8 100644 --- a/.gitignore +++ b/.gitignore @@ -38,4 +38,7 @@ tools/headerversions contrib/pylightning/build/ contrib/pylightning/dist/ contrib/pylightning/pylightning.egg-info/ +contrib/pyln-*/build/ +contrib/pyln-*/dist/ +contrib/pyln-*/pyln_*.egg-info/ devtools/create-gossipstore diff --git a/contrib/pylightning/requirements.txt b/contrib/pylightning/requirements.txt index 5daa26d68..e69de29bb 100644 --- a/contrib/pylightning/requirements.txt +++ b/contrib/pylightning/requirements.txt @@ -1,2 +0,0 @@ -cryptography==2.7 -coincurve==12.0.0 diff --git a/contrib/pyln-client/README.md b/contrib/pyln-client/README.md new file mode 100644 index 000000000..9777fd449 --- /dev/null +++ b/contrib/pyln-client/README.md @@ -0,0 +1,101 @@ +# pyln-client: A python client library for lightningd + +This package implements the Unix socket based JSON-RPC protocol that +`lightningd` exposes to the rest of the world. It can be used to call +arbitrary functions on the RPC interface, and serves as a basis for plugins +written in python. + + +## Installation + +`pyln-client` is available on `pip`: + +``` +pip install pyln-client +``` + +Alternatively you can also install the development version to get access to +currently unreleased features by checking out the c-lightning source code and +installing into your python3 environment: + +```bash +git clone https://github.com/ElementsProject/lightning.git +cd lightning/contrib/pyln-client +python3 setup.py develop +``` + +This will add links to the library into your environment so changing the +checked out source code will also result in the environment picking up these +changes. Notice however that unreleased versions may change API without +warning, so test thoroughly with the released version. + +## Examples + + +### Using the JSON-RPC client +```py +""" +Generate invoice on one daemon and pay it on the other +""" +from pyln.client import LightningRpc +import random + +# Create two instances of the LightningRpc object using two different c-lightning daemons on your computer +l1 = LightningRpc("/tmp/lightning1/lightning-rpc") +l5 = LightningRpc("/tmp/lightning5/lightning-rpc") + +info5 = l5.getinfo() +print(info5) + +# Create invoice for test payment +invoice = l5.invoice(100, "lbl{}".format(random.random()), "testpayment") +print(invoice) + +# Get route to l1 +route = l1.getroute(info5['id'], 100, 1) +print(route) + +# Pay invoice +print(l1.sendpay(route['route'], invoice['payment_hash'])) +``` + +### Writing a plugin + +Plugins are programs that `lightningd` can be configured to execute alongside +the main daemon. They allow advanced interactions with and customizations to +the daemon. + +```python +#!/usr/bin/env python3 +from pyln.client import Plugin + +plugin = Plugin() + +@plugin.method("hello") +def hello(plugin, name="world"): + """This is the documentation string for the hello-function. + + It gets reported as the description when registering the function + as a method with `lightningd`. + + """ + greeting = plugin.get_option('greeting') + s = '{} {}'.format(greeting, name) + plugin.log(s) + return s + + +@plugin.init() +def init(options, configuration, plugin): + plugin.log("Plugin helloworld.py initialized") + + +@plugin.subscribe("connect") +def on_connect(plugin, id, address): + plugin.log("Received connect event for peer {}".format(id)) + + +plugin.add_option('greeting', 'Hello', 'The greeting I should use.') +plugin.run() + +``` diff --git a/contrib/pyln-client/pyln/client/__init__.py b/contrib/pyln-client/pyln/client/__init__.py new file mode 100644 index 000000000..0c8a5c505 --- /dev/null +++ b/contrib/pyln-client/pyln/client/__init__.py @@ -0,0 +1,10 @@ +from lightning import LightningRpc, Plugin, RpcError, Millisatoshi, __version__, monkey_patch + +__all__ = [ + "LightningRpc", + "Plugin", + "RpcError", + "Millisatoshi", + "__version__", + "monkey_patch" +] diff --git a/contrib/pyln-client/requirements.txt b/contrib/pyln-client/requirements.txt new file mode 100644 index 000000000..010c05fc7 --- /dev/null +++ b/contrib/pyln-client/requirements.txt @@ -0,0 +1 @@ +pylightning==0.0.7.3 diff --git a/contrib/pyln-client/setup.py b/contrib/pyln-client/setup.py new file mode 100644 index 000000000..8650962ea --- /dev/null +++ b/contrib/pyln-client/setup.py @@ -0,0 +1,24 @@ +from setuptools import setup +from pyln import client +import io + + +with io.open('README.md', encoding='utf-8') as f: + long_description = f.read() + +with io.open('requirements.txt', encoding='utf-8') as f: + requirements = [r for r in f.read().split('\n') if len(r)] + +setup(name='pyln-client', + version=client.__version__, + description='Client library for lightningd', + long_description=long_description, + long_description_content_type='text/markdown', + url='http://github.com/ElementsProject/lightning', + author='Christian Decker', + author_email='decker.christian@gmail.com', + license='MIT', + packages=['pyln.client'], + scripts=[], + zip_safe=True, + install_requires=requirements) diff --git a/contrib/pyln-proto/README.md b/contrib/pyln-proto/README.md new file mode 100644 index 000000000..497ab3c0b --- /dev/null +++ b/contrib/pyln-proto/README.md @@ -0,0 +1,30 @@ +# pyln-proto: Lightning Network protocol implementation + +This package implements some of the Lightning Network protocol in pure +python. It is intended for protocol testing and some minor tooling only. It is +not deemed secure enough to handle any amount of real funds (you have been +warned!). + + +## Installation + +`pyln-proto` is available on `pip`: + +``` +pip install pyln-proto +``` + +Alternatively you can also install the development version to get access to +currently unreleased features by checking out the c-lightning source code and +installing into your python3 environment: + +```bash +git clone https://github.com/ElementsProject/lightning.git +cd lightning/contrib/pyln-proto +python3 setup.py develop +``` + +This will add links to the library into your environment so changing the +checked out source code will also result in the environment picking up these +changes. Notice however that unreleased versions may change API without +warning, so test thoroughly with the released version. diff --git a/contrib/pyln-proto/examples/connect.py b/contrib/pyln-proto/examples/connect.py new file mode 100644 index 000000000..14ecbd767 --- /dev/null +++ b/contrib/pyln-proto/examples/connect.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +"""Simple connect and read test + +Connects to a peer, performs handshake and then just prints all the messages +it gets. + +""" + +from pyln.proto.wire import connect, PrivateKey, PublicKey +from binascii import unhexlify, hexlify + +ls_privkey = PrivateKey(unhexlify( + b'1111111111111111111111111111111111111111111111111111111111111111' +)) +remote_pubkey = PublicKey(unhexlify( + b'03b31e5bbf2cdbe115b485a2b480e70a1ef3951a0dc6df4b1232e0e56f3dce18d6' +)) + +lc = connect(ls_privkey, remote_pubkey, '127.0.0.1', 9375) +lc.send_message(b'\x00\x10\x00\x00\x00\x01\xaa') + +while True: + print(hexlify(lc.read_message()).decode('ASCII')) diff --git a/contrib/pyln-proto/examples/listen.py b/contrib/pyln-proto/examples/listen.py new file mode 100644 index 000000000..10cd5470e --- /dev/null +++ b/contrib/pyln-proto/examples/listen.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""A simple handshake and encryption test. + +This script will listen on port 9736 for incoming Lightning Network protocol +connections, perform the cryptographic handshake, send 10k small pings, and +then exit, closing the connection. This is useful to check the correct +rotation of send- and receive-keys in the implementation. + +""" + + +from pyln.proto.wire import LightningServerSocket, PrivateKey +from binascii import hexlify, unhexlify +import time +import threading + +ls_privkey = PrivateKey(unhexlify( + b'1111111111111111111111111111111111111111111111111111111111111111' +)) +listener = LightningServerSocket(ls_privkey) +print("Node ID: {}".format(ls_privkey.public_key())) + +listener.bind(('0.0.0.0', 9735)) +listener.listen() +c, a = listener.accept() + +c.send_message(b'\x00\x10\x00\x00\x00\x01\xaa') +print(c.read_message()) + +num_pings = 10000 + + +def read_loop(c): + for i in range(num_pings): + print("Recv", i, hexlify(c.read_message())) + + +t = threading.Thread(target=read_loop, args=(c,)) +t.daemon = True +t.start() +for i in range(num_pings): + m = b'\x00\x12\x00\x01\x00\x01\x00' + c.send_message(m) + print("Sent", i, hexlify(m)) + time.sleep(0.01) + +t.join() diff --git a/contrib/pyln-proto/pyln/proto/__init__.py b/contrib/pyln-proto/pyln/proto/__init__.py new file mode 100644 index 000000000..b8023d8bc --- /dev/null +++ b/contrib/pyln-proto/pyln/proto/__init__.py @@ -0,0 +1 @@ +__version__ = '0.0.1' diff --git a/contrib/pyln-proto/pyln/proto/wire.py b/contrib/pyln-proto/pyln/proto/wire.py new file mode 100644 index 000000000..fb7c57c0e --- /dev/null +++ b/contrib/pyln-proto/pyln/proto/wire.py @@ -0,0 +1,395 @@ +from binascii import hexlify +from cryptography.exceptions import InvalidTag +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.hazmat.primitives.ciphers.aead import ChaCha20Poly1305 +from cryptography.hazmat.primitives.kdf.hkdf import HKDF +from cryptography.hazmat.primitives import serialization +from hashlib import sha256 +import coincurve +import os +import socket +import struct +import threading + + +__all__ = [ + 'PrivateKey', + 'PublicKey', + 'Secret', + 'LightningConnection', + 'LightningServerSocket', + 'connect' +] + + +def hkdf(ikm, salt=b"", info=b""): + hkdf = HKDF( + algorithm=hashes.SHA256(), + length=64, + salt=salt, + info=info, + backend=default_backend()) + + return hkdf.derive(ikm) + + +def hkdf_two_keys(ikm, salt): + t = hkdf(ikm, salt) + return t[:32], t[32:] + + +def ecdh(k, rk): + k = coincurve.PrivateKey(secret=k.rawkey) + rk = coincurve.PublicKey(data=rk.serializeCompressed()) + a = k.ecdh(rk.public_key) + return Secret(a) + + +def encryptWithAD(k, n, ad, plaintext): + chacha = ChaCha20Poly1305(k) + return chacha.encrypt(n, plaintext, ad) + + +def decryptWithAD(k, n, ad, ciphertext): + chacha = ChaCha20Poly1305(k) + return chacha.decrypt(n, ciphertext, ad) + + +class PrivateKey(object): + def __init__(self, rawkey): + assert len(rawkey) == 32 and isinstance(rawkey, bytes) + self.rawkey = rawkey + rawkey = int(hexlify(rawkey), base=16) + self.key = ec.derive_private_key(rawkey, ec.SECP256K1(), + default_backend()) + + def serializeCompressed(self): + return self.key.private_bytes(serialization.Encoding.Raw, + serialization.PrivateFormat.Raw, None) + + def public_key(self): + return PublicKey(self.key.public_key()) + + +class Secret(object): + def __init__(self, raw): + assert(len(raw) == 32) + self.raw = raw + + def __str__(self): + return "Secret[0x{}]".format(hexlify(self.raw).decode('ASCII')) + + +class PublicKey(object): + def __init__(self, innerkey): + # We accept either 33-bytes raw keys, or an EC PublicKey as returned + # by cryptography.io + if isinstance(innerkey, bytes): + innerkey = ec.EllipticCurvePublicKey.from_encoded_point( + ec.SECP256K1(), innerkey + ) + + elif not isinstance(innerkey, ec.EllipticCurvePublicKey): + raise ValueError( + "Key must either be bytes or ec.EllipticCurvePublicKey" + ) + self.key = innerkey + + def serializeCompressed(self): + raw = self.key.public_bytes( + serialization.Encoding.X962, + serialization.PublicFormat.CompressedPoint + ) + return raw + + def __str__(self): + return "PublicKey[0x{}]".format( + hexlify(self.serializeCompressed()).decode('ASCII') + ) + + +def Keypair(object): + def __init__(self, priv, pub): + self.priv, self.pub = priv, pub + + +class Sha256Mixer(object): + def __init__(self, base): + self.hash = sha256(base).digest() + + def update(self, data): + h = sha256(self.hash) + h.update(data) + self.hash = h.digest() + return self.hash + + def digest(self): + return self.hash + + def __str__(self): + return "Sha256Mixer[0x{}]".format(hexlify(self.hash).decode('ASCII')) + + +class LightningConnection(object): + def __init__(self, connection, remote_pubkey, local_privkey, is_initiator): + self.connection = connection + self.chaining_key = None + self.handshake_hash = None + self.local_privkey = local_privkey + self.local_pubkey = self.local_privkey.public_key() + self.remote_pubkey = remote_pubkey + self.is_initiator = is_initiator + self.init_handshake() + self.rn, self.sn = 0, 0 + self.send_lock, self.recv_lock = threading.Lock(), threading.Lock() + + @classmethod + def nonce(cls, n): + """Transforms a numeric nonce into a byte formatted one + + Nonce n encoded as 32 zero bits, followed by a little-endian 64-bit + value. Note: this follows the Noise Protocol convention, rather than + our normal endian. + """ + return b'\x00' * 4 + struct.pack("