From e74cf33f9092858c7d70e7627347ba344388a4a2 Mon Sep 17 00:00:00 2001 From: fiatjaf <fiatjaf@alhur.es> Date: Sat, 3 Oct 2020 17:27:55 -0300 Subject: [PATCH] broken invoice listener on c-lightning and other fixes around wallets. --- .env.example | 6 +-- docs/devs/installation.md | 2 +- docs/guide/installation.md | 2 +- docs/guide/wallets.md | 2 +- lnbits/wallets/clightning.py | 40 ++++++++++++++---- lnbits/wallets/lnbits.py | 2 + lnbits/wallets/lndgrpc.py | 80 +++++++++++++++++++++++------------- lnbits/wallets/lndrest.py | 20 ++++----- lnbits/wallets/lntxbot.py | 2 + 9 files changed, 105 insertions(+), 51 deletions(-) diff --git a/.env.example b/.env.example index c67baa7..bfaccf6 100644 --- a/.env.example +++ b/.env.example @@ -36,11 +36,11 @@ LNBITS_ADMIN_MACAROON=LNBITS_ADMIN_MACAROON LND_GRPC_ENDPOINT=127.0.0.1 LND_GRPC_PORT=11009 LND_GRPC_CERT="/home/bob/.config/Zap/lnd/bitcoin/mainnet/wallet-1/data/chain/bitcoin/mainnet/tls.cert" -LND_ADMIN_MACAROON="/home/bob/.config/Zap/lnd/bitcoin/mainnet/wallet-1/data/chain/bitcoin/mainnet/admin.macaroon" -LND_INVOICE_MACAROON="/home/bob/.config/Zap/lnd/bitcoin/mainnet/wallet-1/data/chain/bitcoin/mainnet/invoice.macaroon" +LND_GRPC_ADMIN_MACAROON="/home/bob/.config/Zap/lnd/bitcoin/mainnet/wallet-1/data/chain/bitcoin/mainnet/admin.macaroon" +LND_GRPC_INVOICE_MACAROON="/home/bob/.config/Zap/lnd/bitcoin/mainnet/wallet-1/data/chain/bitcoin/mainnet/invoice.macaroon" # LndRestWallet -LND_REST_ENDPOINT=https://localhost:8080/ +LND_REST_ENDPOINT=https://127.0.0.1:8080/ LND_REST_CERT="/home/bob/.config/Zap/lnd/bitcoin/mainnet/wallet-1/data/chain/bitcoin/mainnet/tls.cert" LND_REST_ADMIN_MACAROON="HEXSTRING" LND_REST_INVOICE_MACAROON="HEXSTRING" diff --git a/docs/devs/installation.md b/docs/devs/installation.md index a48365a..27efe13 100644 --- a/docs/devs/installation.md +++ b/docs/devs/installation.md @@ -34,7 +34,7 @@ You will need to copy `.env.example` to `.env`, then set variables there.  You might also need to install additional packages, depending on the [backend wallet](../guide/wallets.md) you use. -E.g. when you want to use LND you have to `pipenv run pip install lnd-grpc`. +E.g. when you want to use LND you have to `pipenv run pip install lndgrpc`. Take a look at [Polar][polar] for an excellent way of spinning up a Lightning Network dev environment. diff --git a/docs/guide/installation.md b/docs/guide/installation.md index b7adb79..f539572 100644 --- a/docs/guide/installation.md +++ b/docs/guide/installation.md @@ -31,5 +31,5 @@ You might also need to install additional packages, depending on the chosen back E.g. when you want to use LND you have to run: ```sh -./venv/bin/pip install lnd-grpc +./venv/bin/pip install lndgrpc ``` diff --git a/docs/guide/wallets.md b/docs/guide/wallets.md index 986711f..1d7c11d 100644 --- a/docs/guide/wallets.md +++ b/docs/guide/wallets.md @@ -29,7 +29,7 @@ Using this wallet requires the installation of the `pylightning` Python package. ### LND (gRPC) -Using this wallet requires the installation of the `lnd-grpc` Python package. +Using this wallet requires the installation of the `lndgrpc` Python package. - `LNBITS_BACKEND_WALLET_CLASS`: **LndWallet** - `LND_GRPC_ENDPOINT`: ip_address diff --git a/lnbits/wallets/clightning.py b/lnbits/wallets/clightning.py index 37f9021..14dae31 100644 --- a/lnbits/wallets/clightning.py +++ b/lnbits/wallets/clightning.py @@ -3,7 +3,9 @@ try: except ImportError: # pragma: nocover LightningRpc = None +import asyncio import random +import json from os import getenv from typing import Optional, AsyncGenerator @@ -15,7 +17,8 @@ class CLightningWallet(Wallet): if LightningRpc is None: # pragma: nocover raise ImportError("The `pylightning` library must be installed to use `CLightningWallet`.") - self.ln = LightningRpc(getenv("CLIGHTNING_RPC")) + self.rpc = getenv("CLIGHTNING_RPC") + self.ln = LightningRpc(self.rpc) # check description_hash support (could be provided by a plugin) self.supports_description_hash = False @@ -31,8 +34,10 @@ class CLightningWallet(Wallet): # check last payindex so we can listen from that point on self.last_pay_index = 0 invoices = self.ln.listinvoices() - if len(invoices["invoices"]): - self.last_pay_index = invoices["invoices"][-1]["pay_index"] + for inv in invoices["invoices"][::-1]: + if "pay_index" in inv: + self.last_pay_index = inv["pay_index"] + break def create_invoice( self, amount: int, memo: Optional[str] = None, description_hash: Optional[bytes] = None @@ -45,7 +50,8 @@ class CLightningWallet(Wallet): if not self.supports_description_hash: raise Unsupported("description_hash") - r = self.ln.call("invoicewithdescriptionhash", [msat, label, memo]) + params = [msat, label, description_hash.hex()] + r = self.ln.call("invoicewithdescriptionhash", params) return InvoiceResponse(True, label, r["bolt11"], "") else: r = self.ln.invoice(msat, label, memo, exposeprivatechannels=True) @@ -56,15 +62,14 @@ class CLightningWallet(Wallet): def pay_invoice(self, bolt11: str) -> PaymentResponse: r = self.ln.pay(bolt11) - ok, checking_id, fee_msat, error_message = True, r["payment_hash"], r["msatoshi_sent"] - r["msatoshi"], None - return PaymentResponse(ok, checking_id, fee_msat, error_message) + return PaymentResponse(True, r["payment_hash"], r["msatoshi_sent"] - r["msatoshi"], None) def get_invoice_status(self, checking_id: str) -> PaymentStatus: r = self.ln.listinvoices(checking_id) if not r["invoices"]: return PaymentStatus(False) if r["invoices"][0]["label"] == checking_id: - return PaymentStatus(r["pays"][0]["status"] == "paid") + return PaymentStatus(r["invoices"][0]["status"] == "paid") raise KeyError("supplied an invalid checking_id") def get_payment_status(self, checking_id: str) -> PaymentStatus: @@ -81,7 +86,28 @@ class CLightningWallet(Wallet): raise KeyError("supplied an invalid checking_id") async def paid_invoices_stream(self) -> AsyncGenerator[str, None]: + reader, writer = await asyncio.open_unix_connection(self.rpc) + + i = 0 while True: + call = json.dumps( + { + "method": "waitanyinvoice", + "id": 0, + "params": [self.last_pay_index], + } + ) + + print(call) + writer.write(call.encode("ascii")) + await writer.drain() + + data = await reader.readuntil(b"\n\n") + print(data) + paid = json.loads(data.decode("ascii")) + paid = self.ln.waitanyinvoice(self.last_pay_index) self.last_pay_index = paid["pay_index"] yield paid["label"] + + i += 1 diff --git a/lnbits/wallets/lnbits.py b/lnbits/wallets/lnbits.py index 2e405ee..fc348bd 100644 --- a/lnbits/wallets/lnbits.py +++ b/lnbits/wallets/lnbits.py @@ -1,3 +1,4 @@ +import asyncio from os import getenv from typing import Optional, Dict, AsyncGenerator from requests import get, post @@ -67,4 +68,5 @@ class LNbitsWallet(Wallet): async def paid_invoices_stream(self) -> AsyncGenerator[str, None]: print("lnbits does not support paid invoices stream yet") + await asyncio.sleep(5) yield "" diff --git a/lnbits/wallets/lndgrpc.py b/lnbits/wallets/lndgrpc.py index eebb754..99467bb 100644 --- a/lnbits/wallets/lndgrpc.py +++ b/lnbits/wallets/lndgrpc.py @@ -1,9 +1,12 @@ try: - import lnd_grpc # type: ignore + import lndgrpc # type: ignore + from lndgrpc.common import ln # type: ignore except ImportError: # pragma: nocover - lnd_grpc = None + lndgrpc = None +import binascii import base64 +import hashlib from os import getenv from typing import Optional, Dict, AsyncGenerator @@ -28,63 +31,82 @@ def stringify_checking_id(r_hash: bytes) -> str: class LndWallet(Wallet): def __init__(self): - if lnd_grpc is None: # pragma: nocover - raise ImportError("The `lnd-grpc` library must be installed to use `LndWallet`.") + if lndgrpc is None: # pragma: nocover + raise ImportError("The `lndgrpc` library must be installed to use `LndWallet`.") endpoint = getenv("LND_GRPC_ENDPOINT") endpoint = endpoint[:-1] if endpoint.endswith("/") else endpoint port = getenv("LND_GRPC_PORT") cert = getenv("LND_GRPC_CERT") or getenv("LND_CERT") - auth_admin = getenv("LND_ADMIN_MACAROON") - auth_invoices = getenv("LND_INVOICE_MACAROON") + auth_admin = getenv("LND_GRPC_ADMIN_MACAROON") or getenv("LND_ADMIN_MACAROON") + auth_invoices = getenv("LND_GRPC_INVOICE_MACAROON") or getenv("LND_INVOICE_MACAROON") network = getenv("LND_GRPC_NETWORK", "mainnet") - self.admin_rpc = lnd_grpc.Client( - lnd_dir=None, - macaroon_path=auth_admin, - tls_cert_path=cert, + self.admin_rpc = lndgrpc.LNDClient( + endpoint + ":" + port, + cert_filepath=cert, network=network, - grpc_host=endpoint, - grpc_port=port, + macaroon_filepath=auth_admin, ) - self.invoices_rpc = lnd_grpc.Client( - lnd_dir=None, - macaroon_path=auth_invoices, - tls_cert_path=cert, + self.invoices_rpc = lndgrpc.LNDClient( + endpoint + ":" + port, + cert_filepath=cert, network=network, - grpc_host=endpoint, - grpc_port=port, + macaroon_filepath=auth_invoices, + ) + + self.async_rpc = lndgrpc.AsyncLNDClient( + endpoint + ":" + port, + cert_filepath=cert, + network=network, + macaroon_filepath=auth_invoices, ) def create_invoice( self, amount: int, memo: Optional[str] = None, description_hash: Optional[bytes] = None ) -> InvoiceResponse: params: Dict = {"value": amount, "expiry": 600, "private": True} + if description_hash: params["description_hash"] = description_hash # as bytes directly else: params["memo"] = memo or "" - resp = self.invoices_rpc.add_invoice(**params) + + try: + req = ln.Invoice(**params) + resp = self.invoices_rpc._ln_stub.AddInvoice(req) + except Exception as exc: + error_message = str(exc) + return InvoiceResponse(False, None, None, error_message) checking_id = stringify_checking_id(resp.r_hash) payment_request = str(resp.payment_request) return InvoiceResponse(True, checking_id, payment_request, None) def pay_invoice(self, bolt11: str) -> PaymentResponse: - resp = self.admin_rpc.pay_invoice(payment_request=bolt11) + resp = self.admin_rpc.send_payment(payment_request=bolt11) if resp.payment_error: return PaymentResponse(False, "", 0, resp.payment_error) - checking_id = stringify_checking_id(resp.payment_hash) + r_hash = hashlib.sha256(resp.payment_preimage).digest() + checking_id = stringify_checking_id(r_hash) return PaymentResponse(True, checking_id, 0, None) def get_invoice_status(self, checking_id: str) -> PaymentStatus: - r_hash = parse_checking_id(checking_id) - for _response in self.invoices_rpc.subscribe_single_invoice(r_hash): - if _response.state == 1: - return PaymentStatus(True) + try: + r_hash = parse_checking_id(checking_id) + if len(r_hash) != 32: + raise binascii.Error + except binascii.Error: + # this may happen if we switch between backend wallets + # that use different checking_id formats + return PaymentStatus(None) + + resp = self.invoices_rpc.lookup_invoice(r_hash.hex()) + if resp.settled: + return PaymentStatus(True) return PaymentStatus(None) @@ -92,7 +114,9 @@ class LndWallet(Wallet): return PaymentStatus(True) async def paid_invoices_stream(self) -> AsyncGenerator[str, None]: - for paid in self.invoices_rpc.SubscribeInvoices(): - print("PAID", paid) - checking_id = stringify_checking_id(paid.r_hash) + async for inv in self.async_rpc._ln_stub.SubscribeInvoices(ln.InvoiceSubscription()): + if not inv.settled: + continue + + checking_id = stringify_checking_id(inv.r_hash) yield checking_id diff --git a/lnbits/wallets/lndrest.py b/lnbits/wallets/lndrest.py index ef45a8f..64001f3 100644 --- a/lnbits/wallets/lndrest.py +++ b/lnbits/wallets/lndrest.py @@ -15,8 +15,12 @@ class LndRestWallet(Wallet): endpoint = getenv("LND_REST_ENDPOINT") self.endpoint = endpoint[:-1] if endpoint.endswith("/") else endpoint - self.auth_admin = {"Grpc-Metadata-macaroon": getenv("LND_REST_ADMIN_MACAROON")} - self.auth_invoice = {"Grpc-Metadata-macaroon": getenv("LND_REST_INVOICE_MACAROON")} + self.auth_admin = { + "Grpc-Metadata-macaroon": getenv("LND_ADMIN_MACAROON") or getenv("LND_REST_ADMIN_MACAROON"), + } + self.auth_invoice = { + "Grpc-Metadata-macaroon": getenv("LND_INVOICE_MACAROON") or getenv("LND_REST_INVOICE_MACAROON") + } self.auth_cert = getenv("LND_REST_CERT") def create_invoice( @@ -111,17 +115,13 @@ class LndRestWallet(Wallet): async with httpx.AsyncClient(timeout=None, headers=self.auth_admin, verify=self.auth_cert) as client: async with client.stream("GET", url) as r: - print("ok") - print(r) - print(r.is_error) - print("ok") async for line in r.aiter_lines(): - print("line", line) try: - event = json.loads(line)["result"] - print(event) + inv = json.loads(line)["result"] + if not inv["settled"]: + continue except: continue - payment_hash = bolt11.decode(event["payment_request"]).payment_hash + payment_hash = base64.b64decode(inv["r_hash"]).hex() yield payment_hash diff --git a/lnbits/wallets/lntxbot.py b/lnbits/wallets/lntxbot.py index a8e7527..45eb854 100644 --- a/lnbits/wallets/lntxbot.py +++ b/lnbits/wallets/lntxbot.py @@ -1,3 +1,4 @@ +import asyncio from os import getenv from typing import Optional, Dict, AsyncGenerator from requests import post @@ -78,4 +79,5 @@ class LntxbotWallet(Wallet): async def paid_invoices_stream(self) -> AsyncGenerator[str, None]: print("lntxbot does not support paid invoices stream yet") + await asyncio.sleep(5) yield ""