broken invoice listener on c-lightning and other fixes around wallets.

.env.example

@@ -36,11 +36,11 @@ LNBITS_ADMIN_MACAROON=LNBITS_ADMIN_MACAROON
 LND_GRPC_ENDPOINT=127.0.0.1
 LND_GRPC_PORT=11009
 LND_GRPC_CERT="/home/bob/.config/Zap/lnd/bitcoin/mainnet/wallet-1/data/chain/bitcoin/mainnet/tls.cert"
-LND_ADMIN_MACAROON="/home/bob/.config/Zap/lnd/bitcoin/mainnet/wallet-1/data/chain/bitcoin/mainnet/admin.macaroon"
-LND_INVOICE_MACAROON="/home/bob/.config/Zap/lnd/bitcoin/mainnet/wallet-1/data/chain/bitcoin/mainnet/invoice.macaroon"
+LND_GRPC_ADMIN_MACAROON="/home/bob/.config/Zap/lnd/bitcoin/mainnet/wallet-1/data/chain/bitcoin/mainnet/admin.macaroon"
+LND_GRPC_INVOICE_MACAROON="/home/bob/.config/Zap/lnd/bitcoin/mainnet/wallet-1/data/chain/bitcoin/mainnet/invoice.macaroon"
 
 # LndRestWallet
-LND_REST_ENDPOINT=https://localhost:8080/
+LND_REST_ENDPOINT=https://127.0.0.1:8080/
 LND_REST_CERT="/home/bob/.config/Zap/lnd/bitcoin/mainnet/wallet-1/data/chain/bitcoin/mainnet/tls.cert"
 LND_REST_ADMIN_MACAROON="HEXSTRING"
 LND_REST_INVOICE_MACAROON="HEXSTRING"

docs/devs/installation.md

@@ -34,7 +34,7 @@ You will need to copy `.env.example` to `.env`, then set variables there.
 ![Files](https://i.imgur.com/ri2zOe8.png)
 
 You might also need to install additional packages, depending on the [backend wallet](../guide/wallets.md) you use.
-E.g. when you want to use LND you have to `pipenv run pip install lnd-grpc`.
+E.g. when you want to use LND you have to `pipenv run pip install lndgrpc`.
 
 Take a look at [Polar][polar] for an excellent way of spinning up a Lightning Network dev environment.
 

docs/guide/installation.md

@@ -31,5 +31,5 @@ You might also need to install additional packages, depending on the chosen back
 E.g. when you want to use LND you have to run:
 
 ```sh
-./venv/bin/pip install lnd-grpc
+./venv/bin/pip install lndgrpc
 ```

docs/guide/wallets.md

@@ -29,7 +29,7 @@ Using this wallet requires the installation of the `pylightning` Python package.
 
 ### LND (gRPC)
 
-Using this wallet requires the installation of the `lnd-grpc` Python package.
+Using this wallet requires the installation of the `lndgrpc` Python package.
 
 - `LNBITS_BACKEND_WALLET_CLASS`: **LndWallet**
 - `LND_GRPC_ENDPOINT`: ip_address

lnbits/wallets/clightning.py

@@ -3,7 +3,9 @@ try:
 except ImportError:  # pragma: nocover
     LightningRpc = None
 
+import asyncio
 import random
+import json
 
 from os import getenv
 from typing import Optional, AsyncGenerator
@@ -15,7 +17,8 @@ class CLightningWallet(Wallet):
         if LightningRpc is None:  # pragma: nocover
             raise ImportError("The `pylightning` library must be installed to use `CLightningWallet`.")
 
-        self.ln = LightningRpc(getenv("CLIGHTNING_RPC"))
+        self.rpc = getenv("CLIGHTNING_RPC")
+        self.ln = LightningRpc(self.rpc)
 
         # check description_hash support (could be provided by a plugin)
         self.supports_description_hash = False
@@ -31,8 +34,10 @@ class CLightningWallet(Wallet):
         # check last payindex so we can listen from that point on
         self.last_pay_index = 0
         invoices = self.ln.listinvoices()
-        if len(invoices["invoices"]):
-            self.last_pay_index = invoices["invoices"][-1]["pay_index"]
+        for inv in invoices["invoices"][::-1]:
+            if "pay_index" in inv:
+                self.last_pay_index = inv["pay_index"]
+                break
 
     def create_invoice(
         self, amount: int, memo: Optional[str] = None, description_hash: Optional[bytes] = None
@@ -45,7 +50,8 @@ class CLightningWallet(Wallet):
                 if not self.supports_description_hash:
                     raise Unsupported("description_hash")
 
-                r = self.ln.call("invoicewithdescriptionhash", [msat, label, memo])
+                params = [msat, label, description_hash.hex()]
+                r = self.ln.call("invoicewithdescriptionhash", params)
                 return InvoiceResponse(True, label, r["bolt11"], "")
             else:
                 r = self.ln.invoice(msat, label, memo, exposeprivatechannels=True)
@@ -56,15 +62,14 @@ class CLightningWallet(Wallet):
 
     def pay_invoice(self, bolt11: str) -> PaymentResponse:
         r = self.ln.pay(bolt11)
-        ok, checking_id, fee_msat, error_message = True, r["payment_hash"], r["msatoshi_sent"] - r["msatoshi"], None
-        return PaymentResponse(ok, checking_id, fee_msat, error_message)
+        return PaymentResponse(True, r["payment_hash"], r["msatoshi_sent"] - r["msatoshi"], None)
 
     def get_invoice_status(self, checking_id: str) -> PaymentStatus:
         r = self.ln.listinvoices(checking_id)
         if not r["invoices"]:
             return PaymentStatus(False)
         if r["invoices"][0]["label"] == checking_id:
-            return PaymentStatus(r["pays"][0]["status"] == "paid")
+            return PaymentStatus(r["invoices"][0]["status"] == "paid")
         raise KeyError("supplied an invalid checking_id")
 
     def get_payment_status(self, checking_id: str) -> PaymentStatus:
@@ -81,7 +86,28 @@ class CLightningWallet(Wallet):
         raise KeyError("supplied an invalid checking_id")
 
     async def paid_invoices_stream(self) -> AsyncGenerator[str, None]:
+        reader, writer = await asyncio.open_unix_connection(self.rpc)
+
+        i = 0
         while True:
+            call = json.dumps(
+                {
+                    "method": "waitanyinvoice",
+                    "id": 0,
+                    "params": [self.last_pay_index],
+                }
+            )
+
+            print(call)
+            writer.write(call.encode("ascii"))
+            await writer.drain()
+
+            data = await reader.readuntil(b"\n\n")
+            print(data)
+            paid = json.loads(data.decode("ascii"))
+
             paid = self.ln.waitanyinvoice(self.last_pay_index)
             self.last_pay_index = paid["pay_index"]
             yield paid["label"]
+
+            i += 1

lnbits/wallets/lnbits.py

@@ -1,3 +1,4 @@
+import asyncio
 from os import getenv
 from typing import Optional, Dict, AsyncGenerator
 from requests import get, post
@@ -67,4 +68,5 @@ class LNbitsWallet(Wallet):
 
     async def paid_invoices_stream(self) -> AsyncGenerator[str, None]:
         print("lnbits does not support paid invoices stream yet")
+        await asyncio.sleep(5)
         yield ""

lnbits/wallets/lndgrpc.py

@@ -1,9 +1,12 @@
 try:
-    import lnd_grpc  # type: ignore
+    import lndgrpc  # type: ignore
+    from lndgrpc.common import ln  # type: ignore
 except ImportError:  # pragma: nocover
-    lnd_grpc = None
+    lndgrpc = None
 
+import binascii
 import base64
+import hashlib
 from os import getenv
 from typing import Optional, Dict, AsyncGenerator
 
@@ -28,62 +31,81 @@ def stringify_checking_id(r_hash: bytes) -> str:
 
 class LndWallet(Wallet):
     def __init__(self):
-        if lnd_grpc is None:  # pragma: nocover
-            raise ImportError("The `lnd-grpc` library must be installed to use `LndWallet`.")
+        if lndgrpc is None:  # pragma: nocover
+            raise ImportError("The `lndgrpc` library must be installed to use `LndWallet`.")
 
         endpoint = getenv("LND_GRPC_ENDPOINT")
         endpoint = endpoint[:-1] if endpoint.endswith("/") else endpoint
         port = getenv("LND_GRPC_PORT")
         cert = getenv("LND_GRPC_CERT") or getenv("LND_CERT")
-        auth_admin = getenv("LND_ADMIN_MACAROON")
-        auth_invoices = getenv("LND_INVOICE_MACAROON")
+        auth_admin = getenv("LND_GRPC_ADMIN_MACAROON") or getenv("LND_ADMIN_MACAROON")
+        auth_invoices = getenv("LND_GRPC_INVOICE_MACAROON") or getenv("LND_INVOICE_MACAROON")
         network = getenv("LND_GRPC_NETWORK", "mainnet")
 
-        self.admin_rpc = lnd_grpc.Client(
-            lnd_dir=None,
-            macaroon_path=auth_admin,
-            tls_cert_path=cert,
+        self.admin_rpc = lndgrpc.LNDClient(
+            endpoint + ":" + port,
+            cert_filepath=cert,
             network=network,
-            grpc_host=endpoint,
-            grpc_port=port,
+            macaroon_filepath=auth_admin,
         )
 
-        self.invoices_rpc = lnd_grpc.Client(
-            lnd_dir=None,
-            macaroon_path=auth_invoices,
-            tls_cert_path=cert,
+        self.invoices_rpc = lndgrpc.LNDClient(
+            endpoint + ":" + port,
+            cert_filepath=cert,
             network=network,
-            grpc_host=endpoint,
-            grpc_port=port,
+            macaroon_filepath=auth_invoices,
+        )
+
+        self.async_rpc = lndgrpc.AsyncLNDClient(
+            endpoint + ":" + port,
+            cert_filepath=cert,
+            network=network,
+            macaroon_filepath=auth_invoices,
         )
 
     def create_invoice(
         self, amount: int, memo: Optional[str] = None, description_hash: Optional[bytes] = None
     ) -> InvoiceResponse:
         params: Dict = {"value": amount, "expiry": 600, "private": True}
+
         if description_hash:
             params["description_hash"] = description_hash  # as bytes directly
         else:
             params["memo"] = memo or ""
-        resp = self.invoices_rpc.add_invoice(**params)
+
+        try:
+            req = ln.Invoice(**params)
+            resp = self.invoices_rpc._ln_stub.AddInvoice(req)
+        except Exception as exc:
+            error_message = str(exc)
+            return InvoiceResponse(False, None, None, error_message)
 
         checking_id = stringify_checking_id(resp.r_hash)
         payment_request = str(resp.payment_request)
         return InvoiceResponse(True, checking_id, payment_request, None)
 
     def pay_invoice(self, bolt11: str) -> PaymentResponse:
-        resp = self.admin_rpc.pay_invoice(payment_request=bolt11)
+        resp = self.admin_rpc.send_payment(payment_request=bolt11)
 
         if resp.payment_error:
             return PaymentResponse(False, "", 0, resp.payment_error)
 
-        checking_id = stringify_checking_id(resp.payment_hash)
+        r_hash = hashlib.sha256(resp.payment_preimage).digest()
+        checking_id = stringify_checking_id(r_hash)
         return PaymentResponse(True, checking_id, 0, None)
 
     def get_invoice_status(self, checking_id: str) -> PaymentStatus:
+        try:
             r_hash = parse_checking_id(checking_id)
-        for _response in self.invoices_rpc.subscribe_single_invoice(r_hash):
-            if _response.state == 1:
+            if len(r_hash) != 32:
+                raise binascii.Error
+        except binascii.Error:
+            # this may happen if we switch between backend wallets
+            # that use different checking_id formats
+            return PaymentStatus(None)
+
+        resp = self.invoices_rpc.lookup_invoice(r_hash.hex())
+        if resp.settled:
             return PaymentStatus(True)
 
         return PaymentStatus(None)
@@ -92,7 +114,9 @@ class LndWallet(Wallet):
         return PaymentStatus(True)
 
     async def paid_invoices_stream(self) -> AsyncGenerator[str, None]:
-        for paid in self.invoices_rpc.SubscribeInvoices():
-            print("PAID", paid)
-            checking_id = stringify_checking_id(paid.r_hash)
+        async for inv in self.async_rpc._ln_stub.SubscribeInvoices(ln.InvoiceSubscription()):
+            if not inv.settled:
+                continue
+
+            checking_id = stringify_checking_id(inv.r_hash)
             yield checking_id

lnbits/wallets/lndrest.py

@@ -15,8 +15,12 @@ class LndRestWallet(Wallet):
 
         endpoint = getenv("LND_REST_ENDPOINT")
         self.endpoint = endpoint[:-1] if endpoint.endswith("/") else endpoint
-        self.auth_admin = {"Grpc-Metadata-macaroon": getenv("LND_REST_ADMIN_MACAROON")}
-        self.auth_invoice = {"Grpc-Metadata-macaroon": getenv("LND_REST_INVOICE_MACAROON")}
+        self.auth_admin = {
+            "Grpc-Metadata-macaroon": getenv("LND_ADMIN_MACAROON") or getenv("LND_REST_ADMIN_MACAROON"),
+        }
+        self.auth_invoice = {
+            "Grpc-Metadata-macaroon": getenv("LND_INVOICE_MACAROON") or getenv("LND_REST_INVOICE_MACAROON")
+        }
         self.auth_cert = getenv("LND_REST_CERT")
 
     def create_invoice(
@@ -111,17 +115,13 @@ class LndRestWallet(Wallet):
 
         async with httpx.AsyncClient(timeout=None, headers=self.auth_admin, verify=self.auth_cert) as client:
             async with client.stream("GET", url) as r:
-                print("ok")
-                print(r)
-                print(r.is_error)
-                print("ok")
                 async for line in r.aiter_lines():
-                    print("line", line)
                     try:
-                        event = json.loads(line)["result"]
-                        print(event)
+                        inv = json.loads(line)["result"]
+                        if not inv["settled"]:
+                            continue
                     except:
                         continue
 
-                    payment_hash = bolt11.decode(event["payment_request"]).payment_hash
+                    payment_hash = base64.b64decode(inv["r_hash"]).hex()
                     yield payment_hash

lnbits/wallets/lntxbot.py

@@ -1,3 +1,4 @@
+import asyncio
 from os import getenv
 from typing import Optional, Dict, AsyncGenerator
 from requests import post
@@ -78,4 +79,5 @@ class LntxbotWallet(Wallet):
 
     async def paid_invoices_stream(self) -> AsyncGenerator[str, None]:
         print("lntxbot does not support paid invoices stream yet")
+        await asyncio.sleep(5)
         yield ""