lightning:

* store invoices for both directions
* do not store lightning_payments_inflight, lightning_payments_completed in lnworker
* payment history is returned by get_payments method of LNChannel
* command line: lightning history, lightning_invoices
* re-enable push_msat

electrum/commands.py

@@ -47,6 +47,7 @@ from .wallet import Abstract_Wallet, create_new_wallet, restore_wallet_from_text
 from .address_synchronizer import TX_HEIGHT_LOCAL
 from .import lightning
 from .mnemonic import Mnemonic
+from .lnutil import SENT, RECEIVED
 
 if TYPE_CHECKING:
     from .network import Network
@@ -108,6 +109,8 @@ class Commands:
         self.wallet = wallet
         self.network = network
         self._callback = callback
+        if self.wallet:
+            self.lnworker = self.wallet.lnworker
 
     def _run(self, method, args, password_getter, **kwargs):
         """This wrapper is called from the Qt python console."""
@@ -766,33 +769,33 @@ class Commands:
     # lightning network commands
     @command('wpn')
     def open_channel(self, connection_string, amount, channel_push=0, password=None):
-        return self.wallet.lnworker.open_channel(connection_string, satoshis(amount), satoshis(channel_push), password)
+        return self.lnworker.open_channel(connection_string, satoshis(amount), satoshis(channel_push), password)
 
     @command('wn')
     def reestablish_channel(self):
-        self.wallet.lnworker.reestablish_channel()
+        self.lnworker.reestablish_channel()
 
     @command('wn')
     def lnpay(self, invoice):
-        addr, peer, f = self.wallet.lnworker.pay(invoice)
+        addr, peer, f = self.lnworker.pay(invoice)
         return f.result()
 
     @command('wn')
     def addinvoice(self, requested_amount, message):
         # using requested_amount because it is documented in param_descriptions
-        return self.wallet.lnworker.add_invoice(satoshis(requested_amount), message)
+        return self.lnworker.add_invoice(satoshis(requested_amount), message)
 
     @command('wn')
     def nodeid(self):
-        return bh2u(self.wallet.lnworker.node_keypair.pubkey)
+        return bh2u(self.lnworker.node_keypair.pubkey)
 
     @command('w')
     def listchannels(self):
-        return list(self.wallet.lnworker.list_channels())
+        return list(self.lnworker.list_channels())
 
     @command('wn')
     def dumpgraph(self):
-        return list(map(bh2u, self.wallet.lnworker.channel_db.nodes.keys()))
+        return list(map(bh2u, self.lnworker.channel_db.nodes.keys()))
 
     @command('n')
     def inject_fees(self, fees):
@@ -805,47 +808,35 @@ class Commands:
         self.network.path_finder.blacklist.clear()
 
     @command('w')
-    def listinvoices(self):
-        report = self.wallet.lnworker._list_invoices()
-        return '\n'.join(self._format_ln_invoices(report))
-
-    def _format_ln_invoices(self, report):
-        from .lnutil import SENT
-        if report['settled']:
-            yield 'Settled invoices:'
-            yield '-----------------'
-            for date, direction, htlc, preimage in sorted(report['settled']):
-                # astimezone converts to local time
-                # replace removes the tz info since we don't need to display it
-                yield 'Paid at: ' + date.astimezone().replace(tzinfo=None).isoformat(sep=' ', timespec='minutes')
-                yield 'We paid' if direction == SENT else 'They paid'
-                yield str(htlc)
-                yield 'Preimage: ' + (bh2u(preimage) if preimage else 'Not available') # if delete_invoice was called
-                yield ''
-        if report['unsettled']:
-            yield 'Your unsettled invoices:'
-            yield '------------------------'
-            for addr, preimage, pay_req in report['unsettled']:
-                yield pay_req
-                yield str(addr)
-                yield 'Preimage: ' + bh2u(preimage)
-                yield ''
-        if report['inflight']:
-            yield 'Outgoing payments in progress:'
-            yield '------------------------------'
-            for addr, htlc, direction in report['inflight']:
-                yield str(addr)
-                yield str(htlc)
-                yield ''
+    def lightning_invoices(self):
+        from .util import pr_tooltips
+        out = []
+        for payment_hash, (preimage, pay_req, direction, pay_timestamp) in self.lnworker.invoices.items():
+            status = pr_tooltips[self.lnworker.get_invoice_status(payment_hash)]
+            out.append({'payment_hash':payment_hash, 'invoice':pay_req, 'preimage':preimage, 'status':status, 'direction':direction})
+        return out
 
+    @command('w')
+    def lightning_history(self):
+        out = []
+        for chan_id, htlc, direction, status in self.lnworker.get_payments().values():
+            item = {
+                'direction': 'sent' if direction == SENT else 'received',
+                'status':status,
+                'amout_msat':htlc.amount_msat,
+                'payment_hash':bh2u(htlc.payment_hash),
+                'chan_id':bh2u(chan_id),
+                'htlc_id':htlc.htlc_id,
+                'cltv_expiry':htlc.cltv_expiry
+            }
+            out.append(item)
+        return out
 
     @command('wn')
     def closechannel(self, channel_point, force=False):
         chan_id = bytes(reversed(bfh(channel_point)))
-        if force:
-            return self.network.run_from_another_thread(self.wallet.lnworker.force_close_channel(chan_id))
-        else:
-            return self.network.run_from_another_thread(self.wallet.lnworker.close_channel(chan_id))
+        coro = self.lnworker.force_close_channel(chan_id) if force else self.lnworker.force_close_channel(chan_id)
+        return self.network.run_from_another_thread(coro)
 
 def eval_bool(x: str) -> bool:
     if x == 'false': return False

electrum/gui/qt/channel_details.py

@@ -56,11 +56,8 @@ class ChannelDetailsDialog(QtWidgets.QDialog):
         parentItem = model.invisibleRootItem()
         folder_types = {'settled': _('Fulfilled HTLCs'), 'inflight': _('HTLCs in current commitment transaction')}
         self.folders = {}
-
         self.keyname_rows = {}
 
-        invoices = dict(self.window.wallet.lnworker.invoices)
-
         for keyname, i in folder_types.items():
             myFont=QtGui.QFont()
             myFont.setBold(True)
@@ -70,21 +67,24 @@ class ChannelDetailsDialog(QtWidgets.QDialog):
             self.folders[keyname] = folder
             mapping = {}
             num = 0
-            if keyname == 'inflight':
-                for lnaddr, i, direction in htlcs[keyname]:
+
+        invoices = dict(self.window.wallet.lnworker.invoices)
+        for pay_hash, item in htlcs.items():
+            chan_id, i, direction, status = item
+            if pay_hash in invoices:
+                preimage, invoice, direction, timestamp = invoices[pay_hash]
+                lnaddr = lndecode(invoice)
+            if status == 'inflight':
                 it = self.make_inflight(lnaddr, i, direction)
-                    self.folders[keyname].appendRow(it)
+                self.folders['inflight'].appendRow(it)
                 mapping[i.payment_hash] = num
                 num += 1
-            elif keyname == 'settled':
-                for date, direction, i, preimage in htlcs[keyname]:
+            elif status == 'settled':
                 it = self.make_htlc_item(i, direction)
-                    hex_pay_hash = bh2u(i.payment_hash)
-                    if hex_pay_hash in invoices:
                 # if we made the invoice and still have it, we can show more info
-                        invoice = invoices[hex_pay_hash][1]
+                if pay_hash in invoices:
                     self.append_lnaddr(it, lndecode(invoice))
-                    self.folders[keyname].appendRow(it)
+                self.folders['settled'].appendRow(it)
                 mapping[i.payment_hash] = num
                 num += 1
 
@@ -171,8 +171,8 @@ class ChannelDetailsDialog(QtWidgets.QDialog):
         # add htlc tree view to vbox (wouldn't scale correctly in QFormLayout)
         form_layout.addRow(_('Payments (HTLCs):'), None)
         w = QtWidgets.QTreeView(self)
-        htlcs = window.wallet.lnworker._list_invoices(chan_id)
-        w.setModel(self.make_model(htlcs))
+        htlc_dict = chan.get_payments()
+        w.setModel(self.make_model(htlc_dict))
         w.header().setSectionResizeMode(0, QtWidgets.QHeaderView.ResizeToContents)
         vbox.addWidget(w)
 

electrum/gui/qt/main_window.py

@@ -62,7 +62,7 @@ from electrum.util import (format_time, format_satoshis, format_fee_satoshis,
                            UnknownBaseUnit, DECIMAL_POINT_DEFAULT, UserFacingException,
                            get_new_wallet_name, send_exception_to_crash_reporter,
                            InvalidBitcoinURI, InvoiceError)
-from electrum.lnutil import PaymentFailure
+from electrum.lnutil import PaymentFailure, SENT, RECEIVED
 from electrum.transaction import Transaction, TxOutput
 from electrum.address_synchronizer import AddTransactionException
 from electrum.wallet import (Multisig_Wallet, CannotBumpFee, Abstract_Wallet,
@@ -1941,6 +1941,8 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
         #self.amount_e.textEdited.emit("")
         self.payto_e.is_lightning = True
         self.show_send_tab_onchain_fees(False)
+        # save
+        self.wallet.lnworker.save_invoice(None, invoice, SENT)
 
     def show_send_tab_onchain_fees(self, b: bool):
         self.feecontrol_fields.setVisible(b)

electrum/gui/qt/request_list.py

@@ -31,7 +31,8 @@ from PyQt5.QtCore import Qt, QItemSelectionModel
 
 from electrum.i18n import _
 from electrum.util import format_time, age
-from electrum.util import PR_UNPAID, PR_EXPIRED, PR_PAID, PR_UNKNOWN, PR_INFLIGHT
+from electrum.util import PR_UNPAID, PR_EXPIRED, PR_PAID, PR_UNKNOWN, PR_INFLIGHT, pr_tooltips
+from electrum.lnutil import SENT, RECEIVED
 from electrum.plugin import run_hook
 from electrum.wallet import InternalAddressCorruption
 from electrum.bitcoin import COIN
@@ -95,7 +96,7 @@ class RequestList(MyTreeView):
                 return
             req = self.parent.get_request_URI(key)
         elif request_type == REQUEST_TYPE_LN:
-            preimage, req = self.wallet.lnworker.invoices.get(key, (None, None))
+            preimage, req, direction, pay_timestamp = self.wallet.lnworker.invoices.get(key, (None, None, None))
             if req is None:
                 self.update()
                 return
@@ -145,7 +146,9 @@ class RequestList(MyTreeView):
         self.filter()
         # lightning
         lnworker = self.wallet.lnworker
-        for key, (preimage_hex, invoice) in lnworker.invoices.items():
+        for key, (preimage_hex, invoice, direction, pay_timestamp) in lnworker.invoices.items():
+            if direction == SENT:
+                continue
             status = lnworker.get_invoice_status(key)
             lnaddr = lndecode(invoice, expected_hrp=constants.net.SEGWIT_HRP)
             amount_sat = lnaddr.amount*COIN if lnaddr.amount else None
@@ -181,7 +184,7 @@ class RequestList(MyTreeView):
         if request_type == REQUEST_TYPE_BITCOIN:
             req = self.wallet.receive_requests.get(addr)
         elif request_type == REQUEST_TYPE_LN:
-            preimage, req = self.wallet.lnworker.invoices.get(addr)
+            preimage, req, direction, pay_timestamp = self.wallet.lnworker.invoices.get(addr)
         if req is None:
             self.update()
             return

electrum/gui/qt/util.py

@@ -47,12 +47,6 @@ pr_icons = {
     PR_INFLIGHT:"lightning.png",
 }
 
-pr_tooltips = {
-    PR_UNPAID:_('Pending'),
-    PR_PAID:_('Paid'),
-    PR_EXPIRED:_('Expired'),
-    PR_INFLIGHT:_('Inflight')
-}
 
 expiration_values = [
     (_('1 hour'), 60*60),

electrum/lnbase.py

@@ -420,7 +420,7 @@ class Peer(PrintError):
     @log_exceptions
     async def channel_establishment_flow(self, password: Optional[str], funding_sat: int,
                                          push_msat: int, temp_channel_id: bytes) -> Channel:
-        assert push_msat == 0, "push_msat not supported currently"
+        #assert push_msat == 0, "push_msat not supported currently"
         wallet = self.lnworker.wallet
         # dry run creating funding tx to see if we even have enough funds
         funding_tx_test = wallet.mktx([TxOutput(bitcoin.TYPE_ADDRESS, wallet.dummy_address(), funding_sat)],
@@ -549,7 +549,7 @@ class Peer(PrintError):
             raise Exception('wrong chain_hash')
         funding_sat = int.from_bytes(payload['funding_satoshis'], 'big')
         push_msat = int.from_bytes(payload['push_msat'], 'big')
-        assert push_msat == 0, "push_msat not supported currently"
+        #assert push_msat == 0, "push_msat not supported currently"
         feerate = int.from_bytes(payload['feerate_per_kw'], 'big')
 
         temp_chan_id = payload['temporary_channel_id']

electrum/lnchan.py

@@ -171,6 +171,17 @@ class Channel(PrintError):
         self.local_commitment = None
         self.remote_commitment = None
 
+    def get_payments(self):
+        out = {}
+        for subject in LOCAL, REMOTE:
+            log = self.hm.log[subject]
+            for htlc_id, htlc in log.get('adds', {}).items():
+                rhash = bh2u(htlc.payment_hash)
+                status = 'settled' if htlc_id in log.get('settles',{}) else 'inflight'
+                direction = SENT if subject is LOCAL else RECEIVED
+                out[rhash] = (self.channel_id, htlc, direction, status)
+        return out
+
     def set_local_commitment(self, ctx):
         ctn = extract_ctn_from_tx_and_chan(ctx, self)
         assert self.signature_fits(ctx), (self.log[LOCAL])

electrum/lnworker.py

@@ -19,6 +19,7 @@ import dns.exception
 
 from . import constants
 from . import keystore
+from .util import PR_UNPAID, PR_EXPIRED, PR_PAID, PR_UNKNOWN, PR_INFLIGHT
 from .keystore import BIP32_KeyStore
 from .bitcoin import COIN
 from .transaction import Transaction
@@ -66,10 +67,7 @@ class LNWorker(PrintError):
 
     def __init__(self, wallet: 'Abstract_Wallet'):
         self.wallet = wallet
-        # invoices we are currently trying to pay (might be pending HTLCs on a commitment transaction)
-        self.invoices = self.wallet.storage.get('lightning_invoices', {})  # type: Dict[str, Tuple[str,str]]  # RHASH -> (preimage, invoice)
-        self.inflight = self.wallet.storage.get('lightning_payments_inflight', {}) # type: Dict[bytes, Tuple[str, Optional[int], str]]
-        self.completed = self.wallet.storage.get('lightning_payments_completed', {})
+        self.invoices = self.wallet.storage.get('lightning_invoices', {})  # type: Dict[str, Tuple[str,str]]  # RHASH -> (preimage, invoice, direction, pay_timestamp)
         self.sweep_address = wallet.get_receiving_address()
         self.lock = threading.RLock()
         self.ln_keystore = self._read_ln_keystore()
@@ -122,73 +120,34 @@ class LNWorker(PrintError):
             self.wallet.storage.write()
             self.print_error('saved lightning gossip timestamp')
 
-    def payment_completed(self, chan, direction, htlc, preimage):
-        assert type(direction) is Direction
-        key = bh2u(htlc.payment_hash)
+    def payment_completed(self, chan, direction, htlc, _preimage):
         chan_id = chan.channel_id
+        key = bh2u(htlc.payment_hash)
+        if key not in self.invoices:
+            return
+        preimage, invoice, direction, timestamp = self.invoices.get(key)
         if direction == SENT:
-            assert htlc.payment_hash not in self.invoices
-            self.inflight.pop(key)
-            self.wallet.storage.put('lightning_payments_inflight', self.inflight)
-        if not preimage:
-            preimage, _addr = self.get_invoice(htlc.payment_hash)
-        tupl = (time.time(), direction, json.loads(encoder.encode(htlc)), bh2u(preimage), bh2u(chan_id))
-        self.completed[key] = tupl
-        self.wallet.storage.put('lightning_payments_completed', self.completed)
+            preimage = _preimage
+        now = time.time()
+        self.invoices[key] = preimage, invoice, direction, now
+        self.wallet.storage.put('lightning_invoices', self.invoices)
         self.wallet.storage.write()
-        self.network.trigger_callback('ln_payment_completed', tupl[0], direction, htlc, preimage, chan_id)
+        self.network.trigger_callback('ln_payment_completed', now, direction, htlc, preimage, chan_id)
 
-    def get_invoice_status(self, key):
-        from electrum.util import PR_UNPAID, PR_EXPIRED, PR_PAID, PR_UNKNOWN, PR_INFLIGHT
-        if key in self.completed:
-            return PR_PAID
-        elif key in self.inflight:
-            return PR_INFLIGHT
-        elif key in self.invoices:
-            return PR_UNPAID
-        else:
+    def get_invoice_status(self, payment_hash):
+        if payment_hash not in self.invoices:
             return PR_UNKNOWN
+        preimage, _addr, direction, timestamp = self.invoices.get(payment_hash)
+        if timestamp is None:
+            return PR_UNPAID
+        return PR_PAID
 
-    def _list_invoices(self, chan_id=None):
-        invoices  = dict(self.invoices)
-        settled = []
-        unsettled = []
-        inflight = []
-        for date, direction, htlc, hex_preimage, hex_chan_id in self.completed.values():
-            direction = Direction(direction)
-            if chan_id is not None:
-                if bfh(hex_chan_id) != chan_id:
-                    continue
-            htlcobj = UpdateAddHtlc(*htlc)
-            if direction == RECEIVED:
-                preimage = bfh(invoices.pop(bh2u(htlcobj.payment_hash))[0])
-            else:
-                preimage = bfh(hex_preimage)
-            # FIXME use fromisoformat when minimum Python is 3.7
-            settled.append((datetime.fromtimestamp(date, timezone.utc), direction, htlcobj, preimage))
-        for preimage, pay_req in invoices.values():
-            addr = lndecode(pay_req, expected_hrp=constants.net.SEGWIT_HRP)
-            unsettled.append((addr, bfh(preimage), pay_req))
-        for pay_req, amount_sat, this_chan_id in self.inflight.values():
-            if chan_id is not None and bfh(this_chan_id) != chan_id:
-                continue
-            addr = lndecode(pay_req, expected_hrp=constants.net.SEGWIT_HRP)
-            if amount_sat is not None:
-                addr.amount = Decimal(amount_sat) / COIN
-            htlc = self.find_htlc_for_addr(addr, None if chan_id is None else [chan_id])
-            if not htlc:
-                self.print_error('Warning, in-flight HTLC not found in any channel')
-            inflight.append((addr, htlc, SENT))
-        # not adding received htlcs to inflight because they should have been settled
-        # immediatly and therefore let's not spend time trying to show it in the GUI
-        return {'settled': settled, 'unsettled': unsettled, 'inflight': inflight}
-
-    def find_htlc_for_addr(self, addr, whitelist=None):
-        channels = [y for x,y in self.channels.items() if whitelist is None or x in whitelist]
-        for chan in channels:
-            for htlc in chan.hm.log[LOCAL]['adds'].values():
-                if htlc.payment_hash == addr.paymenthash:
-                    return htlc
+    def get_payments(self):
+        # note: with AMP we will have several channels per payment
+        out = {}
+        for chan in self.channels.values():
+            out.update(chan.get_payments())
+        return out
 
     def _read_ln_keystore(self) -> BIP32_KeyStore:
         xprv = self.wallet.storage.get('lightning_privkey2')
@@ -447,9 +406,6 @@ class LNWorker(PrintError):
                 break
         else:
             assert False, 'Found route with short channel ID we don\'t have: ' + repr(route[0].short_channel_id)
-        self.inflight[bh2u(addr.paymenthash)] = (invoice, amount_sat, bh2u(chan_id))
-        self.wallet.storage.put('lightning_payments_inflight', self.inflight)
-        self.wallet.storage.write()
         return addr, peer, self._pay_to_route(route, addr)
 
     async def _pay_to_route(self, route, addr):
@@ -545,14 +501,20 @@ class LNWorker(PrintError):
                                         ('c', MIN_FINAL_CLTV_EXPIRY_FOR_INVOICE)]
                                        + routing_hints),
                            self.node_keypair.privkey)
-        self.invoices[bh2u(RHASH)] = (bh2u(payment_preimage), pay_req)
+
+        self.save_invoice(bh2u(payment_preimage), pay_req, RECEIVED)
+        return pay_req
+
+    def save_invoice(self, preimage, invoice, direction):
+        lnaddr = lndecode(invoice, expected_hrp=constants.net.SEGWIT_HRP)
+        key = bh2u(lnaddr.paymenthash)
+        self.invoices[key] = preimage, invoice, direction, None
         self.wallet.storage.put('lightning_invoices', self.invoices)
         self.wallet.storage.write()
-        return pay_req
 
     def get_invoice(self, payment_hash: bytes) -> Tuple[bytes, LnAddr]:
         try:
-            preimage_hex, pay_req = self.invoices[bh2u(payment_hash)]
+            preimage_hex, pay_req, direction,timestamp = self.invoices[bh2u(payment_hash)]
             preimage = bfh(preimage_hex)
             assert sha256(preimage) == payment_hash
             return preimage, lndecode(pay_req, expected_hrp=constants.net.SEGWIT_HRP)

electrum/util.py

@@ -80,6 +80,14 @@ PR_UNKNOWN = 2     # sent but not propagated
 PR_PAID    = 3     # send and propagated
 PR_INFLIGHT = 4    # lightning
 
+pr_tooltips = {
+    PR_UNPAID:_('Pending'),
+    PR_PAID:_('Paid'),
+    PR_UNKNOWN:_('Unknown'),
+    PR_EXPIRED:_('Expired'),
+    PR_INFLIGHT:_('Inflight')
+}
+
 
 class UnknownBaseUnit(Exception): pass