lnhtlc: don't throw away fee updates or htlcs

also add inject_fees debug command

electrum/commands.py

@@ -787,6 +787,12 @@ class Commands:
     def listchannels(self):
         return self.wallet.lnworker.list_channels()
 
+    @command('n')
+    def inject_fees(self, fees):
+        import ast
+        self.network.config.fee_estimates = ast.literal_eval(fees)
+        self.network.notify('fee')
+
 def eval_bool(x: str) -> bool:
     if x == 'false': return False
     if x == 'true': return True

electrum/lnhtlc.py

@@ -2,7 +2,7 @@
 from collections import namedtuple
 import binascii
 import json
-from enum import IntFlag
+from enum import Enum, auto
 
 from .util import bfh, PrintError, bh2u
 from .bitcoin import Hash
@@ -16,33 +16,54 @@ from .lnutil import sign_and_get_sig_string
 from .lnutil import make_htlc_tx_with_open_channel, make_commitment, make_received_htlc, make_offered_htlc
 from .lnutil import HTLC_TIMEOUT_WEIGHT, HTLC_SUCCESS_WEIGHT
 from .lnutil import funding_output_script, extract_ctn_from_tx_and_chan
+from .lnutil import LOCAL, REMOTE, SENT, RECEIVED
 from .transaction import Transaction
 
 
 SettleHtlc = namedtuple("SettleHtlc", ["htlc_id"])
 RevokeAndAck = namedtuple("RevokeAndAck", ["per_commitment_secret", "next_per_commitment_point"])
 
-class FeeUpdateProgress(IntFlag):
+class FeeUpdateProgress(Enum):
-    FUNDEE_SIGNED = 1
+    FUNDEE_SIGNED = auto()
-    FUNDEE_ACKED =  2
+    FUNDEE_ACKED =  auto()
-    FUNDER_SIGNED = 4
+    FUNDER_SIGNED = auto()
+    COMMITTED = auto()
 
-class HTLCOwner(IntFlag):
+FUNDEE_SIGNED = FeeUpdateProgress.FUNDEE_SIGNED
-    LOCAL = 1
+FUNDEE_ACKED = FeeUpdateProgress.FUNDEE_ACKED
-    REMOTE = -LOCAL
+FUNDER_SIGNED = FeeUpdateProgress.FUNDER_SIGNED
+COMMITTED = FeeUpdateProgress.COMMITTED
 
-    SENT = LOCAL
+class FeeUpdate:
-    RECEIVED = REMOTE
 
-SENT = HTLCOwner.SENT
+    def __init__(self, chan, feerate):
-RECEIVED = HTLCOwner.RECEIVED
+        self.rate = feerate
-LOCAL = HTLCOwner.LOCAL
+        self.proposed = chan.remote_state.ctn if not chan.constraints.is_initiator else chan.local_state.ctn
-REMOTE = HTLCOwner.REMOTE
+        self.progress = {FUNDEE_SIGNED: None, FUNDEE_ACKED: None, FUNDER_SIGNED: None, COMMITTED: None}
+        self.chan = chan
 
-class FeeUpdate:
+    @property
-    def __init__(self, rate):
+    def height(self):
-        self.rate = rate
+        return self.chan.current_height[LOCAL if self.chan.constraints.is_initiator else REMOTE]
-        self.progress = 0
+
+    def set(self, field):
+        self.progress[field] = self.height
+
+    def is_proposed(self):
+        return self.progress[COMMITTED] is None and self.proposed is not None and self.proposed <= self.height
+
+    def had(self, field):
+        return self.progress[field] is not None and self.height >= self.progress[field]
+
+    def pending_feerate(self, subject):
+        if not self.is_proposed():
+            return
+        if self.had(FUNDEE_ACKED):
+            return self.rate
+        if subject == REMOTE and self.chan.constraints.is_initiator:
+            return self.rate
+        if subject == LOCAL and not self.chan.constraints.is_initiator:
+            return self.rate
 
 class UpdateAddHtlc:
     def __init__(self, amount_msat, payment_hash, cltv_expiry):
@@ -89,22 +110,6 @@ def typeWrap(k, v, local):
     return v
 
 class HTLCStateMachine(PrintError):
-    @property
-    def pending_remote_feerate(self):
-        if self.pending_fee is not None:
-            if self.constraints.is_initiator or (self.pending_fee.progress & FeeUpdateProgress.FUNDEE_ACKED):
-                return self.pending_fee.rate
-        return self.remote_state.feerate
-
-    @property
-    def pending_local_feerate(self):
-        if self.pending_fee is not None:
-            if not self.constraints.is_initiator:
-                return self.pending_fee.rate
-            if self.constraints.is_initiator and (self.pending_fee.progress & FeeUpdateProgress.FUNDEE_ACKED):
-                return self.pending_fee.rate
-        return self.local_state.feerate
-
     def lookup_htlc(self, log, htlc_id):
         assert type(htlc_id) is int
         for htlc in log:
@@ -153,7 +158,7 @@ class HTLCStateMachine(PrintError):
 
         self.name = name
 
-        self.pending_fee = None
+        self.fee_mgr = []
 
         self.local_commitment = self.pending_local_commitment
         self.remote_commitment = self.pending_remote_commitment
@@ -235,7 +240,7 @@ class HTLCStateMachine(PrintError):
 
         for_us = False
 
-        feerate = self.pending_remote_feerate
+        feerate = self.pending_feerate(REMOTE)
 
         htlcsigs = []
         for we_receive, htlcs in zip([True, False], [self.htlcs_in_remote, self.htlcs_in_local]):
@@ -253,11 +258,12 @@ class HTLCStateMachine(PrintError):
                 htlc_sig = ecc.sig_string_from_der_sig(sig[:-1])
                 htlcsigs.append(htlc_sig)
 
-        if self.pending_fee:
+        for pending_fee in self.fee_mgr:
-            if not self.constraints.is_initiator:
+            if pending_fee.is_proposed():
-                self.pending_fee.progress |= FeeUpdateProgress.FUNDEE_SIGNED
+                if not self.constraints.is_initiator:
-            if self.constraints.is_initiator and (self.pending_fee.progress & FeeUpdateProgress.FUNDEE_ACKED):
+                    pending_fee.set(FUNDEE_SIGNED)
-                self.pending_fee.progress |= FeeUpdateProgress.FUNDER_SIGNED
+                if self.constraints.is_initiator and pending_fee.had(FUNDEE_ACKED):
+                    pending_fee.set(FUNDER_SIGNED)
 
         if self.lnwatcher:
             self.lnwatcher.process_new_offchain_ctx(self, pending_remote_commitment, ours=False)
@@ -304,11 +310,12 @@ class HTLCStateMachine(PrintError):
 
         # TODO check htlc in htlcs_in_local
 
-        if self.pending_fee:
+        for pending_fee in self.fee_mgr:
-            if not self.constraints.is_initiator:
+            if pending_fee.is_proposed():
-                self.pending_fee.progress |= FeeUpdateProgress.FUNDEE_SIGNED
+                if not self.constraints.is_initiator:
-            if self.constraints.is_initiator and (self.pending_fee.progress & FeeUpdateProgress.FUNDEE_ACKED):
+                    pending_fee.set(FUNDEE_SIGNED)
-                self.pending_fee.progress |= FeeUpdateProgress.FUNDER_SIGNED
+                if self.constraints.is_initiator and pending_fee.had(FUNDEE_ACKED):
+                    pending_fee.set(FUNDER_SIGNED)
 
         if self.lnwatcher:
             self.lnwatcher.process_new_offchain_ctx(self, pending_local_commitment, ours=True)
@@ -332,14 +339,14 @@ class HTLCStateMachine(PrintError):
         new_local_feerate = self.local_state.feerate
         new_remote_feerate = self.remote_state.feerate
 
-        if self.pending_fee is not None:
+        for pending_fee in self.fee_mgr:
-            if not self.constraints.is_initiator and (self.pending_fee.progress & FeeUpdateProgress.FUNDEE_SIGNED):
+            if not self.constraints.is_initiator and pending_fee.had(FUNDEE_SIGNED):
-                new_local_feerate = new_remote_feerate = self.pending_fee.rate
+                new_local_feerate = new_remote_feerate = pending_fee.rate
-                self.pending_fee = None
+                pending_fee.set(COMMITTED)
                 print("FEERATE CHANGE COMPLETE (non-initiator)")
-            if self.constraints.is_initiator and (self.pending_fee.progress & FeeUpdateProgress.FUNDER_SIGNED):
+            if self.constraints.is_initiator and pending_fee.had(FUNDER_SIGNED):
-                new_local_feerate = new_remote_feerate = self.pending_fee.rate
+                new_local_feerate = new_remote_feerate = pending_fee.rate
-                self.pending_fee = None
+                pending_fee.set(COMMITTED)
                 print("FEERATE CHANGE COMPLETE (initiator)")
 
         self.local_state=self.local_state._replace(
@@ -424,9 +431,10 @@ class HTLCStateMachine(PrintError):
             amount_msat = self.local_state.amount_msat + (received_this_batch - sent_this_batch)
         )
 
-        if self.pending_fee:
+        for pending_fee in self.fee_mgr:
-            if self.constraints.is_initiator:
+            if pending_fee.is_proposed():
-                self.pending_fee.progress |= FeeUpdateProgress.FUNDEE_ACKED
+                if self.constraints.is_initiator:
+                    pending_fee.set(FUNDEE_ACKED)
 
         self.local_commitment = self.pending_local_commitment
         self.remote_commitment = self.pending_remote_commitment
@@ -463,7 +471,7 @@ class HTLCStateMachine(PrintError):
         local_htlc_pubkey = derive_pubkey(self.local_config.htlc_basepoint.pubkey, this_point)
         local_revocation_pubkey = derive_blinded_pubkey(self.local_config.revocation_basepoint.pubkey, this_point)
 
-        feerate = self.pending_remote_feerate
+        feerate = self.pending_feerate(REMOTE)
 
         htlcs_in_local = []
         for htlc in self.htlcs_in_local:
@@ -484,6 +492,20 @@ class HTLCStateMachine(PrintError):
             remote_msat, local_msat, htlcs_in_local + htlcs_in_remote)
         return commit
 
+    def pending_feerate(self, subject):
+        candidate = None
+        for pending_fee in self.fee_mgr:
+            x = pending_fee.pending_feerate(subject)
+            if x is not None:
+                candidate = x
+
+        feerate = candidate if candidate is not None else self._committed_feerate[subject]
+        return feerate
+
+    @property
+    def _committed_feerate(self):
+        return {LOCAL: self.local_state.feerate, REMOTE: self.remote_state.feerate}
+
     @property
     def pending_local_commitment(self):
         remote_msat, local_msat = self.amounts()
@@ -496,7 +518,7 @@ class HTLCStateMachine(PrintError):
         local_htlc_pubkey = derive_pubkey(self.local_config.htlc_basepoint.pubkey, this_point)
         remote_revocation_pubkey = derive_blinded_pubkey(self.remote_config.revocation_basepoint.pubkey, this_point)
 
-        feerate = self.pending_local_feerate
+        feerate = self.pending_feerate(LOCAL)
 
         htlcs_in_local = []
         for htlc in self.htlcs_in_local:
@@ -569,7 +591,7 @@ class HTLCStateMachine(PrintError):
         self.print_error("receive_htlc_settle")
         htlc = self.lookup_htlc(self.log[LOCAL], htlc_index)
         assert htlc.payment_hash == sha256(preimage)
-        assert len([x.htlc_id == htlc_index for x in self.log[LOCAL]]) == 1
+        assert len([x for x in self.log[LOCAL] if x.htlc_id == htlc_index and type(x) is UpdateAddHtlc]) == 1, (self.log[LOCAL], htlc_index)
         self.log[REMOTE].append(SettleHtlc(htlc_index))
 
     def fail_htlc(self, htlc):
@@ -586,15 +608,17 @@ class HTLCStateMachine(PrintError):
     def pending_local_fee(self):
         return self.constraints.capacity - sum(x[2] for x in self.pending_local_commitment.outputs())
 
-    def update_fee(self, fee):
+    def update_fee(self, feerate):
         if not self.constraints.is_initiator:
             raise Exception("only initiator can update_fee, this counterparty is not initiator")
-        self.pending_fee = FeeUpdate(rate=fee)
+        pending_fee = FeeUpdate(self, feerate)
+        self.fee_mgr.append(pending_fee)
 
-    def receive_update_fee(self, fee):
+    def receive_update_fee(self, feerate):
         if self.constraints.is_initiator:
             raise Exception("only the non-initiator can receive_update_fee, this counterparty is initiator")
-        self.pending_fee = FeeUpdate(rate=fee)
+        pending_fee = FeeUpdate(self, feerate)
+        self.fee_mgr.append(pending_fee)
 
     def to_save(self):
         return {
@@ -650,7 +674,7 @@ class HTLCStateMachine(PrintError):
             local_msat,
             remote_msat,
             conf.dust_limit_sat,
-            chan.pending_local_feerate if for_us else chan.pending_remote_feerate,
+            chan.pending_feerate(LOCAL if for_us else REMOTE),
             for_us,
             chan.constraints.is_initiator,
             htlcs=htlcs)

electrum/lnutil.py

@@ -1,3 +1,4 @@
+from enum import IntFlag
 import json
 from collections import namedtuple
 from typing import NamedTuple
@@ -249,7 +250,7 @@ def make_htlc_tx_with_open_channel(chan, pcp, for_us, we_receive, amount_msat, c
     is_htlc_success = for_us == we_receive
     htlc_tx_output = make_htlc_tx_output(
         amount_msat = amount_msat,
-        local_feerate = chan.pending_local_feerate if for_us else chan.pending_remote_feerate,
+        local_feerate = chan.pending_feerate(LOCAL if for_us else REMOTE),
         revocationpubkey=revocation_pubkey,
         local_delayedpubkey=delayedpubkey,
         success = is_htlc_success,
@@ -432,3 +433,15 @@ def get_compressed_pubkey_from_bech32(bech32_pubkey: str) -> bytes:
 
 
 class PaymentFailure(Exception): pass
+
+class HTLCOwner(IntFlag):
+    LOCAL = 1
+    REMOTE = -LOCAL
+
+    SENT = LOCAL
+    RECEIVED = REMOTE
+
+SENT = HTLCOwner.SENT
+RECEIVED = HTLCOwner.RECEIVED
+LOCAL = HTLCOwner.LOCAL
+REMOTE = HTLCOwner.REMOTE

electrum/tests/test_lnhtlc.py

@@ -258,14 +258,14 @@ class TestLNBaseHTLCStateMachine(unittest.TestCase):
         return fee
 
     def test_UpdateFeeSenderCommits(self):
-        old_feerate = self.alice_channel.pending_local_feerate
+        old_feerate = self.alice_channel.pending_feerate(LOCAL)
         fee = self.alice_to_bob_fee_update()
 
         alice_channel, bob_channel = self.alice_channel, self.bob_channel
 
-        self.assertEqual(self.alice_channel.pending_local_feerate, old_feerate)
+        self.assertEqual(self.alice_channel.pending_feerate(LOCAL), old_feerate)
         alice_sig, alice_htlc_sigs = alice_channel.sign_next_commitment()
-        self.assertEqual(self.alice_channel.pending_local_feerate, old_feerate)
+        self.assertEqual(self.alice_channel.pending_feerate(LOCAL), old_feerate)
 
         bob_channel.receive_new_commitment(alice_sig, alice_htlc_sigs)