lnrouter+lnworker: use liquidity hints

Adds liquidity hints for the sending capabilities of routing channels in the
graph. The channel blacklist is incorporated into liquidity hints.
Liquidity hints are updated when a payment fails with a temporary
channel failure or when it succeeds. Liquidity hints are used to give a
penalty in the _edge_cost heuristics used by the pathfinding algorithm.
The base penalty in (_edge_cost) is removed because it is now part of the
liquidity penalty. We don't return early from get_distances, as we want
to explore all channels.

electrum/commands.py

@@ -1089,7 +1089,7 @@ class Commands:
 
     @command('n')
     async def clear_ln_blacklist(self):
-        self.network.path_finder.blacklist.clear()
+        self.network.path_finder.liquidity_hints.clear_blacklist()
 
     @command('w')
     async def list_invoices(self, wallet: Abstract_Wallet = None):

electrum/lnrouter.py

@@ -27,9 +27,11 @@ import queue
 from collections import defaultdict
 from typing import Sequence, List, Tuple, Optional, Dict, NamedTuple, TYPE_CHECKING, Set
 import time
+from threading import RLock
 import attr
+from math import inf
 
-from .util import bh2u, profiler
+from .util import bh2u, profiler, with_lock
 from .logging import Logger
 from .lnutil import (NUM_MAX_EDGES_IN_PAYMENT_PATH, ShortChannelID, LnFeatures,
                      NBLOCK_CLTV_EXPIRY_TOO_FAR_INTO_FUTURE)
@@ -38,6 +40,10 @@ from .channel_db import ChannelDB, Policy, NodeInfo
 if TYPE_CHECKING:
     from .lnchannel import Channel
 
+DEFAULT_PENALTY_BASE_MSAT = 500  # how much base fee we apply for unknown sending capability of a channel
+DEFAULT_PENALTY_PROPORTIONAL_MILLIONTH = 100  # how much relative fee we apply for unknown sending capability of a channel
+BLACKLIST_DURATION = 3600  # how long (in seconds) a channel remains blacklisted
+
 
 class NoChannelPolicy(Exception):
     def __init__(self, short_channel_id: bytes):
@@ -161,12 +167,247 @@ def is_fee_sane(fee_msat: int, *, payment_amount_msat: int) -> bool:
     return False
 
 
+class LiquidityHint:
+    """Encodes the amounts that can and cannot be sent over the direction of a
+    channel and whether the channel is blacklisted.
+
+    A LiquidityHint is the value of a dict, which is keyed to node ids and the
+    channel.
+    """
+    def __init__(self):
+        # use "can_send_forward + can_send_backward < cannot_send_forward + cannot_send_backward" as a sanity check?
+        self._can_send_forward = None
+        self._cannot_send_forward = None
+        self._can_send_backward = None
+        self._cannot_send_backward = None
+        self.is_blacklisted = False
+        self.timestamp = 0
+
+    @property
+    def can_send_forward(self):
+        return self._can_send_forward
+
+    @can_send_forward.setter
+    def can_send_forward(self, amount):
+        # we don't want to record less significant info
+        # (sendable amount is lower than known sendable amount):
+        if self._can_send_forward and self._can_send_forward > amount:
+            return
+        self._can_send_forward = amount
+        # we make a sanity check that sendable amount is lower than not sendable amount
+        if self._cannot_send_forward and self._can_send_forward > self._cannot_send_forward:
+            self._cannot_send_forward = None
+
+    @property
+    def can_send_backward(self):
+        return self._can_send_backward
+
+    @can_send_backward.setter
+    def can_send_backward(self, amount):
+        if self._can_send_backward and self._can_send_backward > amount:
+            return
+        self._can_send_backward = amount
+        if self._cannot_send_backward and self._can_send_backward > self._cannot_send_backward:
+            self._cannot_send_backward = None
+
+    @property
+    def cannot_send_forward(self):
+        return self._cannot_send_forward
+
+    @cannot_send_forward.setter
+    def cannot_send_forward(self, amount):
+        # we don't want to record less significant info
+        # (not sendable amount is higher than known not sendable amount):
+        if self._cannot_send_forward and self._cannot_send_forward < amount:
+            return
+        self._cannot_send_forward = amount
+        if self._can_send_forward and self._can_send_forward > self._cannot_send_forward:
+            self._can_send_forward = None
+        # if we can't send over the channel, we should be able to send in the
+        # reverse direction
+        self.can_send_backward = amount
+
+    @property
+    def cannot_send_backward(self):
+        return self._cannot_send_backward
+
+    @cannot_send_backward.setter
+    def cannot_send_backward(self, amount):
+        if self._cannot_send_backward and self._cannot_send_backward < amount:
+            return
+        self._cannot_send_backward = amount
+        if self._can_send_backward and self._can_send_backward > self._cannot_send_backward:
+            self._can_send_backward = None
+        self.can_send_forward = amount
+
+    def can_send(self, is_forward_direction: bool):
+        # make info invalid after some time?
+        if is_forward_direction:
+            return self.can_send_forward
+        else:
+            return self.can_send_backward
+
+    def cannot_send(self, is_forward_direction: bool):
+        # make info invalid after some time?
+        if is_forward_direction:
+            return self.cannot_send_forward
+        else:
+            return self.cannot_send_backward
+
+    def update_can_send(self, is_forward_direction: bool, amount: int):
+        if is_forward_direction:
+            self.can_send_forward = amount
+        else:
+            self.can_send_backward = amount
+
+    def update_cannot_send(self, is_forward_direction: bool, amount: int):
+        if is_forward_direction:
+            self.cannot_send_forward = amount
+        else:
+            self.cannot_send_backward = amount
+
+    def __repr__(self):
+        return f"forward: can send: {self._can_send_forward}, cannot send: {self._cannot_send_forward}, \n" \
+               f"backward: can send: {self._can_send_backward} cannot send: {self._cannot_send_backward}, \n" \
+               f"blacklisted: {self.is_blacklisted}"
+
+
+class LiquidityHintMgr:
+    """Implements liquidity hints for channels in the graph.
+
+    This class can be used to update liquidity information about channels in the
+    graph. Implements a penalty function for edge weighting in the pathfinding
+    algorithm that favors channels which can route payments and penalizes
+    channels that cannot.
+    """
+    # TODO: incorporate in-flight htlcs
+    # TODO: use timestamps for can/not_send to make them None after some time?
+    # TODO: hints based on node pairs only (shadow channels, non-strict forwarding)?
+    def __init__(self):
+        self.lock = RLock()
+        self._liquidity_hints: Dict[ShortChannelID, LiquidityHint] = {}
+
+    @with_lock
+    def get_hint(self, channel_id: ShortChannelID):
+        hint = self._liquidity_hints.get(channel_id)
+        if not hint:
+            hint = LiquidityHint()
+            self._liquidity_hints[channel_id] = hint
+        return hint
+
+    @with_lock
+    def update_can_send(self, node_from: bytes, node_to: bytes, channel_id: ShortChannelID, amount: int):
+        hint = self.get_hint(channel_id)
+        hint.update_can_send(node_from < node_to, amount)
+
+    @with_lock
+    def update_cannot_send(self, node_from: bytes, node_to: bytes, channel_id: ShortChannelID, amount: int):
+        hint = self.get_hint(channel_id)
+        hint.update_cannot_send(node_from < node_to, amount)
+
+    def penalty(self, node_from: bytes, node_to: bytes, channel_id: ShortChannelID, amount: int) -> float:
+        """Gives a penalty when sending from node1 to node2 over channel_id with an
+        amount in units of millisatoshi.
+
+        The penalty depends on the can_send and cannot_send values that was
+        possibly recorded in previous payment attempts.
+
+        A channel that can send an amount is assigned a penalty of zero, a
+        channel that cannot send an amount is assigned an infinite penalty.
+        If the sending amount lies between can_send and cannot_send, there's
+        uncertainty and we give a default penalty. The default penalty
+        serves the function of giving a positive offset (the Dijkstra
+        algorithm doesn't work with negative weights), from which we can discount
+        from. There is a competition between low-fee channels and channels where
+        we know with some certainty that they can support a payment. The penalty
+        ultimately boils down to: how much more fees do we want to pay for
+        certainty of payment success? This can be tuned via DEFAULT_PENALTY_BASE_MSAT
+        and DEFAULT_PENALTY_PROPORTIONAL_MILLIONTH. A base _and_ relative penalty
+        was chosen such that the penalty will be able to compete with the regular
+        base and relative fees.
+        """
+        # we only evaluate hints here, so use dict get (to not create many hints with self.get_hint)
+        hint = self._liquidity_hints.get(channel_id)
+        if not hint:
+            can_send, cannot_send = None, None
+        else:
+            can_send = hint.can_send(node_from < node_to)
+            cannot_send = hint.cannot_send(node_from < node_to)
+
+        # if we know nothing about the channel, return a default penalty
+        if (can_send, cannot_send) == (None, None):
+            return fee_for_edge_msat(amount, DEFAULT_PENALTY_BASE_MSAT, DEFAULT_PENALTY_PROPORTIONAL_MILLIONTH)
+        # next cases are with half information
+        elif can_send and not cannot_send:
+            if amount <= can_send:
+                return 0
+            else:
+                return fee_for_edge_msat(amount, DEFAULT_PENALTY_BASE_MSAT, DEFAULT_PENALTY_PROPORTIONAL_MILLIONTH)
+        elif not can_send and cannot_send:
+            if amount >= cannot_send:
+                return inf
+            else:
+                return fee_for_edge_msat(amount, DEFAULT_PENALTY_BASE_MSAT, DEFAULT_PENALTY_PROPORTIONAL_MILLIONTH)
+        # we know how much we can/cannot send
+        elif can_send and cannot_send:
+            if amount <= can_send:
+                return 0
+            elif amount < cannot_send:
+                return fee_for_edge_msat(amount, DEFAULT_PENALTY_BASE_MSAT, DEFAULT_PENALTY_PROPORTIONAL_MILLIONTH)
+            else:
+                return inf
+        return 0
+
+    @with_lock
+    def add_to_blacklist(self, node_from: bytes, node_to: bytes, channel_id: ShortChannelID):
+        hint = self.get_hint(channel_id)
+        hint.is_blacklisted = True
+        now = int(time.time())
+        hint.timestamp = now
+
+    @with_lock
+    def get_blacklist(self) -> Set[ShortChannelID]:
+        now = int(time.time())
+        return set(k for k, v in self._liquidity_hints.items() if now - v.timestamp < BLACKLIST_DURATION)
+
+    @with_lock
+    def clear_blacklist(self):
+        for k, v in self._liquidity_hints.items():
+            v.is_blacklisted = False
+
+    def __repr__(self):
+        string = "liquidity hints:\n"
+        if self._liquidity_hints:
+            for k, v in self._liquidity_hints.items():
+                string += f"{k}: {v}\n"
+        return string
+
 
 class LNPathFinder(Logger):
 
     def __init__(self, channel_db: ChannelDB):
         Logger.__init__(self)
         self.channel_db = channel_db
+        self.liquidity_hints = LiquidityHintMgr()
+
+    def update_liquidity_hints(
+            self,
+            route: LNPaymentRoute,
+            amount_msat: int,
+            failing_channel: ShortChannelID=None
+    ):
+        # go through the route and record successes until the failing channel is reached,
+        # for the failing channel, add a cannot_send liquidity hint
+        # note: actual routable amounts are slightly different than reported here
+        # as fees would need to be added
+        for r in route:
+            if r.short_channel_id != failing_channel:
+                self.logger.info(f"report {r.short_channel_id} to be able to forward {amount_msat} msat")
+                self.liquidity_hints.update_can_send(r.start_node, r.end_node, r.short_channel_id, amount_msat)
+            else:
+                self.logger.info(f"report {r.short_channel_id} to be unable to forward {amount_msat} msat")
+                self.liquidity_hints.update_cannot_send(r.start_node, r.end_node, r.short_channel_id, amount_msat)
+                break
 
     def _edge_cost(
             self,
@@ -221,19 +462,20 @@ class LNPathFinder(Logger):
                 node_info=node_info)
         if not route_edge.is_sane_to_use(payment_amt_msat):
             return float('inf'), 0  # thanks but no thanks
-
         # Distance metric notes:  # TODO constants are ad-hoc
         # ( somewhat based on https://github.com/lightningnetwork/lnd/pull/1358 )
         # - Edges have a base cost. (more edges -> less likely none will fail)
         # - The larger the payment amount, and the longer the CLTV,
         #   the more irritating it is if the HTLC gets stuck.
         # - Paying lower fees is better. :)
-        base_cost = 500  # one more edge ~ paying 500 msat more fees
         if ignore_costs:
-            return base_cost, 0
+            return DEFAULT_PENALTY_BASE_MSAT, 0
         fee_msat = route_edge.fee_for_edge(payment_amt_msat)
         cltv_cost = route_edge.cltv_expiry_delta * payment_amt_msat * 15 / 1_000_000_000
-        overall_cost = base_cost + fee_msat + cltv_cost
+        # the liquidty penalty takes care we favor edges that should be able to forward
+        # the payment and penalize edges that cannot
+        liquidity_penalty = self.liquidity_hints.penalty(start_node, end_node, short_channel_id, payment_amt_msat)
+        overall_cost = fee_msat + cltv_cost + liquidity_penalty
         return overall_cost, fee_msat
 
     def get_distances(
@@ -243,7 +485,6 @@ class LNPathFinder(Logger):
             nodeB: bytes,
             invoice_amount_msat: int,
             my_channels: Dict[ShortChannelID, 'Channel'] = None,
-            blacklist: Set[ShortChannelID] = None,
             private_route_edges: Dict[ShortChannelID, RouteEdge] = None,
     ) -> Dict[bytes, PathEdge]:
         # note: we don't lock self.channel_db, so while the path finding runs,
@@ -252,6 +493,7 @@ class LNPathFinder(Logger):
         # run Dijkstra
         # The search is run in the REVERSE direction, from nodeB to nodeA,
         # to properly calculate compound routing fees.
+        blacklist = self.liquidity_hints.get_blacklist()
         distance_from_start = defaultdict(lambda: float('inf'))
         distance_from_start[nodeB] = 0
         prev_node = {}  # type: Dict[bytes, PathEdge]
@@ -316,7 +558,6 @@ class LNPathFinder(Logger):
             nodeB: bytes,
             invoice_amount_msat: int,
             my_channels: Dict[ShortChannelID, 'Channel'] = None,
-            blacklist: Set[ShortChannelID] = None,
             private_route_edges: Dict[ShortChannelID, RouteEdge] = None,
     ) -> Optional[LNPaymentPath]:
         """Return a path from nodeA to nodeB."""
@@ -331,7 +572,6 @@ class LNPathFinder(Logger):
             nodeB=nodeB,
             invoice_amount_msat=invoice_amount_msat,
             my_channels=my_channels,
-            blacklist=blacklist,
             private_route_edges=private_route_edges)
 
         if nodeA not in prev_node:
@@ -394,7 +634,6 @@ class LNPathFinder(Logger):
             invoice_amount_msat: int,
             path = None,
             my_channels: Dict[ShortChannelID, 'Channel'] = None,
-            blacklist: Set[ShortChannelID] = None,
             private_route_edges: Dict[ShortChannelID, RouteEdge] = None,
     ) -> Optional[LNPaymentRoute]:
         route = None
@@ -404,7 +643,6 @@ class LNPathFinder(Logger):
                 nodeB=nodeB,
                 invoice_amount_msat=invoice_amount_msat,
                 my_channels=my_channels,
-                blacklist=blacklist,
                 private_route_edges=private_route_edges)
         if path:
             route = self.create_route_from_path(

electrum/lnutil.py

@@ -1354,16 +1354,3 @@ class OnionFailureCodeMetaFlag(IntFlag):
     UPDATE   = 0x1000
 
 
-class ChannelBlackList:
-
-    def __init__(self):
-        self.blacklist = dict() # short_chan_id -> timestamp
-
-    def add(self, short_channel_id: ShortChannelID):
-        now = int(time.time())
-        self.blacklist[short_channel_id] = now
-
-    def get_current_list(self) -> Set[ShortChannelID]:
-        BLACKLIST_DURATION = 3600
-        now = int(time.time())
-        return set(k for k, t in self.blacklist.items() if now - t < BLACKLIST_DURATION)

electrum/lnworker.py

@@ -1208,6 +1208,11 @@ class LNWallet(LNWorker):
                 raise Exception(f"amount_inflight={amount_inflight} < 0")
             log.append(htlc_log)
             if htlc_log.success:
+                # TODO: report every route to liquidity hints for mpp
+                # even in the case of success, we report channels of the
+                # route as being able to send the same amount in the future,
+                # as we assume to not know the capacity
+                self.network.path_finder.update_liquidity_hints(htlc_log.route, htlc_log.amount_msat)
                 return
             # htlc failed
             if len(log) >= attempts:
@@ -1235,7 +1240,7 @@ class LNWallet(LNWorker):
                     raise PaymentFailure(failure_msg.code_name())
             else:
                 self.handle_error_code_from_failed_htlc(
-                    route=route, sender_idx=sender_idx, failure_msg=failure_msg)
+                    route=route, sender_idx=sender_idx, failure_msg=failure_msg, amount=htlc_log.amount_msat)
 
     async def pay_to_route(
             self, *,
@@ -1281,7 +1286,8 @@ class LNWallet(LNWorker):
             *,
             route: LNPaymentRoute,
             sender_idx: int,
-            failure_msg: OnionRoutingFailure) -> None:
+            failure_msg: OnionRoutingFailure,
+            amount: int) -> None:
         code, data = failure_msg.code, failure_msg.data
         # TODO can we use lnmsg.OnionWireSerializer here?
         # TODO update onion_wire.csv
@@ -1294,40 +1300,53 @@ class LNWallet(LNWorker):
             OnionFailureCode.EXPIRY_TOO_SOON: 0,
             OnionFailureCode.CHANNEL_DISABLED: 2,
         }
-        blacklist = False
-        update = False
+
+        # determine a fallback channel to blacklist if we don't get the erring
+        # channel via the payload
+        if sender_idx is None:
+            raise PaymentFailure(failure_msg.code_name())
+        try:
+            fallback_channel = route[sender_idx + 1].short_channel_id
+            node_from = route[sender_idx].start_node
+            node_to = route[sender_idx].end_node
+        except IndexError:
+            raise PaymentFailure(f'payment destination reported error: {failure_msg.code_name()}') from None
+
+        # TODO: handle unknown next peer?
+        # handle failure codes that include a channel update
         if code in failure_codes:
             offset = failure_codes[code]
             channel_update_len = int.from_bytes(data[offset:offset+2], byteorder="big")
             channel_update_as_received = data[offset+2: offset+2+channel_update_len]
             payload = self._decode_channel_update_msg(channel_update_as_received)
+
             if payload is None:
-                self.logger.info(f'could not decode channel_update for failed htlc: {channel_update_as_received.hex()}')
-                blacklist = True
+                self.logger.info(f'could not decode channel_update for failed htlc: '
+                                 f'{channel_update_as_received.hex()}')
+                self.network.path_finder.channel_blacklist.add(fallback_channel)
             else:
+                # apply the channel update or get blacklisted
                 blacklist, update = self._handle_chanupd_from_failed_htlc(
                     payload, route=route, sender_idx=sender_idx)
-        else:
-            blacklist = True
 
-        if blacklist:
-            # blacklist channel after reporter node
-            # TODO this should depend on the error (even more granularity)
-            # also, we need finer blacklisting (directed edges; nodes)
-            if sender_idx is None:
-                raise PaymentFailure(failure_msg.code_name())
-            try:
-                short_chan_id = route[sender_idx + 1].short_channel_id
-            except IndexError:
-                raise PaymentFailure(f'payment destination reported error: {failure_msg.code_name()}') from None
-            # TODO: for MPP we need to save the amount for which
-            # we saw temporary channel failure
-            self.logger.info(f'blacklisting channel {short_chan_id}')
-            self.network.channel_blacklist.add(short_chan_id)
-
-        # we should not continue if we did not blacklist or update anything
-        if not (blacklist or update):
-            raise PaymentFailure(failure_msg.code_name())
+                # we interpret a temporary channel failure as a liquidity issue
+                # in the channel and update our liquidity hints accordingly
+                if code == OnionFailureCode.TEMPORARY_CHANNEL_FAILURE:
+                    self.network.path_finder.update_liquidity_hints(
+                        route,
+                        amount,
+                        failing_channel=ShortChannelID(payload['short_channel_id']))
+                elif blacklist:
+                    self.network.path_finder.liquidity_hints.add_to_blacklist(
+                        node_from, node_to, payload['short_channel_id'])
+
+                # if we can't decide on some action, we are stuck
+                if not (blacklist or update):
+                    raise PaymentFailure(failure_msg.code_name())
+
+        # for errors that do not include a channel update
+        else:
+            self.network.path_finder.liquidity_hints.add_to_blacklist(node_from, node_to, fallback_channel)
 
     def _handle_chanupd_from_failed_htlc(self, payload, *, route, sender_idx) -> Tuple[bool, bool]:
         blacklist = False
@@ -1597,7 +1616,6 @@ class LNWallet(LNWorker):
             chan.short_channel_id: chan for chan in channels
             if chan.short_channel_id is not None
         }
-        blacklist = self.network.channel_blacklist.get_current_list()
         # Collect all private edges from route hints.
         # Note: if some route hints are multiple edges long, and these paths cross each other,
         #       we allow our path finding to cross the paths; i.e. the route hints are not isolated.
@@ -1629,8 +1647,7 @@ class LNWallet(LNWorker):
                         fee_proportional_millionths=fee_proportional_millionths,
                         cltv_expiry_delta=cltv_expiry_delta,
                         node_features=node_info.features if node_info else 0)
-                if route_edge.short_channel_id not in blacklist:
-                    private_route_edges[route_edge.short_channel_id] = route_edge
+                private_route_edges[route_edge.short_channel_id] = route_edge
                 start_node = end_node
         # now find a route, end to end: between us and the recipient
         try:
@@ -1640,7 +1657,6 @@ class LNWallet(LNWorker):
                 invoice_amount_msat=amount_msat,
                 path=full_path,
                 my_channels=scid_to_my_channels,
-                blacklist=blacklist,
                 private_route_edges=private_route_edges)
         except NoChannelPolicy as e:
             raise NoPathFound() from e

electrum/network.py

@@ -62,7 +62,6 @@ from .version import PROTOCOL_VERSION
 from .simple_config import SimpleConfig
 from .i18n import _
 from .logging import get_logger, Logger
-from .lnutil import ChannelBlackList
 
 if TYPE_CHECKING:
     from .channel_db import ChannelDB
@@ -350,7 +349,6 @@ class Network(Logger, NetworkRetryManager[ServerAddr]):
         self._has_ever_managed_to_connect_to_server = False
 
         # lightning network
-        self.channel_blacklist = ChannelBlackList()
         if self.config.get('run_watchtower', False):
             from . import lnwatcher
             self.local_watchtower = lnwatcher.WatchTower(self)

electrum/tests/test_lnpeer.py

@@ -33,7 +33,7 @@ from electrum import lnmsg
 from electrum.logging import console_stderr_handler, Logger
 from electrum.lnworker import PaymentInfo, RECEIVED
 from electrum.lnonion import OnionFailureCode
-from electrum.lnutil import ChannelBlackList, derive_payment_secret_from_payment_preimage
+from electrum.lnutil import derive_payment_secret_from_payment_preimage
 from electrum.lnutil import LOCAL, REMOTE
 from electrum.invoices import PR_PAID, PR_UNPAID
 
@@ -66,7 +66,6 @@ class MockNetwork:
         self.path_finder = LNPathFinder(self.channel_db)
         self.tx_queue = tx_queue
         self._blockchain = MockBlockchain()
-        self.channel_blacklist = ChannelBlackList()
 
     @property
     def callback_lock(self):
@@ -807,7 +806,7 @@ class TestPeer(TestCaseForTestnet):
             run(f())
 
     @needs_test_with_all_chacha20_implementations
-    def test_payment_with_temp_channel_failure(self):
+    def test_payment_with_temp_channel_failure_and_liquidty_hints(self):
         # prepare channels such that a temporary channel failure happens at c->d
         funds_distribution = {
             'ac': (200_000_000, 200_000_000),  # low fees
@@ -831,6 +830,27 @@ class TestPeer(TestCaseForTestnet):
             self.assertEqual(PR_PAID, graph.w_d.get_payment_status(lnaddr.paymenthash))
             self.assertEqual(OnionFailureCode.TEMPORARY_CHANNEL_FAILURE, log[0].failure_msg.code)
             self.assertEqual(OnionFailureCode.TEMPORARY_CHANNEL_FAILURE, log[1].failure_msg.code)
+
+            liquidity_hints = graph.w_a.network.path_finder.liquidity_hints
+            pubkey_a = graph.w_a.node_keypair.pubkey
+            pubkey_b = graph.w_b.node_keypair.pubkey
+            pubkey_c = graph.w_c.node_keypair.pubkey
+            pubkey_d = graph.w_d.node_keypair.pubkey
+            # check liquidity hints for failing route:
+            hint_ac = liquidity_hints.get_hint(graph.chan_ac.short_channel_id)
+            hint_cd = liquidity_hints.get_hint(graph.chan_cd.short_channel_id)
+            self.assertEqual(amount_to_pay, hint_ac.can_send(pubkey_a < pubkey_c))
+            self.assertEqual(None, hint_ac.cannot_send(pubkey_a < pubkey_c))
+            self.assertEqual(None, hint_cd.can_send(pubkey_c < pubkey_d))
+            self.assertEqual(amount_to_pay, hint_cd.cannot_send(pubkey_c < pubkey_d))
+            # check liquidity hints for successful route:
+            hint_ab = liquidity_hints.get_hint(graph.chan_ab.short_channel_id)
+            hint_bd = liquidity_hints.get_hint(graph.chan_bd.short_channel_id)
+            self.assertEqual(amount_to_pay, hint_ab.can_send(pubkey_a < pubkey_b))
+            self.assertEqual(None, hint_ab.cannot_send(pubkey_a < pubkey_b))
+            self.assertEqual(amount_to_pay, hint_bd.can_send(pubkey_b < pubkey_d))
+            self.assertEqual(None, hint_bd.cannot_send(pubkey_b < pubkey_d))
+
             raise PaymentDone()
         async def f():
             async with TaskGroup() as group:

electrum/tests/test_lnrouter.py

@@ -1,3 +1,4 @@
+from math import inf
 import unittest
 import tempfile
 import shutil
@@ -11,7 +12,7 @@ from electrum.lnonion import (OnionHopsDataSingle, new_onion_packet,
 from electrum import bitcoin, lnrouter
 from electrum.constants import BitcoinTestnet
 from electrum.simple_config import SimpleConfig
-from electrum.lnrouter import PathEdge
+from electrum.lnrouter import PathEdge, LiquidityHintMgr, DEFAULT_PENALTY_PROPORTIONAL_MILLIONTH, DEFAULT_PENALTY_BASE_MSAT, fee_for_edge_msat
 
 from . import TestCaseForTestnet
 from .test_bitcoin import needs_test_with_all_chacha20_implementations
@@ -153,6 +154,104 @@ class Test_LNRouter(TestCaseForTestnet):
         self.cdb.stop()
         asyncio.run_coroutine_threadsafe(self.cdb.stopped_event.wait(), self.asyncio_loop).result()
 
+    def test_find_path_liquidity_hints_failure(self):
+        self.prepare_graph()
+        amount_to_send = 100000
+
+        """
+        assume failure over channel 2, B -> E
+        A -3-> B |-2-> E
+        A -6-> D -5-> E  <= chosen path
+        A -6-> D -4-> C -7-> E
+        A -3-> B -1-> C -7-> E
+        A -6-> D -4-> C -1-> B -2-> E
+        A -3-> B -1-> C -4-> D -5-> E
+        """
+        self.path_finder.liquidity_hints.update_cannot_send(node('b'), node('e'), channel(2), amount_to_send - 1)
+        path = self.path_finder.find_path_for_payment(
+            nodeA=node('a'),
+            nodeB=node('e'),
+            invoice_amount_msat=amount_to_send)
+        self.assertEqual(channel(6), path[0].short_channel_id)
+        self.assertEqual(channel(5), path[1].short_channel_id)
+
+        """
+        assume failure over channel 5, D -> E
+        A -3-> B |-2-> E
+        A -6-> D |-5-> E
+        A -6-> D -4-> C -7-> E
+        A -3-> B -1-> C -7-> E  <= chosen path
+        A -6-> D -4-> C -1-> B |-2-> E
+        A -3-> B -1-> C -4-> D |-5-> E
+        """
+        self.path_finder.liquidity_hints.update_cannot_send(node('d'), node('e'), channel(5), amount_to_send - 1)
+        path = self.path_finder.find_path_for_payment(
+            nodeA=node('a'),
+            nodeB=node('e'),
+            invoice_amount_msat=amount_to_send)
+        self.assertEqual(channel(3), path[0].short_channel_id)
+        self.assertEqual(channel(1), path[1].short_channel_id)
+        self.assertEqual(channel(7), path[2].short_channel_id)
+
+        """
+        assume success over channel 4, D -> C
+        A -3-> B |-2-> E
+        A -6-> D |-5-> E
+        A -6-> D -4-> C -7-> E  <= chosen path
+        A -3-> B -1-> C -7-> E
+        A -6-> D -4-> C -1-> B |-2-> E
+        A -3-> B -1-> C -4-> D |-5-> E
+        """
+        self.path_finder.liquidity_hints.update_can_send(node('d'), node('c'), channel(4), amount_to_send + 1000)
+        path = self.path_finder.find_path_for_payment(
+            nodeA=node('a'),
+            nodeB=node('e'),
+            invoice_amount_msat=amount_to_send)
+        self.assertEqual(channel(6), path[0].short_channel_id)
+        self.assertEqual(channel(4), path[1].short_channel_id)
+        self.assertEqual(channel(7), path[2].short_channel_id)
+
+        self.cdb.stop()
+        asyncio.run_coroutine_threadsafe(self.cdb.stopped_event.wait(), self.asyncio_loop).result()
+
+    def test_liquidity_hints(self):
+        liquidity_hints = LiquidityHintMgr()
+        node_from = bytes(0)
+        node_to = bytes(1)
+        channel_id = ShortChannelID.from_components(0, 0, 0)
+        amount_to_send = 1_000_000
+
+        # check default penalty
+        self.assertEqual(
+            fee_for_edge_msat(amount_to_send, DEFAULT_PENALTY_BASE_MSAT, DEFAULT_PENALTY_PROPORTIONAL_MILLIONTH),
+            liquidity_hints.penalty(node_from, node_to, channel_id, amount_to_send)
+        )
+        liquidity_hints.update_can_send(node_from, node_to, channel_id, 1_000_000)
+        liquidity_hints.update_cannot_send(node_from, node_to, channel_id, 2_000_000)
+        hint = liquidity_hints.get_hint(channel_id)
+        self.assertEqual(1_000_000, hint.can_send(node_from < node_to))
+        self.assertEqual(None, hint.cannot_send(node_to < node_from))
+        self.assertEqual(2_000_000, hint.cannot_send(node_from < node_to))
+        # the can_send backward hint is set automatically
+        self.assertEqual(2_000_000, hint.can_send(node_to < node_from))
+
+        # check penalties
+        self.assertEqual(0., liquidity_hints.penalty(node_from, node_to, channel_id, 1_000_000))
+        self.assertEqual(650, liquidity_hints.penalty(node_from, node_to, channel_id, 1_500_000))
+        self.assertEqual(inf, liquidity_hints.penalty(node_from, node_to, channel_id, 2_000_000))
+
+        # test that we don't overwrite significant info with less significant info
+        liquidity_hints.update_can_send(node_from, node_to, channel_id, 500_000)
+        hint = liquidity_hints.get_hint(channel_id)
+        self.assertEqual(1_000_000, hint.can_send(node_from < node_to))
+
+        # test case when can_send > cannot_send
+        liquidity_hints.update_can_send(node_from, node_to, channel_id, 3_000_000)
+        hint = liquidity_hints.get_hint(channel_id)
+        self.assertEqual(3_000_000, hint.can_send(node_from < node_to))
+        self.assertEqual(None, hint.cannot_send(node_from < node_to))
+
+
     @needs_test_with_all_chacha20_implementations
     def test_new_onion_packet_legacy(self):
         # test vector from bolt-04