Browse Source
Introduces LNRater, which analyzes the Lightning Network graph for potential nodes to connect to by taking into account channel capacities, channel open times and fee policies. A score is constructed to assign a scalar to each node, which is then used to perform a weighted random sampling of the nodes.patch-4
bitromortac
4 years ago
4 changed files with 288 additions and 15 deletions
@ -0,0 +1,255 @@ |
|||||
|
# Copyright (C) 2020 The Electrum developers |
||||
|
# Distributed under the MIT software license, see the accompanying |
||||
|
# file LICENCE or http://www.opensource.org/licenses/mit-license.php |
||||
|
""" |
||||
|
lnrater.py contains Lightning Network node rating functionality. |
||||
|
""" |
||||
|
|
||||
|
import asyncio |
||||
|
from collections import defaultdict |
||||
|
from pprint import pformat |
||||
|
from random import choices |
||||
|
from statistics import mean, median, stdev |
||||
|
from typing import TYPE_CHECKING, Dict, NamedTuple, Tuple, List |
||||
|
import time |
||||
|
|
||||
|
from .logging import Logger |
||||
|
from .util import profiler |
||||
|
from .lnrouter import fee_for_edge_msat |
||||
|
|
||||
|
if TYPE_CHECKING: |
||||
|
from .network import Network |
||||
|
from .channel_db import Policy |
||||
|
from .lnchannel import ShortChannelID |
||||
|
from .lnworker import LNWallet |
||||
|
|
||||
|
|
||||
|
MONTH_IN_BLOCKS = 6 * 24 * 30 |
||||
|
# the scores are only updated after this time interval |
||||
|
RATER_UPDATE_TIME_SEC = 10 * 60 |
||||
|
# amount used for calculating an effective relative fee |
||||
|
FEE_AMOUNT_MSAT = 100_000_000 |
||||
|
|
||||
|
# define some numbers for minimal requirements of good nodes |
||||
|
# exclude nodes with less number of channels |
||||
|
EXCLUDE_NUM_CHANNELS = 15 |
||||
|
# exclude nodes with less mean capacity |
||||
|
EXCLUDE_MEAN_CAPACITY_MSAT = 1_000_000_000 |
||||
|
# exclude nodes which are young |
||||
|
EXCLUDE_NODE_AGE = 2 * MONTH_IN_BLOCKS |
||||
|
# exclude nodes which have young mean channel age |
||||
|
EXCLUDE_MEAN_CHANNEL_AGE = EXCLUDE_NODE_AGE |
||||
|
# exclude nodes which charge a high fee |
||||
|
EXCLUCE_EFFECTIVE_FEE_RATE = 0.001500 |
||||
|
# exclude nodes whose last channel open was a long time ago |
||||
|
EXCLUDE_BLOCKS_LAST_CHANNEL = 3 * MONTH_IN_BLOCKS |
||||
|
|
||||
|
|
||||
|
class NodeStats(NamedTuple): |
||||
|
number_channels: int |
||||
|
# capacity related |
||||
|
total_capacity_msat: int |
||||
|
median_capacity_msat: float |
||||
|
mean_capacity_msat: float |
||||
|
# block height related |
||||
|
node_age_block_height: int |
||||
|
mean_channel_age_block_height: float |
||||
|
blocks_since_last_channel: int |
||||
|
# fees |
||||
|
mean_fee_rate: float |
||||
|
|
||||
|
|
||||
|
def weighted_sum(numbers: List[float], weights: List[float]) -> float: |
||||
|
running_sum = 0.0 |
||||
|
for n, w in zip(numbers, weights): |
||||
|
running_sum += n * w |
||||
|
return running_sum/sum(weights) |
||||
|
|
||||
|
|
||||
|
class LNRater(Logger): |
||||
|
def __init__(self, lnworker: 'LNWallet', network: 'Network'): |
||||
|
"""LNRater can be used to suggest nodes to open up channels with. |
||||
|
|
||||
|
The graph is analyzed and some heuristics are applied to sort out nodes |
||||
|
that are deemed to be bad routers or unmaintained. |
||||
|
""" |
||||
|
Logger.__init__(self) |
||||
|
self.lnworker = lnworker |
||||
|
self.network = network |
||||
|
self.channel_db = self.network.channel_db |
||||
|
|
||||
|
self._node_stats: Dict[bytes, NodeStats] = {} # node_id -> NodeStats |
||||
|
self._node_ratings: Dict[bytes, float] = {} # node_id -> float |
||||
|
self._policies_by_nodes: Dict[bytes, List[Tuple[ShortChannelID, Policy]]] = defaultdict(list) # node_id -> (short_channel_id, policy) |
||||
|
self._last_analyzed = 0 # timestamp |
||||
|
self._last_progress_percent = 0 |
||||
|
|
||||
|
def maybe_analyze_graph(self): |
||||
|
asyncio.run(self._maybe_analyze_graph()) |
||||
|
|
||||
|
def analyze_graph(self): |
||||
|
"""Forces a graph analysis, e.g., due to external triggers like |
||||
|
the graph info reaching 50%.""" |
||||
|
asyncio.run(self._analyze_graph()) |
||||
|
|
||||
|
async def _maybe_analyze_graph(self): |
||||
|
"""Analyzes the graph when in early sync stage (>30%) or when caching |
||||
|
time expires.""" |
||||
|
# gather information about graph sync status |
||||
|
current_channels, total, progress_percent = self.network.lngossip.get_sync_progress_estimate() |
||||
|
|
||||
|
# gossip sync progress state could be None when not started, but channel |
||||
|
# db already knows something about the graph, which is why we allow to |
||||
|
# evaluate the graph early |
||||
|
if progress_percent is not None or self.channel_db.num_nodes > 500: |
||||
|
progress_percent = progress_percent or 0 # convert None to 0 |
||||
|
now = time.time() |
||||
|
# graph should have changed significantly during the sync progress |
||||
|
# or last analysis was a long time ago |
||||
|
if (30 <= progress_percent and progress_percent - self._last_progress_percent >= 10 or |
||||
|
self._last_analyzed + RATER_UPDATE_TIME_SEC < now): |
||||
|
await self._analyze_graph() |
||||
|
self._last_progress_percent = progress_percent |
||||
|
self._last_analyzed = now |
||||
|
|
||||
|
async def _analyze_graph(self): |
||||
|
await self.channel_db.data_loaded.wait() |
||||
|
self._collect_policies_by_node() |
||||
|
loop = asyncio.get_running_loop() |
||||
|
# the analysis is run in an executor because it's costly |
||||
|
await loop.run_in_executor(None, self._collect_purged_stats) |
||||
|
self._rate_nodes() |
||||
|
now = time.time() |
||||
|
self._last_analyzed = now |
||||
|
|
||||
|
def _collect_policies_by_node(self): |
||||
|
policies = self.channel_db.get_node_policies() |
||||
|
for pv, p in policies.items(): |
||||
|
# append tuples of ShortChannelID and Policy |
||||
|
self._policies_by_nodes[pv[0]].append((pv[1], p)) |
||||
|
|
||||
|
@profiler |
||||
|
def _collect_purged_stats(self): |
||||
|
"""Traverses through the graph and sorts out nodes.""" |
||||
|
current_height = self.network.get_local_height() |
||||
|
node_infos = self.channel_db.get_node_infos() |
||||
|
|
||||
|
for n, channel_policies in self._policies_by_nodes.items(): |
||||
|
try: |
||||
|
# use policies synonymously to channels |
||||
|
num_channels = len(channel_policies) |
||||
|
|
||||
|
# save some time for nodes we are not interested in: |
||||
|
if num_channels < EXCLUDE_NUM_CHANNELS: |
||||
|
continue |
||||
|
|
||||
|
# analyze block heights |
||||
|
block_heights = [p[0].block_height for p in channel_policies] |
||||
|
node_age_bh = current_height - min(block_heights) |
||||
|
if node_age_bh < EXCLUDE_NODE_AGE: |
||||
|
continue |
||||
|
mean_channel_age_bh = current_height - mean(block_heights) |
||||
|
if mean_channel_age_bh < EXCLUDE_MEAN_CHANNEL_AGE: |
||||
|
continue |
||||
|
blocks_since_last_channel = current_height - max(block_heights) |
||||
|
if blocks_since_last_channel > EXCLUDE_BLOCKS_LAST_CHANNEL: |
||||
|
continue |
||||
|
|
||||
|
# analyze capacities |
||||
|
capacities = [p[1].htlc_maximum_msat for p in channel_policies] |
||||
|
if None in capacities: |
||||
|
continue |
||||
|
total_capacity = sum(capacities) |
||||
|
|
||||
|
mean_capacity = total_capacity / num_channels if num_channels else 0 |
||||
|
if mean_capacity < EXCLUDE_MEAN_CAPACITY_MSAT: |
||||
|
continue |
||||
|
median_capacity = median(capacities) |
||||
|
|
||||
|
# analyze fees |
||||
|
effective_fee_rates = [fee_for_edge_msat( |
||||
|
FEE_AMOUNT_MSAT, |
||||
|
p[1].fee_base_msat, |
||||
|
p[1].fee_proportional_millionths) / FEE_AMOUNT_MSAT for p in channel_policies] |
||||
|
mean_fees_rate = mean(effective_fee_rates) |
||||
|
if mean_fees_rate > EXCLUCE_EFFECTIVE_FEE_RATE: |
||||
|
continue |
||||
|
|
||||
|
self._node_stats[n] = NodeStats( |
||||
|
number_channels=num_channels, |
||||
|
total_capacity_msat=total_capacity, |
||||
|
median_capacity_msat=median_capacity, |
||||
|
mean_capacity_msat=mean_capacity, |
||||
|
node_age_block_height=node_age_bh, |
||||
|
mean_channel_age_block_height=mean_channel_age_bh, |
||||
|
blocks_since_last_channel=blocks_since_last_channel, |
||||
|
mean_fee_rate=mean_fees_rate |
||||
|
) |
||||
|
|
||||
|
except Exception as e: |
||||
|
self.logger.exception("Could not use channel policies for " |
||||
|
"calculating statistics.") |
||||
|
self.logger.debug(pformat(channel_policies)) |
||||
|
continue |
||||
|
|
||||
|
self.logger.info(f"node statistics done, calculated statistics" |
||||
|
f"for {len(self._node_stats)} nodes") |
||||
|
|
||||
|
def _rate_nodes(self): |
||||
|
"""Rate nodes by collected statistics.""" |
||||
|
|
||||
|
max_capacity = 0 |
||||
|
max_num_chan = 0 |
||||
|
min_fee_rate = float('inf') |
||||
|
for stats in self._node_stats.values(): |
||||
|
max_capacity = max(max_capacity, stats.total_capacity_msat) |
||||
|
max_num_chan = max(max_num_chan, stats.number_channels) |
||||
|
min_fee_rate = min(min_fee_rate, stats.mean_fee_rate) |
||||
|
|
||||
|
for n, stats in self._node_stats.items(): |
||||
|
heuristics = [] |
||||
|
heuristics_weights = [] |
||||
|
|
||||
|
# example of how we could construct a scalar score for nodes |
||||
|
# this is probably not what we want to to, this is roughly |
||||
|
# preferential attachment |
||||
|
|
||||
|
# number of channels |
||||
|
heuristics.append(stats.number_channels / max_num_chan) |
||||
|
heuristics_weights.append(0.2) |
||||
|
# total capacity |
||||
|
heuristics.append(stats.total_capacity_msat / max_capacity) |
||||
|
heuristics_weights.append(0.8) |
||||
|
# inverse fees |
||||
|
fees = min(1E-6, min_fee_rate) / max(1E-10, stats.mean_fee_rate) |
||||
|
heuristics.append(fees) |
||||
|
heuristics_weights.append(1.0) |
||||
|
|
||||
|
self._node_ratings[n] = weighted_sum(heuristics, heuristics_weights) |
||||
|
|
||||
|
def suggest_node_channel_open(self) -> Tuple[bytes, NodeStats]: |
||||
|
node_keys = list(self._node_stats.keys()) |
||||
|
node_ratings = list(self._node_ratings.values()) |
||||
|
channel_peers = self.lnworker.channel_peers() |
||||
|
|
||||
|
while True: |
||||
|
# randomly pick nodes weighted by node_rating |
||||
|
pk = choices(node_keys, weights=node_ratings, k=1)[0] |
||||
|
|
||||
|
# don't want to connect to nodes we are already connected to |
||||
|
if pk not in channel_peers: |
||||
|
break |
||||
|
|
||||
|
node_infos = self.channel_db.get_node_infos() |
||||
|
self.logger.info( |
||||
|
f"node rating for {node_infos[pk].alias}:\n" |
||||
|
f"{pformat(self._node_stats[pk])} (score {self._node_ratings[pk]})") |
||||
|
|
||||
|
return pk, self._node_stats[pk] |
||||
|
|
||||
|
def suggest_peer(self): |
||||
|
self.maybe_analyze_graph() |
||||
|
if self._node_ratings: |
||||
|
return self.suggest_node_channel_open()[0] |
||||
|
else: |
||||
|
return None |
Loading…
Reference in new issue