Browse Source
- The splitting algorithm is redesigned to use random distribution of subsplittings over channels. - Splittings can include multiple subamounts within a channel. - The single-channel splittings are implicitly activated once the liquidity hints don't support payments of large size.patch-4
bitromortac
4 years ago
3 changed files with 217 additions and 276 deletions
@ -1,259 +1,181 @@ |
|||
import random |
|||
import math |
|||
from typing import List, Tuple, Optional, Sequence, Dict, TYPE_CHECKING |
|||
from typing import List, Tuple, Dict, NamedTuple |
|||
from collections import defaultdict |
|||
|
|||
from .util import profiler |
|||
from .lnutil import NoPathFound |
|||
|
|||
PART_PENALTY = 1.0 # 1.0 results in avoiding splits |
|||
MIN_PART_MSAT = 10_000_000 # we don't want to split indefinitely |
|||
MIN_PART_SIZE_MSAT = 10_000_000 # we don't want to split indefinitely |
|||
EXHAUST_DECAY_FRACTION = 10 # fraction of the local balance that should be reserved if possible |
|||
|
|||
# these parameters determine the granularity of the newly suggested configurations |
|||
REDISTRIBUTION_FRACTION = 50 |
|||
SPLIT_FRACTION = 50 |
|||
RELATIVE_SPLIT_SPREAD = 0.3 # deviation from the mean when splitting amounts into parts |
|||
|
|||
# these parameters affect the computational work in the probabilistic algorithm |
|||
STARTING_CONFIGS = 50 |
|||
CANDIDATES_PER_LEVEL = 10 |
|||
REDISTRIBUTE = 20 |
|||
|
|||
# maximum number of parts for splitting |
|||
MAX_PARTS = 5 |
|||
|
|||
|
|||
def unique_hierarchy(hierarchy: Dict[int, List[Dict[Tuple[bytes, bytes], int]]]) -> Dict[int, List[Dict[Tuple[bytes, bytes], int]]]: |
|||
new_hierarchy = defaultdict(list) |
|||
for number_parts, configs in hierarchy.items(): |
|||
unique_configs = set() |
|||
for config in configs: |
|||
# config dict can be out of order, so sort, otherwise not unique |
|||
unique_configs.add(tuple((c, config[c]) for c in sorted(config.keys()))) |
|||
for unique_config in sorted(unique_configs): |
|||
new_hierarchy[number_parts].append( |
|||
{t[0]: t[1] for t in unique_config}) |
|||
return new_hierarchy |
|||
|
|||
|
|||
def single_node_hierarchy(hierarchy: Dict[int, List[Dict[Tuple[bytes, bytes], int]]]) -> Dict[int, List[Dict[Tuple[bytes, bytes], int]]]: |
|||
new_hierarchy = defaultdict(list) |
|||
for number_parts, configs in hierarchy.items(): |
|||
for config in configs: |
|||
# determine number of nodes in configuration |
|||
if number_nonzero_nodes(config) > 1: |
|||
continue |
|||
new_hierarchy[number_parts].append(config) |
|||
return new_hierarchy |
|||
CANDIDATES_PER_LEVEL = 20 |
|||
MAX_PARTS = 5 # maximum number of parts for splitting |
|||
|
|||
|
|||
# maps a channel (channel_id, node_id) to a list of amounts |
|||
SplitConfig = Dict[Tuple[bytes, bytes], List[int]] |
|||
# maps a channel (channel_id, node_id) to the funds it has available |
|||
ChannelsFundsInfo = Dict[Tuple[bytes, bytes], int] |
|||
|
|||
|
|||
class SplitConfigRating(NamedTuple): |
|||
config: SplitConfig |
|||
rating: float |
|||
|
|||
|
|||
def split_amount_normal(total_amount: int, num_parts: int) -> List[int]: |
|||
"""Splits an amount into about `num_parts` parts, where the parts are split |
|||
randomly (normally distributed around amount/num_parts with certain spread).""" |
|||
parts = [] |
|||
avg_amount = total_amount / num_parts |
|||
# roughly reach total_amount |
|||
while total_amount - sum(parts) > avg_amount: |
|||
amount_to_add = int(abs(random.gauss(avg_amount, RELATIVE_SPLIT_SPREAD * avg_amount))) |
|||
if sum(parts) + amount_to_add < total_amount: |
|||
parts.append(amount_to_add) |
|||
# add what's missing |
|||
parts.append(total_amount - sum(parts)) |
|||
return parts |
|||
|
|||
|
|||
def number_parts(config: SplitConfig) -> int: |
|||
return sum([len(v) for v in config.values() if sum(v)]) |
|||
|
|||
|
|||
def number_nonzero_channels(config: SplitConfig) -> int: |
|||
return len([v for v in config.values() if sum(v)]) |
|||
|
|||
|
|||
def number_nonzero_nodes(config: SplitConfig) -> int: |
|||
# using a set comprehension |
|||
return len({nodeid for (_, nodeid), amounts in config.items() if sum(amounts)}) |
|||
|
|||
|
|||
def total_config_amount(config: SplitConfig) -> int: |
|||
return sum([sum(c) for c in config.values()]) |
|||
|
|||
|
|||
def number_nonzero_parts(configuration: Dict[Tuple[bytes, bytes], int]) -> int: |
|||
return len([v for v in configuration.values() if v]) |
|||
def is_any_amount_smaller_than_min_part_size(config: SplitConfig) -> bool: |
|||
smaller = False |
|||
for amounts in config.values(): |
|||
if any([amount < MIN_PART_SIZE_MSAT for amount in amounts]): |
|||
smaller |= True |
|||
return smaller |
|||
|
|||
|
|||
def number_nonzero_nodes(configuration: Dict[Tuple[bytes, bytes], int]) -> int: |
|||
return len({nodeid for (_, nodeid), amount in configuration.items() if amount > 0}) |
|||
def remove_duplicates(configs: List[SplitConfig]) -> List[SplitConfig]: |
|||
unique_configs = set() |
|||
for config in configs: |
|||
# sort keys and values |
|||
config_sorted_values = {k: sorted(v) for k, v in config.items()} |
|||
config_sorted_keys = {k: config_sorted_values[k] for k in sorted(config_sorted_values.keys())} |
|||
hashable_config = tuple((c, tuple(sorted(config[c]))) for c in config_sorted_keys) |
|||
unique_configs.add(hashable_config) |
|||
unique_configs = [{c[0]: list(c[1]) for c in config} for config in unique_configs] |
|||
return unique_configs |
|||
|
|||
|
|||
def create_starting_split_hierarchy(amount_msat: int, channels_with_funds: Dict[Tuple[bytes, bytes], int]): |
|||
"""Distributes the amount to send to a single or more channels in several |
|||
ways (randomly).""" |
|||
# TODO: find all possible starting configurations deterministically |
|||
# could try all permutations |
|||
def remove_multiple_nodes(configs: List[SplitConfig]) -> List[SplitConfig]: |
|||
return [config for config in configs if number_nonzero_nodes(config) == 1] |
|||
|
|||
split_hierarchy = defaultdict(list) |
|||
|
|||
def remove_single_part_configs(configs: List[SplitConfig]) -> List[SplitConfig]: |
|||
return [config for config in configs if number_parts(config) != 1] |
|||
|
|||
|
|||
def rate_config( |
|||
config: SplitConfig, |
|||
channels_with_funds: ChannelsFundsInfo) -> float: |
|||
"""Defines an objective function to rate a configuration. |
|||
|
|||
We calculate the normalized L2 norm for a configuration and |
|||
add a part penalty for each nonzero amount. The consequence is that |
|||
amounts that are equally distributed and have less parts are rated |
|||
lowest (best). A penalty depending on the total amount sent over a channel |
|||
counteracts channel exhaustion.""" |
|||
rating = 0 |
|||
total_amount = total_config_amount(config) |
|||
|
|||
for channel, amounts in config.items(): |
|||
funds = channels_with_funds[channel] |
|||
if amounts: |
|||
for amount in amounts: |
|||
rating += amount * amount / (total_amount * total_amount) # penalty to favor equal distribution of amounts |
|||
rating += PART_PENALTY * PART_PENALTY # penalty for each part |
|||
decay = funds / EXHAUST_DECAY_FRACTION |
|||
rating += math.exp((sum(amounts) - funds) / decay) # penalty for channel exhaustion |
|||
return rating |
|||
|
|||
|
|||
def suggest_splits( |
|||
amount_msat: int, channels_with_funds: ChannelsFundsInfo, |
|||
exclude_single_part_payments=False, |
|||
exclude_multinode_payments=False |
|||
) -> List[SplitConfigRating]: |
|||
"""Breaks amount_msat into smaller pieces and distributes them over the |
|||
channels according to the funds they can send. |
|||
|
|||
Individual channels may be assigned multiple parts. The split configurations |
|||
are returned in sorted order, from best to worst rating. |
|||
|
|||
Single part payments can be excluded, since they represent legacy payments. |
|||
Split configurations that send via multiple nodes can be excluded as well. |
|||
""" |
|||
|
|||
configs = [] |
|||
channels_order = list(channels_with_funds.keys()) |
|||
|
|||
for _ in range(STARTING_CONFIGS): |
|||
# shuffle to have different starting points |
|||
random.shuffle(channels_order) |
|||
|
|||
configuration = {} |
|||
amount_added = 0 |
|||
for c in channels_order: |
|||
s = channels_with_funds[c] |
|||
if amount_added == amount_msat: |
|||
configuration[c] = 0 |
|||
else: |
|||
amount_to_add = amount_msat - amount_added |
|||
amt = min(s, amount_to_add) |
|||
configuration[c] = amt |
|||
amount_added += amt |
|||
if amount_added != amount_msat: |
|||
raise NoPathFound("Channels don't have enough sending capacity.") |
|||
split_hierarchy[number_nonzero_parts(configuration)].append(configuration) |
|||
|
|||
return unique_hierarchy(split_hierarchy) |
|||
|
|||
|
|||
def balances_are_not_ok(proposed_balance_from, channel_from, proposed_balance_to, channel_to, channels_with_funds): |
|||
check = ( |
|||
proposed_balance_to < MIN_PART_MSAT or |
|||
proposed_balance_to > channels_with_funds[channel_to] or |
|||
proposed_balance_from < MIN_PART_MSAT or |
|||
proposed_balance_from > channels_with_funds[channel_from] |
|||
) |
|||
return check |
|||
|
|||
|
|||
def propose_new_configuration(channels_with_funds: Dict[Tuple[bytes, bytes], int], configuration: Dict[Tuple[bytes, bytes], int], |
|||
amount_msat: int, preserve_number_parts=True) -> Dict[Tuple[bytes, bytes], int]: |
|||
"""Randomly alters a split configuration. If preserve_number_parts, the |
|||
configuration stays within the same class of number of splits.""" |
|||
|
|||
# there are three basic operations to reach different split configurations: |
|||
# redistribute, split, swap |
|||
|
|||
def redistribute(config: dict): |
|||
# we redistribute the amount from a nonzero channel to a nonzero channel |
|||
redistribution_amount = amount_msat // REDISTRIBUTION_FRACTION |
|||
nonzero = [ck for ck, cv in config.items() if |
|||
cv >= redistribution_amount] |
|||
if len(nonzero) == 1: # we only have a single channel, so we can't redistribute |
|||
return config |
|||
|
|||
channel_from = random.choice(nonzero) |
|||
channel_to = random.choice(nonzero) |
|||
if channel_from == channel_to: |
|||
return config |
|||
proposed_balance_from = config[channel_from] - redistribution_amount |
|||
proposed_balance_to = config[channel_to] + redistribution_amount |
|||
if balances_are_not_ok(proposed_balance_from, channel_from, proposed_balance_to, channel_to, channels_with_funds): |
|||
return config |
|||
else: |
|||
config[channel_from] = proposed_balance_from |
|||
config[channel_to] = proposed_balance_to |
|||
assert sum([cv for cv in config.values()]) == amount_msat |
|||
return config |
|||
|
|||
def split(config: dict): |
|||
# we split off a certain amount from a nonzero channel and put it into a |
|||
# zero channel |
|||
nonzero = [ck for ck, cv in config.items() if cv != 0] |
|||
zero = [ck for ck, cv in config.items() if cv == 0] |
|||
try: |
|||
channel_from = random.choice(nonzero) |
|||
channel_to = random.choice(zero) |
|||
except IndexError: |
|||
return config |
|||
delta = config[channel_from] // SPLIT_FRACTION |
|||
proposed_balance_from = config[channel_from] - delta |
|||
proposed_balance_to = config[channel_to] + delta |
|||
if balances_are_not_ok(proposed_balance_from, channel_from, proposed_balance_to, channel_to, channels_with_funds): |
|||
return config |
|||
else: |
|||
config[channel_from] = proposed_balance_from |
|||
config[channel_to] = proposed_balance_to |
|||
assert sum([cv for cv in config.values()]) == amount_msat |
|||
return config |
|||
|
|||
def swap(config: dict): |
|||
# we swap the amounts from a single channel with another channel |
|||
nonzero = [ck for ck, cv in config.items() if cv != 0] |
|||
all = list(config.keys()) |
|||
|
|||
channel_from = random.choice(nonzero) |
|||
channel_to = random.choice(all) |
|||
|
|||
proposed_balance_to = config[channel_from] |
|||
proposed_balance_from = config[channel_to] |
|||
if balances_are_not_ok(proposed_balance_from, channel_from, proposed_balance_to, channel_to, channels_with_funds): |
|||
return config |
|||
else: |
|||
config[channel_to] = proposed_balance_to |
|||
config[channel_from] = proposed_balance_from |
|||
return config |
|||
|
|||
initial_number_parts = number_nonzero_parts(configuration) |
|||
|
|||
for _ in range(REDISTRIBUTE): |
|||
configuration = redistribute(configuration) |
|||
if not preserve_number_parts and number_nonzero_parts( |
|||
configuration) == initial_number_parts: |
|||
configuration = split(configuration) |
|||
configuration = swap(configuration) |
|||
|
|||
return configuration |
|||
|
|||
|
|||
@profiler |
|||
def suggest_splits(amount_msat: int, channels_with_funds: Dict[Tuple[bytes, bytes], int], |
|||
exclude_single_parts=True, single_node=False) \ |
|||
-> Sequence[Tuple[Dict[Tuple[bytes, bytes], int], float]]: |
|||
"""Creates split configurations for a payment over channels. Single channel |
|||
payments are excluded by default. channels_with_funds is keyed by |
|||
(channelid, nodeid).""" |
|||
|
|||
def rate_configuration(config: dict) -> float: |
|||
"""Defines an objective function to rate a split configuration. |
|||
|
|||
We calculate the normalized L2 norm for a split configuration and |
|||
add a part penalty for each nonzero amount. The consequence is that |
|||
amounts that are equally distributed and have less parts are rated |
|||
lowest.""" |
|||
F = 0 |
|||
total_amount = sum([v for v in config.values()]) |
|||
|
|||
for channel, amount in config.items(): |
|||
funds = channels_with_funds[channel] |
|||
if amount: |
|||
F += amount * amount / (total_amount * total_amount) # a penalty to favor distribution of amounts |
|||
F += PART_PENALTY * PART_PENALTY # a penalty for each part |
|||
decay = funds / EXHAUST_DECAY_FRACTION |
|||
F += math.exp((amount - funds) / decay) # a penalty for channel saturation |
|||
|
|||
return F |
|||
|
|||
def rated_sorted_configurations(hierarchy: dict) -> Sequence[Tuple[Dict[Tuple[bytes, bytes], int], float]]: |
|||
"""Cleans up duplicate splittings, rates and sorts them according to |
|||
the rating. A lower rating is a better configuration.""" |
|||
hierarchy = unique_hierarchy(hierarchy) |
|||
rated_configs = [] |
|||
for level, configs in hierarchy.items(): |
|||
for config in configs: |
|||
rated_configs.append((config, rate_configuration(config))) |
|||
sorted_rated_configs = sorted(rated_configs, key=lambda c: c[1], reverse=False) |
|||
return sorted_rated_configs |
|||
|
|||
# create initial guesses |
|||
split_hierarchy = create_starting_split_hierarchy(amount_msat, channels_with_funds) |
|||
|
|||
# randomize initial guesses and generate splittings of different split |
|||
# levels up to number of channels |
|||
for level in range(2, min(MAX_PARTS, len(channels_with_funds) + 1)): |
|||
# generate a set of random configurations for each level |
|||
for _ in range(CANDIDATES_PER_LEVEL): |
|||
configurations = unique_hierarchy(split_hierarchy).get(level, None) |
|||
if configurations: # we have a splitting of the desired number of parts |
|||
configuration = random.choice(configurations) |
|||
# generate new splittings preserving the number of parts |
|||
configuration = propose_new_configuration( |
|||
channels_with_funds, configuration, amount_msat, |
|||
preserve_number_parts=True) |
|||
else: |
|||
# go one level lower and look for valid splittings, |
|||
# try to go one level higher by splitting a single outgoing amount |
|||
configurations = unique_hierarchy(split_hierarchy).get(level - 1, None) |
|||
if not configurations: |
|||
continue |
|||
configuration = random.choice(configurations) |
|||
# generate new splittings going one level higher in the number of parts |
|||
configuration = propose_new_configuration( |
|||
channels_with_funds, configuration, amount_msat, |
|||
preserve_number_parts=False) |
|||
|
|||
# add the newly found configuration (doesn't matter if nothing changed) |
|||
split_hierarchy[number_nonzero_parts(configuration)].append(configuration) |
|||
|
|||
if exclude_single_parts: |
|||
# we only want to return configurations that have at least two parts |
|||
try: |
|||
del split_hierarchy[1] |
|||
except: |
|||
pass |
|||
|
|||
if single_node: |
|||
# we only take configurations that send to a single node |
|||
split_hierarchy = single_node_hierarchy(split_hierarchy) |
|||
|
|||
return rated_sorted_configurations(split_hierarchy) |
|||
# generate multiple configurations to get more configurations (there is randomness in this loop) |
|||
for _ in range(CANDIDATES_PER_LEVEL): |
|||
# we want to have configurations with no splitting to many splittings |
|||
for target_parts in range(1, MAX_PARTS): |
|||
config = defaultdict(list) # type: SplitConfig |
|||
|
|||
# randomly split amount into target_parts chunks |
|||
split_amounts = split_amount_normal(amount_msat, target_parts) |
|||
# randomly distribute amounts over channels |
|||
for amount in split_amounts: |
|||
random.shuffle(channels_order) |
|||
# we check each channel and try to put the funds inside, break if we succeed |
|||
for c in channels_order: |
|||
if sum(config[c]) + amount <= channels_with_funds[c]: |
|||
config[c].append(amount) |
|||
break |
|||
# if we don't succeed to put the amount anywhere, |
|||
# we try to fill up channels and put the rest somewhere else |
|||
else: |
|||
distribute_amount = amount |
|||
for c in channels_order: |
|||
funds_left = channels_with_funds[c] - sum(config[c]) |
|||
# it would be good to not fill the full channel if possible |
|||
add_amount = min(funds_left, distribute_amount) |
|||
config[c].append(add_amount) |
|||
distribute_amount -= add_amount |
|||
if distribute_amount == 0: |
|||
break |
|||
if total_config_amount(config) != amount_msat: |
|||
raise NoPathFound('Cannot distribute payment over channels.') |
|||
if target_parts > 1 and is_any_amount_smaller_than_min_part_size(config): |
|||
continue |
|||
assert total_config_amount(config) == amount_msat |
|||
configs.append(config) |
|||
|
|||
configs = remove_duplicates(configs) |
|||
|
|||
# we only take configurations that send via a single node (but there can be multiple parts) |
|||
if exclude_multinode_payments: |
|||
configs = remove_multiple_nodes(configs) |
|||
|
|||
if exclude_single_part_payments: |
|||
configs = remove_single_part_configs(configs) |
|||
|
|||
rated_configs = [SplitConfigRating( |
|||
config=c, |
|||
rating=rate_config(c, channels_with_funds) |
|||
) for c in configs] |
|||
rated_configs.sort(key=lambda x: x.rating) |
|||
|
|||
return rated_configs |
|||
|
|||
Loading…
Reference in new issue