Browse Source
- The splitting algorithm is redesigned to use random distribution of subsplittings over channels. - Splittings can include multiple subamounts within a channel. - The single-channel splittings are implicitly activated once the liquidity hints don't support payments of large size.patch-4
bitromortac
4 years ago
3 changed files with 217 additions and 276 deletions
@ -1,259 +1,181 @@ |
|||||
import random |
import random |
||||
import math |
import math |
||||
from typing import List, Tuple, Optional, Sequence, Dict, TYPE_CHECKING |
from typing import List, Tuple, Dict, NamedTuple |
||||
from collections import defaultdict |
from collections import defaultdict |
||||
|
|
||||
from .util import profiler |
|
||||
from .lnutil import NoPathFound |
from .lnutil import NoPathFound |
||||
|
|
||||
PART_PENALTY = 1.0 # 1.0 results in avoiding splits |
PART_PENALTY = 1.0 # 1.0 results in avoiding splits |
||||
MIN_PART_MSAT = 10_000_000 # we don't want to split indefinitely |
MIN_PART_SIZE_MSAT = 10_000_000 # we don't want to split indefinitely |
||||
EXHAUST_DECAY_FRACTION = 10 # fraction of the local balance that should be reserved if possible |
EXHAUST_DECAY_FRACTION = 10 # fraction of the local balance that should be reserved if possible |
||||
|
RELATIVE_SPLIT_SPREAD = 0.3 # deviation from the mean when splitting amounts into parts |
||||
# these parameters determine the granularity of the newly suggested configurations |
|
||||
REDISTRIBUTION_FRACTION = 50 |
|
||||
SPLIT_FRACTION = 50 |
|
||||
|
|
||||
# these parameters affect the computational work in the probabilistic algorithm |
# these parameters affect the computational work in the probabilistic algorithm |
||||
STARTING_CONFIGS = 50 |
CANDIDATES_PER_LEVEL = 20 |
||||
CANDIDATES_PER_LEVEL = 10 |
MAX_PARTS = 5 # maximum number of parts for splitting |
||||
REDISTRIBUTE = 20 |
|
||||
|
|
||||
# maximum number of parts for splitting |
|
||||
MAX_PARTS = 5 |
|
||||
|
|
||||
|
# maps a channel (channel_id, node_id) to a list of amounts |
||||
|
SplitConfig = Dict[Tuple[bytes, bytes], List[int]] |
||||
|
# maps a channel (channel_id, node_id) to the funds it has available |
||||
|
ChannelsFundsInfo = Dict[Tuple[bytes, bytes], int] |
||||
|
|
||||
def unique_hierarchy(hierarchy: Dict[int, List[Dict[Tuple[bytes, bytes], int]]]) -> Dict[int, List[Dict[Tuple[bytes, bytes], int]]]: |
|
||||
new_hierarchy = defaultdict(list) |
|
||||
for number_parts, configs in hierarchy.items(): |
|
||||
unique_configs = set() |
|
||||
for config in configs: |
|
||||
# config dict can be out of order, so sort, otherwise not unique |
|
||||
unique_configs.add(tuple((c, config[c]) for c in sorted(config.keys()))) |
|
||||
for unique_config in sorted(unique_configs): |
|
||||
new_hierarchy[number_parts].append( |
|
||||
{t[0]: t[1] for t in unique_config}) |
|
||||
return new_hierarchy |
|
||||
|
|
||||
|
class SplitConfigRating(NamedTuple): |
||||
|
config: SplitConfig |
||||
|
rating: float |
||||
|
|
||||
def single_node_hierarchy(hierarchy: Dict[int, List[Dict[Tuple[bytes, bytes], int]]]) -> Dict[int, List[Dict[Tuple[bytes, bytes], int]]]: |
|
||||
new_hierarchy = defaultdict(list) |
|
||||
for number_parts, configs in hierarchy.items(): |
|
||||
for config in configs: |
|
||||
# determine number of nodes in configuration |
|
||||
if number_nonzero_nodes(config) > 1: |
|
||||
continue |
|
||||
new_hierarchy[number_parts].append(config) |
|
||||
return new_hierarchy |
|
||||
|
|
||||
|
def split_amount_normal(total_amount: int, num_parts: int) -> List[int]: |
||||
|
"""Splits an amount into about `num_parts` parts, where the parts are split |
||||
|
randomly (normally distributed around amount/num_parts with certain spread).""" |
||||
|
parts = [] |
||||
|
avg_amount = total_amount / num_parts |
||||
|
# roughly reach total_amount |
||||
|
while total_amount - sum(parts) > avg_amount: |
||||
|
amount_to_add = int(abs(random.gauss(avg_amount, RELATIVE_SPLIT_SPREAD * avg_amount))) |
||||
|
if sum(parts) + amount_to_add < total_amount: |
||||
|
parts.append(amount_to_add) |
||||
|
# add what's missing |
||||
|
parts.append(total_amount - sum(parts)) |
||||
|
return parts |
||||
|
|
||||
def number_nonzero_parts(configuration: Dict[Tuple[bytes, bytes], int]) -> int: |
|
||||
return len([v for v in configuration.values() if v]) |
|
||||
|
|
||||
|
def number_parts(config: SplitConfig) -> int: |
||||
|
return sum([len(v) for v in config.values() if sum(v)]) |
||||
|
|
||||
def number_nonzero_nodes(configuration: Dict[Tuple[bytes, bytes], int]) -> int: |
|
||||
return len({nodeid for (_, nodeid), amount in configuration.items() if amount > 0}) |
|
||||
|
|
||||
|
def number_nonzero_channels(config: SplitConfig) -> int: |
||||
|
return len([v for v in config.values() if sum(v)]) |
||||
|
|
||||
def create_starting_split_hierarchy(amount_msat: int, channels_with_funds: Dict[Tuple[bytes, bytes], int]): |
|
||||
"""Distributes the amount to send to a single or more channels in several |
|
||||
ways (randomly).""" |
|
||||
# TODO: find all possible starting configurations deterministically |
|
||||
# could try all permutations |
|
||||
|
|
||||
split_hierarchy = defaultdict(list) |
def number_nonzero_nodes(config: SplitConfig) -> int: |
||||
channels_order = list(channels_with_funds.keys()) |
# using a set comprehension |
||||
|
return len({nodeid for (_, nodeid), amounts in config.items() if sum(amounts)}) |
||||
|
|
||||
for _ in range(STARTING_CONFIGS): |
|
||||
# shuffle to have different starting points |
|
||||
random.shuffle(channels_order) |
|
||||
|
|
||||
configuration = {} |
def total_config_amount(config: SplitConfig) -> int: |
||||
amount_added = 0 |
return sum([sum(c) for c in config.values()]) |
||||
for c in channels_order: |
|
||||
s = channels_with_funds[c] |
|
||||
if amount_added == amount_msat: |
def is_any_amount_smaller_than_min_part_size(config: SplitConfig) -> bool: |
||||
configuration[c] = 0 |
smaller = False |
||||
else: |
for amounts in config.values(): |
||||
amount_to_add = amount_msat - amount_added |
if any([amount < MIN_PART_SIZE_MSAT for amount in amounts]): |
||||
amt = min(s, amount_to_add) |
smaller |= True |
||||
configuration[c] = amt |
return smaller |
||||
amount_added += amt |
|
||||
if amount_added != amount_msat: |
|
||||
raise NoPathFound("Channels don't have enough sending capacity.") |
|
||||
split_hierarchy[number_nonzero_parts(configuration)].append(configuration) |
|
||||
|
|
||||
return unique_hierarchy(split_hierarchy) |
|
||||
|
|
||||
|
|
||||
def balances_are_not_ok(proposed_balance_from, channel_from, proposed_balance_to, channel_to, channels_with_funds): |
|
||||
check = ( |
|
||||
proposed_balance_to < MIN_PART_MSAT or |
|
||||
proposed_balance_to > channels_with_funds[channel_to] or |
|
||||
proposed_balance_from < MIN_PART_MSAT or |
|
||||
proposed_balance_from > channels_with_funds[channel_from] |
|
||||
) |
|
||||
return check |
|
||||
|
|
||||
|
|
||||
def propose_new_configuration(channels_with_funds: Dict[Tuple[bytes, bytes], int], configuration: Dict[Tuple[bytes, bytes], int], |
|
||||
amount_msat: int, preserve_number_parts=True) -> Dict[Tuple[bytes, bytes], int]: |
|
||||
"""Randomly alters a split configuration. If preserve_number_parts, the |
|
||||
configuration stays within the same class of number of splits.""" |
|
||||
|
|
||||
# there are three basic operations to reach different split configurations: |
|
||||
# redistribute, split, swap |
|
||||
|
|
||||
def redistribute(config: dict): |
|
||||
# we redistribute the amount from a nonzero channel to a nonzero channel |
|
||||
redistribution_amount = amount_msat // REDISTRIBUTION_FRACTION |
|
||||
nonzero = [ck for ck, cv in config.items() if |
|
||||
cv >= redistribution_amount] |
|
||||
if len(nonzero) == 1: # we only have a single channel, so we can't redistribute |
|
||||
return config |
|
||||
|
|
||||
channel_from = random.choice(nonzero) |
|
||||
channel_to = random.choice(nonzero) |
|
||||
if channel_from == channel_to: |
|
||||
return config |
|
||||
proposed_balance_from = config[channel_from] - redistribution_amount |
|
||||
proposed_balance_to = config[channel_to] + redistribution_amount |
|
||||
if balances_are_not_ok(proposed_balance_from, channel_from, proposed_balance_to, channel_to, channels_with_funds): |
|
||||
return config |
|
||||
else: |
|
||||
config[channel_from] = proposed_balance_from |
|
||||
config[channel_to] = proposed_balance_to |
|
||||
assert sum([cv for cv in config.values()]) == amount_msat |
|
||||
return config |
|
||||
|
|
||||
def split(config: dict): |
|
||||
# we split off a certain amount from a nonzero channel and put it into a |
|
||||
# zero channel |
|
||||
nonzero = [ck for ck, cv in config.items() if cv != 0] |
|
||||
zero = [ck for ck, cv in config.items() if cv == 0] |
|
||||
try: |
|
||||
channel_from = random.choice(nonzero) |
|
||||
channel_to = random.choice(zero) |
|
||||
except IndexError: |
|
||||
return config |
|
||||
delta = config[channel_from] // SPLIT_FRACTION |
|
||||
proposed_balance_from = config[channel_from] - delta |
|
||||
proposed_balance_to = config[channel_to] + delta |
|
||||
if balances_are_not_ok(proposed_balance_from, channel_from, proposed_balance_to, channel_to, channels_with_funds): |
|
||||
return config |
|
||||
else: |
|
||||
config[channel_from] = proposed_balance_from |
|
||||
config[channel_to] = proposed_balance_to |
|
||||
assert sum([cv for cv in config.values()]) == amount_msat |
|
||||
return config |
|
||||
|
|
||||
def swap(config: dict): |
|
||||
# we swap the amounts from a single channel with another channel |
|
||||
nonzero = [ck for ck, cv in config.items() if cv != 0] |
|
||||
all = list(config.keys()) |
|
||||
|
|
||||
channel_from = random.choice(nonzero) |
|
||||
channel_to = random.choice(all) |
|
||||
|
|
||||
proposed_balance_to = config[channel_from] |
|
||||
proposed_balance_from = config[channel_to] |
|
||||
if balances_are_not_ok(proposed_balance_from, channel_from, proposed_balance_to, channel_to, channels_with_funds): |
|
||||
return config |
|
||||
else: |
|
||||
config[channel_to] = proposed_balance_to |
|
||||
config[channel_from] = proposed_balance_from |
|
||||
return config |
|
||||
|
|
||||
initial_number_parts = number_nonzero_parts(configuration) |
def remove_duplicates(configs: List[SplitConfig]) -> List[SplitConfig]: |
||||
|
unique_configs = set() |
||||
|
for config in configs: |
||||
|
# sort keys and values |
||||
|
config_sorted_values = {k: sorted(v) for k, v in config.items()} |
||||
|
config_sorted_keys = {k: config_sorted_values[k] for k in sorted(config_sorted_values.keys())} |
||||
|
hashable_config = tuple((c, tuple(sorted(config[c]))) for c in config_sorted_keys) |
||||
|
unique_configs.add(hashable_config) |
||||
|
unique_configs = [{c[0]: list(c[1]) for c in config} for config in unique_configs] |
||||
|
return unique_configs |
||||
|
|
||||
for _ in range(REDISTRIBUTE): |
|
||||
configuration = redistribute(configuration) |
|
||||
if not preserve_number_parts and number_nonzero_parts( |
|
||||
configuration) == initial_number_parts: |
|
||||
configuration = split(configuration) |
|
||||
configuration = swap(configuration) |
|
||||
|
|
||||
return configuration |
def remove_multiple_nodes(configs: List[SplitConfig]) -> List[SplitConfig]: |
||||
|
return [config for config in configs if number_nonzero_nodes(config) == 1] |
||||
|
|
||||
|
|
||||
@profiler |
def remove_single_part_configs(configs: List[SplitConfig]) -> List[SplitConfig]: |
||||
def suggest_splits(amount_msat: int, channels_with_funds: Dict[Tuple[bytes, bytes], int], |
return [config for config in configs if number_parts(config) != 1] |
||||
exclude_single_parts=True, single_node=False) \ |
|
||||
-> Sequence[Tuple[Dict[Tuple[bytes, bytes], int], float]]: |
|
||||
"""Creates split configurations for a payment over channels. Single channel |
|
||||
payments are excluded by default. channels_with_funds is keyed by |
|
||||
(channelid, nodeid).""" |
|
||||
|
|
||||
def rate_configuration(config: dict) -> float: |
|
||||
"""Defines an objective function to rate a split configuration. |
|
||||
|
|
||||
We calculate the normalized L2 norm for a split configuration and |
def rate_config( |
||||
|
config: SplitConfig, |
||||
|
channels_with_funds: ChannelsFundsInfo) -> float: |
||||
|
"""Defines an objective function to rate a configuration. |
||||
|
|
||||
|
We calculate the normalized L2 norm for a configuration and |
||||
add a part penalty for each nonzero amount. The consequence is that |
add a part penalty for each nonzero amount. The consequence is that |
||||
amounts that are equally distributed and have less parts are rated |
amounts that are equally distributed and have less parts are rated |
||||
lowest.""" |
lowest (best). A penalty depending on the total amount sent over a channel |
||||
F = 0 |
counteracts channel exhaustion.""" |
||||
total_amount = sum([v for v in config.values()]) |
rating = 0 |
||||
|
total_amount = total_config_amount(config) |
||||
|
|
||||
for channel, amount in config.items(): |
for channel, amounts in config.items(): |
||||
funds = channels_with_funds[channel] |
funds = channels_with_funds[channel] |
||||
if amount: |
if amounts: |
||||
F += amount * amount / (total_amount * total_amount) # a penalty to favor distribution of amounts |
for amount in amounts: |
||||
F += PART_PENALTY * PART_PENALTY # a penalty for each part |
rating += amount * amount / (total_amount * total_amount) # penalty to favor equal distribution of amounts |
||||
|
rating += PART_PENALTY * PART_PENALTY # penalty for each part |
||||
decay = funds / EXHAUST_DECAY_FRACTION |
decay = funds / EXHAUST_DECAY_FRACTION |
||||
F += math.exp((amount - funds) / decay) # a penalty for channel saturation |
rating += math.exp((sum(amounts) - funds) / decay) # penalty for channel exhaustion |
||||
|
return rating |
||||
|
|
||||
return F |
|
||||
|
|
||||
def rated_sorted_configurations(hierarchy: dict) -> Sequence[Tuple[Dict[Tuple[bytes, bytes], int], float]]: |
def suggest_splits( |
||||
"""Cleans up duplicate splittings, rates and sorts them according to |
amount_msat: int, channels_with_funds: ChannelsFundsInfo, |
||||
the rating. A lower rating is a better configuration.""" |
exclude_single_part_payments=False, |
||||
hierarchy = unique_hierarchy(hierarchy) |
exclude_multinode_payments=False |
||||
rated_configs = [] |
) -> List[SplitConfigRating]: |
||||
for level, configs in hierarchy.items(): |
"""Breaks amount_msat into smaller pieces and distributes them over the |
||||
for config in configs: |
channels according to the funds they can send. |
||||
rated_configs.append((config, rate_configuration(config))) |
|
||||
sorted_rated_configs = sorted(rated_configs, key=lambda c: c[1], reverse=False) |
|
||||
return sorted_rated_configs |
|
||||
|
|
||||
# create initial guesses |
Individual channels may be assigned multiple parts. The split configurations |
||||
split_hierarchy = create_starting_split_hierarchy(amount_msat, channels_with_funds) |
are returned in sorted order, from best to worst rating. |
||||
|
|
||||
# randomize initial guesses and generate splittings of different split |
Single part payments can be excluded, since they represent legacy payments. |
||||
# levels up to number of channels |
Split configurations that send via multiple nodes can be excluded as well. |
||||
for level in range(2, min(MAX_PARTS, len(channels_with_funds) + 1)): |
""" |
||||
# generate a set of random configurations for each level |
|
||||
|
configs = [] |
||||
|
channels_order = list(channels_with_funds.keys()) |
||||
|
|
||||
|
# generate multiple configurations to get more configurations (there is randomness in this loop) |
||||
for _ in range(CANDIDATES_PER_LEVEL): |
for _ in range(CANDIDATES_PER_LEVEL): |
||||
configurations = unique_hierarchy(split_hierarchy).get(level, None) |
# we want to have configurations with no splitting to many splittings |
||||
if configurations: # we have a splitting of the desired number of parts |
for target_parts in range(1, MAX_PARTS): |
||||
configuration = random.choice(configurations) |
config = defaultdict(list) # type: SplitConfig |
||||
# generate new splittings preserving the number of parts |
|
||||
configuration = propose_new_configuration( |
# randomly split amount into target_parts chunks |
||||
channels_with_funds, configuration, amount_msat, |
split_amounts = split_amount_normal(amount_msat, target_parts) |
||||
preserve_number_parts=True) |
# randomly distribute amounts over channels |
||||
|
for amount in split_amounts: |
||||
|
random.shuffle(channels_order) |
||||
|
# we check each channel and try to put the funds inside, break if we succeed |
||||
|
for c in channels_order: |
||||
|
if sum(config[c]) + amount <= channels_with_funds[c]: |
||||
|
config[c].append(amount) |
||||
|
break |
||||
|
# if we don't succeed to put the amount anywhere, |
||||
|
# we try to fill up channels and put the rest somewhere else |
||||
else: |
else: |
||||
# go one level lower and look for valid splittings, |
distribute_amount = amount |
||||
# try to go one level higher by splitting a single outgoing amount |
for c in channels_order: |
||||
configurations = unique_hierarchy(split_hierarchy).get(level - 1, None) |
funds_left = channels_with_funds[c] - sum(config[c]) |
||||
if not configurations: |
# it would be good to not fill the full channel if possible |
||||
|
add_amount = min(funds_left, distribute_amount) |
||||
|
config[c].append(add_amount) |
||||
|
distribute_amount -= add_amount |
||||
|
if distribute_amount == 0: |
||||
|
break |
||||
|
if total_config_amount(config) != amount_msat: |
||||
|
raise NoPathFound('Cannot distribute payment over channels.') |
||||
|
if target_parts > 1 and is_any_amount_smaller_than_min_part_size(config): |
||||
continue |
continue |
||||
configuration = random.choice(configurations) |
assert total_config_amount(config) == amount_msat |
||||
# generate new splittings going one level higher in the number of parts |
configs.append(config) |
||||
configuration = propose_new_configuration( |
|
||||
channels_with_funds, configuration, amount_msat, |
configs = remove_duplicates(configs) |
||||
preserve_number_parts=False) |
|
||||
|
# we only take configurations that send via a single node (but there can be multiple parts) |
||||
# add the newly found configuration (doesn't matter if nothing changed) |
if exclude_multinode_payments: |
||||
split_hierarchy[number_nonzero_parts(configuration)].append(configuration) |
configs = remove_multiple_nodes(configs) |
||||
|
|
||||
if exclude_single_parts: |
if exclude_single_part_payments: |
||||
# we only want to return configurations that have at least two parts |
configs = remove_single_part_configs(configs) |
||||
try: |
|
||||
del split_hierarchy[1] |
rated_configs = [SplitConfigRating( |
||||
except: |
config=c, |
||||
pass |
rating=rate_config(c, channels_with_funds) |
||||
|
) for c in configs] |
||||
if single_node: |
rated_configs.sort(key=lambda x: x.rating) |
||||
# we only take configurations that send to a single node |
|
||||
split_hierarchy = single_node_hierarchy(split_hierarchy) |
return rated_configs |
||||
|
|
||||
return rated_sorted_configurations(split_hierarchy) |
|
||||
|
|||||
Loading…
Reference in new issue