From 4bf0bc1f288af64dc012f0f2b7aa91236fa22d04 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 27 Sep 2019 09:34:34 +0930 Subject: [PATCH] gossipd: age txout_failures map. We do this by keeping a current and an old map, and moving the current to old every hour or 10,000 entries. Signed-off-by: Rusty Russell --- gossipd/gossipd.c | 1 + gossipd/routing.c | 48 ++++++++- gossipd/routing.h | 8 +- gossipd/test/run-bench-find_route.c | 12 ++- gossipd/test/run-find_route-specific.c | 11 +- gossipd/test/run-find_route.c | 11 +- gossipd/test/run-overlong.c | 12 ++- gossipd/test/run-txout_failure.c | 135 +++++++++++++++++++++++++ 8 files changed, 230 insertions(+), 8 deletions(-) create mode 100644 gossipd/test/run-txout_failure.c diff --git a/gossipd/gossipd.c b/gossipd/gossipd.c index d21e55448..1990e7cfa 100644 --- a/gossipd/gossipd.c +++ b/gossipd/gossipd.c @@ -938,6 +938,7 @@ static struct io_plan *gossip_init(struct io_conn *conn, chainparams_by_chainhash(&daemon->chain_hash), &daemon->id, &daemon->peers, + &daemon->timers, take(dev_gossip_time), dev_fast_gossip, dev_fast_gossip_prune); diff --git a/gossipd/routing.c b/gossipd/routing.c index d4e66922f..cce6615d2 100644 --- a/gossipd/routing.c +++ b/gossipd/routing.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -234,10 +235,48 @@ static void memleak_help_routing_tables(struct htable *memtable, } #endif /* DEVELOPER */ +/* Once an hour, or at 10000 entries, we expire old ones */ +static void txout_failure_age(struct routing_state *rstate) +{ + uintmap_clear(&rstate->txout_failures_old); + rstate->txout_failures_old = rstate->txout_failures; + uintmap_init(&rstate->txout_failures); + rstate->num_txout_failures = 0; + + rstate->txout_failure_timer = new_reltimer(rstate->timers, + rstate, time_from_sec(3600), + txout_failure_age, rstate); +} + +static void add_to_txout_failures(struct routing_state *rstate, + const struct short_channel_id *scid) +{ + if (uintmap_add(&rstate->txout_failures, scid->u64, true) + && ++rstate->num_txout_failures == 10000) { + tal_free(rstate->txout_failure_timer); + txout_failure_age(rstate); + } +} + +static bool in_txout_failures(struct routing_state *rstate, + const struct short_channel_id *scid) +{ + if (uintmap_get(&rstate->txout_failures, scid->u64)) + return true; + + /* If we were going to expire it, we no longer are. */ + if (uintmap_get(&rstate->txout_failures_old, scid->u64)) { + add_to_txout_failures(rstate, scid); + return true; + } + return false; +} + struct routing_state *new_routing_state(const tal_t *ctx, const struct chainparams *chainparams, const struct node_id *local_id, struct list_head *peers, + struct timers *timers, const u32 *dev_gossip_time TAKES, bool dev_fast_gossip, bool dev_fast_gossip_prune) @@ -245,6 +284,7 @@ struct routing_state *new_routing_state(const tal_t *ctx, struct routing_state *rstate = tal(ctx, struct routing_state); rstate->nodes = new_node_map(rstate); rstate->gs = gossip_store_new(rstate, peers); + rstate->timers = timers; rstate->chainparams = chainparams; rstate->local_id = *local_id; rstate->local_channel_announced = false; @@ -254,8 +294,10 @@ struct routing_state *new_routing_state(const tal_t *ctx, uintmap_init(&rstate->chanmap); uintmap_init(&rstate->unupdated_chanmap); local_chan_map_init(&rstate->local_chan_map); + rstate->num_txout_failures = 0; uintmap_init(&rstate->txout_failures); - + uintmap_init(&rstate->txout_failures_old); + txout_failure_age(rstate); rstate->pending_node_map = tal(ctx, struct pending_node_map); pending_node_map_init(rstate->pending_node_map); @@ -1835,7 +1877,7 @@ bool handle_pending_cannouncement(struct routing_state *rstate, type_to_string(pending, struct short_channel_id, scid)); tal_free(pending); - uintmap_add(&rstate->txout_failures, scid->u64, true); + add_to_txout_failures(rstate, scid); return false; } @@ -2213,7 +2255,7 @@ u8 *handle_channel_update(struct routing_state *rstate, const u8 *update TAKES, /* If we dropped the matching announcement for this channel due to the * txout query failing, don't report failure, it's just too noisy on * mainnet */ - if (uintmap_get(&rstate->txout_failures, short_channel_id.u64)) + if (in_txout_failures(rstate, &short_channel_id)) return NULL; /* If we have an unvalidated channel, just queue on that */ diff --git a/gossipd/routing.h b/gossipd/routing.h index 168861848..f7bc88f49 100644 --- a/gossipd/routing.h +++ b/gossipd/routing.h @@ -248,6 +248,9 @@ struct routing_state { /* Which chain we're on */ const struct chainparams *chainparams; + /* TImers base from struct gossipd. */ + struct timers *timers; + /* All known nodes. */ struct node_map *nodes; @@ -275,7 +278,9 @@ struct routing_state { /* Cache for txout queries that failed. Allows us to skip failed * checks if we get another announcement for the same scid. */ - UINTMAP(bool) txout_failures; + size_t num_txout_failures; + UINTMAP(bool) txout_failures, txout_failures_old; + struct oneshot *txout_failure_timer; /* A map of local channels by short_channel_ids */ struct local_chan_map local_chan_map; @@ -324,6 +329,7 @@ struct routing_state *new_routing_state(const tal_t *ctx, const struct chainparams *chainparams, const struct node_id *local_id, struct list_head *peers, + struct timers *timers, const u32 *dev_gossip_time TAKES, bool dev_fast_gossip, bool dev_fast_gossip_prune); diff --git a/gossipd/test/run-bench-find_route.c b/gossipd/test/run-bench-find_route.c index a9255557b..958ded7a6 100644 --- a/gossipd/test/run-bench-find_route.c +++ b/gossipd/test/run-bench-find_route.c @@ -87,6 +87,15 @@ void memleak_remove_intmap_(struct htable *memtable UNNEEDED, const struct intma { fprintf(stderr, "memleak_remove_intmap_ called!\n"); abort(); } #endif +/* NOOP for new_reltimer_ */ +struct oneshot *new_reltimer_(struct timers *timers UNNEEDED, + const tal_t *ctx UNNEEDED, + struct timerel expire UNNEEDED, + void (*cb)(void *) UNNEEDED, void *arg UNNEEDED) +{ + return NULL; +} + /* Updates existing route if required. */ static void add_connection(struct routing_state *rstate, const struct node_id *nodes, @@ -191,7 +200,8 @@ int main(int argc, char *argv[]) setup_tmpctx(); me = nodeid(0); - rstate = new_routing_state(tmpctx, NULL, &me, 0, NULL, false, false); + rstate = new_routing_state(tmpctx, NULL, &me, NULL, NULL, NULL, + false, false); opt_register_noarg("--perfme", opt_set_bool, &perfme, "Run perfme-start and perfme-stop around benchmark"); diff --git a/gossipd/test/run-find_route-specific.c b/gossipd/test/run-find_route-specific.c index cf50a2171..bc40e0398 100644 --- a/gossipd/test/run-find_route-specific.c +++ b/gossipd/test/run-find_route-specific.c @@ -76,6 +76,15 @@ void memleak_remove_intmap_(struct htable *memtable UNNEEDED, const struct intma { fprintf(stderr, "memleak_remove_intmap_ called!\n"); abort(); } #endif +/* NOOP for new_reltimer_ */ +struct oneshot *new_reltimer_(struct timers *timers UNNEEDED, + const tal_t *ctx UNNEEDED, + struct timerel expire UNNEEDED, + void (*cb)(void *) UNNEEDED, void *arg UNNEEDED) +{ + return NULL; +} + const void *trc; static struct half_chan * @@ -146,7 +155,7 @@ int main(void) strlen("02cca6c5c966fcf61d121e3a70e03a1cd9eeeea024b26ea666ce974d43b242e636"), &d); - rstate = new_routing_state(tmpctx, NULL, &a, 0, NULL, false, false); + rstate = new_routing_state(tmpctx, NULL, &a, NULL, NULL, NULL, false, false); /* [{'active': True, 'short_id': '6990:2:1/1', 'fee_per_kw': 10, 'delay': 5, 'message_flags': 0, 'channel_flags': 1, 'destination': '0230ad0e74ea03976b28fda587bb75bdd357a1938af4424156a18265167f5e40ae', 'source': '02ea622d5c8d6143f15ed3ce1d501dd0d3d09d3b1c83a44d0034949f8a9ab60f06', 'last_update': 1504064344}, */ diff --git a/gossipd/test/run-find_route.c b/gossipd/test/run-find_route.c index 79241e329..b80285bb8 100644 --- a/gossipd/test/run-find_route.c +++ b/gossipd/test/run-find_route.c @@ -74,6 +74,15 @@ void memleak_remove_intmap_(struct htable *memtable UNNEEDED, const struct intma { fprintf(stderr, "memleak_remove_intmap_ called!\n"); abort(); } #endif +/* NOOP for new_reltimer_ */ +struct oneshot *new_reltimer_(struct timers *timers UNNEEDED, + const tal_t *ctx UNNEEDED, + struct timerel expire UNNEEDED, + void (*cb)(void *) UNNEEDED, void *arg UNNEEDED) +{ + return NULL; +} + static void node_id_from_privkey(const struct privkey *p, struct node_id *id) { struct pubkey k; @@ -181,7 +190,7 @@ int main(void) memset(&tmp, 'a', sizeof(tmp)); node_id_from_privkey(&tmp, &a); - rstate = new_routing_state(tmpctx, NULL, &a, 0, NULL, false, false); + rstate = new_routing_state(tmpctx, NULL, &a, NULL, NULL, NULL, false, false); new_node(rstate, &a); diff --git a/gossipd/test/run-overlong.c b/gossipd/test/run-overlong.c index a2a17e6bd..135e0f989 100644 --- a/gossipd/test/run-overlong.c +++ b/gossipd/test/run-overlong.c @@ -74,6 +74,15 @@ void memleak_remove_intmap_(struct htable *memtable UNNEEDED, const struct intma { fprintf(stderr, "memleak_remove_intmap_ called!\n"); abort(); } #endif +/* NOOP for new_reltimer_ */ +struct oneshot *new_reltimer_(struct timers *timers UNNEEDED, + const tal_t *ctx UNNEEDED, + struct timerel expire UNNEEDED, + void (*cb)(void *) UNNEEDED, void *arg UNNEEDED) +{ + return NULL; +} + static void node_id_from_privkey(const struct privkey *p, struct node_id *id) { struct pubkey k; @@ -105,7 +114,8 @@ int main(void) node_id_from_privkey(&tmp, &ids[i]); } /* We are node 0 */ - rstate = new_routing_state(tmpctx, NULL, &ids[0], 0, NULL, false, false); + rstate = new_routing_state(tmpctx, NULL, &ids[0], NULL, NULL, NULL, + false, false); for (size_t i = 0; i < NUM_NODES; i++) { struct chan *chan; diff --git a/gossipd/test/run-txout_failure.c b/gossipd/test/run-txout_failure.c new file mode 100644 index 000000000..258c33b05 --- /dev/null +++ b/gossipd/test/run-txout_failure.c @@ -0,0 +1,135 @@ +#include "../routing.c" +#include "../common/timeout.c" +#include + +/* AUTOGENERATED MOCKS START */ +/* Generated stub for cupdate_different */ +bool cupdate_different(struct gossip_store *gs UNNEEDED, + const struct half_chan *hc UNNEEDED, + const u8 *cupdate UNNEEDED) +{ fprintf(stderr, "cupdate_different called!\n"); abort(); } +/* Generated stub for fromwire_gossipd_local_add_channel */ +bool fromwire_gossipd_local_add_channel(const void *p UNNEEDED, struct short_channel_id *short_channel_id UNNEEDED, struct node_id *remote_node_id UNNEEDED, struct amount_sat *satoshis UNNEEDED) +{ fprintf(stderr, "fromwire_gossipd_local_add_channel called!\n"); abort(); } +/* Generated stub for fromwire_wireaddr */ +bool fromwire_wireaddr(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, struct wireaddr *addr UNNEEDED) +{ fprintf(stderr, "fromwire_wireaddr called!\n"); abort(); } +/* Generated stub for gossip_store_add */ +u64 gossip_store_add(struct gossip_store *gs UNNEEDED, const u8 *gossip_msg UNNEEDED, + u32 timestamp UNNEEDED, const u8 *addendum UNNEEDED) +{ fprintf(stderr, "gossip_store_add called!\n"); abort(); } +/* Generated stub for gossip_store_add_private_update */ +u64 gossip_store_add_private_update(struct gossip_store *gs UNNEEDED, const u8 *update UNNEEDED) +{ fprintf(stderr, "gossip_store_add_private_update called!\n"); abort(); } +/* Generated stub for gossip_store_delete */ +void gossip_store_delete(struct gossip_store *gs UNNEEDED, + struct broadcastable *bcast UNNEEDED, + int type UNNEEDED) +{ fprintf(stderr, "gossip_store_delete called!\n"); abort(); } +/* Generated stub for gossip_store_get */ +const u8 *gossip_store_get(const tal_t *ctx UNNEEDED, + struct gossip_store *gs UNNEEDED, + u64 offset UNNEEDED) +{ fprintf(stderr, "gossip_store_get called!\n"); abort(); } +/* Generated stub for gossip_store_get_private_update */ +const u8 *gossip_store_get_private_update(const tal_t *ctx UNNEEDED, + struct gossip_store *gs UNNEEDED, + u64 offset UNNEEDED) +{ fprintf(stderr, "gossip_store_get_private_update called!\n"); abort(); } +/* Generated stub for memleak_add_helper_ */ +void memleak_add_helper_(const tal_t *p UNNEEDED, void (*cb)(struct htable *memtable UNNEEDED, + const tal_t *)){ } +/* Generated stub for memleak_remove_htable */ +void memleak_remove_htable(struct htable *memtable UNNEEDED, const struct htable *ht UNNEEDED) +{ fprintf(stderr, "memleak_remove_htable called!\n"); abort(); } +/* Generated stub for memleak_remove_intmap_ */ +void memleak_remove_intmap_(struct htable *memtable UNNEEDED, const struct intmap *m UNNEEDED) +{ fprintf(stderr, "memleak_remove_intmap_ called!\n"); abort(); } +/* Generated stub for nannounce_different */ +bool nannounce_different(struct gossip_store *gs UNNEEDED, + const struct node *node UNNEEDED, + const u8 *nannounce UNNEEDED) +{ fprintf(stderr, "nannounce_different called!\n"); abort(); } +/* Generated stub for onion_type_name */ +const char *onion_type_name(int e UNNEEDED) +{ fprintf(stderr, "onion_type_name called!\n"); abort(); } +/* Generated stub for sanitize_error */ +char *sanitize_error(const tal_t *ctx UNNEEDED, const u8 *errmsg UNNEEDED, + struct channel_id *channel_id UNNEEDED) +{ fprintf(stderr, "sanitize_error called!\n"); abort(); } +/* Generated stub for status_fmt */ +void status_fmt(enum log_level level UNNEEDED, const char *fmt UNNEEDED, ...) + +{ fprintf(stderr, "status_fmt called!\n"); abort(); } +/* Generated stub for towire_errorfmt */ +u8 *towire_errorfmt(const tal_t *ctx UNNEEDED, + const struct channel_id *channel UNNEEDED, + const char *fmt UNNEEDED, ...) +{ fprintf(stderr, "towire_errorfmt called!\n"); abort(); } +/* Generated stub for towire_gossip_store_channel_amount */ +u8 *towire_gossip_store_channel_amount(const tal_t *ctx UNNEEDED, struct amount_sat satoshis UNNEEDED) +{ fprintf(stderr, "towire_gossip_store_channel_amount called!\n"); abort(); } +/* AUTOGENERATED MOCKS END */ + +/* NOOP stub for gossip_store_new */ +struct gossip_store *gossip_store_new(struct routing_state *rstate UNNEEDED, + struct list_head *peers UNNEEDED) +{ + return NULL; +} + +int main(void) +{ + struct routing_state *rstate; + struct timers timers; + struct timer *t; + struct short_channel_id scid1, scid2; + + setup_locale(); + setup_tmpctx(); + + timers_init(&timers, time_mono()); + /* Random uninitalized node_id, we don't reference it. */ + rstate = new_routing_state(tmpctx, NULL, tal(tmpctx, struct node_id), + NULL, &timers, NULL, false, false); + + scid1.u64 = 100; + scid2.u64 = 200; + assert(!in_txout_failures(rstate, &scid1)); + assert(!in_txout_failures(rstate, &scid2)); + + add_to_txout_failures(rstate, &scid1); + assert(in_txout_failures(rstate, &scid1)); + assert(!in_txout_failures(rstate, &scid2)); + assert(rstate->num_txout_failures == 1); + + add_to_txout_failures(rstate, &scid2); + assert(in_txout_failures(rstate, &scid1)); + assert(in_txout_failures(rstate, &scid2)); + assert(rstate->num_txout_failures == 2); + + /* Move time forward 1 hour. */ + t = timers_expire(&timers, timemono_add(time_mono(), + time_from_sec(3601))); + assert(t); + timer_expired(NULL, t); + + /* Still there, just old. Refresh scid1 */ + assert(rstate->num_txout_failures == 0); + assert(in_txout_failures(rstate, &scid1)); + assert(rstate->num_txout_failures == 1); + + t = timers_expire(&timers, timemono_add(time_mono(), + time_from_sec(3601))); + assert(t); + timer_expired(NULL, t); + + assert(rstate->num_txout_failures == 0); + assert(in_txout_failures(rstate, &scid1)); + assert(rstate->num_txout_failures == 1); + assert(!in_txout_failures(rstate, &scid2)); + + tal_free(tmpctx); + timers_cleanup(&timers); + return 0; +}