gossipd: age txout_failures map.

We do this by keeping a current and an old map, and moving the current to old
every hour or 10,000 entries.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

gossipd/gossipd.c

@@ -938,6 +938,7 @@ static struct io_plan *gossip_init(struct io_conn *conn,
 					   chainparams_by_chainhash(&daemon->chain_hash),
 					   &daemon->id,
 					   &daemon->peers,
+					   &daemon->timers,
 					   take(dev_gossip_time),
 					   dev_fast_gossip,
 					   dev_fast_gossip_prune);

gossipd/routing.c

@@ -10,6 +10,7 @@
 #include <common/memleak.h>
 #include <common/pseudorand.h>
 #include <common/status.h>
+#include <common/timeout.h>
 #include <common/type_to_string.h>
 #include <common/wire_error.h>
 #include <common/wireaddr.h>
@@ -234,10 +235,48 @@ static void memleak_help_routing_tables(struct htable *memtable,
 }
 #endif /* DEVELOPER */
 
+/* Once an hour, or at 10000 entries, we expire old ones */
+static void txout_failure_age(struct routing_state *rstate)
+{
+	uintmap_clear(&rstate->txout_failures_old);
+	rstate->txout_failures_old = rstate->txout_failures;
+	uintmap_init(&rstate->txout_failures);
+	rstate->num_txout_failures = 0;
+
+	rstate->txout_failure_timer = new_reltimer(rstate->timers,
+						   rstate, time_from_sec(3600),
+						   txout_failure_age, rstate);
+}
+
+static void add_to_txout_failures(struct routing_state *rstate,
+				  const struct short_channel_id *scid)
+{
+	if (uintmap_add(&rstate->txout_failures, scid->u64, true)
+	    && ++rstate->num_txout_failures == 10000) {
+		tal_free(rstate->txout_failure_timer);
+		txout_failure_age(rstate);
+	}
+}
+
+static bool in_txout_failures(struct routing_state *rstate,
+			      const struct short_channel_id *scid)
+{
+	if (uintmap_get(&rstate->txout_failures, scid->u64))
+		return true;
+
+	/* If we were going to expire it, we no longer are. */
+	if (uintmap_get(&rstate->txout_failures_old, scid->u64)) {
+		add_to_txout_failures(rstate, scid);
+		return true;
+	}
+	return false;
+}
+
 struct routing_state *new_routing_state(const tal_t *ctx,
 					const struct chainparams *chainparams,
 					const struct node_id *local_id,
 					struct list_head *peers,
+					struct timers *timers,
 					const u32 *dev_gossip_time TAKES,
 					bool dev_fast_gossip,
 					bool dev_fast_gossip_prune)
@@ -245,6 +284,7 @@ struct routing_state *new_routing_state(const tal_t *ctx,
 	struct routing_state *rstate = tal(ctx, struct routing_state);
 	rstate->nodes = new_node_map(rstate);
 	rstate->gs = gossip_store_new(rstate, peers);
+	rstate->timers = timers;
 	rstate->chainparams = chainparams;
 	rstate->local_id = *local_id;
 	rstate->local_channel_announced = false;
@@ -254,8 +294,10 @@ struct routing_state *new_routing_state(const tal_t *ctx,
 	uintmap_init(&rstate->chanmap);
 	uintmap_init(&rstate->unupdated_chanmap);
 	local_chan_map_init(&rstate->local_chan_map);
+	rstate->num_txout_failures = 0;
 	uintmap_init(&rstate->txout_failures);
-
+	uintmap_init(&rstate->txout_failures_old);
+	txout_failure_age(rstate);
 	rstate->pending_node_map = tal(ctx, struct pending_node_map);
 	pending_node_map_init(rstate->pending_node_map);
 
@@ -1835,7 +1877,7 @@ bool handle_pending_cannouncement(struct routing_state *rstate,
 			     type_to_string(pending, struct short_channel_id,
 					    scid));
 		tal_free(pending);
-		uintmap_add(&rstate->txout_failures, scid->u64, true);
+		add_to_txout_failures(rstate, scid);
 		return false;
 	}
 
@@ -2213,7 +2255,7 @@ u8 *handle_channel_update(struct routing_state *rstate, const u8 *update TAKES,
 	/* If we dropped the matching announcement for this channel due to the
 	 * txout query failing, don't report failure, it's just too noisy on
 	 * mainnet */
-	if (uintmap_get(&rstate->txout_failures, short_channel_id.u64))
+	if (in_txout_failures(rstate, &short_channel_id))
 		return NULL;
 
 	/* If we have an unvalidated channel, just queue on that */

gossipd/routing.h

@@ -248,6 +248,9 @@ struct routing_state {
 	/* Which chain we're on */
 	const struct chainparams *chainparams;
 
+	/* TImers base from struct gossipd. */
+	struct timers *timers;
+
 	/* All known nodes. */
 	struct node_map *nodes;
 
@@ -275,7 +278,9 @@ struct routing_state {
 
 	/* Cache for txout queries that failed. Allows us to skip failed
 	 * checks if we get another announcement for the same scid. */
-	UINTMAP(bool) txout_failures;
+	size_t num_txout_failures;
+	UINTMAP(bool) txout_failures, txout_failures_old;
+	struct oneshot *txout_failure_timer;
 
         /* A map of local channels by short_channel_ids */
 	struct local_chan_map local_chan_map;
@@ -324,6 +329,7 @@ struct routing_state *new_routing_state(const tal_t *ctx,
 					const struct chainparams *chainparams,
 					const struct node_id *local_id,
 					struct list_head *peers,
+					struct timers *timers,
 					const u32 *dev_gossip_time TAKES,
 					bool dev_fast_gossip,
 					bool dev_fast_gossip_prune);

gossipd/test/run-bench-find_route.c

@@ -87,6 +87,15 @@ void memleak_remove_intmap_(struct htable *memtable UNNEEDED, const struct intma
 { fprintf(stderr, "memleak_remove_intmap_ called!\n"); abort(); }
 #endif
 
+/* NOOP for new_reltimer_ */
+struct oneshot *new_reltimer_(struct timers *timers UNNEEDED,
+			      const tal_t *ctx UNNEEDED,
+			      struct timerel expire UNNEEDED,
+			      void (*cb)(void *) UNNEEDED, void *arg UNNEEDED)
+{
+	return NULL;
+}
+
 /* Updates existing route if required. */
 static void add_connection(struct routing_state *rstate,
 			   const struct node_id *nodes,
@@ -191,7 +200,8 @@ int main(int argc, char *argv[])
 	setup_tmpctx();
 
 	me = nodeid(0);
-	rstate = new_routing_state(tmpctx, NULL, &me, 0, NULL, false, false);
+	rstate = new_routing_state(tmpctx, NULL, &me, NULL, NULL, NULL,
+				   false, false);
 	opt_register_noarg("--perfme", opt_set_bool, &perfme,
 			   "Run perfme-start and perfme-stop around benchmark");
 

gossipd/test/run-find_route-specific.c

@@ -76,6 +76,15 @@ void memleak_remove_intmap_(struct htable *memtable UNNEEDED, const struct intma
 { fprintf(stderr, "memleak_remove_intmap_ called!\n"); abort(); }
 #endif
 
+/* NOOP for new_reltimer_ */
+struct oneshot *new_reltimer_(struct timers *timers UNNEEDED,
+			      const tal_t *ctx UNNEEDED,
+			      struct timerel expire UNNEEDED,
+			      void (*cb)(void *) UNNEEDED, void *arg UNNEEDED)
+{
+	return NULL;
+}
+
 const void *trc;
 
 static struct half_chan *
@@ -146,7 +155,7 @@ int main(void)
 			   strlen("02cca6c5c966fcf61d121e3a70e03a1cd9eeeea024b26ea666ce974d43b242e636"),
 			   &d);
 
-	rstate = new_routing_state(tmpctx, NULL, &a, 0, NULL, false, false);
+	rstate = new_routing_state(tmpctx, NULL, &a, NULL, NULL, NULL, false, false);
 
 	/* [{'active': True, 'short_id': '6990:2:1/1', 'fee_per_kw': 10, 'delay': 5, 'message_flags': 0, 'channel_flags': 1, 'destination': '0230ad0e74ea03976b28fda587bb75bdd357a1938af4424156a18265167f5e40ae', 'source': '02ea622d5c8d6143f15ed3ce1d501dd0d3d09d3b1c83a44d0034949f8a9ab60f06', 'last_update': 1504064344}, */
 

gossipd/test/run-find_route.c

@@ -74,6 +74,15 @@ void memleak_remove_intmap_(struct htable *memtable UNNEEDED, const struct intma
 { fprintf(stderr, "memleak_remove_intmap_ called!\n"); abort(); }
 #endif
 
+/* NOOP for new_reltimer_ */
+struct oneshot *new_reltimer_(struct timers *timers UNNEEDED,
+			      const tal_t *ctx UNNEEDED,
+			      struct timerel expire UNNEEDED,
+			      void (*cb)(void *) UNNEEDED, void *arg UNNEEDED)
+{
+	return NULL;
+}
+
 static void node_id_from_privkey(const struct privkey *p, struct node_id *id)
 {
 	struct pubkey k;
@@ -181,7 +190,7 @@ int main(void)
 
 	memset(&tmp, 'a', sizeof(tmp));
 	node_id_from_privkey(&tmp, &a);
-	rstate = new_routing_state(tmpctx, NULL, &a, 0, NULL, false, false);
+	rstate = new_routing_state(tmpctx, NULL, &a, NULL, NULL, NULL, false, false);
 
 	new_node(rstate, &a);
 

gossipd/test/run-overlong.c

@@ -74,6 +74,15 @@ void memleak_remove_intmap_(struct htable *memtable UNNEEDED, const struct intma
 { fprintf(stderr, "memleak_remove_intmap_ called!\n"); abort(); }
 #endif
 
+/* NOOP for new_reltimer_ */
+struct oneshot *new_reltimer_(struct timers *timers UNNEEDED,
+			      const tal_t *ctx UNNEEDED,
+			      struct timerel expire UNNEEDED,
+			      void (*cb)(void *) UNNEEDED, void *arg UNNEEDED)
+{
+	return NULL;
+}
+
 static void node_id_from_privkey(const struct privkey *p, struct node_id *id)
 {
 	struct pubkey k;
@@ -105,7 +114,8 @@ int main(void)
 		node_id_from_privkey(&tmp, &ids[i]);
 	}
 	/* We are node 0 */
-	rstate = new_routing_state(tmpctx, NULL, &ids[0], 0, NULL, false, false);
+	rstate = new_routing_state(tmpctx, NULL, &ids[0], NULL, NULL, NULL,
+				   false, false);
 
 	for (size_t i = 0; i < NUM_NODES; i++) {
 		struct chan *chan;

gossipd/test/run-txout_failure.c

@@ -0,0 +1,135 @@
+#include "../routing.c"
+#include "../common/timeout.c"
+#include <stdio.h>
+
+/* AUTOGENERATED MOCKS START */
+/* Generated stub for cupdate_different */
+bool cupdate_different(struct gossip_store *gs UNNEEDED,
+		       const struct half_chan *hc UNNEEDED,
+		       const u8 *cupdate UNNEEDED)
+{ fprintf(stderr, "cupdate_different called!\n"); abort(); }
+/* Generated stub for fromwire_gossipd_local_add_channel */
+bool fromwire_gossipd_local_add_channel(const void *p UNNEEDED, struct short_channel_id *short_channel_id UNNEEDED, struct node_id *remote_node_id UNNEEDED, struct amount_sat *satoshis UNNEEDED)
+{ fprintf(stderr, "fromwire_gossipd_local_add_channel called!\n"); abort(); }
+/* Generated stub for fromwire_wireaddr */
+bool fromwire_wireaddr(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, struct wireaddr *addr UNNEEDED)
+{ fprintf(stderr, "fromwire_wireaddr called!\n"); abort(); }
+/* Generated stub for gossip_store_add */
+u64 gossip_store_add(struct gossip_store *gs UNNEEDED, const u8 *gossip_msg UNNEEDED,
+		     u32 timestamp UNNEEDED, const u8 *addendum UNNEEDED)
+{ fprintf(stderr, "gossip_store_add called!\n"); abort(); }
+/* Generated stub for gossip_store_add_private_update */
+u64 gossip_store_add_private_update(struct gossip_store *gs UNNEEDED, const u8 *update UNNEEDED)
+{ fprintf(stderr, "gossip_store_add_private_update called!\n"); abort(); }
+/* Generated stub for gossip_store_delete */
+void gossip_store_delete(struct gossip_store *gs UNNEEDED,
+			 struct broadcastable *bcast UNNEEDED,
+			 int type UNNEEDED)
+{ fprintf(stderr, "gossip_store_delete called!\n"); abort(); }
+/* Generated stub for gossip_store_get */
+const u8 *gossip_store_get(const tal_t *ctx UNNEEDED,
+			   struct gossip_store *gs UNNEEDED,
+			   u64 offset UNNEEDED)
+{ fprintf(stderr, "gossip_store_get called!\n"); abort(); }
+/* Generated stub for gossip_store_get_private_update */
+const u8 *gossip_store_get_private_update(const tal_t *ctx UNNEEDED,
+					  struct gossip_store *gs UNNEEDED,
+					  u64 offset UNNEEDED)
+{ fprintf(stderr, "gossip_store_get_private_update called!\n"); abort(); }
+/* Generated stub for memleak_add_helper_ */
+void memleak_add_helper_(const tal_t *p UNNEEDED, void (*cb)(struct htable *memtable UNNEEDED,
+						    const tal_t *)){ }
+/* Generated stub for memleak_remove_htable */
+void memleak_remove_htable(struct htable *memtable UNNEEDED, const struct htable *ht UNNEEDED)
+{ fprintf(stderr, "memleak_remove_htable called!\n"); abort(); }
+/* Generated stub for memleak_remove_intmap_ */
+void memleak_remove_intmap_(struct htable *memtable UNNEEDED, const struct intmap *m UNNEEDED)
+{ fprintf(stderr, "memleak_remove_intmap_ called!\n"); abort(); }
+/* Generated stub for nannounce_different */
+bool nannounce_different(struct gossip_store *gs UNNEEDED,
+			 const struct node *node UNNEEDED,
+			 const u8 *nannounce UNNEEDED)
+{ fprintf(stderr, "nannounce_different called!\n"); abort(); }
+/* Generated stub for onion_type_name */
+const char *onion_type_name(int e UNNEEDED)
+{ fprintf(stderr, "onion_type_name called!\n"); abort(); }
+/* Generated stub for sanitize_error */
+char *sanitize_error(const tal_t *ctx UNNEEDED, const u8 *errmsg UNNEEDED,
+		     struct channel_id *channel_id UNNEEDED)
+{ fprintf(stderr, "sanitize_error called!\n"); abort(); }
+/* Generated stub for status_fmt */
+void status_fmt(enum log_level level UNNEEDED, const char *fmt UNNEEDED, ...)
+
+{ fprintf(stderr, "status_fmt called!\n"); abort(); }
+/* Generated stub for towire_errorfmt */
+u8 *towire_errorfmt(const tal_t *ctx UNNEEDED,
+		    const struct channel_id *channel UNNEEDED,
+		    const char *fmt UNNEEDED, ...)
+{ fprintf(stderr, "towire_errorfmt called!\n"); abort(); }
+/* Generated stub for towire_gossip_store_channel_amount */
+u8 *towire_gossip_store_channel_amount(const tal_t *ctx UNNEEDED, struct amount_sat satoshis UNNEEDED)
+{ fprintf(stderr, "towire_gossip_store_channel_amount called!\n"); abort(); }
+/* AUTOGENERATED MOCKS END */
+
+/* NOOP stub for gossip_store_new */
+struct gossip_store *gossip_store_new(struct routing_state *rstate UNNEEDED,
+				      struct list_head *peers UNNEEDED)
+{
+	return NULL;
+}
+
+int main(void)
+{
+	struct routing_state *rstate;
+	struct timers timers;
+	struct timer *t;
+	struct short_channel_id scid1, scid2;
+
+	setup_locale();
+	setup_tmpctx();
+
+	timers_init(&timers, time_mono());
+	/* Random uninitalized node_id, we don't reference it. */
+	rstate = new_routing_state(tmpctx, NULL, tal(tmpctx, struct node_id),
+				   NULL, &timers, NULL, false, false);
+
+	scid1.u64 = 100;
+	scid2.u64 = 200;
+	assert(!in_txout_failures(rstate, &scid1));
+	assert(!in_txout_failures(rstate, &scid2));
+
+	add_to_txout_failures(rstate, &scid1);
+	assert(in_txout_failures(rstate, &scid1));
+	assert(!in_txout_failures(rstate, &scid2));
+	assert(rstate->num_txout_failures == 1);
+
+	add_to_txout_failures(rstate, &scid2);
+	assert(in_txout_failures(rstate, &scid1));
+	assert(in_txout_failures(rstate, &scid2));
+	assert(rstate->num_txout_failures == 2);
+
+	/* Move time forward 1 hour. */
+	t = timers_expire(&timers, timemono_add(time_mono(),
+						time_from_sec(3601)));
+	assert(t);
+	timer_expired(NULL, t);
+
+	/* Still there, just old.  Refresh scid1 */
+	assert(rstate->num_txout_failures == 0);
+	assert(in_txout_failures(rstate, &scid1));
+	assert(rstate->num_txout_failures == 1);
+
+	t = timers_expire(&timers, timemono_add(time_mono(),
+						time_from_sec(3601)));
+	assert(t);
+	timer_expired(NULL, t);
+
+	assert(rstate->num_txout_failures == 0);
+	assert(in_txout_failures(rstate, &scid1));
+	assert(rstate->num_txout_failures == 1);
+	assert(!in_txout_failures(rstate, &scid2));
+
+	tal_free(tmpctx);
+	timers_cleanup(&timers);
+	return 0;
+}