gossipd: restore a flag for fast pruning

I was seeing some accidental pruning under load / Travis, and in
particular we stopped accepting channel_updates because they were 103
seconds old.  But making it too long makes the prune test untenable,
so restore a separate flag that this test can use.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

common/gossip_constants.h

@@ -63,16 +63,16 @@
  *     - MAY prune the channel.
  *     - MAY ignore the channel.
  */
-#define GOSSIP_PRUNE_INTERVAL(dev_fast_gossip_flag) \
-	DEV_FAST_GOSSIP(dev_fast_gossip_flag, 90, 1209600)
+#define GOSSIP_PRUNE_INTERVAL(dev_fast_gossip_prune_flag) \
+	DEV_FAST_GOSSIP(dev_fast_gossip_prune_flag, 60, 1209600)
 
 /* How long after seeing lockin until we announce the channel. */
 #define GOSSIP_ANNOUNCE_DELAY(dev_fast_gossip_flag) \
 	DEV_FAST_GOSSIP(dev_fast_gossip_flag, 1, 60)
 
 /* How long before deadline should we send refresh update? 1 day normally */
-#define GOSSIP_BEFORE_DEADLINE(dev_fast_gossip_flag) \
-	DEV_FAST_GOSSIP(dev_fast_gossip_flag, 30, 24*60*60)
+#define GOSSIP_BEFORE_DEADLINE(dev_fast_gossip_prune_flag) \
+	DEV_FAST_GOSSIP(dev_fast_gossip_prune_flag, 30, 24*60*60)
 
 /* How many seconds per token?  Normally 1 hour. */
 #define GOSSIP_TOKEN_TIME(dev_fast_gossip_flag) \

gossipd/gossip_wire.csv

@@ -14,6 +14,7 @@ msgdata,gossipctl_init,num_announcable,u16,
 msgdata,gossipctl_init,announcable,wireaddr,num_announcable
 msgdata,gossipctl_init,dev_gossip_time,?u32,
 msgdata,gossipctl_init,dev_fast_gossip,bool,
+msgdata,gossipctl_init,dev_fast_gossip_prune,bool,
 
 # In developer mode, we can mess with time.
 msgtype,gossip_dev_set_time,3001

gossipd/gossipd.c

@@ -776,11 +776,11 @@ static void gossip_refresh_network(struct daemon *daemon)
 
 	/* Send out 1 day before deadline */
 	highwater = now - (GOSSIP_PRUNE_INTERVAL(daemon->rstate->dev_fast_gossip)
-			   - GOSSIP_BEFORE_DEADLINE(daemon->rstate->dev_fast_gossip));
+			   - GOSSIP_BEFORE_DEADLINE(daemon->rstate->dev_fast_gossip_prune));
 
 	/* Schedule next run now */
 	notleak(new_reltimer(&daemon->timers, daemon,
-			     time_from_sec(GOSSIP_PRUNE_INTERVAL(daemon->rstate->dev_fast_gossip)/4),
+			     time_from_sec(GOSSIP_PRUNE_INTERVAL(daemon->rstate->dev_fast_gossip_prune)/4),
 			     gossip_refresh_network, daemon));
 
 	/* Find myself in the network */
@@ -920,7 +920,7 @@ static struct io_plan *gossip_init(struct io_conn *conn,
 				   const u8 *msg)
 {
 	u32 *dev_gossip_time;
-	bool dev_fast_gossip;
+	bool dev_fast_gossip, dev_fast_gossip_prune;
 
 	if (!fromwire_gossipctl_init(daemon, msg,
 				     &daemon->chain_hash,
@@ -928,7 +928,9 @@ static struct io_plan *gossip_init(struct io_conn *conn,
 				     daemon->rgb,
 				     daemon->alias,
 				     &daemon->announcable,
-				     &dev_gossip_time, &dev_fast_gossip)) {
+				     &dev_gossip_time,
+				     &dev_fast_gossip,
+				     &dev_fast_gossip_prune)) {
 		master_badmsg(WIRE_GOSSIPCTL_INIT, msg);
 	}
 
@@ -937,7 +939,8 @@ static struct io_plan *gossip_init(struct io_conn *conn,
 					   &daemon->id,
 					   &daemon->peers,
 					   take(dev_gossip_time),
-					   dev_fast_gossip);
+					   dev_fast_gossip,
+					   dev_fast_gossip_prune);
 
 	/* Load stored gossip messages */
 	if (!gossip_store_load(daemon->rstate, daemon->rstate->gs))
@@ -952,7 +955,7 @@ static struct io_plan *gossip_init(struct io_conn *conn,
 
 	/* Start the twice- weekly refresh timer. */
 	notleak(new_reltimer(&daemon->timers, daemon,
-			     time_from_sec(GOSSIP_PRUNE_INTERVAL(daemon->rstate->dev_fast_gossip) / 4),
+			     time_from_sec(GOSSIP_PRUNE_INTERVAL(daemon->rstate->dev_fast_gossip_prune) / 4),
 			     gossip_refresh_network, daemon));
 
 	return daemon_conn_read_next(conn, daemon->master);

gossipd/routing.c

@@ -207,7 +207,7 @@ static bool timestamp_reasonable(struct routing_state *rstate, u32 timestamp)
 	if (timestamp > now + 24*60*60)
 		return false;
 	/* More than 2 weeks behind? */
-	if (timestamp < now - GOSSIP_PRUNE_INTERVAL(rstate->dev_fast_gossip))
+	if (timestamp < now - GOSSIP_PRUNE_INTERVAL(rstate->dev_fast_gossip_prune))
 		return false;
 	return true;
 }
@@ -238,7 +238,8 @@ struct routing_state *new_routing_state(const tal_t *ctx,
 					const struct node_id *local_id,
 					struct list_head *peers,
 					const u32 *dev_gossip_time TAKES,
-					bool dev_fast_gossip)
+					bool dev_fast_gossip,
+					bool dev_fast_gossip_prune)
 {
 	struct routing_state *rstate = tal(ctx, struct routing_state);
 	rstate->nodes = new_node_map(rstate);
@@ -265,6 +266,7 @@ struct routing_state *new_routing_state(const tal_t *ctx,
 	} else
 		rstate->gossip_time = NULL;
 	rstate->dev_fast_gossip = dev_fast_gossip;
+	rstate->dev_fast_gossip_prune = dev_fast_gossip_prune;
 #endif
 	tal_add_destructor(rstate, destroy_routing_state);
 	memleak_add_helper(rstate, memleak_help_routing_tables);
@@ -2030,7 +2032,7 @@ bool routing_add_channel_update(struct routing_state *rstate,
 		}
 
 		/* Allow redundant updates once every 7 days */
-		if (timestamp < hc->bcast.timestamp + GOSSIP_PRUNE_INTERVAL(rstate->dev_fast_gossip) / 2
+		if (timestamp < hc->bcast.timestamp + GOSSIP_PRUNE_INTERVAL(rstate->dev_fast_gossip_prune) / 2
 		    && !cupdate_different(rstate->gs, hc, update)) {
 			status_debug("Ignoring redundant update for %s/%u"
 				     " (last %u, now %u)",
@@ -2382,7 +2384,7 @@ bool routing_add_node_announcement(struct routing_state *rstate,
 		}
 
 		/* Allow redundant updates once every 7 days */
-		if (timestamp < node->bcast.timestamp + GOSSIP_PRUNE_INTERVAL(rstate->dev_fast_gossip) / 2
+		if (timestamp < node->bcast.timestamp + GOSSIP_PRUNE_INTERVAL(rstate->dev_fast_gossip_prune) / 2
 		    && !nannounce_different(rstate->gs, node, msg)) {
 			status_debug("Ignoring redundant nannounce for %s"
 				     " (last %u, now %u)",
@@ -2730,7 +2732,7 @@ void route_prune(struct routing_state *rstate)
 {
 	u64 now = gossip_time_now(rstate).ts.tv_sec;
 	/* Anything below this highwater mark ought to be pruned */
-	const s64 highwater = now - GOSSIP_PRUNE_INTERVAL(rstate->dev_fast_gossip);
+	const s64 highwater = now - GOSSIP_PRUNE_INTERVAL(rstate->dev_fast_gossip_prune);
 	struct chan **pruned = tal_arr(tmpctx, struct chan *, 0);
 	u64 idx;
 

gossipd/routing.h

@@ -286,6 +286,9 @@ struct routing_state {
 
 	/* Speed up gossip. */
 	bool dev_fast_gossip;
+
+	/* Speed up pruning. */
+	bool dev_fast_gossip_prune;
 #endif
 };
 
@@ -322,7 +325,8 @@ struct routing_state *new_routing_state(const tal_t *ctx,
 					const struct node_id *local_id,
 					struct list_head *peers,
 					const u32 *dev_gossip_time TAKES,
-					bool dev_fast_gossip);
+					bool dev_fast_gossip,
+					bool dev_fast_gossip_prune);
 
 /**
  * Add a new bidirectional channel from id1 to id2 with the given

gossipd/test/run-bench-find_route.c

@@ -191,7 +191,7 @@ int main(int argc, char *argv[])
 	setup_tmpctx();
 
 	me = nodeid(0);
-	rstate = new_routing_state(tmpctx, NULL, &me, 0, NULL, NULL);
+	rstate = new_routing_state(tmpctx, NULL, &me, 0, NULL, false, false);
 	opt_register_noarg("--perfme", opt_set_bool, &perfme,
 			   "Run perfme-start and perfme-stop around benchmark");
 

gossipd/test/run-find_route-specific.c

@@ -146,7 +146,7 @@ int main(void)
 			   strlen("02cca6c5c966fcf61d121e3a70e03a1cd9eeeea024b26ea666ce974d43b242e636"),
 			   &d);
 
-	rstate = new_routing_state(tmpctx, NULL, &a, 0, NULL, NULL);
+	rstate = new_routing_state(tmpctx, NULL, &a, 0, NULL, false, false);
 
 	/* [{'active': True, 'short_id': '6990:2:1/1', 'fee_per_kw': 10, 'delay': 5, 'message_flags': 0, 'channel_flags': 1, 'destination': '0230ad0e74ea03976b28fda587bb75bdd357a1938af4424156a18265167f5e40ae', 'source': '02ea622d5c8d6143f15ed3ce1d501dd0d3d09d3b1c83a44d0034949f8a9ab60f06', 'last_update': 1504064344}, */
 

gossipd/test/run-find_route.c

@@ -181,7 +181,7 @@ int main(void)
 
 	memset(&tmp, 'a', sizeof(tmp));
 	node_id_from_privkey(&tmp, &a);
-	rstate = new_routing_state(tmpctx, NULL, &a, 0, NULL, NULL);
+	rstate = new_routing_state(tmpctx, NULL, &a, 0, NULL, false, false);
 
 	new_node(rstate, &a);
 

gossipd/test/run-overlong.c

@@ -105,7 +105,7 @@ int main(void)
 		node_id_from_privkey(&tmp, &ids[i]);
 	}
 	/* We are node 0 */
-	rstate = new_routing_state(tmpctx, NULL, &ids[0], 0, NULL, NULL);
+	rstate = new_routing_state(tmpctx, NULL, &ids[0], 0, NULL, false, false);
 
 	for (size_t i = 0; i < NUM_NODES; i++) {
 		struct chan *chan;

lightningd/closing_control.c

@@ -289,12 +289,7 @@ void peer_start_closingd(struct channel *channel,
 				      p2wpkh_for_keyidx(tmpctx, ld,
 							channel->final_key_idx),
 				      &last_remote_per_commit_secret,
-#if DEVELOPER
-				      ld->dev_fast_gossip
-#else
-				      false
-#endif
-		);
+				      IFDEV(ld->dev_fast_gossip, false));
 
 	/* We don't expect a response: it will give us feedback on
 	 * signatures sent and received, then closing_complete. */

lightningd/gossip_control.c

@@ -219,7 +219,8 @@ void gossip_init(struct lightningd *ld, int connectd_fd)
 	    ld->alias,
 	    ld->announcable,
 	    IFDEV(ld->dev_gossip_time ? &ld->dev_gossip_time: NULL, NULL),
-	    IFDEV(ld->dev_fast_gossip, false));
+	    IFDEV(ld->dev_fast_gossip, false),
+	    IFDEV(ld->dev_fast_gossip_prune, false));
 	subd_send_msg(ld->gossip, msg);
 }
 

lightningd/lightningd.c

@@ -120,6 +120,7 @@ static struct lightningd *new_lightningd(const tal_t *ctx)
 	ld->dev_allow_localhost = false;
 	ld->dev_gossip_time = 0;
 	ld->dev_fast_gossip = false;
+	ld->dev_fast_gossip_prune = false;
 	ld->dev_force_privkey = NULL;
 	ld->dev_force_bip32_seed = NULL;
 	ld->dev_force_channel_secrets = NULL;

lightningd/lightningd.h

@@ -203,6 +203,7 @@ struct lightningd {
 
 	/* Speedup gossip propagation, for testing. */
 	bool dev_fast_gossip;
+	bool dev_fast_gossip_prune;
 
 	/* Things we've marked as not leaking. */
 	const void **notleaks;

lightningd/options.c

@@ -416,7 +416,10 @@ static void dev_register_opts(struct lightningd *ld)
 
 	opt_register_noarg("--dev-fast-gossip", opt_set_bool,
 			   &ld->dev_fast_gossip,
-			   "Make gossip broadcast 1 second, pruning 14 seconds");
+			   "Make gossip broadcast 1 second, etc");
+	opt_register_noarg("--dev-fast-gossip-prune", opt_set_bool,
+			   &ld->dev_fast_gossip_prune,
+			   "Make gossip pruning 30 seconds");
 
 	opt_register_arg("--dev-gossip-time", opt_set_u32, opt_show_u32,
 			 &ld->dev_gossip_time,

tests/test_gossip.py

@@ -18,11 +18,11 @@ with open('config.vars') as configfile:
 DEVELOPER = os.getenv("DEVELOPER", config['DEVELOPER']) == "1"
 
 
-@unittest.skipIf(not DEVELOPER, "needs --dev-fast-gossip for fast pruning")
+@unittest.skipIf(not DEVELOPER, "needs --dev-fast-gossip-prune")
 def test_gossip_pruning(node_factory, bitcoind):
     """ Create channel and see it being updated in time before pruning
     """
-    l1, l2, l3 = node_factory.get_nodes(3)
+    l1, l2, l3 = node_factory.get_nodes(3, opts={'dev-fast-gossip-prune': None})
 
     l1.rpc.connect(l2.info['id'], 'localhost', l2.port)
     l2.rpc.connect(l3.info['id'], 'localhost', l3.port)
@@ -37,10 +37,10 @@ def test_gossip_pruning(node_factory, bitcoind):
     wait_for(lambda: [c['active'] for c in l2.rpc.listchannels()['channels']] == [True] * 4)
     wait_for(lambda: [c['active'] for c in l3.rpc.listchannels()['channels']] == [True] * 4)
 
-    # All of them should send a keepalive message (after ~60 seconds)
+    # All of them should send a keepalive message (after 30 seconds)
     l1.daemon.wait_for_logs([
         'Sending keepalive channel_update for {}'.format(scid1),
-    ], timeout=90)
+    ], timeout=50)
     l2.daemon.wait_for_logs([
         'Sending keepalive channel_update for {}'.format(scid1),
         'Sending keepalive channel_update for {}'.format(scid2),
@@ -49,12 +49,12 @@ def test_gossip_pruning(node_factory, bitcoind):
         'Sending keepalive channel_update for {}'.format(scid2),
     ])
 
-    # Now kill l2, so that l1 and l3 will prune from their view after 90 seconds
+    # Now kill l2, so that l1 and l3 will prune from their view after 60 seconds
     l2.stop()
 
-    # We check every 90/4 seconds, and takes 90 seconds since last update.
+    # We check every 60/4 seconds, and takes 60 seconds since last update.
     l1.daemon.wait_for_log("Pruning channel {} from network view".format(scid2),
-                           timeout=120)
+                           timeout=80)
     l3.daemon.wait_for_log("Pruning channel {} from network view".format(scid1))
 
     assert scid2 not in [c['short_channel_id'] for c in l1.rpc.listchannels()['channels']]