per-peer-daemons: tie in gossip filter.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

channeld/Makefile

@@ -48,6 +48,7 @@ CHANNELD_COMMON_OBJS :=				\
 	common/features.o			\
 	common/gen_status_wire.o		\
 	common/gen_peer_status_wire.o		\
+	common/gossip_rcvd_filter.o		\
 	common/gossip_store.o			\
 	common/htlc_state.o			\
 	common/htlc_trim.o			\

closingd/Makefile

@@ -57,6 +57,7 @@ CLOSINGD_COMMON_OBJS :=				\
 	common/features.o			\
 	common/gen_peer_status_wire.o		\
 	common/gen_status_wire.o		\
+	common/gossip_rcvd_filter.o		\
 	common/gossip_store.o			\
 	common/htlc_wire.o			\
 	common/key_derive.o			\
@@ -66,6 +67,7 @@ CLOSINGD_COMMON_OBJS :=				\
 	common/peer_failed.o			\
 	common/per_peer_state.o			\
 	common/permute_tx.o			\
+	common/pseudorand.o			\
 	common/read_peer_msg.o			\
 	common/socket_close.o			\
 	common/status.o				\

common/gossip_store.c

@@ -1,6 +1,7 @@
 #include <assert.h>
 #include <ccan/crc32c/crc32c.h>
 #include <common/features.h>
+#include <common/gossip_rcvd_filter.h>
 #include <common/gossip_store.h>
 #include <common/per_peer_state.h>
 #include <common/status.h>
@@ -122,6 +123,12 @@ u8 *gossip_store_next(const tal_t *ctx, struct per_peer_state *pps)
 						 0, SEEK_CUR) - msglen,
 				      tal_hex(tmpctx, msg));
 
+		/* Don't send back gossip they sent to us! */
+		if (gossip_rcvd_filter_del(pps->grf, msg)) {
+			msg = tal_free(msg);
+			continue;
+		}
+
 		/* Ignore gossipd internal messages. */
 		type = fromwire_peektype(msg);
 		if (type != WIRE_CHANNEL_ANNOUNCEMENT

common/per_peer_state.c

@@ -1,5 +1,6 @@
 #include <assert.h>
 #include <ccan/fdpass/fdpass.h>
+#include <common/gossip_rcvd_filter.h>
 #include <common/per_peer_state.h>
 #include <unistd.h>
 #include <wire/wire.h>
@@ -22,6 +23,7 @@ struct per_peer_state *new_per_peer_state(const tal_t *ctx,
 	pps->cs = *cs;
 	pps->gs = NULL;
 	pps->peer_fd = pps->gossip_fd = pps->gossip_store_fd = -1;
+	pps->grf = new_gossip_rcvd_filter(pps);
 	tal_add_destructor(pps, destroy_per_peer_state);
 	return pps;
 }
@@ -70,6 +72,7 @@ void towire_per_peer_state(u8 **pptr, const struct per_peer_state *pps)
 	towire_bool(pptr, pps->gs != NULL);
 	if (pps->gs)
 		towire_gossip_state(pptr, pps->gs);
+	/* We don't pass the gossip_rcvd_filter: it's merely an optimization */
 }
 
 void per_peer_state_fdpass_send(int fd, const struct per_peer_state *pps)
@@ -138,4 +141,5 @@ void per_peer_state_reset_gossip_timer(struct per_peer_state *pps)
 	t = time_from_msec(pps->dev_gossip_broadcast_msec);
 #endif
 	pps->gs->next_gossip = timemono_add(time_mono(), t);
+	gossip_rcvd_filter_age(pps->grf);
 }

common/per_peer_state.h

@@ -20,6 +20,8 @@ struct per_peer_state {
 	struct crypto_state cs;
 	/* NULL if it's not initialized yet */
 	struct gossip_state *gs;
+	/* Cache of msgs we have received, to avoid re-xmitting from store */
+	struct gossip_rcvd_filter *grf;
 #if DEVELOPER
 	/* Normally 60000, but adjustable for dev mode */
 	u32 dev_gossip_broadcast_msec;

common/read_peer_msg.c

@@ -1,5 +1,6 @@
 #include <ccan/fdpass/fdpass.h>
 #include <common/crypto_sync.h>
+#include <common/gossip_rcvd_filter.h>
 #include <common/gossip_store.h>
 #include <common/peer_failed.h>
 #include <common/per_peer_state.h>
@@ -166,6 +167,7 @@ bool handle_peer_gossip_or_error(struct per_peer_state *pps,
 	if (handle_timestamp_filter(pps, msg))
 		return true;
 	else if (is_msg_for_gossipd(msg)) {
+		gossip_rcvd_filter_add(pps->grf, msg);
 		wire_sync_write(pps->gossip_fd, msg);
 		/* wire_sync_write takes, so don't take again. */
 		return true;

connectd/Makefile

@@ -51,6 +51,7 @@ CONNECTD_COMMON_OBJS :=				\
 	common/dev_disconnect.o			\
 	common/features.o			\
 	common/gen_status_wire.o		\
+	common/gossip_rcvd_filter.o		\
 	common/key_derive.o			\
 	common/memleak.o			\
 	common/msg_queue.o			\

devtools/Makefile

@@ -16,10 +16,12 @@ DEVTOOLS_COMMON_OBJS :=				\
 	common/crypto_state.o			\
 	common/decode_short_channel_ids.o	\
 	common/features.o			\
+	common/gossip_rcvd_filter.o		\
 	common/hash_u5.o			\
 	common/memleak.o			\
 	common/node_id.o			\
 	common/per_peer_state.o			\
+	common/pseudorand.o			\
 	common/json.o				\
 	common/json_helpers.o			\
 	common/type_to_string.o			\

gossipd/Makefile

@@ -51,6 +51,7 @@ GOSSIPD_COMMON_OBJS :=				\
 	common/dev_disconnect.o			\
 	common/features.o			\
 	common/gen_status_wire.o		\
+	common/gossip_rcvd_filter.o		\
 	common/key_derive.o			\
 	common/memleak.o			\
 	common/msg_queue.o			\

lightningd/Makefile

@@ -31,6 +31,7 @@ LIGHTNINGD_COMMON_OBJS :=			\
 	common/funding_tx.o			\
 	common/gen_peer_status_wire.o		\
 	common/gen_status_wire.o		\
+	common/gossip_rcvd_filter.o		\
 	common/hash_u5.o			\
 	common/htlc_state.o			\
 	common/htlc_trim.o			\

openingd/Makefile

@@ -51,6 +51,7 @@ OPENINGD_COMMON_OBJS :=				\
 	common/funding_tx.o			\
 	common/gen_status_wire.o		\
 	common/gen_peer_status_wire.o		\
+	common/gossip_rcvd_filter.o		\
 	common/gossip_store.o			\
 	common/htlc_wire.o			\
 	common/initial_channel.o		\

openingd/openingd.c

@@ -21,6 +21,7 @@
 #include <common/features.h>
 #include <common/funding_tx.h>
 #include <common/gen_peer_status_wire.h>
+#include <common/gossip_rcvd_filter.h>
 #include <common/gossip_store.h>
 #include <common/initial_channel.h>
 #include <common/key_derive.h>
@@ -377,6 +378,7 @@ static u8 *opening_negotiate_msg(const tal_t *ctx, struct state *state,
 
 		/* Some messages go straight to gossipd. */
 		if (is_msg_for_gossipd(msg)) {
+			gossip_rcvd_filter_add(state->pps->grf, msg);
 			wire_sync_write(state->pps->gossip_fd, take(msg));
 			continue;
 		}

tests/test_gossip.py

@@ -978,8 +978,13 @@ def test_node_reannounce(node_factory, bitcoind):
     nannouncement = l2.daemon.wait_for_log(r'{}.*\[IN\] 0101.*{}'.format(l1.info['id'], l1.info['id'])).split('[IN] ')[1]
     wait_for(lambda: only_one(l2.rpc.listnodes(l1.info['id'])['nodes'])['alias'] == 'SENIORBEAM')
 
-    # Restart should re-xmit exact same update on reconnect.
-    l1.restart()
+    # Restart should re-xmit exact same update on reconnect, but make sure
+    # l2 doesn't send it first!
+    l1.stop()
+    l2.stop()
+    os.remove(os.path.join(l2.daemon.lightning_dir, 'gossip_store'))
+    l2.start()
+    l1.start()
 
     # l1 should retransmit it exactly the same (no timestamp change!)
     l2.daemon.wait_for_log(r'{}.*\[IN\] {}'.format(l1.info['id'], nannouncement))
@@ -1377,7 +1382,6 @@ def test_gossip_announce_unknown_block(node_factory, bitcoind):
     sync_blockheight(bitcoind, [l1])
 
 
-@pytest.mark.xfail(strict=True)
 @unittest.skipIf(not DEVELOPER, "gossip without DEVELOPER=1 is slow")
 def test_gossip_no_backtalk(node_factory):
     l1, l2 = node_factory.line_graph(2, wait_for_announce=True)