common/gossip_rcvd_filter: cache for received gossip.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

common/Makefile

@@ -20,6 +20,7 @@ COMMON_SRC_NOGEN :=				\
 	common/dev_disconnect.c			\
 	common/features.c			\
 	common/funding_tx.c			\
+	common/gossip_rcvd_filter.c		\
 	common/gossip_store.c			\
 	common/hash_u5.c			\
 	common/htlc_state.c			\

common/gossip_rcvd_filter.c

@@ -0,0 +1,117 @@
+#include <ccan/crypto/siphash24/siphash24.h>
+#include <ccan/htable/htable.h>
+#include <ccan/structeq/structeq.h>
+#include <common/gossip_rcvd_filter.h>
+#include <common/memleak.h>
+#include <common/pseudorand.h>
+#include <wire/gen_peer_wire.h>
+
+static u64 msg_key(const u8 *msg)
+{
+	return siphash24(siphash_seed(), msg, tal_bytelen(msg));
+}
+
+static size_t rehash(const void *key, void *unused)
+{
+	return *(u64 *)key;
+}
+
+static void destroy_msg_map(struct htable *ht)
+{
+	htable_clear(ht);
+}
+
+static struct htable *new_msg_map(const tal_t *ctx)
+{
+	struct htable *ht = tal(ctx, struct htable);
+
+	htable_init(ht, rehash, NULL);
+	tal_add_destructor(ht, destroy_msg_map);
+	return ht;
+}
+
+/* We age by keeping two maps, a current and an old one */
+struct gossip_rcvd_filter {
+	struct htable *cur, *old;
+};
+
+#if DEVELOPER
+static void memleak_help_gossip_rcvd_filter(struct htable *memtable,
+					    struct gossip_rcvd_filter *grf)
+{
+	memleak_remove_htable(memtable, grf->cur);
+	memleak_remove_htable(memtable, grf->old);
+}
+#endif
+
+struct gossip_rcvd_filter *new_gossip_rcvd_filter(const tal_t *ctx)
+{
+	struct gossip_rcvd_filter *f = tal(ctx, struct gossip_rcvd_filter);
+
+	f->cur = new_msg_map(f);
+	f->old = new_msg_map(f);
+	memleak_add_helper(f, memleak_help_gossip_rcvd_filter);
+	return f;
+}
+
+static bool extract_msg_key(const u8 *msg, u64 *key)
+{
+	int type = fromwire_peektype(msg);
+
+	if (type != WIRE_CHANNEL_ANNOUNCEMENT
+	    && type != WIRE_NODE_ANNOUNCEMENT
+	    && type != WIRE_CHANNEL_UPDATE)
+		return false;
+
+	*key = msg_key(msg);
+	return true;
+}
+
+/* Add a gossip msg to the received map */
+void gossip_rcvd_filter_add(struct gossip_rcvd_filter *f, const u8 *msg)
+{
+	u64 key;
+
+	/* We don't attach destructor here directly to tag; would be neat,
+	 * but it's also an extra allocation */
+	if (extract_msg_key(msg, &key))
+		htable_add(f->cur, key, tal_dup(f->cur, u64, &key));
+}
+
+/* htable is fast, but it's also horribly manual. */
+static bool msg_map_remove(struct htable *ht, u64 key)
+{
+	struct htable_iter i;
+	u64 *c;
+
+	for (c = htable_firstval(ht, &i, key);
+	     c;
+	     c = htable_nextval(ht, &i, key)) {
+		if (*c == key) {
+			htable_del(ht, key, c);
+			tal_free(c);
+			return true;
+		}
+	}
+	return false;
+}
+
+/* Is a gossip msg in the received map? (Removes it) */
+bool gossip_rcvd_filter_del(struct gossip_rcvd_filter *f, const u8 *msg)
+{
+	u64 key;
+
+	if (!extract_msg_key(msg, &key))
+		return false;
+
+	/* Look in both for gossip. */
+	return msg_map_remove(f->cur, key) || msg_map_remove(f->old, key);
+}
+
+/* Flush out old entries. */
+void gossip_rcvd_filter_age(struct gossip_rcvd_filter *f)
+{
+	tal_free(f->old);
+	f->old = f->cur;
+	f->cur = new_msg_map(f);
+}

common/gossip_rcvd_filter.h

@@ -0,0 +1,22 @@
+/* This implements a cheap gossip cache, so we can recognize what gossip
+ * msgs this peer sent us, thus avoid retransmitting gossip it sent. */
+#ifndef LIGHTNING_COMMON_GOSSIP_RCVD_FILTER_H
+#define LIGHTNING_COMMON_GOSSIP_RCVD_FILTER_H
+#include "config.h"
+#include <ccan/short_types/short_types.h>
+#include <ccan/tal/tal.h>
+
+struct gossip_rcvd_filter;
+
+struct gossip_rcvd_filter *new_gossip_rcvd_filter(const tal_t *ctx);
+
+/* Add a gossip msg to the received map */
+void gossip_rcvd_filter_add(struct gossip_rcvd_filter *map, const u8 *msg);
+
+/* Is a gossip msg in the received map? (Removes it) */
+bool gossip_rcvd_filter_del(struct gossip_rcvd_filter *map, const u8 *msg);
+
+/* Flush out old entries. */
+void gossip_rcvd_filter_age(struct gossip_rcvd_filter *map);
+
+#endif /* LIGHTNING_COMMON_GOSSIP_RCVD_FILTER_H */

common/test/run-gossip_rcvd_filter.c

@@ -0,0 +1,134 @@
+#include "../gossip_rcvd_filter.c"
+#include "../pseudorand.c"
+#include "../../wire/fromwire.c"
+#include <common/utils.h>
+#include <stdio.h>
+
+/* AUTOGENERATED MOCKS START */
+/* Generated stub for bigsize_get */
+size_t bigsize_get(const u8 *p UNNEEDED, size_t max UNNEEDED, bigsize_t *val UNNEEDED)
+{ fprintf(stderr, "bigsize_get called!\n"); abort(); }
+/* Generated stub for memleak_remove_htable */
+void memleak_remove_htable(struct htable *memtable UNNEEDED, const struct htable *ht UNNEEDED)
+{ fprintf(stderr, "memleak_remove_htable called!\n"); abort(); }
+/* AUTOGENERATED MOCKS END */
+
+void memleak_add_helper_(const tal_t *p UNNEEDED,
+			 void (*cb)(struct htable *memtable,
+				    const tal_t *) UNNEEDED)
+{
+}
+
+static u8 *mkgossip(const tal_t *ctx, const char *str)
+{
+	return tal_hexdata(ctx, str, strlen(str));
+}
+
+int main(void)
+{
+	const tal_t *ctx = tal(NULL, char);
+	struct gossip_rcvd_filter *f = new_gossip_rcvd_filter(ctx);
+	const u8 *msg[3], *badmsg;
+
+	setup_locale();
+
+	msg[0] = mkgossip(ctx, "0100231024fcd59aa58ca8e2ed8f71e07843fc576dd6b2872681960ce64f5f3cd3b5386211a103736bf1de2c03a74f5885d50ea30d21a82e4389339ad13149ac7f52942e6ff0778952b7cb001350d1e2edd25cee80c4c64d624a0273be5436923f5524f1f7e4586007203b2f2c47d6863052529321ebb8e0a171ed013c889bbeaa7a462e9826861c608428509804eb4dd5f75dc5e6baa03205933759fd7abcb2e0304b1a895abb7de3d24e92ade99a6a14f51ac9852ef3daf68b8ad40459d8a6124f23e1271537347b6a1bc9bff1f3e6f60e93177b3bf1d53e76771be9c974ba6b1c6d4916762c0867c13f3617e4893f6272c64fa360aaf6c2a94af7739c498bab3600006fe28c0ab6f1b372c1a6a246ae63f74f931e8365e15a089c68d619000000000008a8090000510000020cf679b34b5819dfbaf68663bdd636c92117b6c04981940e878818b736c0cdb702809e936f0e82dfce13bcc47c77112db068f569e1db29e7bf98bcdd68b838ee8402590349bcd37d04b81022e52bd65d5119a43e0d7b78f526971ede38d2cd1c0d9e02eec6ac38e4acad847cd42b0946977896527b9e1f7dd59525a1a1344a3cea7fa3");
+	msg[1] = mkgossip(ctx, "0102ccc0a84e4ce09f522f7765db7c30b822ebb346eb17dda92612d03cc8e53ee1454b6c9a918a60ac971e623fd056687f17a01d3c7e805723f7b68be0e8544013546fe28c0ab6f1b372c1a6a246ae63f74f931e8365e15a089c68d619000000000008a80900005100005d06bacc0102009000000000000003e8000003e8000000010000000005e69ec0");
+	msg[2] = mkgossip(ctx, "01017914e70e3ef662e8d75166ea64465f8679d042bdc26b91c7de6d2c5bdd1f73654f4df7c010ea4c41538bbc5f0f6528dfd48097c7c18f3febae4dc36819550c5900005c8d5bac020cf679b34b5819dfbaf68663bdd636c92117b6c04981940e878818b736c0cdb73399ff5468655f4c656176696e677300000000000000000000000000000000000000000007018e3b64fd2607");
+
+	/* Not a gossip msg. */
+	badmsg = tal_hexdata(ctx, "00100000", strlen("00100000"));
+
+	gossip_rcvd_filter_add(f, msg[0]);
+	assert(htable_count(f->cur) == 1);
+	assert(htable_count(f->old) == 0);
+
+	gossip_rcvd_filter_add(f, msg[1]);
+	assert(htable_count(f->cur) == 2);
+	assert(htable_count(f->old) == 0);
+
+	gossip_rcvd_filter_add(f, msg[2]);
+	assert(htable_count(f->cur) == 3);
+	assert(htable_count(f->old) == 0);
+
+	gossip_rcvd_filter_add(f, badmsg);
+	assert(htable_count(f->cur) == 3);
+	assert(htable_count(f->old) == 0);
+
+	assert(gossip_rcvd_filter_del(f, msg[0]));
+	assert(htable_count(f->cur) == 2);
+	assert(htable_count(f->old) == 0);
+	assert(!gossip_rcvd_filter_del(f, msg[0]));
+	assert(htable_count(f->cur) == 2);
+	assert(htable_count(f->old) == 0);
+	assert(gossip_rcvd_filter_del(f, msg[1]));
+	assert(htable_count(f->cur) == 1);
+	assert(htable_count(f->old) == 0);
+	assert(!gossip_rcvd_filter_del(f, msg[1]));
+	assert(htable_count(f->cur) == 1);
+	assert(htable_count(f->old) == 0);
+	assert(gossip_rcvd_filter_del(f, msg[2]));
+	assert(htable_count(f->cur) == 0);
+	assert(htable_count(f->old) == 0);
+	assert(!gossip_rcvd_filter_del(f, msg[2]));
+	assert(htable_count(f->cur) == 0);
+	assert(htable_count(f->old) == 0);
+	assert(!gossip_rcvd_filter_del(f, badmsg));
+	assert(htable_count(f->cur) == 0);
+	assert(htable_count(f->old) == 0);
+
+	/* Re-add them, and age. */
+	gossip_rcvd_filter_add(f, msg[0]);
+	gossip_rcvd_filter_add(f, msg[1]);
+	gossip_rcvd_filter_add(f, msg[2]);
+	assert(htable_count(f->cur) == 3);
+	assert(htable_count(f->old) == 0);
+
+	gossip_rcvd_filter_age(f);
+	assert(htable_count(f->cur) == 0);
+	assert(htable_count(f->old) == 3);
+
+	/* Delete 1 and 2. */
+	assert(gossip_rcvd_filter_del(f, msg[2]));
+	assert(gossip_rcvd_filter_del(f, msg[1]));
+	assert(htable_count(f->cur) == 0);
+	assert(htable_count(f->old) == 1);
+	assert(!gossip_rcvd_filter_del(f, msg[2]));
+	assert(!gossip_rcvd_filter_del(f, msg[1]));
+	assert(htable_count(f->cur) == 0);
+	assert(htable_count(f->old) == 1);
+	assert(!gossip_rcvd_filter_del(f, badmsg));
+	assert(htable_count(f->cur) == 0);
+	assert(htable_count(f->old) == 1);
+
+	/* Re-add 2, and age. */
+	gossip_rcvd_filter_add(f, msg[2]);
+	assert(htable_count(f->cur) == 1);
+	assert(htable_count(f->old) == 1);
+
+	gossip_rcvd_filter_age(f);
+	assert(htable_count(f->cur) == 0);
+	assert(htable_count(f->old) == 1);
+
+	/* Now, only 2 remains. */
+	assert(!gossip_rcvd_filter_del(f, msg[0]));
+	assert(!gossip_rcvd_filter_del(f, msg[1]));
+	assert(gossip_rcvd_filter_del(f, msg[2]));
+	assert(!gossip_rcvd_filter_del(f, msg[2]));
+	assert(htable_count(f->cur) == 0);
+	assert(htable_count(f->old) == 0);
+
+	/* They should have no children, and f should only have 2. */
+	assert(!tal_first(f->cur));
+	assert(!tal_first(f->old));
+
+	assert((tal_first(f) == f->cur
+		&& tal_next(f->cur) == f->old
+		&& tal_next(f->old) == NULL)
+	       || (tal_first(f) == f->old
+		   && tal_next(f->old) == f->cur
+		   && tal_next(f->cur) == NULL));
+
+	tal_free(ctx);
+	return 0;
+}