gossip: Use the custom gossip wire msg to wrap channel_announcements

This stores and reads the channel_announcements in the wrapping message which
allows us to store associated data with the raw channel_announcements.

The gossip_store applies channel_announcements directly but it also returns it,
and it gets discarded as a duplicate. In the next commit we'll have gossip_store
apply all changes, bypassing verification, so the duplication is only temporary.

Signed-off-by: Christian Decker <decker.christian@gmail.com>

gossipd/gossip.c

@@ -1530,7 +1530,7 @@ static bool master_conn_idle(struct io_conn *conn UNUSED,
 {
 	const u8 *msg;
 	struct daemon *daemon = container_of(dc, struct daemon, master);
-	msg = gossip_store_read_next(tmpctx, daemon->rstate->store);
+	msg = gossip_store_read_next(tmpctx, daemon->rstate, daemon->rstate->store);
 
 	if (msg) {
 		handle_gossip_msg(daemon, msg, false);

gossipd/gossip_store.c

@@ -4,7 +4,10 @@
 #include <ccan/read_write_all/read_write_all.h>
 #include <common/status.h>
 #include <fcntl.h>
+#include <gossipd/gen_gossip_wire.h>
 #include <unistd.h>
+#include <wire/gen_peer_wire.h>
+#include <wire/wire.h>
 
 #define GOSSIP_STORE_FILENAME "gossip_store"
 static u8 gossip_store_version = 0x01;
@@ -56,11 +59,21 @@ void gossip_store_append(struct gossip_store *gs, const u8 *msg)
 		gs->write_pos += sizeof(belen) + msglen;
 }
 
-const u8 *gossip_store_read_next(const tal_t *ctx, struct gossip_store *gs)
+void gossip_store_add_channel_announcement(struct gossip_store *gs, const u8 *gossip_msg, u64 satoshis)
+{
+	u8 *msg = towire_gossip_store_channel_announcement(NULL, gossip_msg, satoshis);
+	gossip_store_append(gs, msg);
+	tal_free(msg);
+}
+
+const u8 *gossip_store_read_next(const tal_t *ctx, struct routing_state *rstate,
+				 struct gossip_store *gs)
 {
 	beint32_t belen;
 	u32 msglen;
-	u8 *msg;
+	u8 *msg, *gossip_msg;
+	u64 satoshis;
+	enum gossip_wire_type type;
 
 	/* Did we already reach the end of the gossip_store? */
 	if (gs->read_pos == -1)
@@ -84,8 +97,19 @@ const u8 *gossip_store_read_next(const tal_t *ctx, struct gossip_store *gs)
 		gs->write_pos = gs->read_pos;
 		gs->read_pos = -1;
 		ftruncate(gs->fd, gs->write_pos);
-	} else
+		return NULL;
+	}
+
 	gs->read_pos += sizeof(belen) + msglen;
+	type = fromwire_peektype(msg);
+
+	if (type == WIRE_GOSSIP_STORE_CHANNEL_ANNOUNCEMENT) {
+		fromwire_gossip_store_channel_announcement(msg, msg, &gossip_msg, &satoshis);
+		routing_add_channel_announcement(rstate, gossip_msg, satoshis);
+
+		/* No harm in returning it, it'll get discarded as a duplicate */
+		return gossip_msg;
+	}
 
 	return msg;
 }

gossipd/gossip_store.h

@@ -5,11 +5,13 @@
 
 #include <ccan/short_types/short_types.h>
 #include <ccan/tal/tal.h>
+#include <gossipd/routing.h>
 
 /**
  * gossip_store -- On-disk storage related information
  */
 struct gossip_store;
+struct routing_state;
 
 struct gossip_store *gossip_store_new(const tal_t *ctx);
 
@@ -30,6 +32,12 @@ void gossip_store_append(struct gossip_store *gs, const u8 *msg);
  * @return The gossip message allocated from `ctx`, `NULL` if no more messages are
  *         available.
  */
-const u8 *gossip_store_read_next(const tal_t *ctx, struct gossip_store *gs);
+const u8 *gossip_store_read_next(const tal_t *ctx, struct routing_state *rstate,
+				 struct gossip_store *gs);
 
+/**
+ * Store a channel_announcement with all its extra data
+ */
+void gossip_store_add_channel_announcement(struct gossip_store *gs,
+					   const u8 *gossip_msg, u64 satoshis);
 #endif /* GOSSIPD_GOSSIP_STORE_H */

gossipd/routing.c

@@ -833,7 +833,7 @@ bool handle_pending_cannouncement(struct routing_state *rstate,
 	}
 
 	if (pending->store)
-		gossip_store_append(rstate->store, pending->announce);
+		gossip_store_add_channel_announcement(rstate->store, pending->announce, satoshis);
 	routing_add_channel_announcement(rstate, pending->announce, satoshis);
 
 	local = pubkey_eq(&pending->node_id_1, &rstate->local_id) ||

gossipd/test/run-bench-find_route.c

@@ -62,9 +62,15 @@ bool fromwire_channel_announcement(const tal_t *ctx UNNEEDED, const void *p UNNE
 /* Generated stub for fromwire_channel_update */
 bool fromwire_channel_update(const void *p UNNEEDED, secp256k1_ecdsa_signature *signature UNNEEDED, struct bitcoin_blkid *chain_hash UNNEEDED, struct short_channel_id *short_channel_id UNNEEDED, u32 *timestamp UNNEEDED, u16 *flags UNNEEDED, u16 *cltv_expiry_delta UNNEEDED, u64 *htlc_minimum_msat UNNEEDED, u32 *fee_base_msat UNNEEDED, u32 *fee_proportional_millionths UNNEEDED)
 { fprintf(stderr, "fromwire_channel_update called!\n"); abort(); }
+/* Generated stub for fromwire_gossip_store_channel_announcement */
+bool fromwire_gossip_store_channel_announcement(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, u8 **announcement UNNEEDED, u64 *satoshis UNNEEDED)
+{ fprintf(stderr, "fromwire_gossip_store_channel_announcement called!\n"); abort(); }
 /* Generated stub for fromwire_node_announcement */
 bool fromwire_node_announcement(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, secp256k1_ecdsa_signature *signature UNNEEDED, u8 **features UNNEEDED, u32 *timestamp UNNEEDED, struct pubkey *node_id UNNEEDED, u8 rgb_color[3] UNNEEDED, u8 alias[32] UNNEEDED, u8 **addresses UNNEEDED)
 { fprintf(stderr, "fromwire_node_announcement called!\n"); abort(); }
+/* Generated stub for fromwire_peektype */
+int fromwire_peektype(const u8 *cursor UNNEEDED)
+{ fprintf(stderr, "fromwire_peektype called!\n"); abort(); }
 /* Generated stub for fromwire_u8 */
 u8 fromwire_u8(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
 { fprintf(stderr, "fromwire_u8 called!\n"); abort(); }
@@ -93,6 +99,9 @@ u8 *towire_errorfmt(const tal_t *ctx UNNEEDED,
 		    const struct channel_id *channel UNNEEDED,
 		    const char *fmt UNNEEDED, ...)
 { fprintf(stderr, "towire_errorfmt called!\n"); abort(); }
+/* Generated stub for towire_gossip_store_channel_announcement */
+u8 *towire_gossip_store_channel_announcement(const tal_t *ctx UNNEEDED, const u8 *announcement UNNEEDED, u64 satoshis UNNEEDED)
+{ fprintf(stderr, "towire_gossip_store_channel_announcement called!\n"); abort(); }
 /* AUTOGENERATED MOCKS END */
 
 /* Updates existing route if required. */

gossipd/test/run-find_route-specific.c

@@ -26,9 +26,15 @@ bool fromwire_channel_announcement(const tal_t *ctx UNNEEDED, const void *p UNNE
 /* Generated stub for fromwire_channel_update */
 bool fromwire_channel_update(const void *p UNNEEDED, secp256k1_ecdsa_signature *signature UNNEEDED, struct bitcoin_blkid *chain_hash UNNEEDED, struct short_channel_id *short_channel_id UNNEEDED, u32 *timestamp UNNEEDED, u16 *flags UNNEEDED, u16 *cltv_expiry_delta UNNEEDED, u64 *htlc_minimum_msat UNNEEDED, u32 *fee_base_msat UNNEEDED, u32 *fee_proportional_millionths UNNEEDED)
 { fprintf(stderr, "fromwire_channel_update called!\n"); abort(); }
+/* Generated stub for fromwire_gossip_store_channel_announcement */
+bool fromwire_gossip_store_channel_announcement(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, u8 **announcement UNNEEDED, u64 *satoshis UNNEEDED)
+{ fprintf(stderr, "fromwire_gossip_store_channel_announcement called!\n"); abort(); }
 /* Generated stub for fromwire_node_announcement */
 bool fromwire_node_announcement(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, secp256k1_ecdsa_signature *signature UNNEEDED, u8 **features UNNEEDED, u32 *timestamp UNNEEDED, struct pubkey *node_id UNNEEDED, u8 rgb_color[3] UNNEEDED, u8 alias[32] UNNEEDED, u8 **addresses UNNEEDED)
 { fprintf(stderr, "fromwire_node_announcement called!\n"); abort(); }
+/* Generated stub for fromwire_peektype */
+int fromwire_peektype(const u8 *cursor UNNEEDED)
+{ fprintf(stderr, "fromwire_peektype called!\n"); abort(); }
 /* Generated stub for fromwire_u8 */
 u8 fromwire_u8(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
 { fprintf(stderr, "fromwire_u8 called!\n"); abort(); }
@@ -57,6 +63,9 @@ u8 *towire_errorfmt(const tal_t *ctx UNNEEDED,
 		    const struct channel_id *channel UNNEEDED,
 		    const char *fmt UNNEEDED, ...)
 { fprintf(stderr, "towire_errorfmt called!\n"); abort(); }
+/* Generated stub for towire_gossip_store_channel_announcement */
+u8 *towire_gossip_store_channel_announcement(const tal_t *ctx UNNEEDED, const u8 *announcement UNNEEDED, u64 satoshis UNNEEDED)
+{ fprintf(stderr, "towire_gossip_store_channel_announcement called!\n"); abort(); }
 /* AUTOGENERATED MOCKS END */
 
 const void *trc;

gossipd/test/run-find_route.c

@@ -24,9 +24,15 @@ bool fromwire_channel_announcement(const tal_t *ctx UNNEEDED, const void *p UNNE
 /* Generated stub for fromwire_channel_update */
 bool fromwire_channel_update(const void *p UNNEEDED, secp256k1_ecdsa_signature *signature UNNEEDED, struct bitcoin_blkid *chain_hash UNNEEDED, struct short_channel_id *short_channel_id UNNEEDED, u32 *timestamp UNNEEDED, u16 *flags UNNEEDED, u16 *cltv_expiry_delta UNNEEDED, u64 *htlc_minimum_msat UNNEEDED, u32 *fee_base_msat UNNEEDED, u32 *fee_proportional_millionths UNNEEDED)
 { fprintf(stderr, "fromwire_channel_update called!\n"); abort(); }
+/* Generated stub for fromwire_gossip_store_channel_announcement */
+bool fromwire_gossip_store_channel_announcement(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, u8 **announcement UNNEEDED, u64 *satoshis UNNEEDED)
+{ fprintf(stderr, "fromwire_gossip_store_channel_announcement called!\n"); abort(); }
 /* Generated stub for fromwire_node_announcement */
 bool fromwire_node_announcement(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, secp256k1_ecdsa_signature *signature UNNEEDED, u8 **features UNNEEDED, u32 *timestamp UNNEEDED, struct pubkey *node_id UNNEEDED, u8 rgb_color[3] UNNEEDED, u8 alias[32] UNNEEDED, u8 **addresses UNNEEDED)
 { fprintf(stderr, "fromwire_node_announcement called!\n"); abort(); }
+/* Generated stub for fromwire_peektype */
+int fromwire_peektype(const u8 *cursor UNNEEDED)
+{ fprintf(stderr, "fromwire_peektype called!\n"); abort(); }
 /* Generated stub for fromwire_u8 */
 u8 fromwire_u8(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
 { fprintf(stderr, "fromwire_u8 called!\n"); abort(); }
@@ -55,6 +61,9 @@ u8 *towire_errorfmt(const tal_t *ctx UNNEEDED,
 		    const struct channel_id *channel UNNEEDED,
 		    const char *fmt UNNEEDED, ...)
 { fprintf(stderr, "towire_errorfmt called!\n"); abort(); }
+/* Generated stub for towire_gossip_store_channel_announcement */
+u8 *towire_gossip_store_channel_announcement(const tal_t *ctx UNNEEDED, const u8 *announcement UNNEEDED, u64 satoshis UNNEEDED)
+{ fprintf(stderr, "towire_gossip_store_channel_announcement called!\n"); abort(); }
 /* AUTOGENERATED MOCKS END */
 
 /* Updates existing route if required. */