gossipd: push our own gossip messages harder.

I had a report of a 0.7.2 user whose node hadn't appeared on 1ml.  Their
node_announcement wasn't visible to my node, either.

I suspect this is a consequence of recent version reducing the amount of
gossip they send, as well as large nodes increasingly turning off gossip
altogether from some peers (as we do).  We should ignore timestamp filters
for our own channels: the easiest way to do this is to push them out
directly from gossipd (other messages are sent via the store).

We change channeld to wrap the local channel_announcements: previously
we just handed it to gossipd as for any other gossip message we received
from our peer.  Now gossipd knows to push it out, as it's local.

This interferes with the logic in tests/test_misc.py::test_htlc_send_timeout
which expects the node_announcement message last, so we generalize
that too.

[ Thanks to @trueptolmy for bugfix! ]
Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

CHANGELOG.md

@@ -33,6 +33,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - JSON-API: `pay` can exclude error nodes if the failcode of `sendpay` has the NODE bit set
 - JSON API: The `plugin` command now returns on error. A timeout of 20 seconds is added to `start` and `startdir` subcommands at the end of which the plugin is errored if it did not complete the handshake with `lightningd`.
 - JSON API: The `plugin` command does not allow to start static plugins after `lightningd` startup anymore.
+- Protocol: We now push our own gossip to all peers, independent of their filter.
 - Protocol: Now follows spec in responses to short channel id queries on unknown chainhashes; correspondingly, disconnects from peers that signal they do not maintain up-to-date information for the requested chain.
 - Tor: We default now with autotor to generate if possible temporary ED25519-V3 onions.  You can use new option `enable-autotor-v2-mode` to fallback to V2 RSA1024 mode.
 

channeld/channeld.c

@@ -441,7 +441,9 @@ static void announce_channel(struct peer *peer)
 
 	cannounce = create_channel_announcement(tmpctx, peer);
 
-	wire_sync_write(peer->pps->gossip_fd, cannounce);
+	wire_sync_write(peer->pps->gossip_fd,
+			take(towire_gossipd_local_channel_announcement(NULL,
+								       cannounce)));
 	send_channel_update(peer, 0);
 }
 

gossipd/gossip_generation.c

@@ -202,12 +202,13 @@ static void update_own_node_announcement(struct daemon *daemon)
 
 	/* This injects it into the routing code in routing.c; it should not
 	 * reject it! */
-	err = handle_node_announcement(daemon->rstate, take(nannounce),
+	err = handle_node_announcement(daemon->rstate, nannounce,
 				       NULL, NULL);
 	if (err)
 		status_failed(STATUS_FAIL_INTERNAL_ERROR,
 			      "rejected own node announcement: %s",
 			      tal_hex(tmpctx, err));
+	push_gossip(daemon, take(nannounce));
 }
 
 /* Should we announce our own node?  Called at strategic places. */
@@ -355,7 +356,7 @@ static void update_local_channel(struct local_cupdate *lc /* frees! */)
 	}
 
 	msg = wire_sync_read(tmpctx, HSM_FD);
-	if (!msg || !fromwire_hsm_cupdate_sig_reply(NULL, msg, &update)) {
+	if (!msg || !fromwire_hsm_cupdate_sig_reply(tmpctx, msg, &update)) {
 		status_failed(STATUS_FAIL_HSM_IO,
 			      "Reading cupdate_sig_req: %s",
 			      strerror(errno));
@@ -383,7 +384,7 @@ static void update_local_channel(struct local_cupdate *lc /* frees! */)
 	 * discard it (eg. non-public channel), but it should not complain
 	 * about it being invalid! __func__ is a magic C constant which
 	 * expands to this function name. */
-	msg = handle_channel_update(daemon->rstate, take(update),
+	msg = handle_channel_update(daemon->rstate, update,
 				    find_peer(daemon,
 					      &chan->nodes[!direction]->id),
 				    NULL);
@@ -397,6 +398,10 @@ static void update_local_channel(struct local_cupdate *lc /* frees! */)
 			       * tmpctx, so it's actually OK. */
 			      tal_hex(tmpctx, update),
 			      tal_hex(tmpctx, msg));
+
+	if (is_chan_public(chan))
+		push_gossip(daemon, take(update));
+
 	tal_free(lc);
 }
 

gossipd/gossip_peerd_wire.csv

@@ -31,3 +31,8 @@ msgdata,gossipd_local_channel_update,htlc_maximum_msat,amount_msat,
 msgtype,gossipd_new_store_fd,3505
 # How much shorter the new store is, so you can offset streaming.
 msgdata,gossipd_new_store_fd,offset_shorter,u64,
+
+# Send this channel_announcement
+msgtype,gossipd_local_channel_announcement,3506
+msgdata,gossipd_local_channel_announcement,len,u16,
+msgdata,gossipd_local_channel_announcement,cannount,u8,len

gossipd/gossipd.c

@@ -402,6 +402,49 @@ static u8 *handle_node_announce(struct peer *peer, const u8 *msg)
 	return err;
 }
 
+/*~ Large peers often turn off gossip msgs (using timestamp_filter) from most
+ * of their peers, however if the gossip is about us, we should spray it to
+ * everyone whether they've set the filter or not, otherwise it might not
+ * propagate! */
+void push_gossip(struct daemon *daemon, const u8 *msg TAKES)
+{
+	struct peer *peer;
+
+	if (taken(msg))
+		tal_steal(tmpctx, msg);
+
+	list_for_each(&daemon->peers, peer, list)
+		queue_peer_msg(peer, msg);
+}
+
+static bool handle_local_channel_announcement(struct daemon *daemon,
+					      struct peer *peer,
+					      const u8 *msg)
+{
+	u8 *cannouncement;
+	const u8 *err;
+
+	if (!fromwire_gossipd_local_channel_announcement(msg, msg,
+							 &cannouncement)) {
+		status_broken("peer %s bad local_channel_announcement %s",
+			      type_to_string(tmpctx, struct node_id, &peer->id),
+			      tal_hex(tmpctx, msg));
+		return false;
+	}
+
+	err = handle_channel_announcement_msg(peer, cannouncement);
+	if (err) {
+		status_broken("peer %s invalid local_channel_announcement %s (%s)",
+			      type_to_string(tmpctx, struct node_id, &peer->id),
+			      tal_hex(tmpctx, msg),
+			      tal_hex(tmpctx, err));
+		return false;
+	}
+
+	push_gossip(daemon, take(cannouncement));
+	return true;
+}
+
 /*~ This is where the per-peer daemons send us messages.  It's either forwarded
  * gossip, or a request for information.  We deliberately use non-overlapping
  * message types so we can distinguish them. */
@@ -479,6 +522,9 @@ static struct io_plan *peer_msg_in(struct io_conn *conn,
 	case WIRE_GOSSIPD_LOCAL_CHANNEL_UPDATE:
 		ok = handle_local_channel_update(peer->daemon, &peer->id, msg);
 		goto handled_cmd;
+	case WIRE_GOSSIPD_LOCAL_CHANNEL_ANNOUNCEMENT:
+		ok = handle_local_channel_announcement(peer->daemon, peer, msg);
+		goto handled_cmd;
 
 	/* These are the ones we send, not them */
 	case WIRE_GOSSIPD_GET_UPDATE_REPLY:

gossipd/gossipd.h

@@ -125,6 +125,9 @@ void peer_supplied_good_gossip(struct peer *peer, size_t amount);
 struct peer *random_peer(struct daemon *daemon,
 			 bool (*check_peer)(const struct peer *peer));
 
+/* Push this gossip out to all peers immediately. */
+void push_gossip(struct daemon *daemon, const u8 *msg TAKES);
+
 /* Queue a gossip message for the peer: the subdaemon on the other end simply
  * forwards it to the peer. */
 void queue_peer_msg(struct peer *peer, const u8 *msg TAKES);

gossipd/test/run-crc32_of_update.c

@@ -75,6 +75,9 @@ void *notleak_(const void *ptr UNNEEDED, bool plus_children UNNEEDED)
 /* Generated stub for peer_supplied_good_gossip */
 void peer_supplied_good_gossip(struct peer *peer UNNEEDED, size_t amount UNNEEDED)
 { fprintf(stderr, "peer_supplied_good_gossip called!\n"); abort(); }
+/* Generated stub for push_gossip */
+void push_gossip(struct daemon *daemon UNNEEDED, const u8 *msg TAKES UNNEEDED)
+{ fprintf(stderr, "push_gossip called!\n"); abort(); }
 /* Generated stub for queue_peer_from_store */
 void queue_peer_from_store(struct peer *peer UNNEEDED,
 			   const struct broadcastable *bcast UNNEEDED)

lightningd/test/run-invoice-select-inchan.c

@@ -46,6 +46,9 @@ bool channel_tell_depth(struct lightningd *ld UNNEEDED,
 				 const struct bitcoin_txid *txid UNNEEDED,
 				 u32 depth UNNEEDED)
 { fprintf(stderr, "channel_tell_depth called!\n"); abort(); }
+/* Generated stub for command_check_only */
+bool command_check_only(const struct command *cmd UNNEEDED)
+{ fprintf(stderr, "command_check_only called!\n"); abort(); }
 /* Generated stub for command_fail */
 struct command_result *command_fail(struct command *cmd UNNEEDED, int code UNNEEDED,
 				    const char *fmt UNNEEDED, ...)
@@ -314,6 +317,13 @@ struct command_result *param_array(struct command *cmd UNNEEDED, const char *nam
 				   const char *buffer UNNEEDED, const jsmntok_t *tok UNNEEDED,
 				   const jsmntok_t **arr UNNEEDED)
 { fprintf(stderr, "param_array called!\n"); abort(); }
+/* Generated stub for param_bitcoin_address */
+struct command_result *param_bitcoin_address(struct command *cmd UNNEEDED,
+					     const char *name UNNEEDED,
+					     const char *buffer UNNEEDED,
+					     const jsmntok_t *tok UNNEEDED,
+					     const u8 **scriptpubkey UNNEEDED)
+{ fprintf(stderr, "param_bitcoin_address called!\n"); abort(); }
 /* Generated stub for param_bool */
 struct command_result *param_bool(struct command *cmd UNNEEDED, const char *name UNNEEDED,
 				  const char *buffer UNNEEDED, const jsmntok_t *tok UNNEEDED,
@@ -616,16 +626,6 @@ struct txowatch *watch_txo(const tal_t *ctx UNNEEDED,
 						   size_t input_num UNNEEDED,
 						   const struct block *block))
 { fprintf(stderr, "watch_txo called!\n"); abort(); }
-/* Generated stub for param_bitcoin_address */
-struct command_result *param_bitcoin_address(struct command *cmd UNNEEDED,
-					     const char *name UNNEEDED,
-					     const char *buffer UNNEEDED,
-					     const jsmntok_t *tok UNNEEDED,
-					     const u8 **scriptpubkey UNNEEDED)
-{ fprintf(stderr, "param_bitcoin_address called!\n"); abort(); }
-/* Generated stub for command_check_only */
-bool command_check_only(const struct command *cmd UNNEEDED)
-{ fprintf(stderr, "command_check_only called!\n"); abort(); }
 /* AUTOGENERATED MOCKS END */
 
 #if DEVELOPER

lightningd/test/run-jsonrpc.c

@@ -30,6 +30,9 @@ const char *feerate_name(enum feerate feerate UNNEEDED)
 /* Generated stub for fmt_wireaddr_without_port */
 char *fmt_wireaddr_without_port(const tal_t *ctx UNNEEDED, const struct wireaddr *a UNNEEDED)
 { fprintf(stderr, "fmt_wireaddr_without_port called!\n"); abort(); }
+/* Generated stub for get_chainparams */
+const struct chainparams *get_chainparams(const struct lightningd *ld UNNEEDED)
+{ fprintf(stderr, "get_chainparams called!\n"); abort(); }
 /* Generated stub for json_to_pubkey */
 bool json_to_pubkey(const char *buffer UNNEEDED, const jsmntok_t *tok UNNEEDED,
 		    struct pubkey *pubkey UNNEEDED)
@@ -99,9 +102,6 @@ struct command_result *param_tok(struct command *cmd UNNEEDED, const char *name
 				 const char *buffer UNNEEDED, const jsmntok_t * tok UNNEEDED,
 				 const jsmntok_t **out UNNEEDED)
 { fprintf(stderr, "param_tok called!\n"); abort(); }
-/* Generated stub for get_chainparams */
-const struct chainparams *get_chainparams(const struct lightningd *ld UNNEEDED)
-{ fprintf(stderr, "get_chainparams called!\n"); abort(); }
 /* AUTOGENERATED MOCKS END */
 
 bool deprecated_apis;

tests/test_misc.py

@@ -1200,7 +1200,7 @@ def test_htlc_send_timeout(node_factory, bitcoind):
     timedout = False
     while not timedout:
         try:
-            l2.daemon.wait_for_log(r'channeld-{} chan #[0-9]*:\[IN\] 0101'.format(l3.info['id']), timeout=30)
+            l2.daemon.wait_for_log(r'channeld-{} chan #[0-9]*:\[IN\] '.format(l3.info['id']), timeout=30)
         except TimeoutError:
             timedout = True
 

wallet/test/run-wallet.c

@@ -62,6 +62,9 @@ bool channel_tell_depth(struct lightningd *ld UNNEEDED,
 				 const struct bitcoin_txid *txid UNNEEDED,
 				 u32 depth UNNEEDED)
 { fprintf(stderr, "channel_tell_depth called!\n"); abort(); }
+/* Generated stub for command_check_only */
+bool command_check_only(const struct command *cmd UNNEEDED)
+{ fprintf(stderr, "command_check_only called!\n"); abort(); }
 /* Generated stub for command_fail */
 struct command_result *command_fail(struct command *cmd UNNEEDED, int code UNNEEDED,
 				    const char *fmt UNNEEDED, ...)
@@ -325,6 +328,12 @@ char *json_strdup(const tal_t *ctx UNNEEDED, const char *buffer UNNEEDED, const
 /* Generated stub for json_stream_success */
 struct json_stream *json_stream_success(struct command *cmd UNNEEDED)
 { fprintf(stderr, "json_stream_success called!\n"); abort(); }
+/* Generated stub for json_to_address_scriptpubkey */
+enum address_parse_result json_to_address_scriptpubkey(const tal_t *ctx UNNEEDED,
+			     const struct chainparams *chainparams UNNEEDED,
+			     const char *buffer UNNEEDED,
+			     const jsmntok_t *tok UNNEEDED, const u8 **scriptpubkey UNNEEDED)
+{ fprintf(stderr, "json_to_address_scriptpubkey called!\n"); abort(); }
 /* Generated stub for json_to_bool */
 bool json_to_bool(const char *buffer UNNEEDED, const jsmntok_t *tok UNNEEDED, bool *b UNNEEDED)
 { fprintf(stderr, "json_to_bool called!\n"); abort(); }
@@ -412,6 +421,13 @@ void outpointfilter_remove(struct outpointfilter *of UNNEEDED,
 bool param(struct command *cmd UNNEEDED, const char *buffer UNNEEDED,
 	   const jsmntok_t params[] UNNEEDED, ...)
 { fprintf(stderr, "param called!\n"); abort(); }
+/* Generated stub for param_bitcoin_address */
+struct command_result *param_bitcoin_address(struct command *cmd UNNEEDED,
+					     const char *name UNNEEDED,
+					     const char *buffer UNNEEDED,
+					     const jsmntok_t *tok UNNEEDED,
+					     const u8 **scriptpubkey UNNEEDED)
+{ fprintf(stderr, "param_bitcoin_address called!\n"); abort(); }
 /* Generated stub for param_bool */
 struct command_result *param_bool(struct command *cmd UNNEEDED, const char *name UNNEEDED,
 				  const char *buffer UNNEEDED, const jsmntok_t *tok UNNEEDED,
@@ -615,22 +631,6 @@ struct txowatch *watch_txo(const tal_t *ctx UNNEEDED,
 						   size_t input_num UNNEEDED,
 						   const struct block *block))
 { fprintf(stderr, "watch_txo called!\n"); abort(); }
-/* Generated stub for param_bitcoin_address */
-struct command_result *param_bitcoin_address(struct command *cmd UNNEEDED,
-					     const char *name UNNEEDED,
-					     const char *buffer UNNEEDED,
-					     const jsmntok_t *tok UNNEEDED,
-					     const u8 **scriptpubkey UNNEEDED)
-{ fprintf(stderr, "param_bitcoin_address called!\n"); abort(); }
-/* Generated stub for json_tok_address_scriptpubkey */
-enum address_parse_result json_to_address_scriptpubkey(const tal_t *ctx UNNEEDED,
-			     const struct chainparams *chainparams UNNEEDED,
-			     const char *buffer UNNEEDED,
-			     const jsmntok_t *tok UNNEEDED, const u8 **scriptpubkey UNNEEDED)
-{ fprintf(stderr, "json_tok_address_scriptpubkey called!\n"); abort(); }
-/* Generated stub for command_check_only */
-bool command_check_only(const struct command *cmd UNNEEDED)
-{ fprintf(stderr, "command_check_only called!\n"); abort(); }
 /* AUTOGENERATED MOCKS END */
 
 #if DEVELOPER