gossipd: gossip_queries_ex is not longer experimental.

The master spec has some typos which make it not parse, so I created
a PR and generated the CSV from that:

https://github.com/lightningnetwork/lightning-rfc/pull/673

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

CHANGELOG.md

@@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Protocol: we now retransmit `funding_locked` upon reconnection while closing if there was no update
 - Plugin: new notifications `sendpay_success` and `sendpay_failure`.
+- Protocol: we now offer `option_gossip_queries_ex` for finegrained gossip control.
 
 ### Changed
 

common/decode_short_channel_ids.c

@@ -63,7 +63,6 @@ struct short_channel_id *decode_short_ids(const tal_t *ctx, const u8 *encoded)
 	return NULL;
 }
 
-#if EXPERIMENTAL_FEATURES
 bigsize_t *decode_scid_query_flags(const tal_t *ctx,
 				   const struct tlv_query_short_channel_ids_tlvs_query_flags *qf)
 {
@@ -104,4 +103,3 @@ bigsize_t *decode_scid_query_flags(const tal_t *ctx,
 	}
 	return NULL;
 }
-#endif /* EXPERIMENTAL_FEATURES */

common/decode_short_channel_ids.h

@@ -41,9 +41,7 @@ enum scid_query_flag {
 	SCID_QF_NODE2 = 0x10,
 };
 
-#if EXPERIMENTAL_FEATURES
 bigsize_t *decode_scid_query_flags(const tal_t *ctx,
 				   const struct tlv_query_short_channel_ids_tlvs_query_flags *qf);
-#endif /* EXPERIMENTAL_FEATURES */
 
 #endif /* LIGHTNING_COMMON_DECODE_SHORT_CHANNEL_IDS_H */

common/features.c

@@ -9,8 +9,8 @@ static const u32 our_localfeatures[] = {
 	OPTIONAL_FEATURE(LOCAL_INITIAL_ROUTING_SYNC),
 	OPTIONAL_FEATURE(LOCAL_UPFRONT_SHUTDOWN_SCRIPT),
 	OPTIONAL_FEATURE(LOCAL_GOSSIP_QUERIES),
-#if EXPERIMENTAL_FEATURES
 	OPTIONAL_FEATURE(LOCAL_GOSSIP_QUERIES_EX),
+#if EXPERIMENTAL_FEATURES
 	OPTIONAL_FEATURE(LOCAL_STATIC_REMOTEKEY),
 #endif
 };

gossipd/gossipd.c

@@ -77,34 +77,6 @@ static u32 max_encoding_bytes = -1U;
 static bool suppress_gossip = false;
 #endif
 
-#if EXPERIMENTAL_FEATURES == 0
-/* We want these definitions for convenience, even if we never encode/decode
- * them when not EXPERIMENTAL_FEATURES */
-struct tlv_reply_channel_range_tlvs_timestamps_tlv {
-        u8 encoding_type;
-        u8 *encoded_timestamps;
-};
-
-struct tlv_reply_channel_range_tlvs_checksums_tlv {
-        struct channel_update_checksums *checksums;
-};
-
-struct channel_update_timestamps {
-        u32 timestamp_node_id_1;
-        u32 timestamp_node_id_2;
-};
-struct channel_update_checksums {
-        u32 checksum_node_id_1;
-        u32 checksum_node_id_2;
-};
-
-static void towire_channel_update_timestamps(u8 **p,
-					     const struct channel_update_timestamps *channel_update_timestamps)
-{
-	abort();
-}
-#endif
-
 /*~ How gossipy do we ask a peer to be? */
 enum gossip_level {
 	/* Give us everything since epoch */
@@ -518,13 +490,8 @@ static bool query_short_channel_ids(struct daemon *daemon,
 		return false;
 	}
 
-#if EXPERIMENTAL_FEATURES
 	msg = towire_query_short_channel_ids(NULL, &daemon->chain_hash,
 					     encoded, NULL);
-#else
-	msg = towire_query_short_channel_ids(NULL, &daemon->chain_hash,
-					     encoded);
-#endif
 	queue_peer_msg(peer, take(msg));
 	peer->scid_query_outstanding = true;
 	peer->scid_query_was_internal = internal;
@@ -623,7 +590,6 @@ static const u8 *handle_query_short_channel_ids(struct peer *peer, const u8 *msg
 	struct short_channel_id *scids;
 	bigsize_t *flags;
 
-#if EXPERIMENTAL_FEATURES
 	struct tlv_query_short_channel_ids_tlvs *tlvs
 		= tlv_query_short_channel_ids_tlvs_new(tmpctx);
 
@@ -651,14 +617,6 @@ static const u8 *handle_query_short_channel_ids(struct peer *peer, const u8 *msg
 		}
 	} else
 		flags = NULL;
-#else
-	if (!fromwire_query_short_channel_ids(tmpctx, msg, &chain, &encoded)) {
-		return towire_errorfmt(peer, NULL,
-				       "Bad query_short_channel_ids %s",
-				       tal_hex(tmpctx, msg));
-	}
-	flags = NULL;
-#endif
 
 	if (!bitcoin_blkid_eq(&peer->daemon->chain_hash, &chain)) {
 		status_debug("%s sent query_short_channel_ids chainhash %s",
@@ -772,7 +730,6 @@ static void reply_channel_range(struct peer *peer,
 	 *   - otherwise:
 	 *     - SHOULD set `complete` to 1.
 	 */
-#if EXPERIMENTAL_FEATURES
  	struct tlv_reply_channel_range_tlvs *tlvs
  		= tlv_reply_channel_range_tlvs_new(tmpctx);
 	tlvs->timestamps_tlv = timestamps;
@@ -783,13 +740,6 @@ static void reply_channel_range(struct peer *peer,
 					     first_blocknum,
 					     number_of_blocks,
 					     1, encoded_scids, tlvs);
-#else
-	u8 *msg = towire_reply_channel_range(NULL,
-					     &peer->daemon->chain_hash,
-					     first_blocknum,
-					     number_of_blocks,
-					     1, encoded_scids);
-#endif
 	queue_peer_msg(peer, take(msg));
 }
 
@@ -1001,8 +951,6 @@ static u8 *handle_query_channel_range(struct peer *peer, const u8 *msg)
 	u32 first_blocknum, number_of_blocks, tail_blocks;
 	struct short_channel_id last_scid;
 	enum query_option_flags query_option_flags;
-
-#if EXPERIMENTAL_FEATURES
 	struct tlv_query_channel_range_tlvs *tlvs
 		= tlv_query_channel_range_tlvs_new(msg);
 
@@ -1017,15 +965,6 @@ static u8 *handle_query_channel_range(struct peer *peer, const u8 *msg)
 		query_option_flags = tlvs->query_option->query_option_flags;
 	else
 		query_option_flags = 0;
-#else
-	if (!fromwire_query_channel_range(msg, &chain_hash,
-					  &first_blocknum, &number_of_blocks)) {
-		return towire_errorfmt(peer, NULL,
-				       "Bad query_channel_range %s",
-				       tal_hex(tmpctx, msg));
-	}
-	query_option_flags = 0;
-#endif
 
 	/* If they ask for the wrong chain, we give an empty response
 	 * with the `complete` flag unset */
@@ -1034,13 +973,8 @@ static u8 *handle_query_channel_range(struct peer *peer, const u8 *msg)
 			     type_to_string(tmpctx, struct node_id, &peer->id),
 			     type_to_string(tmpctx, struct bitcoin_blkid,
 					    &chain_hash));
-#if EXPERIMENTAL_FEATURES
 		u8 *end = towire_reply_channel_range(NULL, &chain_hash, first_blocknum,
 		                                     number_of_blocks, false, NULL, NULL);
-#else
-		u8 *end = towire_reply_channel_range(NULL, &chain_hash, first_blocknum,
-		                                     number_of_blocks, false, NULL);
-#endif
 		queue_peer_msg(peer, take(end));
 		return NULL;
 	}
@@ -1082,10 +1016,9 @@ static const u8 *handle_reply_channel_range(struct peer *peer, const u8 *msg)
 	struct short_channel_id *scids;
 	size_t n;
 	unsigned long b;
-
-#if EXPERIMENTAL_FEATURES
 	struct tlv_reply_channel_range_tlvs *tlvs
 		= tlv_reply_channel_range_tlvs_new(tmpctx);
+
 	if (!fromwire_reply_channel_range(tmpctx, msg, &chain, &first_blocknum,
 					  &number_of_blocks, &complete,
 					  &encoded, tlvs)) {
@@ -1093,15 +1026,6 @@ static const u8 *handle_reply_channel_range(struct peer *peer, const u8 *msg)
 				       "Bad reply_channel_range w/tlvs %s",
 				       tal_hex(tmpctx, msg));
 	}
-#else
-	if (!fromwire_reply_channel_range(tmpctx, msg, &chain, &first_blocknum,
-					  &number_of_blocks, &complete,
-					  &encoded)) {
-		return towire_errorfmt(peer, NULL,
-				       "Bad reply_channel_range %s",
-				       tal_hex(tmpctx, msg));
-	}
-#endif
 
 	if (!bitcoin_blkid_eq(&peer->daemon->chain_hash, &chain)) {
 		return towire_errorfmt(peer, NULL,
@@ -2531,14 +2455,9 @@ static struct io_plan *query_channel_range(struct io_conn *conn,
 	status_debug("sending query_channel_range for blocks %u+%u",
 		     first_blocknum, number_of_blocks);
 
-#if EXPERIMENTAL_FEATURES
 	msg = towire_query_channel_range(NULL, &daemon->chain_hash,
 					 first_blocknum, number_of_blocks,
 					 NULL);
-#else
-	msg = towire_query_channel_range(NULL, &daemon->chain_hash,
-					 first_blocknum, number_of_blocks);
-#endif
 	queue_peer_msg(peer, take(msg));
 	peer->range_first_blocknum = first_blocknum;
 	peer->range_end_blocknum = first_blocknum + number_of_blocks;

gossipd/test/run-crc32_of_update.c

@@ -33,6 +33,10 @@ void daemon_conn_wake(struct daemon_conn *dc UNNEEDED)
 /* Generated stub for daemon_shutdown */
 void daemon_shutdown(void)
 { fprintf(stderr, "daemon_shutdown called!\n"); abort(); }
+/* Generated stub for decode_scid_query_flags */
+bigsize_t *decode_scid_query_flags(const tal_t *ctx UNNEEDED,
+				   const struct tlv_query_short_channel_ids_tlvs_query_flags *qf UNNEEDED)
+{ fprintf(stderr, "decode_scid_query_flags called!\n"); abort(); }
 /* Generated stub for decode_short_ids */
 struct short_channel_id *decode_short_ids(const tal_t *ctx UNNEEDED, const u8 *encoded UNNEEDED)
 { fprintf(stderr, "decode_short_ids called!\n"); abort(); }
@@ -326,13 +330,6 @@ bool wire_sync_write(int fd UNNEEDED, const void *msg TAKES UNNEEDED)
 { fprintf(stderr, "wire_sync_write called!\n"); abort(); }
 /* AUTOGENERATED MOCKS END */
 
-#if EXPERIMENTAL_FEATURES
-/* Generated stub for decode_scid_query_flags */
-bigsize_t *decode_scid_query_flags(const tal_t *ctx UNNEEDED,
-				   const struct tlv_query_short_channel_ids_tlvs_query_flags *qf UNNEEDED)
-{ fprintf(stderr, "decode_scid_query_flags called!\n"); abort(); }
-#endif /* EXPERIMENTAL_FEATURES */
-
 int main(void)
 {
 	u8 *update;

gossipd/test/run-extended-info.c

@@ -1,12 +1,5 @@
 #include "config.h"
 
-#if EXPERIMENTAL_FEATURES == 0
-/* Can't run without EXPERIMENTAL_FEATURES */
-int main(void)
-{
-	return 0;
-}
-#else
 #define main gossipd_main
 int gossipd_main(int argc, char *argv[]);
 
@@ -783,4 +776,3 @@ int main(void)
 	tal_free(toks);
 	return 0;
 }
-#endif /* EXPERIMENTAL_FEATURES */

tests/test_connection.py

@@ -1382,7 +1382,7 @@ def test_forget_channel(node_factory):
 
 def test_peerinfo(node_factory, bitcoind):
     l1, l2 = node_factory.line_graph(2, fundchannel=False, opts={'may_reconnect': True})
-    lfeatures = 'aa'
+    lfeatures = '08aa'
     if EXPERIMENTAL_FEATURES:
         lfeatures = '28aa'
     # Gossiping but no node announcement yet
@@ -1638,8 +1638,8 @@ def test_dataloss_protection(node_factory, bitcoind):
         # features 1, 3, 5, 7, 11 and 13 (0x28aa).
         lf = "28aa"
     else:
-        # features 1, 3, 5 and 7 (0xaa).
-        lf = "aa"
+        # features 1, 3, 5, 7 and 11 (0x08aa).
+        lf = "08aa"
     l1.rpc.connect(l2.info['id'], 'localhost', l2.port)
     # l1 should send out WIRE_INIT (0010)
     l1.daemon.wait_for_log(r"\[OUT\] 0010"

tests/test_misc.py

@@ -1446,8 +1446,8 @@ def test_list_features_only(node_factory):
     expected = ['option_data_loss_protect/odd',
                 'option_initial_routing_sync/odd',
                 'option_upfront_shutdown_script/odd',
-                'option_gossip_queries/odd']
+                'option_gossip_queries/odd',
+                'option_gossip_queries_ex/odd']
     if EXPERIMENTAL_FEATURES:
-        expected += ['option_gossip_queries_ex/odd',
-                     'option_static_remotekey/odd']
+        expected += ['option_static_remotekey/odd']
     assert features == expected

wire/extracted_peer_experimental_legacy

@@ -10,46 +10,3 @@
 +msgdata,channel_reestablish,my_current_per_commitment_point,point,,option_data_loss_protect,option_static_remotekey
  msgtype,announcement_signatures,259
  msgdata,announcement_signatures,channel_id,channel_id,
-@@ -154,6 +168,11 @@
- msgdata,query_short_channel_ids,chain_hash,chain_hash,
- msgdata,query_short_channel_ids,len,u16,
- msgdata,query_short_channel_ids,encoded_short_ids,byte,len
-+# pr-557-tlvfixes, rebased with minor fixes
-+msgdata,query_short_channel_ids,tlvs,query_short_channel_ids_tlvs,
-+tlvtype,query_short_channel_ids_tlvs,query_flags,1
-+tlvdata,query_short_channel_ids_tlvs,query_flags,encoding_type,byte,
-+tlvdata,query_short_channel_ids_tlvs,query_flags,encoded_query_flags,byte,...
- msgtype,reply_short_channel_ids_end,262,gossip_queries
- msgdata,reply_short_channel_ids_end,chain_hash,chain_hash,
- msgdata,reply_short_channel_ids_end,complete,byte,
-@@ -161,6 +179,10 @@
- msgdata,query_channel_range,chain_hash,chain_hash,
- msgdata,query_channel_range,first_blocknum,u32,
- msgdata,query_channel_range,number_of_blocks,u32,
-+# pr-557-tlvfixes, rebased with minor fixes
-+msgdata,query_channel_range,tlvs,query_channel_range_tlvs,
-+tlvtype,query_channel_range_tlvs,query_option,1
-+tlvdata,query_channel_range_tlvs,query_option,query_option_flags,varint,
- msgtype,reply_channel_range,264,gossip_queries
- msgdata,reply_channel_range,chain_hash,chain_hash,
- msgdata,reply_channel_range,first_blocknum,u32,
-@@ -168,6 +189,19 @@
- msgdata,reply_channel_range,complete,byte,
- msgdata,reply_channel_range,len,u16,
- msgdata,reply_channel_range,encoded_short_ids,byte,len
-+# pr-557-tlvfixes, rebased with minor fixes
-+msgdata,reply_channel_range,tlvs,reply_channel_range_tlvs,
-+tlvtype,reply_channel_range_tlvs,timestamps_tlv,1
-+tlvdata,reply_channel_range_tlvs,timestamps_tlv,encoding_type,byte,
-+tlvdata,reply_channel_range_tlvs,timestamps_tlv,encoded_timestamps,byte,...
-+tlvtype,reply_channel_range_tlvs,checksums_tlv,3
-+tlvdata,reply_channel_range_tlvs,checksums_tlv,checksums,channel_update_checksums,...
-+subtype,channel_update_timestamps
-+subtypedata,channel_update_timestamps,timestamp_node_id_1,u32,
-+subtypedata,channel_update_timestamps,timestamp_node_id_2,u32,
-+subtype,channel_update_checksums
-+subtypedata,channel_update_checksums,checksum_node_id_1,u32,
-+subtypedata,channel_update_checksums,checksum_node_id_2,u32,
- msgtype,gossip_timestamp_filter,265,gossip_queries
- msgdata,gossip_timestamp_filter,chain_hash,chain_hash,
- msgdata,gossip_timestamp_filter,first_timestamp,u32,

wire/extracted_peer_wire_csv

@@ -168,6 +168,10 @@ msgtype,query_short_channel_ids,261,gossip_queries
 msgdata,query_short_channel_ids,chain_hash,chain_hash,
 msgdata,query_short_channel_ids,len,u16,
 msgdata,query_short_channel_ids,encoded_short_ids,byte,len
+msgdata,query_short_channel_ids,tlvs,query_short_channel_ids_tlvs,
+tlvtype,query_short_channel_ids_tlvs,query_flags,1
+tlvdata,query_short_channel_ids_tlvs,query_flags,encoding_type,u8,
+tlvdata,query_short_channel_ids_tlvs,query_flags,encoded_query_flags,byte,...
 msgtype,reply_short_channel_ids_end,262,gossip_queries
 msgdata,reply_short_channel_ids_end,chain_hash,chain_hash,
 msgdata,reply_short_channel_ids_end,complete,byte,
@@ -175,6 +179,9 @@ msgtype,query_channel_range,263,gossip_queries
 msgdata,query_channel_range,chain_hash,chain_hash,
 msgdata,query_channel_range,first_blocknum,u32,
 msgdata,query_channel_range,number_of_blocks,u32,
+msgdata,query_channel_range,tlvs,query_channel_range_tlvs,
+tlvtype,query_channel_range_tlvs,query_option,1
+tlvdata,query_channel_range_tlvs,query_option,query_option_flags,varint,
 msgtype,reply_channel_range,264,gossip_queries
 msgdata,reply_channel_range,chain_hash,chain_hash,
 msgdata,reply_channel_range,first_blocknum,u32,
@@ -182,6 +189,18 @@ msgdata,reply_channel_range,number_of_blocks,u32,
 msgdata,reply_channel_range,complete,byte,
 msgdata,reply_channel_range,len,u16,
 msgdata,reply_channel_range,encoded_short_ids,byte,len
+msgdata,reply_channel_range,tlvs,reply_channel_range_tlvs,
+tlvtype,reply_channel_range_tlvs,timestamps_tlv,1
+tlvdata,reply_channel_range_tlvs,timestamps_tlv,encoding_type,u8,
+tlvdata,reply_channel_range_tlvs,timestamps_tlv,encoded_timestamps,byte,...
+tlvtype,reply_channel_range_tlvs,checksums_tlv,3
+tlvdata,reply_channel_range_tlvs,checksums_tlv,checksums,channel_update_checksums,...
+subtype,channel_update_timestamps
+subtypedata,channel_update_timestamps,timestamp_node_id_1,u32,
+subtypedata,channel_update_timestamps,timestamp_node_id_2,u32,
+subtype,channel_update_checksums
+subtypedata,channel_update_checksums,checksum_node_id_1,u32,
+subtypedata,channel_update_checksums,checksum_node_id_2,u32,
 msgtype,gossip_timestamp_filter,265,gossip_queries
 msgdata,gossip_timestamp_filter,chain_hash,chain_hash,
 msgdata,gossip_timestamp_filter,first_timestamp,u32,