bolt: update to BOLT spec changes (extract format + type specifications)

updates the bolt version to 6639cef095a2ecc7b8f0c48c6e7f2f906fbfbc58. this requires us to use the new bolt parser at generate-bolt.py and updates to all of the type specifications (ie. from u8 -> byte)
6 years ago · 5c07afac7d
16 changed files with 315 additions and 288 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

 ### Changed

+- build: now requires `python3-mako` to be installed, i.e. `sudo apt-get install python3-mako`
+
 ### Deprecated

 Note: You should always set `allow-deprecated-apis=false` to test for
--- a/11
+++ b/11
@ -15,7 +15,7 @@ CCANDIR := ccan

 # Where we keep the BOLT RFCs
 BOLTDIR := ../lightning-rfc/
-BOLTVERSION := 309e86d471faf90c1f6c531701c16887684badb9
+BOLTVERSION := 6639cef095a2ecc7b8f0c48c6e7f2f906fbfbc58

 -include config.vars

@ -185,6 +185,8 @@ ALL_GEN_HEADERS += gen_version.h
 CDUMP_OBJS := ccan-cdump.o ccan-strmap.o

 WIRE_GEN := tools/generate-wire.py
+BOLT_GEN := tools/generate-bolts.py
+BOLT_DEPS := $(BOLT_GEN)

 ALL_PROGRAMS =

@ -272,6 +274,13 @@ check-hdr-include-order/%: %
 check-makefile:
 	@if [ x"$(CCANDIR)/config.h `find $(CCANDIR)/ccan -name '*.h' | grep -v /test/ | LC_ALL=C sort | tr '\n' ' '`" != x"$(CCAN_HEADERS) " ]; then echo CCAN_HEADERS incorrect; exit 1; fi

+# Check if they've installed 'mako' dependency, useful for
+# keeping your build from clobbering itself
+check-bolt-dependency:
+	@python3 -c "import mako"
+
+BOLT_DEPS += check-bolt-dependency
+
 # Any mention of BOLT# must be followed by an exact quote, modulo whitespace.
 bolt-check/%: % bolt-precheck tools/check-bolt
 	@if [ -d .tmp.lightningrfc ]; then tools/check-bolt .tmp.lightningrfc $<; else echo "Not checking BOLTs: BOLTDIR $(BOLTDIR) does not exist" >&2; fi
--- a/README.md
+++ b/README.md
@ -59,7 +59,8 @@ For the impatient here's the gist of it for Ubuntu and Debian:
    sudo apt-get update
    sudo apt-get install -y \
      autoconf automake build-essential git libtool libgmp-dev \
-      libsqlite3-dev python python3 net-tools zlib1g-dev libsodium-dev
+      libsqlite3-dev python python3 net-tools zlib1g-dev libsodium-dev \
+      python3-mako
    git clone https://github.com/ElementsProject/lightning.git
    cd lightning
    ./configure
--- a/common/channel_config.h
+++ b/common/channel_config.h
@ -9,28 +9,28 @@
 *
 * 1. type: 32 (`open_channel`)
 * 2. data:
- *    * [`32`:`chain_hash`]
- *    * [`32`:`temporary_channel_id`]
- *    * [`8`:`funding_satoshis`]
- *    * [`8`:`push_msat`]
- *    * [`8`:`dust_limit_satoshis`]
- *    * [`8`:`max_htlc_value_in_flight_msat`]
- *    * [`8`:`channel_reserve_satoshis`]
- *    * [`8`:`htlc_minimum_msat`]
- *    * [`4`:`feerate_per_kw`]
- *    * [`2`:`to_self_delay`]
- *    * [`2`:`max_accepted_htlcs`]
+ *    * [`chain_hash`:`chain_hash`]
+ *    * [`32*byte`:`temporary_channel_id`]
+ *    * [`u64`:`funding_satoshis`]
+ *    * [`u64`:`push_msat`]
+ *    * [`u64`:`dust_limit_satoshis`]
+ *    * [`u64`:`max_htlc_value_in_flight_msat`]
+ *    * [`u64`:`channel_reserve_satoshis`]
+ *    * [`u64`:`htlc_minimum_msat`]
+ *    * [`u32`:`feerate_per_kw`]
+ *    * [`u16`:`to_self_delay`]
+ *    * [`u16`:`max_accepted_htlcs`]
 *...
 * 1. type: 33 (`accept_channel`)
 * 2. data:
- *    * [`32`:`temporary_channel_id`]
- *    * [`8`:`dust_limit_satoshis`]
- *    * [`8`:`max_htlc_value_in_flight_msat`]
- *    * [`8`:`channel_reserve_satoshis`]
- *    * [`8`:`htlc_minimum_msat`]
- *    * [`4`:`minimum_depth`]
- *    * [`2`:`to_self_delay`]
- *    * [`2`:`max_accepted_htlcs`]
+ *    * [`32*byte`:`temporary_channel_id`]
+ *    * [`u64`:`dust_limit_satoshis`]
+ *    * [`u64`:`max_htlc_value_in_flight_msat`]
+ *    * [`u64`:`channel_reserve_satoshis`]
+ *    * [`u64`:`htlc_minimum_msat`]
+ *    * [`u32`:`minimum_depth`]
+ *    * [`u16`:`to_self_delay`]
+ *    * [`u16`:`max_accepted_htlcs`]
 */
 struct channel_config {
 	/* Database ID */
--- a/common/sphinx.c
+++ b/common/sphinx.c
@ -464,11 +464,11 @@ u8 *create_onionreply(const tal_t *ctx, const struct secret *shared_secret,
 	 * the following fields:
 	 *
 	 * 1. data:
-	 *    * [`32`:`hmac`]
-	 *    * [`2`:`failure_len`]
-	 *    * [`failure_len`:`failuremsg`]
-	 *    * [`2`:`pad_len`]
-	 *    * [`pad_len`:`pad`]
+	 *    * [`32*byte`:`hmac`]
+	 *    * [`u16`:`failure_len`]
+	 *    * [`failure_len*byte`:`failuremsg`]
+	 *    * [`u16`:`pad_len`]
+	 *    * [`pad_len*byte`:`pad`]
 	 */
 	towire_u16(&payload, msglen);
 	towire(&payload, failure_msg, msglen);
--- a/common/sphinx.h
+++ b/common/sphinx.h
@ -42,9 +42,9 @@ enum route_next_case {
 *
 * 1. type: `hops_data`
 * 2. data:
- *    * [`1`:`realm`]
- *    * [`32`:`per_hop`]
- *    * [`32`:`HMAC`]
+ *    * [`byte`:`realm`]
+ *    * [`32*byte`:`per_hop`]
+ *    * [`32*byte`:`HMAC`]
 *    * ...
 *    * `filler`
 *
@ -59,10 +59,10 @@ enum route_next_case {
 *
 * 1. type: `per_hop` (for `realm` 0)
 * 2. data:
- *    * [`8`:`short_channel_id`]
- *    * [`8`:`amt_to_forward`]
- *    * [`4`:`outgoing_cltv_value`]
- *    * [`12`:`padding`]
+ *    * [`short_channel_id`:`short_channel_id`]
+ *    * [`u64`:`amt_to_forward`]
+ *    * [`u32`:`outgoing_cltv_value`]
+ *    * [`12*byte`:`padding`]
 */
 struct hop_data {
 	u8 realm;
--- a/devtools/Makefile
+++ b/devtools/Makefile
@ -4,6 +4,8 @@ DEVTOOLS := devtools/bolt11-cli devtools/decodemsg devtools/onion devtools/dump-
 DEVTOOLS_TOOL_SRC := $(DEVTOOLS:=.c)
 DEVTOOLS_TOOL_OBJS := $(DEVTOOLS_TOOL_SRC:.c=.o)

+DEVTOOLS_BOLT_DEPS += $(BOLT_DEPS) tools/gen/print_impl_template tools/gen/print_header_template
+
 DEVTOOLS_COMMON_OBJS :=				\
 	common/amount.o				\
 	common/base32.o				\
@ -24,17 +26,17 @@ DEVTOOLS_COMMON_OBJS :=				\

 devtools-all: $(DEVTOOLS)

-devtools/gen_print_wire.h: $(WIRE_GEN) wire/gen_peer_wire_csv
-	$(WIRE_GEN) --bolt --printwire --header $@ wire_type < wire/gen_peer_wire_csv > $@
+devtools/gen_print_wire.h: $(DEVTOOL_BOLT_DEPS) wire/gen_peer_wire_csv
+	$(BOLT_GEN) -P --page header $@ wire_type < wire/gen_peer_wire_csv > $@

-devtools/gen_print_wire.c: $(WIRE_GEN) wire/gen_peer_wire_csv
-	$(WIRE_GEN) --bolt --printwire ${@:.c=.h} wire_type < wire/gen_peer_wire_csv > $@
+devtools/gen_print_wire.c: $(DEVTOOL_BOLT_DEPS) wire/gen_peer_wire_csv
+	$(BOLT_GEN) -P --page impl ${@:.c=.h} wire_type < wire/gen_peer_wire_csv > $@

-devtools/gen_print_onion_wire.h: $(WIRE_GEN) wire/gen_onion_wire_csv
-	$(WIRE_GEN) --bolt --printwire --header $@ onion_type < wire/gen_onion_wire_csv > $@
+devtools/gen_print_onion_wire.h: $(DEVTOOL_BOLT_DEPS) wire/gen_onion_wire_csv
+	$(BOLT_GEN) -P --page header $@ onion_type < wire/gen_onion_wire_csv > $@

-devtools/gen_print_onion_wire.c: $(WIRE_GEN) wire/gen_onion_wire_csv
-	$(WIRE_GEN) --bolt --printwire ${@:.c=.h} onion_type < wire/gen_onion_wire_csv > $@
+devtools/gen_print_onion_wire.c: $(DEVTOOL_BOLT_DEPS) wire/gen_onion_wire_csv
+	$(BOLT_GEN) -P --page impl ${@:.c=.h} onion_type < wire/gen_onion_wire_csv > $@

 devtools/bolt11-cli: $(DEVTOOLS_OBJS) $(DEVTOOLS_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o devtools/bolt11-cli.o

--- a/doc/HACKING.md
+++ b/doc/HACKING.md
@ -44,8 +44,10 @@ Here's a list of parts, with notes:
 * tools/ - tools for building
  - check-bolt.c: check the source code contains correct BOLT quotes
    (as used by check-source)
-  - generate-wire.py: generate marshal/unmarshal routines from
-    extracts from BOLT specs, and as specified by subdaemons.
+  - generate-wire.py: generates wire marshal/unmarshal-ing
+    routines for subdaemons.
+  - generate-bolts.py: generate marshal/unmarshal routines
+    from extracted BOLT specs.
  - mockup.sh / update-mocks.sh: tools to generate mock functions for
    unit tests.

@ -59,7 +61,7 @@ Here's a list of parts, with notes:

 * contrib/ - python support and other stuff which doesn't belong :)

-* wire/ - basic marshalling/un
+* wire/ - basic marshalling/un for messages defined in the BOLTs

 * common/ - routines needed by any two or more of the directories below

@ -216,6 +218,15 @@ There are three kinds of tests:
 Our Travis CI instance (see `.travis.yml`) runs all these for each
 pull request.

+
+Making BOLT Modifications
+-------------------------
+
+All of code for marshalling/unmarshalling BOLT protocol messages is generated
+directly from the spec. These are pegged to the BOLTVERSION, as specified in
+`Makefile`.
+
+
 Source code analysis
 --------------------
 An updated version of the NCC source code analysis tool is available at
--- a/gossipd/gossip_constants.h
+++ b/gossipd/gossip_constants.h
@ -7,9 +7,9 @@
 *...
 * 1. type: `onion_packet`
 * 2. data:
- *    * [`1`:`version`]
- *    * [`33`:`public_key`]
- *    * [`20*65`:`hops_data`]
+ *    * [`byte`:`version`]
+ *    * [`point`:`public_key`]
+ *    * [`1300*byte`:`hops_data`]
 */
 #define ROUTING_MAX_HOPS 20

--- a/gossipd/gossipd.c
+++ b/gossipd/gossipd.c
@ -587,9 +587,9 @@ static bool query_short_channel_ids(struct daemon *daemon,
 	 *
 	 * 1. type: 261 (`query_short_channel_ids`) (`gossip_queries`)
 	 * 2. data:
-	 *     * [`32`:`chain_hash`]
-	 *     * [`2`:`len`]
-	 *     * [`len`:`encoded_short_ids`]
+	 *     * [`chain_hash`:`chain_hash`]
+	 *     * [`u16`:`len`]
+	 *     * [`len*byte`:`encoded_short_ids`]
 	 */
 	const size_t reply_overhead = 32 + 2;
 	const size_t max_encoded_bytes = 65535 - 2 - reply_overhead;
@ -841,12 +841,12 @@ static bool queue_channel_ranges(struct peer *peer,
 	 *
 	 * 1. type: 264 (`reply_channel_range`) (`gossip_queries`)
 	 * 2. data:
-	 *   * [`32`:`chain_hash`]
-	 *   * [`4`:`first_blocknum`]
-	 *   * [`4`:`number_of_blocks`]
-	 *   * [`1`:`complete`]
-	 *   * [`2`:`len`]
-	 *   * [`len`:`encoded_short_ids`]
+	 *   * [`chain_hash`:`chain_hash`]
+	 *   * [`u32`:`first_blocknum`]
+	 *   * [`u32`:`number_of_blocks`]
+	 *   * [`byte`:`complete`]
+	 *   * [`u16`:`len`]
+	 *   * [`len*byte`:`encoded_short_ids`]
 	 */
 	const size_t reply_overhead = 32 + 4 + 4 + 1 + 2;
 	const size_t max_encoded_bytes = 65535 - 2 - reply_overhead;
--- a/lightningd/options.c
+++ b/lightningd/options.c
@ -252,7 +252,7 @@ static char *opt_set_alias(const char *arg, struct lightningd *ld)
 	ld->alias = tal_free(ld->alias);
 	/* BOLT #7:
 	 *
-	 *    * [`32`:`alias`]
+	 *    * [`32*byte`:`alias`]
 	 *...
 	 *  - MUST set `alias` to a valid UTF-8 string, with any
 	 *   `alias` trailing-bytes equal to 0.
--- a/lightningd/peer_htlcs.c
+++ b/lightningd/peer_htlcs.c
@ -282,7 +282,7 @@ static void handle_localpay(struct htlc_in *hin,
 	 *
 	 * 1. type: 19 (`final_incorrect_htlc_amount`)
 	 * 2. data:
-	 *    * [`8`:`incoming_htlc_amt`]
+	 *    * [`u64`:`incoming_htlc_amt`]
 	 *
 	 * The amount in the HTLC doesn't match the value in the onion.
 	 */
@ -295,7 +295,7 @@ static void handle_localpay(struct htlc_in *hin,
 	 *
 	 * 1. type: 18 (`final_incorrect_cltv_expiry`)
 	 * 2. data:
-	 *    * [`4`:`cltv_expiry`]
+	 *    * [`u32`:`cltv_expiry`]
 	 *
 	 * The CLTV expiry in the HTLC doesn't match the value in the onion.
 	 */
--- a/lightningd/ping.c
+++ b/lightningd/ping.c
@ -104,9 +104,9 @@ static struct command_result *json_ping(struct command *cmd,
 	 *...
 	 * 1. type: 18 (`ping`)
 	 * 2. data:
-	 *    * [`2`:`num_pong_bytes`]
-	 *    * [`2`:`byteslen`]
-	 *    * [`byteslen`:`ignored`]
+	 *    * [`u16`:`num_pong_bytes`]
+	 *    * [`u16`:`byteslen`]
+	 *    * [`byteslen*byte`:`ignored`]
 	 */
 	if (*len > 65535 - 2 - 2 - 2) {
 		return command_fail(cmd, JSONRPC2_INVALID_PARAMS,
--- a/wire/Makefile
+++ b/wire/Makefile
@ -28,16 +28,18 @@ ALL_GEN_HEADERS += $(WIRE_GEN_HEADERS)
 # They may not have the bolts.
 BOLT_EXTRACT=$(LOCAL_BOLTDIR)/tools/extract-formats.py

+WIRE_BOLT_DEPS := $(BOLT_DEPS) tools/gen/impl_template tools/gen/header_template
+
 # Explicit command to re-extract CSV from BOLTs.
 extract-bolt-csv: bolt-precheck
-	@$(BOLT_EXTRACT) --message-fields --message-types $(LOCAL_BOLTDIR)/0[127]*.md > wire/extracted_peer_wire_csv
+	@$(BOLT_EXTRACT) $(LOCAL_BOLTDIR)/0[127]*.md > wire/extracted_peer_wire_csv
 	@echo '#include <wire/onion_defs.h>' > wire/extracted_onion_wire_csv
-	@$(BOLT_EXTRACT) --message-fields --message-types $(LOCAL_BOLTDIR)/04*.md >> wire/extracted_onion_wire_csv
+	@$(BOLT_EXTRACT) $(LOCAL_BOLTDIR)/04*.md >> wire/extracted_onion_wire_csv

 # Explicit command to add diff of BOLT CSV's to an experimental_csv file
 extract-experimental-bolt-csv:
-	@$(BOLT_EXTRACT) --message-fields --message-types $(LOCAL_BOLTDIR)/0[127]*.md | diff -u wire/extracted_peer_wire_csv - >>wire/extracted_peer_experimental_csv | if [ $$? -lt 0 ];then exit 1;fi
-	@{ echo '#include <wire/onion_defs.h>'; $(BOLT_EXTRACT) --message-fields --message-types $(LOCAL_BOLTDIR)/04*.md; } | diff -u wire/extracted_onion_wire_csv - >>wire/extracted_onion_experimental_csv | if [ $$? -lt 0 ];then exit 1;fi
+	@$(BOLT_EXTRACT) $(LOCAL_BOLTDIR)/0[127]*.md | diff -u wire/extracted_peer_wire_csv - >>wire/extracted_peer_experimental_csv | if [ $$? -lt 0 ];then exit 1;fi
+	@{ echo '#include <wire/onion_defs.h>'; $(BOLT_EXTRACT) $(LOCAL_BOLTDIR)/04*.md; } | diff -u wire/extracted_onion_wire_csv - >>wire/extracted_onion_experimental_csv | if [ $$? -lt 0 ];then exit 1;fi

 wire/extracted_peer_experimental_csv:
 	@touch $@
@ -60,17 +62,17 @@ wire/gen_onion_wire_csv: wire/extracted_onion_wire_csv
 	@cp $< $@
 endif

-wire/gen_peer_wire.h: wire/gen_peer_wire_csv $(WIRE_GEN)
-	$(WIRE_GEN) --bolt --header $@ wire_type < $< > $@
+wire/gen_peer_wire.h: wire/gen_peer_wire_csv $(WIRE_BOLT_DEPS)
+	$(BOLT_GEN) -s --page header $@ wire_type < $< > $@

-wire/gen_peer_wire.c: wire/gen_peer_wire_csv $(WIRE_GEN)
-	$(WIRE_GEN) --bolt ${@:.c=.h} wire_type < $< > $@
+wire/gen_peer_wire.c: wire/gen_peer_wire_csv $(WIRE_BOLT_DEPS)
+	$(BOLT_GEN) --page impl ${@:.c=.h} wire_type < $< > $@

-wire/gen_onion_wire.h: wire/gen_onion_wire_csv $(WIRE_GEN)
-	$(WIRE_GEN) --bolt --header $@ onion_type < $< > $@
+wire/gen_onion_wire.h: wire/gen_onion_wire_csv $(WIRE_BOLT_DEPS)
+	$(BOLT_GEN) -s --page header $@ onion_type < $< > $@

-wire/gen_onion_wire.c: wire/gen_onion_wire_csv $(WIRE_GEN)
-	$(WIRE_GEN) --bolt ${@:.c=.h} onion_type < $< > $@
+wire/gen_onion_wire.c: wire/gen_onion_wire_csv $(WIRE_BOLT_DEPS)
+	$(BOLT_GEN) --page impl ${@:.c=.h} onion_type < $< > $@

 check-source: $(WIRE_SRC:%=check-src-include-order/%) $(WIRE_HEADERS_NOGEN:%=check-hdr-include-order/%)

--- a/wire/extracted_onion_wire_csv
+++ b/wire/extracted_onion_wire_csv
@ -1,41 +1,41 @@
 #include <wire/onion_defs.h>
-invalid_realm,PERM|1
-temporary_node_failure,NODE|2
-permanent_node_failure,PERM|NODE|2
-required_node_feature_missing,PERM|NODE|3
-invalid_onion_version,BADONION|PERM|4
-invalid_onion_version,0,sha256_of_onion,32
-invalid_onion_hmac,BADONION|PERM|5
-invalid_onion_hmac,0,sha256_of_onion,32
-invalid_onion_key,BADONION|PERM|6
-invalid_onion_key,0,sha256_of_onion,32
-temporary_channel_failure,UPDATE|7
-temporary_channel_failure,0,len,2
-temporary_channel_failure,2,channel_update,len
-permanent_channel_failure,PERM|8
-required_channel_feature_missing,PERM|9
-unknown_next_peer,PERM|10
-amount_below_minimum,UPDATE|11
-amount_below_minimum,0,htlc_msat,8
-amount_below_minimum,8,len,2
-amount_below_minimum,10,channel_update,len
-fee_insufficient,UPDATE|12
-fee_insufficient,0,htlc_msat,8
-fee_insufficient,8,len,2
-fee_insufficient,10,channel_update,len
-incorrect_cltv_expiry,UPDATE|13
-incorrect_cltv_expiry,0,cltv_expiry,4
-incorrect_cltv_expiry,4,len,2
-incorrect_cltv_expiry,6,channel_update,len
-expiry_too_soon,UPDATE|14
-expiry_too_soon,0,len,2
-expiry_too_soon,2,channel_update,len
-incorrect_or_unknown_payment_details,PERM|15
-incorrect_or_unknown_payment_details,0,htlc_msat,8
-final_expiry_too_soon,17
-final_incorrect_cltv_expiry,18
-final_incorrect_cltv_expiry,0,cltv_expiry,4
-final_incorrect_htlc_amount,19
-final_incorrect_htlc_amount,0,incoming_htlc_amt,8
-channel_disabled,UPDATE|20
-expiry_too_far,21
+msgtype,invalid_realm,PERM|1
+msgtype,temporary_node_failure,NODE|2
+msgtype,permanent_node_failure,PERM|NODE|2
+msgtype,required_node_feature_missing,PERM|NODE|3
+msgtype,invalid_onion_version,BADONION|PERM|4
+msgdata,invalid_onion_version,sha256_of_onion,sha256,
+msgtype,invalid_onion_hmac,BADONION|PERM|5
+msgdata,invalid_onion_hmac,sha256_of_onion,sha256,
+msgtype,invalid_onion_key,BADONION|PERM|6
+msgdata,invalid_onion_key,sha256_of_onion,sha256,
+msgtype,temporary_channel_failure,UPDATE|7
+msgdata,temporary_channel_failure,len,u16,
+msgdata,temporary_channel_failure,channel_update,byte,len
+msgtype,permanent_channel_failure,PERM|8
+msgtype,required_channel_feature_missing,PERM|9
+msgtype,unknown_next_peer,PERM|10
+msgtype,amount_below_minimum,UPDATE|11
+msgdata,amount_below_minimum,htlc_msat,u64,
+msgdata,amount_below_minimum,len,u16,
+msgdata,amount_below_minimum,channel_update,byte,len
+msgtype,fee_insufficient,UPDATE|12
+msgdata,fee_insufficient,htlc_msat,u64,
+msgdata,fee_insufficient,len,u16,
+msgdata,fee_insufficient,channel_update,byte,len
+msgtype,incorrect_cltv_expiry,UPDATE|13
+msgdata,incorrect_cltv_expiry,cltv_expiry,u32,
+msgdata,incorrect_cltv_expiry,len,u16,
+msgdata,incorrect_cltv_expiry,channel_update,byte,len
+msgtype,expiry_too_soon,UPDATE|14
+msgdata,expiry_too_soon,len,u16,
+msgdata,expiry_too_soon,channel_update,byte,len
+msgtype,incorrect_or_unknown_payment_details,PERM|15
+msgdata,incorrect_or_unknown_payment_details,htlc_msat,u64,
+msgtype,final_expiry_too_soon,17
+msgtype,final_incorrect_cltv_expiry,18
+msgdata,final_incorrect_cltv_expiry,cltv_expiry,u32,
+msgtype,final_incorrect_htlc_amount,19
+msgdata,final_incorrect_htlc_amount,incoming_htlc_amt,u64,
+msgtype,channel_disabled,UPDATE|20
+msgtype,expiry_too_far,21
--- a/wire/extracted_peer_wire_csv
+++ b/wire/extracted_peer_wire_csv
@ -1,174 +1,174 @@
-init,16
-init,0,gflen,2
-init,2,globalfeatures,gflen
-init,2+gflen,lflen,2
-init,4+gflen,localfeatures,lflen
-error,17
-error,0,channel_id,32
-error,32,len,2
-error,34,data,len
-ping,18
-ping,0,num_pong_bytes,2
-ping,2,byteslen,2
-ping,4,ignored,byteslen
-pong,19
-pong,0,byteslen,2
-pong,2,ignored,byteslen
-open_channel,32
-open_channel,0,chain_hash,32
-open_channel,32,temporary_channel_id,32
-open_channel,64,funding_satoshis,8
-open_channel,72,push_msat,8
-open_channel,80,dust_limit_satoshis,8
-open_channel,88,max_htlc_value_in_flight_msat,8
-open_channel,96,channel_reserve_satoshis,8
-open_channel,104,htlc_minimum_msat,8
-open_channel,112,feerate_per_kw,4
-open_channel,116,to_self_delay,2
-open_channel,118,max_accepted_htlcs,2
-open_channel,120,funding_pubkey,33
-open_channel,153,revocation_basepoint,33
-open_channel,186,payment_basepoint,33
-open_channel,219,delayed_payment_basepoint,33
-open_channel,252,htlc_basepoint,33
-open_channel,285,first_per_commitment_point,33
-open_channel,318,channel_flags,1
-open_channel,319,shutdown_len,2,option_upfront_shutdown_script
-open_channel,321,shutdown_scriptpubkey,shutdown_len,option_upfront_shutdown_script
-accept_channel,33
-accept_channel,0,temporary_channel_id,32
-accept_channel,32,dust_limit_satoshis,8
-accept_channel,40,max_htlc_value_in_flight_msat,8
-accept_channel,48,channel_reserve_satoshis,8
-accept_channel,56,htlc_minimum_msat,8
-accept_channel,64,minimum_depth,4
-accept_channel,68,to_self_delay,2
-accept_channel,70,max_accepted_htlcs,2
-accept_channel,72,funding_pubkey,33
-accept_channel,105,revocation_basepoint,33
-accept_channel,138,payment_basepoint,33
-accept_channel,171,delayed_payment_basepoint,33
-accept_channel,204,htlc_basepoint,33
-accept_channel,237,first_per_commitment_point,33
-accept_channel,270,shutdown_len,2,option_upfront_shutdown_script
-accept_channel,272,shutdown_scriptpubkey,shutdown_len,option_upfront_shutdown_script
-funding_created,34
-funding_created,0,temporary_channel_id,32
-funding_created,32,funding_txid,32
-funding_created,64,funding_output_index,2
-funding_created,66,signature,64
-funding_signed,35
-funding_signed,0,channel_id,32
-funding_signed,32,signature,64
-funding_locked,36
-funding_locked,0,channel_id,32
-funding_locked,32,next_per_commitment_point,33
-shutdown,38
-shutdown,0,channel_id,32
-shutdown,32,len,2
-shutdown,34,scriptpubkey,len
-closing_signed,39
-closing_signed,0,channel_id,32
-closing_signed,32,fee_satoshis,8
-closing_signed,40,signature,64
-update_add_htlc,128
-update_add_htlc,0,channel_id,32
-update_add_htlc,32,id,8
-update_add_htlc,40,amount_msat,8
-update_add_htlc,48,payment_hash,32
-update_add_htlc,80,cltv_expiry,4
-update_add_htlc,84,onion_routing_packet,1366
-update_fulfill_htlc,130
-update_fulfill_htlc,0,channel_id,32
-update_fulfill_htlc,32,id,8
-update_fulfill_htlc,40,payment_preimage,32
-update_fail_htlc,131
-update_fail_htlc,0,channel_id,32
-update_fail_htlc,32,id,8
-update_fail_htlc,40,len,2
-update_fail_htlc,42,reason,len
-update_fail_malformed_htlc,135
-update_fail_malformed_htlc,0,channel_id,32
-update_fail_malformed_htlc,32,id,8
-update_fail_malformed_htlc,40,sha256_of_onion,32
-update_fail_malformed_htlc,72,failure_code,2
-commitment_signed,132
-commitment_signed,0,channel_id,32
-commitment_signed,32,signature,64
-commitment_signed,96,num_htlcs,2
-commitment_signed,98,htlc_signature,num_htlcs*64
-revoke_and_ack,133
-revoke_and_ack,0,channel_id,32
-revoke_and_ack,32,per_commitment_secret,32
-revoke_and_ack,64,next_per_commitment_point,33
-update_fee,134
-update_fee,0,channel_id,32
-update_fee,32,feerate_per_kw,4
-channel_reestablish,136
-channel_reestablish,0,channel_id,32
-channel_reestablish,32,next_local_commitment_number,8
-channel_reestablish,40,next_remote_revocation_number,8
-channel_reestablish,48,your_last_per_commitment_secret,32,option_data_loss_protect
-channel_reestablish,80,my_current_per_commitment_point,33,option_data_loss_protect
-announcement_signatures,259
-announcement_signatures,0,channel_id,32
-announcement_signatures,32,short_channel_id,8
-announcement_signatures,40,node_signature,64
-announcement_signatures,104,bitcoin_signature,64
-channel_announcement,256
-channel_announcement,0,node_signature_1,64
-channel_announcement,64,node_signature_2,64
-channel_announcement,128,bitcoin_signature_1,64
-channel_announcement,192,bitcoin_signature_2,64
-channel_announcement,256,len,2
-channel_announcement,258,features,len
-channel_announcement,258+len,chain_hash,32
-channel_announcement,290+len,short_channel_id,8
-channel_announcement,298+len,node_id_1,33
-channel_announcement,331+len,node_id_2,33
-channel_announcement,364+len,bitcoin_key_1,33
-channel_announcement,397+len,bitcoin_key_2,33
-node_announcement,257
-node_announcement,0,signature,64
-node_announcement,64,flen,2
-node_announcement,66,features,flen
-node_announcement,66+flen,timestamp,4
-node_announcement,70+flen,node_id,33
-node_announcement,103+flen,rgb_color,3
-node_announcement,106+flen,alias,32
-node_announcement,138+flen,addrlen,2
-node_announcement,140+flen,addresses,addrlen
-channel_update,258
-channel_update,0,signature,64
-channel_update,64,chain_hash,32
-channel_update,96,short_channel_id,8
-channel_update,104,timestamp,4
-channel_update,108,message_flags,1
-channel_update,109,channel_flags,1
-channel_update,110,cltv_expiry_delta,2
-channel_update,112,htlc_minimum_msat,8
-channel_update,120,fee_base_msat,4
-channel_update,124,fee_proportional_millionths,4
-channel_update,128,htlc_maximum_msat,8,option_channel_htlc_max
-query_short_channel_ids,261
-query_short_channel_ids,0,chain_hash,32
-query_short_channel_ids,32,len,2
-query_short_channel_ids,34,encoded_short_ids,len
-reply_short_channel_ids_end,262
-reply_short_channel_ids_end,0,chain_hash,32
-reply_short_channel_ids_end,32,complete,1
-query_channel_range,263
-query_channel_range,0,chain_hash,32
-query_channel_range,32,first_blocknum,4
-query_channel_range,36,number_of_blocks,4
-reply_channel_range,264
-reply_channel_range,0,chain_hash,32
-reply_channel_range,32,first_blocknum,4
-reply_channel_range,36,number_of_blocks,4
-reply_channel_range,40,complete,1
-reply_channel_range,41,len,2
-reply_channel_range,43,encoded_short_ids,len
-gossip_timestamp_filter,265
-gossip_timestamp_filter,0,chain_hash,32
-gossip_timestamp_filter,32,first_timestamp,4
-gossip_timestamp_filter,36,timestamp_range,4
+msgtype,init,16
+msgdata,init,gflen,u16,
+msgdata,init,globalfeatures,byte,gflen
+msgdata,init,lflen,u16,
+msgdata,init,localfeatures,byte,lflen
+msgtype,error,17
+msgdata,error,channel_id,channel_id,
+msgdata,error,len,u16,
+msgdata,error,data,byte,len
+msgtype,ping,18
+msgdata,ping,num_pong_bytes,u16,
+msgdata,ping,byteslen,u16,
+msgdata,ping,ignored,byte,byteslen
+msgtype,pong,19
+msgdata,pong,byteslen,u16,
+msgdata,pong,ignored,byte,byteslen
+msgtype,open_channel,32
+msgdata,open_channel,chain_hash,chain_hash,
+msgdata,open_channel,temporary_channel_id,byte,32
+msgdata,open_channel,funding_satoshis,u64,
+msgdata,open_channel,push_msat,u64,
+msgdata,open_channel,dust_limit_satoshis,u64,
+msgdata,open_channel,max_htlc_value_in_flight_msat,u64,
+msgdata,open_channel,channel_reserve_satoshis,u64,
+msgdata,open_channel,htlc_minimum_msat,u64,
+msgdata,open_channel,feerate_per_kw,u32,
+msgdata,open_channel,to_self_delay,u16,
+msgdata,open_channel,max_accepted_htlcs,u16,
+msgdata,open_channel,funding_pubkey,pubkey,
+msgdata,open_channel,revocation_basepoint,point,
+msgdata,open_channel,payment_basepoint,point,
+msgdata,open_channel,delayed_payment_basepoint,point,
+msgdata,open_channel,htlc_basepoint,point,
+msgdata,open_channel,first_per_commitment_point,point,
+msgdata,open_channel,channel_flags,byte,
+msgdata,open_channel,shutdown_len,u16,,option_upfront_shutdown_script
+msgdata,open_channel,shutdown_scriptpubkey,byte,shutdown_len,option_upfront_shutdown_script
+msgtype,accept_channel,33
+msgdata,accept_channel,temporary_channel_id,byte,32
+msgdata,accept_channel,dust_limit_satoshis,u64,
+msgdata,accept_channel,max_htlc_value_in_flight_msat,u64,
+msgdata,accept_channel,channel_reserve_satoshis,u64,
+msgdata,accept_channel,htlc_minimum_msat,u64,
+msgdata,accept_channel,minimum_depth,u32,
+msgdata,accept_channel,to_self_delay,u16,
+msgdata,accept_channel,max_accepted_htlcs,u16,
+msgdata,accept_channel,funding_pubkey,pubkey,
+msgdata,accept_channel,revocation_basepoint,point,
+msgdata,accept_channel,payment_basepoint,point,
+msgdata,accept_channel,delayed_payment_basepoint,point,
+msgdata,accept_channel,htlc_basepoint,point,
+msgdata,accept_channel,first_per_commitment_point,point,
+msgdata,accept_channel,shutdown_len,u16,,option_upfront_shutdown_script
+msgdata,accept_channel,shutdown_scriptpubkey,byte,shutdown_len,option_upfront_shutdown_script
+msgtype,funding_created,34
+msgdata,funding_created,temporary_channel_id,byte,32
+msgdata,funding_created,funding_txid,sha256,
+msgdata,funding_created,funding_output_index,u16,
+msgdata,funding_created,signature,signature,
+msgtype,funding_signed,35
+msgdata,funding_signed,channel_id,channel_id,
+msgdata,funding_signed,signature,signature,
+msgtype,funding_locked,36
+msgdata,funding_locked,channel_id,channel_id,
+msgdata,funding_locked,next_per_commitment_point,point,
+msgtype,shutdown,38
+msgdata,shutdown,channel_id,channel_id,
+msgdata,shutdown,len,u16,
+msgdata,shutdown,scriptpubkey,byte,len
+msgtype,closing_signed,39
+msgdata,closing_signed,channel_id,channel_id,
+msgdata,closing_signed,fee_satoshis,u64,
+msgdata,closing_signed,signature,signature,
+msgtype,update_add_htlc,128
+msgdata,update_add_htlc,channel_id,channel_id,
+msgdata,update_add_htlc,id,u64,
+msgdata,update_add_htlc,amount_msat,u64,
+msgdata,update_add_htlc,payment_hash,sha256,
+msgdata,update_add_htlc,cltv_expiry,u32,
+msgdata,update_add_htlc,onion_routing_packet,byte,1366
+msgtype,update_fulfill_htlc,130
+msgdata,update_fulfill_htlc,channel_id,channel_id,
+msgdata,update_fulfill_htlc,id,u64,
+msgdata,update_fulfill_htlc,payment_preimage,preimage,
+msgtype,update_fail_htlc,131
+msgdata,update_fail_htlc,channel_id,channel_id,
+msgdata,update_fail_htlc,id,u64,
+msgdata,update_fail_htlc,len,u16,
+msgdata,update_fail_htlc,reason,byte,len
+msgtype,update_fail_malformed_htlc,135
+msgdata,update_fail_malformed_htlc,channel_id,channel_id,
+msgdata,update_fail_malformed_htlc,id,u64,
+msgdata,update_fail_malformed_htlc,sha256_of_onion,sha256,
+msgdata,update_fail_malformed_htlc,failure_code,u16,
+msgtype,commitment_signed,132
+msgdata,commitment_signed,channel_id,channel_id,
+msgdata,commitment_signed,signature,signature,
+msgdata,commitment_signed,num_htlcs,u16,
+msgdata,commitment_signed,htlc_signature,signature,num_htlcs
+msgtype,revoke_and_ack,133
+msgdata,revoke_and_ack,channel_id,channel_id,
+msgdata,revoke_and_ack,per_commitment_secret,secret,
+msgdata,revoke_and_ack,next_per_commitment_point,point,
+msgtype,update_fee,134
+msgdata,update_fee,channel_id,channel_id,
+msgdata,update_fee,feerate_per_kw,u32,
+msgtype,channel_reestablish,136
+msgdata,channel_reestablish,channel_id,channel_id,
+msgdata,channel_reestablish,next_local_commitment_number,u64,
+msgdata,channel_reestablish,next_remote_revocation_number,u64,
+msgdata,channel_reestablish,your_last_per_commitment_secret,secret,,option_data_loss_protect
+msgdata,channel_reestablish,my_current_per_commitment_point,point,,option_data_loss_protect
+msgtype,announcement_signatures,259
+msgdata,announcement_signatures,channel_id,channel_id,
+msgdata,announcement_signatures,short_channel_id,short_channel_id,
+msgdata,announcement_signatures,node_signature,signature,
+msgdata,announcement_signatures,bitcoin_signature,signature,
+msgtype,channel_announcement,256
+msgdata,channel_announcement,node_signature_1,signature,
+msgdata,channel_announcement,node_signature_2,signature,
+msgdata,channel_announcement,bitcoin_signature_1,signature,
+msgdata,channel_announcement,bitcoin_signature_2,signature,
+msgdata,channel_announcement,len,u16,
+msgdata,channel_announcement,features,byte,len
+msgdata,channel_announcement,chain_hash,chain_hash,
+msgdata,channel_announcement,short_channel_id,short_channel_id,
+msgdata,channel_announcement,node_id_1,pubkey,
+msgdata,channel_announcement,node_id_2,pubkey,
+msgdata,channel_announcement,bitcoin_key_1,pubkey,
+msgdata,channel_announcement,bitcoin_key_2,pubkey,
+msgtype,node_announcement,257
+msgdata,node_announcement,signature,signature,
+msgdata,node_announcement,flen,u16,
+msgdata,node_announcement,features,byte,flen
+msgdata,node_announcement,timestamp,u32,
+msgdata,node_announcement,node_id,pubkey,
+msgdata,node_announcement,rgb_color,byte,3
+msgdata,node_announcement,alias,byte,32
+msgdata,node_announcement,addrlen,u16,
+msgdata,node_announcement,addresses,byte,addrlen
+msgtype,channel_update,258
+msgdata,channel_update,signature,signature,
+msgdata,channel_update,chain_hash,chain_hash,
+msgdata,channel_update,short_channel_id,short_channel_id,
+msgdata,channel_update,timestamp,u32,
+msgdata,channel_update,message_flags,byte,
+msgdata,channel_update,channel_flags,byte,
+msgdata,channel_update,cltv_expiry_delta,u16,
+msgdata,channel_update,htlc_minimum_msat,u64,
+msgdata,channel_update,fee_base_msat,u32,
+msgdata,channel_update,fee_proportional_millionths,u32,
+msgdata,channel_update,htlc_maximum_msat,u64,,option_channel_htlc_max
+msgtype,query_short_channel_ids,261,gossip_queries
+msgdata,query_short_channel_ids,chain_hash,chain_hash,
+msgdata,query_short_channel_ids,len,u16,
+msgdata,query_short_channel_ids,encoded_short_ids,byte,len
+msgtype,reply_short_channel_ids_end,262,gossip_queries
+msgdata,reply_short_channel_ids_end,chain_hash,chain_hash,
+msgdata,reply_short_channel_ids_end,complete,byte,
+msgtype,query_channel_range,263,gossip_queries
+msgdata,query_channel_range,chain_hash,chain_hash,
+msgdata,query_channel_range,first_blocknum,u32,
+msgdata,query_channel_range,number_of_blocks,u32,
+msgtype,reply_channel_range,264,gossip_queries
+msgdata,reply_channel_range,chain_hash,chain_hash,
+msgdata,reply_channel_range,first_blocknum,u32,
+msgdata,reply_channel_range,number_of_blocks,u32,
+msgdata,reply_channel_range,complete,byte,
+msgdata,reply_channel_range,len,u16,
+msgdata,reply_channel_range,encoded_short_ids,byte,len
+msgtype,gossip_timestamp_filter,265,gossip_queries
+msgdata,gossip_timestamp_filter,chain_hash,chain_hash,
+msgdata,gossip_timestamp_filter,first_timestamp,u32,
+msgdata,gossip_timestamp_filter,timestamp_range,u32,