generate-wire.py: add --bolt arg, use size->type hacks only when that's specified.

For our own internal comms CSVs, we should always name explicit types.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

channeld/channel_wire.csv

@@ -8,27 +8,27 @@ channel_normal_operation,11001
 channel_init,1000
 channel_init,,chain_hash,struct sha256_double
 channel_init,,funding_txid,struct sha256_double
-channel_init,,funding_txout,2
-channel_init,,funding_satoshi,8
+channel_init,,funding_txout,u16
+channel_init,,funding_satoshi,u64
 channel_init,,our_config,struct channel_config
 channel_init,,their_config,struct channel_config
-channel_init,,feerate_per_kw,4
+channel_init,,feerate_per_kw,u32
 channel_init,,first_commit_sig,secp256k1_ecdsa_signature
 channel_init,,crypto_state,struct crypto_state
-channel_init,,remote_fundingkey,33
-channel_init,,revocation_basepoint,33
-channel_init,,payment_basepoint,33
-channel_init,,delayed_payment_basepoint,33
-channel_init,,remote_per_commit,33
-channel_init,,old_remote_per_commit,33
+channel_init,,remote_fundingkey,struct pubkey
+channel_init,,revocation_basepoint,struct pubkey
+channel_init,,payment_basepoint,struct pubkey
+channel_init,,delayed_payment_basepoint,struct pubkey
+channel_init,,remote_per_commit,struct pubkey
+channel_init,,old_remote_per_commit,struct pubkey
 channel_init,,funder,enum side
-channel_init,,fee_base,4
-channel_init,,fee_proportional,4
-channel_init,,local_msatoshi,8
+channel_init,,fee_base,u32
+channel_init,,fee_proportional,u32
+channel_init,,local_msatoshi,u64
 channel_init,,seed,struct privkey
 channel_init,,local_node_id,struct pubkey
 channel_init,,remote_node_id,struct pubkey
-channel_init,,commit_msec,4
+channel_init,,commit_msec,u32
 channel_init,,cltv_delta,u16
 channel_init,,last_was_revoke,bool
 channel_init,,num_last_sent_commit,u16
@@ -66,32 +66,32 @@ channel_funding_announce_depth,1003
 
 # Tell channel to offer this htlc
 channel_offer_htlc,1004
-channel_offer_htlc,,amount_msat,8
-channel_offer_htlc,,cltv_expiry,4
-channel_offer_htlc,,payment_hash,32
+channel_offer_htlc,,amount_msat,u64
+channel_offer_htlc,,cltv_expiry,u32
+channel_offer_htlc,,payment_hash,struct sha256
 channel_offer_htlc,,onion_routing_packet,1366*u8
 
 # Reply; synchronous since IDs have to increment.
 channel_offer_htlc_reply,1104
-channel_offer_htlc_reply,,id,8
+channel_offer_htlc_reply,,id,u64
 # Zero failure code means success.
-channel_offer_htlc_reply,,failure_code,2
-channel_offer_htlc_reply,,failurestrlen,2
+channel_offer_htlc_reply,,failure_code,u16
+channel_offer_htlc_reply,,failurestrlen,u16
 channel_offer_htlc_reply,,failurestr,failurestrlen*u8
 
 # Main daemon found out the preimage for an htlc
 #include <bitcoin/preimage.h>
 channel_fulfill_htlc,1005
-channel_fulfill_htlc,,id,8
+channel_fulfill_htlc,,id,u64
 channel_fulfill_htlc,,payment_preimage,struct preimage
 
 # Main daemon says HTLC failed
 channel_fail_htlc,1006
-channel_fail_htlc,,id,8
+channel_fail_htlc,,id,u64
 # If malformed is non-zero, it's a BADONION code
 channel_fail_htlc,,malformed,u16
 # Otherwise, error_pkt contains failreason.
-channel_fail_htlc,,len,2
+channel_fail_htlc,,len,u16
 channel_fail_htlc,,error_pkt,len*u8
 
 # Ping/pong test.

gossipd/gossip_wire.csv

@@ -1,31 +1,31 @@
 # Peers can give a bad message, we close their fd, but no harm done.
 gossipstatus_peer_bad_msg,13000
-gossipstatus_peer_bad_msg,,unique_id,8
-gossipstatus_peer_bad_msg,,len,2
+gossipstatus_peer_bad_msg,,unique_id,u64
+gossipstatus_peer_bad_msg,,len,u16
 gossipstatus_peer_bad_msg,,err,len*u8
 
 # Misc problems like opening control fd.
 gossipstatus_peer_failed,13001
-gossipstatus_peer_failed,,unique_id,8
-gossipstatus_peer_failed,,len,2
+gossipstatus_peer_failed,,unique_id,u64
+gossipstatus_peer_failed,,len,u16
 gossipstatus_peer_failed,,err,len*u8
 
 #include <common/cryptomsg.h>
 
 # Initialize the gossip daemon
 gossipctl_init,3000
-gossipctl_init,,broadcast_interval,4
+gossipctl_init,,broadcast_interval,u32
 gossipctl_init,,chain_hash,struct sha256_double
 
 # These take an fd, but have no response
 # (if it is to move onto a channel, we get a status msg).
 gossipctl_new_peer,3001
-gossipctl_new_peer,,unique_id,8
+gossipctl_new_peer,,unique_id,u64
 gossipctl_new_peer,,crypto_state,struct crypto_state
 
 # Tell it to release a peer which has initialized.
 gossipctl_release_peer,3002
-gossipctl_release_peer,,unique_id,8
+gossipctl_release_peer,,unique_id,u64
 
 # This releases the peer and returns the cryptostate (followed two fds: peer and gossip)
 gossipctl_release_peer_reply,3102
@@ -36,17 +36,17 @@ gossipctl_release_peer_replyfail,3202
 
 # This is where we save a peer's features.
 #gossipstatus_peer_features,3001
-#gossipstatus_peer_features,,unique_id,8
-#gossipstatus_peer_features,,gflen,2
-#gossipstatus_peer_features,,globalfeatures,gflen
-#gossipstatus_peer_features,,lflen,2
-#gossipstatus_peer_features,,localfeatures,lflen
+#gossipstatus_peer_features,,unique_id,u64
+#gossipstatus_peer_features,,gflen,u16
+#gossipstatus_peer_features,,globalfeatures,gflen*u8
+#gossipstatus_peer_features,,lflen,u16
+#gossipstatus_peer_features,,localfeatures,lflen*u8
 
 # Peer can send non-gossip packet (usually an open_channel) (followed two fds: peer and gossip)
 gossipstatus_peer_nongossip,3004
-gossipstatus_peer_nongossip,,unique_id,8
+gossipstatus_peer_nongossip,,unique_id,u64
 gossipstatus_peer_nongossip,,crypto_state,struct crypto_state
-gossipstatus_peer_nongossip,,len,2
+gossipstatus_peer_nongossip,,len,u16
 gossipstatus_peer_nongossip,,msg,len*u8
 
 # Pass JSON-RPC getnodes call through
@@ -94,12 +94,12 @@ gossip_resolve_channel_reply,,keys,num_keys*struct pubkey
 # The main daemon forward some gossip message to gossipd, allows injecting
 # arbitrary gossip messages.
 gossip_forwarded_msg,3010
-gossip_forwarded_msg,,msglen,2
-gossip_forwarded_msg,,msg,msglen
+gossip_forwarded_msg,,msglen,u16
+gossip_forwarded_msg,,msg,msglen*u8
 
 # If peer is still connected, fail it (master does this for reconnect)
 gossipctl_fail_peer,3011
-gossipctl_fail_peer,,unique_id,8
+gossipctl_fail_peer,,unique_id,u64
 
 # Get a gossip fd for this peer (it has reconnected)
 gossipctl_get_peer_gossipfd,3012

handshaked/handshake_wire.csv

@@ -10,22 +10,22 @@ respr_act_three,14013
 success,4000
 
 handshake_responder,4001
-handshake_responder,,my_id,33
+handshake_responder,,my_id,struct pubkey
 handshake_responder_reply,4101
-handshake_responder_reply,,initiator_id,33
+handshake_responder_reply,,initiator_id,struct pubkey
 handshake_responder_reply,,cs,struct crypto_state
-handshake_responder_reply,,gflen,2
-handshake_responder_reply,,globalfeatures,gflen
-handshake_responder_reply,,lflen,2
-handshake_responder_reply,,localfeatures,lflen
+handshake_responder_reply,,gflen,u16
+handshake_responder_reply,,globalfeatures,gflen*u8
+handshake_responder_reply,,lflen,u16
+handshake_responder_reply,,localfeatures,lflen*u8
 
 handshake_initiator,4002
-handshake_initiator,,my_id,33
-handshake_initiator,,responder_id,33
+handshake_initiator,,my_id,struct pubkey
+handshake_initiator,,responder_id,struct pubkey
 
 handshake_initiator_reply,4102
 handshake_initiator_reply,,cs,struct crypto_state
-handshake_initiator_reply,,gflen,2
-handshake_initiator_reply,,globalfeatures,gflen
-handshake_initiator_reply,,lflen,2
-handshake_initiator_reply,,localfeatures,lflen
+handshake_initiator_reply,,gflen,u16
+handshake_initiator_reply,,globalfeatures,gflen*u8
+handshake_initiator_reply,,lflen,u16
+handshake_initiator_reply,,localfeatures,lflen*u8

hsmd/hsm_client_wire_csv

@@ -7,15 +7,15 @@ hsm_ecdh_resp,,ss,struct secret
 hsm_cannouncement_sig_req,2
 hsm_cannouncement_sig_req,,bitcoin_id,struct pubkey
 hsm_cannouncement_sig_req,,calen,u16
-hsm_cannouncement_sig_req,,ca,calen
+hsm_cannouncement_sig_req,,ca,calen*u8
 
 hsm_cannouncement_sig_reply,102
-hsm_cannouncement_sig_reply,,node_signature,64
+hsm_cannouncement_sig_reply,,node_signature,secp256k1_ecdsa_signature
 
 hsm_cupdate_sig_req,3
 hsm_cupdate_sig_req,,culen,u16
-hsm_cupdate_sig_req,,cu,culen
+hsm_cupdate_sig_req,,cu,culen*u8
 
 hsm_cupdate_sig_reply,103
 hsm_cupdate_sig_reply,,culen,u16
-hsm_cupdate_sig_reply,,cu,culen
+hsm_cupdate_sig_reply,,cu,culen*u8

hsmd/hsm_wire.csv

@@ -1,7 +1,7 @@
 # Clients should not give a bad request but not the HSM's decision to crash.
 hsmstatus_client_bad_request,1000
-hsmstatus_client_bad_request,,unique_id,8
-hsmstatus_client_bad_request,,len,2
+hsmstatus_client_bad_request,,unique_id,u64
+hsmstatus_client_bad_request,,len,u16
 hsmstatus_client_bad_request,,msg,len*u8
 
 # Start the HSM.
@@ -10,13 +10,13 @@ hsmctl_init,,new,bool
 
 #include <common/bip32.h>
 hsmctl_init_reply,101
-hsmctl_init_reply,,node_id,33
+hsmctl_init_reply,,node_id,struct pubkey
 hsmctl_init_reply,,peer_seed,struct secret
 hsmctl_init_reply,,bip32,struct ext_key
 
 # ECDH returns an fd.
 hsmctl_hsmfd_ecdh,3
-hsmctl_hsmfd_ecdh,,unique_id,8
+hsmctl_hsmfd_ecdh,,unique_id,u64
 
 # No contents, just an fd.
 hsmctl_hsmfd_ecdh_fd_reply,103
@@ -25,28 +25,28 @@ hsmctl_hsmfd_ecdh_fd_reply,103
 #include <common/utxo.h>
 # FIXME: This should also take their commit sig & details, to verify.
 hsmctl_sign_funding,4
-hsmctl_sign_funding,,satoshi_out,8
-hsmctl_sign_funding,,change_out,8
-hsmctl_sign_funding,,change_keyindex,4
-hsmctl_sign_funding,,our_pubkey,33
-hsmctl_sign_funding,,their_pubkey,33
-hsmctl_sign_funding,,num_inputs,2
+hsmctl_sign_funding,,satoshi_out,u64
+hsmctl_sign_funding,,change_out,u64
+hsmctl_sign_funding,,change_keyindex,u32
+hsmctl_sign_funding,,our_pubkey,struct pubkey
+hsmctl_sign_funding,,their_pubkey,struct pubkey
+hsmctl_sign_funding,,num_inputs,u16
 hsmctl_sign_funding,,inputs,num_inputs*struct utxo
 
 hsmctl_sign_funding_reply,104
-hsmctl_sign_funding_reply,,num_sigs,2
+hsmctl_sign_funding_reply,,num_sigs,u16
 hsmctl_sign_funding_reply,,sig,num_sigs*secp256k1_ecdsa_signature
 
 # Request a client socket for a `channeld`, allows signing announcements 
 hsmctl_hsmfd_channeld,5
-hsmctl_hsmfd_channeld,,unique_id,8
+hsmctl_hsmfd_channeld,,unique_id,u64
 
 # Empty reply, just an fd
 hsmctl_hsmfd_channeld_reply,105
 
 # Master asks the HSM to sign a node_announcement
 hsmctl_node_announcement_sig_req,6
-hsmctl_node_announcement_sig_req,,annlen,2
+hsmctl_node_announcement_sig_req,,annlen,u16
 hsmctl_node_announcement_sig_req,,announcement,annlen*u8
 
 hsmctl_node_announcement_sig_reply,106
@@ -54,13 +54,13 @@ hsmctl_node_announcement_sig_reply,,signature,secp256k1_ecdsa_signature
 
 # Sign a withdrawal request
 hsmctl_sign_withdrawal,7
-hsmctl_sign_withdrawal,,satoshi_out,8
-hsmctl_sign_withdrawal,,change_out,8
-hsmctl_sign_withdrawal,,change_keyindex,4
+hsmctl_sign_withdrawal,,satoshi_out,u64
+hsmctl_sign_withdrawal,,change_out,u64
+hsmctl_sign_withdrawal,,change_keyindex,u32
 hsmctl_sign_withdrawal,,pkh,20*u8
-hsmctl_sign_withdrawal,,num_inputs,2
+hsmctl_sign_withdrawal,,num_inputs,u16
 hsmctl_sign_withdrawal,,inputs,num_inputs*struct utxo
 
 hsmctl_sign_withdrawal_reply,107
-hsmctl_sign_withdrawal_reply,,num_sigs,2
+hsmctl_sign_withdrawal_reply,,num_sigs,u16
 hsmctl_sign_withdrawal_reply,,sig,num_sigs*secp256k1_ecdsa_signature

openingd/opening_wire.csv

@@ -2,12 +2,12 @@
 #include <common/channel_config.h>
 opening_init,6000
 # Which network are we configured for (as index into the chainparams)?
-opening_init,,network_index,4
+opening_init,,network_index,u32
 # Base configuration we'll offer (channel reserve will vary with amount)
 opening_init,,our_config,struct channel_config
 # Minimum/maximum configuration values we'll accept
-opening_init,,max_to_self_delay,4
-opening_init,,min_effective_htlc_capacity_msat,8
+opening_init,,max_to_self_delay,u32
+opening_init,,min_effective_htlc_capacity_msat,u64
 opening_init,,crypto_state,struct crypto_state
 # Seed to generate all the keys from
 opening_init,,seed,struct privkey
@@ -16,10 +16,10 @@ opening_init,,seed,struct privkey
 #include <common/htlc_wire.h>
 # This means we offer the open.
 opening_funder,6001
-opening_funder,,funding_satoshis,8
-opening_funder,,push_msat,8
-opening_funder,,feerate_per_kw,4
-opening_funder,,max_minimum_depth,4
+opening_funder,,funding_satoshis,u64
+opening_funder,,push_msat,u64
+opening_funder,,feerate_per_kw,u32
+opening_funder,,max_minimum_depth,u32
 opening_funder,,change_satoshis,u64
 opening_funder,,change_keyindex,u32
 opening_funder,,channel_flags,u8
@@ -34,21 +34,21 @@ opening_funder_reply,,their_config,struct channel_config
 opening_funder_reply,,first_commit,struct bitcoin_tx
 opening_funder_reply,,first_commit_sig,secp256k1_ecdsa_signature
 opening_funder_reply,,crypto_state,struct crypto_state
-opening_funder_reply,,revocation_basepoint,33
-opening_funder_reply,,payment_basepoint,33
-opening_funder_reply,,delayed_payment_basepoint,33
-opening_funder_reply,,their_per_commit_point,33
-opening_funder_reply,,minimum_depth,4
-opening_funder_reply,,remote_fundingkey,33
+opening_funder_reply,,revocation_basepoint,struct pubkey
+opening_funder_reply,,payment_basepoint,struct pubkey
+opening_funder_reply,,delayed_payment_basepoint,struct pubkey
+opening_funder_reply,,their_per_commit_point,struct pubkey
+opening_funder_reply,,minimum_depth,u32
+opening_funder_reply,,remote_fundingkey,struct pubkey
 opening_funder_reply,,funding_txid,struct sha256_double
-opening_funder_reply,,feerate_per_kw,4
+opening_funder_reply,,feerate_per_kw,u32
 
 # This means they offer the open (contains their offer packet)
 opening_fundee,6003
-opening_fundee,,minimum_depth,4
-opening_fundee,,min_feerate,4
-opening_fundee,,max_feerate,4
-opening_fundee,,len,2
+opening_fundee,,minimum_depth,u32
+opening_fundee,,min_feerate,u32
+opening_fundee,,max_feerate,u32
+opening_fundee,,len,u16
 opening_fundee,,msg,len*u8
 
 # This gives their txid and info, means we can send funding_signed: we're done.
@@ -57,17 +57,17 @@ opening_fundee_reply,,their_config,struct channel_config
 opening_fundee_reply,,first_commit,struct bitcoin_tx
 opening_fundee_reply,,first_commit_sig,secp256k1_ecdsa_signature
 opening_fundee_reply,,crypto_state,struct crypto_state
-opening_fundee_reply,,revocation_basepoint,33
-opening_fundee_reply,,payment_basepoint,33
-opening_fundee_reply,,delayed_payment_basepoint,33
-opening_fundee_reply,,their_per_commit_point,33
-opening_fundee_reply,,remote_fundingkey,33
+opening_fundee_reply,,revocation_basepoint,struct pubkey
+opening_fundee_reply,,payment_basepoint,struct pubkey
+opening_fundee_reply,,delayed_payment_basepoint,struct pubkey
+opening_fundee_reply,,their_per_commit_point,struct pubkey
+opening_fundee_reply,,remote_fundingkey,struct pubkey
 opening_fundee_reply,,funding_txid,struct sha256_double
 opening_fundee_reply,,funding_txout,u16
-opening_fundee_reply,,funding_satoshis,8
-opening_fundee_reply,,push_msat,8
+opening_fundee_reply,,funding_satoshis,u64
+opening_fundee_reply,,push_msat,u64
 opening_fundee_reply,,channel_flags,u8
-opening_fundee_reply,,feerate_per_kw,4
+opening_fundee_reply,,feerate_per_kw,u32
 # The (encrypted) funding signed message: send this and we're committed.
 opening_fundee_reply,,msglen,u16
 opening_fundee_reply,,funding_signed_msg,msglen*u8

tools/generate-wire.py

@@ -34,7 +34,6 @@ varlen_structs = [
 class FieldType(object):
     def __init__(self,name):
         self.name = name
-        self.tsize = FieldType._typesize(name)
 
     def is_assignable(self):
         return self.name in ['u8', 'u16', 'u32', 'u64', 'bool'] or self.name.startswith('enum ')
@@ -116,27 +115,26 @@ class Field(object):
             else:
                 self.num_elems = int(number)
             size = size.split('*')[1]
-        else:
-            if size == prevname:
-                # Raw length field, implies u8.
-                self.lenvar = size
-                size = 'u8'
+        elif options.bolt and size == prevname:
+            # Raw length field, implies u8.
+            self.lenvar = size
+            size = '1'
 
-        try:
-            # Just a number?  Guess based on size.
+        # Bolts use just a number: Guess type based on size.
+        if options.bolt:
             base_size = int(size)
             self.fieldtype = Field._guess_type(message,self.name,base_size)
             # There are some arrays which we have to guess, based on sizes.
-            if base_size % self.fieldtype.tsize != 0:
+            tsize = FieldType._typesize(self.fieldtype.name)
+            if base_size % tsize != 0:
                 raise ValueError('Invalid size {} for {}.{} not a multiple of {}'
                                  .format(base_size,
                                          self.message,
                                          self.name,
-                                         self.fieldtype.tsize))
-            self.num_elems = int(base_size / self.fieldtype.tsize)
-
-        except ValueError:
-            # Not a number; must be a type.
+                                         tsize))
+            self.num_elems = int(base_size / tsize)
+        else:
+            # Real typename.
             self.fieldtype = FieldType(size)
 
     def basetype(self):
@@ -227,8 +225,8 @@ class Message(object):
         for f in self.fields:
             if f.name == field.lenvar:
                 if f.fieldtype.name != 'u16':
-                    raise ValueError('Field {} has non-u16 length variable {}'
-                                     .format(field.name, field.lenvar))
+                    raise ValueError('Field {} has non-u16 length variable {} (type {})'
+                                     .format(field.name, field.lenvar, f.fieldtype.name))
 
                 if f.is_array() or f.is_variable_size():
                     raise ValueError('Field {} has non-simple length variable {}'
@@ -391,6 +389,7 @@ class Message(object):
 
 parser = argparse.ArgumentParser(description='Generate C from from CSV')
 parser.add_argument('--header', action='store_true', help="Create wire header")
+parser.add_argument('--bolt', action='store_true', help="Generate wire-format for BOLT")
 parser.add_argument('headerfilename', help='The filename of the header')
 parser.add_argument('enumname', help='The name of the enum to produce')
 parser.add_argument('files', nargs='*', help='Files to read in (or stdin)')

wire/Makefile

@@ -34,16 +34,16 @@ wire/gen_onion_wire_csv: FORCE
 	@set -e; if [ -f $(BOLT_EXTRACT) ]; then for f in $(BOLTDIR)/04*.md $(BOLT_EXTRACT); do if [ $$f -nt $@ -o ! -f $@ ]; then echo '#include <wire/onion_defs.h>' > $@ && $(BOLT_EXTRACT) --message-fields --message-types --check-alignment $(BOLTDIR)/04*.md >> $@; break; fi; done; fi
 
 wire/gen_peer_wire.h: $(WIRE_GEN) wire/gen_peer_wire_csv
-	$(WIRE_GEN) --header $@ wire_type < wire/gen_peer_wire_csv > $@
+	$(WIRE_GEN) --bolt --header $@ wire_type < wire/gen_peer_wire_csv > $@
 
 wire/gen_peer_wire.c: $(WIRE_GEN) wire/gen_peer_wire_csv
-	$(WIRE_GEN) ${@:.c=.h} wire_type < wire/gen_peer_wire_csv > $@
+	$(WIRE_GEN) --bolt ${@:.c=.h} wire_type < wire/gen_peer_wire_csv > $@
 
 wire/gen_onion_wire.h: $(WIRE_GEN) wire/gen_onion_wire_csv
-	$(WIRE_GEN) --header $@ onion_type < wire/gen_onion_wire_csv > $@
+	$(WIRE_GEN) --bolt --header $@ onion_type < wire/gen_onion_wire_csv > $@
 
 wire/gen_onion_wire.c: $(WIRE_GEN) wire/gen_onion_wire_csv
-	$(WIRE_GEN) ${@:.c=.h} onion_type < wire/gen_onion_wire_csv > $@
+	$(WIRE_GEN) --bolt ${@:.c=.h} onion_type < wire/gen_onion_wire_csv > $@
 
 check-source: $(WIRE_SRC:%=check-src-include-order/%) $(WIRE_HEADERS_NOGEN:%=check-hdr-include-order/%)