tools/generate-wire.py: allow typename instead of type sizes.

We use the fourth value (size) to determine the type, unless the fifth
value is suppled.  That's silly: allow the fourth value to be a typename,
since that's the only reason we care about the size at all!

Unfortunately there are places in the spec where we use a raw fieldname
without '*1' for a length, so we have to distingish this from the
typename case.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

lightningd/channel/channel_wire.csv

@@ -19,25 +19,25 @@ channel_normal_operation,1001
 # Begin!  You're still waiting for the tx to be buried though (passes
 # gossipd-client fd)
 channel_init,1
-channel_init,0,funding_txid,32,struct sha256_double
+channel_init,0,funding_txid,struct sha256_double
 channel_init,32,funding_txout,2
-channel_init,34,our_config,36,struct channel_config
-channel_init,70,their_config,36,struct channel_config
-channel_init,106,first_commit_sig,64,secp256k1_ecdsa_signature
-channel_init,166,crypto_state,144,struct crypto_state
+channel_init,34,our_config,struct channel_config
+channel_init,70,their_config,struct channel_config
+channel_init,106,first_commit_sig,secp256k1_ecdsa_signature
+channel_init,166,crypto_state,struct crypto_state
 channel_init,310,remote_fundingkey,33
 channel_init,343,revocation_basepoint,33
 channel_init,376,payment_basepoint,33
 channel_init,409,delayed_payment_basepoint,33
 channel_init,442,their_per_commit_point,33
-channel_init,475,am_funder,1,bool
+channel_init,475,am_funder,bool
 channel_init,476,feerate,4
 channel_init,480,funding_satoshi,8
 channel_init,488,push_msat,8
-channel_init,496,seed,32,struct privkey
-channel_init,529,local_node_id,33,struct pubkey
-channel_init,562,remote_node_id,33,struct pubkey
+channel_init,496,seed,struct privkey
+channel_init,529,local_node_id,struct pubkey
+channel_init,562,remote_node_id,struct pubkey
 
 # Tx is deep enough, go!
 channel_funding_locked,2
-channel_funding_locked,0,short_channel_id,8,struct short_channel_id
+channel_funding_locked,0,short_channel_id,struct short_channel_id

lightningd/gossip/gossip_wire.csv

@@ -9,7 +9,7 @@ gossipstatus_fdpass_failed,0x8004
 gossipstatus_peer_bad_msg,1000
 gossipstatus_peer_bad_msg,0,unique_id,8
 gossipstatus_peer_bad_msg,8,len,2
-gossipstatus_peer_bad_msg,10,err,len,u8
+gossipstatus_peer_bad_msg,10,err,len*u8
 
 #include <lightningd/cryptomsg.h>
 
@@ -17,7 +17,7 @@ gossipstatus_peer_bad_msg,10,err,len,u8
 # (if it is to move onto a channel, we get a status msg).
 gossipctl_new_peer,1
 gossipctl_new_peer,0,unique_id,8
-gossipctl_new_peer,8,crypto_state,144,struct crypto_state
+gossipctl_new_peer,8,crypto_state,struct crypto_state
 
 # Tell it to release a peer which has initialized.
 gossipctl_release_peer,2
@@ -26,7 +26,7 @@ gossipctl_release_peer,0,unique_id,8
 # This releases the peer and returns the cryptostate (followed by fd)
 gossipctl_release_peer_reply,102
 gossipctl_release_peer_reply,0,unique_id,8
-gossipctl_release_peer_reply,8,crypto_state,144,struct crypto_state
+gossipctl_release_peer_reply,8,crypto_state,struct crypto_state
 
 # This is where we save a peer's features.
 #gossipstatus_peer_features,1
@@ -43,13 +43,13 @@ gossipstatus_peer_ready,0,unique_id,8
 # Peer can send non-gossip packet (usually an open_channel) (followed by fd)
 gossipstatus_peer_nongossip,4
 gossipstatus_peer_nongossip,0,unique_id,8
-gossipstatus_peer_nongossip,10,crypto_state,144,struct crypto_state
+gossipstatus_peer_nongossip,10,crypto_state,struct crypto_state
 gossipstatus_peer_nongossip,154,len,2
-gossipstatus_peer_nongossip,156,msg,len,u8
+gossipstatus_peer_nongossip,156,msg,len*u8
 
 # Pass JSON-RPC getnodes call through
 gossip_getnodes_request,5
 
 gossip_getnodes_reply,105
-gossip_getnodes_reply,0,replen,2,u16
-gossip_getnodes_reply,2,reply,replen,u8
+gossip_getnodes_reply,0,replen,u16
+gossip_getnodes_reply,2,reply,replen*u8

lightningd/handshake/handshake_wire.csv

@@ -36,12 +36,12 @@ handshake_responder,1
 handshake_responder,1,my_id,33
 handshake_responder_reply,101
 handshake_responder_reply,0,initiator_id,33
-handshake_responder_reply,33,cs,144,struct crypto_state
+handshake_responder_reply,33,cs,struct crypto_state
 
 handshake_initiator,2
 handshake_initiator,0,my_id,33
 handshake_initiator,33,responder_id,33
 
 handshake_initiator_reply,102
-handshake_initiator_reply,0,cs,144,struct crypto_state
+handshake_initiator_reply,0,cs,struct crypto_state
 

lightningd/hsm/hsm_wire.csv

@@ -9,17 +9,17 @@ hsmstatus_key_failed,0x8004
 hsmstatus_client_bad_request,1000
 hsmstatus_client_bad_request,0,unique-id,8
 hsmstatus_client_bad_request,8,len,2
-hsmstatus_client_bad_request,10,msg,len,u8
+hsmstatus_client_bad_request,10,msg,len*u8
 
 # Start the HSM.
 hsmctl_init,1
-hsmctl_init,0,new,1,bool
+hsmctl_init,0,new,bool
 
 hsmctl_init_reply,101
 hsmctl_init_reply,0,node_id,33
-hsmctl_init_reply,33,peer_seed,32,struct privkey
+hsmctl_init_reply,33,peer_seed,struct privkey
 hsmctl_init_reply,65,bip32_len,2
-hsmctl_init_reply,67,bip32_seed,bip32_len*1,u8
+hsmctl_init_reply,67,bip32_seed,bip32_len*u8
 
 # ECDH returns an fd.
 hsmctl_hsmfd_ecdh,3
@@ -38,10 +38,10 @@ hsmctl_sign_funding,16,change_keyindex,4
 hsmctl_sign_funding,20,our_pubkey,33
 hsmctl_sign_funding,52,their_pubkey,33
 hsmctl_sign_funding,85,num_inputs,2
-hsmctl_sign_funding,87,inputs,num_inputs*49,struct utxo
+hsmctl_sign_funding,87,inputs,num_inputs*struct utxo
 
 hsmctl_sign_funding_reply,104
 hsmctl_sign_funding_reply,0,num_sigs,2
-hsmctl_sign_funding_reply,0,sig,num_sigs*64,secp256k1_ecdsa_signature
+hsmctl_sign_funding_reply,0,sig,num_sigs*secp256k1_ecdsa_signature
 
 

lightningd/opening/opening_wire.csv

@@ -15,13 +15,13 @@ opening_peer_bad_initial_message,0x8014
 #include <lightningd/channel_config.h>
 opening_init,0
 # Base configuration we'll offer (channel reserve will vary with amount)
-opening_init,0,our_config,36,struct channel_config
+opening_init,0,our_config,struct channel_config
 # Minimum/maximum configuration values we'll accept
 opening_init,36,max_to_self_delay,4
 opening_init,40,min_effective_htlc_capacity_msat,8
-opening_init,48,crypto_state,144,struct crypto_state
+opening_init,48,crypto_state,struct crypto_state
 # Seed to generate all the keys from
-opening_init,196,seed,32,struct privkey
+opening_init,196,seed,struct privkey
 
 # This means we offer the open.
 opening_open,1
@@ -37,14 +37,14 @@ opening_open_reply,0,remote_fundingkey,33
 
 # Now we give the funding txid and outnum.
 opening_open_funding,2
-opening_open_funding,0,txid,32,struct sha256_double
-opening_open_funding,32,txout,2,u16
+opening_open_funding,0,txid,struct sha256_double
+opening_open_funding,32,txout,u16
 
 # This gives their sig, means we can broadcast tx: we're done.
 opening_open_funding_reply,102
-opening_open_funding_reply,0,their_config,36,struct channel_config
-opening_open_funding_reply,36,first_commit_sig,64,secp256k1_ecdsa_signature
-opening_open_funding_reply,100,crypto_state,144,struct crypto_state
+opening_open_funding_reply,0,their_config,struct channel_config
+opening_open_funding_reply,36,first_commit_sig,secp256k1_ecdsa_signature
+opening_open_funding_reply,100,crypto_state,struct crypto_state
 opening_open_funding_reply,244,revocation_basepoint,33
 opening_open_funding_reply,277,payment_basepoint,33
 opening_open_funding_reply,310,delayed_payment_basepoint,33
@@ -55,20 +55,20 @@ opening_accept,3
 opening_accept,0,min_feerate,4
 opening_accept,4,max_feerate,4
 opening_accept,8,len,2
-opening_accept,10,msg,len,u8
+opening_accept,10,msg,len*u8
 
 # This gives the txid of their funding tx to watch.
 opening_accept_reply,103
-opening_accept_reply,0,funding_txid,32,struct sha256_double
+opening_accept_reply,0,funding_txid,struct sha256_double
 
 # Acknowledge watch is in place, now can send sig.
 opening_accept_finish,4
 
 opening_accept_finish_reply,104
-opening_accept_finish_reply,32,funding_txout,2,u16
-opening_accept_finish_reply,0,their_config,36,struct channel_config
-opening_accept_finish_reply,36,first_commit_sig,64,secp256k1_ecdsa_signature
-opening_accept_finish_reply,100,crypto_state,144,struct crypto_state
+opening_accept_finish_reply,32,funding_txout,u16
+opening_accept_finish_reply,0,their_config,struct channel_config
+opening_accept_finish_reply,36,first_commit_sig,secp256k1_ecdsa_signature
+opening_accept_finish_reply,100,crypto_state,struct crypto_state
 opening_accept_finish_reply,244,remote_fundingkey,33
 opening_accept_finish_reply,277,revocation_basepoint,33
 opening_accept_finish_reply,310,payment_basepoint,33

tools/generate-wire.py

@@ -81,40 +81,48 @@ sizetypemap = {
     1: FieldType('u8')
 }
 
+# It would be nicer if we had put '*1' in spec and disallowed bare lenvar.
+# In practice we only recognize raw lenvar when it's the previous field.
+
+# size := baresize | arraysize
+# baresize := simplesize | lenvar
+# simplesize := number | type
+# arraysize := lenvar '*' simplesize
 class Field(object):
-    def __init__(self, message, name, size, comments, typename=None):
+    def __init__(self, message, name, size, comments, prevname):
         self.message = message
         self.comments = comments
         self.name = name.replace('-', '_')
         self.is_len_var = False
         self.lenvar = None
+        self.num_elems = 1
+
+        # If it's an arraysize, swallow prefix.
+        if '*' in size:
+            self.lenvar = size.split('*')[0].replace('-','_')
+            size = size.split('*')[1]
+        else:
+            if size == prevname:
+                # Raw length field, implies u8.
+                self.lenvar = size.replace('-','_')
+                size = 'u8'
 
-        # Size could be a literal number (eg. 33), or a field (eg 'len'), or
-        # a multiplier of a field (eg. num-htlc-timeouts*64).
         try:
+            # Just a number?  Guess based on size.
             base_size = int(size)
-        except ValueError:
-            # If it's a multiplicitive expression, must end in basesize.
-            if '*' in size:
-                base_size = int(size.split('*')[1])
-                self.lenvar = size.split('*')[0]
-            else:
-                base_size = 0
-                self.lenvar = size
-            self.lenvar = self.lenvar.replace('-','_')
-
-        if typename is None:
             self.fieldtype = Field._guess_type(message,self.name,base_size)
-        else:
-            self.fieldtype = FieldType(typename)
+            # There are some arrays which we have to guess, based on sizes.
+            if base_size % self.fieldtype.tsize != 0:
+                raise ValueError('Invalid size {} for {}.{} not a multiple of {}'
+                                 .format(base_size,
+                                         self.message,
+                                         self.name,
+                                         self.fieldtype.tsize))
+            self.num_elems = int(base_size / self.fieldtype.tsize)
 
-        # Unknown types are assumed to have base_size: div by 0 if that's unknown.
-        if self.fieldtype.tsize == 0:
-            self.fieldtype.tsize = base_size
-
-        if base_size % self.fieldtype.tsize != 0:
-            raise ValueError('Invalid size {} for {}.{} not a multiple of {}'.format(base_size,self.message,self.name,self.fieldtype.tsize))
-        self.num_elems = int(base_size / self.fieldtype.tsize)
+        except ValueError:
+            # Not a number; must be a type.
+            self.fieldtype = FieldType(size)
 
     def is_padding(self):
         return self.name.startswith('pad')
@@ -360,6 +368,7 @@ options = parser.parse_args()
 messages = []
 comments = []
 includes = []
+prevfield = None
 
 # Read csv lines.  Single comma is the message values, more is offset/len.
 for line in fileinput.input(options.files):
@@ -382,18 +391,19 @@ for line in fileinput.input(options.files):
         # eg commit_sig,132
         messages.append(Message(parts[0],Enumtype("WIRE_" + parts[0].upper(), parts[1]), comments))
         comments=[]
-    else:
+        prevfield = None
+    elif len(parts) == 4:
         # eg commit_sig,0,channel-id,8 OR
-        #    commit_sig,0,channel-id,8,u64
+        #    commit_sig,0,channel-id,u64
         for m in messages:
             if m.name == parts[0]:
-                if len(parts) == 4:
-                    m.addField(Field(parts[0], parts[2], parts[3], comments))
-                else:
-                    m.addField(Field(parts[0], parts[2], parts[3], comments,
-                                     parts[4]))
+                m.addField(Field(parts[0], parts[2], parts[3], comments, prevfield))
+                prevfield = parts[2]
                 break
         comments=[]
+    else:
+        raise ValueError('Line {} malformed'.format(line.rstrip()))
+        
 
 header_template = """#ifndef LIGHTNING_{idem}
 #define LIGHTNING_{idem}