tools/generate-wire.py: handle implicit tlv length fields.

TLVs have an implicit `len` field, so allow expressions containing
that (eg. `len-1`), but assume it means "the remainder of the
message".

This means in most places, f.size() needs an fallback for the
implicit-length case.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

tools/gen/impl_template

@@ -31,9 +31,9 @@ const char *${enum_set['name']}_name(int e)
 <%def name="towire_subtype_field(fieldname, f, ptr)">\
 % if f.is_array() or f.is_varlen():
     % if f.type_obj.has_array_helper():
-towire_${f.type_obj.name}_array(${ptr}, ${fieldname}, ${f.size()});
+towire_${f.type_obj.name}_array(${ptr}, ${fieldname}, ${f.size('tal_count(' + fieldname + ')')});
     % else:
-for (size_t i = 0; i < ${f.size()}; i++)
+for (size_t i = 0; i < ${f.size('tal_count(' + fieldname + ')')}; i++)
 	% if f.type_obj.is_assignable() or f.type_obj.has_len_fields():
 		towire_${f.type_obj.name}(${ptr}, ${fieldname}[i]);
 	% else:
@@ -55,13 +55,17 @@ towire_${f.type_obj.name}(${ptr}, ${'' if f.type_obj.is_assignable() else '&'}${
 	typename += ' *'
 %>\
 % if f.is_varlen():
-${fieldname} = ${f.len_field} ? tal_arr(${ctx}, ${typename}, ${f.len_field}) : NULL;
+${fieldname} = ${f.size('*plen')} ? tal_arr(${ctx}, ${typename}, ${f.size('*plen')}) : NULL;
 % endif
 % if f.is_array() or f.is_varlen():
     % if f.type_obj.has_array_helper():
-fromwire_${type_}_array(cursor, plen, ${fieldname}, ${f.size()});
+fromwire_${type_}_array(cursor, plen, ${fieldname}, ${f.size('*plen')});
     % else:
+        % if f.is_implicit_len():
+for (size_t i = 0; *plen != 0; i++)
+	% else:
 for (size_t i = 0; i < ${f.size()}; i++)
+        % endif
 	% if f.type_obj.is_assignable():
 		(${fieldname})[i] = fromwire_${type_}(cursor, plen);
 	% elif f.is_varlen() and f.type_obj.is_varsize():
@@ -223,9 +227,9 @@ u8 *towire_${msg.name}(const tal_t *ctx${''.join([f.arg_desc_to() for f in msg.f
 	    % endfor
 	% if f.is_array() or f.is_varlen():
 	    % if f.type_obj.has_array_helper():
-	towire_${f.type_obj.name}_array(&p, ${f.name}, ${f.size()});
+	towire_${f.type_obj.name}_array(&p, ${f.name}, ${f.size('tal_count(' + f.name + ')')});
 	    % else:
-	for (size_t i = 0; i < ${f.size()}; i++)
+	for (size_t i = 0; i < ${f.size('tal_count(' + f.name + ')')}; i++)
 		% if f.type_obj.is_assignable() or f.type_obj.is_varsize():
 		towire_${f.type_obj.name}(&p, ${f.name}[i]);
 		% else:
@@ -274,7 +278,7 @@ bool fromwire_${msg.name}(${'const tal_t *ctx, ' if msg.needs_context() else ''}
 	    % endfor
 	% if f.is_varlen():
 	// 2nd case ${f.name}
-	*${f.name} = ${f.len_field} ? tal_arr(ctx, ${typename}, ${f.len_field}) : NULL;
+	*${f.name} = ${f.size('plen')} ? tal_arr(ctx, ${typename}, ${f.size('plen')}) : NULL;
 	% endif
 	% if f.len_field_of:
 	${f.name} = fromwire_${type_}(&cursor, &plen);
@@ -282,9 +286,13 @@ bool fromwire_${msg.name}(${'const tal_t *ctx, ' if msg.needs_context() else ''}
 	fromwire_tlvs(&cursor, &plen, tlvs_${f.type_obj.tlv.name}, ARRAY_SIZE(tlvs_${f.type_obj.tlv.name}), ${f.name});
 	% elif f.is_array() or f.is_varlen():
 	    % if f.type_obj.has_array_helper():
-	fromwire_${type_}_array(&cursor, &plen, ${'*' if f.is_varlen() else ''}${f.name}, ${f.size()});
+	fromwire_${type_}_array(&cursor, &plen, ${'*' if f.is_varlen() else ''}${f.name}, ${f.size('plen')});
 	    % else:
+	        % if f.is_implicit_len():
+	for (size_t i = 0; plen != 0; i++)
+	    	% else:
 	for (size_t i = 0; i < ${f.size()}; i++)
+	        % endif
 		% if not varsized and not f.type_obj.is_assignable():
 		    % if f.is_varlen():			
 		fromwire_${type_}(&cursor, &plen, *${f.name} + i);

tools/gen/print_impl_template

@@ -46,10 +46,14 @@ void print${options.enum_name}_message(const u8 *msg)
 	printwire_tlvs(tal_fmt(NULL, "%s.${f.name}", fieldname), ${cursor}, ${plen}, print_tlvs_${f.type_obj.tlv.name}, ARRAY_SIZE(print_tlvs_${f.type_obj.tlv.name}));
     % elif f.is_array() or f.is_varlen():
 	% if f.type_obj.has_array_helper():
-	printwire_${f.type_obj.name}_array(tal_fmt(NULL, "%s.${f.name}", fieldname), ${cursor}, ${plen}, ${f.size()});
+	printwire_${f.type_obj.name}_array(tal_fmt(NULL, "%s.${f.name}", fieldname), ${cursor}, ${plen}, ${f.size('*' + plen)});
 	% else:
 	printf("[");
+            % if f.is_implicit_len():
+	for (size_t i = 0; i < *${plen}; i++) {
+	    % else:
 	for (size_t i = 0; i < ${f.size()}; i++) {
+	    % endif
 	    % if f.type_obj.is_subtype():
 		printf("{\n");
 		printwire_${f.type_obj.name}(tal_fmt(NULL, "%s.${f.name}", fieldname), ${cursor}, ${plen});

tools/generate-wire.py

@@ -45,6 +45,7 @@ class Field(object):
         self.count = 1
         self.len_field_of = None
         self.len_field = None
+        self.implicit_len = False
 
         self.extension_names = extensions
         self.is_optional = optional
@@ -69,19 +70,30 @@ class Field(object):
         # the len-field caches our name
         len_field.len_field_of = self.name
 
+    def add_implicit_len(self):
+        self.count = False
+        self.implicit_len = True
+
     def is_array(self):
         return self.count > 1
 
     def is_varlen(self):
         return not self.count
 
+    def is_implicit_len(self):
+        return self.implicit_len
+
     def is_extension(self):
         return bool(self.extension_names)
 
-    def size(self):
+    def size(self, implicit_expression=None):
         if self.count:
             return self.count
-        return self.len_field
+        if self.len_field:
+            return self.len_field
+        assert self.is_implicit_len()
+        assert implicit_expression
+        return implicit_expression
 
     def needs_context(self):
         """ A field needs a context if it's varsized """
@@ -122,17 +134,27 @@ class FieldSet(object):
         self.len_fields = {}
 
     def add_data_field(self, field_name, type_obj, count=1,
-                       extensions=[], comments=[], optional=False):
+                       extensions=[], comments=[], optional=False,
+                       implicit_len_ok=False):
         field = Field(field_name, type_obj, extensions=extensions,
                       field_comments=comments, optional=optional)
         if bool(count):
             try:
                 field.add_count(int(count))
             except ValueError:
-                len_field = self.find_data_field(count)
-                field.add_len_field(len_field)
-                self.len_fields[len_field.name] = len_field
-
+                if count in self.fields:
+                    len_field = self.find_data_field(count)
+                    field.add_len_field(len_field)
+                    self.len_fields[len_field.name] = len_field
+                else:
+                    # '...' means "rest of TLV"
+                    assert implicit_len_ok
+                    assert count == '...'
+                    field.add_implicit_len()
+
+        # You can't have any fields after an implicit-length field.
+        if len(self.fields) != 0:
+            assert not self.fields[next(reversed(self.fields))].is_implicit_len()
         self.fields[field_name] = field
 
     def find_data_field(self, field_name):
@@ -537,7 +559,7 @@ def main(options, args=None, output=sys.stdout, lines=None):
                     count = tokens[5]
 
                 msg.add_data_field(tokens[3], type_obj, count, comments=list(comment_set),
-                                   optional=optional)
+                                   optional=optional, implicit_len_ok=True)
                 comment_set = []
             elif token_type == 'msgtype':
                 master.add_message(tokens[1:], comments=list(comment_set))