bolt-gen: handle csv inline comments

The bolts don't have in-line comments, but the internal wire
message CSVs do. This adds the 'comments' back in to the
generated docs.

tools/gen/header_template

@@ -14,10 +14,9 @@ ${i}
 % for enum_set in enum_sets:
 enum ${enum_set['name']} {
     % for msg in enum_set['set']:
-        ## TODO: add back comments for a message
-        ##% for comment in msg.comments:
-        ##/* ${comment} */
-        ##% endfor
+        % for comment in msg.msg_comments:
+        /* ${comment} */
+        % endfor
         ${msg.enum_name()} = ${msg.number},
     % endfor
 };
@@ -32,6 +31,9 @@ const char *${enum_set['name']}_name(int e);
 % for struct in structs:
 struct ${struct.struct_name()} {
         % for f in struct.fields.values():
+            % for c in f.field_comments:
+        /* ${c} */
+            % endfor
         % if bool(f.len_field_of):
 <% continue %>
         % endif
@@ -59,11 +61,17 @@ struct ${tlv.name} {
 /* SUBTYPE: ${subtype.name.upper()} */
 void towire_${subtype.name}(u8 **p, const struct ${subtype.name} *${subtype.name});
 bool fromwire_${subtype.name}(${'const tal_t *ctx, ' if subtype.has_len_fields() else '' }const u8 **cursor, size_t *plen, struct ${subtype.name} *${subtype.name});
+    % for c in subtype.type_comments:
+/* ${c} */
+    % endfor
 
 % endfor
 % endif
 % for msg in messages:
 /* WIRE: ${msg.name.upper()} */
+    % for c in msg.msg_comments:
+/* ${c} */
+    % endfor
 u8 *towire_${msg.name}(const tal_t *ctx${''.join([f.arg_desc_to() for f in msg.fields.values() if not f.is_optional])});
 bool fromwire_${msg.name}(${'const tal_t *ctx, ' if msg.has_len_fields() else ''}const void *p${''.join([f.arg_desc_from() for f in msg.fields.values() if not f.is_optional])});
 

tools/gen/impl_template

@@ -28,6 +28,9 @@ const char *${enum_set['name']}_name(int e)
 ## (shared between here and header_template)
 % for subtype in subtypes:
 /* SUBTYPE: ${subtype.name.upper()} */
+    % for c in subtype.type_comments:
+/* ${c} */
+    % endfor
 <% static = '' if options.expose_subtypes else 'static ' %>
 ${static}void towire_${subtype.name}(u8 **p, const struct ${subtype.name} *${subtype.name})
 {
@@ -37,7 +40,9 @@ ${static}void towire_${subtype.name}(u8 **p, const struct ${subtype.name} *${sub
 
 ## FIXME: abstract this out? (semi-shared with towire_msg, minus the optional bits)
 	% for f in subtype.fields.values():
-## FIXME: add field level comments
+	    % for c in f.field_comments:
+	/* ${c} */
+	    % endfor
 <%
     fieldname = '{}->{}'.format(subtype.name,f.name)
 %> \
@@ -66,8 +71,10 @@ ${static}bool fromwire_${subtype.name}(${'const tal_t *ctx, ' if subtype.has_len
 	${f.type_obj.type_name()} ${f.name};
 	% endfor
 
-## FIXME: field level comments
 	% for f in subtype.fields.values():
+	    % for c in f.field_comments:
+	/* ${c} */
+	    % endfor
 <%
     fieldname = '{}->{}'.format(subtype.name,f.name)
     typename = f.type_obj.type_name()
@@ -105,12 +112,15 @@ ${static}bool fromwire_${subtype.name}(${'const tal_t *ctx, ' if subtype.has_len
 	%endif
 	% endfor
 
-	return cursor != NULL;
+	return ${subtype.name};
 }
 
 % endfor
 % for msg in messages:
 /* WIRE: ${msg.name.upper()} */
+    % for c in msg.msg_comments:
+/* ${c} */
+    % endfor
 u8 *towire_${msg.name}(const tal_t *ctx${''.join([f.arg_desc_to() for f in msg.fields.values() if not f.is_optional])})
 {
 ## FIXME: we're ignoring TLV's rn
@@ -121,7 +131,9 @@ u8 *towire_${msg.name}(const tal_t *ctx${''.join([f.arg_desc_to() for f in msg.f
 
 	towire_u16(&p, ${msg.enum_name()});
 	% for f in msg.fields.values():
-## FIXME: add field level comments
+	    % for c in f.field_comments:
+	/* ${c} */
+	    % endfor
 	% if f.is_array() or f.has_len_field(): ## multiples?
 	    % if f.type_obj.has_array_helper():
 	towire_${f.type_obj.name}_array(&p, ${f.name}, ${f.size()});
@@ -168,6 +180,9 @@ bool fromwire_${msg.name}(${'const tal_t *ctx, ' if msg.has_len_fields() else ''
 		typename = typename + ' *'
 	    type_ = f.type_obj.name
 	%> \
+	    % for c in f.field_comments:
+	/* ${c} */
+	    % endfor
 	% if f.has_len_field():
 	// 2nd case ${f.name}
 	*${f.name} = ${f.len_field} ? tal_arr(ctx, ${typename}, ${f.len_field}) : NULL;
@@ -209,4 +224,3 @@ bool fromwire_${msg.name}(${'const tal_t *ctx, ' if msg.has_len_fields() else ''
 }
 
 % endfor
-##${func_decls}

tools/generate-bolts.py

@@ -33,18 +33,21 @@ def next_line(args, lines):
         lines = fileinput.input(args)
 
     for i, line in enumerate(lines):
+        if not bool(line.strip()):
+            continue
         yield i, line.strip()
 
 
 # Class definitions, to keep things classy
 class Field(object):
-    def __init__(self, name, type_obj, optional=False):
+    def __init__(self, name, type_obj, optional=False, field_comments=[]):
         self.name = name
         self.type_obj = type_obj
         self.count = 1
         self.is_optional = optional
         self.len_field_of = None
         self.len_field = None
+        self.field_comments = field_comments
 
     def add_count(self, count):
         self.count = int(count)
@@ -103,12 +106,12 @@ class FieldSet(object):
         self.optional_fields = False
         self.len_fields = {}
 
-    def add_data_field(self, field_name, type_obj, count=1, is_optional=[]):
+    def add_data_field(self, field_name, type_obj, count=1, is_optional=[], comments=[]):
         # FIXME: use this somewhere?
         if is_optional:
             self.optional_fields = True
 
-        field = Field(field_name, type_obj, bool(is_optional))
+        field = Field(field_name, type_obj, bool(is_optional), comments)
         if bool(count):
             try:
                 field.add_count(int(count))
@@ -198,9 +201,10 @@ class Type(FieldSet):
         self.depends_on = {}
         # FIXME: internal msgs can be enums
         self.is_enum = False
+        self.type_comments = []
 
-    def add_data_field(self, field_name, type_obj, count=1, is_optional=[]):
-        FieldSet.add_data_field(self, field_name, type_obj, count, is_optional)
+    def add_data_field(self, field_name, type_obj, count=1, is_optional=[], comments=[]):
+        FieldSet.add_data_field(self, field_name, type_obj, count, is_optional, comments)
         if type_obj.name not in self.depends_on:
             self.depends_on[type_obj.name] = type_obj
 
@@ -218,17 +222,26 @@ class Type(FieldSet):
         return self.name
 
     def subtype_deps(self):
-        return [dep for dep in self.depends_on.values() if dep.is_subtype()]
+        return [dep for dep in self.depends_on.values() if dep.is_gen_subtype()]
 
-    def is_subtype(self):
+    def is_gen_subtype(self):
+        """ is this a 'genuine' subtype; i.e. will be generated """
         return bool(self.fields)
 
+    def is_subtype(self):
+        return self.is_gen_subtype()
+
     def is_assignable(self):
+        """ Generally typedef's and enums """
         return self.name in self.assignables or self.is_enum
 
+    def add_comments(self, comments):
+        self.type_comments = comments
+
 
 class Message(FieldSet):
-    def __init__(self, name, number, option=[], enum_prefix='wire', struct_prefix=None):
+    def __init__(self, name, number, option=[], enum_prefix='wire',
+                 struct_prefix=None, comments=[]):
         FieldSet.__init__(self)
         self.name = name
         self.number = number
@@ -236,6 +249,7 @@ class Message(FieldSet):
         self.option = option[0] if len(option) else None
         self.struct_prefix = struct_prefix
         self.enumname = None
+        self.msg_comments = comments
 
     def has_option(self):
         return self.option is not None
@@ -255,10 +269,11 @@ class Tlv(object):
         self.name = name
         self.messages = {}
 
-    def add_message(self, tokens):
+    def add_message(self, tokens, comments=[]):
         """ tokens -> (name, value[, option]) """
-        self.messages[tokens[0]] = Message(tokens[0], tokens[1], tokens[2:],
-                                           self.name, self.name)
+        self.messages[tokens[0]] = Message(tokens[0], tokens[1], option=tokens[2:],
+                                           enum_prefix=self.name, struct_prefix=self.name,
+                                           comments=comments)
 
     def find_message(self, name):
         return self.messages[name]
@@ -279,9 +294,10 @@ class Master(object):
             self.tlvs[tlv_name] = Tlv(tlv_name)
         return self.tlvs[tlv_name]
 
-    def add_message(self, tokens):
+    def add_message(self, tokens, comments=[]):
         """ tokens -> (name, value[, option])"""
-        self.messages[tokens[0]] = Message(tokens[0], tokens[1], tokens[2:])
+        self.messages[tokens[0]] = Message(tokens[0], tokens[1], option=tokens[2:],
+                                           comments=comments)
 
     def add_extension_msg(self, name, msg):
         self.extension_msgs[name] = msg
@@ -363,6 +379,8 @@ class Master(object):
 def main(options, args=None, output=sys.stdout, lines=None):
     genline = next_line(args, lines)
 
+    comment_set = []
+
     # Create a new 'master' that serves as the coordinator for the file generation
     master = Master()
     try:
@@ -371,7 +389,10 @@ def main(options, args=None, output=sys.stdout, lines=None):
             tokens = line.split(',')
             token_type = tokens[0]
             if token_type == 'subtype':
-                master.add_type(tokens[1])
+                subtype, _ = master.add_type(tokens[1])
+
+                subtype.add_comments(list(comment_set))
+                comment_set = []
             elif token_type == 'subtypedata':
                 subtype = master.find_type(tokens[1])
                 if not subtype:
@@ -382,10 +403,14 @@ def main(options, args=None, output=sys.stdout, lines=None):
                     count = 1
                 else:
                     count = tokens[4]
-                subtype.add_data_field(tokens[2], type_obj, count)
+
+                subtype.add_data_field(tokens[2], type_obj, count, list(comment_set))
+                comment_set = []
             elif token_type == 'tlvtype':
                 tlv = master.add_tlv(tokens[1])
-                tlv.add_message(tokens[2:])
+                tlv.add_message(tokens[2:], comments=list(comment_set))
+
+                comment_set = []
             elif token_type == 'tlvdata':
                 type_obj, collapse = master.add_type(tokens[4], tokens[3])
                 tlv = master.find_tlv(tokens[1])
@@ -400,9 +425,12 @@ def main(options, args=None, output=sys.stdout, lines=None):
                     count = 1
                 else:
                     count = tokens[5]
-                msg.add_data_field(tokens[3], type_obj, count)
+
+                msg.add_data_field(tokens[3], type_obj, count, list(comment_set))
+                comment_set = []
             elif token_type == 'msgtype':
-                master.add_message(tokens[1:])
+                master.add_message(tokens[1:], comments=list(comment_set))
+                comment_set = []
             elif token_type == 'msgdata':
                 msg = master.find_message(tokens[1])
                 if not msg:
@@ -431,9 +459,13 @@ def main(options, args=None, output=sys.stdout, lines=None):
                     count = 1
                 else:
                     count = tokens[4]
-                msg.add_data_field(tokens[2], type_obj, count)
+
+                msg.add_data_field(tokens[2], type_obj, count, list(comment_set))
+                comment_set = []
             elif token_type.startswith('#include'):
                 master.add_include(token_type)
+            elif token_type.startswith('#'):
+                comment_set.append(token_type[1:])
             else:
                 raise ValueError('Unknown token type {} on line {}:{}'.format(token_type, ln, line))