From 6c240ab589388a6319ff5736bee35944120c681e Mon Sep 17 00:00:00 2001
From: lisa neigut <niftynei@gmail.com>
Date: Sun, 14 Jul 2019 19:10:01 -0500
Subject: [PATCH] bolts: new parsing script and templates for new bolt format

the RFC's extract-format.py is switching to a new format.
this script can correctly parse them.

mostly moves logic over from generate-wire.py, uses a
Python formatting libarary called mako, which needs to be
installed prior to running this script.

you can add it to your system with

    sudo apt-get install python3-mako
---
 tools/gen/header_template       |  71 +++++
 tools/gen/impl_template         | 212 +++++++++++++++
 tools/gen/print_header_template |  20 ++
 tools/gen/print_impl_template   | 109 ++++++++
 tools/generate-bolts.py         | 455 ++++++++++++++++++++++++++++++++
 5 files changed, 867 insertions(+)
 create mode 100644 tools/gen/header_template
 create mode 100644 tools/gen/impl_template
 create mode 100644 tools/gen/print_header_template
 create mode 100644 tools/gen/print_impl_template
 create mode 100755 tools/generate-bolts.py

diff --git a/tools/gen/header_template b/tools/gen/header_template
new file mode 100644
index 000000000..d37c4a77f
--- /dev/null
+++ b/tools/gen/header_template
@@ -0,0 +1,71 @@
+/* This file was generated by generate-bolts.py */
+/* Do not modify this file! Modify the _csv file it was generated from. */
+/* Original template can be found at tools/gen/header_template */
+
+#ifndef LIGHTNING_${idem}
+#define LIGHTNING_${idem}
+#include <ccan/tal/tal.h>
+#include <wire/wire.h>
+% for i in includes:
+${i}
+% endfor
+
+## Enum sets for wire messages & tlvs
+% for enum_set in enum_sets:
+enum ${enum_set['name']} {
+    % for msg in enum_set['set']:
+        ## TODO: add back comments for a message
+        ##% for comment in msg.comments:
+        ##/* ${comment} */
+        ##% endfor
+        ${msg.enum_name()} = ${msg.number},
+    % endfor
+};
+
+%endfor
+## The 'name' functions for the enums
+% for enum_set in enum_sets:
+const char *${enum_set['name']}_name(int e);
+% endfor
+
+## Structs for subtypes + tlv messages
+% for struct in structs:
+struct ${struct.struct_name()} {
+        % for f in struct.fields.values():
+        % if f.is_len_field:
+<% continue %>
+        % endif
+            % if f.has_len_field():
+        ${f.type_obj.type_name()} *${f.name};
+            % elif f.is_array():
+        ${f.type_obj.type_name()} ${f.name}[${f.count}];
+            % else:
+        ${f.type_obj.type_name()} ${f.name};
+            % endif
+        % endfor
+};
+% endfor
+## Structs for TLV types!
+% for tlv in tlvs:
+struct ${tlv.name} {
+    % for msg_name in tlv.messages.keys():
+        struct ${tlv.name}_${msg_name} *${msg_name};
+    % endfor
+};
+% endfor
+
+% if options.expose_subtypes and bool(subtypes):
+% for subtype in subtypes:
+/* SUBTYPE: ${subtype.name.upper()} */
+void towire_${subtype.name}(u8 **p, const struct ${subtype.name} *${subtype.name});
+bool fromwire_${subtype.name}(${'const tal_t *ctx, ' if subtype.has_len_fields() else '' }const u8 **cursor, size_t *plen, struct ${subtype.name} *${subtype.name});
+
+% endfor
+% endif
+% for msg in messages:
+/* WIRE: ${msg.name.upper()} */
+u8 *towire_${msg.name}(const tal_t *ctx${''.join([f.arg_desc_to() for f in msg.fields.values() if not f.is_optional])});
+bool fromwire_${msg.name}(${'const tal_t *ctx, ' if msg.has_len_fields() else ''}const void *p${''.join([f.arg_desc_from() for f in msg.fields.values() if not f.is_optional])});
+
+% endfor
+#endif /* LIGHTNING_${idem} */
diff --git a/tools/gen/impl_template b/tools/gen/impl_template
new file mode 100644
index 000000000..52f61c8ba
--- /dev/null
+++ b/tools/gen/impl_template
@@ -0,0 +1,212 @@
+/* This file was generated by generate-bolts.py */
+/* Do not modify this file! Modify the _csv file it was generated from. */
+/* Original template can be found at tools/gen/impl_template */
+
+#include <${header_filename}>
+#include <ccan/mem/mem.h>
+#include <ccan/tal/str/str.h>
+#include <stdio.h>
+
+% for enum_set in enum_sets:
+const char *${enum_set['name']}_name(int e)
+{
+	static char invalidbuf[sizeof("INVALID ") + STR_MAX_CHARS(e)];
+
+	switch ((enum ${enum_set['name']})e) {
+    % for msg in enum_set['set']:
+	case ${msg.enum_name()}: return "${msg.enum_name()}";
+    % endfor
+	}
+
+	snprintf(invalidbuf, sizeof(invalidbuf), "INVALID %i", e);
+	return invalidbuf;
+}
+
+% endfor
+
+## FIXME: extract out partials for the method declarations
+## (shared between here and header_template)
+% for subtype in subtypes:
+/* SUBTYPE: ${subtype.name.upper()} */
+<% static = '' if options.expose_subtypes else 'static ' %>
+${static}void towire_${subtype.name}(u8 **p, const struct ${subtype.name} *${subtype.name})
+{
+	% for f in subtype.get_len_fields():
+	${f.type_obj.type_name()} ${f.name} = tal_count(${subtype.name}->${f.len_field_of});
+	% endfor
+
+## FIXME: abstract this out? (semi-shared with towire_msg, minus the optional bits)
+	% for f in subtype.fields.values():
+## FIXME: add field level comments
+<%
+    fieldname = '{}->{}'.format(subtype.name,f.name)
+%> \
+	% if f.is_array() or f.has_len_field(): ## multiples?
+	    % if f.type_obj.has_array_helper():
+	towire_${f.type_obj.name}_array(&p, ${fieldname}, ${f.size()});
+	    % else:
+	for (size_t i = 0; i < ${f.size()}; i++)
+		% if f.type_obj.is_assignable() or f.type_obj.has_len_fields():
+		towire_${f.type_obj.name}(p, ${fieldname}[i]);
+		% else:
+		towire_${f.type_obj.name}(p, ${fieldname} + i);
+		% endif
+	    % endif
+	% elif f.len_field_of:
+	towire_${f.type_obj.name}(p, ${f.name});
+	% else:
+	towire_${f.type_obj.name}(&p, ${'' if f.type_obj.is_assignable() else '*'}${fieldname});
+	% endif
+	% endfor
+}
+${static}bool fromwire_${subtype.name}(${'const tal_t *ctx, ' if subtype.has_len_fields() else '' }const u8 **cursor, size_t *plen, struct ${subtype.name} *${subtype.name})
+{
+	## Length field declarations
+	% for f in subtype.get_len_fields():
+	${f.type_obj.type_name()} ${f.name};
+	% endfor
+
+## FIXME: field level comments
+	% for f in subtype.fields.values():
+<%
+    fieldname = '{}->{}'.format(subtype.name,f.name)
+    typename = f.type_obj.type_name()
+    type_ = f.type_obj.name
+%> \
+	% if f.has_len_field():
+	${fieldname} = ${f.len_field} ? tal_arr(ctx, ${typename}${' *' if f.type_obj.has_len_fields() else ''}, ${f.len_field}) : NULL;
+	% endif
+<%
+    if f.is_array():
+	fieldname = '*' + fieldname
+	ctx = 'ctx'
+    else:
+	ctx = fieldname
+%> \
+	% if f.is_array() or f.has_len_field():
+	    % if f.type_obj.has_array_helper():
+	fromwire_${type_}_array(cursor, plen, ${fieldname}, ${f.size()});
+	    % else:
+	for (size_t i = 0; i < ${f.size()}; i++)
+		% if f.type_obj.is_assignable():
+		(${fieldname})[i] = fromwire_${type_}(cursor, plen);
+		% elif f.has_len_field():
+		(${fieldname})[i] = fromwire_${type_}(${ctx}, cursor, plen);
+		% else:
+		fromwire_${type_}(${ctx}, cursor, plen, ${fieldname} + i);
+		% endif
+	    % endif
+	% else:
+	    % if f.type_obj.is_assignable():
+	${ f.name if f.len_field_of else fieldname} = fromwire_${type_}(cursor, plen);
+	    % else:
+	fromwire_${type_}(cursor, plen, &${fieldname});
+	    % endif
+	%endif
+	% endfor
+
+	return cursor != NULL;
+}
+
+% endfor
+% for msg in messages:
+/* WIRE: ${msg.name.upper()} */
+u8 *towire_${msg.name}(const tal_t *ctx${''.join([f.arg_desc_to() for f in msg.fields.values() if not f.is_optional])})
+{
+## FIXME: we're ignoring TLV's rn
+	% for f in msg.get_len_fields():
+	${f.type_obj.type_name()} ${f.name} = tal_count(${f.len_field_of});
+	% endfor
+	u8 *p = tal_arr(ctx, u8, 0);
+
+	towire_u16(&p, ${msg.enum_name()});
+	% for f in msg.fields.values():
+## FIXME: add field level comments
+	% if f.is_array() or f.has_len_field(): ## multiples?
+	    % if f.type_obj.has_array_helper():
+	towire_${f.type_obj.name}_array(&p, ${f.name}, ${f.size()});
+	    % else:
+	for (size_t i = 0; i < ${f.size()}; i++)
+		% if f.type_obj.is_assignable() or f.type_obj.has_len_fields():
+		towire_${f.type_obj.name}(&p, ${f.name}[i]);
+		% else:
+		towire_${f.type_obj.name}(&p, ${f.name} + i);
+		% endif
+	    % endif
+        % elif f.is_optional: ## is optional?
+	if (!${f.name})
+		towire_bool(&p, false);
+	else {
+		towire_bool(&p, true);
+		towire_${f.type_obj.name}(&p, ${'*' if f.type_obj.is_assignable() else ''}${f.name});
+	}
+	% else: ## all other cases
+	towire_${f.type_obj.name}(&p, ${f.name});
+	% endif
+	% endfor
+
+	return memcheck(p, tal_count(p));
+}
+bool fromwire_${msg.name}(${'const tal_t *ctx, ' if msg.has_len_fields() else ''}const void *p${''.join([f.arg_desc_from() for f in msg.fields.values() if not f.is_optional])})
+{
+	% if msg.get_len_fields():
+	    % for f in msg.get_len_fields():
+	${f.type_obj.type_name()} ${f.name};
+	    % endfor
+
+	% endif
+	const u8 *cursor = p;
+	size_t plen = tal_count(p);
+
+	if (fromwire_u16(&cursor, &plen) != ${msg.enum_name()})
+		return false;
+## FIXME: TLV has been omitted
+	% for f in msg.fields.values():
+<%
+	    typename = f.type_obj.type_name()
+	    if f.type_obj.has_len_fields():
+		typename = typename + ' *'
+	    type_ = f.type_obj.name
+	%> \
+	% if f.has_len_field():
+	// 2nd case ${f.name}
+	*${f.name} = ${f.len_field} ? tal_arr(ctx, ${typename}, ${f.len_field}) : NULL;
+	% endif
+	% if f.len_field_of:
+	${f.name} = fromwire_${type_}(&cursor, &plen);
+	% elif f.is_array() or f.has_len_field():
+<%
+	    if f.has_len_field():
+		fieldname = '*' + f.name
+		ctx = fieldname
+	    else:
+		fieldname = f.name
+		ctx = 'ctx'
+	%> \
+	    % if f.type_obj.has_array_helper():
+	fromwire_${type_}_array(&cursor, &plen, ${fieldname}, ${f.size()});
+	    % else:
+	for (size_t i = 0; i < ${f.size()}; i++)
+		% if f.type_obj.is_assignable():
+		(${fieldname})[i] = fromwire_${type_}(&cursor, &plen);
+		## FIXME: case for 'varlen' structs
+		## (${fieldname})[i] = fromwire_${type_}(${ctx}, &cursor, &plen);
+		% else:
+		fromwire_${type_}(${ctx + ', ' if f.type_obj.is_subtype() else ''}&cursor, &plen, ${fieldname} + i);
+		% endif
+	    % endif
+	% else:
+## FIXME: leaves out optional fields + 'varlen' structs
+	    %if f.type_obj.is_assignable():
+	*${f.name} = fromwire_${type_}(&cursor, &plen);
+	    % else:
+	fromwire_${type_}(&cursor, &plen, ${f.name});
+	## assignment
+	    % endif
+	% endif
+	% endfor
+	return cursor != NULL;
+}
+
+% endfor
+##${func_decls}
diff --git a/tools/gen/print_header_template b/tools/gen/print_header_template
new file mode 100644
index 000000000..6b8fa63e7
--- /dev/null
+++ b/tools/gen/print_header_template
@@ -0,0 +1,20 @@
+/* This file was generated by generate-bolts.py */
+/* Do not modify this file! Modify the _csv file it was generated from. */
+/* Template located at tools/gen/print_header_template */
+#ifndef LIGHTNING_${idem}
+#define LIGHTNING_${idem}
+#include <ccan/tal/tal.h>
+#include <devtools/print_wire.h>
+% for i in includes:
+${i}
+% endfor
+
+void print${options.enum_name}_message(const u8 *msg);
+
+void print${options.enum_name}_tlv_message(const char *tlv_name, const u8 *msg);
+
+% for msg in messages:
+void printwire_${msg.name}(const char *fieldname, const u8 *cursor);
+
+% endfor
+#endif /* LIGHTNING_${idem} */
diff --git a/tools/gen/print_impl_template b/tools/gen/print_impl_template
new file mode 100644
index 000000000..7d45b65c4
--- /dev/null
+++ b/tools/gen/print_impl_template
@@ -0,0 +1,109 @@
+/* This file was generated by generate-bolts.py */
+/* Do not modify this file! Modify the _csv file it was generated from. */
+
+#include "${options.header_filename}"
+#include <ccan/mem/mem.h>
+#include <ccan/tal/str/str.h>
+#include <common/utils.h>
+#include <inttypes.h>
+#include <stdio.h>
+
+void print${options.enum_name}_message(const u8 *msg)
+{
+	switch ((enum ${options.enum_name})fromwire_peektype(msg)) {
+% for msg in enum_sets[0]['set']:
+		case ${msg.enum_name()}:
+			printf("${msg.enum_name()}:\n");
+			printwire_${msg.name}("${msg.name}", msg);
+			return;
+% endfor
+	}
+
+	printf("UNKNOWN: %s\\n", tal_hex(msg, msg));
+}
+
+void print${options.enum_name}_tlv_message(const char *tlv_name, const u8 *msg)
+{
+% if not bool(tlvs):
+	printf("~~ No TLV definition found for %s ~~\\n", tlv_name);
+% else:
+    % for tlv in tlvs:
+	if (strcmp(tlv_name, "${tlv.name}") == 0) {
+		printwire_${tlv.name}("${tlv.name}", msg);
+		return;
+	}	
+    % endfor
+	printf("ERR: Unknown TLV message type: %s\n", tlv_name);
+% endif
+}
+
+## 'component' for 'truncate check 
+<%def name="truncate_check(nested=False)">
+	if (!${ '*' if nested else '' }cursor) {
+		printf("**TRUNCATED**\n");
+		return;
+	}
+</%def> \
+## definition for printing field sets
+<%def name="print_fieldset(fields, nested, cursor, plen)">
+## FIXME: optional field handling omitted since we only generate these for bolts rn
+% for f in fields:
+    % if f.len_field_of:
+	${f.type_obj.type_name()} ${f.name} = fromwire_${f.type_obj.name}(${cursor}, ${plen});${truncate_check(nested)} <% continue %> \
+    % endif	
+	printf("${f.name}=");
+    % if f.is_array() or f.has_len_field():
+	% if f.type_obj.has_array_helper():
+	printwire_${f.type_obj.name}_array(tal_fmt(NULL, "%s.${f.name}", fieldname), ${cursor}, ${plen}, ${f.size()});
+	% else:
+	printf("[");
+	for (size_t i = 0; i < ${f.size()}; i++) {
+		${f.type_obj.type_name()} v;
+	    % if f.type_obj.is_assignable():
+		v = fromwire_${f.type_obj.name}(${cursor}, ${plen});
+	    % else:
+		fromwire_${f.type_obj.name}(${cursor}, ${plen}, &v);
+	    % endif
+${truncate_check(nested)} \
+		printwire_${f.type_obj.name}(tal_fmt(NULL, "%s.${f.name}", fieldname), &v);
+	}
+	printf("]");
+	% endif
+${truncate_check(nested)} \
+    % else:
+	    % if f.type_obj.is_assignable():
+	${f.type_obj.type_name()} ${f.name} = fromwire_${f.type_obj.name}(${cursor}, ${plen});
+	    % else:
+	${f.type_obj.type_name()} ${f.name};
+	fromwire_${f.type_obj.name}(${cursor}, ${plen}, &${f.name});
+	    % endif
+	printwire_${f.type_obj.name}(tal_fmt(NULL, "%s.${f.name}", fieldname), &${f.name}); ${truncate_check(nested)} \
+    % endif
+% endfor
+</%def> \
+
+## Definitions for 'subtypes'
+% for subtype in subtypes:
+static void printwire_${subtype.name}(const char *fieldname, const u9 **cursor, size_t *plen)
+{
+	${print_fieldset(subtype.fields.values(), True, 'cursor', 'plen')}
+}
+% endfor
+
+## FIXME: handling for tlv's :/
+% for msg in messages:
+void printwire_${msg.name}(const char *fieldname, const u8 *cursor)
+{
+
+	size_t plen = tal_count(cursor);
+	if (fromwire_u16(&cursor, &plen) != ${msg.enum_name()}) {
+		printf("WRONG TYPE?!\n");
+		return;
+	}
+${print_fieldset(msg.fields.values(), False, '&cursor', '&plen')}
+
+## Length check
+	if (plen != 0)
+		printf("EXTRA: %s\n", tal_hexstr(NULL, cursor, plen));
+}
+% endfor
diff --git a/tools/generate-bolts.py b/tools/generate-bolts.py
new file mode 100755
index 000000000..2bf476adf
--- /dev/null
+++ b/tools/generate-bolts.py
@@ -0,0 +1,455 @@
+#! /usr/bin/env python3
+# Script to parse spec output CSVs and produce C files.
+# Released by lisa neigut under CC0:
+# https://creativecommons.org/publicdomain/zero/1.0/
+#
+# Reads from stdin, outputs C header or body file.
+#
+# Standard message types:
+#   msgtype,<msgname>,<value>[,<option>]
+#   msgdata,<msgname>,<fieldname>,<typename>,[<count>][,<option>]
+#
+# TLV types:
+#   tlvtype,<tlvstreamname>,<tlvname>,<value>[,<option>]
+#   tlvdata,<tlvstreamname>,<tlvname>,<fieldname>,<typename>,[<count>][,<option>]
+#
+# Subtypes:
+#   subtype,<subtypename>
+#   subtypedata,<subtypename>,<fieldname>,<typename>,[<count>]
+
+from argparse import ArgumentParser, REMAINDER
+import copy
+import fileinput
+from mako.template import Template
+import os
+import re
+import sys
+
+
+# Generator to give us one line at a time.
+def next_line(args, lines):
+    if lines is None:
+        lines = fileinput.input(args)
+
+    for i, line in enumerate(lines):
+        yield i, line.strip()
+
+
+# Class definitions, to keep things classy
+class Field(object):
+    def __init__(self, name, type_obj, optional=False):
+        self.name = name
+        self.type_obj = type_obj
+        self.count = 1
+        self.is_optional = optional
+        self.len_field_of = None
+        self.len_field = None
+
+    def add_count(self, count):
+        self.count = int(count)
+
+    def add_len_field(self, len_field):
+        self.count = False
+        # we cache our len-field's name
+        self.len_field = len_field.name
+        # the len-field caches our name
+        len_field.len_field_of = self.name
+
+    def is_array(self):
+        return self.count > 1
+
+    def has_len_field(self):
+        return not self.count
+
+    def is_optional(self):
+        return self.is_optional
+
+    def size(self):
+        if self.count:
+            return self.count
+        return self.len_field
+
+    def arg_desc_to(self):
+        if self.len_field_of:
+            return ''
+        type_name = self.type_obj.type_name()
+        if self.is_array():
+            return ', const {} {}[{}]'.format(type_name, self.name, self.count)
+        if self.type_obj.is_assignable() and not self.has_len_field():
+            return ', {} {}'.format(type_name, self.name)
+        # Are we a variable number of objects with a variable number of things?
+        if self.has_len_field() and self.type_obj.has_len_fields():
+            return ', {} **{}'.format(type_name, self.name)
+        return ', const {} *{}'.format(type_name, self.name)
+
+    def arg_desc_from(self):
+        if self.len_field_of:
+            return ''
+        type_name = self.type_obj.type_name()
+        if self.is_array():
+            return ', {} {}[{}]'.format(type_name, self.name, self.count)
+        ptrs = '*'
+        if self.has_len_field():
+            ptrs += '*'
+        if self.is_optional or self.type_obj.has_len_fields():
+            ptrs += '*'
+        return ', {} {}{}'.format(type_name, ptrs, self.name)
+
+
+class FieldSet(object):
+    def __init__(self):
+        self.fields = {}
+        self.optional_fields = False
+        self.len_fields = {}
+
+    def add_data_field(self, field_name, type_obj, count=1, is_optional=[]):
+        # FIXME: use this somewhere?
+        if is_optional:
+            self.optional_fields = True
+
+        field = Field(field_name, type_obj, bool(is_optional))
+        if bool(count):
+            try:
+                field.add_count(int(count))
+            except ValueError:
+                len_field = self.find_data_field(count)
+                if not len_field:
+                    raise ValueError("No length field found with name {} for {}:{}"
+                                     .format(count, self.name, field_name))
+                field.add_len_field(len_field)
+                self.len_fields[len_field.name] = len_field
+
+        self.fields[field_name] = field
+
+    def find_data_field(self, field_name):
+        return self.fields[field_name]
+
+    def get_len_fields(self):
+        return list(self.len_fields.values())
+
+    def has_len_fields(self):
+        return bool(self.len_fields)
+
+
+class Type(FieldSet):
+    assignables = [
+        'u8',
+        'u16',
+        'u32',
+        'u64',
+        'bool',
+        'amount_sat',
+        'amount_msat',
+        # FIXME: omits var_int
+    ]
+
+    typedefs = [
+        'u8',
+        'u16',
+        'u32',
+        'u64',
+        'bool',
+        'secp256k1_ecdsa_signature',
+    ]
+
+    # Some BOLT types are re-typed based on their field name
+    # ('fieldname partial', 'original type'): ('true type', 'collapse array?')
+    name_field_map = {
+        ('txid', 'sha256'): ('bitcoin_txid', False),
+        ('amt', 'u64'): ('amount_msat', False),
+        ('msat', 'u64'): ('amount_msat', False),
+        ('satoshis', 'u64'): ('amount_sat', False),
+        ('node_id', 'pubkey'): ('node_id', False),
+        ('temporary_channel_id', 'u8'): ('channel_id', True),
+    }
+
+    # For BOLT specified types, a few type names need to be simply 'remapped'
+    # 'original type': 'true type'
+    name_remap = {
+        'byte': 'u8',
+        'signature': 'secp256k1_ecdsa_signature',
+        'chain_hash': 'bitcoin_blkid',
+        'point': 'pubkey',
+        # FIXME: omits 'pad'
+    }
+
+    @staticmethod
+    def true_type(type_name, field_name=None):
+        """ Returns 'true' type of a given type and a flag if
+            we've remapped a variable size/array type to a single struct
+            (an example of this is 'temporary_channel_id' which is specified
+            as a 32*byte, but we re-map it to a channel_id
+        """
+        if type_name in Type.name_remap:
+            type_name = Type.name_remap[type_name]
+
+        if field_name:
+            for (partial, t), true_type in Type.name_field_map.items():
+                if partial in field_name and t == type_name:
+                    return true_type
+        return (type_name, False)
+
+    def __init__(self, name):
+        FieldSet.__init__(self)
+        self.name = name
+        self.depends_on = {}
+        # FIXME: internal msgs can be enums
+        self.is_enum = False
+
+    def add_data_field(self, field_name, type_obj, count=1, is_optional=[]):
+        FieldSet.add_data_field(self, field_name, type_obj, count, is_optional)
+        if type_obj.name not in self.depends_on:
+            self.depends_on[type_obj.name] = type_obj
+
+    def type_name(self):
+        if self.name in self.typedefs:
+            return self.name
+        prefix = 'enum ' if self.is_enum else 'struct '
+        return prefix + self.name
+
+    # We only accelerate the u8 case: it's common and trivial.
+    def has_array_helper(self):
+        return self.name in ['u8']
+
+    def struct_name(self):
+        return self.name
+
+    def subtype_deps(self):
+        return [dep for dep in self.depends_on.values() if dep.is_subtype()]
+
+    def is_subtype(self):
+        return bool(self.fields)
+
+    def is_assignable(self):
+        return self.name in self.assignables or self.is_enum
+
+
+class Message(FieldSet):
+    def __init__(self, name, number, option=[], enum_prefix='wire', struct_prefix=None):
+        FieldSet.__init__(self)
+        self.name = name
+        self.number = number
+        self.enum_prefix = enum_prefix
+        self.option = option[0] if len(option) else None
+        self.struct_prefix = struct_prefix
+        self.enumname = None
+
+    def has_option(self):
+        return self.option is not None
+
+    def enum_name(self):
+        name = self.enumname if self.enumname else self.name
+        return "{}_{}".format(self.enum_prefix, name).upper()
+
+    def struct_name(self):
+        if self.struct_prefix:
+            return self.struct_prefix + "_" + self.name
+        return self.name
+
+
+class Tlv(object):
+    def __init__(self, name):
+        self.name = name
+        self.messages = {}
+
+    def add_message(self, tokens):
+        """ tokens -> (name, value[, option]) """
+        self.messages[tokens[0]] = Message(tokens[0], tokens[1], tokens[2:],
+                                           self.name, self.name)
+
+    def find_message(self, name):
+        return self.messages[name]
+
+
+class Master(object):
+    types = {}
+    tlvs = {}
+    messages = {}
+    extension_msgs = {}
+    inclusions = []
+
+    def add_include(self, inclusion):
+        self.inclusions.append(inclusion)
+
+    def add_tlv(self, tlv_name):
+        if tlv_name not in self.tlvs:
+            self.tlvs[tlv_name] = Tlv(tlv_name)
+        return self.tlvs[tlv_name]
+
+    def add_message(self, tokens):
+        """ tokens -> (name, value[, option])"""
+        self.messages[tokens[0]] = Message(tokens[0], tokens[1], tokens[2:])
+
+    def add_extension_msg(self, name, msg):
+        self.extension_msgs[name] = msg
+
+    def add_type(self, type_name, field_name=None):
+        # Check for special type name re-mapping
+        type_name, collapse_original = Type.true_type(type_name, field_name)
+
+        if type_name not in self.types:
+            self.types[type_name] = Type(type_name)
+        return self.types[type_name], collapse_original
+
+    def find_type(self, type_name):
+        return self.types[type_name]
+
+    def find_message(self, msg_name):
+        if msg_name in self.messages:
+            return self.messages[msg_name]
+        if msg_name in self.extension_msgs:
+            return self.extension_msgs[msg_name]
+        return None
+
+    def find_tlv(self, tlv_name):
+        return self.tlvs[tlv_name]
+
+    def get_ordered_subtypes(self):
+        """ We want to order subtypes such that the 'no dependency'
+        types are printed first """
+        subtypes = [s for s in self.types.values() if s.is_subtype()]
+
+        # Start with subtypes without subtype dependencies
+        sorted_types = [s for s in subtypes if not len(s.subtype_deps())]
+        unsorted = [s for s in subtypes if len(s.subtype_deps())]
+        while len(unsorted):
+            names = [s.name for s in sorted_types]
+            for s in list(unsorted):
+                if all([dependency.name in names for dependency in s.subtype_deps()]):
+                    sorted_types.append(s)
+                    unsorted.remove(s)
+        return sorted_types
+
+    def tlv_messages(self):
+        return [m for tlv in self.tlvs.values() for m in tlv.messages.values()]
+
+    def find_template(self, options):
+        dirpath = os.path.dirname(os.path.abspath(__file__))
+        filename = dirpath + '/gen/{}{}_template'.format(
+            'print_' if options.print_wire else '', options.page)
+
+        return Template(filename=filename)
+
+    def write(self, options, output):
+        template = self.find_template(options)
+        enum_sets = []
+        enum_sets.append({
+            'name': options.enum_name,
+            'set': self.messages.values(),
+        })
+        for tlv in self.tlvs.values():
+            enum_sets.append({
+                'name': tlv.name,
+                'set': tlv.messages.values(),
+            })
+        stuff = {}
+        stuff['options'] = options
+        stuff['idem'] = re.sub(r'[^A-Z]+', '_', options.header_filename.upper())
+        stuff['header_filename'] = options.header_filename
+        stuff['includes'] = self.inclusions
+        stuff['enum_sets'] = enum_sets
+        subtypes = self.get_ordered_subtypes()
+        stuff['structs'] = subtypes + self.tlv_messages()
+        stuff['tlvs'] = self.tlvs.values()
+        stuff['messages'] = list(self.messages.values()) + list(self.extension_msgs.values())
+        stuff['subtypes'] = subtypes
+
+        print(template.render(**stuff), file=output)
+
+
+def main(options, args=None, output=sys.stdout, lines=None):
+    genline = next_line(args, lines)
+
+    # Create a new 'master' that serves as the coordinator for the file generation
+    master = Master()
+    try:
+        while True:
+            ln, line = next(genline)
+            tokens = line.split(',')
+            token_type = tokens[0]
+            if token_type == 'subtype':
+                master.add_type(tokens[1])
+            elif token_type == 'subtypedata':
+                subtype = master.find_type(tokens[1])
+                if not subtype:
+                    raise ValueError('Unknown subtype {} for data.\nat {}:{}'
+                                     .format(tokens[1], ln, line))
+                type_obj, collapse = master.add_type(tokens[3], tokens[2])
+                if collapse:
+                    count = 1
+                else:
+                    count = tokens[4]
+                subtype.add_data_field(tokens[2], type_obj, count)
+            elif token_type == 'tlvtype':
+                tlv = master.add_tlv(tokens[1])
+                tlv.add_message(tokens[2:])
+            elif token_type == 'tlvdata':
+                type_obj, collapse = master.add_type(tokens[4], tokens[3])
+                tlv = master.find_tlv(tokens[1])
+                if not tlv:
+                    raise ValueError('tlvdata for unknown tlv {}.\nat {}:{}'
+                                     .format(tokens[1], ln, line))
+                msg = tlv.find_message(tokens[2])
+                if not msg:
+                    raise ValueError('tlvdata for unknown tlv-message {}.\nat {}:{}'
+                                     .format(tokens[2], ln, line))
+                if collapse:
+                    count = 1
+                else:
+                    count = tokens[5]
+                msg.add_data_field(tokens[3], type_obj, count)
+            elif token_type == 'msgtype':
+                master.add_message(tokens[1:])
+            elif token_type == 'msgdata':
+                msg = master.find_message(tokens[1])
+                if not msg:
+                    raise ValueError('Unknown message type {}. {}:{}'.format(tokens[1], ln, line))
+                type_obj, collapse = master.add_type(tokens[3], tokens[2])
+
+                # if this is an 'extension' field*, we want to add a new 'message' type
+                # in the future, extensions will be handled as TLV's
+                #
+                # *(in the spec they're called 'optional', but that term is overloaded
+                #   in that internal wire messages have 'optional' fields that are treated
+                #   differently. for the sake of clarity here, for bolt-wire messages,
+                #   we'll refer to 'optional' message fields as 'extensions')
+                #
+                if bool(tokens[5:]):  # is an extension field
+                    extension_name = "{}_{}".format(tokens[1], tokens[5])
+                    orig_msg = msg
+                    msg = master.find_message(extension_name)
+                    if not msg:
+                        msg = copy.deepcopy(orig_msg)
+                        msg.enumname = msg.name
+                        msg.name = extension_name
+                        master.add_extension_msg(msg.name, msg)
+
+                if collapse:
+                    count = 1
+                else:
+                    count = tokens[4]
+                msg.add_data_field(tokens[2], type_obj, count)
+            elif token_type.startswith('#include'):
+                master.add_include(token_type)
+            else:
+                raise ValueError('Unknown token type {} on line {}:{}'.format(token_type, ln, line))
+
+    except StopIteration:
+        pass
+
+    master.write(options, output)
+
+
+if __name__ == "__main__":
+    parser = ArgumentParser()
+    parser.add_argument("-s", "--expose-subtypes", help="print subtypes in header",
+                        action="store_true", default=False)
+    parser.add_argument("-P", "--print_wire", help="generate wire printing source files",
+                        action="store_true", default=False)
+    parser.add_argument("--page", choices=['header', 'impl'], help="page to print")
+    parser.add_argument('header_filename', help='The filename of the header')
+    parser.add_argument('enum_name', help='The name of the enum to produce')
+    parser.add_argument("files", help='Files to read in (or stdin)', nargs=REMAINDER)
+    parsed_args = parser.parse_args()
+
+    main(parsed_args, parsed_args.files)