diff --git a/tools/gen/header_template b/tools/gen/header_template index e03168d64..112cf8aba 100644 --- a/tools/gen/header_template +++ b/tools/gen/header_template @@ -5,6 +5,7 @@ #ifndef LIGHTNING_${idem} #define LIGHTNING_${idem} #include +#include #include % for i in includes: ${i} @@ -49,15 +50,18 @@ struct ${struct.struct_name()} { % endfor }; % endfor -## Structs for TLV types! -% for tlv in tlvs: -struct ${tlv.name} { - % for msg_name in tlv.messages.keys(): - struct ${tlv.name}_${msg_name} *${msg_name}; +## Structs for TLVs +% for tlv in tlvs.values(): +struct ${tlv.struct_name()} { + % for msg in tlv.messages.values(): + struct ${msg.struct_name()} *${msg.name}; % endfor }; % endfor +% for tlv in tlvs.values(): +struct ${tlv.struct_name()} *${tlv.struct_name()}_new(const tal_t *ctx); +% endfor % if options.expose_subtypes and bool(subtypes): % for subtype in subtypes: /* SUBTYPE: ${subtype.name.upper()} */ diff --git a/tools/gen/impl_template b/tools/gen/impl_template index 518078615..3853e2155 100644 --- a/tools/gen/impl_template +++ b/tools/gen/impl_template @@ -3,6 +3,7 @@ /* Original template can be found at tools/gen/impl_template */ #include <${header_filename}> +#include #include #include #include @@ -25,9 +26,62 @@ const char *${enum_set['name']}_name(int e) return invalidbuf; } % endfor +## START PARTIALS +## Subtype and TLV-msg towire_ +<%def name="towire_subtype_field(fieldname, f, ptr)">\ +% if f.is_array() or f.is_varlen(): + % if f.type_obj.has_array_helper(): +towire_${f.type_obj.name}_array(${ptr}, ${fieldname}, ${f.size()}); + % else: +for (size_t i = 0; i < ${f.size()}; i++) + % if f.type_obj.is_assignable() or f.type_obj.has_len_fields(): + towire_${f.type_obj.name}(${ptr}, ${fieldname}[i]); + % else: + towire_${f.type_obj.name}(${ptr}, ${fieldname} + i); + % endif + % endif +% elif f.len_field_of: +towire_${f.type_obj.name}(${ptr}, ${f.name}); +% else: +towire_${f.type_obj.name}(${ptr}, ${'' if f.type_obj.is_assignable() else '&'}${fieldname}); +% endif + +## Subtype and TLV-msg fromwire_ +<%def name="fromwire_subtype_field(fieldname, f, ctx)">\ +<% + type_ = f.type_obj.name + typename = f.type_obj.type_name() +%>\ +% if f.is_varlen(): +${'*' if f.type_obj.is_varsize() else ''}${fieldname} = ${f.len_field} ? tal_arr(${ctx}, ${typename}, ${f.len_field}) : NULL; +% endif +% if f.is_array() or f.is_varlen(): + % if f.type_obj.has_array_helper(): +fromwire_${type_}_array(cursor, plen, ${fieldname}, ${f.size()}); + % else: +for (size_t i = 0; i < ${f.size()}; i++) + % if f.type_obj.is_assignable(): + (${fieldname})[i] = fromwire_${type_}(cursor, plen); + % elif f.is_varlen() and f.type_obj.is_varsize(): + (${fieldname})[i] = fromwire_${type_}(${ctx}, cursor, plen); + % else: + fromwire_${type_}(cursor, plen, ${fieldname} + i); + % endif + % endif +% else: + % if f.type_obj.is_assignable(): +${ f.name if f.len_field_of else fieldname} = fromwire_${type_}(cursor, plen); + % elif f.type_obj.is_varsize(): +${fieldname} = *fromwire_${type_}(${ctx}, cursor, plen); + % else: +fromwire_${type_}(cursor, plen, &${fieldname}); + % endif +%endif + +## END PARTIALS ## FIXME: extract out partials for the method declarations ## (shared between here and header_template) -% for subtype in subtypes: +% for subtype in subtypes: ## START Subtypes /* SUBTYPE: ${subtype.name.upper()} */ % for c in subtype.type_comments: @@ -49,22 +103,7 @@ ${static}void towire_${subtype.name}(u8 **p, const ${subtype.type_name()} *${sub <% fieldname = '{}->{}'.format(subtype.name,f.name) %>\ - % if f.is_array() or f.is_varlen(): - % if f.type_obj.has_array_helper(): - towire_${f.type_obj.name}_array(p, ${fieldname}, ${f.size()}); - % else: - for (size_t i = 0; i < ${f.size()}; i++) - % if f.type_obj.is_assignable() or f.type_obj.has_len_fields(): - towire_${f.type_obj.name}(p, ${fieldname}[i]); - % else: - towire_${f.type_obj.name}(p, ${fieldname} + i); - % endif - % endif - % elif f.len_field_of: - towire_${f.type_obj.name}(p, ${f.name}); - % else: - towire_${f.type_obj.name}(p, ${'' if f.type_obj.is_assignable() else '&'}${fieldname}); - % endif + ${towire_subtype_field(fieldname, f, 'p')}\ % endfor } % if subtype.is_varsize(): @@ -88,41 +127,9 @@ ${static}void fromwire_${subtype.name}(${'const tal_t *ctx, ' if subtype.needs_c % endfor <% fieldname = '{}->{}'.format(subtype.name,f.name) - ctx = fieldname - if f.is_array(): - fieldname = '*' + fieldname - if f.type_obj.is_varsize(): - typename += ' *' - type_ = f.type_obj.name - typename = f.type_obj.type_name() + ctx = subtype.name %> \ - % if f.is_varlen(): - ${'*' if f.type_obj.is_varsize() else ''}${fieldname} = ${f.len_field} ? tal_arr(${subtype.name}, ${typename}, ${f.len_field}) : NULL; - % endif - % if f.is_array() or f.is_varlen(): - % if f.type_obj.has_array_helper(): - fromwire_${type_}_array(cursor, plen, ${fieldname}, ${f.size()}); - % else: - for (size_t i = 0; i < ${f.size()}; i++) - % if f.type_obj.is_assignable(): - (${fieldname})[i] = fromwire_${type_}(cursor, plen); - % elif f.is_varlen() and f.type_obj.is_varsize(): - (${fieldname})[i] = fromwire_${type_}(${ctx}, cursor, plen); - % elif f.is_varlen(): - fromwire_${type_}(cursor, plen, ${fieldname} + i); - % else: - fromwire_${type_}(${ctx}, cursor, plen, ${fieldname} + i); - % endif - % endif - % else: - % if f.type_obj.is_assignable(): - ${ f.name if f.len_field_of else fieldname} = fromwire_${type_}(cursor, plen); - % elif f.type_obj.is_varsize(): - ${fieldname} = *fromwire_${type_}(ctx, cursor, plen); - % else: - fromwire_${type_}(cursor, plen, &${fieldname}); - % endif - %endif + ${fromwire_subtype_field(fieldname, f, ctx)}\ % endfor % if subtype.is_varsize(): @@ -139,7 +146,61 @@ ${static}void fromwire_${subtype.name}(${'const tal_t *ctx, ' if subtype.needs_c fromwire_${type_}(${'ctx, ' if f.needs_context() else ''}&cursor, &plen, ${'*' if f.is_optional else ''}${f.name}); % endif -% for msg in messages: +% for tlv in tlvs.values(): ## START TLV's + +struct ${tlv.struct_name()} *${tlv.struct_name()}_new(const tal_t *ctx) +{ + /* Initialize everything to NULL. (Quiet, C pedants!) */ + return talz(ctx, struct ${tlv.struct_name()}); +} + + % for msg in tlv.messages.values(): +/* ${tlv.name.upper()} MSG: ${msg.name} */ +static u8 *towire_${msg.struct_name()}(const tal_t *ctx, const void *vrecord) +{ + const struct ${tlv.struct_name()} *r = vrecord; + u8 *ptr; + + if (!r->${msg.name}) + return NULL; + + % for f in msg.get_len_fields(): + ${f.type_obj.type_name()} ${f.name} = tal_count(r->${msg.name}->${f.len_field_of}); + % endfor + + ptr = tal_arr(ctx, u8, 0); + % for f in msg.fields.values(): +<% fieldname = 'r->{}->{}'.format(msg.name, f.name) %>\ + ${towire_subtype_field(fieldname, f, '&ptr')}\ + % endfor + return ptr; +} +static void fromwire_${msg.struct_name()}(const u8 **cursor, size_t *plen, void *vrecord) +{ + struct ${tlv.struct_name()} *r = vrecord; + ## Length field declarations + % for f in msg.get_len_fields(): + ${f.type_obj.type_name()} ${f.name}; + % endfor + + r->${msg.name} = tal(r, struct ${msg.struct_name()}); + % for f in msg.fields.values(): +<% + fieldname = 'r->{}->{}'.format(msg.name, f.name) + ctx = 'r->{}'.format(msg.name) +%>\ + ${fromwire_subtype_field(fieldname, f, ctx)}\ + % endfor +} + % endfor + +static const struct tlv_record_type tlvs_${tlv.name}[] = { + % for msg in tlv.messages.values(): + { ${msg.number}, towire_${msg.struct_name()}, fromwire_${msg.struct_name()} }, + % endfor +}; +% endfor ## END TLV's +% for msg in messages: ## START Wire Messages /* WIRE: ${msg.name.upper()} */ % for c in msg.msg_comments: @@ -169,6 +230,8 @@ u8 *towire_${msg.name}(const tal_t *ctx${''.join([f.arg_desc_to() for f in msg.f towire_${f.type_obj.name}(&p, ${f.name} + i); % endif % endif + % elif f.type_obj.is_tlv(): + towire_tlvs(&p, tlvs_${f.type_obj.tlv.name}, ARRAY_SIZE(tlvs_${f.type_obj.tlv.name}), ${f.name}); % elif f.is_optional: ## is optional? if (!${f.name}) towire_bool(&p, false); @@ -196,7 +259,6 @@ bool fromwire_${msg.name}(${'const tal_t *ctx, ' if msg.needs_context() else ''} if (fromwire_u16(&cursor, &plen) != ${msg.enum_name()}) return false; -## FIXME: TLV has been omitted % for f in msg.fields.values(): <% typename = f.type_obj.type_name() @@ -214,6 +276,8 @@ bool fromwire_${msg.name}(${'const tal_t *ctx, ' if msg.needs_context() else ''} % endif % if f.len_field_of: ${f.name} = fromwire_${type_}(&cursor, &plen); + % elif f.type_obj.is_tlv(): + fromwire_tlvs(&cursor, &plen, tlvs_${f.type_obj.tlv.name}, ARRAY_SIZE(tlvs_${f.type_obj.tlv.name}), ${f.name}); % elif f.is_array() or f.is_varlen(): % if f.type_obj.has_array_helper(): fromwire_${type_}_array(&cursor, &plen, ${'*' if f.is_varlen() else ''}${f.name}, ${f.size()}); @@ -246,4 +310,4 @@ bool fromwire_${msg.name}(${'const tal_t *ctx, ' if msg.needs_context() else ''} % endfor return cursor != NULL; } -% endfor +% endfor ## END Wire Messages diff --git a/tools/generate-bolts.py b/tools/generate-bolts.py index dc4e42657..be50eb1de 100755 --- a/tools/generate-bolts.py +++ b/tools/generate-bolts.py @@ -226,6 +226,7 @@ class Type(FieldSet): # FIXME: internal msgs can be enums self.is_enum = False self.type_comments = [] + self.tlv = False def add_data_field(self, field_name, type_obj, count=1, is_extension=[], comments=[], optional=False): @@ -237,14 +238,20 @@ class Type(FieldSet): def type_name(self): if self.name in self.typedefs: return self.name - prefix = 'enum ' if self.is_enum else 'struct ' - return prefix + self.name + if self.is_enum: + prefix = 'enum ' + else: + prefix = 'struct ' + + return prefix + self.struct_name() # We only accelerate the u8 case: it's common and trivial. def has_array_helper(self): return self.name in ['u8'] def struct_name(self): + if self.is_tlv(): + return self.tlv.struct_name() return self.name def subtype_deps(self): @@ -268,6 +275,12 @@ class Type(FieldSet): def add_comments(self, comments): self.type_comments = comments + def mark_tlv(self, tlv): + self.tlv = tlv + + def is_tlv(self): + return bool(self.tlv) + class Message(FieldSet): def __init__(self, name, number, option=[], enum_prefix='wire', @@ -302,9 +315,13 @@ class Tlv(object): def add_message(self, tokens, comments=[]): """ tokens -> (name, value[, option]) """ self.messages[tokens[0]] = Message(tokens[0], tokens[1], option=tokens[2:], - enum_prefix=self.name, struct_prefix=self.name, + enum_prefix=self.name, + struct_prefix=self.struct_name(), comments=comments) + def struct_name(self): + return "tlv_{}".format(self.name) + def find_message(self, name): return self.messages[name] @@ -326,6 +343,10 @@ class Master(object): def add_tlv(self, tlv_name): if tlv_name not in self.tlvs: self.tlvs[tlv_name] = Tlv(tlv_name) + + if tlv_name not in self.types: + self.types[tlv_name] = Type(tlv_name) + return self.tlvs[tlv_name] def add_message(self, tokens, comments=[]): @@ -387,6 +408,13 @@ class Master(object): return Template(filename=filename) + def post_process(self): + """ method to handle any 'post processing' that needs to be done. + for now, we just need match up types to TLVs """ + for tlv_name, tlv in self.tlvs.items(): + if tlv_name in self.types: + self.types[tlv_name].mark_tlv(tlv) + def write(self, options, output): template = self.find_template(options) enum_sets = [] @@ -394,11 +422,6 @@ class Master(object): 'name': options.enum_name, 'set': self.messages.values(), }) - for tlv in self.tlvs.values(): - enum_sets.append({ - 'name': tlv.name, - 'set': tlv.messages.values(), - }) stuff = {} stuff['top_comments'] = self.top_comments stuff['options'] = options @@ -408,7 +431,7 @@ class Master(object): stuff['enum_sets'] = enum_sets subtypes = self.get_ordered_subtypes() stuff['structs'] = subtypes + self.tlv_messages() - stuff['tlvs'] = self.tlvs.values() + stuff['tlvs'] = self.tlvs stuff['messages'] = list(self.messages.values()) + list(self.extension_msgs.values()) stuff['subtypes'] = subtypes @@ -527,6 +550,7 @@ def main(options, args=None, output=sys.stdout, lines=None): except StopIteration: pass + master.post_process() master.write(options, output) diff --git a/tools/test/Makefile b/tools/test/Makefile index 9ab911a01..82e8bb139 100644 --- a/tools/test/Makefile +++ b/tools/test/Makefile @@ -20,7 +20,7 @@ TOOLS_WIRE_DEPS := $(BOLT_DEPS) tools/test/test_cases $(wildcard tools/gen/*_tem $(TOOL_TEST_SRC): $(TOOL_GEN_HEADER) $(TOOL_TEST_OBJS): $(TOOL_GEN_SRC) -$(TOOL_TEST_PROGRAMS): $(TOOL_TEST_COMMON_OBJS) +$(TOOL_TEST_PROGRAMS): $(TOOL_TEST_COMMON_OBJS) tools/test/gen_test.o $(TOOL_GEN_SRC) $(TOOL_GEN_HEADER): $(TOOLS_WIRE_DEPS) tools/test/gen_test.h: diff --git a/tools/test/run-test-wire.c b/tools/test/run-test-wire.c index f43d7c413..25041ec9e 100644 --- a/tools/test/run-test-wire.c +++ b/tools/test/run-test-wire.c @@ -1,12 +1,25 @@ -#include "gen_test.c" +#include "gen_test.h" #include #include #include +/* AUTOGENERATED MOCKS START */ +/* AUTOGENERATED MOCKS END */ + int main(void) { setup_locale(); + void *ctx = tal(NULL, char); + + struct tlv_n1 *n1 = tlv_n1_new(ctx); + struct tlv_n2 *n2 = tlv_n2_new(ctx); + struct tlv_n3 *n3 = tlv_n3_new(ctx); + + assert(n1); + assert(n2); + assert(n3); + tal_free(ctx); } diff --git a/tools/test/test_cases b/tools/test/test_cases index 29f9de681..e5dd3cf42 100644 --- a/tools/test/test_cases +++ b/tools/test/test_cases @@ -39,6 +39,17 @@ msgdata,test_msg,test_sbt_varsize_struct,subtype_varsize_struct, msgdata,test_msg,test_sbt_varsize_var_assign,subtype_var_assign, msgdata,test_msg,test_sbt_var_len,subtype_var_len, msgdata,test_msg,test_sbt_varlen_varsize,subtype_varlen_varsize, +msgdata,test_msg,test_sbt_arrays,subtype_arrays, + +msgtype,test_tlv1,2 +msgdata,test_tlv1,test_struct,test_short_id, +msgdata,test_tlv1,tlv,n1, + +msgtype,test_tlv2,3 +msgdata,test_tlv2,tlv,n2, + +msgtype,test_tlv3,4 +msgdata,test_tlv3,tlv,n3, # A subtype with a nested subtype subtype,subtype_nested @@ -51,9 +62,16 @@ subtypedata,subtype_varsize_struct,field_0,test_features, # A subtype to test varsize things of # - A two level subtype comment subtype,subtype_var_assign -# variable length set of assignables +# variable length set of assignables (helper) subtypedata,subtype_var_assign,len_1,u8, subtypedata,subtype_var_assign,field_1,u16,len_1 +# variable length set of assignables (helper) + +# Subtype to test arrays of things +subtype,subtype_arrays +subtypedata,subtype_arrays,field1,u8,2 +subtypedata,subtype_arrays,field2,u16,2 +subtypedata,subtype_arrays,field3,test_short_id,2 # variable length set of structs subtype,subtype_var_len @@ -75,3 +93,39 @@ subtypedata,test_features,len_lf,u16, subtypedata,test_features,localfeatures,u8,len_lf subtypedata,test_features,len_gf,u16, subtypedata,test_features,globalfeatures,u8,len_gf + +# TLV's! +tlvtype,n1,tlv1,1 +tlvtype,n1,tlv2,2 +# neseted subtype +tlvdata,n1,tlv2,scid,test_short_id, +tlvtype,n1,tlv3,3 +tlvdata,n1,tlv3,features,test_features, +tlvdata,n1,tlv3,amount_msat_1,u64, +tlvdata,n1,tlv3,amount_msat_2,u64, +tlvtype,n1,tlv4,254 +tlvdata,n1,tlv4,cltv_delta,u16, +tlvtype,n2,tlv1,0 +tlvtype,n2,tlv2,11 +# TLV with all varsized and array structs +tlvtype,n3,tlv3,1 +tlvdata,n3,tlv3,subtype,test_short_id, +tlvdata,n3,tlv3,varlen_subtype,subtype_var_len, +tlvdata,n3,tlv3,varlen_assigned,subtype_var_assign, +tlvdata,n3,tlv3,test_sbt_varlen_varsize,subtype_varlen_varsize, +# array assignable +tlvdata,n3,tlv3,arr_assign,u32,2 +# array structs +tlvdata,n3,tlv3,arr_struct,test_short_id,2 +# varlen assignable (helper) +tlvdata,n3,tlv3,len_lf,u16, +tlvdata,n3,tlv3,localfeatures,u8,len_lf +# varlen assignable (no-helper) +tlvdata,n3,tlv3,len_varlen_ass,u16, +tlvdata,n3,tlv3,varlen_ass,u64,len_varlen_ass +# varlen structs +tlvdata,n3,tlv3,len_varlen,u8, +tlvdata,n3,tlv3,varlen_struct,test_short_id,len_varlen +# varlen varsized structs +tlvdata,n3,tlv3,len_varlenvarsize,u8, +tlvdata,n3,tlv3,varlen_varsize,test_features,len_varlenvarsize