From 74df755a297115651ecd98dadd78df6801e1a21e Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Wed, 15 Mar 2017 10:43:03 +0100 Subject: [PATCH 1/5] gossip: Added nested message types for getnodes query Added a struct that represents a single entry in a `getnodes` reply, so that we can append one after the other and parse them again on the other side. --- lightningd/Makefile | 1 + lightningd/gossip_msg.c | 27 +++++++++++++++++++++++++++ lightningd/gossip_msg.h | 15 +++++++++++++++ 3 files changed, 43 insertions(+) create mode 100644 lightningd/gossip_msg.c create mode 100644 lightningd/gossip_msg.h diff --git a/lightningd/Makefile b/lightningd/Makefile index b5cbe6fde..79748dc9b 100644 --- a/lightningd/Makefile +++ b/lightningd/Makefile @@ -45,6 +45,7 @@ LIGHTNINGD_LIB_SRC := \ lightningd/debug.c \ lightningd/derive_basepoints.c \ lightningd/funding_tx.c \ + lightningd/gossip_msg.c \ lightningd/htlc_tx.c \ lightningd/key_derive.c \ lightningd/msg_queue.c \ diff --git a/lightningd/gossip_msg.c b/lightningd/gossip_msg.c new file mode 100644 index 000000000..4d3698851 --- /dev/null +++ b/lightningd/gossip_msg.c @@ -0,0 +1,27 @@ +#include +#include + +void fromwire_gossip_getnodes_entry(const u8 **pptr, size_t *max, struct gossip_getnodes_entry *entry) +{ + u8 hostnamelen; + fromwire_pubkey(pptr, max, &entry->nodeid); + hostnamelen = fromwire_u8(pptr, max); + entry->hostname = tal_arr(entry, char, hostnamelen); + fromwire_u8_array(pptr, max, (u8*)entry->hostname, hostnamelen); + entry->port = fromwire_u16(pptr, max); +} +void towire_gossip_getnodes_entry(u8 **pptr, const struct gossip_getnodes_entry *entry) +{ + u8 hostnamelen; + towire_pubkey(pptr, &entry->nodeid); + if (entry->hostname) { + hostnamelen = strlen(entry->hostname); + towire_u8(pptr, hostnamelen); + towire_u8_array(pptr, (u8*)entry->hostname, hostnamelen); + }else { + /* If we don't have a hostname just write an empty string */ + hostnamelen = 0; + towire_u8(pptr, hostnamelen); + } + towire_u16(pptr, entry->port); +} diff --git a/lightningd/gossip_msg.h b/lightningd/gossip_msg.h new file mode 100644 index 000000000..3a543f297 --- /dev/null +++ b/lightningd/gossip_msg.h @@ -0,0 +1,15 @@ +#ifndef LIGHTNING_LIGHTNINGD_GOSSIP_MSG_H +#define LIGHTNING_LIGHTNINGD_GOSSIP_MSG_H +#include "config.h" +#include + +struct gossip_getnodes_entry { + struct pubkey nodeid; + char *hostname; + u16 port; +}; + +void fromwire_gossip_getnodes_entry(const u8 **pptr, size_t *max, struct gossip_getnodes_entry *entry); +void towire_gossip_getnodes_entry(u8 **pptr, const struct gossip_getnodes_entry *entry); + +#endif /* LIGHTNING_LIGHTGNINGD_GOSSIP_MSG_H */ From d6918dc4ef80a062889f3a5bb8538e269b6fd9e8 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Sun, 12 Mar 2017 13:39:23 +0100 Subject: [PATCH 2/5] gossip: Implemented `getnodes` JSON-RPC call --- lightningd/gossip/gossip.c | 26 ++++++++++++++++ lightningd/gossip/gossip_wire.csv | 7 +++++ lightningd/gossip_control.c | 51 +++++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+) diff --git a/lightningd/gossip/gossip.c b/lightningd/gossip/gossip.c index 07119ec1f..e1640b26e 100644 --- a/lightningd/gossip/gossip.c +++ b/lightningd/gossip/gossip.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -438,6 +439,27 @@ static struct io_plan *release_peer(struct io_conn *conn, struct daemon *daemon, "Unknown peer %"PRIu64, unique_id); } +static struct io_plan *getnodes(struct io_conn *conn, struct daemon *daemon) +{ + tal_t *tmpctx = tal_tmpctx(daemon); + u8 *out, *reply = tal_arr(tmpctx, u8, 0); + struct node *n; + struct node_map_iter i; + + n = node_map_first(daemon->rstate->nodes, &i); + while (n != NULL) { + struct gossip_getnodes_entry entry; + entry.nodeid = n->id; + entry.hostname = n->hostname; + entry.port = n->port; + towire_gossip_getnodes_entry(&reply, &entry); + n = node_map_next(daemon->rstate->nodes, &i); + } + out = towire_gossip_getnodes_reply(daemon, reply); + tal_free(tmpctx); + return io_write_wire(conn, take(out), next_req_in, daemon); +} + static struct io_plan *recv_req(struct io_conn *conn, struct daemon *daemon) { enum gossip_wire_type t = fromwire_peektype(daemon->msg_in); @@ -451,7 +473,11 @@ static struct io_plan *recv_req(struct io_conn *conn, struct daemon *daemon) case WIRE_GOSSIPCTL_RELEASE_PEER: return release_peer(conn, daemon, daemon->msg_in); + case WIRE_GOSSIP_GETNODES_REQUEST: + return getnodes(conn, daemon); + case WIRE_GOSSIPCTL_RELEASE_PEER_REPLY: + case WIRE_GOSSIP_GETNODES_REPLY: case WIRE_GOSSIPSTATUS_INIT_FAILED: case WIRE_GOSSIPSTATUS_BAD_NEW_PEER_REQUEST: case WIRE_GOSSIPSTATUS_BAD_RELEASE_REQUEST: diff --git a/lightningd/gossip/gossip_wire.csv b/lightningd/gossip/gossip_wire.csv index 8cf74da74..d6f3a3135 100644 --- a/lightningd/gossip/gossip_wire.csv +++ b/lightningd/gossip/gossip_wire.csv @@ -46,3 +46,10 @@ gossipstatus_peer_nongossip,0,unique_id,8 gossipstatus_peer_nongossip,10,crypto_state,144,struct crypto_state gossipstatus_peer_nongossip,154,len,2 gossipstatus_peer_nongossip,156,msg,len,u8 + +# Pass JSON-RPC getnodes call through +gossip_getnodes_request,5 + +gossip_getnodes_reply,105 +gossip_getnodes_reply,0,replen,2,u16 +gossip_getnodes_reply,2,reply,replen,u8 diff --git a/lightningd/gossip_control.c b/lightningd/gossip_control.c index a3ddf2b31..406e872ba 100644 --- a/lightningd/gossip_control.c +++ b/lightningd/gossip_control.c @@ -9,6 +9,7 @@ #include #include #include +#include #include static void gossip_finished(struct subd *gossip, int status) @@ -118,8 +119,10 @@ static enum subd_msg_ret gossip_msg(struct subd *gossip, /* These are messages we send, not them. */ case WIRE_GOSSIPCTL_NEW_PEER: case WIRE_GOSSIPCTL_RELEASE_PEER: + case WIRE_GOSSIP_GETNODES_REQUEST: /* This is a reply, so never gets through to here. */ case WIRE_GOSSIPCTL_RELEASE_PEER_REPLY: + case WIRE_GOSSIP_GETNODES_REPLY: break; case WIRE_GOSSIPSTATUS_PEER_BAD_MSG: peer_bad_message(gossip, msg); @@ -147,3 +150,51 @@ void gossip_init(struct lightningd *ld) if (!ld->gossip) err(1, "Could not subdaemon gossip"); } + +static bool json_getnodes_reply(struct subd *gossip, const u8 *reply, + struct command *cmd) +{ + u8 *inner; + const u8 *cursor; + size_t max; + + struct json_result *response = new_json_result(cmd); + fromwire_gossip_getnodes_reply(reply, reply, NULL, &inner); + max = tal_len(inner); + cursor = inner; + json_object_start(response, NULL); + json_array_start(response, "nodes"); + + while (max > 0) { + struct gossip_getnodes_entry *entry = tal(reply, struct gossip_getnodes_entry); + fromwire_gossip_getnodes_entry(&cursor, &max, entry); + json_object_start(response, NULL); + json_add_pubkey(response, "nodeid", &entry->nodeid); + if (tal_len(entry->hostname) > 0) { + json_add_string(response, "hostname", entry->hostname); + } else { + json_add_null(response, "hostname"); + } + json_add_num(response, "port", entry->port); + json_object_end(response); + tal_free(entry); + } + json_array_end(response); + json_object_end(response); + command_success(cmd, response); + tal_free(reply); + return true; +} + +static void json_getnodes(struct command *cmd, const char *buffer, + const jsmntok_t *params) +{ + struct lightningd *ld = ld_from_dstate(cmd->dstate); + u8 *req = towire_gossip_getnodes_request(cmd); + subd_req(ld->gossip, req, -1, NULL, json_getnodes_reply, cmd); +} + +static const struct json_command getnodes_command = { + "getnodes", json_getnodes, "Retrieve all nodes in our local network view", + "Returns a list of all nodes that we know about"}; +AUTODATA(json_command, &getnodes_command); From 9768e4c9072c07df923900ba53e583f78d617e4d Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Thu, 16 Mar 2017 14:35:23 +1030 Subject: [PATCH 3/5] tools/generate-wire.py: open-code array handling for non-u8. Except for the trivial case of u8 arrays, have the generator create the loop code for the array iteration. This removes some trivial helpers, and avoids us having to write more. Signed-off-by: Rusty Russell --- lightningd/utxo.c | 17 ---------------- lightningd/utxo.h | 5 ----- tools/generate-wire.py | 46 +++++++++++++++++++++++++++++++++--------- wire/fromwire.c | 42 -------------------------------------- wire/towire.c | 42 -------------------------------------- wire/wire.h | 16 --------------- 6 files changed, 37 insertions(+), 131 deletions(-) diff --git a/lightningd/utxo.c b/lightningd/utxo.c index 4091eff65..b8c688f46 100644 --- a/lightningd/utxo.c +++ b/lightningd/utxo.c @@ -18,20 +18,3 @@ void fromwire_utxo(const u8 **ptr, size_t *max, struct utxo *utxo) utxo->keyindex = fromwire_u32(ptr, max); utxo->is_p2sh = fromwire_bool(ptr, max); } - -void fromwire_utxo_array(const u8 **ptr, size_t *max, - struct utxo *utxo, size_t num) -{ - size_t i; - - for (i = 0; i < num; i++) - fromwire_utxo(ptr, max, &utxo[i]); -} - -void towire_utxo_array(u8 **pptr, const struct utxo *utxo, size_t num) -{ - size_t i; - - for (i = 0; i < num; i++) - towire_utxo(pptr, &utxo[i]); -} diff --git a/lightningd/utxo.h b/lightningd/utxo.h index a2eb5ca7d..0b4ef0c61 100644 --- a/lightningd/utxo.h +++ b/lightningd/utxo.h @@ -16,9 +16,4 @@ struct utxo { void towire_utxo(u8 **pptr, const struct utxo *utxo); void fromwire_utxo(const u8 **ptr, size_t *max, struct utxo *utxo); - -void fromwire_utxo_array(const u8 **ptr, size_t *max, - struct utxo *utxo, size_t num); - -void towire_utxo_array(u8 **pptr, const struct utxo *utxo, size_t num); #endif /* LIGHTNING_LIGHTNINGD_UTXO_H */ diff --git a/tools/generate-wire.py b/tools/generate-wire.py index deef23433..ac7dc4431 100755 --- a/tools/generate-wire.py +++ b/tools/generate-wire.py @@ -31,6 +31,10 @@ class FieldType(object): def is_assignable(self): return self.name in ['u8', 'u16', 'u32', 'u64', 'bool'] + # We only accelerate the u8 case: it's common and trivial. + def has_array_helper(self): + return self.name in ['u8'] + # Returns base size @staticmethod def _typesize(typename): @@ -127,6 +131,9 @@ class Field(object): return False return self.fieldtype.is_assignable() + def has_array_helper(self): + return self.fieldtype.has_array_helper() + # Returns FieldType @staticmethod def _guess_type(message, fieldname, base_size): @@ -209,6 +216,20 @@ class Message(object): self.has_variable_fields = True self.fields.append(field) + def print_fromwire_array(self, subcalls, basetype, f, name, num_elems): + if f.has_array_helper(): + subcalls.append('\tfromwire_{}_array(&cursor, plen, {}, {});' + .format(basetype, name, num_elems)) + else: + subcalls.append('\tfor (size_t i = 0; i < {}; i++)' + .format(num_elems)) + if f.is_assignable(): + subcalls.append('\t\t{}[i] = fromwire_{}(&cursor, plen);' + .format(name, basetype)) + else: + subcalls.append('\t\tfromwire_{}(&cursor, plen, {} + i);' + .format(basetype, name)) + def print_fromwire(self,is_header): ctx_arg = 'const tal_t *ctx, ' if self.has_variable_fields else '' @@ -240,15 +261,15 @@ class Message(object): subcalls.append('\tfromwire_pad(&cursor, plen, {});' .format(f.num_elems)) elif f.is_array(): - subcalls.append("\t//1th case {name}".format(name=f.name)) - subcalls.append('\tfromwire_{}_array(&cursor, plen, {}, {});' - .format(basetype, f.name, f.num_elems)) + self.print_fromwire_array(subcalls, basetype, f, f.name, + f.num_elems) elif f.is_variable_size(): subcalls.append("\t//2th case {name}".format(name=f.name)) subcalls.append('\t*{} = tal_arr(ctx, {}, {});' .format(f.name, f.fieldtype.name, f.lenvar)) - subcalls.append('\tfromwire_{}_array(&cursor, plen, *{}, {});' - .format(basetype, f.name, f.lenvar)) + + self.print_fromwire_array(subcalls, basetype, f, '*'+f.name, + f.lenvar) elif f.is_assignable(): subcalls.append("\t//3th case {name}".format(name=f.name)) if f.is_len_var: @@ -271,6 +292,15 @@ class Message(object): subcalls='\n'.join(subcalls) ) + def print_towire_array(self, subcalls, basetype, f, num_elems): + if f.has_array_helper(): + subcalls.append('\ttowire_{}_array(&p, {}, {});' + .format(basetype, f.name, num_elems)) + else: + subcalls.append('\tfor (size_t i = 0; i < {}; i++)\n' + '\t\ttowire_{}(&p, {} + i);' + .format(num_elems, basetype, f.name)) + def print_towire(self,is_header): template = towire_header_templ if is_header else towire_impl_templ args = [] @@ -304,11 +334,9 @@ class Message(object): subcalls.append('\ttowire_pad(&p, {});' .format(f.num_elems)) elif f.is_array(): - subcalls.append('\ttowire_{}_array(&p, {}, {});' - .format(basetype, f.name, f.num_elems)) + self.print_towire_array(subcalls, basetype, f, f.num_elems) elif f.is_variable_size(): - subcalls.append('\ttowire_{}_array(&p, {}, {});' - .format(basetype, f.name, f.lenvar)) + self.print_towire_array(subcalls, basetype, f, f.lenvar) else: subcalls.append('\ttowire_{}(&p, {});' .format(basetype, f.name)) diff --git a/wire/fromwire.c b/wire/fromwire.c index ab49e4a2e..3859525fa 100644 --- a/wire/fromwire.c +++ b/wire/fromwire.c @@ -160,53 +160,11 @@ void fromwire_u8_array(const u8 **cursor, size_t *max, u8 *arr, size_t num) fromwire(cursor, max, arr, num); } -void fromwire_u32_array(const u8 **cursor, size_t *max, u32 *arr, size_t num) -{ - size_t i; - - for (i = 0; i < num; i++) - arr[i] = fromwire_u32(cursor, max); -} - -void fromwire_u64_array(const u8 **cursor, size_t *max, u64 *arr, size_t num) -{ - size_t i; - - for (i = 0; i < num; i++) - arr[i] = fromwire_u64(cursor, max); -} - -void fromwire_bool_array(const u8 **cursor, size_t *max, bool *arr, size_t num) -{ - size_t i; - - for (i = 0; i < num; i++) - arr[i] = fromwire_bool(cursor, max); -} - void fromwire_pad(const u8 **cursor, size_t *max, size_t num) { fromwire(cursor, max, NULL, num); } -void fromwire_secp256k1_ecdsa_signature_array(const u8 **cursor, size_t *max, - secp256k1_ecdsa_signature *arr, size_t num) -{ - size_t i; - - for (i = 0; i < num; i++) - fromwire_secp256k1_ecdsa_signature(cursor, max, arr + i); -} - -void fromwire_sha256_double_array(const u8 **cursor, size_t *max, - struct sha256_double *arr, size_t num) -{ - size_t i; - - for (i = 0; i < num; i++) - fromwire_sha256_double(cursor, max, arr + i); -} - static char *fmt_short_channel_id(const tal_t *ctx, const struct short_channel_id *id) { diff --git a/wire/towire.c b/wire/towire.c index db999e75e..3ec232cda 100644 --- a/wire/towire.c +++ b/wire/towire.c @@ -104,30 +104,6 @@ void towire_u8_array(u8 **pptr, const u8 *arr, size_t num) towire(pptr, arr, num); } -void towire_u32_array(u8 **pptr, const u32 *arr, size_t num) -{ - size_t i; - - for (i = 0; i < num; i++) - towire_u32(pptr, arr[i]); -} - -void towire_u64_array(u8 **pptr, const u64 *arr, size_t num) -{ - size_t i; - - for (i = 0; i < num; i++) - towire_u64(pptr, arr[i]); -} - -void towire_bool_array(u8 **pptr, const bool *arr, size_t num) -{ - size_t i; - - for (i = 0; i < num; i++) - towire_bool(pptr, arr[i]); -} - void towire_pad(u8 **pptr, size_t num) { /* Simply insert zeros. */ @@ -136,21 +112,3 @@ void towire_pad(u8 **pptr, size_t num) tal_resize(pptr, oldsize + num); memset(*pptr + oldsize, 0, num); } - -void towire_secp256k1_ecdsa_signature_array(u8 **pptr, - const secp256k1_ecdsa_signature *arr, size_t num) -{ - size_t i; - - for (i = 0; i < num; i++) - towire_secp256k1_ecdsa_signature(pptr, arr+i); -} - -void towire_sha256_double_array(u8 **pptr, - const struct sha256_double *arr, size_t num) -{ - size_t i; - - for (i = 0; i < num; i++) - towire_sha256_double(pptr, arr+i); -} diff --git a/wire/wire.h b/wire/wire.h index bd974b086..86639f82b 100644 --- a/wire/wire.h +++ b/wire/wire.h @@ -43,14 +43,6 @@ void towire_pad(u8 **pptr, size_t num); void towire_bool(u8 **pptr, bool v); void towire_u8_array(u8 **pptr, const u8 *arr, size_t num); -void towire_u32_array(u8 **pptr, const u32 *arr, size_t num); -void towire_u64_array(u8 **pptr, const u64 *arr, size_t num); -void towire_bool_array(u8 **pptr, const bool *arr, size_t num); -void towire_secp256k1_ecdsa_signature_array(u8 **pptr, - const secp256k1_ecdsa_signature *arr, size_t num); -void towire_sha256_double_array(u8 **pptr, - const struct sha256_double *arr, size_t num); - const u8 *fromwire(const u8 **cursor, size_t *max, void *copy, size_t n); u8 fromwire_u8(const u8 **cursor, size_t *max); @@ -73,12 +65,4 @@ void fromwire_ipv6(const u8 **cursor, size_t *max, struct ipv6 *ipv6); void fromwire_pad(const u8 **cursor, size_t *max, size_t num); void fromwire_u8_array(const u8 **cursor, size_t *max, u8 *arr, size_t num); -void fromwire_u32_array(const u8 **cursor, size_t *max, u32 *arr, size_t num); -void fromwire_u64_array(const u8 **cursor, size_t *max, u64 *arr, size_t num); -void fromwire_bool_array(const u8 **cursor, size_t *max, bool *arr, size_t num); - -void fromwire_secp256k1_ecdsa_signature_array(const u8 **cursor, size_t *max, - secp256k1_ecdsa_signature *arr, size_t num); -void fromwire_sha256_double_array(const u8 **cursor, size_t *max, - struct sha256_double *arr, size_t num); #endif /* LIGHTNING_WIRE_WIRE_H */ From e042198cf8c94768c199eb2af07d2f8b160184b2 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Thu, 16 Mar 2017 14:35:25 +1030 Subject: [PATCH 4/5] tools/generate-wire.py: allow typename instead of type sizes. We use the fourth value (size) to determine the type, unless the fifth value is suppled. That's silly: allow the fourth value to be a typename, since that's the only reason we care about the size at all! Unfortunately there are places in the spec where we use a raw fieldname without '*1' for a length, so we have to distingish this from the typename case. Signed-off-by: Rusty Russell --- lightningd/channel/channel_wire.csv | 20 +++---- lightningd/gossip/gossip_wire.csv | 14 ++--- lightningd/handshake/handshake_wire.csv | 4 +- lightningd/hsm/hsm_wire.csv | 12 ++--- lightningd/opening/opening_wire.csv | 28 +++++----- tools/generate-wire.py | 70 ++++++++++++++----------- 6 files changed, 79 insertions(+), 69 deletions(-) diff --git a/lightningd/channel/channel_wire.csv b/lightningd/channel/channel_wire.csv index bb07a0b2f..e895feddb 100644 --- a/lightningd/channel/channel_wire.csv +++ b/lightningd/channel/channel_wire.csv @@ -19,25 +19,25 @@ channel_normal_operation,1001 # Begin! You're still waiting for the tx to be buried though (passes # gossipd-client fd) channel_init,1 -channel_init,0,funding_txid,32,struct sha256_double +channel_init,0,funding_txid,struct sha256_double channel_init,32,funding_txout,2 -channel_init,34,our_config,36,struct channel_config -channel_init,70,their_config,36,struct channel_config -channel_init,106,first_commit_sig,64,secp256k1_ecdsa_signature -channel_init,166,crypto_state,144,struct crypto_state +channel_init,34,our_config,struct channel_config +channel_init,70,their_config,struct channel_config +channel_init,106,first_commit_sig,secp256k1_ecdsa_signature +channel_init,166,crypto_state,struct crypto_state channel_init,310,remote_fundingkey,33 channel_init,343,revocation_basepoint,33 channel_init,376,payment_basepoint,33 channel_init,409,delayed_payment_basepoint,33 channel_init,442,their_per_commit_point,33 -channel_init,475,am_funder,1,bool +channel_init,475,am_funder,bool channel_init,476,feerate,4 channel_init,480,funding_satoshi,8 channel_init,488,push_msat,8 -channel_init,496,seed,32,struct privkey -channel_init,529,local_node_id,33,struct pubkey -channel_init,562,remote_node_id,33,struct pubkey +channel_init,496,seed,struct privkey +channel_init,529,local_node_id,struct pubkey +channel_init,562,remote_node_id,struct pubkey # Tx is deep enough, go! channel_funding_locked,2 -channel_funding_locked,0,short_channel_id,8,struct short_channel_id \ No newline at end of file +channel_funding_locked,0,short_channel_id,struct short_channel_id diff --git a/lightningd/gossip/gossip_wire.csv b/lightningd/gossip/gossip_wire.csv index d6f3a3135..b162816e5 100644 --- a/lightningd/gossip/gossip_wire.csv +++ b/lightningd/gossip/gossip_wire.csv @@ -9,7 +9,7 @@ gossipstatus_fdpass_failed,0x8004 gossipstatus_peer_bad_msg,1000 gossipstatus_peer_bad_msg,0,unique_id,8 gossipstatus_peer_bad_msg,8,len,2 -gossipstatus_peer_bad_msg,10,err,len,u8 +gossipstatus_peer_bad_msg,10,err,len*u8 #include @@ -17,7 +17,7 @@ gossipstatus_peer_bad_msg,10,err,len,u8 # (if it is to move onto a channel, we get a status msg). gossipctl_new_peer,1 gossipctl_new_peer,0,unique_id,8 -gossipctl_new_peer,8,crypto_state,144,struct crypto_state +gossipctl_new_peer,8,crypto_state,struct crypto_state # Tell it to release a peer which has initialized. gossipctl_release_peer,2 @@ -26,7 +26,7 @@ gossipctl_release_peer,0,unique_id,8 # This releases the peer and returns the cryptostate (followed by fd) gossipctl_release_peer_reply,102 gossipctl_release_peer_reply,0,unique_id,8 -gossipctl_release_peer_reply,8,crypto_state,144,struct crypto_state +gossipctl_release_peer_reply,8,crypto_state,struct crypto_state # This is where we save a peer's features. #gossipstatus_peer_features,1 @@ -43,13 +43,13 @@ gossipstatus_peer_ready,0,unique_id,8 # Peer can send non-gossip packet (usually an open_channel) (followed by fd) gossipstatus_peer_nongossip,4 gossipstatus_peer_nongossip,0,unique_id,8 -gossipstatus_peer_nongossip,10,crypto_state,144,struct crypto_state +gossipstatus_peer_nongossip,10,crypto_state,struct crypto_state gossipstatus_peer_nongossip,154,len,2 -gossipstatus_peer_nongossip,156,msg,len,u8 +gossipstatus_peer_nongossip,156,msg,len*u8 # Pass JSON-RPC getnodes call through gossip_getnodes_request,5 gossip_getnodes_reply,105 -gossip_getnodes_reply,0,replen,2,u16 -gossip_getnodes_reply,2,reply,replen,u8 +gossip_getnodes_reply,0,replen,u16 +gossip_getnodes_reply,2,reply,replen*u8 diff --git a/lightningd/handshake/handshake_wire.csv b/lightningd/handshake/handshake_wire.csv index 2a46dee57..8937a26bb 100644 --- a/lightningd/handshake/handshake_wire.csv +++ b/lightningd/handshake/handshake_wire.csv @@ -36,12 +36,12 @@ handshake_responder,1 handshake_responder,1,my_id,33 handshake_responder_reply,101 handshake_responder_reply,0,initiator_id,33 -handshake_responder_reply,33,cs,144,struct crypto_state +handshake_responder_reply,33,cs,struct crypto_state handshake_initiator,2 handshake_initiator,0,my_id,33 handshake_initiator,33,responder_id,33 handshake_initiator_reply,102 -handshake_initiator_reply,0,cs,144,struct crypto_state +handshake_initiator_reply,0,cs,struct crypto_state diff --git a/lightningd/hsm/hsm_wire.csv b/lightningd/hsm/hsm_wire.csv index be25d8e6b..5294eeb16 100644 --- a/lightningd/hsm/hsm_wire.csv +++ b/lightningd/hsm/hsm_wire.csv @@ -9,17 +9,17 @@ hsmstatus_key_failed,0x8004 hsmstatus_client_bad_request,1000 hsmstatus_client_bad_request,0,unique-id,8 hsmstatus_client_bad_request,8,len,2 -hsmstatus_client_bad_request,10,msg,len,u8 +hsmstatus_client_bad_request,10,msg,len*u8 # Start the HSM. hsmctl_init,1 -hsmctl_init,0,new,1,bool +hsmctl_init,0,new,bool hsmctl_init_reply,101 hsmctl_init_reply,0,node_id,33 -hsmctl_init_reply,33,peer_seed,32,struct privkey +hsmctl_init_reply,33,peer_seed,struct privkey hsmctl_init_reply,65,bip32_len,2 -hsmctl_init_reply,67,bip32_seed,bip32_len*1,u8 +hsmctl_init_reply,67,bip32_seed,bip32_len*u8 # ECDH returns an fd. hsmctl_hsmfd_ecdh,3 @@ -38,10 +38,10 @@ hsmctl_sign_funding,16,change_keyindex,4 hsmctl_sign_funding,20,our_pubkey,33 hsmctl_sign_funding,52,their_pubkey,33 hsmctl_sign_funding,85,num_inputs,2 -hsmctl_sign_funding,87,inputs,num_inputs*49,struct utxo +hsmctl_sign_funding,87,inputs,num_inputs*struct utxo hsmctl_sign_funding_reply,104 hsmctl_sign_funding_reply,0,num_sigs,2 -hsmctl_sign_funding_reply,0,sig,num_sigs*64,secp256k1_ecdsa_signature +hsmctl_sign_funding_reply,0,sig,num_sigs*secp256k1_ecdsa_signature diff --git a/lightningd/opening/opening_wire.csv b/lightningd/opening/opening_wire.csv index ea7280e63..11b51a2f4 100644 --- a/lightningd/opening/opening_wire.csv +++ b/lightningd/opening/opening_wire.csv @@ -15,13 +15,13 @@ opening_peer_bad_initial_message,0x8014 #include opening_init,0 # Base configuration we'll offer (channel reserve will vary with amount) -opening_init,0,our_config,36,struct channel_config +opening_init,0,our_config,struct channel_config # Minimum/maximum configuration values we'll accept opening_init,36,max_to_self_delay,4 opening_init,40,min_effective_htlc_capacity_msat,8 -opening_init,48,crypto_state,144,struct crypto_state +opening_init,48,crypto_state,struct crypto_state # Seed to generate all the keys from -opening_init,196,seed,32,struct privkey +opening_init,196,seed,struct privkey # This means we offer the open. opening_open,1 @@ -37,14 +37,14 @@ opening_open_reply,0,remote_fundingkey,33 # Now we give the funding txid and outnum. opening_open_funding,2 -opening_open_funding,0,txid,32,struct sha256_double -opening_open_funding,32,txout,2,u16 +opening_open_funding,0,txid,struct sha256_double +opening_open_funding,32,txout,u16 # This gives their sig, means we can broadcast tx: we're done. opening_open_funding_reply,102 -opening_open_funding_reply,0,their_config,36,struct channel_config -opening_open_funding_reply,36,first_commit_sig,64,secp256k1_ecdsa_signature -opening_open_funding_reply,100,crypto_state,144,struct crypto_state +opening_open_funding_reply,0,their_config,struct channel_config +opening_open_funding_reply,36,first_commit_sig,secp256k1_ecdsa_signature +opening_open_funding_reply,100,crypto_state,struct crypto_state opening_open_funding_reply,244,revocation_basepoint,33 opening_open_funding_reply,277,payment_basepoint,33 opening_open_funding_reply,310,delayed_payment_basepoint,33 @@ -55,20 +55,20 @@ opening_accept,3 opening_accept,0,min_feerate,4 opening_accept,4,max_feerate,4 opening_accept,8,len,2 -opening_accept,10,msg,len,u8 +opening_accept,10,msg,len*u8 # This gives the txid of their funding tx to watch. opening_accept_reply,103 -opening_accept_reply,0,funding_txid,32,struct sha256_double +opening_accept_reply,0,funding_txid,struct sha256_double # Acknowledge watch is in place, now can send sig. opening_accept_finish,4 opening_accept_finish_reply,104 -opening_accept_finish_reply,32,funding_txout,2,u16 -opening_accept_finish_reply,0,their_config,36,struct channel_config -opening_accept_finish_reply,36,first_commit_sig,64,secp256k1_ecdsa_signature -opening_accept_finish_reply,100,crypto_state,144,struct crypto_state +opening_accept_finish_reply,32,funding_txout,u16 +opening_accept_finish_reply,0,their_config,struct channel_config +opening_accept_finish_reply,36,first_commit_sig,secp256k1_ecdsa_signature +opening_accept_finish_reply,100,crypto_state,struct crypto_state opening_accept_finish_reply,244,remote_fundingkey,33 opening_accept_finish_reply,277,revocation_basepoint,33 opening_accept_finish_reply,310,payment_basepoint,33 diff --git a/tools/generate-wire.py b/tools/generate-wire.py index ac7dc4431..f92961dc1 100755 --- a/tools/generate-wire.py +++ b/tools/generate-wire.py @@ -81,40 +81,48 @@ sizetypemap = { 1: FieldType('u8') } +# It would be nicer if we had put '*1' in spec and disallowed bare lenvar. +# In practice we only recognize raw lenvar when it's the previous field. + +# size := baresize | arraysize +# baresize := simplesize | lenvar +# simplesize := number | type +# arraysize := lenvar '*' simplesize class Field(object): - def __init__(self, message, name, size, comments, typename=None): + def __init__(self, message, name, size, comments, prevname): self.message = message self.comments = comments self.name = name.replace('-', '_') self.is_len_var = False self.lenvar = None + self.num_elems = 1 + + # If it's an arraysize, swallow prefix. + if '*' in size: + self.lenvar = size.split('*')[0].replace('-','_') + size = size.split('*')[1] + else: + if size == prevname: + # Raw length field, implies u8. + self.lenvar = size.replace('-','_') + size = 'u8' - # Size could be a literal number (eg. 33), or a field (eg 'len'), or - # a multiplier of a field (eg. num-htlc-timeouts*64). try: + # Just a number? Guess based on size. base_size = int(size) - except ValueError: - # If it's a multiplicitive expression, must end in basesize. - if '*' in size: - base_size = int(size.split('*')[1]) - self.lenvar = size.split('*')[0] - else: - base_size = 0 - self.lenvar = size - self.lenvar = self.lenvar.replace('-','_') - - if typename is None: self.fieldtype = Field._guess_type(message,self.name,base_size) - else: - self.fieldtype = FieldType(typename) + # There are some arrays which we have to guess, based on sizes. + if base_size % self.fieldtype.tsize != 0: + raise ValueError('Invalid size {} for {}.{} not a multiple of {}' + .format(base_size, + self.message, + self.name, + self.fieldtype.tsize)) + self.num_elems = int(base_size / self.fieldtype.tsize) - # Unknown types are assumed to have base_size: div by 0 if that's unknown. - if self.fieldtype.tsize == 0: - self.fieldtype.tsize = base_size - - if base_size % self.fieldtype.tsize != 0: - raise ValueError('Invalid size {} for {}.{} not a multiple of {}'.format(base_size,self.message,self.name,self.fieldtype.tsize)) - self.num_elems = int(base_size / self.fieldtype.tsize) + except ValueError: + # Not a number; must be a type. + self.fieldtype = FieldType(size) def is_padding(self): return self.name.startswith('pad') @@ -360,6 +368,7 @@ options = parser.parse_args() messages = [] comments = [] includes = [] +prevfield = None # Read csv lines. Single comma is the message values, more is offset/len. for line in fileinput.input(options.files): @@ -382,18 +391,19 @@ for line in fileinput.input(options.files): # eg commit_sig,132 messages.append(Message(parts[0],Enumtype("WIRE_" + parts[0].upper(), parts[1]), comments)) comments=[] - else: + prevfield = None + elif len(parts) == 4: # eg commit_sig,0,channel-id,8 OR - # commit_sig,0,channel-id,8,u64 + # commit_sig,0,channel-id,u64 for m in messages: if m.name == parts[0]: - if len(parts) == 4: - m.addField(Field(parts[0], parts[2], parts[3], comments)) - else: - m.addField(Field(parts[0], parts[2], parts[3], comments, - parts[4])) + m.addField(Field(parts[0], parts[2], parts[3], comments, prevfield)) + prevfield = parts[2] break comments=[] + else: + raise ValueError('Line {} malformed'.format(line.rstrip())) + header_template = """#ifndef LIGHTNING_{idem} #define LIGHTNING_{idem} From de651bf8fb1fe63be17941f4a40882b43f5c6fa6 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Thu, 16 Mar 2017 14:35:26 +1030 Subject: [PATCH 5/5] gossip: don't use hand-coded nested messages for getnodes array. We can simply tell it 'nodes' is 'num_nodes*struct gossip_getnodes_entry'. Signed-off-by: Rusty Russell --- lightningd/gossip/gossip.c | 17 ++++++++++------- lightningd/gossip/gossip_wire.csv | 5 +++-- lightningd/gossip_control.c | 29 +++++++++++++---------------- 3 files changed, 26 insertions(+), 25 deletions(-) diff --git a/lightningd/gossip/gossip.c b/lightningd/gossip/gossip.c index e1640b26e..8e836aaff 100644 --- a/lightningd/gossip/gossip.c +++ b/lightningd/gossip/gossip.c @@ -442,20 +442,23 @@ static struct io_plan *release_peer(struct io_conn *conn, struct daemon *daemon, static struct io_plan *getnodes(struct io_conn *conn, struct daemon *daemon) { tal_t *tmpctx = tal_tmpctx(daemon); - u8 *out, *reply = tal_arr(tmpctx, u8, 0); + u8 *out; struct node *n; struct node_map_iter i; + struct gossip_getnodes_entry *nodes; + size_t node_count = 0; + nodes = tal_arr(tmpctx, struct gossip_getnodes_entry, node_count); n = node_map_first(daemon->rstate->nodes, &i); while (n != NULL) { - struct gossip_getnodes_entry entry; - entry.nodeid = n->id; - entry.hostname = n->hostname; - entry.port = n->port; - towire_gossip_getnodes_entry(&reply, &entry); + tal_resize(&nodes, node_count + 1); + nodes[node_count].nodeid = n->id; + nodes[node_count].hostname = n->hostname; + nodes[node_count].port = n->port; + node_count++; n = node_map_next(daemon->rstate->nodes, &i); } - out = towire_gossip_getnodes_reply(daemon, reply); + out = towire_gossip_getnodes_reply(daemon, nodes); tal_free(tmpctx); return io_write_wire(conn, take(out), next_req_in, daemon); } diff --git a/lightningd/gossip/gossip_wire.csv b/lightningd/gossip/gossip_wire.csv index b162816e5..70e0f7c6b 100644 --- a/lightningd/gossip/gossip_wire.csv +++ b/lightningd/gossip/gossip_wire.csv @@ -50,6 +50,7 @@ gossipstatus_peer_nongossip,156,msg,len*u8 # Pass JSON-RPC getnodes call through gossip_getnodes_request,5 +#include gossip_getnodes_reply,105 -gossip_getnodes_reply,0,replen,u16 -gossip_getnodes_reply,2,reply,replen*u8 +gossip_getnodes_reply,0,num_nodes,u16 +gossip_getnodes_reply,2,nodes,num_nodes*struct gossip_getnodes_entry diff --git a/lightningd/gossip_control.c b/lightningd/gossip_control.c index 406e872ba..ebe943ff2 100644 --- a/lightningd/gossip_control.c +++ b/lightningd/gossip_control.c @@ -154,35 +154,32 @@ void gossip_init(struct lightningd *ld) static bool json_getnodes_reply(struct subd *gossip, const u8 *reply, struct command *cmd) { - u8 *inner; - const u8 *cursor; - size_t max; - + struct gossip_getnodes_entry *nodes; struct json_result *response = new_json_result(cmd); - fromwire_gossip_getnodes_reply(reply, reply, NULL, &inner); - max = tal_len(inner); - cursor = inner; + size_t i; + + if (!fromwire_gossip_getnodes_reply(reply, reply, NULL, &nodes)) { + command_fail(cmd, "Malformed gossip_getnodes response"); + return true; + } + json_object_start(response, NULL); json_array_start(response, "nodes"); - while (max > 0) { - struct gossip_getnodes_entry *entry = tal(reply, struct gossip_getnodes_entry); - fromwire_gossip_getnodes_entry(&cursor, &max, entry); + for (i = 0; i < tal_count(nodes); i++) { json_object_start(response, NULL); - json_add_pubkey(response, "nodeid", &entry->nodeid); - if (tal_len(entry->hostname) > 0) { - json_add_string(response, "hostname", entry->hostname); + json_add_pubkey(response, "nodeid", &nodes[i].nodeid); + if (tal_len(nodes[i].hostname) > 0) { + json_add_string(response, "hostname", nodes[i].hostname); } else { json_add_null(response, "hostname"); } - json_add_num(response, "port", entry->port); + json_add_num(response, "port", nodes[i].port); json_object_end(response); - tal_free(entry); } json_array_end(response); json_object_end(response); command_success(cmd, response); - tal_free(reply); return true; }