onion_message: support variable-length onion messages.

Updated to the BOLT, and a few tweaks, and we can send giant onion_messages.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

gossipd/gossipd.c

@@ -444,18 +444,18 @@ static u8 *handle_onion_message(struct peer *peer, const u8 *msg)
 	struct secret ss, *blinding_ss;
 	struct pubkey *blinding_in;
 	struct route_step *rs;
-	u8 onion[TOTAL_PACKET_SIZE(ROUTING_INFO_SIZE)];
+	u8 *onion;
 	const u8 *cursor;
 	size_t max, maxlen;
 	struct tlv_onionmsg_payload *om;
 	struct tlv_onion_message_tlvs *tlvs = tlv_onion_message_tlvs_new(msg);
 
 	/* FIXME: ratelimit! */
-	if (!fromwire_onion_message(msg, onion, tlvs))
+	if (!fromwire_onion_message(msg, msg, &onion, tlvs))
 		return towire_errorfmt(peer, NULL, "Bad onion_message");
 
 	/* We unwrap the onion now. */
-	op = parse_onionpacket(tmpctx, onion, TOTAL_PACKET_SIZE(ROUTING_INFO_SIZE), &badreason);
+	op = parse_onionpacket(tmpctx, onion, tal_bytelen(onion), &badreason);
 	if (!op) {
 		status_debug("peer %s: onion msg: can't parse onionpacket: %s",
 			     type_to_string(tmpctx, struct node_id, &peer->id),
@@ -664,11 +664,11 @@ static struct io_plan *onionmsg_req(struct io_conn *conn, struct daemon *daemon,
 				    const u8 *msg)
 {
 	struct node_id id;
-	u8 onion_routing_packet[TOTAL_PACKET_SIZE(ROUTING_INFO_SIZE)];
+	u8 *onion_routing_packet;
 	struct pubkey *blinding;
 	struct peer *peer;
 
-	if (!fromwire_gossipd_send_onionmsg(msg, msg, &id, onion_routing_packet,
+	if (!fromwire_gossipd_send_onionmsg(msg, msg, &id, &onion_routing_packet,
 					    &blinding))
 		master_badmsg(WIRE_GOSSIPD_SEND_ONIONMSG, msg);
 

gossipd/gossipd_wire.csv

@@ -152,10 +152,12 @@ msgtype,gossipd_got_onionmsg_forward,3143
 msgdata,gossipd_got_onionmsg_forward,next_scid,?short_channel_id,
 msgdata,gossipd_got_onionmsg_forward,next_node_id,?node_id,
 msgdata,gossipd_got_onionmsg_forward,next_blinding,?pubkey,
-msgdata,gossipd_got_onionmsg_forward,next_onion,u8,1366
+msgdata,gossipd_got_onionmsg_forward,next_onion_len,u16,
+msgdata,gossipd_got_onionmsg_forward,next_onion,u8,next_onion_len
 
 # Lightningd tells us to send a onion message.
 msgtype,gossipd_send_onionmsg,3040
 msgdata,gossipd_send_onionmsg,id,node_id,
-msgdata,gossipd_send_onionmsg,onion,u8,1366
+msgdata,gossipd_send_onionmsg,onion_len,u16,
+msgdata,gossipd_send_onionmsg,onion,u8,onion_len
 msgdata,gossipd_send_onionmsg,blinding,?pubkey,

gossipd/gossipd_wiregen.c

@@ -928,8 +928,9 @@ bool fromwire_gossipd_got_onionmsg_to_us(const tal_t *ctx, const void *p, struct
 }
 
 /* WIRE: GOSSIPD_GOT_ONIONMSG_FORWARD */
-u8 *towire_gossipd_got_onionmsg_forward(const tal_t *ctx, const struct short_channel_id *next_scid, const struct node_id *next_node_id, const struct pubkey *next_blinding, const u8 next_onion[1366])
+u8 *towire_gossipd_got_onionmsg_forward(const tal_t *ctx, const struct short_channel_id *next_scid, const struct node_id *next_node_id, const struct pubkey *next_blinding, const u8 *next_onion)
 {
+	u16 next_onion_len = tal_count(next_onion);
 	u8 *p = tal_arr(ctx, u8, 0);
 
 	towire_u16(&p, WIRE_GOSSIPD_GOT_ONIONMSG_FORWARD);
@@ -951,12 +952,15 @@ u8 *towire_gossipd_got_onionmsg_forward(const tal_t *ctx, const struct short_cha
 		towire_bool(&p, true);
 		towire_pubkey(&p, next_blinding);
 	}
-	towire_u8_array(&p, next_onion, 1366);
+	towire_u16(&p, next_onion_len);
+	towire_u8_array(&p, next_onion, next_onion_len);
 
 	return memcheck(p, tal_count(p));
 }
-bool fromwire_gossipd_got_onionmsg_forward(const tal_t *ctx, const void *p, struct short_channel_id **next_scid, struct node_id **next_node_id, struct pubkey **next_blinding, u8 next_onion[1366])
+bool fromwire_gossipd_got_onionmsg_forward(const tal_t *ctx, const void *p, struct short_channel_id **next_scid, struct node_id **next_node_id, struct pubkey **next_blinding, u8 **next_onion)
 {
+	u16 next_onion_len;
+
 	const u8 *cursor = p;
 	size_t plen = tal_count(p);
 
@@ -980,19 +984,24 @@ bool fromwire_gossipd_got_onionmsg_forward(const tal_t *ctx, const void *p, stru
 		*next_blinding = tal(ctx, struct pubkey);
 		fromwire_pubkey(&cursor, &plen, *next_blinding);
 	}
- 	fromwire_u8_array(&cursor, &plen, next_onion, 1366);
+ 	next_onion_len = fromwire_u16(&cursor, &plen);
+ 	// 2nd case next_onion
+	*next_onion = next_onion_len ? tal_arr(ctx, u8, next_onion_len) : NULL;
+	fromwire_u8_array(&cursor, &plen, *next_onion, next_onion_len);
 	return cursor != NULL;
 }
 
 /* WIRE: GOSSIPD_SEND_ONIONMSG */
 /* Lightningd tells us to send a onion message. */
-u8 *towire_gossipd_send_onionmsg(const tal_t *ctx, const struct node_id *id, const u8 onion[1366], const struct pubkey *blinding)
+u8 *towire_gossipd_send_onionmsg(const tal_t *ctx, const struct node_id *id, const u8 *onion, const struct pubkey *blinding)
 {
+	u16 onion_len = tal_count(onion);
 	u8 *p = tal_arr(ctx, u8, 0);
 
 	towire_u16(&p, WIRE_GOSSIPD_SEND_ONIONMSG);
 	towire_node_id(&p, id);
-	towire_u8_array(&p, onion, 1366);
+	towire_u16(&p, onion_len);
+	towire_u8_array(&p, onion, onion_len);
 	if (!blinding)
 		towire_bool(&p, false);
 	else {
@@ -1002,15 +1011,20 @@ u8 *towire_gossipd_send_onionmsg(const tal_t *ctx, const struct node_id *id, con
 
 	return memcheck(p, tal_count(p));
 }
-bool fromwire_gossipd_send_onionmsg(const tal_t *ctx, const void *p, struct node_id *id, u8 onion[1366], struct pubkey **blinding)
+bool fromwire_gossipd_send_onionmsg(const tal_t *ctx, const void *p, struct node_id *id, u8 **onion, struct pubkey **blinding)
 {
+	u16 onion_len;
+
 	const u8 *cursor = p;
 	size_t plen = tal_count(p);
 
 	if (fromwire_u16(&cursor, &plen) != WIRE_GOSSIPD_SEND_ONIONMSG)
 		return false;
  	fromwire_node_id(&cursor, &plen, id);
- 	fromwire_u8_array(&cursor, &plen, onion, 1366);
+ 	onion_len = fromwire_u16(&cursor, &plen);
+ 	// 2nd case onion
+	*onion = onion_len ? tal_arr(ctx, u8, onion_len) : NULL;
+	fromwire_u8_array(&cursor, &plen, *onion, onion_len);
  	if (!fromwire_bool(&cursor, &plen))
 		*blinding = NULL;
 	else {
@@ -1019,4 +1033,4 @@ bool fromwire_gossipd_send_onionmsg(const tal_t *ctx, const void *p, struct node
 	}
 	return cursor != NULL;
 }
-// SHA256STAMP:45335ac4c553938ed7c2d63344d80465eca1ff70ab8ec750e3d0305f81acdad5
+// SHA256STAMP:6ed2cbe23f1e0cda995d8b62631875aef762ee4e1cd7109188d855d23a1752d0

gossipd/gossipd_wiregen.h

@@ -208,14 +208,14 @@ u8 *towire_gossipd_got_onionmsg_to_us(const tal_t *ctx, const struct pubkey *bli
 bool fromwire_gossipd_got_onionmsg_to_us(const tal_t *ctx, const void *p, struct pubkey **blinding_in, struct pubkey **reply_blinding, struct onionmsg_path ***reply_path, u8 **rawmsg);
 
 /* WIRE: GOSSIPD_GOT_ONIONMSG_FORWARD */
-u8 *towire_gossipd_got_onionmsg_forward(const tal_t *ctx, const struct short_channel_id *next_scid, const struct node_id *next_node_id, const struct pubkey *next_blinding, const u8 next_onion[1366]);
+u8 *towire_gossipd_got_onionmsg_forward(const tal_t *ctx, const struct short_channel_id *next_scid, const struct node_id *next_node_id, const struct pubkey *next_blinding, const u8 *next_onion);
-bool fromwire_gossipd_got_onionmsg_forward(const tal_t *ctx, const void *p, struct short_channel_id **next_scid, struct node_id **next_node_id, struct pubkey **next_blinding, u8 next_onion[1366]);
+bool fromwire_gossipd_got_onionmsg_forward(const tal_t *ctx, const void *p, struct short_channel_id **next_scid, struct node_id **next_node_id, struct pubkey **next_blinding, u8 **next_onion);
 
 /* WIRE: GOSSIPD_SEND_ONIONMSG */
 /*  Lightningd tells us to send a onion message. */
-u8 *towire_gossipd_send_onionmsg(const tal_t *ctx, const struct node_id *id, const u8 onion[1366], const struct pubkey *blinding);
+u8 *towire_gossipd_send_onionmsg(const tal_t *ctx, const struct node_id *id, const u8 *onion, const struct pubkey *blinding);
-bool fromwire_gossipd_send_onionmsg(const tal_t *ctx, const void *p, struct node_id *id, u8 onion[1366], struct pubkey **blinding);
+bool fromwire_gossipd_send_onionmsg(const tal_t *ctx, const void *p, struct node_id *id, u8 **onion, struct pubkey **blinding);
 
 
 #endif /* LIGHTNING_GOSSIPD_GOSSIPD_WIREGEN_H */
-// SHA256STAMP:45335ac4c553938ed7c2d63344d80465eca1ff70ab8ec750e3d0305f81acdad5
+// SHA256STAMP:6ed2cbe23f1e0cda995d8b62631875aef762ee4e1cd7109188d855d23a1752d0

lightningd/onion_message.c

@@ -138,11 +138,11 @@ void handle_onionmsg_forward(struct lightningd *ld, const u8 *msg)
 	struct short_channel_id *next_scid;
 	struct node_id *next_node;
 	struct pubkey *next_blinding;
-	u8 onion[TOTAL_PACKET_SIZE(ROUTING_INFO_SIZE)];
+	u8 *onion;
 
 	if (!fromwire_gossipd_got_onionmsg_forward(msg, msg, &next_scid,
 						   &next_node,
-						   &next_blinding, onion)) {
+						   &next_blinding, &onion)) {
 		log_broken(ld->log, "bad got_onionmsg_forward: %s",
 			   tal_hex(tmpctx, msg));
 		return;
@@ -398,6 +398,7 @@ static struct command_result *json_send_onion_message(struct command *cmd,
 	struct onionpacket *op;
 	struct secret *path_secrets;
 	struct node_id first_id;
+	size_t onion_size;
 
 	if (!param(cmd, buffer, params,
 		   p_req("hops", param_hops, &hops),
@@ -432,7 +433,15 @@ static struct command_result *json_send_onion_message(struct command *cmd,
 				hops[i].rawtlv, tal_bytelen(hops[i].rawtlv));
 		sphinx_add_hop(sphinx_path, &hops[i].id, take(tlv_with_len));
 	}
-	op = create_onionpacket(tmpctx, sphinx_path, ROUTING_INFO_SIZE, &path_secrets);
+	/* BOLT-offers #4:
+	 * - SHOULD set `len` to 1366 or 32834.
+	 */
+	if (sphinx_path_payloads_size(sphinx_path) <= ROUTING_INFO_SIZE)
+		onion_size = ROUTING_INFO_SIZE;
+	else
+		onion_size = 32768;
+
+	op = create_onionpacket(tmpctx, sphinx_path, onion_size, &path_secrets);
 	if (!op)
 		return command_fail(cmd, JSONRPC2_INVALID_PARAMS,
 				    "Creating onion failed (tlvs too long?)");

tests/plugins/onionmessage-reply.py

@@ -14,6 +14,7 @@ def on_onion_message(plugin, onion_message, **kwargs):
         return
 
     plugin.rpc.call('sendonionmessage', [onion_message['reply_path']])
+    plugin.log("Got onion_message invoice '{}'".format(onion_message['invoice']))
     plugin.log("Sent reply via {}".format(onion_message['reply_path']))
     return {"result": "continue"}
 

tests/test_misc.py

@@ -2348,15 +2348,18 @@ def test_sendonionmessage_reply(node_factory):
     # First hop can't be blinded!
     assert p1 == l2.info['id']
 
+    # Also tests oversize payload which won't fit in 1366-byte onion.
     l1.rpc.call('sendonionmessage',
                 {'hops':
                  [{'id': l2.info['id']},
-                  {'id': l3.info['id']}],
+                  {'id': l3.info['id'],
+                   'invoice': '77' * 15000}],
                  'reply_path':
                  {'blinding': blinding,
                   'path': [{'id': p1, 'enctlv': p1enc}, {'id': p2}]}})
 
     assert l3.daemon.wait_for_log('Got onionmsg reply_blinding reply_path')
+    assert l3.daemon.wait_for_log("Got onion_message invoice '{}'".format('77' * 15000))
     assert l3.daemon.wait_for_log('Sent reply via')
     assert l1.daemon.wait_for_log('Got onionmsg')
 

wire/extracted_peer_experimental_varonionmessage

@@ -0,0 +1,12 @@
+--- wire/peer_exp_wire.csv	2020-12-07 14:17:07.322636209 +1030
++++ -	2020-12-08 11:05:50.990733421 +1030
+@@ -310,7 +211,8 @@
+ msgdata,gossip_timestamp_filter,first_timestamp,u32,
+ msgdata,gossip_timestamp_filter,timestamp_range,u32,
+ msgtype,onion_message,385,option_onion_messages
+-msgdata,onion_message,onionmsg,byte,1366
++msgdata,onion_message,len,u16,
++msgdata,onion_message,onionmsg,byte,len
+ msgdata,onion_message,onion_message_tlvs,onion_message_tlvs,
+ tlvtype,onion_message_tlvs,blinding,2
+ tlvdata,onion_message_tlvs,blinding,blinding,point,