spec: update to experimental BOLTs with secret/total_amount.

Also pulls in a new onion error (mpp_timeout).  We change our
route_step_decode_end() to always return the total_msat and optional
secret.

We check total_amount (to prohibit mpp), but we do nothing with
secret for now other than hand it to the htlc_accepted hook.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

channeld/channeld.c

@@ -961,6 +961,11 @@ static u8 *make_failmsg(const tal_t *ctx,
 		/* FIXME: wire this into tlv parser somehow. */
 		msg = towire_invalid_onion_payload(ctx, 0, 0);
 		goto done;
+#if EXPERIMENTAL_FEATURES
+	case WIRE_MPP_TIMEOUT:
+		msg = towire_mpp_timeout(ctx);
+		goto done;
+#endif /* EXPERIMENTAL_FEATURES */
 	}
 	status_failed(STATUS_FAIL_INTERNAL_ERROR,
 		      "Asked to create failmsg %u (%s)",

common/sphinx.c

@@ -192,16 +192,25 @@ void sphinx_add_nonfinal_hop(struct sphinx_path *path,
 	}
 }
 
-void sphinx_add_final_hop(struct sphinx_path *path,
+bool sphinx_add_final_hop(struct sphinx_path *path,
 			  const struct pubkey *pubkey,
 			  bool use_tlv,
 			  struct amount_msat forward,
-			  u32 outgoing_cltv)
+			  u32 outgoing_cltv,
+			  struct amount_msat total_msat,
+			  const struct secret *payment_secret)
 {
+	/* These go together! */
+	if (!payment_secret)
+		assert(amount_msat_eq(total_msat, forward));
+
 	if (use_tlv) {
 		struct tlv_tlv_payload *tlv = tlv_tlv_payload_new(tmpctx);
 		struct tlv_tlv_payload_amt_to_forward tlv_amt;
 		struct tlv_tlv_payload_outgoing_cltv_value tlv_cltv;
+#if EXPERIMENTAL_FEATURES
+		struct tlv_tlv_payload_payment_data tlv_pdata;
+#endif
 
 		/* BOLT #4:
 		 *
@@ -216,12 +225,27 @@ void sphinx_add_final_hop(struct sphinx_path *path,
 		tlv->amt_to_forward = &tlv_amt;
 		tlv->outgoing_cltv_value = &tlv_cltv;
 
+#if EXPERIMENTAL_FEATURES
+		if (payment_secret) {
+			tlv_pdata.payment_secret = *payment_secret;
+			tlv_pdata.total_msat = total_msat.millisatoshis; /* Raw: TLV convert */
+			tlv->payment_data = &tlv_pdata;
+		}
+#else
+		/* Wihtout EXPERIMENTAL_FEATURES, we can't send payment_secret */
+		if (payment_secret)
+			return false;
+#endif
 		sphinx_add_tlv_hop(path, pubkey, tlv);
 	} else {
 		static struct short_channel_id all_zero_scid;
+		/* No payment secrets in legacy format. */
+		if (payment_secret)
+			return false;
 		sphinx_add_v0_hop(path, pubkey, &all_zero_scid,
 				  forward, outgoing_cltv);
 	}
+	return true;
 }
 
 /* Small helper to append data to a buffer and update the position
@@ -693,6 +717,12 @@ static void route_step_decode(struct route_step *rs)
 	case SPHINX_V0_PAYLOAD:
 		rs->amt_to_forward = &rs->payload.v0.amt_forward;
 		rs->outgoing_cltv = &rs->payload.v0.outgoing_cltv;
+		rs->payment_secret = NULL;
+		/* BOLT-e36f7b6517e1173dcbd49da3b516cfe1f48ae556 #4:
+		 * - if it is the final node:
+		 *   - MUST treat `total_msat` as if it were equal to
+		 *     `amt_to_forward` if it is not present. */
+		rs->total_msat = rs->amt_to_forward;
 		if (rs->nextcase == ONION_FORWARD) {
 			rs->forward_channel = &rs->payload.v0.channel_id;
 		} else {
@@ -722,6 +752,23 @@ static void route_step_decode(struct route_step *rs)
 						   ->short_channel_id;
 		else
 			rs->forward_channel = NULL;
+
+		rs->payment_secret = NULL;
+		/* BOLT-e36f7b6517e1173dcbd49da3b516cfe1f48ae556 #4:
+		 * - if it is the final node:
+		 *   - MUST treat `total_msat` as if it were equal to
+		 *     `amt_to_forward` if it is not present. */
+		rs->total_msat = rs->amt_to_forward;
+
+#if EXPERIMENTAL_FEATURES
+		if (rs->payload.tlv->payment_data) {
+			rs->payment_secret
+				= &rs->payload.tlv->payment_data->payment_secret;
+			rs->total_msat = tal(rs, struct amount_msat);
+			rs->total_msat->millisatoshis /* Raw: tu64 on wire */
+				= rs->payload.tlv->payment_data->total_msat;
+		}
+#endif
 		break;
 	case SPHINX_INVALID_PAYLOAD:
 	case SPHINX_RAW_PAYLOAD:

common/sphinx.h

@@ -89,6 +89,8 @@ struct route_step {
 	struct amount_msat *amt_to_forward;
 	u32 *outgoing_cltv;
 	struct short_channel_id *forward_channel;
+	struct secret *payment_secret;
+	struct amount_msat *total_msat;
 };
 
 /**
@@ -243,10 +245,12 @@ void sphinx_add_nonfinal_hop(struct sphinx_path *path,
 /**
  * Add a final hop to the path.
  */
-void sphinx_add_final_hop(struct sphinx_path *path,
+bool sphinx_add_final_hop(struct sphinx_path *path,
 			  const struct pubkey *pubkey,
 			  bool use_tlv,
 			  struct amount_msat forward,
-			  u32 outgoing_cltv);
+			  u32 outgoing_cltv,
+			  struct amount_msat total_msat,
+			  const struct secret *payment_secret);
 
 #endif /* LIGHTNING_COMMON_SPHINX_H */

common/test/run-sphinx.c

@@ -18,6 +18,9 @@ bool amount_asset_is_main(struct amount_asset *asset UNNEEDED)
 /* Generated stub for amount_asset_to_sat */
 struct amount_sat amount_asset_to_sat(struct amount_asset *asset UNNEEDED)
 { fprintf(stderr, "amount_asset_to_sat called!\n"); abort(); }
+/* Generated stub for amount_msat_eq */
+bool amount_msat_eq(struct amount_msat a UNNEEDED, struct amount_msat b UNNEEDED)
+{ fprintf(stderr, "amount_msat_eq called!\n"); abort(); }
 /* Generated stub for amount_msat_from_u64 */
 void amount_msat_from_u64(struct amount_msat *msat UNNEEDED, u64 millisatoshis UNNEEDED)
 { fprintf(stderr, "amount_msat_from_u64 called!\n"); abort(); }
@@ -52,6 +55,9 @@ bigsize_t fromwire_bigsize(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
 /* Generated stub for fromwire_fail */
 const void *fromwire_fail(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
 { fprintf(stderr, "fromwire_fail called!\n"); abort(); }
+/* Generated stub for fromwire_secret */
+void fromwire_secret(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, struct secret *secret UNNEEDED)
+{ fprintf(stderr, "fromwire_secret called!\n"); abort(); }
 /* Generated stub for fromwire_sha256 */
 void fromwire_sha256(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, struct sha256 *sha256 UNNEEDED)
 { fprintf(stderr, "fromwire_sha256 called!\n"); abort(); }
@@ -89,6 +95,9 @@ void towire_bigsize(u8 **pptr UNNEEDED, const bigsize_t val UNNEEDED)
 /* Generated stub for towire_pad */
 void towire_pad(u8 **pptr UNNEEDED, size_t num UNNEEDED)
 { fprintf(stderr, "towire_pad called!\n"); abort(); }
+/* Generated stub for towire_secret */
+void towire_secret(u8 **pptr UNNEEDED, const struct secret *secret UNNEEDED)
+{ fprintf(stderr, "towire_secret called!\n"); abort(); }
 /* Generated stub for towire_sha256 */
 void towire_sha256(u8 **pptr UNNEEDED, const struct sha256 *sha256 UNNEEDED)
 { fprintf(stderr, "towire_sha256 called!\n"); abort(); }

devtools/onion.c

@@ -72,12 +72,13 @@ static void do_generate(int argc, char **argv,
 			struct amount_msat amt;
 			bool use_tlv = streq(argv[1 + i] + klen, "/tlv");
 
+			/* FIXME: support secret and and total_msat */
 			memset(&scid, i, sizeof(scid));
 			amt.millisatoshis = i; /* Raw: test code */
 			if (i == num_hops - 1)
 				sphinx_add_final_hop(sp, &path[i],
 						     use_tlv,
-						     amt, i);
+						     amt, i, amt, NULL);
 			else
 				sphinx_add_nonfinal_hop(sp, &path[i],
 							use_tlv,

doc/PLUGINS.md

@@ -631,6 +631,8 @@ For detailed information about each field please refer to [BOLT 04 of the specif
      and should match the incoming funds in case we are the recipient.
  - `outgoing_cltv_value` determines what the CLTV value for the HTLC that we
      forward to the next hop should be.
+ - `total_msat` specifies the total amount to pay, if present.
+ - `payment_secret` specifies the payment secret (which the payer should have obtained from the invoice), if present.
  - `next_onion` is the fully processed onion that we should be sending to the
    next hop as part of the outgoing HTLC. Processed in this case means that we
    took the incoming onion, decrypted it, extracted the payload destined for

lightningd/pay.c

@@ -681,10 +681,17 @@ send_payment(struct lightningd *ld,
 	ret = pubkey_from_node_id(&pubkey, &ids[i]);
 	assert(ret);
 
-	sphinx_add_final_hop(path, &pubkey,
-			     should_use_tlv(route[i].style),
-			     route[i].amount,
-			     base_expiry + route[i].delay);
+	/* FIXME: This can't fail if payment_secret total_msat == amount, and
+	 * payment_secret is NULL, but it will later when we modify code. */
+	if (!sphinx_add_final_hop(path, &pubkey,
+				  should_use_tlv(route[i].style),
+				  route[i].amount,
+				  base_expiry + route[i].delay,
+				  route[i].amount, NULL)) {
+		return command_fail(cmd, PAY_DESTINATION_PERM_FAIL,
+				    "Destination does not support"
+				    " payment_secret");
+	}
 
 	/* Now, do we already have a payment? */
 	payment = wallet_payment_by_hash(tmpctx, ld->wallet, rhash);

lightningd/peer_htlcs.c

@@ -280,7 +280,9 @@ static void handle_localpay(struct htlc_in *hin,
 			    u32 cltv_expiry,
 			    const struct sha256 *payment_hash,
 			    struct amount_msat amt_to_forward,
-			    u32 outgoing_cltv_value)
+			    u32 outgoing_cltv_value,
+			    struct amount_msat total_msat,
+			    const struct secret *payment_secret)
 {
 	enum onion_type failcode;
 	struct lightningd *ld = hin->key.channel->peer->ld;
@@ -298,6 +300,18 @@ static void handle_localpay(struct htlc_in *hin,
 		goto fail;
 	}
 
+	/* BOLT- #4:
+	 * - if it does not support `basic_mpp`:
+	 *    - MUST fail the HTLC if `total_msat` is not exactly equal to
+	 *    `amt_to_forward`.
+	 */
+	if (!amount_msat_eq(amt_to_forward, total_msat)) {
+		/* FIXME: Ideally, we use WIRE_INVALID_ONION_PAYLOAD and
+		 * point at the total_msat field */
+		failcode = WIRE_FINAL_INCORRECT_HTLC_AMOUNT;
+		goto fail;
+	}
+
 	/* BOLT #4:
 	 *
 	 * 1. type: 18 (`final_incorrect_cltv_expiry`)
@@ -598,6 +612,9 @@ fail:
 struct gossip_resolve {
 	struct short_channel_id next_channel;
 	struct amount_msat amt_to_forward;
+	struct amount_msat total_msat;
+	/* Only set if TLV specifies it */
+	const struct secret *payment_secret;
 	u32 outgoing_cltv_value;
 	u8 *next_onion;
 	struct htlc_in *hin;
@@ -752,6 +769,12 @@ static void htlc_accepted_hook_serialize(struct htlc_accepted_hook_payload *p,
 					  *rs->amt_to_forward);
 	if (rs->outgoing_cltv)
 		json_add_u32(s, "outgoing_cltv_value", *rs->outgoing_cltv);
+	/* These are specified together in TLV, so only print total_msat if
+	 * payment_secret set (ie. modern, and final hop) */
+	if (rs->payment_secret) {
+		json_add_amount_msat_only(s, "total_msat", *rs->total_msat);
+		json_add_secret(s, "payment_secret", rs->payment_secret);
+	}
 	json_add_hex_talarr(s, "next_onion", p->next_onion);
 	json_add_secret(s, "shared_secret", hin->shared_secret);
 	json_object_end(s);
@@ -844,7 +867,9 @@ htlc_accepted_hook_callback(struct htlc_accepted_hook_payload *request,
 		} else
 			handle_localpay(hin, hin->cltv_expiry, &hin->payment_hash,
 					*rs->amt_to_forward,
-					*rs->outgoing_cltv);
+					*rs->outgoing_cltv,
+					*rs->total_msat,
+					rs->payment_secret);
 		break;
 	case htlc_accepted_fail:
 		log_debug(channel->log,

wire/extracted_onion_experimental_e36f7b6517e1173dcbd49da3b516cfe1f48ae556

@@ -0,0 +1,17 @@
+--- wire/extracted_onion_wire_csv	2019-11-04 15:38:24.345401216 +1030
++++ -	2019-11-06 14:40:16.145483573 +1030
+@@ -5,6 +5,9 @@
+ tlvdata,tlv_payload,outgoing_cltv_value,outgoing_cltv_value,tu32,
+ tlvtype,tlv_payload,short_channel_id,6
+ tlvdata,tlv_payload,short_channel_id,short_channel_id,short_channel_id,
++tlvtype,tlv_payload,payment_data,8
++tlvdata,tlv_payload,payment_data,payment_secret,byte,32
++tlvdata,tlv_payload,payment_data,total_msat,tu64,
+ msgtype,invalid_realm,PERM|1
+ msgtype,temporary_node_failure,NODE|2
+ msgtype,permanent_node_failure,PERM|NODE|2
+@@ -48,3 +51,4 @@
+ msgtype,invalid_onion_payload,PERM|22
+ msgdata,invalid_onion_payload,type,varint,
+ msgdata,invalid_onion_payload,offset,u16,
++msgtype,mpp_timeout,23