channeld: simplify loading of pre-existing HTLCs.

We currently abuse the added_htlc and failed_htlc messages to tell channeld
about existing htlcs when it restarts.  It's clearer to have an explicit
'existing_htlc' type which contains all the information for this case.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

channeld/channel_wire.csv

@@ -41,16 +41,8 @@ msgdata,channel_init,next_index_local,u64,
 msgdata,channel_init,next_index_remote,u64,
 msgdata,channel_init,revocations_received,u64,
 msgdata,channel_init,next_htlc_id,u64,
-msgdata,channel_init,num_htlcs,u16,
-msgdata,channel_init,htlcs,added_htlc,num_htlcs
-msgdata,channel_init,htlc_states,enum htlc_state,num_htlcs
-msgdata,channel_init,num_fulfilled,u16,
-msgdata,channel_init,fulfilled,fulfilled_htlc,num_fulfilled
-msgdata,channel_init,fulfilled_sides,enum side,num_fulfilled
-msgdata,channel_init,num_failed_in,u16,
-msgdata,channel_init,failed_in,failed_htlc,num_failed_in
-msgdata,channel_init,num_failed_out,u16,
-msgdata,channel_init,failed_out,u64,num_failed_out
+msgdata,channel_init,num_existing_htlcs,u16,
+msgdata,channel_init,htlcs,existing_htlc,num_existing_htlcs
 msgdata,channel_init,local_funding_locked,bool,
 msgdata,channel_init,remote_funding_locked,bool,
 msgdata,channel_init,funding_short_id,short_channel_id,

channeld/channeld.c

@@ -3036,12 +3036,7 @@ static void init_channel(struct peer *peer)
 	struct channel_config conf[NUM_SIDES];
 	struct bitcoin_txid funding_txid;
 	enum side funder;
-	enum htlc_state *hstates;
-	struct fulfilled_htlc *fulfilled;
-	enum side *fulfilled_sides;
-	struct failed_htlc **failed_in;
-	u64 *failed_out;
-	struct added_htlc *htlcs;
+	struct existing_htlc **htlcs;
 	bool reconnected;
 	u8 *funding_signed;
 	const u8 *msg;
@@ -3092,11 +3087,6 @@ static void init_channel(struct peer *peer)
 				   &peer->revocations_received,
 				   &peer->htlc_id,
 				   &htlcs,
-				   &hstates,
-				   &fulfilled,
-				   &fulfilled_sides,
-				   &failed_in,
-				   &failed_out,
 				   &peer->funding_locked[LOCAL],
 				   &peer->funding_locked[REMOTE],
 				   &peer->short_channel_ids[LOCAL],
@@ -3175,23 +3165,13 @@ static void init_channel(struct peer *peer)
 					 option_static_remotekey,
 					 funder);
 
-	if (!channel_force_htlcs(peer->channel, htlcs, hstates,
-				 fulfilled, fulfilled_sides,
-				 cast_const2(const struct failed_htlc **,
-					     failed_in),
-				 failed_out))
+	if (!channel_force_htlcs(peer->channel,
+			 cast_const2(const struct existing_htlc **, htlcs)))
 		status_failed(STATUS_FAIL_INTERNAL_ERROR,
 			      "Could not restore HTLCs");
 
 	/* We don't need these any more, so free them. */
 	tal_free(htlcs);
-	tal_free(hstates);
-	tal_free(fulfilled);
-	tal_free(fulfilled_sides);
-	tal_free(failed_in);
-	tal_free(failed_out);
-	tal_free(remote_ann_node_sig);
-	tal_free(remote_ann_bitcoin_sig);
 
 	peer->channel_direction = node_id_idx(&peer->node_ids[LOCAL],
 					      &peer->node_ids[REMOTE]);

channeld/full_channel.c

@@ -1185,31 +1185,22 @@ static bool adjust_balance(struct balance view_owed[NUM_SIDES][NUM_SIDES],
 }
 
 bool channel_force_htlcs(struct channel *channel,
-			 const struct added_htlc *htlcs,
-			 const enum htlc_state *hstates,
-			 const struct fulfilled_htlc *fulfilled,
-			 const enum side *fulfilled_sides,
-			 const struct failed_htlc **failed_in,
-			 const u64 *failed_out)
+			 const struct existing_htlc **htlcs)
 {
-	size_t i;
-	struct htlc *htlc;
-	struct htlc_map_iter it;
 	struct balance view_owed[NUM_SIDES][NUM_SIDES];
 
-	if (tal_count(hstates) != tal_count(htlcs)) {
-		status_broken("#hstates %zu != #htlcs %zu",
-			     tal_count(hstates), tal_count(htlcs));
-		return false;
-	}
-
-	if (tal_count(fulfilled) != tal_count(fulfilled_sides)) {
-		status_broken("#fulfilled sides %zu != #fulfilled %zu",
-			     tal_count(fulfilled_sides), tal_count(fulfilled));
-		return false;
+	/* You'd think, since we traverse HTLCs in ID order, this would never
+	 * go negative.  But this ignores the fact that HTLCs ids from each
+	 * side have no correlation with each other.  Copy into struct balance,
+	 * to allow transient underflow. */
+	for (int view = 0; view < NUM_SIDES; view++) {
+		for (int side = 0; side < NUM_SIDES; side++) {
+			to_balance(&view_owed[view][side],
+				   channel->view[view].owed[side]);
+		}
 	}
 
-	for (i = 0; i < tal_count(htlcs); i++) {
+	for (size_t i = 0; i < tal_count(htlcs); i++) {
 		enum channel_add_err e;
 		struct htlc *htlc;
 
@@ -1217,125 +1208,30 @@ bool channel_force_htlcs(struct channel *channel,
 			     " id=%"PRIu64" amount=%s cltv=%u"
 			     " payment_hash=%s",
 			     i, tal_count(htlcs),
-			     htlcs[i].id,
+			     htlcs[i]->id,
 			     type_to_string(tmpctx, struct amount_msat,
-					    &htlcs[i].amount),
-			     htlcs[i].cltv_expiry,
+					    &htlcs[i]->amount),
+			     htlcs[i]->cltv_expiry,
 			     type_to_string(tmpctx, struct sha256,
-					    &htlcs[i].payment_hash));
+					    &htlcs[i]->payment_hash));
 
-		e = add_htlc(channel, hstates[i],
-			     htlcs[i].id, htlcs[i].amount,
-			     htlcs[i].cltv_expiry,
-			     &htlcs[i].payment_hash,
-			     htlcs[i].onion_routing_packet, &htlc, false, NULL);
+		e = add_htlc(channel, htlcs[i]->state,
+			     htlcs[i]->id, htlcs[i]->amount,
+			     htlcs[i]->cltv_expiry,
+			     &htlcs[i]->payment_hash,
+			     htlcs[i]->onion_routing_packet, &htlc, false, NULL);
 		if (e != CHANNEL_ERR_ADD_OK) {
 			status_broken("%s HTLC %"PRIu64" failed error %u",
-				     htlc_state_owner(hstates[i]) == LOCAL
-				     ? "out" : "in", htlcs[i].id, e);
+				     htlc_state_owner(htlcs[i]->state) == LOCAL
+				     ? "out" : "in", htlcs[i]->id, e);
 			return false;
 		}
-	}
-
-	for (i = 0; i < tal_count(fulfilled); i++) {
-		struct htlc *htlc = channel_get_htlc(channel,
-						     fulfilled_sides[i],
-						     fulfilled[i].id);
-		if (!htlc) {
-			status_broken("Fulfill %s HTLC %"PRIu64" not found",
-				     fulfilled_sides[i] == LOCAL ? "out" : "in",
-				     fulfilled[i].id);
-			return false;
-		}
-		if (htlc->r) {
-			status_broken("Fulfill %s HTLC %"PRIu64" already fulfilled",
-				     fulfilled_sides[i] == LOCAL ? "out" : "in",
-				     fulfilled[i].id);
-			return false;
-		}
-		if (htlc->failed) {
-			status_broken("Fulfill %s HTLC %"PRIu64" already failed",
-				     fulfilled_sides[i] == LOCAL ? "out" : "in",
-				     fulfilled[i].id);
-			return false;
-		}
-		if (!htlc_has(htlc, HTLC_REMOVING)) {
-			status_broken("Fulfill %s HTLC %"PRIu64" state %s",
-				     fulfilled_sides[i] == LOCAL ? "out" : "in",
-				     fulfilled[i].id,
-				     htlc_state_name(htlc->state));
-			return false;
-		}
-		htlc->r = tal_dup(htlc, struct preimage,
-				  &fulfilled[i].payment_preimage);
-	}
+		if (htlcs[i]->payment_preimage)
+			htlc->r = tal_dup(htlc, struct preimage,
+					  htlcs[i]->payment_preimage);
+		if (htlcs[i]->failed)
+			htlc->failed = tal_steal(htlc, htlcs[i]->failed);
 
-	for (i = 0; i < tal_count(failed_in); i++) {
-		struct htlc *htlc;
-		htlc = channel_get_htlc(channel, REMOTE, failed_in[i]->id);
-		if (!htlc) {
-			status_broken("Fail in HTLC %"PRIu64" not found",
-				     failed_in[i]->id);
-			return false;
-		}
-		if (htlc->r) {
-			status_broken("Fail in HTLC %"PRIu64" already fulfilled",
-				     failed_in[i]->id);
-			return false;
-		}
-		if (htlc->failed) {
-			status_broken("Fail in HTLC %"PRIu64" already failed_in",
-				     failed_in[i]->id);
-			return false;
-		}
-		htlc->failed = tal_steal(htlc, failed_in[i]);
-	}
-
-	for (i = 0; i < tal_count(failed_out); i++) {
-		struct htlc *htlc;
-
-		htlc = channel_get_htlc(channel, LOCAL, failed_out[i]);
-		if (!htlc) {
-			status_broken("Fail out HTLC %"PRIu64" not found",
-				      failed_out[i]);
-			return false;
-		}
-		if (htlc->r) {
-			status_broken("Fail out HTLC %"PRIu64" already fulfilled",
-				      failed_out[i]);
-			return false;
-		}
-		if (htlc->failed) {
-			status_broken("Fail out HTLC %"PRIu64" already failed",
-				      failed_out[i]);
-			return false;
-		}
-		if (!htlc_has(htlc, HTLC_REMOVING)) {
-			status_broken("Fail out HTLC %"PRIu64" state %s",
-				      failed_out[i],
-				      htlc_state_name(htlc->state));
-			return false;
-		}
-
-		/* Now, we don't really care why our htlcs failed: lightningd
-		 * already knows.  Just mark it failed using anything. */
-		htlc->failed = tal(htlc, struct failed_htlc);
-	}
-
-	/* You'd think, since we traverse HTLCs in ID order, this would never
-	 * go negative.  But this ignores the fact that HTLCs ids from each
-	 * side have no correlation with each other.  Copy into struct balance,
-	 * to allow transient underflow. */
-	for (int view = 0; view < NUM_SIDES; view++) {
-		for (int side = 0; side < NUM_SIDES; side++) {
-			to_balance(&view_owed[view][side],
-				   channel->view[view].owed[side]);
-		}
-	}
-
-	for (htlc = htlc_map_first(channel->htlcs, &it);
-	     htlc;
-	     htlc = htlc_map_next(channel->htlcs, &it)) {
 		if (!adjust_balance(view_owed, htlc))
 			return false;
 	}

channeld/full_channel.h

@@ -7,6 +7,8 @@
 #include <common/initial_channel.h>
 #include <common/sphinx.h>
 
+struct existing_htlc;
+
 /**
  * new_full_channel: Given initial fees and funding, what is initial state?
  * @ctx: tal context to allocate return value from.
@@ -227,22 +229,12 @@ size_t num_channel_htlcs(const struct channel *channel);
 /**
  * channel_force_htlcs: force these htlcs into the (new) channel
  * @channel: the channel
- * @htlcs: the htlcs to add (tal_arr)
- * @hstates: the states for the htlcs (tal_arr of same size)
- * @fulfilled: htlcs of those which are fulfilled
- * @fulfilled_sides: sides for ids in @fulfilled
- * @failed_in: incoming htlcs which are failed (stolen!)
- * @failed_out: outgoing htlc ids which are failed
+ * @htlcs: the htlcs to add (tal_arr) elements stolen.
  *
  * This is used for restoring a channel state.
  */
 bool channel_force_htlcs(struct channel *channel,
-			 const struct added_htlc *htlcs,
-			 const enum htlc_state *hstates,
-			 const struct fulfilled_htlc *fulfilled,
-			 const enum side *fulfilled_sides,
-			 const struct failed_htlc **failed_in,
-			 const u64 *failed_out);
+			 const struct existing_htlc **htlcs);
 
 /**
  * dump_htlcs: debugging dump of all HTLCs

common/htlc_wire.c

@@ -1,11 +1,65 @@
 #include <bitcoin/tx.h>
 #include <ccan/array_size/array_size.h>
+#include <ccan/cast/cast.h>
 #include <ccan/crypto/shachain/shachain.h>
 #include <common/htlc_wire.h>
 #include <common/memleak.h>
 #include <common/onionreply.h>
 #include <wire/wire.h>
 
+struct failed_htlc *failed_htlc_dup(const tal_t *ctx,
+				    const struct failed_htlc *f TAKES)
+{
+	struct failed_htlc *newf;
+
+	if (taken(f))
+		return cast_const(struct failed_htlc *, tal_steal(ctx, f));
+	newf = tal(ctx, struct failed_htlc);
+	newf->id = f->id;
+	if (f->sha256_of_onion)
+		newf->sha256_of_onion = tal_dup(newf, struct sha256, f->sha256_of_onion);
+	else
+		newf->sha256_of_onion = NULL;
+	newf->badonion = f->badonion;
+	if (f->onion)
+		newf->onion = dup_onionreply(newf, f->onion);
+	else
+		newf->onion = NULL;
+	return newf;
+}
+
+struct existing_htlc *new_existing_htlc(const tal_t *ctx,
+					u64 id,
+					enum htlc_state state,
+					struct amount_msat amount,
+					const struct sha256 *payment_hash,
+					u32 cltv_expiry,
+					const u8 onion_routing_packet[TOTAL_PACKET_SIZE],
+					const struct preimage *preimage,
+					const struct failed_htlc *failed TAKES)
+{
+	struct existing_htlc *existing = tal(ctx, struct existing_htlc);
+
+	existing->id = id;
+	existing->state = state;
+	existing->amount = amount;
+	existing->cltv_expiry = cltv_expiry;
+	existing->payment_hash = *payment_hash;
+	memcpy(existing->onion_routing_packet, onion_routing_packet,
+	       sizeof(existing->onion_routing_packet));
+	if (preimage)
+		existing->payment_preimage
+			= tal_dup(existing, struct preimage, preimage);
+	else
+		existing->payment_preimage = NULL;
+	if (failed)
+		existing->failed = failed_htlc_dup(existing, failed);
+	else
+		existing->failed = NULL;
+
+	return existing;
+}
+
 /* FIXME: We could adapt tools/generate-wire.py to generate structures
  * and code like this. */
 void towire_added_htlc(u8 **pptr, const struct added_htlc *added)
@@ -18,6 +72,27 @@ void towire_added_htlc(u8 **pptr, const struct added_htlc *added)
 	       sizeof(added->onion_routing_packet));
 }
 
+void towire_existing_htlc(u8 **pptr, const struct existing_htlc *existing)
+{
+	towire_u8(pptr, existing->state);
+	towire_u64(pptr, existing->id);
+	towire_amount_msat(pptr, existing->amount);
+ 	towire_sha256(pptr, &existing->payment_hash);
+	towire_u32(pptr, existing->cltv_expiry);
+	towire(pptr, existing->onion_routing_packet,
+	       sizeof(existing->onion_routing_packet));
+	if (existing->payment_preimage) {
+		towire_bool(pptr, true);
+		towire_preimage(pptr, existing->payment_preimage);
+	} else
+		towire_bool(pptr, false);
+	if (existing->failed) {
+		towire_bool(pptr, true);
+		towire_failed_htlc(pptr, existing->failed);
+	} else
+		towire_bool(pptr, false);
+}
+
 void towire_fulfilled_htlc(u8 **pptr, const struct fulfilled_htlc *fulfilled)
 {
 	towire_u64(pptr, fulfilled->id);
@@ -79,6 +154,30 @@ void fromwire_added_htlc(const u8 **cursor, size_t *max,
 		 sizeof(added->onion_routing_packet));
 }
 
+struct existing_htlc *fromwire_existing_htlc(const tal_t *ctx,
+					     const u8 **cursor, size_t *max)
+{
+	struct existing_htlc *existing = tal(ctx, struct existing_htlc);
+
+	existing->state = fromwire_u8(cursor, max);
+	existing->id = fromwire_u64(cursor, max);
+	existing->amount = fromwire_amount_msat(cursor, max);
+	fromwire_sha256(cursor, max, &existing->payment_hash);
+	existing->cltv_expiry = fromwire_u32(cursor, max);
+	fromwire(cursor, max, existing->onion_routing_packet,
+		 sizeof(existing->onion_routing_packet));
+	if (fromwire_bool(cursor, max)) {
+		existing->payment_preimage = tal(existing, struct preimage);
+		fromwire_preimage(cursor, max, existing->payment_preimage);
+	} else
+		existing->payment_preimage = NULL;
+	if (fromwire_bool(cursor, max))
+		existing->failed = fromwire_failed_htlc(existing, cursor, max);
+	else
+		existing->failed = NULL;
+	return existing;
+}
+
 void fromwire_fulfilled_htlc(const u8 **cursor, size_t *max,
 			     struct fulfilled_htlc *fulfilled)
 {

common/htlc_wire.h

@@ -20,6 +20,20 @@ struct added_htlc {
 	u8 onion_routing_packet[TOTAL_PACKET_SIZE];
 };
 
+/* This is how lightningd tells us about HTLCs which already exist at startup */
+struct existing_htlc {
+	u64 id;
+	enum htlc_state state;
+	struct amount_msat amount;
+	struct sha256 payment_hash;
+	u32 cltv_expiry;
+	u8 onion_routing_packet[TOTAL_PACKET_SIZE];
+	/* If fulfilled, this is non-NULL */
+	struct preimage *payment_preimage;
+	/* If failed, this is set */
+	const struct failed_htlc *failed;
+};
+
 struct fulfilled_htlc {
 	u64 id;
 	struct preimage payment_preimage;
@@ -45,7 +59,20 @@ struct changed_htlc {
 	u64 id;
 };
 
+struct existing_htlc *new_existing_htlc(const tal_t *ctx,
+					u64 id,
+					enum htlc_state state,
+					struct amount_msat amount,
+					const struct sha256 *payment_hash,
+					u32 cltv_expiry,
+					const u8 onion_routing_packet[TOTAL_PACKET_SIZE],
+					const struct preimage *preimage,
+					const struct failed_htlc *failed TAKES);
+
+struct failed_htlc *failed_htlc_dup(const tal_t *ctx, const struct failed_htlc *f TAKES);
+
 void towire_added_htlc(u8 **pptr, const struct added_htlc *added);
+void towire_existing_htlc(u8 **pptr, const struct existing_htlc *existing);
 void towire_fulfilled_htlc(u8 **pptr, const struct fulfilled_htlc *fulfilled);
 void towire_failed_htlc(u8 **pptr, const struct failed_htlc *failed);
 void towire_changed_htlc(u8 **pptr, const struct changed_htlc *changed);
@@ -55,6 +82,8 @@ void towire_shachain(u8 **pptr, const struct shachain *shachain);
 
 void fromwire_added_htlc(const u8 **cursor, size_t *max,
 			 struct added_htlc *added);
+struct existing_htlc *fromwire_existing_htlc(const tal_t *ctx,
+					     const u8 **cursor, size_t *max);
 void fromwire_fulfilled_htlc(const u8 **cursor, size_t *max,
 			     struct fulfilled_htlc *fulfilled);
 struct failed_htlc *fromwire_failed_htlc(const tal_t *ctx, const u8 **cursor,

devtools/mkcommit.c

@@ -10,6 +10,7 @@
  */
 #include <bitcoin/script.h>
 #include <bitcoin/tx.h>
+#include <ccan/cast/cast.h>
 #include <ccan/opt/opt.h>
 #include <ccan/err/err.h>
 #include <ccan/str/hex/hex.h>
@@ -163,55 +164,52 @@ static int parse_config(char *argv[],
 	return argnum;
 }
 
-static int parse_htlc(char *argv[],
-		      struct added_htlc **htlcs,
-		      enum htlc_state **htlc_states,
-		      struct preimage **preimages)
+static int parse_htlc(char *argv[], struct existing_htlc ***htlcs)
 {
-	struct added_htlc add;
+	struct existing_htlc *exist = tal(*htlcs, struct existing_htlc);
 	int argnum = 0;
-	struct preimage preimage;
 
-	add.id = tal_count(*htlcs);
+	exist->id = tal_count(*htlcs);
 	if (streq(argv[argnum], "local"))
-		tal_arr_expand(htlc_states, SENT_ADD_ACK_REVOCATION);
+		exist->state = SENT_ADD_ACK_REVOCATION;
 	else if (streq(argv[argnum], "remote"))
-		tal_arr_expand(htlc_states, RCVD_ADD_ACK_REVOCATION);
+		exist->state = RCVD_ADD_ACK_REVOCATION;
 	else
 		errx(1, "Bad htlc offer: %s should be 'local' or 'remote'",
 		     argv[argnum]);
 	argnum++;
+	exist->payment_preimage = tal(*htlcs, struct preimage);
 	if (!hex_decode(argv[argnum], strlen(argv[argnum]),
-			&preimage, sizeof(preimage)))
+			exist->payment_preimage, sizeof(*exist->payment_preimage)))
 		errx(1, "Bad payment-preimage %s", argv[argnum]);
-	tal_arr_expand(preimages, preimage);
-	sha256(&add.payment_hash, &preimage, sizeof(preimage));
+
+	sha256(&exist->payment_hash, exist->payment_preimage,
+	       sizeof(*exist->payment_preimage));
 	argnum++;
-	if (!parse_amount_msat(&add.amount,
+	if (!parse_amount_msat(&exist->amount,
 			       argv[argnum], strlen(argv[argnum])))
 		errx(1, "Bad htlc amount %s", argv[argnum]);
 	argnum++;
-	add.cltv_expiry = atoi(argv[argnum]);
+	exist->cltv_expiry = atoi(argv[argnum]);
 	argnum++;
 
 	printf("# HTLC %"PRIu64": %s amount=%s preimage=%s payment_hash=%s cltv=%u\n",
-	       add.id, argv[0],
-	       type_to_string(tmpctx, struct amount_msat, &add.amount),
-	       type_to_string(tmpctx, struct preimage, &preimage),
-	       type_to_string(tmpctx, struct sha256, &add.payment_hash),
-	       add.cltv_expiry);
+	       exist->id, argv[0],
+	       type_to_string(tmpctx, struct amount_msat, &exist->amount),
+	       type_to_string(tmpctx, struct preimage, exist->payment_preimage),
+	       type_to_string(tmpctx, struct sha256, &exist->payment_hash),
+	       exist->cltv_expiry);
 
-	tal_arr_expand(htlcs, add);
+	tal_arr_expand(htlcs, exist);
 	return argnum;
 }
 
 static const struct preimage *preimage_of(const struct sha256 *hash,
-					  const struct added_htlc *htlcs,
-					  const struct preimage *preimages)
+					  const struct existing_htlc **htlcs)
 {
-	for (size_t i = 0; i < tal_count(preimages); i++)
-		if (sha256_eq(hash, &htlcs[i].payment_hash))
-			return preimages + i;
+	for (size_t i = 0; i < tal_count(htlcs); i++)
+		if (sha256_eq(hash, &htlcs[i]->payment_hash))
+			return htlcs[i]->payment_preimage;
 	abort();
 }
 
@@ -263,9 +261,7 @@ int main(int argc, char *argv[])
 	u8 **witness;
 	const u8 **wscripts;
 	struct channel *channel;
-	struct added_htlc *htlcs = tal_arr(NULL, struct added_htlc, 0);
-	enum htlc_state *hstates = tal_arr(NULL, enum htlc_state, 0);
-	struct preimage *preimages = tal_arr(NULL, struct preimage, 0);
+	struct existing_htlc **htlcs = tal_arr(NULL, struct existing_htlc *, 0);
 	const struct htlc **htlcmap;
 	struct privkey local_htlc_privkey, remote_htlc_privkey;
 	struct pubkey local_htlc_pubkey, remote_htlc_pubkey;
@@ -348,7 +344,7 @@ int main(int argc, char *argv[])
 	while (argnum < argc) {
 		if (argnum + 4 > argc)
 			opt_usage_exit_fail("Too few arguments for htlc");
-		argnum += parse_htlc(argv + argnum, &htlcs, &hstates, &preimages);
+		argnum += parse_htlc(argv + argnum, &htlcs);
 	}
 	printf("\n");
 
@@ -393,7 +389,8 @@ int main(int argc, char *argv[])
 				   option_static_remotekey,
 				   fee_payer);
 
-	if (!channel_force_htlcs(channel, htlcs, hstates, NULL, NULL, NULL, NULL))
+	if (!channel_force_htlcs(channel,
+			 cast_const2(const struct existing_htlc **, htlcs)))
 		errx(1, "Cannot add HTLCs");
 
 	u8 *funding_wscript = bitcoin_redeem_2of2(NULL,
@@ -504,7 +501,7 @@ int main(int argc, char *argv[])
 			witness = bitcoin_witness_htlc_success_tx(NULL,
 								  &local_htlc_sig,
 								  &remote_htlc_sig,
-								  preimage_of(&htlcmap[i]->rhash, htlcs, preimages),
+								  preimage_of(&htlcmap[i]->rhash, cast_const2(const struct existing_htlc **, htlcs)),
 								  wscripts[1+i]);
 		bitcoin_tx_input_set_witness(local_txs[1+i], 0, witness);
 		printf("htlc tx for output %zu: %s\n",
@@ -615,7 +612,7 @@ int main(int argc, char *argv[])
 			witness = bitcoin_witness_htlc_success_tx(NULL,
 								  &remote_htlc_sig,
 								  &local_htlc_sig,
-								  preimage_of(&htlcmap[i]->rhash, htlcs, preimages),
+								  preimage_of(&htlcmap[i]->rhash, cast_const2(const struct existing_htlc **, htlcs)),
 								  wscripts[1+i]);
 		bitcoin_tx_input_set_witness(remote_txs[1+i], 0, witness);
 		printf("htlc tx for output %zu: %s\n",

lightningd/channel_control.c

@@ -377,12 +377,7 @@ void peer_start_channeld(struct channel *channel,
 {
 	u8 *initmsg;
 	int hsmfd;
-	struct added_htlc *htlcs;
-	enum htlc_state *htlc_states;
-	struct fulfilled_htlc *fulfilled_htlcs;
-	enum side *fulfilled_sides;
-	const struct failed_htlc **failed_in;
-	u64 *failed_out;
+	const struct existing_htlc **htlcs;
 	struct short_channel_id scid;
 	u64 num_revocations;
 	struct lightningd *ld = channel->peer->ld;
@@ -420,8 +415,7 @@ void peer_start_channeld(struct channel *channel,
 		return;
 	}
 
-	peer_htlcs(tmpctx, channel, &htlcs, &htlc_states, &fulfilled_htlcs,
-		   &fulfilled_sides, &failed_in, &failed_out);
+	htlcs = peer_htlcs(tmpctx, channel);
 
 	if (channel->scid) {
 		scid = *channel->scid;
@@ -503,9 +497,7 @@ void peer_start_channeld(struct channel *channel,
 				      channel->next_index[REMOTE],
 				      num_revocations,
 				      channel->next_htlc_id,
-				      htlcs, htlc_states,
-				      fulfilled_htlcs, fulfilled_sides,
-				      failed_in, failed_out,
+				      htlcs,
 				      channel->scid != NULL,
 				      channel->remote_funding_locked,
 				      &scid,

lightningd/peer_htlcs.c

@@ -1975,123 +1975,77 @@ void peer_got_revoke(struct channel *channel, const u8 *msg)
 	wallet_channel_save(ld->wallet, channel);
 }
 
-static void add_htlc(struct added_htlc **htlcs,
-		     enum htlc_state **htlc_states,
-		     u64 id,
-		     struct amount_msat amount,
-		     const struct sha256 *payment_hash,
-		     u32 cltv_expiry,
-		     const u8 onion_routing_packet[TOTAL_PACKET_SIZE],
-		     enum htlc_state state)
-{
-	struct added_htlc a;
-
-	a.id = id;
-	a.amount = amount;
-	a.payment_hash = *payment_hash;
-	a.cltv_expiry = cltv_expiry;
-	memcpy(a.onion_routing_packet, onion_routing_packet,
-	       sizeof(a.onion_routing_packet));
-
-	tal_arr_expand(htlcs, a);
-	tal_arr_expand(htlc_states, state);
-}
-
-static void add_fulfill(u64 id, enum side side,
-			const struct preimage *payment_preimage,
-			struct fulfilled_htlc **fulfilled_htlcs,
-			enum side **fulfilled_sides)
-{
-	struct fulfilled_htlc f;
-
-	f.id = id;
-	f.payment_preimage = *payment_preimage;
-
-	tal_arr_expand(fulfilled_htlcs, f);
-	tal_arr_expand(fulfilled_sides, side);
-}
-
-static void add_fail(struct htlc_in *hin,
-		     const struct onionreply *failonion,
-		     const struct failed_htlc ***failed_htlcs)
-{
-	struct failed_htlc *newf;
-
-	newf = mk_failed_htlc(*failed_htlcs, hin, failonion);
-	tal_arr_expand(failed_htlcs, newf);
-}
-
-static void add_fail_badonion(struct htlc_in *hin,
-			      enum onion_type badonion,
-			      const struct failed_htlc ***failed_htlcs)
-{
-	struct failed_htlc *newf;
-
-	newf = mk_failed_htlc_badonion(*failed_htlcs, hin, badonion);
-	tal_arr_expand(failed_htlcs, newf);
-}
 
 /* FIXME: Load direct from db. */
-void peer_htlcs(const tal_t *ctx,
-		const struct channel *channel,
-		struct added_htlc **htlcs,
-		enum htlc_state **htlc_states,
-		struct fulfilled_htlc **fulfilled_htlcs,
-		enum side **fulfilled_sides,
-		const struct failed_htlc ***failed_in,
-		u64 **failed_out)
+const struct existing_htlc **peer_htlcs(const tal_t *ctx,
+					const struct channel *channel)
 {
+	struct existing_htlc **htlcs;
 	struct htlc_in_map_iter ini;
 	struct htlc_out_map_iter outi;
 	struct htlc_in *hin;
 	struct htlc_out *hout;
 	struct lightningd *ld = channel->peer->ld;
 
-	*htlcs = tal_arr(ctx, struct added_htlc, 0);
-	*htlc_states = tal_arr(ctx, enum htlc_state, 0);
-	*fulfilled_htlcs = tal_arr(ctx, struct fulfilled_htlc, 0);
-	*fulfilled_sides = tal_arr(ctx, enum side, 0);
-	*failed_in = tal_arr(ctx, const struct failed_htlc *, 0);
-	*failed_out = tal_arr(ctx, u64, 0);
+	htlcs = tal_arr(ctx, struct existing_htlc *, 0);
 
 	for (hin = htlc_in_map_first(&ld->htlcs_in, &ini);
 	     hin;
 	     hin = htlc_in_map_next(&ld->htlcs_in, &ini)) {
+		struct failed_htlc *f;
+		struct existing_htlc *existing;
+
 		if (hin->key.channel != channel)
 			continue;
 
-		add_htlc(htlcs, htlc_states,
-			 hin->key.id, hin->msat, &hin->payment_hash,
-			 hin->cltv_expiry, hin->onion_routing_packet,
-			 hin->hstate);
-
 		if (hin->badonion)
-			add_fail_badonion(hin, hin->badonion, failed_in);
-		if (hin->failonion)
-			add_fail(hin, hin->failonion, failed_in);
-		if (hin->preimage)
-			add_fulfill(hin->key.id, REMOTE, hin->preimage,
-				    fulfilled_htlcs, fulfilled_sides);
+			f = take(mk_failed_htlc_badonion(NULL, hin, hin->badonion));
+		else if (hin->failonion)
+			f = take(mk_failed_htlc(NULL, hin, hin->failonion));
+		else
+			f = NULL;
+
+		existing = new_existing_htlc(htlcs, hin->key.id, hin->hstate,
+					     hin->msat, &hin->payment_hash,
+					     hin->cltv_expiry,
+					     hin->onion_routing_packet,
+					     hin->preimage,
+					     f);
+		tal_arr_expand(&htlcs, existing);
 	}
 
 	for (hout = htlc_out_map_first(&ld->htlcs_out, &outi);
 	     hout;
 	     hout = htlc_out_map_next(&ld->htlcs_out, &outi)) {
+		struct failed_htlc *f;
+		struct existing_htlc *existing;
+
 		if (hout->key.channel != channel)
 			continue;
 
-		add_htlc(htlcs, htlc_states,
-			 hout->key.id, hout->msat, &hout->payment_hash,
-			 hout->cltv_expiry, hout->onion_routing_packet,
-			 hout->hstate);
-
-		if (hout->failonion || hout->failmsg)
-			tal_arr_expand(failed_out, hout->key.id);
+		/* Note that channeld doesn't actually care *why* outgoing
+		 * HTLCs failed, so just use a dummy here. */
+		if (hout->failonion || hout->failmsg) {
+			f = take(tal(NULL, struct failed_htlc));
+			f->id = hout->key.id;
+			f->sha256_of_onion = tal(f, struct sha256);
+			memset(f->sha256_of_onion, 0,
+			       sizeof(*f->sha256_of_onion));
+			f->badonion = BADONION;
+			f->onion = NULL;
+		} else
+			f = NULL;
 
-		if (hout->preimage)
-			add_fulfill(hout->key.id, LOCAL, hout->preimage,
-				    fulfilled_htlcs, fulfilled_sides);
+		existing = new_existing_htlc(htlcs, hout->key.id, hout->hstate,
+					     hout->msat, &hout->payment_hash,
+					     hout->cltv_expiry,
+					     hout->onion_routing_packet,
+					     hout->preimage,
+					     f);
+		tal_arr_expand(&htlcs, existing);
 	}
+
+	return cast_const2(const struct existing_htlc **, htlcs);
 }
 
 /* If channel is NULL, free them all (for shutdown) */

lightningd/peer_htlcs.h

@@ -29,14 +29,8 @@ struct channel_info {
 };
 
 /* Get all HTLCs for a peer, to send in init message. */
-void peer_htlcs(const tal_t *ctx,
-		const struct channel *channel,
-		struct added_htlc **htlcs,
-		enum htlc_state **htlc_states,
-		struct fulfilled_htlc **fulfilled_htlcs,
-		enum side **fulfilled_sides,
-		const struct failed_htlc ***failed_in,
-		u64 **failed_out);
+const struct existing_htlc **peer_htlcs(const tal_t *ctx,
+					const struct channel *channel);
 
 void free_htlcs(struct lightningd *ld, const struct channel *channel);
 

tools/generate-wire.py

@@ -216,6 +216,7 @@ class Type(FieldSet):
         'gossip_getnodes_entry',
         'gossip_getchannels_entry',
         'failed_htlc',
+        'existing_htlc',
         'utxo',
         'bitcoin_tx',
         'wirestring',