wally: update to the latest wally version

Includes:
psbt: Use renamed functions for new wally version
psbt: Set the transaction directly to avoid script workarounds
psbt: Use low-S grinding when computing signatures
tx: Use wally_tx_clone from libwally now that its exported

Signed-off-by: Jon Griffiths <jon_p_griffiths@yahoo.com>

.gitmodules

@@ -9,6 +9,5 @@
 	url = https://github.com/ianlancetaylor/libbacktrace.git
 [submodule "external/libwally-core"]
 	path = external/libwally-core
-	url = https://github.com/niftynei/libwally-core.git
+	url = https://github.com/ElementsProject/libwally-core.git
 	ignore = dirty
-	branch = nifty/blessed-branch

bitcoin/psbt.c

@@ -44,69 +44,45 @@ struct wally_psbt *new_psbt(const tal_t *ctx, const struct wally_tx *wtx)
 {
 	struct wally_psbt *psbt;
 	int wally_err;
-	u8 **scripts;
-	size_t *script_lens;
-	struct wally_tx_witness_stack **witnesses;
 
 	if (is_elements(chainparams))
-		wally_err = wally_psbt_elements_init_alloc(wtx->num_inputs, wtx->num_outputs, 0, &psbt);
+		wally_err = wally_psbt_elements_init_alloc(0, wtx->num_inputs, wtx->num_outputs, 0, &psbt);
 	else
-		wally_err = wally_psbt_init_alloc(wtx->num_inputs, wtx->num_outputs, 0, &psbt);
+		wally_err = wally_psbt_init_alloc(0, wtx->num_inputs, wtx->num_outputs, 0, &psbt);
 	assert(wally_err == WALLY_OK);
 	tal_add_destructor(psbt, psbt_destroy);
 
-	/* we can't have scripts on the psbt's global tx,
-	 * so we erase them/stash them until after it's been populated */
-	scripts = tal_arr(NULL, u8 *, wtx->num_inputs);
-	script_lens = tal_arr(NULL, size_t, wtx->num_inputs);
-	witnesses = tal_arr(NULL, struct wally_tx_witness_stack *, wtx->num_inputs);
-	for (size_t i = 0; i < wtx->num_inputs; i++) {
-		scripts[i] = (u8 *)wtx->inputs[i].script;
-		wtx->inputs[i].script = NULL;
-		script_lens[i] = wtx->inputs[i].script_len;
-		wtx->inputs[i].script_len = 0;
-		witnesses[i] = wtx->inputs[i].witness;
-		wtx->inputs[i].witness = NULL;
-	}
-
-	wally_err = wally_psbt_set_global_tx(psbt, cast_const(struct wally_tx *, wtx));
+	/* Set directly: avoids psbt checks for non-NULL scripts/witnesses */
+	wally_err = wally_tx_clone_alloc(wtx, 0, &psbt->tx);
 	assert(wally_err == WALLY_OK);
+	/* Inputs/outs are pre-allocated above, 'add' them as empty dummies */
+	psbt->num_inputs = wtx->num_inputs;
+	psbt->num_outputs = wtx->num_outputs;
 
-	/* set the scripts + witnesses back */
 	for (size_t i = 0; i < wtx->num_inputs; i++) {
-		int wally_err;
-
-		wtx->inputs[i].script = (unsigned char *)scripts[i];
-		wtx->inputs[i].script_len = script_lens[i];
-		wtx->inputs[i].witness = witnesses[i];
-
 		/* add these scripts + witnesses to the psbt */
-		if (scripts[i]) {
+		if (wtx->inputs[i].script) {
 			wally_err =
-				wally_psbt_input_set_final_script_sig(&psbt->inputs[i],
-								      (unsigned char *)scripts[i],
-								      script_lens[i]);
+				wally_psbt_input_set_final_scriptsig(&psbt->inputs[i],
+								     wtx->inputs[i].script,
+								     wtx->inputs[i].script_len);
 			assert(wally_err == WALLY_OK);
 		}
-		if (witnesses[i]) {
+		if (wtx->inputs[i].witness) {
 			wally_err =
 				wally_psbt_input_set_final_witness(&psbt->inputs[i],
-								   witnesses[i]);
+								   wtx->inputs[i].witness);
 			assert(wally_err == WALLY_OK);
 		}
 	}
 
-	tal_free(witnesses);
-	tal_free(scripts);
-	tal_free(script_lens);
-
 	return tal_steal(ctx, psbt);
 }
 
 bool psbt_is_finalized(struct wally_psbt *psbt)
 {
 	for (size_t i = 0; i < psbt->num_inputs; i++) {
-		if (!psbt->inputs[i].final_script_sig &&
+		if (!psbt->inputs[i].final_scriptsig &&
 				!psbt->inputs[i].final_witness)
 			return false;
 	}
@@ -235,35 +211,27 @@ void psbt_input_add_pubkey(struct wally_psbt *psbt, size_t in,
 	/* we serialize the compressed version of the key, wally likes this */
 	pubkey_to_der(pk_der, pubkey);
 
-	if (!psbt->inputs[in].keypaths)
-		if (wally_keypath_map_init_alloc(1, &psbt->inputs[in].keypaths) != WALLY_OK)
-			abort();
-
-	wally_err = wally_add_new_keypath(psbt->inputs[in].keypaths,
+	wally_err = wally_psbt_input_add_keypath_item(&psbt->inputs[in],
 						      pk_der, sizeof(pk_der),
 						      fingerprint, sizeof(fingerprint),
 						      empty_path, ARRAY_SIZE(empty_path));
-
 	assert(wally_err == WALLY_OK);
 }
 
-bool psbt_input_set_partial_sig(struct wally_psbt *psbt, size_t in,
+bool psbt_input_set_signature(struct wally_psbt *psbt, size_t in,
 			      const struct pubkey *pubkey,
 			      const struct bitcoin_signature *sig)
 {
 	u8 pk_der[PUBKEY_CMPR_LEN];
 
 	assert(in < psbt->num_inputs);
-	if (!psbt->inputs[in].partial_sigs)
-		if (wally_partial_sigs_map_init_alloc(1, &psbt->inputs[in].partial_sigs) != WALLY_OK)
-			return false;
 
 	/* we serialize the compressed version of the key, wally likes this */
 	pubkey_to_der(pk_der, pubkey);
-	wally_psbt_input_set_sighash_type(&psbt->inputs[in], sig->sighash_type);
-	return wally_add_new_partial_sig(psbt->inputs[in].partial_sigs,
+	wally_psbt_input_set_sighash(&psbt->inputs[in], sig->sighash_type);
+	return wally_psbt_input_add_signature(&psbt->inputs[in],
 					      pk_der, sizeof(pk_der),
-					 cast_const(unsigned char *, sig->s.data),
+					      sig->s.data,
 					      sizeof(sig->s.data)) == WALLY_OK;
 }
 
@@ -383,11 +351,10 @@ void psbt_elements_input_init_witness(struct wally_psbt *psbt, size_t in,
 			asset, nonce);
 
 	if (asset->value > 0)
-		wally_psbt_elements_input_set_value(&psbt->inputs[in],
-						    asset->value);
+		wally_psbt_input_set_value(&psbt->inputs[in], asset->value);
 
 	/* PSET expects an asset tag without the prefix */
-	if (wally_psbt_elements_input_set_asset(&psbt->inputs[in],
+	if (wally_psbt_input_set_asset(&psbt->inputs[in],
 				       asset->asset + 1,
 				       ELEMENTS_ASSET_LEN - 1) != WALLY_OK)
 		abort();
@@ -403,7 +370,7 @@ void psbt_elements_input_init(struct wally_psbt *psbt, size_t in,
 					  asset, nonce);
 
 	if (asset->value > 0) {
-		if (wally_psbt_elements_input_set_value(
+		if (wally_psbt_input_set_value(
 					&psbt->inputs[in],
 					asset->value) != WALLY_OK)
 			abort();
@@ -412,8 +379,7 @@ void psbt_elements_input_init(struct wally_psbt *psbt, size_t in,
 
 	/* PSET expects an asset tag without the prefix */
 	/* FIXME: Verify that we're sending unblinded asset tag */
-	if (wally_psbt_elements_input_set_asset(
-					&psbt->inputs[in],
+	if (wally_psbt_input_set_asset(&psbt->inputs[in],
 				       asset->asset + 1,
 				       ELEMENTS_ASSET_LEN - 1) != WALLY_OK)
 		abort();
@@ -440,9 +406,9 @@ struct amount_sat psbt_input_get_amount(struct wally_psbt *psbt,
 			wally_tx_output_get_amount(psbt->inputs[in].witness_utxo);
 		assert(amount_asset_is_main(&amt_asset));
 		val = amount_asset_to_sat(&amt_asset);
-	} else if (psbt->inputs[in].non_witness_utxo) {
+	} else if (psbt->inputs[in].utxo) {
 		int idx = psbt->tx->inputs[in].index;
-		struct wally_tx *prev_tx = psbt->inputs[in].non_witness_utxo;
+		struct wally_tx *prev_tx = psbt->inputs[in].utxo;
 		val = amount_sat(prev_tx->outputs[idx].satoshi);
 	} else
 		abort();
@@ -464,14 +430,14 @@ struct wally_tx *psbt_finalize(struct wally_psbt *psbt, bool finalize_in_place)
 	} else
 		tmppsbt = cast_const(struct wally_psbt *, psbt);
 
-	if (wally_finalize_psbt(tmppsbt) != WALLY_OK) {
+	if (wally_psbt_finalize(tmppsbt) != WALLY_OK) {
 		if (!finalize_in_place)
 			wally_psbt_free(tmppsbt);
 		return NULL;
 	}
 
 	if (psbt_is_finalized(tmppsbt)
-		&& wally_extract_psbt(tmppsbt, &wtx) == WALLY_OK) {
+		&& wally_psbt_extract(tmppsbt, &wtx) == WALLY_OK) {
 		if (!finalize_in_place)
 			wally_psbt_free(tmppsbt);
 		return wtx;
@@ -494,7 +460,7 @@ char *psbt_to_b64(const tal_t *ctx, const struct wally_psbt *psbt)
 	char *serialized_psbt, *ret_val;
 	int ret;
 
-	ret = wally_psbt_to_base64(cast_const(struct wally_psbt *, psbt),
+	ret = wally_psbt_to_base64(cast_const(struct wally_psbt *, psbt), 0,
 				   &serialized_psbt);
 	assert(ret == WALLY_OK);
 
@@ -512,7 +478,7 @@ const u8 *psbt_get_bytes(const tal_t *ctx, const struct wally_psbt *psbt,
 	 */
 	size_t room = 1024 * 1000;
 	u8 *pbt_bytes = tal_arr(ctx, u8, room);
-	if (wally_psbt_to_bytes(psbt, pbt_bytes, room, bytes_written) != WALLY_OK) {
+	if (wally_psbt_to_bytes(psbt, 0, pbt_bytes, room, bytes_written) != WALLY_OK) {
 		/* something went wrong. bad libwally ?? */
 		abort();
 	}
@@ -564,7 +530,7 @@ struct wally_psbt *fromwire_wally_psbt(const tal_t *ctx,
 	/* Re-marshall for sanity check! */
 	u8 *tmpbuf = tal_arr(NULL, u8, psbt_byte_len);
 	size_t written;
-	if (wally_psbt_to_bytes(psbt, tmpbuf, psbt_byte_len, &written) != WALLY_OK) {
+	if (wally_psbt_to_bytes(psbt, 0, tmpbuf, psbt_byte_len, &written) != WALLY_OK) {
 		tal_free(tmpbuf);
 		tal_free(psbt);
 		return fromwire_fail(cursor, max);

bitcoin/psbt.h

@@ -52,7 +52,7 @@ void psbt_rm_output(struct wally_psbt *psbt,
 void psbt_input_add_pubkey(struct wally_psbt *psbt, size_t in,
 			   const struct pubkey *pubkey);
 
-WARN_UNUSED_RESULT bool psbt_input_set_partial_sig(struct wally_psbt *psbt, size_t in,
+WARN_UNUSED_RESULT bool psbt_input_set_signature(struct wally_psbt *psbt, size_t in,
 						 const struct pubkey *pubkey,
 						 const struct bitcoin_signature *sig);
 

bitcoin/tx.c

@@ -16,21 +16,6 @@
 
 #define SEGREGATED_WITNESS_FLAG 0x1
 
-/* FIXME: When wally exposes this, we will clash and can remove this one */
-int wally_tx_clone(struct wally_tx *tx, struct wally_tx **output)
-{
-	u8 *txlin = linearize_wtx(NULL, tx);
-	int flags = WALLY_TX_FLAG_USE_WITNESS;
-	int ret;
-
-	if (chainparams->is_elements)
-		flags |= WALLY_TX_FLAG_USE_ELEMENTS;
-
-	ret = wally_tx_from_bytes(txlin, tal_bytelen(txlin), flags, output);
-	tal_free(txlin);
-	return ret;
-}
-
 struct bitcoin_tx_output *new_tx_output(const tal_t *ctx,
 					struct amount_sat amount,
 					const u8 *script)
@@ -393,7 +378,7 @@ void bitcoin_tx_input_set_script(struct bitcoin_tx *tx, int innum, u8 *script)
 	/* Also add to the psbt */
 	assert(innum < tx->psbt->num_inputs);
 	in = &tx->psbt->inputs[innum];
-	wally_psbt_input_set_final_script_sig(in, script, tal_bytelen(script));
+	wally_psbt_input_set_final_scriptsig(in, script, tal_bytelen(script));
 }
 
 const u8 *bitcoin_tx_input_get_witness(const tal_t *ctx,
@@ -540,7 +525,7 @@ struct bitcoin_tx *bitcoin_tx_with_psbt(const tal_t *ctx, struct wally_psbt *psb
 					   psbt->tx->locktime);
 	wally_tx_free(tx->wtx);
 	tx->wtx = psbt_finalize(psbt, false);
-	if (!tx->wtx && wally_tx_clone(psbt->tx, &tx->wtx) != WALLY_OK)
+	if (!tx->wtx && wally_tx_clone_alloc(psbt->tx, 0, &tx->wtx) != WALLY_OK)
 		return NULL;
 
 	tal_free(tx->psbt);

bitcoin/tx.h

@@ -233,8 +233,6 @@ void towire_bitcoin_txid(u8 **pptr, const struct bitcoin_txid *txid);
 void towire_bitcoin_tx(u8 **pptr, const struct bitcoin_tx *tx);
 void towire_bitcoin_tx_output(u8 **pptr, const struct bitcoin_tx_output *output);
 
-int wally_tx_clone(struct wally_tx *tx, struct wally_tx **output);
-
 /* Various weights of transaction parts. */
 size_t bitcoin_tx_core_weight(size_t num_inputs, size_t num_outputs);
 size_t bitcoin_tx_output_weight(size_t outscript_len);

channeld/channeld.c

@@ -1291,7 +1291,7 @@ static void handle_peer_commit_sig(struct peer *peer, const u8 *msg)
 			peer->next_index[LOCAL], LOCAL);
 
 	/* Set the commit_sig on the commitment tx psbt */
-	if (!psbt_input_set_partial_sig(txs[0]->psbt, 0,
+	if (!psbt_input_set_signature(txs[0]->psbt, 0,
 				      &peer->channel->funding_pubkey[REMOTE],
 				      &commit_sig))
 		status_failed(STATUS_FAIL_INTERNAL_ERROR,

external/libwally-core

@@ -1 +1 @@
-Subproject commit 1f45aef1e990e945710691e0e3637312f9a84e73
+Subproject commit 058aad016040c493f076215397ad76784144ce4e

hsmd/hsmd.c

@@ -1531,7 +1531,7 @@ static void sign_our_inputs(struct utxo **utxos, struct wally_psbt *psbt)
 			/* This line is basically the entire reason we have
 			 * to iterate through to match the psbt input
 			 * to the UTXO -- otherwise we would just
-			 * call wally_sign_psbt for every utxo privkey
+			 * call wally_psbt_sign for every utxo privkey
 			 * and be done with it. We can't do that though
 			 * because any UTXO that's derived from channel_info
 			 * requires the HSM to find the pubkey, and we
@@ -1539,8 +1539,9 @@ static void sign_our_inputs(struct utxo **utxos, struct wally_psbt *psbt)
 			 * of complexity in the calling code */
 			psbt_input_add_pubkey(psbt, j, &pubkey);
 
-			if (wally_sign_psbt(psbt, privkey.secret.data,
-					    sizeof(privkey.secret.data)) != WALLY_OK)
+			if (wally_psbt_sign(psbt, privkey.secret.data,
+					    sizeof(privkey.secret.data),
+					    EC_FLAG_GRIND_R) != WALLY_OK)
 				status_broken("Received wally_err attempting to "
 					      "sign utxo with key %s. PSBT: %s",
 					      type_to_string(tmpctx, struct pubkey,

openingd/openingd.c

@@ -839,7 +839,7 @@ static bool funder_finalize_channel_setup(struct state *state,
 	}
 
 	/* We save their sig to our first commitment tx */
-	if (!psbt_input_set_partial_sig((*tx)->psbt, 0,
+	if (!psbt_input_set_signature((*tx)->psbt, 0,
 				      &state->their_funding_pubkey,
 				      sig))
 		status_failed(STATUS_FAIL_INTERNAL_ERROR,

wallet/db.c

@@ -1284,7 +1284,7 @@ void migrate_last_tx_to_psbt(struct lightningd *ld, struct db *db,
 			abort();
 
 		last_sig.sighash_type = SIGHASH_ALL;
-		if (!psbt_input_set_partial_sig(last_tx->psbt, 0,
+		if (!psbt_input_set_signature(last_tx->psbt, 0,
 					      &remote_funding_pubkey, &last_sig))
 			abort();
 		psbt_input_add_pubkey(last_tx->psbt, 0,