coin moves: log all withdrawals when confirmed in a block

This moves the notification for our coin spends from when it's
successfully submited to the mempool to when they're confirmed in a
block.

We also add an 'informational' notice tagged as `spend_track` which
can be used to track which transaction a wallet output was spent in.

common/coin_mvt.c

@@ -19,6 +19,7 @@ static const char *mvt_tags[] = {
 	"journal_entry",
 	"onchain_htlc",
 	"pushed",
+	"spend_track",
 };
 const char *mvt_tag_str(enum mvt_tag tag)
 {

common/coin_mvt.h

@@ -29,6 +29,7 @@ enum mvt_tag {
 	JOURNAL = 6,
 	ONCHAIN_HTLC = 7,
 	PUSHED = 8,
+	SPEND_TRACK = 9,
 };
 
 enum mvt_unit_type {

lightningd/chaintopology.c

@@ -14,6 +14,7 @@
 #include <ccan/build_assert/build_assert.h>
 #include <ccan/io/io.h>
 #include <ccan/tal/str/str.h>
+#include <common/coin_mvt.h>
 #include <common/configdir.h>
 #include <common/json_command.h>
 #include <common/jsonrpc_errors.h>
@@ -23,6 +24,7 @@
 #include <common/utils.h>
 #include <inttypes.h>
 #include <lightningd/channel_control.h>
+#include <lightningd/coin_mvts.h>
 #include <lightningd/gossip_control.h>
 #include <lightningd/io_loop_with_timers.h>
 
@@ -660,6 +662,92 @@ static void updates_complete(struct chain_topology *topo)
 	next_topology_timer(topo);
 }
 
+static void record_output_spend(struct lightningd *ld,
+				const struct bitcoin_txid *txid,
+				const struct bitcoin_txid *utxo_txid,
+				u32 vout, u32 blockheight,
+				struct amount_sat *input_amt)
+{
+	struct utxo *utxo;
+	struct chain_coin_mvt *mvt;
+	u8 *ctx = tal(NULL, u8);
+
+	utxo = wallet_utxo_get(ctx, ld->wallet, utxo_txid, vout);
+	if (!utxo)
+		log_broken(ld->log, "No record of utxo %s:%d",
+			    type_to_string(tmpctx, struct bitcoin_txid,
+					   utxo_txid),
+			    vout);
+
+	*input_amt = utxo->amount;
+	mvt = new_chain_coin_mvt_sat(ctx, "wallet", txid,
+				     utxo_txid, vout, NULL,
+				     blockheight,
+				     SPEND_TRACK, AMOUNT_SAT(0),
+				     false, BTC);
+	if (!mvt)
+		fatal("unable to convert %s to msat",
+		      type_to_string(tmpctx, struct amount_sat,
+				     input_amt));
+	notify_chain_mvt(ld, mvt);
+	tal_free(ctx);
+}
+
+static void record_tx_outs_and_fees(struct lightningd *ld, const struct bitcoin_tx *tx,
+				    struct bitcoin_txid *txid, u32 blockheight,
+				    struct amount_sat inputs_total)
+{
+	struct amount_sat fee;
+	struct chain_coin_mvt *mvt;
+	size_t i;
+	u8 *ctx = tal(NULL, u8);
+
+	if (!tx)
+		log_broken(ld->log, "We have no record of transaction %s",
+			   type_to_string(ctx, struct bitcoin_txid, txid));
+
+	/* We record every output on this transaction as a withdraw */
+	/* FIXME: collaborative tx will need to keep track of which
+	 *        outputs are ours */
+	for (i = 0; i < tx->wtx->num_outputs; i++) {
+		struct amount_asset asset;
+		struct amount_sat outval;
+		if (elements_tx_output_is_fee(tx, i))
+			continue;
+		asset = bitcoin_tx_output_get_amount(tx, i);
+		assert(amount_asset_is_main(&asset));
+		outval = amount_asset_to_sat(&asset);
+		mvt = new_chain_coin_mvt_sat(ctx, "wallet", txid,
+					     txid, i, NULL,
+					     blockheight, WITHDRAWAL,
+					     outval, false, BTC);
+		if (!mvt)
+			fatal("unable to convert %s to msat",
+			      type_to_string(tmpctx, struct amount_sat, &fee));
+
+		notify_chain_mvt(ld, mvt);
+	}
+
+	fee = bitcoin_tx_compute_fee_w_inputs(tx, inputs_total);
+
+	/* Note that to figure out the *total* 'onchain'
+	 * cost of a channel, you'll want to also include
+	 * fees logged here, to the 'wallet' account (for funding tx).
+	 * You can do this in post by accounting for any 'chain_fees' logged for
+	 * the funding txid when looking at a channel. */
+	mvt = new_chain_coin_mvt_sat(ctx, "wallet", txid,
+				     NULL, 0, NULL, blockheight,
+				     CHAIN_FEES, fee, false, BTC);
+
+	if (!mvt)
+		fatal("unable to convert %s to msat",
+		      type_to_string(tmpctx, struct amount_sat, &fee));
+
+	notify_chain_mvt(ld, mvt);
+
+	tal_free(ctx);
+}
+
 /**
  * topo_update_spends -- Tell the wallet about all spent outpoints
  */
@@ -668,19 +756,56 @@ static void topo_update_spends(struct chain_topology *topo, struct block *b)
 	const struct short_channel_id *scid;
 	for (size_t i = 0; i < tal_count(b->full_txs); i++) {
 		const struct bitcoin_tx *tx = b->full_txs[i];
+		bool our_tx = false;
+		struct bitcoin_txid txid;
+		struct amount_sat inputs_total = AMOUNT_SAT(0);
+
+		bitcoin_txid(tx, &txid);
+
 		for (size_t j = 0; j < tx->wtx->num_inputs; j++) {
 			const struct wally_tx_input *input = &tx->wtx->inputs[j];
-			struct bitcoin_txid txid;
-			bitcoin_tx_input_get_txid(tx, j, &txid);
+			struct bitcoin_txid outpoint_txid;
+			bool our_spend;
+
+			bitcoin_tx_input_get_txid(tx, j, &outpoint_txid);
 
 			scid = wallet_outpoint_spend(topo->ld->wallet, tmpctx,
-						     b->height, &txid,
-						     input->index);
+						     b->height, &outpoint_txid,
+						     input->index,
+						     &our_spend);
 			if (scid) {
 				gossipd_notify_spend(topo->bitcoind->ld, scid);
 				tal_free(scid);
 			}
+
+			our_tx |= our_spend;
+			if (our_spend) {
+				struct amount_sat input_amt;
+				bool ok;
+
+				record_output_spend(topo->ld, &txid, &outpoint_txid,
+						    input->index, b->height, &input_amt);
+				ok = amount_sat_add(&inputs_total, inputs_total, input_amt);
+				assert(ok);
+			} else if (our_tx)
+				log_broken(topo->ld->log, "Recording fee spend for tx %s but "
+					   "our wallet did not contribute input %s:%d",
+					   type_to_string(tmpctx, struct bitcoin_txid,
+							  &txid),
+					   type_to_string(tmpctx, struct bitcoin_txid,
+							  &outpoint_txid),
+					   input->index);
+
 		}
+
+		/* For now we assume that if one of the spent utxos
+		 * in this tx is 'ours', that we own all of the
+		 * utxos and therefore paid all of the fees
+		 * FIXME: update once interactive tx construction
+		 * is a reality */
+		if (our_tx)
+			record_tx_outs_and_fees(topo->ld, tx, &txid,
+						b->height, inputs_total);
 	}
 }
 

tests/test_misc.py

@@ -604,7 +604,13 @@ def test_withdraw_misc(node_factory, bitcoind, chainparams):
     with pytest.raises(RpcError, match=r'Cannot afford transaction'):
         l1.rpc.withdraw(waddr, 'all')
 
+    # Coins aren't counted as moved until we receive notice they've
+    # been mined.
+    assert account_balance(l1, 'wallet') == 11974560000
+    bitcoind.generate_block(1)
+    sync_blockheight(bitcoind, [l1])
     assert account_balance(l1, 'wallet') == 0
+
     wallet_moves = [
         {'type': 'chain_mvt', 'credit': 2000000000, 'debit': 0, 'tag': 'deposit'},
         {'type': 'chain_mvt', 'credit': 2000000000, 'debit': 0, 'tag': 'deposit'},
@@ -616,25 +622,40 @@ def test_withdraw_misc(node_factory, bitcoind, chainparams):
         {'type': 'chain_mvt', 'credit': 2000000000, 'debit': 0, 'tag': 'deposit'},
         {'type': 'chain_mvt', 'credit': 2000000000, 'debit': 0, 'tag': 'deposit'},
         {'type': 'chain_mvt', 'credit': 2000000000, 'debit': 0, 'tag': 'deposit'},
+        {'type': 'chain_mvt', 'credit': 0, 'debit': 0, 'tag': 'spend_track'},
+        {'type': 'chain_mvt', 'credit': 0, 'debit': 0, 'tag': 'spend_track'},
         {'type': 'chain_mvt', 'credit': 0, 'debit': 1993730000, 'tag': 'withdrawal'},
         {'type': 'chain_mvt', 'credit': 0, 'debit': 2000000000, 'tag': 'withdrawal'},
         {'type': 'chain_mvt', 'credit': 0, 'debit': 6270000, 'tag': 'chain_fees'},
-        {'type': 'chain_mvt', 'credit': 1993730000, 'debit': 0, 'tag': 'deposit'},
+        {'type': 'chain_mvt', 'credit': 0, 'debit': 0, 'tag': 'spend_track'},
+        {'type': 'chain_mvt', 'credit': 0, 'debit': 0, 'tag': 'spend_track'},
         {'type': 'chain_mvt', 'credit': 0, 'debit': 1993730000, 'tag': 'withdrawal'},
         {'type': 'chain_mvt', 'credit': 0, 'debit': 2000000000, 'tag': 'withdrawal'},
         {'type': 'chain_mvt', 'credit': 0, 'debit': 6270000, 'tag': 'chain_fees'},
         {'type': 'chain_mvt', 'credit': 1993730000, 'debit': 0, 'tag': 'deposit'},
+        {'type': 'chain_mvt', 'credit': 1993730000, 'debit': 0, 'tag': 'deposit'},
+        {'type': 'chain_mvt', 'credit': 0, 'debit': 0, 'tag': 'spend_track'},
+        {'type': 'chain_mvt', 'credit': 0, 'debit': 0, 'tag': 'spend_track'},
         {'type': 'chain_mvt', 'credit': 0, 'debit': 1993730000, 'tag': 'withdrawal'},
         {'type': 'chain_mvt', 'credit': 0, 'debit': 2000000000, 'tag': 'withdrawal'},
         {'type': 'chain_mvt', 'credit': 0, 'debit': 6270000, 'tag': 'chain_fees'},
         {'type': 'chain_mvt', 'credit': 1993730000, 'debit': 0, 'tag': 'deposit'},
+        {'type': 'chain_mvt', 'credit': 0, 'debit': 0, 'tag': 'spend_track'},
+        {'type': 'chain_mvt', 'credit': 0, 'debit': 0, 'tag': 'spend_track'},
         {'type': 'chain_mvt', 'credit': 0, 'debit': 1993370000, 'tag': 'withdrawal'},
         {'type': 'chain_mvt', 'credit': 0, 'debit': 2000000000, 'tag': 'withdrawal'},
         {'type': 'chain_mvt', 'credit': 0, 'debit': 6630000, 'tag': 'chain_fees'},
         {'type': 'chain_mvt', 'credit': 1993370000, 'debit': 0, 'tag': 'deposit'},
+        {'type': 'chain_mvt', 'credit': 0, 'debit': 0, 'tag': 'spend_track'},
+        {'type': 'chain_mvt', 'credit': 0, 'debit': 0, 'tag': 'spend_track'},
+        {'type': 'chain_mvt', 'credit': 0, 'debit': 0, 'tag': 'spend_track'},
+        {'type': 'chain_mvt', 'credit': 0, 'debit': 0, 'tag': 'spend_track'},
+        {'type': 'chain_mvt', 'credit': 0, 'debit': 0, 'tag': 'spend_track'},
+        {'type': 'chain_mvt', 'credit': 0, 'debit': 0, 'tag': 'spend_track'},
         {'type': 'chain_mvt', 'credit': 0, 'debit': 11961030000, 'tag': 'withdrawal'},
         {'type': 'chain_mvt', 'credit': 0, 'debit': 13530000, 'tag': 'chain_fees'},
         {'type': 'chain_mvt', 'credit': 11961030000, 'debit': 0, 'tag': 'deposit'},
+        {'type': 'chain_mvt', 'credit': 0, 'debit': 0, 'tag': 'spend_track'},
         {'type': 'chain_mvt', 'credit': 0, 'debit': 11957378000, 'tag': 'withdrawal'},
         {'type': 'chain_mvt', 'credit': 0, 'debit': 3652000, 'tag': 'chain_fees'},
     ]

tests/test_plugin.py

@@ -1381,6 +1381,7 @@ def test_coin_movement_notices(node_factory, bitcoind):
     ]
     l2_wallet_mvts = [
         {'type': 'chain_mvt', 'credit': 2000000000, 'debit': 0, 'tag': 'deposit'},
+        {'type': 'chain_mvt', 'credit': 0, 'debit': 0, 'tag': 'spend_track'},
         {'type': 'chain_mvt', 'credit': 0, 'debit': 995418000, 'tag': 'withdrawal'},
         {'type': 'chain_mvt', 'credit': 0, 'debit': 1000000000, 'tag': 'withdrawal'},
         {'type': 'chain_mvt', 'credit': 0, 'debit': 4582000, 'tag': 'chain_fees'},

wallet/wallet.c

@@ -314,6 +314,46 @@ struct utxo **wallet_get_unconfirmed_closeinfo_utxos(const tal_t *ctx,
 	return results;
 }
 
+struct utxo *wallet_utxo_get(const tal_t *ctx, struct wallet *w,
+			     const struct bitcoin_txid *txid,
+			     u32 outnum)
+{
+	struct db_stmt *stmt;
+	struct utxo *utxo;
+
+	stmt = db_prepare_v2(w->db, SQL("SELECT"
+					"  prev_out_tx"
+					", prev_out_index"
+					", value"
+					", type"
+					", status"
+					", keyindex"
+					", channel_id"
+					", peer_id"
+					", commitment_point"
+					", confirmation_height"
+					", spend_height"
+					", scriptpubkey"
+					" FROM outputs"
+					" WHERE prev_out_tx = ?"
+					" AND prev_out_index = ?"));
+
+	db_bind_sha256d(stmt, 0, &txid->shad);
+	db_bind_int(stmt, 1, outnum);
+
+	db_query_prepared(stmt);
+
+	if (!db_step(stmt)) {
+		tal_free(stmt);
+		return NULL;
+	}
+
+	utxo = wallet_stmt2output(ctx, stmt);
+	tal_free(stmt);
+
+	return utxo;
+}
+
 /**
  * unreserve_utxo - Mark a reserved UTXO as available again
  */
@@ -2904,7 +2944,8 @@ void wallet_blocks_rollback(struct wallet *w, u32 height)
 
 const struct short_channel_id *
 wallet_outpoint_spend(struct wallet *w, const tal_t *ctx, const u32 blockheight,
-		      const struct bitcoin_txid *txid, const u32 outnum)
+		      const struct bitcoin_txid *txid, const u32 outnum,
+		      bool *our_spend)
 {
 	struct short_channel_id *scid;
 	struct db_stmt *stmt;
@@ -2921,7 +2962,10 @@ wallet_outpoint_spend(struct wallet *w, const tal_t *ctx, const u32 blockheight,
 		db_bind_int(stmt, 2, outnum);
 
 		db_exec_prepared_v2(take(stmt));
-	}
+
+		*our_spend = true;
+	} else
+		*our_spend = false;
 
 	if (outpointfilter_matches(w->utxoset_outpoints, txid, outnum)) {
 		stmt = db_prepare_v2(w->db, SQL("UPDATE utxoset "

wallet/wallet.h

@@ -384,6 +384,14 @@ struct utxo **wallet_get_utxos(const tal_t *ctx, struct wallet *w,
 struct utxo **wallet_get_unconfirmed_closeinfo_utxos(const tal_t *ctx,
 						     struct wallet *w);
 
+/** wallet_utxo_get - Retrive a utxo.
+ *
+ * Returns a utxo, or NULL if not found.
+ */
+struct utxo *wallet_utxo_get(const tal_t *ctx, struct wallet *w,
+			     const struct bitcoin_txid *txid,
+			     u32 outnum);
+
 const struct utxo **wallet_select_coins(const tal_t *ctx, struct wallet *w,
 					bool with_change,
 					struct amount_sat value,
@@ -1091,12 +1099,14 @@ bool wallet_have_block(struct wallet *w, u32 blockheight);
  * Given the outpoint (txid, outnum), and the blockheight, mark the
  * corresponding DB entries as spent at the blockheight.
  *
+ * @our_spend - set to true if found in our wallet's output set, false otherwise
  * @return scid The short_channel_id corresponding to the spent outpoint, if
  *         any.
  */
 const struct short_channel_id *
 wallet_outpoint_spend(struct wallet *w, const tal_t *ctx, const u32 blockheight,
-		      const struct bitcoin_txid *txid, const u32 outnum);
+		      const struct bitcoin_txid *txid, const u32 outnum,
+		      bool *our_spend);
 
 struct outpoint *wallet_outpoint_for_scid(struct wallet *w, tal_t *ctx,
 					  const struct short_channel_id *scid);

wallet/walletrpc.c

@@ -6,7 +6,6 @@
 #include <ccan/tal/str/str.h>
 #include <common/addr.h>
 #include <common/bech32.h>
-#include <common/coin_mvt.h>
 #include <common/json_command.h>
 #include <common/json_helpers.h>
 #include <common/jsonrpc_errors.h>
@@ -23,7 +22,6 @@
 #include <inttypes.h>
 #include <lightningd/bitcoind.h>
 #include <lightningd/chaintopology.h>
-#include <lightningd/coin_mvts.h>
 #include <lightningd/hsm_control.h>
 #include <lightningd/json.h>
 #include <lightningd/jsonrpc.h>
@@ -36,77 +34,6 @@
 #include <wally_bip32.h>
 #include <wire/wire_sync.h>
 
-static struct amount_sat compute_fee(const struct bitcoin_tx *tx,
-				     struct unreleased_tx *utx)
-{
-	size_t i;
-	bool ok;
-	struct amount_sat input_sum = AMOUNT_SAT(0);
-
-	/* ok so, the easiest thing to do is to add up all the inputs,
-	 * separately, and then compute the fee from the outputs.
-	 * 'normally' we'd just pass in the tx to `bitcoin_compute_fee`
-	 * but due to how serialization works, the input amounts aren't
-	 * preserved here */
-	/* FIXME: use `bitcoin_compute_fee` once input amounts
-	 * are preserved across the wire */
-	for (i = 0; i < tal_count(utx->wtx->utxos); i++) {
-		ok = amount_sat_add(&input_sum, input_sum,
-				    utx->wtx->utxos[i]->amount);
-		assert(ok);
-	}
-
-	return bitcoin_tx_compute_fee_w_inputs(tx, input_sum);
-}
-static void record_coin_moves(struct lightningd *ld,
-			      struct unreleased_tx *utx)
-{
-	struct amount_sat fees;
-	struct chain_coin_mvt *mvt;
-	size_t i;
-
-	/* record each of the outputs as a withdrawal */
-	for (i = 0; i < utx->tx->wtx->num_outputs; i++) {
-		struct amount_asset asset;
-		struct amount_sat sats;
-		asset = bitcoin_tx_output_get_amount(utx->tx, i);
-		if (elements_tx_output_is_fee(utx->tx, i) ||
-		    !amount_asset_is_main(&asset)) {
-			/* FIXME: handle non-btc withdrawals */
-			continue;
-		}
-		sats = amount_asset_to_sat(&asset);
-		mvt = new_chain_coin_mvt_sat(utx, "wallet", &utx->txid,
-					     &utx->txid, i, NULL, 0,
-					     WITHDRAWAL, sats,
-					     false, BTC);
-		if (!mvt)
-			fatal("unable to convert %s to msat",
-			      type_to_string(tmpctx, struct amount_sat,
-					     &sats));
-		notify_chain_mvt(ld, mvt);
-	}
-
-	/* we can't use bitcoin_tx_compute_fee because the input
-	 * amounts aren't set... */
-	fees = compute_fee(utx->tx, utx);
-
-	/* Note that to figure out the *total* 'onchain'
-	 * cost of a channel, you'll want to also include
-	 * fees logged here, to the 'wallet' account (for funding tx).
-	 * You can do this in post by accounting for any 'chain_fees' logged for
-	 * the funding txid when looking at a channel. */
-	mvt = new_chain_coin_mvt_sat(utx, "wallet", &utx->txid,
-				     NULL, 0, NULL, 0, CHAIN_FEES,
-				     fees, false, BTC);
-
-	if (!mvt)
-		fatal("unable to convert %s to msat",
-		      type_to_string(tmpctx, struct amount_sat, &fees));
-
-	notify_chain_mvt(ld, mvt);
-}
-
 /**
  * wallet_withdrawal_broadcast - The tx has been broadcast (or it failed)
  *
@@ -130,7 +57,6 @@ static void wallet_withdrawal_broadcast(struct bitcoind *bitcoind UNUSED,
 	if (success) {
 		/* Mark used outputs as spent */
 		wallet_confirm_utxos(ld->wallet, utx->wtx->utxos);
-		record_coin_moves(ld, utx);
 
 		/* Extract the change output and add it to the DB */
 		wallet_extract_owned_outputs(ld->wallet, utx->tx, NULL, &change);