From de86e29e16628eccbc5c4b049ecca0a80bf57e46 Mon Sep 17 00:00:00 2001 From: lisa neigut Date: Fri, 3 Apr 2020 16:56:57 -0500 Subject: [PATCH] coin moves: log all withdrawals when confirmed in a block This moves the notification for our coin spends from when it's successfully submited to the mempool to when they're confirmed in a block. We also add an 'informational' notice tagged as `spend_track` which can be used to track which transaction a wallet output was spent in. --- common/coin_mvt.c | 1 + common/coin_mvt.h | 1 + lightningd/chaintopology.c | 133 +++++++++++++++++++++++++++++++++++-- tests/test_misc.py | 23 ++++++- tests/test_plugin.py | 1 + wallet/wallet.c | 48 ++++++++++++- wallet/wallet.h | 12 +++- wallet/walletrpc.c | 74 --------------------- 8 files changed, 211 insertions(+), 82 deletions(-) diff --git a/common/coin_mvt.c b/common/coin_mvt.c index 073636ad9..1c7ab3bb3 100644 --- a/common/coin_mvt.c +++ b/common/coin_mvt.c @@ -19,6 +19,7 @@ static const char *mvt_tags[] = { "journal_entry", "onchain_htlc", "pushed", + "spend_track", }; const char *mvt_tag_str(enum mvt_tag tag) { diff --git a/common/coin_mvt.h b/common/coin_mvt.h index 8a3528543..c09596063 100644 --- a/common/coin_mvt.h +++ b/common/coin_mvt.h @@ -29,6 +29,7 @@ enum mvt_tag { JOURNAL = 6, ONCHAIN_HTLC = 7, PUSHED = 8, + SPEND_TRACK = 9, }; enum mvt_unit_type { diff --git a/lightningd/chaintopology.c b/lightningd/chaintopology.c index adf2b4cba..d5bcb2419 100644 --- a/lightningd/chaintopology.c +++ b/lightningd/chaintopology.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -23,6 +24,7 @@ #include #include #include +#include #include #include @@ -660,6 +662,92 @@ static void updates_complete(struct chain_topology *topo) next_topology_timer(topo); } +static void record_output_spend(struct lightningd *ld, + const struct bitcoin_txid *txid, + const struct bitcoin_txid *utxo_txid, + u32 vout, u32 blockheight, + struct amount_sat *input_amt) +{ + struct utxo *utxo; + struct chain_coin_mvt *mvt; + u8 *ctx = tal(NULL, u8); + + utxo = wallet_utxo_get(ctx, ld->wallet, utxo_txid, vout); + if (!utxo) + log_broken(ld->log, "No record of utxo %s:%d", + type_to_string(tmpctx, struct bitcoin_txid, + utxo_txid), + vout); + + *input_amt = utxo->amount; + mvt = new_chain_coin_mvt_sat(ctx, "wallet", txid, + utxo_txid, vout, NULL, + blockheight, + SPEND_TRACK, AMOUNT_SAT(0), + false, BTC); + if (!mvt) + fatal("unable to convert %s to msat", + type_to_string(tmpctx, struct amount_sat, + input_amt)); + notify_chain_mvt(ld, mvt); + tal_free(ctx); +} + +static void record_tx_outs_and_fees(struct lightningd *ld, const struct bitcoin_tx *tx, + struct bitcoin_txid *txid, u32 blockheight, + struct amount_sat inputs_total) +{ + struct amount_sat fee; + struct chain_coin_mvt *mvt; + size_t i; + u8 *ctx = tal(NULL, u8); + + if (!tx) + log_broken(ld->log, "We have no record of transaction %s", + type_to_string(ctx, struct bitcoin_txid, txid)); + + /* We record every output on this transaction as a withdraw */ + /* FIXME: collaborative tx will need to keep track of which + * outputs are ours */ + for (i = 0; i < tx->wtx->num_outputs; i++) { + struct amount_asset asset; + struct amount_sat outval; + if (elements_tx_output_is_fee(tx, i)) + continue; + asset = bitcoin_tx_output_get_amount(tx, i); + assert(amount_asset_is_main(&asset)); + outval = amount_asset_to_sat(&asset); + mvt = new_chain_coin_mvt_sat(ctx, "wallet", txid, + txid, i, NULL, + blockheight, WITHDRAWAL, + outval, false, BTC); + if (!mvt) + fatal("unable to convert %s to msat", + type_to_string(tmpctx, struct amount_sat, &fee)); + + notify_chain_mvt(ld, mvt); + } + + fee = bitcoin_tx_compute_fee_w_inputs(tx, inputs_total); + + /* Note that to figure out the *total* 'onchain' + * cost of a channel, you'll want to also include + * fees logged here, to the 'wallet' account (for funding tx). + * You can do this in post by accounting for any 'chain_fees' logged for + * the funding txid when looking at a channel. */ + mvt = new_chain_coin_mvt_sat(ctx, "wallet", txid, + NULL, 0, NULL, blockheight, + CHAIN_FEES, fee, false, BTC); + + if (!mvt) + fatal("unable to convert %s to msat", + type_to_string(tmpctx, struct amount_sat, &fee)); + + notify_chain_mvt(ld, mvt); + + tal_free(ctx); +} + /** * topo_update_spends -- Tell the wallet about all spent outpoints */ @@ -668,19 +756,56 @@ static void topo_update_spends(struct chain_topology *topo, struct block *b) const struct short_channel_id *scid; for (size_t i = 0; i < tal_count(b->full_txs); i++) { const struct bitcoin_tx *tx = b->full_txs[i]; + bool our_tx = false; + struct bitcoin_txid txid; + struct amount_sat inputs_total = AMOUNT_SAT(0); + + bitcoin_txid(tx, &txid); + for (size_t j = 0; j < tx->wtx->num_inputs; j++) { const struct wally_tx_input *input = &tx->wtx->inputs[j]; - struct bitcoin_txid txid; - bitcoin_tx_input_get_txid(tx, j, &txid); + struct bitcoin_txid outpoint_txid; + bool our_spend; + + bitcoin_tx_input_get_txid(tx, j, &outpoint_txid); scid = wallet_outpoint_spend(topo->ld->wallet, tmpctx, - b->height, &txid, - input->index); + b->height, &outpoint_txid, + input->index, + &our_spend); if (scid) { gossipd_notify_spend(topo->bitcoind->ld, scid); tal_free(scid); } + + our_tx |= our_spend; + if (our_spend) { + struct amount_sat input_amt; + bool ok; + + record_output_spend(topo->ld, &txid, &outpoint_txid, + input->index, b->height, &input_amt); + ok = amount_sat_add(&inputs_total, inputs_total, input_amt); + assert(ok); + } else if (our_tx) + log_broken(topo->ld->log, "Recording fee spend for tx %s but " + "our wallet did not contribute input %s:%d", + type_to_string(tmpctx, struct bitcoin_txid, + &txid), + type_to_string(tmpctx, struct bitcoin_txid, + &outpoint_txid), + input->index); + } + + /* For now we assume that if one of the spent utxos + * in this tx is 'ours', that we own all of the + * utxos and therefore paid all of the fees + * FIXME: update once interactive tx construction + * is a reality */ + if (our_tx) + record_tx_outs_and_fees(topo->ld, tx, &txid, + b->height, inputs_total); } } diff --git a/tests/test_misc.py b/tests/test_misc.py index e75b249cf..699b25fb4 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -604,7 +604,13 @@ def test_withdraw_misc(node_factory, bitcoind, chainparams): with pytest.raises(RpcError, match=r'Cannot afford transaction'): l1.rpc.withdraw(waddr, 'all') + # Coins aren't counted as moved until we receive notice they've + # been mined. + assert account_balance(l1, 'wallet') == 11974560000 + bitcoind.generate_block(1) + sync_blockheight(bitcoind, [l1]) assert account_balance(l1, 'wallet') == 0 + wallet_moves = [ {'type': 'chain_mvt', 'credit': 2000000000, 'debit': 0, 'tag': 'deposit'}, {'type': 'chain_mvt', 'credit': 2000000000, 'debit': 0, 'tag': 'deposit'}, @@ -616,25 +622,40 @@ def test_withdraw_misc(node_factory, bitcoind, chainparams): {'type': 'chain_mvt', 'credit': 2000000000, 'debit': 0, 'tag': 'deposit'}, {'type': 'chain_mvt', 'credit': 2000000000, 'debit': 0, 'tag': 'deposit'}, {'type': 'chain_mvt', 'credit': 2000000000, 'debit': 0, 'tag': 'deposit'}, + {'type': 'chain_mvt', 'credit': 0, 'debit': 0, 'tag': 'spend_track'}, + {'type': 'chain_mvt', 'credit': 0, 'debit': 0, 'tag': 'spend_track'}, {'type': 'chain_mvt', 'credit': 0, 'debit': 1993730000, 'tag': 'withdrawal'}, {'type': 'chain_mvt', 'credit': 0, 'debit': 2000000000, 'tag': 'withdrawal'}, {'type': 'chain_mvt', 'credit': 0, 'debit': 6270000, 'tag': 'chain_fees'}, - {'type': 'chain_mvt', 'credit': 1993730000, 'debit': 0, 'tag': 'deposit'}, + {'type': 'chain_mvt', 'credit': 0, 'debit': 0, 'tag': 'spend_track'}, + {'type': 'chain_mvt', 'credit': 0, 'debit': 0, 'tag': 'spend_track'}, {'type': 'chain_mvt', 'credit': 0, 'debit': 1993730000, 'tag': 'withdrawal'}, {'type': 'chain_mvt', 'credit': 0, 'debit': 2000000000, 'tag': 'withdrawal'}, {'type': 'chain_mvt', 'credit': 0, 'debit': 6270000, 'tag': 'chain_fees'}, {'type': 'chain_mvt', 'credit': 1993730000, 'debit': 0, 'tag': 'deposit'}, + {'type': 'chain_mvt', 'credit': 1993730000, 'debit': 0, 'tag': 'deposit'}, + {'type': 'chain_mvt', 'credit': 0, 'debit': 0, 'tag': 'spend_track'}, + {'type': 'chain_mvt', 'credit': 0, 'debit': 0, 'tag': 'spend_track'}, {'type': 'chain_mvt', 'credit': 0, 'debit': 1993730000, 'tag': 'withdrawal'}, {'type': 'chain_mvt', 'credit': 0, 'debit': 2000000000, 'tag': 'withdrawal'}, {'type': 'chain_mvt', 'credit': 0, 'debit': 6270000, 'tag': 'chain_fees'}, {'type': 'chain_mvt', 'credit': 1993730000, 'debit': 0, 'tag': 'deposit'}, + {'type': 'chain_mvt', 'credit': 0, 'debit': 0, 'tag': 'spend_track'}, + {'type': 'chain_mvt', 'credit': 0, 'debit': 0, 'tag': 'spend_track'}, {'type': 'chain_mvt', 'credit': 0, 'debit': 1993370000, 'tag': 'withdrawal'}, {'type': 'chain_mvt', 'credit': 0, 'debit': 2000000000, 'tag': 'withdrawal'}, {'type': 'chain_mvt', 'credit': 0, 'debit': 6630000, 'tag': 'chain_fees'}, {'type': 'chain_mvt', 'credit': 1993370000, 'debit': 0, 'tag': 'deposit'}, + {'type': 'chain_mvt', 'credit': 0, 'debit': 0, 'tag': 'spend_track'}, + {'type': 'chain_mvt', 'credit': 0, 'debit': 0, 'tag': 'spend_track'}, + {'type': 'chain_mvt', 'credit': 0, 'debit': 0, 'tag': 'spend_track'}, + {'type': 'chain_mvt', 'credit': 0, 'debit': 0, 'tag': 'spend_track'}, + {'type': 'chain_mvt', 'credit': 0, 'debit': 0, 'tag': 'spend_track'}, + {'type': 'chain_mvt', 'credit': 0, 'debit': 0, 'tag': 'spend_track'}, {'type': 'chain_mvt', 'credit': 0, 'debit': 11961030000, 'tag': 'withdrawal'}, {'type': 'chain_mvt', 'credit': 0, 'debit': 13530000, 'tag': 'chain_fees'}, {'type': 'chain_mvt', 'credit': 11961030000, 'debit': 0, 'tag': 'deposit'}, + {'type': 'chain_mvt', 'credit': 0, 'debit': 0, 'tag': 'spend_track'}, {'type': 'chain_mvt', 'credit': 0, 'debit': 11957378000, 'tag': 'withdrawal'}, {'type': 'chain_mvt', 'credit': 0, 'debit': 3652000, 'tag': 'chain_fees'}, ] diff --git a/tests/test_plugin.py b/tests/test_plugin.py index 0b760f6bb..e8dcea1b6 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -1381,6 +1381,7 @@ def test_coin_movement_notices(node_factory, bitcoind): ] l2_wallet_mvts = [ {'type': 'chain_mvt', 'credit': 2000000000, 'debit': 0, 'tag': 'deposit'}, + {'type': 'chain_mvt', 'credit': 0, 'debit': 0, 'tag': 'spend_track'}, {'type': 'chain_mvt', 'credit': 0, 'debit': 995418000, 'tag': 'withdrawal'}, {'type': 'chain_mvt', 'credit': 0, 'debit': 1000000000, 'tag': 'withdrawal'}, {'type': 'chain_mvt', 'credit': 0, 'debit': 4582000, 'tag': 'chain_fees'}, diff --git a/wallet/wallet.c b/wallet/wallet.c index 3e4e14832..7c79812da 100644 --- a/wallet/wallet.c +++ b/wallet/wallet.c @@ -314,6 +314,46 @@ struct utxo **wallet_get_unconfirmed_closeinfo_utxos(const tal_t *ctx, return results; } +struct utxo *wallet_utxo_get(const tal_t *ctx, struct wallet *w, + const struct bitcoin_txid *txid, + u32 outnum) +{ + struct db_stmt *stmt; + struct utxo *utxo; + + stmt = db_prepare_v2(w->db, SQL("SELECT" + " prev_out_tx" + ", prev_out_index" + ", value" + ", type" + ", status" + ", keyindex" + ", channel_id" + ", peer_id" + ", commitment_point" + ", confirmation_height" + ", spend_height" + ", scriptpubkey" + " FROM outputs" + " WHERE prev_out_tx = ?" + " AND prev_out_index = ?")); + + db_bind_sha256d(stmt, 0, &txid->shad); + db_bind_int(stmt, 1, outnum); + + db_query_prepared(stmt); + + if (!db_step(stmt)) { + tal_free(stmt); + return NULL; + } + + utxo = wallet_stmt2output(ctx, stmt); + tal_free(stmt); + + return utxo; +} + /** * unreserve_utxo - Mark a reserved UTXO as available again */ @@ -2904,7 +2944,8 @@ void wallet_blocks_rollback(struct wallet *w, u32 height) const struct short_channel_id * wallet_outpoint_spend(struct wallet *w, const tal_t *ctx, const u32 blockheight, - const struct bitcoin_txid *txid, const u32 outnum) + const struct bitcoin_txid *txid, const u32 outnum, + bool *our_spend) { struct short_channel_id *scid; struct db_stmt *stmt; @@ -2921,7 +2962,10 @@ wallet_outpoint_spend(struct wallet *w, const tal_t *ctx, const u32 blockheight, db_bind_int(stmt, 2, outnum); db_exec_prepared_v2(take(stmt)); - } + + *our_spend = true; + } else + *our_spend = false; if (outpointfilter_matches(w->utxoset_outpoints, txid, outnum)) { stmt = db_prepare_v2(w->db, SQL("UPDATE utxoset " diff --git a/wallet/wallet.h b/wallet/wallet.h index d68c9d4f3..bc8d0c323 100644 --- a/wallet/wallet.h +++ b/wallet/wallet.h @@ -384,6 +384,14 @@ struct utxo **wallet_get_utxos(const tal_t *ctx, struct wallet *w, struct utxo **wallet_get_unconfirmed_closeinfo_utxos(const tal_t *ctx, struct wallet *w); +/** wallet_utxo_get - Retrive a utxo. + * + * Returns a utxo, or NULL if not found. + */ +struct utxo *wallet_utxo_get(const tal_t *ctx, struct wallet *w, + const struct bitcoin_txid *txid, + u32 outnum); + const struct utxo **wallet_select_coins(const tal_t *ctx, struct wallet *w, bool with_change, struct amount_sat value, @@ -1091,12 +1099,14 @@ bool wallet_have_block(struct wallet *w, u32 blockheight); * Given the outpoint (txid, outnum), and the blockheight, mark the * corresponding DB entries as spent at the blockheight. * + * @our_spend - set to true if found in our wallet's output set, false otherwise * @return scid The short_channel_id corresponding to the spent outpoint, if * any. */ const struct short_channel_id * wallet_outpoint_spend(struct wallet *w, const tal_t *ctx, const u32 blockheight, - const struct bitcoin_txid *txid, const u32 outnum); + const struct bitcoin_txid *txid, const u32 outnum, + bool *our_spend); struct outpoint *wallet_outpoint_for_scid(struct wallet *w, tal_t *ctx, const struct short_channel_id *scid); diff --git a/wallet/walletrpc.c b/wallet/walletrpc.c index df8e52c12..ebd8e96c9 100644 --- a/wallet/walletrpc.c +++ b/wallet/walletrpc.c @@ -6,7 +6,6 @@ #include #include #include -#include #include #include #include @@ -23,7 +22,6 @@ #include #include #include -#include #include #include #include @@ -36,77 +34,6 @@ #include #include -static struct amount_sat compute_fee(const struct bitcoin_tx *tx, - struct unreleased_tx *utx) -{ - size_t i; - bool ok; - struct amount_sat input_sum = AMOUNT_SAT(0); - - /* ok so, the easiest thing to do is to add up all the inputs, - * separately, and then compute the fee from the outputs. - * 'normally' we'd just pass in the tx to `bitcoin_compute_fee` - * but due to how serialization works, the input amounts aren't - * preserved here */ - /* FIXME: use `bitcoin_compute_fee` once input amounts - * are preserved across the wire */ - for (i = 0; i < tal_count(utx->wtx->utxos); i++) { - ok = amount_sat_add(&input_sum, input_sum, - utx->wtx->utxos[i]->amount); - assert(ok); - } - - return bitcoin_tx_compute_fee_w_inputs(tx, input_sum); -} -static void record_coin_moves(struct lightningd *ld, - struct unreleased_tx *utx) -{ - struct amount_sat fees; - struct chain_coin_mvt *mvt; - size_t i; - - /* record each of the outputs as a withdrawal */ - for (i = 0; i < utx->tx->wtx->num_outputs; i++) { - struct amount_asset asset; - struct amount_sat sats; - asset = bitcoin_tx_output_get_amount(utx->tx, i); - if (elements_tx_output_is_fee(utx->tx, i) || - !amount_asset_is_main(&asset)) { - /* FIXME: handle non-btc withdrawals */ - continue; - } - sats = amount_asset_to_sat(&asset); - mvt = new_chain_coin_mvt_sat(utx, "wallet", &utx->txid, - &utx->txid, i, NULL, 0, - WITHDRAWAL, sats, - false, BTC); - if (!mvt) - fatal("unable to convert %s to msat", - type_to_string(tmpctx, struct amount_sat, - &sats)); - notify_chain_mvt(ld, mvt); - } - - /* we can't use bitcoin_tx_compute_fee because the input - * amounts aren't set... */ - fees = compute_fee(utx->tx, utx); - - /* Note that to figure out the *total* 'onchain' - * cost of a channel, you'll want to also include - * fees logged here, to the 'wallet' account (for funding tx). - * You can do this in post by accounting for any 'chain_fees' logged for - * the funding txid when looking at a channel. */ - mvt = new_chain_coin_mvt_sat(utx, "wallet", &utx->txid, - NULL, 0, NULL, 0, CHAIN_FEES, - fees, false, BTC); - - if (!mvt) - fatal("unable to convert %s to msat", - type_to_string(tmpctx, struct amount_sat, &fees)); - - notify_chain_mvt(ld, mvt); -} - /** * wallet_withdrawal_broadcast - The tx has been broadcast (or it failed) * @@ -130,7 +57,6 @@ static void wallet_withdrawal_broadcast(struct bitcoind *bitcoind UNUSED, if (success) { /* Mark used outputs as spent */ wallet_confirm_utxos(ld->wallet, utx->wtx->utxos); - record_coin_moves(ld, utx); /* Extract the change output and add it to the DB */ wallet_extract_owned_outputs(ld->wallet, utx->tx, NULL, &change);