wallet: Move UTXO tracking to DB

Since we have a simple way to query the database for UTXOs we can
simplify some of the coin selection logic. That gets rid of the
in-memory list of UTXOs.

lightningd/build_utxos.c

@@ -7,15 +7,6 @@
 #include <utils.h>
 #include <wally_bip32.h>
 
-struct tracked_utxo {
-	struct list_node list;
-
-	/* Currently being used for a connection. */
-	bool reserved;
-
-	struct utxo utxo;
-};
-
 static void json_newaddr(struct command *cmd,
 			 const char *buffer, const jsmntok_t *params)
 {
@@ -133,27 +124,26 @@ static void json_addfunds(struct command *cmd,
 
 	/* Find an output we know how to spend. */
 	for (output = 0; output < tal_count(tx->output); output++) {
-		struct tracked_utxo *utxo;
+		struct utxo *utxo;
 		u32 index;
 		bool is_p2sh;
 
 		if (!can_spend(ld, tx->output[output].script, &index, &is_p2sh))
 			continue;
 
-		utxo = tal(ld, struct tracked_utxo);
-		utxo->utxo.keyindex = index;
-		utxo->utxo.is_p2sh = is_p2sh;
-		utxo->utxo.amount = tx->output[output].amount;
-		bitcoin_txid(tx, &utxo->utxo.txid);
-		utxo->utxo.outnum = output;
-		utxo->reserved = false;
-		if (!wallet_add_utxo(ld->wallet, &utxo->utxo, p2sh_wpkh)) {
+		utxo = tal(ld, struct utxo);
+		utxo->keyindex = index;
+		utxo->is_p2sh = is_p2sh;
+		utxo->amount = tx->output[output].amount;
+		utxo->status = output_state_available;
+		bitcoin_txid(tx, &utxo->txid);
+		utxo->outnum = output;
+		if (!wallet_add_utxo(ld->wallet, utxo, p2sh_wpkh)) {
 			command_fail(cmd, "Could add outputs to wallet");
 			tal_free(utxo);
 			return;
 		}
-		list_add_tail(&ld->utxos, &utxo->list);
-		total_satoshi += utxo->utxo.amount;
+		total_satoshi += utxo->amount;
 		num_utxos++;
 	}
 
@@ -179,21 +169,9 @@ AUTODATA(json_command, &addfunds_command);
 
 static void unreserve_utxo(struct lightningd *ld, const struct utxo *unres)
 {
-	struct tracked_utxo *utxo;
-
 	assert(wallet_update_output_status(ld->wallet, &unres->txid,
 					   unres->outnum, output_state_reserved,
 					   output_state_available));
-
-	list_for_each(&ld->utxos, utxo, list)
-	{
-		if (unres != &utxo->utxo)
-			continue;
-		assert(utxo->reserved);
-		utxo->reserved = false;
-		return;
-	}
-	abort();
 }
 
 static void destroy_utxos(const struct utxo **utxos, struct lightningd *ld)
@@ -215,26 +193,25 @@ const struct utxo **build_utxos(const tal_t *ctx,
 				u64 *change_satoshis, u32 *change_keyindex)
 {
 	size_t i = 0;
+	struct utxo **available;
 	const struct utxo **utxos = tal_arr(ctx, const struct utxo *, 0);
-	struct tracked_utxo *utxo;
 	/* We assume two outputs for the weight. */
 	u64 satoshi_in = 0, weight = (4 + (8 + 22) * 2 + 4) * 4;
 	u64 bip32_max_index = db_get_intvar(ld->wallet->db, "bip32_max_index", 0);
 
 	tal_add_destructor2(utxos, destroy_utxos, ld);
 
-	list_for_each(&ld->utxos, utxo, list) {
-		u64 fee;
+	db_begin_transaction(ld->wallet->db);
+	available = wallet_get_utxos(utxos, ld->wallet, output_state_available);
 
-		if (utxo->reserved)
-			continue;
+	for (i=0; i<tal_count(available); i++) {
+		u64 fee;
 
 		tal_resize(&utxos, i+1);
-		utxos[i] = &utxo->utxo;
-		utxo->reserved = true;
+		utxos[i] = tal_steal(utxos, available[i]);
 
 		assert(wallet_update_output_status(
-		    ld->wallet, &utxo->utxo.txid, utxo->utxo.outnum,
+		    ld->wallet, &available[i]->txid, available[i]->outnum,
 		    output_state_available, output_state_reserved));
 
 		/* Add this input's weight. */
@@ -258,10 +235,12 @@ const struct utxo **build_utxos(const tal_t *ctx,
 				db_set_intvar(ld->wallet->db, "bip32_max_index", *change_keyindex);
 			}
 
+			db_commit_transaction(ld->wallet->db);
+			tal_free(available);
 			return utxos;
 		}
-		i++;
 	}
-
+	db_rollback_transaction(ld->wallet->db);
+	tal_free(available);
 	return tal_free(utxos);
 }

lightningd/lightningd.c

@@ -108,7 +108,6 @@ static struct lightningd *new_lightningd(const tal_t *ctx)
 	list_head_init(&ld->peers);
 	ld->peer_counter = 0;
 	ld->dev_debug_subdaemon = NULL;
-	list_head_init(&ld->utxos);
 	htlc_end_map_init(&ld->htlc_ends);
 	ld->dev_disconnect_fd = -1;
 	ld->dstate.log_book = new_log_book(&ld->dstate, 20*1024*1024,LOG_INFORM);

lightningd/lightningd.h

@@ -53,9 +53,6 @@ struct lightningd {
 	/* If we have a --dev-disconnect file */
 	int dev_disconnect_fd;
 
-	/* UTXOs we have available to spend. */
-	struct list_head utxos;
-
 	/* HTLCs in flight. */
 	struct htlc_end_map htlc_ends;
 

lightningd/utxo.h

@@ -12,6 +12,7 @@ struct utxo {
 	u64 amount;
 	u32 keyindex;
 	bool is_p2sh;
+	u8 status;
 };
 
 void towire_utxo(u8 **pptr, const struct utxo *utxo);

wallet/wallet.c

@@ -1,5 +1,7 @@
 #include "wallet.h"
 
+#include <ccan/str/hex/hex.h>
+
 struct wallet *wallet_new(const tal_t *ctx, struct log *log)
 {
 	struct wallet *wallet = tal(ctx, struct wallet);
@@ -26,6 +28,23 @@ bool wallet_add_utxo(struct wallet *w, struct utxo *utxo,
 	return result;
 }
 
+/**
+ * wallet_stmt2output - Extract data from stmt and fill a utxo
+ *
+ * Returns true on success.
+ */
+static bool wallet_stmt2output(sqlite3_stmt *stmt, struct utxo *utxo)
+{
+	const unsigned char *hextxid = sqlite3_column_text(stmt, 0);
+	hex_decode((const char*)hextxid, sizeof(utxo->txid) * 2, &utxo->txid, sizeof(utxo->txid));
+	utxo->outnum = sqlite3_column_int(stmt, 1);
+	utxo->amount = sqlite3_column_int(stmt, 2);
+	utxo->is_p2sh = sqlite3_column_int(stmt, 3) == p2sh_wpkh;
+	utxo->status = sqlite3_column_int(stmt, 4);
+	utxo->keyindex = sqlite3_column_int(stmt, 5);
+	return true;
+}
+
 bool wallet_update_output_status(struct wallet *w,
 				 const struct sha256_double *txid,
 				 const u32 outnum, enum output_status oldstatus,
@@ -49,3 +68,27 @@ bool wallet_update_output_status(struct wallet *w,
 	tal_free(tmpctx);
 	return sqlite3_changes(w->db->sql) > 0;
 }
+
+struct utxo **wallet_get_utxos(const tal_t *ctx, struct wallet *w, const enum output_status state)
+{
+	struct utxo **results;
+	int i;
+	sqlite3_stmt *stmt =
+	    db_query(__func__, w->db, "SELECT prev_out_tx, prev_out_index, "
+				      "value, type, status, keyindex FROM "
+				      "outputs WHERE status=%d OR %d=255",
+		     state, state);
+
+	if (!stmt)
+		return NULL;
+
+	results = tal_arr(ctx, struct utxo*, 0);
+	for (i=0; sqlite3_step(stmt) == SQLITE_ROW; i++) {
+		tal_resize(&results, i+1);
+		results[i] = tal(results, struct utxo);
+		wallet_stmt2output(stmt, results[i]);
+	}
+	sqlite3_finalize(stmt);
+
+	return results;
+}

wallet/wallet.h

@@ -65,4 +65,13 @@ bool wallet_update_output_status(struct wallet *w,
 				 const u32 outnum, enum output_status oldstatus,
 				 enum output_status newstatus);
 
+/**
+ * wallet_get_utxos - Retrieve all utxos matching a given state
+ *
+ * Returns a `tal_arr` of `utxo` structs. Double indirection in order
+ * to be able to steal individual elements onto something else.
+ */
+struct utxo **wallet_get_utxos(const tal_t *ctx, struct wallet *w,
+			      const enum output_status state);
+
 #endif /* WALLET_WALLET_H */