wallet: Create and manage a sqlite3 database for the wallet

Makefile

@@ -220,6 +220,7 @@ default: $(PROGRAMS) doc-all daemon-all
 include doc/Makefile
 include bitcoin/Makefile
 include wire/Makefile
+include wallet/Makefile
 include lightningd/Makefile
 
 # Git doesn't maintain timestamps, so we only regen if git says we should.

wallet/.gitignore

@@ -0,0 +1 @@
+*_tests

wallet/Makefile

@@ -0,0 +1,33 @@
+#! /usr/bin/make
+
+# Designed to be run one level up
+wallet-wrongdir:
+	$(MAKE) -C .. lightningd-all
+
+check: wallet/tests
+
+WALLET_LIB_SRC :=					\
+	wallet/db.c
+
+WALLET_LIB_OBJS := $(WALLET_LIB_SRC:.c=.o)
+WALLET_LIB_HEADERS := $(WALLET_LIB_SRC:.c=.h)
+
+WALLET_TEST_SRC := $(wildcard wallet/*_tests.c)
+WALLET_TEST_OBJS := $(WALLET_TEST_SRC:.c=.o)
+WALLET_TEST_PROGRAMS := $(WALLET_TEST_OBJS:.o=)
+
+$(WALLET_TEST_OBJS): $(WALLET_LIB_OBJS)
+
+$(WALLET_TEST_PROGRAMS): $(CCAN_OBJS) daemon/log.o type_to_string.o daemon/pseudorand.o utils.o libsodium.a
+
+$(WALLET_TEST_OBJS): $(CCAN_HEADERS)
+wallet/tests: $(WALLET_TEST_PROGRAMS:%=unittest/%)
+
+check-whitespace: $(WALLET_LIB_SRC:%=check-whitespace/%) $(WALLET_LIB_HEADERS:%=check-whitespace/%)
+
+check-makefile: check-lightningd-makefile
+
+clean: wallet-clean
+
+wallet-clean:
+	$(RM) $(WALLET_LIB_OBJS)

wallet/db.c

@@ -0,0 +1,248 @@
+#include "db.h"
+
+#include "daemon/log.h"
+#include "lightningd/lightningd.h"
+#include <ccan/tal/str/str.h>
+#include <ccan/tal/tal.h>
+
+#define DB_FILE "lightningd.sqlite3"
+
+/* Do not reorder or remove elements from this array, it is used to
+ * migrate existing databases from a previous state, based on the
+ * string indices */
+char *dbmigrations[] = {
+    "CREATE TABLE version (version INTEGER)",
+    "INSERT INTO version VALUES (1)",
+    "CREATE TABLE outputs ( \
+       prev_out_tx BLOB,			 \
+       prev_out_index INTEGER,			 \
+       value INTEGER,				 \
+       type INTEGER,				 \
+       status INTEGER,				 \
+       keyindex INTEGER,			 \
+       PRIMARY KEY (prev_out_tx, prev_out_index) \
+    );",
+    "CREATE TABLE vars (name VARCHAR(32), val VARCHAR(255), PRIMARY KEY (name));",
+    NULL,
+};
+
+bool PRINTF_FMT(3, 4)
+    db_exec(const char *caller, struct db *db, const char *fmt, ...)
+{
+	va_list ap;
+	char *cmd, *errmsg;
+	int err;
+
+	if (db->in_transaction && db->err)
+		return false;
+
+	va_start(ap, fmt);
+	cmd = tal_vfmt(db, fmt, ap);
+	va_end(ap);
+
+	err = sqlite3_exec(db->sql, cmd, NULL, NULL, &errmsg);
+	if (err != SQLITE_OK) {
+		tal_free(db->err);
+		db->err = tal_fmt(db, "%s:%s:%s:%s", caller,
+				  sqlite3_errstr(err), cmd, errmsg);
+		sqlite3_free(errmsg);
+		tal_free(cmd);
+		return false;
+	}
+	tal_free(cmd);
+	return true;
+}
+
+sqlite3_stmt *PRINTF_FMT(3, 4)
+    db_query(const char *caller, struct db *db, const char *fmt, ...)
+{
+	va_list ap;
+	char *query;
+	sqlite3_stmt *stmt;
+	int err;
+
+	if (db->in_transaction && db->err)
+		return NULL;
+
+	va_start(ap, fmt);
+	query = tal_vfmt(db, fmt, ap);
+	va_end(ap);
+
+	err = sqlite3_prepare_v2(db->sql, query, -1, &stmt, NULL);
+	if (err != SQLITE_OK) {
+		db->err = tal_fmt(db, "%s:%s:%s:%s", caller,
+				  sqlite3_errstr(err), query, sqlite3_errmsg(db->sql));
+	}
+	return stmt;
+}
+
+/**
+ * db_clear_error - Clear any errors from previous queries
+ */
+static void db_clear_error(struct db *db)
+{
+	db->err = tal_free(db->err);
+}
+
+
+static void close_db(struct db *db) { sqlite3_close(db->sql); }
+
+/**
+ * db_begin_transaction - Begin a transaction
+ *
+ * We do not support nesting multiple transactions, so make sure that
+ * we are not in a transaction when calling this. Returns true if we
+ * succeeded in starting a transaction.
+ */
+static bool db_begin_transaction(struct db *db)
+{
+	assert(!db->in_transaction);
+	/* Clear any errors from previous transactions and
+	 * non-transactional queries */
+	db_clear_error(db);
+	db->in_transaction = db_exec(__func__, db, "BEGIN TRANSACTION;");
+	return db->in_transaction;
+}
+
+/**
+ * db_commit_transaction - Commit a running transaction
+ *
+ * Requires that we are currently in a transaction. Returns whether
+ * the commit was successful.
+ */
+static bool db_commit_transaction(struct db *db)
+{
+	assert(db->in_transaction);
+	bool ret = db_exec(__func__, db, "COMMIT;");
+	db->in_transaction = false;
+	return ret;
+}
+
+/**
+ * db_rollback_transaction - Whoops... undo! undo!
+ */
+static bool db_rollback_transaction(struct db *db)
+{
+	assert(db->in_transaction);
+	bool ret = db_exec(__func__, db, "ROLLBACK;");
+	db->in_transaction = false;
+	return ret;
+}
+
+/**
+ * db_open - Open or create a sqlite3 database
+ */
+static struct db *db_open(const tal_t *ctx, char *filename)
+{
+	int err;
+	struct db *db;
+	sqlite3 *sql;
+
+	if (SQLITE_VERSION_NUMBER != sqlite3_libversion_number())
+		fatal("SQLITE version mistmatch: compiled %u, now %u",
+		      SQLITE_VERSION_NUMBER, sqlite3_libversion_number());
+
+	int flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE;
+	err = sqlite3_open_v2(filename, &sql, flags, NULL);
+
+	if (err != SQLITE_OK) {
+		fatal("failed to open database %s: %s", filename,
+		      sqlite3_errstr(err));
+	}
+
+	db = tal(ctx, struct db);
+	db->filename = tal_dup_arr(db, char, filename, strlen(filename), 0);
+	db->sql = sql;
+	tal_add_destructor(db, close_db);
+	db->in_transaction = false;
+	db->err = NULL;
+	return db;
+}
+
+/**
+ * db_get_version - Determine the current DB schema version
+ *
+ * Will attempt to determine the current schema version of the
+ * database @db by querying the `version` table. If the table does not
+ * exist it'll return schema version -1, so that migration 0 is
+ * applied, which should create the `version` table.
+ */
+static int db_get_version(struct db *db)
+{
+	int err;
+	u64 res = -1;
+	sqlite3_stmt *stmt =
+	    db_query(__func__, db, "SELECT version FROM version LIMIT 1");
+
+	if (!stmt)
+		return -1;
+
+	err = sqlite3_step(stmt);
+	if (err != SQLITE_ROW) {
+		sqlite3_finalize(stmt);
+		return -1;
+	} else {
+		res = sqlite3_column_int64(stmt, 0);
+		sqlite3_finalize(stmt);
+		return res;
+	}
+}
+
+/**
+ * db_mirgation_count - Count how many migrations are available
+ *
+ * Returns the maximum migration index, i.e., the version number of an
+ * up-to-date database schema.
+ */
+static int db_migration_count(void)
+{
+	int count = 0;
+	while (dbmigrations[count] != NULL)
+		count++;
+	return count - 1;
+}
+
+/**
+ * db_migrate - Apply all remaining migrations from the current version
+ */
+static bool db_migrate(struct db *db)
+{
+	/* Attempt to read the version from the database */
+	int current = db_get_version(db);
+	int available = db_migration_count();
+
+	if (!db_begin_transaction(db)) {
+		/* No need to rollback, we didn't even start... */
+		return false;
+	}
+
+	while (++current <= available) {
+		if (!db_exec(__func__, db, "%s", dbmigrations[current]))
+			goto fail;
+	}
+
+	/* Finally update the version number in the version table */
+	db_exec(__func__, db, "UPDATE version SET version=%d;", available);
+
+	if (!db_commit_transaction(db)) {
+		goto fail;
+	}
+
+	return true;
+fail:
+	db_rollback_transaction(db);
+	return false;
+}
+
+struct db *db_setup(const tal_t *ctx)
+{
+	struct db *db = db_open(ctx, DB_FILE);
+	if (!db) {
+		return db;
+	}
+
+	if (!db_migrate(db)) {
+		return tal_free(db);
+	}
+	return db;
+}

wallet/db.h

@@ -0,0 +1,38 @@
+#ifndef WALLET_DB_H
+#define WALLET_DB_H
+
+#include "config.h"
+#include "daemon/log.h"
+
+#include <sqlite3.h>
+#include <stdbool.h>
+
+struct db {
+	char *filename;
+	bool in_transaction;
+	const char *err;
+	sqlite3 *sql;
+};
+
+/**
+ * db_setup - Open a the lightningd database and update the schema
+ *
+ * Opens the database, creating it if necessary, and applying
+ * migrations until the schema is updated to the current state.
+ *
+ * Params:
+ *  @ctx: the tal_t context to allocate from
+ *  @log: where to log messages to
+ */
+struct db *db_setup(const tal_t *ctx);
+
+/**
+ * db_query - Prepare and execute a query, and return the result
+ */
+sqlite3_stmt *PRINTF_FMT(3, 4)
+	db_query(const char *caller, struct db *db, const char *fmt, ...);
+
+bool PRINTF_FMT(3, 4)
+	db_exec(const char *caller, struct db *db, const char *fmt, ...);
+
+#endif /* WALLET_DB_H */

wallet/db_tests.c

@@ -0,0 +1,50 @@
+#include "db.c"
+
+#include <stdio.h>
+#include <unistd.h>
+
+static struct db *create_test_db(const char *testname)
+{
+	struct db *db;
+	char filename[] = "/tmp/ldb-XXXXXX";
+
+	int fd = mkstemp(filename);
+	if (fd == -1)
+		return NULL;
+	close(fd);
+
+	db = db_open(NULL, filename);
+	return db;
+}
+
+static bool test_empty_db_migrate(void)
+{
+	struct db *db = create_test_db(__func__);
+	if (!db)
+		goto fail;
+
+	if (db_get_version(db) != -1)
+		goto fail;
+
+	if (!db_migrate(db))
+		goto fail;
+
+	if (db_get_version(db) != db_migration_count())
+		goto fail;
+
+	tal_free(db);
+	return true;
+fail:
+	printf("Migration failed with error: %s\n", db->err);
+	tal_free(db);
+	return false;
+}
+
+int main(void)
+{
+	bool ok = true;
+
+	ok &= test_empty_db_migrate();
+
+	return !ok;
+}