refactor: improve database migrations

README.md

@@ -61,6 +61,7 @@ You will need to set the variables in .env.example, and rename the file to .env
 Running the server
 ------------------
 
+    $ flask migrate
     $ flask run
 
 There is an environment variable called `FLASK_ENV` that has to be set to `development`

lnbits/__init__.py

@@ -6,8 +6,7 @@ from flask_compress import Compress
 from flask_talisman import Talisman
 from os import getenv
 
-from .core import core_app
-from .db import init_databases
+from .core import core_app, migrations as core_migrations
 from .helpers import ExtensionManager, megajson
 
 
@@ -70,14 +69,24 @@ assets.url = app.static_url_path
 assets.register("base_css", Bundle("scss/base.scss", filters="pyscss", output="css/base.css"))
 
 
-# init
-# ----
+# commands
+# --------
+
+@app.cli.command("migrate")
+def migrate_databases():
+    """Creates the necessary databases if they don't exist already; or migrates them."""
+    core_migrations.migrate()
 
+    for ext in valid_extensions:
+        try:
+            ext_migrations = importlib.import_module(f"lnbits.extensions.{ext.code}.migrations")
+            ext_migrations.migrate()
+        except Exception:
+            raise ImportError(f"Please make sure that the extension `{ext.code}` has a migrations file.")
 
-@app.before_first_request
-def init():
-    init_databases()
 
+# init
+# ----
 
 if __name__ == "__main__":
     app.run()

lnbits/core/migrations.py

@@ -0,0 +1,65 @@
+from lnbits.db import open_db
+
+
+def m001_initial(db):
+    """
+    Initial LNbits tables.
+    """
+    db.execute("""
+        CREATE TABLE IF NOT EXISTS accounts (
+            id TEXT PRIMARY KEY,
+            email TEXT,
+            pass TEXT
+        );
+    """)
+    db.execute("""
+        CREATE TABLE IF NOT EXISTS extensions (
+            user TEXT NOT NULL,
+            extension TEXT NOT NULL,
+            active BOOLEAN DEFAULT 0,
+
+            UNIQUE (user, extension)
+        );
+    """)
+    db.execute("""
+        CREATE TABLE IF NOT EXISTS wallets (
+            id TEXT PRIMARY KEY,
+            name TEXT NOT NULL,
+            user TEXT NOT NULL,
+            adminkey TEXT NOT NULL,
+            inkey TEXT
+        );
+    """)
+    db.execute("""
+        CREATE TABLE IF NOT EXISTS apipayments (
+            payhash TEXT NOT NULL,
+            amount INTEGER NOT NULL,
+            fee INTEGER NOT NULL DEFAULT 0,
+            wallet TEXT NOT NULL,
+            pending BOOLEAN NOT NULL,
+            memo TEXT,
+            time TIMESTAMP NOT NULL DEFAULT (strftime('%s', 'now')),
+
+            UNIQUE (wallet, payhash)
+        );
+    """)
+    db.execute("""
+        CREATE VIEW IF NOT EXISTS balances AS
+        SELECT wallet, COALESCE(SUM(s), 0) AS balance FROM (
+            SELECT wallet, SUM(amount) AS s  -- incoming
+            FROM apipayments
+            WHERE amount > 0 AND pending = 0  -- don't sum pending
+            GROUP BY wallet
+            UNION ALL
+            SELECT wallet, SUM(amount + fee) AS s  -- outgoing, sum fees
+            FROM apipayments
+            WHERE amount < 0  -- do sum pending
+            GROUP BY wallet
+        )
+        GROUP BY wallet;
+    """)
+
+
+def migrate():
+    with open_db() as db:
+        m001_initial(db)

lnbits/core/schema.sql

@@ -1,47 +0,0 @@
-CREATE TABLE IF NOT EXISTS accounts (
-    id TEXT PRIMARY KEY,
-    email TEXT,
-    pass TEXT
-);
-
-CREATE TABLE IF NOT EXISTS extensions (
-    user TEXT NOT NULL,
-    extension TEXT NOT NULL,
-    active BOOLEAN DEFAULT 0,
-
-    UNIQUE (user, extension)
-);
-
-CREATE TABLE IF NOT EXISTS wallets (
-    id TEXT PRIMARY KEY,
-    name TEXT NOT NULL,
-    user TEXT NOT NULL,
-    adminkey TEXT NOT NULL,
-    inkey TEXT
-);
-
-CREATE TABLE IF NOT EXISTS apipayments (
-    payhash TEXT NOT NULL,
-    amount INTEGER NOT NULL,
-    fee INTEGER NOT NULL DEFAULT 0,
-    wallet TEXT NOT NULL,
-    pending BOOLEAN NOT NULL,
-    memo TEXT,
-    time TIMESTAMP NOT NULL DEFAULT (strftime('%s', 'now')),
-
-    UNIQUE (wallet, payhash)
-);
-
-CREATE VIEW IF NOT EXISTS balances AS
-  SELECT wallet, COALESCE(SUM(s), 0) AS balance FROM (
-      SELECT wallet, SUM(amount) AS s  -- incoming
-      FROM apipayments
-      WHERE amount > 0 AND pending = 0  -- don't sum pending
-      GROUP BY wallet
-    UNION ALL
-      SELECT wallet, SUM(amount + fee) AS s  -- outgoing, sum fees
-      FROM apipayments
-      WHERE amount < 0  -- do sum pending
-      GROUP BY wallet
-  )
-  GROUP BY wallet;

lnbits/db.py

@@ -1,8 +1,7 @@
 import os
 import sqlite3
 
-from .helpers import ExtensionManager
-from .settings import LNBITS_PATH, LNBITS_DATA_FOLDER
+from .settings import LNBITS_DATA_FOLDER
 
 
 class Database:
@@ -19,16 +18,16 @@ class Database:
         self.cursor.close()
         self.connection.close()
 
-    def fetchall(self, query: str, values: tuple) -> list:
+    def fetchall(self, query: str, values: tuple = ()) -> list:
         """Given a query, return cursor.fetchall() rows."""
         self.cursor.execute(query, values)
         return self.cursor.fetchall()
 
-    def fetchone(self, query: str, values: tuple):
+    def fetchone(self, query: str, values: tuple = ()):
         self.cursor.execute(query, values)
         return self.cursor.fetchone()
 
-    def execute(self, query: str, values: tuple) -> None:
+    def execute(self, query: str, values: tuple = ()) -> None:
         """Given a query, cursor.execute() it."""
         self.cursor.execute(query, values)
         self.connection.commit()
@@ -41,22 +40,3 @@ def open_db(db_name: str = "database") -> Database:
 
 def open_ext_db(extension_name: str) -> Database:
     return open_db(f"ext_{extension_name}")
-
-
-def init_databases() -> None:
-    """Creates the necessary databases if they don't exist already."""
-    """TODO: see how we can deal with migrations."""
-
-    schemas = [
-        ("database", os.path.join(LNBITS_PATH, "core", "schema.sql")),
-    ]
-
-    for extension in ExtensionManager().extensions:
-        extension_path = os.path.join(LNBITS_PATH, "extensions", extension.code)
-        schemas.append((f"ext_{extension.code}", os.path.join(extension_path, "schema.sql")))
-
-    for schema in [s for s in schemas if os.path.exists(s[1])]:
-        with open_db(schema[0]) as db:
-            with open(schema[1]) as schemafile:
-                for stmt in schemafile.read().split(";\n\n"):
-                    db.execute(stmt, [])

lnbits/extensions/amilk/migrations.py

@@ -0,0 +1,21 @@
+from lnbits.db import open_ext_db
+
+
+def m001_initial(db):
+    """
+    Initial amilks table.
+    """
+    db.execute("""
+        CREATE TABLE IF NOT EXISTS amilks (
+            id TEXT PRIMARY KEY,
+            wallet TEXT NOT NULL,
+            lnurl TEXT NOT NULL,
+            atime TEXT NOT NULL,
+            amount INTEGER NOT NULL
+        );
+    """)
+
+
+def migrate():
+    with open_ext_db("amilk") as db:
+        m001_initial(db)

lnbits/extensions/amilk/schema.sql

@@ -1,7 +0,0 @@
-CREATE TABLE IF NOT EXISTS amilks (
-  id TEXT PRIMARY KEY,
-  wallet TEXT NOT NULL,
-  lnurl TEXT NOT NULL,
-  atime TEXT NOT NULL,
-  amount INTEGER NOT NULL
-);

lnbits/extensions/example/migrations.py

@@ -0,0 +1,5 @@
+from lnbits.db import open_ext_db
+
+
+def migrate():
+    print("pending")

lnbits/extensions/example/schema.sql

@@ -1 +0,0 @@
-/* create your extensions table and the variables needed here */

lnbits/extensions/paywall/migrations.py

@@ -0,0 +1,22 @@
+from lnbits.db import open_ext_db
+
+
+def m001_initial(db):
+    """
+    Initial paywalls table.
+    """
+    db.execute("""
+        CREATE TABLE IF NOT EXISTS paywalls (
+            id TEXT PRIMARY KEY,
+            wallet TEXT NOT NULL,
+            url TEXT NOT NULL,
+            memo TEXT NOT NULL,
+            amount INTEGER NOT NULL,
+            time TIMESTAMP NOT NULL DEFAULT (strftime('%s', 'now'))
+        );
+    """)
+
+
+def migrate():
+    with open_ext_db("tpos") as db:
+        m001_initial(db)

lnbits/extensions/paywall/schema.sql

@@ -1,8 +0,0 @@
-CREATE TABLE IF NOT EXISTS paywalls (
-  id TEXT PRIMARY KEY,
-  wallet TEXT NOT NULL,
-  url TEXT NOT NULL,
-  memo TEXT NOT NULL,
-  amount INTEGER NOT NULL,
-  time TIMESTAMP NOT NULL DEFAULT (strftime('%s', 'now'))
-);

lnbits/extensions/tpos/migrations.py

@@ -0,0 +1,20 @@
+from lnbits.db import open_ext_db
+
+
+def m001_initial(db):
+    """
+    Initial tposs table.
+    """
+    db.execute("""
+        CREATE TABLE IF NOT EXISTS tposs (
+            id TEXT PRIMARY KEY,
+            wallet TEXT NOT NULL,
+            name TEXT NOT NULL,
+            currency TEXT NOT NULL
+        );
+    """)
+
+
+def migrate():
+    with open_ext_db("tpos") as db:
+        m001_initial(db)

lnbits/extensions/tpos/schema.sql

@@ -1,6 +0,0 @@
-CREATE TABLE IF NOT EXISTS tposs (
-  id TEXT PRIMARY KEY,
-  wallet TEXT NOT NULL,
-  name TEXT NOT NULL,
-  currency TEXT NOT NULL
-);

lnbits/extensions/withdraw/migrations.py

@@ -0,0 +1,37 @@
+from datetime import datetime
+from uuid import uuid4
+
+from lnbits.db import open_ext_db
+
+
+def m001_initial(db):
+    """
+    Initial withdraw table.
+    """
+    db.execute(
+        """
+        CREATE TABLE IF NOT EXISTS withdraws (
+            key INTEGER PRIMARY KEY AUTOINCREMENT,
+            usr TEXT,
+            wal TEXT,
+            walnme TEXT,
+            adm INTEGER,
+            uni TEXT,
+            tit TEXT,
+            maxamt INTEGER,
+            minamt INTEGER,
+            spent INTEGER,
+            inc INTEGER,
+            tme INTEGER,
+            uniq INTEGER DEFAULT 0,
+            withdrawals TEXT,
+            tmestmp INTEGER,
+            rand TEXT
+        );
+        """
+    )
+
+
+def migrate():
+    with open_ext_db("withdraw") as db:
+        m001_initial(db)

lnbits/extensions/withdraw/schema.sql

@@ -1,18 +0,0 @@
-CREATE TABLE IF NOT EXISTS withdraws (
-    key INTEGER PRIMARY KEY AUTOINCREMENT,
-    usr TEXT,
-    wal TEXT,
-    walnme TEXT,
-    adm INTEGER,
-    uni TEXT,
-    tit TEXT,
-    maxamt INTEGER,
-    minamt INTEGER,
-    spent INTEGER,
-    inc INTEGER,
-    tme INTEGER,
-    uniq INTEGER DEFAULT 0,
-    withdrawals TEXT,
-    tmestmp INTEGER,
-    rand TEXT
-);