migration tracking.

Makefile

@@ -8,7 +8,7 @@ prettier: $(shell find lnbits -name "*.js" -name ".html")
 	./node_modules/.bin/prettier --write lnbits/static/js/*.js lnbits/core/static/js/*.js lnbits/extensions/*/templates/*/*.html ./lnbits/core/templates/core/*.html lnbits/templates/*.html lnbits/extensions/*/static/js/*.js
 
 black: $(shell find lnbits -name "*.py")
-	./venv/bin/black --line-length 120 lnbits
+	./venv/bin/black lnbits
 
 mypy: $(shell find lnbits -name "*.py")
 	./venv/bin/mypy lnbits
@@ -17,4 +17,4 @@ checkprettier: $(shell find lnbits -name "*.js" -name ".html")
 	./node_modules/.bin/prettier --check lnbits/static/js/*.js lnbits/core/static/js/*.js lnbits/extensions/*/templates/*/*.html ./lnbits/core/templates/core/*.html lnbits/templates/*.html lnbits/extensions/*/static/js/*.js
 
 checkblack: $(shell find lnbits -name "*.py")
-	./venv/bin/black --check --line-length 120 lnbits
+	./venv/bin/black --check lnbits

docs/devs/extensions.md

@@ -19,7 +19,7 @@ find . -type f -print0 | xargs -0 sed -i 's/example/mysuperplugin/g' # Change al
 Going over the example extension's structure:
 * views_api.py: This is where your public API would go. It will be exposed at "$DOMAIN/$PLUGIN/$ROUTE". For example: https://lnbits.com/mysuperplugin/api/v1/tools.
 * views.py: The `/` path will show up as your plugin's home page in lnbits' UI. Other pages you can define yourself. The `templates` folder should explain itself in relation to this.
-* migrations.py: Create database tables for your plugin. They'll be created when you run `pipenv run flask migrate`.
+* migrations.py: Create database tables for your plugin. They'll be created automatically when you start lnbits.
 
 ... This document is a work-in-progress. Send pull requests if you get stuck, so others don't.
 

docs/devs/installation.md

@@ -45,8 +45,7 @@ Running the server
 LNbits uses [Flask][flask] as an application server.
 
 ```sh
-$ pipenv run flask migrate
-$ pipenv run flask run
+$ pipenv run python main.py
 ```
 
 There is an environment variable called `FLASK_ENV` that has to be set to `development`

docs/guide/installation.md

@@ -19,11 +19,10 @@ $ source ./.venv/bin/activate
 
 You will need to set the variables in `.env.example`, and rename the file to `.env`.
 
-Run the migrations and the Flask server:
+Run the server:
 
 ```sh
-(.venv) $ flask migrate
-(.venv) $ flask run
+(.venv) $ python main.py
 ```
 
 You might also need to install additional packages, depending on the [backend wallet](./wallets.md) you use.

lnbits/__init__.py

@@ -1,4 +1,6 @@
+import re
 import importlib
+import sqlite3
 
 from flask import Flask
 from flask_assets import Environment, Bundle  # type: ignore
@@ -8,9 +10,10 @@ from flask_talisman import Talisman  # type: ignore
 from os import getenv
 from werkzeug.middleware.proxy_fix import ProxyFix
 
-from .core import core_app, migrations as core_migrations
+from .core import core_app
 from .helpers import ExtensionManager
 from .settings import FORCE_HTTPS
+from .db import open_db, open_ext_db
 
 
 disabled_extensions = getenv("LNBITS_DISABLED_EXTENSIONS", "").split(",")
@@ -73,21 +76,40 @@ assets.register("base_css", Bundle("scss/base.scss", filters="pyscss", output="c
 # --------
 
 
-@app.cli.command("migrate")
 def migrate_databases():
     """Creates the necessary databases if they don't exist already; or migrates them."""
-    core_migrations.migrate()
-
-    for ext in valid_extensions:
-        try:
-            ext_migrations = importlib.import_module(f"lnbits.extensions.{ext.code}.migrations")
-            ext_migrations.migrate()
-        except Exception:
-            raise ImportError(f"Please make sure that the extension `{ext.code}` has a migrations file.")
 
+    from .core import migrations as core_migrations
 
-# init
-# ----
-
-if __name__ == "__main__":
-    app.run()
+    with open_db() as core_db:
+        try:
+            rows = core_db.fetchall("SELECT * FROM dbversions")
+        except sqlite3.OperationalError:
+            # migration 3 wasn't ran
+            core_migrations.m000_create_migrations_table(core_db)
+            rows = core_db.fetchall("SELECT * FROM dbversions")
+
+        current_versions = {row["db"]: row["version"] for row in rows}
+        matcher = re.compile(r"^m(\d\d\d)_")
+
+        def run_migration(db, migrations_module):
+            db_name = migrations_module.__name__.split(".")[-2]
+            for key, run_migration in migrations_module.__dict__.items():
+                if match := matcher.match(key):
+                    version = int(match.group(1))
+                    if version > current_versions.get(db_name, 0):
+                        print(f"running migration {db_name}.{version}")
+                        run_migration(db)
+                        core_db.execute(
+                            "INSERT OR REPLACE INTO dbversions (db, version) VALUES (?, ?)", (db_name, version)
+                        )
+
+        run_migration(core_db, core_migrations)
+
+        for ext in valid_extensions:
+            try:
+                ext_migrations = importlib.import_module(f"lnbits.extensions.{ext.code}.migrations")
+                with open_ext_db(ext.code) as db:
+                    run_migration(db, ext_migrations)
+            except ImportError:
+                raise ImportError(f"Please make sure that the extension `{ext.code}` has a migrations file.")

lnbits/core/migrations.py

@@ -1,4 +1,15 @@
-from lnbits.db import open_db
+import sqlite3
+
+
+def m000_create_migrations_table(db):
+    db.execute(
+        """
+    CREATE TABLE dbversions (
+        db TEXT PRIMARY KEY,
+        version INT NOT NULL
+    )
+    """
+    )
 
 
 def m001_initial(db):
@@ -76,35 +87,36 @@ def m002_add_fields_to_apipayments(db):
     Adding fields to apipayments for better accounting,
     and renaming payhash to checking_id since that is what it really is.
     """
-    db.execute("ALTER TABLE apipayments RENAME COLUMN payhash TO checking_id")
-    db.execute("ALTER TABLE apipayments ADD COLUMN hash TEXT")
-    db.execute("CREATE INDEX by_hash ON apipayments (hash)")
-    db.execute("ALTER TABLE apipayments ADD COLUMN preimage TEXT")
-    db.execute("ALTER TABLE apipayments ADD COLUMN bolt11 TEXT")
-    db.execute("ALTER TABLE apipayments ADD COLUMN extra TEXT")
-
-    import json
-
-    rows = db.fetchall("SELECT * FROM apipayments")
-    for row in rows:
-        if not row["memo"] or not row["memo"].startswith("#"):
-            continue
+    try:
+        db.execute("ALTER TABLE apipayments RENAME COLUMN payhash TO checking_id")
+        db.execute("ALTER TABLE apipayments ADD COLUMN hash TEXT")
+        db.execute("CREATE INDEX by_hash ON apipayments (hash)")
+        db.execute("ALTER TABLE apipayments ADD COLUMN preimage TEXT")
+        db.execute("ALTER TABLE apipayments ADD COLUMN bolt11 TEXT")
+        db.execute("ALTER TABLE apipayments ADD COLUMN extra TEXT")
 
-        for ext in ["withdraw", "events", "lnticket", "paywall", "tpos"]:
-            prefix = "#" + ext + " "
-            if row["memo"].startswith(prefix):
-                new = row["memo"][len(prefix) :]
-                db.execute(
-                    """
-                    UPDATE apipayments SET extra = ?, memo = ?
-                    WHERE checking_id = ? AND memo = ?
-                    """,
-                    (json.dumps({"tag": ext}), new, row["checking_id"], row["memo"]),
-                )
-                break
+        import json
 
+        rows = db.fetchall("SELECT * FROM apipayments")
+        for row in rows:
+            if not row["memo"] or not row["memo"].startswith("#"):
+                continue
 
-def migrate():
-    with open_db() as db:
-        m001_initial(db)
-        m002_add_fields_to_apipayments(db)
+            for ext in ["withdraw", "events", "lnticket", "paywall", "tpos"]:
+                prefix = "#" + ext + " "
+                if row["memo"].startswith(prefix):
+                    new = row["memo"][len(prefix) :]
+                    db.execute(
+                        """
+                        UPDATE apipayments SET extra = ?, memo = ?
+                        WHERE checking_id = ? AND memo = ?
+                        """,
+                        (json.dumps({"tag": ext}), new, row["checking_id"], row["memo"]),
+                    )
+                    break
+    except sqlite3.OperationalError:
+        # this is necessary now because it may be the case that this migration will
+        # run twice in some environments.
+        # catching errors like this won't be necessary in anymore now that we
+        # keep track of db versions so no migration ever runs twice.
+        pass

lnbits/extensions/amilk/migrations.py

@@ -1,6 +1,3 @@
-from lnbits.db import open_ext_db
-
-
 def m001_initial(db):
     """
     Initial amilks table.
@@ -16,8 +13,3 @@ def m001_initial(db):
         );
     """
     )
-
-
-def migrate():
-    with open_ext_db("amilk") as db:
-        m001_initial(db)

lnbits/extensions/diagonalley/migrations.py

@@ -1,6 +1,3 @@
-from lnbits.db import open_ext_db
-
-
 def m001_initial(db):
     """
     Initial products table.
@@ -61,8 +58,3 @@ def m001_initial(db):
         );
     """
     )
-
-
-def migrate():
-    with open_ext_db("diagonalley") as db:
-        m001_initial(db)

lnbits/extensions/events/migrations.py

@@ -1,6 +1,3 @@
-from lnbits.db import open_ext_db
-
-
 def m001_initial(db):
 
     db.execute(
@@ -86,9 +83,3 @@ def m002_changed(db):
             ),
         )
     db.execute("DROP TABLE tickets")
-
-
-def migrate():
-    with open_ext_db("events") as db:
-        m001_initial(db)
-        m002_changed(db)

lnbits/extensions/example/migrations.py

@@ -1,5 +0,0 @@
-from lnbits.db import open_ext_db
-
-
-def migrate():
-    print("pending")

lnbits/extensions/lnticket/migrations.py

@@ -1,6 +1,3 @@
-from lnbits.db import open_ext_db
-
-
 def m001_initial(db):
 
     db.execute(
@@ -86,9 +83,3 @@ def m002_changed(db):
             ),
         )
     db.execute("DROP TABLE tickets")
-
-
-def migrate():
-    with open_ext_db("lnticket") as db:
-        m001_initial(db)
-        m002_changed(db)

lnbits/extensions/lnurlp/migrations.py

@@ -1,6 +1,3 @@
-from lnbits.db import open_ext_db
-
-
 def m001_initial(db):
     """
     Initial pay table.
@@ -17,8 +14,3 @@ def m001_initial(db):
         );
         """
     )
-
-
-def migrate():
-    with open_ext_db("lnurlp") as db:
-        m001_initial(db)

lnbits/extensions/paywall/migrations.py

@@ -1,7 +1,5 @@
 from sqlite3 import OperationalError
 
-from lnbits.db import open_ext_db
-
 
 def m001_initial(db):
     """
@@ -65,9 +63,3 @@ def m002_redux(db):
             )
 
         db.execute("DROP TABLE paywalls_old")
-
-
-def migrate():
-    with open_ext_db("paywall") as db:
-        m001_initial(db)
-        m002_redux(db)

lnbits/extensions/tpos/migrations.py

@@ -1,6 +1,3 @@
-from lnbits.db import open_ext_db
-
-
 def m001_initial(db):
     """
     Initial tposs table.
@@ -15,8 +12,3 @@ def m001_initial(db):
         );
     """
     )
-
-
-def migrate():
-    with open_ext_db("tpos") as db:
-        m001_initial(db)

lnbits/extensions/usermanager/migrations.py

@@ -1,6 +1,3 @@
-from lnbits.db import open_ext_db
-
-
 def m001_initial(db):
     """
     Initial users table.
@@ -32,8 +29,3 @@ def m001_initial(db):
         );
     """
     )
-
-
-def migrate():
-    with open_ext_db("usermanager") as db:
-        m001_initial(db)

lnbits/extensions/withdraw/migrations.py

@@ -1,6 +1,3 @@
-from lnbits.db import open_ext_db
-
-
 def m001_initial(db):
     """
     Creates an improved withdraw table and migrates the existing data.
@@ -97,9 +94,3 @@ def m002_change_withdraw_table(db):
             ),
         )
     db.execute("DROP TABLE withdraw_links")
-
-
-def migrate():
-    with open_ext_db("withdraw") as db:
-        m001_initial(db)
-        m002_change_withdraw_table(db)

main.py

@@ -0,0 +1,4 @@
+from lnbits import app, migrate_databases
+
+migrate_databases()
+app.run()