diff --git a/Makefile b/Makefile index fd8c70d..a95fde2 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ prettier: $(shell find lnbits -name "*.js" -name ".html") ./node_modules/.bin/prettier --write lnbits/static/js/*.js lnbits/core/static/js/*.js lnbits/extensions/*/templates/*/*.html ./lnbits/core/templates/core/*.html lnbits/templates/*.html lnbits/extensions/*/static/js/*.js black: $(shell find lnbits -name "*.py") - ./venv/bin/black --line-length 120 lnbits + ./venv/bin/black lnbits mypy: $(shell find lnbits -name "*.py") ./venv/bin/mypy lnbits @@ -17,4 +17,4 @@ checkprettier: $(shell find lnbits -name "*.js" -name ".html") ./node_modules/.bin/prettier --check lnbits/static/js/*.js lnbits/core/static/js/*.js lnbits/extensions/*/templates/*/*.html ./lnbits/core/templates/core/*.html lnbits/templates/*.html lnbits/extensions/*/static/js/*.js checkblack: $(shell find lnbits -name "*.py") - ./venv/bin/black --check --line-length 120 lnbits + ./venv/bin/black --check lnbits diff --git a/docs/devs/extensions.md b/docs/devs/extensions.md index 6dd002a..70e599e 100644 --- a/docs/devs/extensions.md +++ b/docs/devs/extensions.md @@ -19,7 +19,7 @@ find . -type f -print0 | xargs -0 sed -i 's/example/mysuperplugin/g' # Change al Going over the example extension's structure: * views_api.py: This is where your public API would go. It will be exposed at "$DOMAIN/$PLUGIN/$ROUTE". For example: https://lnbits.com/mysuperplugin/api/v1/tools. * views.py: The `/` path will show up as your plugin's home page in lnbits' UI. Other pages you can define yourself. The `templates` folder should explain itself in relation to this. -* migrations.py: Create database tables for your plugin. They'll be created when you run `pipenv run flask migrate`. +* migrations.py: Create database tables for your plugin. They'll be created automatically when you start lnbits. ... This document is a work-in-progress. Send pull requests if you get stuck, so others don't. diff --git a/docs/devs/installation.md b/docs/devs/installation.md index a81b6e0..2a22f7e 100644 --- a/docs/devs/installation.md +++ b/docs/devs/installation.md @@ -45,8 +45,7 @@ Running the server LNbits uses [Flask][flask] as an application server. ```sh -$ pipenv run flask migrate -$ pipenv run flask run +$ pipenv run python main.py ``` There is an environment variable called `FLASK_ENV` that has to be set to `development` diff --git a/docs/guide/installation.md b/docs/guide/installation.md index 0b19ce1..bb517c2 100644 --- a/docs/guide/installation.md +++ b/docs/guide/installation.md @@ -19,11 +19,10 @@ $ source ./.venv/bin/activate You will need to set the variables in `.env.example`, and rename the file to `.env`. -Run the migrations and the Flask server: +Run the server: ```sh -(.venv) $ flask migrate -(.venv) $ flask run +(.venv) $ python main.py ``` You might also need to install additional packages, depending on the [backend wallet](./wallets.md) you use. diff --git a/lnbits/__init__.py b/lnbits/__init__.py index 228de13..f0b52cb 100644 --- a/lnbits/__init__.py +++ b/lnbits/__init__.py @@ -1,4 +1,6 @@ +import re import importlib +import sqlite3 from flask import Flask from flask_assets import Environment, Bundle # type: ignore @@ -8,9 +10,10 @@ from flask_talisman import Talisman # type: ignore from os import getenv from werkzeug.middleware.proxy_fix import ProxyFix -from .core import core_app, migrations as core_migrations +from .core import core_app from .helpers import ExtensionManager from .settings import FORCE_HTTPS +from .db import open_db, open_ext_db disabled_extensions = getenv("LNBITS_DISABLED_EXTENSIONS", "").split(",") @@ -73,21 +76,40 @@ assets.register("base_css", Bundle("scss/base.scss", filters="pyscss", output="c # -------- -@app.cli.command("migrate") def migrate_databases(): """Creates the necessary databases if they don't exist already; or migrates them.""" - core_migrations.migrate() - - for ext in valid_extensions: - try: - ext_migrations = importlib.import_module(f"lnbits.extensions.{ext.code}.migrations") - ext_migrations.migrate() - except Exception: - raise ImportError(f"Please make sure that the extension `{ext.code}` has a migrations file.") + from .core import migrations as core_migrations -# init -# ---- - -if __name__ == "__main__": - app.run() + with open_db() as core_db: + try: + rows = core_db.fetchall("SELECT * FROM dbversions") + except sqlite3.OperationalError: + # migration 3 wasn't ran + core_migrations.m000_create_migrations_table(core_db) + rows = core_db.fetchall("SELECT * FROM dbversions") + + current_versions = {row["db"]: row["version"] for row in rows} + matcher = re.compile(r"^m(\d\d\d)_") + + def run_migration(db, migrations_module): + db_name = migrations_module.__name__.split(".")[-2] + for key, run_migration in migrations_module.__dict__.items(): + if match := matcher.match(key): + version = int(match.group(1)) + if version > current_versions.get(db_name, 0): + print(f"running migration {db_name}.{version}") + run_migration(db) + core_db.execute( + "INSERT OR REPLACE INTO dbversions (db, version) VALUES (?, ?)", (db_name, version) + ) + + run_migration(core_db, core_migrations) + + for ext in valid_extensions: + try: + ext_migrations = importlib.import_module(f"lnbits.extensions.{ext.code}.migrations") + with open_ext_db(ext.code) as db: + run_migration(db, ext_migrations) + except ImportError: + raise ImportError(f"Please make sure that the extension `{ext.code}` has a migrations file.") diff --git a/lnbits/core/migrations.py b/lnbits/core/migrations.py index 78631f4..828d9d4 100644 --- a/lnbits/core/migrations.py +++ b/lnbits/core/migrations.py @@ -1,4 +1,15 @@ -from lnbits.db import open_db +import sqlite3 + + +def m000_create_migrations_table(db): + db.execute( + """ + CREATE TABLE dbversions ( + db TEXT PRIMARY KEY, + version INT NOT NULL + ) + """ + ) def m001_initial(db): @@ -76,35 +87,36 @@ def m002_add_fields_to_apipayments(db): Adding fields to apipayments for better accounting, and renaming payhash to checking_id since that is what it really is. """ - db.execute("ALTER TABLE apipayments RENAME COLUMN payhash TO checking_id") - db.execute("ALTER TABLE apipayments ADD COLUMN hash TEXT") - db.execute("CREATE INDEX by_hash ON apipayments (hash)") - db.execute("ALTER TABLE apipayments ADD COLUMN preimage TEXT") - db.execute("ALTER TABLE apipayments ADD COLUMN bolt11 TEXT") - db.execute("ALTER TABLE apipayments ADD COLUMN extra TEXT") - - import json - - rows = db.fetchall("SELECT * FROM apipayments") - for row in rows: - if not row["memo"] or not row["memo"].startswith("#"): - continue + try: + db.execute("ALTER TABLE apipayments RENAME COLUMN payhash TO checking_id") + db.execute("ALTER TABLE apipayments ADD COLUMN hash TEXT") + db.execute("CREATE INDEX by_hash ON apipayments (hash)") + db.execute("ALTER TABLE apipayments ADD COLUMN preimage TEXT") + db.execute("ALTER TABLE apipayments ADD COLUMN bolt11 TEXT") + db.execute("ALTER TABLE apipayments ADD COLUMN extra TEXT") - for ext in ["withdraw", "events", "lnticket", "paywall", "tpos"]: - prefix = "#" + ext + " " - if row["memo"].startswith(prefix): - new = row["memo"][len(prefix) :] - db.execute( - """ - UPDATE apipayments SET extra = ?, memo = ? - WHERE checking_id = ? AND memo = ? - """, - (json.dumps({"tag": ext}), new, row["checking_id"], row["memo"]), - ) - break + import json + rows = db.fetchall("SELECT * FROM apipayments") + for row in rows: + if not row["memo"] or not row["memo"].startswith("#"): + continue -def migrate(): - with open_db() as db: - m001_initial(db) - m002_add_fields_to_apipayments(db) + for ext in ["withdraw", "events", "lnticket", "paywall", "tpos"]: + prefix = "#" + ext + " " + if row["memo"].startswith(prefix): + new = row["memo"][len(prefix) :] + db.execute( + """ + UPDATE apipayments SET extra = ?, memo = ? + WHERE checking_id = ? AND memo = ? + """, + (json.dumps({"tag": ext}), new, row["checking_id"], row["memo"]), + ) + break + except sqlite3.OperationalError: + # this is necessary now because it may be the case that this migration will + # run twice in some environments. + # catching errors like this won't be necessary in anymore now that we + # keep track of db versions so no migration ever runs twice. + pass diff --git a/lnbits/extensions/amilk/migrations.py b/lnbits/extensions/amilk/migrations.py index 0210fcb..3ab2d4a 100644 --- a/lnbits/extensions/amilk/migrations.py +++ b/lnbits/extensions/amilk/migrations.py @@ -1,6 +1,3 @@ -from lnbits.db import open_ext_db - - def m001_initial(db): """ Initial amilks table. @@ -16,8 +13,3 @@ def m001_initial(db): ); """ ) - - -def migrate(): - with open_ext_db("amilk") as db: - m001_initial(db) diff --git a/lnbits/extensions/diagonalley/migrations.py b/lnbits/extensions/diagonalley/migrations.py index 9981d27..afec1a6 100644 --- a/lnbits/extensions/diagonalley/migrations.py +++ b/lnbits/extensions/diagonalley/migrations.py @@ -1,6 +1,3 @@ -from lnbits.db import open_ext_db - - def m001_initial(db): """ Initial products table. @@ -61,8 +58,3 @@ def m001_initial(db): ); """ ) - - -def migrate(): - with open_ext_db("diagonalley") as db: - m001_initial(db) diff --git a/lnbits/extensions/events/migrations.py b/lnbits/extensions/events/migrations.py index 851c4ae..95e361b 100644 --- a/lnbits/extensions/events/migrations.py +++ b/lnbits/extensions/events/migrations.py @@ -1,6 +1,3 @@ -from lnbits.db import open_ext_db - - def m001_initial(db): db.execute( @@ -86,9 +83,3 @@ def m002_changed(db): ), ) db.execute("DROP TABLE tickets") - - -def migrate(): - with open_ext_db("events") as db: - m001_initial(db) - m002_changed(db) diff --git a/lnbits/extensions/example/migrations.py b/lnbits/extensions/example/migrations.py index 04435aa..e69de29 100644 --- a/lnbits/extensions/example/migrations.py +++ b/lnbits/extensions/example/migrations.py @@ -1,5 +0,0 @@ -from lnbits.db import open_ext_db - - -def migrate(): - print("pending") diff --git a/lnbits/extensions/lnticket/migrations.py b/lnbits/extensions/lnticket/migrations.py index 4d8ab82..b7b7f6b 100644 --- a/lnbits/extensions/lnticket/migrations.py +++ b/lnbits/extensions/lnticket/migrations.py @@ -1,6 +1,3 @@ -from lnbits.db import open_ext_db - - def m001_initial(db): db.execute( @@ -86,9 +83,3 @@ def m002_changed(db): ), ) db.execute("DROP TABLE tickets") - - -def migrate(): - with open_ext_db("lnticket") as db: - m001_initial(db) - m002_changed(db) diff --git a/lnbits/extensions/lnurlp/migrations.py b/lnbits/extensions/lnurlp/migrations.py index b1fe152..cdb8e9a 100644 --- a/lnbits/extensions/lnurlp/migrations.py +++ b/lnbits/extensions/lnurlp/migrations.py @@ -1,6 +1,3 @@ -from lnbits.db import open_ext_db - - def m001_initial(db): """ Initial pay table. @@ -17,8 +14,3 @@ def m001_initial(db): ); """ ) - - -def migrate(): - with open_ext_db("lnurlp") as db: - m001_initial(db) diff --git a/lnbits/extensions/paywall/migrations.py b/lnbits/extensions/paywall/migrations.py index aa63d0a..d1b6a3a 100644 --- a/lnbits/extensions/paywall/migrations.py +++ b/lnbits/extensions/paywall/migrations.py @@ -1,7 +1,5 @@ from sqlite3 import OperationalError -from lnbits.db import open_ext_db - def m001_initial(db): """ @@ -65,9 +63,3 @@ def m002_redux(db): ) db.execute("DROP TABLE paywalls_old") - - -def migrate(): - with open_ext_db("paywall") as db: - m001_initial(db) - m002_redux(db) diff --git a/lnbits/extensions/tpos/migrations.py b/lnbits/extensions/tpos/migrations.py index f7b1a17..1a03ed2 100644 --- a/lnbits/extensions/tpos/migrations.py +++ b/lnbits/extensions/tpos/migrations.py @@ -1,6 +1,3 @@ -from lnbits.db import open_ext_db - - def m001_initial(db): """ Initial tposs table. @@ -15,8 +12,3 @@ def m001_initial(db): ); """ ) - - -def migrate(): - with open_ext_db("tpos") as db: - m001_initial(db) diff --git a/lnbits/extensions/usermanager/migrations.py b/lnbits/extensions/usermanager/migrations.py index 150cc4e..faff3b8 100644 --- a/lnbits/extensions/usermanager/migrations.py +++ b/lnbits/extensions/usermanager/migrations.py @@ -1,6 +1,3 @@ -from lnbits.db import open_ext_db - - def m001_initial(db): """ Initial users table. @@ -32,8 +29,3 @@ def m001_initial(db): ); """ ) - - -def migrate(): - with open_ext_db("usermanager") as db: - m001_initial(db) diff --git a/lnbits/extensions/withdraw/migrations.py b/lnbits/extensions/withdraw/migrations.py index 9b5872d..84924ce 100644 --- a/lnbits/extensions/withdraw/migrations.py +++ b/lnbits/extensions/withdraw/migrations.py @@ -1,6 +1,3 @@ -from lnbits.db import open_ext_db - - def m001_initial(db): """ Creates an improved withdraw table and migrates the existing data. @@ -97,9 +94,3 @@ def m002_change_withdraw_table(db): ), ) db.execute("DROP TABLE withdraw_links") - - -def migrate(): - with open_ext_db("withdraw") as db: - m001_initial(db) - m002_change_withdraw_table(db) diff --git a/main.py b/main.py new file mode 100644 index 0000000..ef15eb7 --- /dev/null +++ b/main.py @@ -0,0 +1,4 @@ +from lnbits import app, migrate_databases + +migrate_databases() +app.run()