refactor: remove databases from code an use schemas instead

All databases are now saved together in the same LNBITS_DATA_FOLDER.
Extensions have to define a schema.yml file for creating the necessary database.

.gitignore

@@ -22,6 +22,5 @@ Pipfile.lock
 .env
 venv
 
-database.sqlite3
-database.sqlite3*
+*.sqlite3
 .pyre*

lnbits/__init__.py

@@ -1,5 +1,4 @@
 import json
-import os
 import requests
 import uuid
 
@@ -9,14 +8,16 @@ from lnurl import Lnurl, LnurlWithdrawResponse
 
 from . import bolt11
 from .core import core_app
-from .db import open_db, open_ext_db
+from .db import init_databases, open_db
 from .extensions.withdraw import withdraw_ext
 from .helpers import megajson
-from .settings import LNBITS_PATH, WALLET, DEFAULT_USER_WALLET_NAME, FEE_RESERVE
+from .settings import WALLET, DEFAULT_USER_WALLET_NAME, FEE_RESERVE
 
 
 app = Flask(__name__)
-Talisman(app, content_security_policy={
+Talisman(
+    app,
+    content_security_policy={
         "default-src": [
             "'self'",
             "'unsafe-eval'",
@@ -28,7 +29,8 @@ Talisman(app, content_security_policy={
             "fonts.gstatic.com",
             "maxcdn.bootstrapcdn.com",
         ]
-})
+    },
+)
 
 # filters
 app.jinja_env.filters["megajson"] = megajson
@@ -40,10 +42,7 @@ app.register_blueprint(withdraw_ext, url_prefix="/withdraw")
 
 @app.before_first_request
 def init():
-    with open_db() as db:
-        with open(os.path.join(LNBITS_PATH, "data", "schema.sql")) as schemafile:
-            for stmt in schemafile.read().split(";\n\n"):
-                db.execute(stmt, [])
+    init_databases()
 
 
 @app.route("/deletewallet")
@@ -447,38 +446,35 @@ def api_checkpending():
 @app.route("/extensions")
 def extensions():
     usr = request.args.get("usr")
-    lnevents = request.args.get("lnevents")
-    lnjoust = request.args.get("lnjoust")
-    withdraw = request.args.get("withdraw")
-    if usr:
-        if not len(usr) > 20:
+    enable = request.args.get("enable")
+    disable = request.args.get("disable")
+    ext = None
+
+    if usr and not len(usr) > 20:
         return redirect(url_for("home"))
 
+    if enable and disable:
+        # TODO: show some kind of error
+        return redirect(url_for("extensions"))
+
     with open_db() as db:
         user_wallets = db.fetchall("SELECT * FROM wallets WHERE user = ?", (usr,))
 
-    with open_ext_db() as ext_db:
-        user_ext = ext_db.fetchall("SELECT * FROM overview WHERE user = ?", (usr,))
-        if not user_ext:
-            ext_db.execute(
+        if enable:
+            ext, value = enable, 1
+        if disable:
+            ext, value = disable, 0
+
+        if ext:
+            db.execute(
                 """
-                INSERT OR IGNORE INTO overview (user) VALUES (?)
+                INSERT OR REPLACE INTO extensions (user, extension, active)
+                VALUES (?, ?, ?)
                 """,
-                (usr,),
+                (usr, ext, value),
             )
-            return redirect(url_for("extensions", usr=usr))
-
-        if lnevents:
-            if int(lnevents) != user_ext[0][1] and int(lnevents) < 2:
-                ext_db.execute("UPDATE overview SET lnevents = ? WHERE user = ?", (int(lnevents), usr,))
-                user_ext = ext_db.fetchall("SELECT * FROM overview WHERE user = ?", (usr,))
-        if lnjoust:
-            if int(lnjoust) != user_ext[0][2] and int(lnjoust) < 2:
-                ext_db.execute("UPDATE overview SET lnjoust = ? WHERE user = ?", (int(lnjoust), usr,))
-                user_ext = ext_db.fetchall("SELECT * FROM overview WHERE user = ?", (usr,))
-        if withdraw:
-            if int(withdraw) != user_ext[0][3] and int(withdraw) < 2:
-                ext_db.execute("UPDATE overview SET withdraw = ? WHERE user = ?", (int(withdraw), usr,))
-                user_ext = ext_db.fetchall("SELECT * FROM overview WHERE user = ?", (usr,))
+
+        user_ext = db.fetchall("SELECT extension FROM extensions WHERE user = ? AND active = 1", (usr,))
+        user_ext = [v[0] for v in user_ext]
 
     return render_template("extensions.html", user_wallets=user_wallets, user=usr, user_ext=user_ext)

lnbits/data/schema.sql

@@ -1,37 +1,45 @@
 CREATE TABLE IF NOT EXISTS accounts (
-    id text PRIMARY KEY,
-    email text,
-    pass text
+    id TEXT PRIMARY KEY,
+    email TEXT,
+    pass TEXT
+);
+
+CREATE TABLE IF NOT EXISTS extensions (
+    user TEXT NOT NULL,
+    extension TEXT NOT NULL,
+    active BOOLEAN DEFAULT 0,
+
+    UNIQUE (user, extension)
 );
 
 CREATE TABLE IF NOT EXISTS wallets (
-    id text PRIMARY KEY,
-    name text NOT NULL,
-    user text NOT NULL,
-    adminkey text NOT NULL,
-    inkey text
+    id TEXT PRIMARY KEY,
+    name TEXT NOT NULL,
+    user TEXT NOT NULL,
+    adminkey TEXT NOT NULL,
+    inkey TEXT
 );
 
 CREATE TABLE IF NOT EXISTS apipayments (
-    payhash text NOT NULL,
-    amount integer NOT NULL,
-    fee integer NOT NULL DEFAULT 0,
-    wallet text NOT NULL,
-    pending boolean NOT NULL,
-    memo text,
-    time timestamp NOT NULL DEFAULT (strftime('%s', 'now')),
+    payhash TEXT NOT NULL,
+    amount INTEGER NOT NULL,
+    fee INTEGER NOT NULL DEFAULT 0,
+    wallet TEXT NOT NULL,
+    pending BOOLEAN NOT NULL,
+    memo TEXT,
+    time TIMESTAMP NOT NULL DEFAULT (strftime('%s', 'now')),
 
     UNIQUE (wallet, payhash)
 );
 
 CREATE VIEW IF NOT EXISTS balances AS
-  SELECT wallet, coalesce(sum(s), 0) AS balance FROM (
-      SELECT wallet, sum(amount) AS s -- incoming
+  SELECT wallet, COALESCE(SUM(s), 0) AS balance FROM (
+      SELECT wallet, SUM(amount) AS s  -- incoming
       FROM apipayments
       WHERE amount > 0 AND pending = 0  -- don't sum pending
       GROUP BY wallet
     UNION ALL
-      SELECT wallet, sum(amount + fee) AS s -- outgoing, sum fees
+      SELECT wallet, SUM(amount + fee) AS s  -- outgoing, sum fees
       FROM apipayments
       WHERE amount < 0  -- do sum pending
       GROUP BY wallet

lnbits/db.py

@@ -1,9 +1,7 @@
 import os
 import sqlite3
 
-from typing import Optional
-
-from .settings import DATABASE_PATH, LNBITS_PATH
+from .settings import LNBITS_PATH, LNBITS_DATA_FOLDER
 
 
 class Database:
@@ -35,11 +33,28 @@ class Database:
         self.connection.commit()
 
 
-def open_db(db_path: str = DATABASE_PATH) -> Database:
+def open_db(db_name: str = "database") -> Database:
+    db_path = os.path.join(LNBITS_DATA_FOLDER, f"{db_name}.sqlite3")
     return Database(db_path=db_path)
 
 
-def open_ext_db(extension: Optional[str] = None) -> Database:
-    if extension:
-        return open_db(os.path.join(LNBITS_PATH, "extensions", extension, "database.sqlite3"))
-    return open_db(os.path.join(LNBITS_PATH, "extensions", "overview.sqlite3"))
+def open_ext_db(extension_name: str) -> Database:
+    return open_db(f"ext_{extension_name}")
+
+
+def init_databases() -> None:
+    """Creates the necessary databases if they don't exist already."""
+    """TODO: see how we can deal with migrations."""
+
+    schemas = [
+        ("database", os.path.join(LNBITS_PATH, "data", "schema.sql")),
+    ]
+
+    for extension in [x[1] for x in os.walk(os.path.join(LNBITS_PATH, "extensions"))][0]:
+        schemas.append((f"ext_{extension}", os.path.join(LNBITS_PATH, "extensions", extension, "schema.sql")))
+
+    for schema in [s for s in schemas if os.path.exists(s[1])]:
+        with open_db(schema[0]) as db:
+            with open(schema[1]) as schemafile:
+                for stmt in schemafile.read().split(";\n\n"):
+                    db.execute(stmt, [])

lnbits/extensions/withdraw/schema.sql

@@ -0,0 +1,18 @@
+CREATE TABLE IF NOT EXISTS withdraws (
+    key INTEGER PRIMARY KEY AUTOINCREMENT,
+    usr TEXT,
+    wal TEXT,
+    walnme TEXT,
+    adm INTEGER,
+    uni TEXT,
+    tit TEXT,
+    maxamt INTEGER,
+    minamt INTEGER,
+    spent INTEGER,
+    inc INTEGER,
+    tme INTEGER,
+    uniq INTEGER DEFAULT 0,
+    withdrawals TEXT,
+    tmestmp INTEGER,
+    rand TEXT
+);

lnbits/extensions/withdraw/templates/withdraw/index.html

@@ -34,9 +34,9 @@
     <i class="fa fa-angle-left pull-right"></i>
   </a>
     <ul class="treeview-menu">
-    {% if user_ext[0][3] %}
+    {% if "withdraw" in user_ext %}
     <li>
-      <a href="{{ url_for('withdraw.index') }}?usr={{ user_ext[0][0]}}"
+      <a href="{{ url_for('withdraw.index') }}?usr={{ user }}"
         ><i class="fa fa-plus"></i> LNURLw</a>
     </li>
     {% endif %}

lnbits/extensions/withdraw/views.py

@@ -21,9 +21,8 @@ def index():
     # Get all the data
     with open_db() as db:
         user_wallets = db.fetchall("SELECT * FROM wallets WHERE user = ?", (usr,))
-
-    with open_ext_db() as ext_db:
-        user_ext = ext_db.fetchall("SELECT * FROM overview WHERE user = ?", (usr,))
+        user_ext = db.fetchall("SELECT * FROM extensions WHERE user = ?", (usr,))
+        user_ext = [v[0] for v in user_ext]
 
     with open_ext_db("withdraw") as withdraw_ext_db:
         user_fau = withdraw_ext_db.fetchall("SELECT * FROM withdraws WHERE usr = ?", (usr,))
@@ -92,10 +91,18 @@ def create():
     dt = datetime.now()
     seconds = dt.timestamp()
 
+    with open_db() as db:
+        user_ext = db.fetchall("SELECT * FROM extensions WHERE user = ?", (usr,))
+        user_ext = [v[0] for v in user_ext]
+
     # Add to DB
-    with open_ext_db("withdraw") as db:
-        db.execute(
-            "INSERT OR IGNORE INTO withdraws (usr, wal, walnme, adm, uni, tit, maxamt, minamt, spent, inc, tme, uniq, withdrawals, tmestmp, rand) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+    with open_ext_db("withdraw") as withdraw_ext_db:
+        withdraw_ext_db.execute(
+            """
+            INSERT OR IGNORE INTO withdraws
+            (usr, wal, walnme, adm, uni, tit, maxamt, minamt, spent, inc, tme, uniq, withdrawals, tmestmp, rand)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+            """,
             (
                 usr,
                 wall[1],
@@ -115,14 +122,8 @@ def create():
             ),
         )
 
-    # Get updated records
-    with open_ext_db() as ext_db:
-        user_ext = ext_db.fetchall("SELECT * FROM overview WHERE user = ?", (usr,))
-        if not user_ext:
-            return jsonify({"ERROR": "NO WALLET USER"}), 401
-
-    with open_ext_db("withdraw") as withdraw_ext_db:
         user_fau = withdraw_ext_db.fetchall("SELECT * FROM withdraws WHERE usr = ?", (usr,))
+
         if not user_fau:
             return jsonify({"ERROR": "NO WALLET USER"}), 401
 

lnbits/extensions/withdraw/views_api.py

@@ -28,7 +28,6 @@ def api_lnurlencode(urlstr, parstr):
         else:
             rand = randar[0]
 
-
     url = url_for("withdraw.api_lnurlfetch", _external=True, urlstr=urlstr, parstr=parstr, rand=rand)
 
     return jsonify({"status": "TRUE", "lnurl": lnurl_encode(url.replace("http", "https"))}), 200
@@ -42,7 +41,6 @@ def api_lnurlfetch(parstr, urlstr, rand):
         return jsonify({"status": "FALSE", "ERROR": "NO WALL ID"}), 200
 
     if not urlstr:
-
         return jsonify({"status": "FALSE", "ERROR": "NO URL"}), 200
 
     with open_ext_db("withdraw") as withdraw_ext_db:
@@ -58,7 +56,6 @@ def api_lnurlfetch(parstr, urlstr, rand):
         default_description="LNbits LNURL withdraw",
     )
 
-
     return res.json(), 200
 
 
@@ -79,11 +76,9 @@ def api_lnurlwithdraw(rand):
         user_fau = withdraw_ext_db.fetchall("SELECT * FROM withdraws WHERE withdrawals = ?", (k1,))
 
         if not user_fau:
-          
             return jsonify({"status": "ERROR", "reason": "NO AUTH"}), 400
 
         if user_fau[0][10] < 1:
-            
             return jsonify({"status": "ERROR", "reason": "withdraw SPENT"}), 400
 
         # Check withdraw time
@@ -110,8 +105,8 @@ def api_lnurlwithdraw(rand):
 
     header = {"Content-Type": "application/json", "Grpc-Metadata-macaroon": str(user_fau[0][4])}
     data = {"payment_request": pr}
-    #this works locally but not being served over host, bug, needs fixing
-    #r = requests.post(url="https://lnbits.com/api/v1/channels/transactions", headers=header, data=json.dumps(data))
+    # this works locally but not being served over host, bug, needs fixing
+    # r = requests.post(url="https://lnbits.com/api/v1/channels/transactions", headers=header, data=json.dumps(data))
     r = requests.post(url=url_for("api_transactions", _external=True), headers=header, data=json.dumps(data))
     r_json = r.json()
 

lnbits/helpers.py

@@ -3,12 +3,9 @@ import sqlite3
 
 
 class MegaEncoder(json.JSONEncoder):
-    def default(self, o):
-        if type(o) == sqlite3.Row:
-            val = {}
-            for k in o.keys():
-                val[k] = o[k]
-            return val
+    def default(self, obj):
+        if isinstance(obj, sqlite3.Row):
+            return {k: obj[k] for k in obj.keys()}
         return obj
 
 

lnbits/settings.py

@@ -9,7 +9,7 @@ WALLET = OpenNodeWallet(endpoint=os.getenv("OPENNODE_API_ENDPOINT"),admin_key=os
 
 
 LNBITS_PATH = os.path.dirname(os.path.realpath(__file__))
-DATABASE_PATH = os.getenv("DATABASE_PATH", os.path.join(LNBITS_PATH, "data", "database.sqlite3"))
+LNBITS_DATA_FOLDER = os.getenv("LNBITS_DATA_FOLDER", os.path.join(LNBITS_PATH, "data"))
 
 DEFAULT_USER_WALLET_NAME = os.getenv("DEFAULT_USER_WALLET_NAME", "Bitcoin LN Wallet")
 FEE_RESERVE = float(os.getenv("FEE_RESERVE", 0))

lnbits/templates/extensions.html

@@ -36,23 +36,23 @@
     <ul class="treeview-menu">
 
 
-    {% if user_ext[0][1] %}
+    {% if "lnevents" in user_ext %}
     <li>
-      <a href="lnevents?usr={{ user_ext[0][0]}}"
+      <a href="lnevents?usr={{ user }}"
         ><i class="fa fa-plus"></i> LNEvents</a>
     </li>
     {% endif %}
 
-    {% if user_ext[0][2] %}
+    {% if "lnjoust" in user_ext %}
     <li>
-      <a href="lnjoust?usr={{ user_ext[0][0]}}"
+      <a href="lnjoust?usr={{ user }}"
         ><i class="fa fa-plus"></i> LNJoust</a>
     </li>
     {% endif %}
 
-    {% if user_ext[0][3] %}
+    {% if "withdraw" in user_ext %}
     <li>
-      <a href="withdraw?usr={{ user_ext[0][0]}}"
+      <a href="withdraw?usr={{ user }}"
         ><i class="fa fa-plus"></i> LNURLw</a>
     </li>
     {% endif %}
@@ -97,8 +97,7 @@
     <!-- Small boxes (Stat box) -->
     <div class="row">
 
-
-        {% if not user_ext[0][3] %}
+        {% if "withdraw" not in user_ext %}
             <div class="col-lg-3 col-xs-6">
               <!-- small box -->
               <div class="small-box bg-blue">
@@ -113,7 +112,7 @@
                 <div class="icon">
                   <i class="ion ion-beer"></i>
                 </div>
-                <a href="extensions?usr={{user}}&withdraw=1" class="small-box-footer">
+                <a href="extensions?usr={{user}}&enable=withdraw" class="small-box-footer">
                   Activate <i class="fa fa-arrow-circle-right"></i>
                 </a>
               </div>
@@ -141,7 +140,7 @@
 
                 </div>
               </a>
-                <a href="extensions?usr={{user}}&withdraw=0" class="small-box-footer">
+                <a href="extensions?usr={{user}}&disable=withdraw" class="small-box-footer">
                   Deactivate <i class="fa fa-arrow-circle-right"></i>
                 </a>